3.5Testing the Test Agent

An often-overlooked concern: how do you test your ReAct agent itself?

Unit Testing the Decision Parser

class TestDecisionParser:
    def test_parses_navigate_command(self):
        action = agent.parse_decision("NAVIGATE https://example.com")
        assert action.type == "NAVIGATE"
        assert action.payload == "https://example.com"

    def test_parses_click_with_complex_selector(self):
        action = agent.parse_decision('CLICK button[data-testid="submit"]')
        assert action.type == "CLICK"
        assert action.payload == 'button[data-testid="submit"]'

    def test_handles_malformed_response(self):
        with pytest.raises(AgentDecisionError):
            agent.parse_decision("I think we should click the button")

    def test_handles_empty_response(self):
        with pytest.raises(AgentDecisionError):
            agent.parse_decision("")

Integration Testing with Mock LLM

class TestReActAgentFlow:
    def test_completes_login_flow(self, mock_browser):
        """Agent should complete a login flow with a scripted LLM."""
        mock_llm = ScriptedLLM([
            'NAVIGATE https://app.example.com/login',
            'TYPE input[name=email] test@test.com',
            'TYPE input[name=password] secret',
            'CLICK button[type=submit]',
            'DONE pass "Successfully navigated to dashboard"',
        ])

        agent = ReActTestAgent(llm=mock_llm, browser=mock_browser)
        result = agent.run("Log in with valid credentials")

        assert result.status == "pass"
        assert result.steps_taken == 5

    def test_times_out_on_stuck_flow(self, mock_browser):
        """Agent should timeout if it cannot complete the objective."""
        mock_llm = ScriptedLLM([
            'CLICK #nonexistent-button',  # Repeated forever
        ] * 20)

        agent = ReActTestAgent(llm=mock_llm, browser=mock_browser, max_steps=5)
        result = agent.run("Click the submit button")

        assert result.status == "TIMEOUT"
        assert result.steps_taken == 5

Regression Testing Agent Behavior

Record agent decisions on a known-good run, then replay to detect regressions:

class TestAgentRegression:
    def test_login_agent_matches_baseline(self, real_browser, real_llm):
        """Agent decisions should not drift from established baseline."""
        agent = ReActTestAgent(llm=real_llm, browser=real_browser)
        result = agent.run("Log in with test@test.com / password123")

        # Save the decision sequence
        decisions = [h["action"] for h in agent.history]

        # Compare against baseline (saved from a known-good run)
        baseline = load_baseline("login_test_baseline.json")

        # Allow some flexibility (exact decisions may vary)
        assert result.status == baseline["status"]
        assert len(decisions) <= baseline["max_steps"] * 1.5  # Allow 50% more steps