9.3Running Multiple Tests with the Harness

In practice, you run many tests through the same harness. Each test gets its own step/token counters but shares the harness configuration.

class TestSuiteRunner:
    def __init__(self, agent_factory, config: HarnessConfig):
        self.agent_factory = agent_factory
        self.config = config

    def run_suite(self, objectives: list[str]) -> SuiteResult:
        results = []
        total_tokens = 0
        suite_start = time.time()

        for objective in objectives:
            # Create fresh agent for each test (isolation)
            agent = self.agent_factory()
            harness = ConstrainedTestHarness(agent, self.config)

            result = harness.execute(objective)
            results.append(result)
            total_tokens += harness.token_count

            # Early exit on critical failures
            if result.status == "BLOCKED":
                break  # Stop if a guardrail violation occurs

        return SuiteResult(
            tests=results,
            total_time=time.time() - suite_start,
            total_tokens=total_tokens,
            pass_count=sum(1 for r in results if r.status == "pass"),
            fail_count=sum(1 for r in results if r.status == "fail"),
            abort_count=sum(1 for r in results if r.status == "ABORTED"),
        )