9.4Harness Observability

A production harness must be observable. Every action, decision, and guardrail check should be logged.

import logging
import json

class ObservableHarness(ConstrainedTestHarness):
    def __init__(self, agent, config, logger=None):
        super().__init__(agent, config)
        self.logger = logger or logging.getLogger("test_harness")
        self.telemetry = []

    def execute(self, test_objective: str) -> TestResult:
        self.logger.info(f"Starting test: {test_objective}")
        self.logger.info(f"Config: max_steps={self.config.max_steps}, "
                        f"timeout={self.config.timeout_seconds}s, "
                        f"token_budget={self.config.max_tokens}")

        result = super().execute(test_objective)

        self.logger.info(f"Test completed: status={result.status}, "
                        f"steps={self.step_count}, "
                        f"tokens={self.token_count}, "
                        f"time={time.time() - self.start_time:.1f}s")

        # Save telemetry for post-run analysis
        self.telemetry.append({
            "objective": test_objective,
            "status": result.status,
            "steps": self.step_count,
            "tokens": self.token_count,
            "duration": time.time() - self.start_time,
            "violations": self.violations,
        })

        return result

    def save_telemetry(self, path: str):
        with open(path, "w") as f:
            json.dump(self.telemetry, f, indent=2)