1.4Testing Feature Flag Behavior

Feature flags themselves need testing. A misconfigured flag can cause partial outages or inconsistent user experiences.

Unit Testing Flag Logic

# test_feature_flags.py
import pytest
from unittest.mock import patch

class TestFeatureFlagBehavior:
    def test_flag_on_uses_new_implementation(self, mock_ld_client):
        """When flag is ON, the new implementation should be used."""
        mock_ld_client.variation.return_value = True

        result = get_ai_summary("test document", {"user_id": "u1", "plan": "beta"})

        assert result.source == "ai-summary-v2"
        mock_ld_client.track.assert_called_with(
            "ai-summary-v2-success", pytest.ANY, metric_value=pytest.ANY
        )

    def test_flag_off_uses_legacy_implementation(self, mock_ld_client):
        """When flag is OFF, the legacy implementation should be used."""
        mock_ld_client.variation.return_value = False

        result = get_ai_summary("test document", {"user_id": "u1", "plan": "free"})

        assert result.source == "legacy-summary"

    def test_flag_on_with_quality_degradation_falls_back(self, mock_ld_client):
        """When flag is ON but quality is poor, should fall back to legacy."""
        mock_ld_client.variation.return_value = True

        with patch("evaluate_summary_quality", return_value=0.4):
            result = get_ai_summary("test document", {"user_id": "u1", "plan": "beta"})

        assert result.source == "legacy-summary"
        mock_ld_client.track.assert_called_with(
            "ai-summary-quality-degraded", pytest.ANY, metric_value=0.4
        )

    def test_flag_on_with_exception_falls_back(self, mock_ld_client):
        """When flag is ON but the new implementation throws, should fall back."""
        mock_ld_client.variation.return_value = True

        with patch("call_new_ai_summary_endpoint", side_effect=TimeoutError):
            result = get_ai_summary("test document", {"user_id": "u1", "plan": "beta"})

        assert result.source == "legacy-summary"
        mock_ld_client.track.assert_called_with("ai-summary-v2-error", pytest.ANY)

Integration Testing: Both Paths

Every feature-flagged code path must have integration test coverage:

@pytest.mark.parametrize("flag_state", [True, False])
def test_summary_endpoint_works_in_both_states(flag_state, test_client, mock_flags):
    """Both code paths must produce valid responses."""
    mock_flags.set("ai-summary-v2", flag_state)

    response = test_client.post("/api/summarize", json={"document": "Test content..."})

    assert response.status_code == 200
    assert "summary" in response.json()
    assert len(response.json()["summary"]) > 0