9.4AI-Assisted Alert Correlation

When a single root cause triggers multiple alerts across services, AI can identify the correlation:

# alert_correlator.py
from datetime import datetime

def correlate_alerts_with_deployments(alerts: list, deployments: list) -> list:
    """
    Use temporal correlation to link alerts with recent deployments.
    If an alert fires within 30 minutes of a deployment to the same service,
    flag it as potentially deployment-related.
    """
    correlations = []

    for alert in alerts:
        alert_time = alert["fired_at"]
        alert_service = alert["service"]

        for deployment in deployments:
            deploy_time = deployment["completed_at"]
            deploy_service = deployment["service"]

            time_delta = (alert_time - deploy_time).total_seconds() / 60

            if deploy_service == alert_service and 0 < time_delta < 30:
                correlations.append({
                    "alert": alert["name"],
                    "deployment": deployment["id"],
                    "service": alert_service,
                    "minutes_after_deploy": round(time_delta, 1),
                    "deploy_commit": deployment["commit_sha"],
                    "confidence": "high" if time_delta < 10 else "medium",
                    "recommendation": (
                        f"Investigate commit {deployment['commit_sha'][:8]} "
                        f"deployed {time_delta:.0f}min before alert"
                    ),
                })

    return correlations

LLM-Enhanced Correlation

For more sophisticated correlation, feed alert context to an LLM:

def llm_correlate_alerts(alerts: list[dict]) -> dict:
    """Use an LLM to find the common root cause across multiple alerts."""
    prompt = f"""You are an SRE analyzing multiple alerts that fired within a short window.

## Active Alerts
{json.dumps(alerts, indent=2, default=str)}

## Task
1. Determine if these alerts share a common root cause.
2. If so, identify the most likely root cause.
3. Rank the alerts by importance (which one is the PRIMARY symptom vs. secondary effects).
4. Suggest an investigation order.

Respond as JSON with fields: "common_root_cause", "confidence", "primary_alert",
"investigation_steps", "likely_fix"."""

    response = client.chat.completions.create(
        model="gpt-5.5",
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"},
        temperature=0.1,
    )
    return json.loads(response.choices[0].message.content)