from __future__ import annotations import tempfile import unittest from datetime import datetime, timedelta, timezone from pathlib import Path from ai_triage.models import ( MonitorConfig, MonitorTarget, WorkflowJob, WorkflowRun, WorkflowStep, ) from ai_triage.monitor import Monitor from ai_triage.orchestrator import EventTriage from ai_triage.state import StateStore NOW = datetime(2026, 7, 18, 2, 0, tzinfo=timezone.utc) TARGET = MonitorTarget( id="dedicated", enabled=True, workflow="scenario.yaml", branch="main", ) def workflow_run( *, run_id: int = 101, attempt: int = 1, conclusion: str = "failure", created_at: datetime | None = None, updated_at: datetime | None = None, ) -> WorkflowRun: return WorkflowRun( id=run_id, status="completed", conclusion=conclusion, run_attempt=attempt, html_url=f"https://github.example/runs/{run_id}", created_at=created_at or NOW - timedelta(hours=1), updated_at=updated_at or NOW + timedelta(minutes=1), ) def workflow_job(job_id: int = 201, conclusion: str = "failure") -> WorkflowJob: return WorkflowJob( id=job_id, name="gcp (case.yaml)", status="completed", conclusion=conclusion, steps=( WorkflowStep(13, "Run scenario cases", "completed", conclusion), ), html_url=f"https://github.example/jobs/{job_id}", ) class FakeGitHub: def __init__(self) -> None: self.runs: dict[str, list[WorkflowRun]] = {TARGET.workflow: []} self.jobs: dict[tuple[int, int], tuple[WorkflowJob, ...]] = {} self.fail_workflows: set[str] = set() self.queries: list[tuple[str, datetime, int]] = [] self.job_queries: list[tuple[int, int, int]] = [] def list_completed_runs( self, workflow: str, *, branch: str | None, created_since: datetime, max_pages: int, ) -> tuple[WorkflowRun, ...]: self.queries.append((workflow, created_since, max_pages)) if workflow in self.fail_workflows: raise RuntimeError("api unavailable token=must-not-leak") return tuple(self.runs.get(workflow, [])) def list_jobs( self, run_id: int, *, attempt_number: int, max_pages: int, ) -> tuple[WorkflowJob, ...]: self.job_queries.append((run_id, attempt_number, max_pages)) return self.jobs.get((run_id, attempt_number), ()) class FakeOrchestrator: def __init__(self) -> None: self.calls: list[str] = [] def triage_event(self, event: object, target: object) -> EventTriage: event_id = getattr(event, "event_id") self.calls.append(event_id) report = { "event_id": event_id, "target_id": getattr(event, "target_id"), "workflow": getattr(event, "workflow"), "run_id": getattr(event, "run").id, "run_attempt": getattr(event, "run").run_attempt, "job_id": None if getattr(event, "job") is None else getattr(event, "job").id, "conclusion": getattr(event, "run").conclusion, "incidents": [], } job_key = "run" if getattr(event, "job") is None else f"job-{getattr(event, 'job').id}" relative = f"{getattr(event, 'target_id')}/run-{getattr(event, 'run').id}-attempt-{getattr(event, 'run').run_attempt}/{job_key}" return EventTriage(report, dict(report), relative) class GroupingOrchestrator(FakeOrchestrator): def __init__(self, grouping: dict[int, tuple[str, str]]) -> None: super().__init__() self.grouping = grouping def triage_event(self, event: object, target: object) -> EventTriage: result = super().triage_event(event, target) job = getattr(event, "job") component, problem_key = self.grouping[job.id] incident = { "failed_step": "Run scenario cases", "triage": { "category": "product_error", "component": component, "problem_key": problem_key, "summary": f"summary for {job.id}", "root_cause": f"root cause for {job.id}", "recommendations": ["inspect the failed request"], "evidence": ["HTTP response was unexpected"], "owner": "component-owner", "owner_type": "component", }, "evidence": { "scenario_yaml": f"case-{job.id}.yaml", "request_method": "GET", "request_url": "https://api.example.invalid/v1/resources", "expected": "200", "actual_status": 500, }, } report = dict(result.report) report["incidents"] = [incident] return EventTriage(report, dict(report), result.relative_directory) class MultiIncidentOrchestrator(GroupingOrchestrator): def triage_event(self, event: object, target: object) -> EventTriage: result = super().triage_event(event, target) first = result.report["incidents"][0] second = { **first, "failed_step": "Run scenario cleanup", "triage": { **first["triage"], "component": "scenario-cleanup", "problem_key": "cleanup-resource-leaked", "summary": "cleanup resource was not removed", }, } report = dict(result.report) report["incidents"] = [first, second] return EventTriage(report, dict(report), result.relative_directory) class FakeNotifier: def __init__( self, *, fail_once: bool = False, fail_on_calls: set[int] | None = None, ) -> None: self.fail_on_calls = set(fail_on_calls or ()) if fail_once: self.fail_on_calls.add(1) self.calls: list[str] = [] self.notifications: list[object] = [] def send_failure(self, notification: object) -> dict[str, object]: self.notifications.append(notification) getter = getattr(notification, "get") event_ids = getter("event_ids") event_id = ( "|".join(str(item) for item in event_ids) if isinstance(event_ids, list) else getter("event_id") ) self.calls.append(str(event_id)) if len(self.calls) in self.fail_on_calls: self.fail_on_calls.remove(len(self.calls)) raise RuntimeError("feishu timeout token=must-not-leak") return {"code": 0} class FakeRandom: def uniform(self, lower: float, upper: float) -> float: return upper class StopAfterWait: def __init__(self) -> None: self.stopped = False self.waits: list[float] = [] def is_set(self) -> bool: return self.stopped def wait(self, timeout: float) -> bool: self.waits.append(timeout) self.stopped = True return True class MonitorTests(unittest.TestCase): def setUp(self) -> None: self.directory = tempfile.TemporaryDirectory() self.root = Path(self.directory.name) self.github = FakeGitHub() self.orchestrator = FakeOrchestrator() self.notifier = FakeNotifier() self.state = StateStore(self.root / "state.sqlite3") self.config = MonitorConfig( poll_interval_seconds=60, lookback_seconds=86_400, max_pages_per_target=3, state_path=str(self.root / "state.sqlite3"), lease_seconds=60, retry_base_seconds=10, retry_max_seconds=60, max_attempts=3, retention_days=30, ) def tearDown(self) -> None: self.state.close() self.directory.cleanup() def monitor( self, *, targets: tuple[MonitorTarget, ...] = (TARGET,), notifier: FakeNotifier | None = None, now: datetime = NOW, ) -> Monitor: return Monitor( self.github, self.state, self.orchestrator, notifier or self.notifier, config=self.config, targets=targets, output_root=self.root / "reports", worker_id="worker-1", now=lambda: now, random_source=FakeRandom(), ) def set_failure( self, run: WorkflowRun, job: WorkflowJob | None = None, *, workflow: str = TARGET.workflow, ) -> None: self.github.runs[workflow] = [run] self.github.jobs[(run.id, run.run_attempt)] = () if job is None else (job,) def test_fresh_store_ignores_old_failure_but_detects_later_failure(self) -> None: self.set_failure(workflow_run(updated_at=NOW - timedelta(minutes=1)), workflow_job()) monitor = self.monitor() first = monitor.scan_once(NOW) self.assertEqual(0, first.discovered) self.set_failure( workflow_run(run_id=102, updated_at=NOW + timedelta(minutes=1)), workflow_job(job_id=202), ) second = monitor.scan_once(NOW + timedelta(minutes=1)) self.assertEqual(1, second.discovered) self.assertEqual(1, second.triaged) self.assertEqual(1, second.notified) self.assertEqual(["dedicated:102:1:202"], self.notifier.calls) self.assertEqual([(102, 1, 3)], self.github.job_queries) def test_run_active_at_bootstrap_is_processed_when_it_later_fails(self) -> None: monitor = self.monitor() monitor.scan_once(NOW) self.set_failure( workflow_run( run_id=103, created_at=NOW - timedelta(hours=2), updated_at=NOW + timedelta(minutes=5), ), workflow_job(job_id=203), ) result = monitor.scan_once(NOW + timedelta(minutes=5)) self.assertEqual(1, result.discovered) self.assertEqual(["dedicated:103:1:203"], self.notifier.calls) def test_restart_does_not_repeat_triage_or_notification(self) -> None: self.monitor().scan_once(NOW) self.set_failure(workflow_run(), workflow_job()) self.monitor(now=NOW + timedelta(minutes=1)).scan_once( NOW + timedelta(minutes=1) ) reopened = self.monitor(now=NOW + timedelta(minutes=2)) result = reopened.scan_once(NOW + timedelta(minutes=2)) self.assertEqual(0, result.discovered) self.assertEqual(1, len(self.orchestrator.calls)) self.assertEqual(1, len(self.notifier.calls)) def test_failed_rerun_is_a_new_event(self) -> None: self.monitor().scan_once(NOW) self.set_failure(workflow_run(attempt=1), workflow_job(job_id=201)) self.monitor().scan_once(NOW + timedelta(minutes=1)) self.set_failure( workflow_run(attempt=2, updated_at=NOW + timedelta(minutes=2)), workflow_job(job_id=301), ) result = self.monitor().scan_once(NOW + timedelta(minutes=2)) self.assertEqual(1, result.discovered) self.assertEqual( ["dedicated:101:1:201", "dedicated:101:2:301"], self.notifier.calls, ) def test_benign_conclusions_are_ignored(self) -> None: self.monitor().scan_once(NOW) for conclusion in ("success", "neutral", "skipped"): with self.subTest(conclusion=conclusion): self.set_failure( workflow_run( run_id=200 + len(conclusion), conclusion=conclusion, updated_at=NOW + timedelta(minutes=1), ), workflow_job(), ) self.assertEqual( 0, self.monitor().scan_once(NOW + timedelta(minutes=1)).discovered, ) def test_target_failure_does_not_block_other_targets(self) -> None: broken = MonitorTarget(True.__str__(), True, "broken.yaml") healthy = MonitorTarget("healthy", True, "healthy.yaml") self.github.fail_workflows.add("broken.yaml") self.github.runs["healthy.yaml"] = [] monitor = self.monitor(targets=(broken, healthy)) result = monitor.scan_once(NOW) self.assertEqual(1, len(result.errors)) self.assertEqual(broken.id, result.errors[0].target_id) self.assertNotIn("must-not-leak", result.errors[0].message) self.assertEqual( ["broken.yaml", "healthy.yaml"], [query[0] for query in self.github.queries], ) def test_notification_retry_does_not_repeat_ai_or_report(self) -> None: failing_notifier = FakeNotifier(fail_once=True) monitor = self.monitor(notifier=failing_notifier) monitor.scan_once(NOW) self.set_failure(workflow_run(), workflow_job()) first = monitor.scan_once(NOW + timedelta(minutes=1)) report_path = self.root / "reports/dedicated/run-101-attempt-1/job-201/triage-report.json" second = monitor.scan_once(NOW + timedelta(minutes=1, seconds=10)) self.assertEqual(1, len(first.errors)) self.assertTrue(report_path.is_file()) self.assertEqual(1, len(self.orchestrator.calls)) self.assertEqual(2, len(failing_notifier.calls)) self.assertEqual(1, second.notified) def test_same_component_failures_share_one_topic(self) -> None: self.monitor().scan_once(NOW) run = workflow_run(updated_at=NOW + timedelta(minutes=1)) jobs = (workflow_job(201), workflow_job(202)) self.github.runs[TARGET.workflow] = [run] self.github.jobs[(run.id, run.run_attempt)] = jobs self.orchestrator = GroupingOrchestrator( { 201: ("dedicated-control-plane", "cluster-stuck-creating"), 202: ("dedicated-control-plane", "cluster-delete-timeout"), } ) result = self.monitor().scan_once(NOW + timedelta(minutes=1)) self.assertEqual(2, result.triaged) self.assertEqual(1, result.notified) self.assertEqual(1, len(self.notifier.notifications)) topic = self.notifier.notifications[0] self.assertEqual(2, len(topic["failures"])) def test_same_problem_across_components_shares_one_topic(self) -> None: self.monitor().scan_once(NOW) run = workflow_run(updated_at=NOW + timedelta(minutes=1)) jobs = (workflow_job(201), workflow_job(202)) self.github.runs[TARGET.workflow] = [run] self.github.jobs[(run.id, run.run_attempt)] = jobs self.orchestrator = GroupingOrchestrator( { 201: ("api-gateway", "upstream-http-503"), 202: ("dedicated-control-plane", "upstream-http-503"), } ) result = self.monitor().scan_once(NOW + timedelta(minutes=1)) self.assertEqual(1, result.notified) self.assertEqual(2, len(self.notifier.notifications[0]["failures"])) def test_unrelated_failures_create_separate_topics(self) -> None: self.monitor().scan_once(NOW) run = workflow_run(updated_at=NOW + timedelta(minutes=1)) jobs = (workflow_job(201), workflow_job(202)) self.github.runs[TARGET.workflow] = [run] self.github.jobs[(run.id, run.run_attempt)] = jobs self.orchestrator = GroupingOrchestrator( { 201: ("api-gateway", "upstream-http-503"), 202: ("scenario-runner", "placeholder-not-bound"), } ) result = self.monitor().scan_once(NOW + timedelta(minutes=1)) self.assertEqual(2, result.notified) self.assertEqual(2, len(self.notifier.notifications)) def test_retry_does_not_resend_an_already_confirmed_unrelated_topic(self) -> None: notifier = FakeNotifier(fail_on_calls={2}) self.monitor(notifier=notifier).scan_once(NOW) run = workflow_run(updated_at=NOW + timedelta(minutes=1)) jobs = (workflow_job(201), workflow_job(202)) self.github.runs[TARGET.workflow] = [run] self.github.jobs[(run.id, run.run_attempt)] = jobs self.orchestrator = GroupingOrchestrator( { 201: ("api-gateway", "upstream-http-503"), 202: ("scenario-runner", "placeholder-not-bound"), } ) first = self.monitor(notifier=notifier).scan_once( NOW + timedelta(minutes=1) ) second = self.monitor(notifier=notifier).scan_once( NOW + timedelta(minutes=1, seconds=10) ) self.assertEqual(1, len(first.errors)) self.assertEqual(1, first.notified) self.assertEqual(1, second.notified) self.assertEqual( [ "dedicated:101:1:201", "dedicated:101:1:202", "dedicated:101:1:202", ], notifier.calls, ) def test_retry_skips_confirmed_incident_topic_from_the_same_job(self) -> None: notifier = FakeNotifier(fail_on_calls={2}) self.monitor(notifier=notifier).scan_once(NOW) run = workflow_run(updated_at=NOW + timedelta(minutes=1)) job = workflow_job(201) self.github.runs[TARGET.workflow] = [run] self.github.jobs[(run.id, run.run_attempt)] = (job,) self.orchestrator = MultiIncidentOrchestrator( {201: ("api-gateway", "upstream-http-503")} ) first = self.monitor(notifier=notifier).scan_once( NOW + timedelta(minutes=1) ) second = self.monitor(notifier=notifier).scan_once( NOW + timedelta(minutes=1, seconds=10) ) self.assertEqual(1, first.notified) self.assertEqual(1, second.notified) self.assertEqual(3, len(notifier.notifications)) summaries = [ topic["failures"][0]["triage"]["summary"] for topic in notifier.notifications ] self.assertEqual( [ "summary for 201", "cleanup resource was not removed", "cleanup resource was not removed", ], summaries, ) def test_continuous_loop_waits_with_bounded_jitter_and_stops(self) -> None: stop = StopAfterWait() cycles: list[object] = [] self.monitor().run(stop, on_cycle=cycles.append) self.assertEqual([65.0], stop.waits) self.assertEqual(1, len(self.github.queries)) self.assertEqual(1, len(cycles)) if __name__ == "__main__": unittest.main()