{ "audit_json": { "artifact_resolution": { "candidate_run_dirs": [ "evidences" ], "manifest_path_mismatch": false, "primary_failure_code": null, "resolution_mode": "workspace_evidence_root", "scanned_scopes": [ { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "resolved", "selected": true, "source_kind": "audit_dir", "source_root": "/data/eval_audits/evalrun-a6921afa263e4be1a7c1bdcc298bbe72/evalcase-94272b66dd89438d8c831759f6bc608d", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 }, { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "skipped_lower_priority", "selected": false, "source_kind": "workspace_dir", "source_root": "/data/workspaces/ws-eval-1784625405607-0089da6b", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 } ], "selected_candidate": { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "selected_scope": "audit_dir", "selected_trace_candidate": { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "status": "resolved" }, "case_id": "representative-business-slow-queries-in-premium-cluster-during-latest-window", "contract": { "actual_cluster_id": "10748116073341772391", "cluster_scope_match": true, "expected_allowed_window_id": "bjt-2026-07-21-1100-5m", "expected_cluster_id": "10748116073341772391", "expected_parent_id": "2026948324023603200", "time_window_match": false }, "critical_evidence_verdict": "pass", "evidence_collection_verdict": "pass", "evidence_routing_verdict": "pass", "execution_contract": { "artifact_facts": { "analysis_patch_present": false, "core_evidence_bundle_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "channel_facts": { "events": true, "logs": false, "query_or_metric": true, "ticket_context": false }, "checks": [ { "description": "The minimal ad-hoc workflow is present.", "id": "required_workflow_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "subset_observed", "path": "", "paths": [ "final_answer_emitted" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "id": "collect_evidence_not_used", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_not_used" ], "matcher": "subset_observed", "path": "", "paths": [ "collect_evidence_not_used" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: collect_evidence_not_used" }, { "description": "Workspace diagnosis report is present.", "id": "required_report_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched execution contract facts: diagnosis_report_present" }, { "description": "At least one query or metric evidence channel is observed.", "id": "query_or_metric_channel_observed", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched execution contract facts: query_or_metric" } ], "status": "pass", "workflow_facts": { "collect_evidence_not_used": true, "final_answer_emitted": true } }, "execution_contract_verdict": "pass", "overall_verdict": "pass", "quality_verdict": "pass", "semantic_shadow": { "available": false, "status": "not_available" }, "source": "deterministic_judge", "strategy": "deterministic_trace_heavy_v1", "workflow_verdict": "pass" }, "facts_json": { "artifact_observations": { "analysis_patch_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "artifacts": { "analysis_patch_path": "", "diagnosis_report_path": "evidences/workspace_evidence_report.json", "evidence_dir": "evidences", "logs_dir": "", "manifest_path": "", "manifest_path_mismatch": false, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "source_root": "/data/eval_audits/evalrun-a6921afa263e4be1a7c1bdcc298bbe72/evalcase-94272b66dd89438d8c831759f6bc608d", "source_scope": "audit_dir", "summary_txt_path": "", "telemetry_path": ".opencode/telemetry.log" }, "case_id": "representative-business-slow-queries-in-premium-cluster-during-latest-window", "cluster": { "actual_cluster_id": "10748116073341772391", "allowed_cluster_ids": [ "10748116073341772391", "2026948324023603200" ], "expected_cluster_id": "10748116073341772391", "expected_parent_id": "2026948324023603200", "matched": true }, "contract": { "actual_cluster_id": "10748116073341772391", "cluster_scope_match": true, "expected_allowed_window_id": "bjt-2026-07-21-1100-5m", "expected_cluster_id": "10748116073341772391", "expected_parent_id": "2026948324023603200", "time_window_match": false }, "critical_evidence": { "configured": true, "matched_check_ids": [ "output_cluster_id_matches_reference", "output_window_start_matches_reference", "output_window_end_matches_reference", "evidence_sufficient_matches_reference", "slow_queries_present", "slow_queries_have_three_unique_digests", "slow_query_digests_match_reference", "slow_query_exec_count_match_reference", "slow_query_avg_time_match_reference" ] }, "evidence": { "analysis_patch_path": "", "core_evidence_bundle_present": false, "decoded_plan_indexes_used": null, "events_attempted_implies_artifact_present": true, "events_channel_observed": true, "events_collection_attempted": false, "events_output_file_present": false, "hotspot_detected": false, "infra_clear": false, "kill_query_signal_present": false, "log_error_counts": {}, "logs_channel_observed": false, "logs_dir_present": false, "memory_tracker_panic_present": false, "metric_evidence": [], "panic_signal_artifact_present": false, "query_evidence": { "slow_query": [ { "avg_query_time": "0.4135382620666667", "digest": "992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577", "exec_count": "90", "max_query_time": "0.912442372", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_aqua` WHERE `id` > 1152921504665110825 AND `id` < 2305843009250084684 AND `created_time` < FROM_UNIXTIME(1779332471) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "31.99205184299999", "total_wait_time": "0.0015970980000000006" }, { "avg_query_time": "0.7144608548636365", "digest": "bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6", "exec_count": "66", "max_query_time": "2.317812442", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_inbetween` WHERE `id` > 4899916394593024407 AND `id` < 6341068275349384987 AND `created_time` < FROM_UNIXTIME(1779332464) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "26.17346202", "total_wait_time": "0.0008578349999999999" }, { "avg_query_time": "0.4492159362", "digest": "d94be824b6c9bd25dd280afec2e0dda0e5b6b8f38d0f912d085a02f137f3130a", "exec_count": "15", "max_query_time": "0.603791282", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_inbetween` WHERE `id` >= 4899916394580972940 AND `id` < 6341068275349384987 AND `created_time` < FROM_UNIXTIME(1779332464) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "3.541772229", "total_wait_time": "0.000046946999999999996" }, { "avg_query_time": "0.47709968999999997", "digest": "004e25591e3f89e48a9f851d43b1bb0d82d6dd7195e7300ba814a7d7aad5f8ba", "exec_count": "12", "max_query_time": "0.550764909", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_aqua` WHERE `id` > 8070450532306169480 AND `created_time` < FROM_UNIXTIME(1779332471) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "5.634322518", "total_wait_time": "0.000325017" }, { "avg_query_time": "0.327395286", "digest": "a9cde48f6a24501b29025d1bcaea103777ac4429ba05a921cbe341de425df1e7", "exec_count": "3", "max_query_time": "0.327395286", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_inbetween` WHERE `id` < 1441151880767086829 AND `created_time` < FROM_UNIXTIME(1779332464) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "0.973772385", "total_wait_time": "0.000089742" }, { "avg_query_time": "2.402072092", "digest": "a46254ac8d0d4e725f900443ea8abea91591509bce8af4a124b5a901c3c8b4f0", "exec_count": "3", "max_query_time": "2.402072092", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_ginto` WHERE `id` > 8646911284822084385 AND `id` < 8935141660931218634 AND `created_time` < FROM_UNIXTIME(1779332378) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "3.249871371", "total_wait_time": "0.000067131" }, { "avg_query_time": "0.401071624", "digest": "4e9e2c20c79b6a52fc2c41dccdfb6009285959e518449ed759769fb7726f07dc", "exec_count": "3", "max_query_time": "0.401071624", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_aqua` WHERE `id` >= 8070450532289975596 AND `created_time` < FROM_UNIXTIME(1779332471) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "0.534", "total_wait_time": "\\N" }, { "avg_query_time": "0.45897648", "digest": "2d8a2f5b6ca9720258c125633d77e2c85c6586e38e115a662588c9263c82a6c3", "exec_count": "3", "max_query_time": "0.45897648", "query_sql": "SELECT\n digest,\n any_value(query) AS sample_sql,\n COUNT(*) AS exec_count,\n MAX(query_time) AS max_query_time,\n AVG(query_time) AS avg_query_time,\n SUM(process_time) AS total_process_time,\n SUM(wait_time) AS total_wait_time\nFROM clinic_data_proxy.slow_query_logs\nWHERE date IN ('20260721')\n AND time >= 1784602800\n AND time <= 1784603100\nGROUP BY digest\nORDER BY exec_count DESC\nLIMIT 20", "sample_sql": "SELECT LOW_PRIORITY SQL_NO_CACHE `id` FROM `c66_seamless_others`.`t_seamless_log_aqua` WHERE `id` >= 3458764513864257921 AND `id` < 4611686018471382252 AND `created_time` < FROM_UNIXTIME(1779332471) ORDER BY `id` ASC LIMIT 500;", "source": "slow_queries", "title": "slow_query_top_by_exec_count", "total_process_time": "0.708", "total_wait_time": "\\N" } ], "topsql": [] }, "query_or_metric_channel_observed": true, "slow_query_top_digest": "992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577", "ticket_context_channel_observed": false, "topsql_collection_error_recorded": false }, "execution_contract": { "artifact_facts": { "analysis_patch_present": false, "core_evidence_bundle_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "channel_facts": { "events": true, "logs": false, "query_or_metric": true, "ticket_context": false }, "checks": [ { "description": "The minimal ad-hoc workflow is present.", "id": "required_workflow_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "subset_observed", "path": "", "paths": [ "final_answer_emitted" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "id": "collect_evidence_not_used", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_not_used" ], "matcher": "subset_observed", "path": "", "paths": [ "collect_evidence_not_used" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: collect_evidence_not_used" }, { "description": "Workspace diagnosis report is present.", "id": "required_report_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched execution contract facts: diagnosis_report_present" }, { "description": "At least one query or metric evidence channel is observed.", "id": "query_or_metric_channel_observed", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched execution contract facts: query_or_metric" } ], "status": "pass", "workflow_facts": { "collect_evidence_not_used": true, "final_answer_emitted": true } }, "output": { "analyze_primary_culprit_asserted": false, "claims_panic_not_confirmable": false, "diagnosis_cpu_hotspot_asserted": false, "diagnosis_kill_query_chain_asserted": false, "diagnosis_memory_tracker_panic_asserted": false, "diagnosis_plan_stats_issue_asserted": false, "diagnosis_single_instance_panic_restart_asserted": false, "fake_alarm_primary_asserted": false, "focuses_on_missing_index": false, "mentions_cpu_saturation": false, "mentions_est_act_divergence": false, "mentions_idle_transaction": false, "mentions_intermediate_result_explosion": false, "mentions_kill_query": false, "mentions_memory_tracker_consume": false, "mentions_p99_latency_spike": false, "mentions_rollback_cleanup": true, "mentions_statistics_uninitialized": false, "mentions_v8_5_2": false, "mentions_v8_5_3_or_fixed_release": false, "mentions_without_fix": false, "missing_index_primary_root_cause_asserted": false, "puller_or_source_side_primary_root_cause_asserted": false, "query_only_primary_root_cause_asserted": false, "wide_scan_primary_root_cause_asserted": false }, "routing": { "analysis_event_items_present": false, "analysis_has_key_evidence": false, "analysis_has_supporting_evidence": false, "analysis_items_shape_valid": false, "analysis_log_items_present": false, "analysis_log_or_event_channel_resolvable": true, "analysis_metric_refs_required": false, "analysis_metric_refs_resolvable": true, "analysis_patch_present": false, "analysis_patch_valid_json": false, "analysis_query_refs_required": false, "analysis_query_refs_resolvable": true, "classification_contract_valid": false, "diagnosis_analyst_contract_present": false, "fault_classification_contract_valid": false, "primary_fault_class_component_restart_or_panic": false, "primary_fault_class_resource_control_or_capacity": false, "primary_fault_class_service_unavailability_or_worker_failure": false, "primary_fault_class_sql_plan_or_query": false, "primary_fault_class_storage_or_infra_io": false, "primary_fault_class_tidb_cpu_gc_or_memory": false, "primary_fault_class_tikv_read_hotspot": false, "primary_fault_class_unknown_insufficient_evidence": false, "selected_diagnosis_skills_present": false, "signal_tags_contract_valid": false }, "runtime_context_present": true, "window": { "actual_end_ts": null, "actual_start_ts": null, "expected_allowed_window_id": "bjt-2026-07-21-1100-5m", "expected_end_ts": 1784603100, "expected_start_ts": 1784602800, "matched_allowed_window": false }, "workflow": { "architecture_context_loaded": false, "collect_evidence_invoked": false, "diagnosis_run_initialized": true, "direct_conclusion_by_investigator": false, "evidence_collection_completed": false, "evidence_collection_started": false, "final_answer_emitted": true, "misdiagnosis_guardrails_loaded": false, "premise_verification_recorded": false, "summary_read_completed": false, "targeted_panic_log_query_completed": false } }, "invalid_subtype": null, "judgment_verdict": "pass", "packet_json": { "artifact_resolution": { "candidate_run_dirs": [ "evidences" ], "manifest_path_mismatch": false, "primary_failure_code": null, "resolution_mode": "workspace_evidence_root", "scanned_scopes": [ { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "resolved", "selected": true, "source_kind": "audit_dir", "source_root": "/data/eval_audits/evalrun-a6921afa263e4be1a7c1bdcc298bbe72/evalcase-94272b66dd89438d8c831759f6bc608d", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 }, { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "skipped_lower_priority", "selected": false, "source_kind": "workspace_dir", "source_root": "/data/workspaces/ws-eval-1784625405607-0089da6b", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 } ], "selected_candidate": { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "selected_scope": "audit_dir", "selected_trace_candidate": { "actual_cluster_id": "10748116073341772391", "actual_end_ts": null, "actual_start_ts": null, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "status": "resolved" }, "contract": { "checks": [ { "description": "Actual diagnosis cluster scope matches the expected cluster id.", "id": "cluster_scope_match", "match": "all_of", "passed": true, "required": true, "rules": [ { "allowed": [], "matched": true, "matched_keys": [ "cluster_scope_match" ], "matcher": "bool_true", "path": "cluster_scope_match", "source": "trace_fact", "value": "" } ], "snippet": "matched trace facts: cluster_scope_match" } ], "status": "pass" }, "critical_evidence": { "checks": [ { "artifact_paths": [], "description": "Output JSON cluster_id matches the runtime reference.", "earned_score": 8, "id": "output_cluster_id_matches_reference", "match": "all_of", "max_score": 8, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "\"10748116073341772391\"", "source": "output_json" } ], "snippet": "\"10748116073341772391\"" }, { "artifact_paths": [], "description": "Output JSON window.start matches the runtime reference.", "earned_score": 6, "id": "output_window_start_matches_reference", "match": "all_of", "max_score": 6, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "\"2026-07-21T03:00:00Z\"", "source": "output_json" } ], "snippet": "\"2026-07-21T03:00:00Z\"" }, { "artifact_paths": [], "description": "Output JSON window.end matches the runtime reference.", "earned_score": 6, "id": "output_window_end_matches_reference", "match": "all_of", "max_score": 6, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "\"2026-07-21T03:05:00Z\"", "source": "output_json" } ], "snippet": "\"2026-07-21T03:05:00Z\"" }, { "artifact_paths": [], "description": "Output JSON evidence_sufficient matches the runtime reference.", "earned_score": 4, "id": "evidence_sufficient_matches_reference", "match": "all_of", "max_score": 4, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "true", "source": "output_json" } ], "snippet": "true" }, { "artifact_paths": [ "output_json" ], "description": "Output JSON contains a non-empty slow query list.", "earned_score": 4, "id": "slow_queries_present", "match": "all_of", "max_score": 4, "passed": true, "required": true, "rules": [ { "artifact_path": "output_json", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_list_nonempty", "snippet": "{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}", "source": "output_json" } ], "snippet": "{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}" }, { "artifact_paths": [ "output_json" ], "description": "Output JSON contains three unique slow-query digests.", "earned_score": 4, "id": "slow_queries_have_three_unique_digests", "match": "all_of", "max_score": 4, "passed": true, "required": true, "rules": [ { "artifact_path": "output_json", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_unique_count_gte", "snippet": "unique_count=3 values=['992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577', 'bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6', 'd94be824b6c9bd25dd280afec2e0dda0e5b6b8f38d0f912d085a02f137f3130a']", "source": "output_json" } ], "snippet": "unique_count=3 values=['992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577', 'bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6', 'd94be824b6c9bd25dd280afec2e0dda0e5b6b8f38d0f912d085a02f137f3130a']" }, { "artifact_paths": [], "description": "Output JSON digest ordering matches the runtime reference.", "earned_score": 24, "id": "slow_query_digests_match_reference", "match": "all_of", "max_score": 24, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_array_matches_reference", "snippet": "[{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}, {\"digest\": \"bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6\", \"exec_count\": 66, \"avg_time_s\": 0.714}, {", "source": "output_json" } ], "snippet": "[{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}, {\"digest\": \"bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6\", \"exec_count\": 66, \"avg_time_s\": 0.714}, {" }, { "artifact_paths": [], "description": "Output JSON exec_count values match the runtime reference.", "earned_score": 12, "id": "slow_query_exec_count_match_reference", "match": "all_of", "max_score": 12, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_array_matches_reference", "snippet": "[{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}, {\"digest\": \"bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6\", \"exec_count\": 66, \"avg_time_s\": 0.714}, {", "source": "output_json" } ], "snippet": "[{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}, {\"digest\": \"bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6\", \"exec_count\": 66, \"avg_time_s\": 0.714}, {" }, { "artifact_paths": [], "description": "Output JSON avg_time_s values match the runtime reference.", "earned_score": 12, "id": "slow_query_avg_time_match_reference", "match": "all_of", "max_score": 12, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_array_matches_reference", "snippet": "[{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}, {\"digest\": \"bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6\", \"exec_count\": 66, \"avg_time_s\": 0.714}, {", "source": "output_json" } ], "snippet": "[{\"digest\": \"992377525bc78cf77741e870a6a509f866749e322c5baf19d55703fc26970577\", \"exec_count\": 90, \"avg_time_s\": 0.414}, {\"digest\": \"bda1f10c99c3bfba07501f9579aeedaff16fcde326a3e99d75dd75adfd880ca6\", \"exec_count\": 66, \"avg_time_s\": 0.714}, {" } ], "max_score": 80, "raw_max_score": 80, "raw_score": 80, "score": 80, "status": "pass" }, "critical_evidence_verdict": "pass", "diagnosis": { "checks": [ { "description": "identify_known_bug_63956", "id": "identify_known_bug_63956", "passed": false, "snippet": null }, { "description": "identify_kill_query_idle_txn_chain", "id": "identify_kill_query_idle_txn_chain", "passed": false, "snippet": null }, { "description": "identify_version_without_fix", "id": "identify_version_without_fix", "passed": false, "snippet": null }, { "description": "recommend_fixed_version_upgrade", "id": "recommend_fixed_version_upgrade", "passed": false, "snippet": null } ], "hard_gate_passed": true, "max_score": 0, "raw_score": 0, "score": 0, "semantic_shadow": { "available": false, "checks": [], "packet_path": "/data/eval_audits/evalrun-a6921afa263e4be1a7c1bdcc298bbe72/evalcase-94272b66dd89438d8c831759f6bc608d/semantic_packet_v1.json", "reason": "semantic_packet_missing", "status": "not_available", "summary": { "disagreement_flags": [], "shadow_max_score": 0, "shadow_score": 0 } }, "status": "pass" }, "evidence_collection": { "checks": [ { "description": "Workspace diagnosis report is present and readable.", "earned_score": 2, "id": "required_report_present", "match": "all_of", "max_score": 2, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched collection facts: diagnosis_report_present" }, { "description": "Query or metric evidence is observed for this ad-hoc request.", "earned_score": 3, "id": "query_or_metric_channel_observed", "match": "all_of", "max_score": 3, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched collection facts: query_or_metric" }, { "description": "If an events collection query is attempted, an events output file is persisted even when the result set is empty.", "earned_score": 2, "id": "attempted_events_collection_persisted", "match": "all_of", "max_score": 2, "passed": true, "required": false, "rules": [ { "matched": true, "matched_keys": [ "events_attempted_implies_artifact_present" ], "matcher": "bool_true", "path": "events_attempted_implies_artifact_present", "paths": [], "source": "evidence_fact", "threshold": null } ], "snippet": "matched collection facts: events_attempted_implies_artifact_present" } ], "max_score": 5, "raw_max_score": 7, "raw_score": 7, "score": 5, "status": "pass" }, "evidence_collection_verdict": "pass", "evidence_routing": { "checks": [ { "description": "Analysis patch is only required to be valid when it exists.", "earned_score": 0, "id": "analysis_patch_valid_when_present", "match": "any_of", "max_score": 0, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "analysis_patch_present" ], "matcher": "bool_false", "path": "analysis_patch_present", "paths": [], "source": "routing_fact" }, { "matched": false, "matched_keys": [], "matcher": "bool_true", "path": "analysis_patch_valid_json", "paths": [], "source": "routing_fact" } ], "snippet": "matched routing facts: analysis_patch_present" } ], "max_score": 0, "raw_max_score": 0, "raw_score": 0, "score": 0, "status": "pass" }, "evidence_routing_verdict": "pass", "execution_contract": { "artifact_facts": { "analysis_patch_present": false, "core_evidence_bundle_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "channel_facts": { "events": true, "logs": false, "query_or_metric": true, "ticket_context": false }, "checks": [ { "description": "The minimal ad-hoc workflow is present.", "id": "required_workflow_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "subset_observed", "path": "", "paths": [ "final_answer_emitted" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "id": "collect_evidence_not_used", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_not_used" ], "matcher": "subset_observed", "path": "", "paths": [ "collect_evidence_not_used" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: collect_evidence_not_used" }, { "description": "Workspace diagnosis report is present.", "id": "required_report_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched execution contract facts: diagnosis_report_present" }, { "description": "At least one query or metric evidence channel is observed.", "id": "query_or_metric_channel_observed", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched execution contract facts: query_or_metric" } ], "status": "pass", "workflow_facts": { "collect_evidence_not_used": true, "final_answer_emitted": true } }, "execution_contract_verdict": "pass", "ground_truth_root_cause": "Correctly identifying the most important business slow queries for the primary premium cluster requires using the exact runtime-resolved window, excluding trivial transaction-control statements, and ranking the remaining digests by exec_count and avg_time.", "overall_verdict": "pass", "quality_verdict": "pass", "scoring": { "failing_hard_gate_score_cap": 69, "final_score": 100, "hard_gate_passed": true, "normalized_total_score": 100, "passing_hard_gate_score_floor": 70, "raw_total_score": 85, "total_max_score": 85 }, "strategy": "deterministic_trace_heavy_v1", "workflow": { "checks": [ { "description": "Final answer is emitted for the ad-hoc query.", "earned_score": 0, "id": "final_answer_emitted", "match": "all_of", "max_score": 0, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "bool_true", "path": "final_answer_emitted", "paths": [], "source": "workflow_fact", "threshold": null } ], "snippet": "matched workflow facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "earned_score": 0, "id": "collect_evidence_not_used", "match": "all_of", "max_score": 0, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_invoked" ], "matcher": "bool_false", "path": "collect_evidence_invoked", "paths": [], "source": "workflow_fact", "threshold": null } ], "snippet": "matched workflow facts: collect_evidence_invoked" } ], "max_score": 0, "raw_max_score": 0, "raw_score": 0, "score": 0, "status": "pass" }, "workflow_verdict": "pass" }, "passed_checks": 0, "primary_failure_code": null, "score": 100, "total_checks": 0 }