{ "audit": { "artifact_resolution": { "candidate_run_dirs": [ "evidences" ], "manifest_path_mismatch": false, "primary_failure_code": null, "resolution_mode": "workspace_evidence_root", "scanned_scopes": [ { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "resolved", "selected": true, "source_kind": "audit_dir", "source_root": "/data/eval_audits/evalrun-9ca3e149083040cfa6e5a554382fc030/evalcase-e2f133c76f9a43689a1e011ae870a994", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 }, { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "skipped_lower_priority", "selected": false, "source_kind": "workspace_dir", "source_root": "/data/workspaces/ws-eval-1784628840145-28bfed9c", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 } ], "selected_candidate": { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "selected_scope": "audit_dir", "selected_trace_candidate": { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "status": "resolved" }, "case_id": "map-selected-tikv-pods-to-worker-nodes-in-saas01", "contract": { "actual_cluster_id": "10989049060142230334", "cluster_scope_match": true, "expected_allowed_window_id": "snapshot-20260721T100400Z", "expected_cluster_id": "10989049060142230334", "expected_parent_id": "", "time_window_match": false }, "critical_evidence_verdict": "pass", "evidence_collection_verdict": "pass", "evidence_routing_verdict": "pass", "execution_contract": { "artifact_facts": { "analysis_patch_present": false, "core_evidence_bundle_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "channel_facts": { "events": true, "logs": false, "query_or_metric": true, "ticket_context": false }, "checks": [ { "description": "The minimal ad-hoc workflow is present.", "id": "required_workflow_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "subset_observed", "path": "", "paths": [ "final_answer_emitted" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "id": "collect_evidence_not_used", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_not_used" ], "matcher": "subset_observed", "path": "", "paths": [ "collect_evidence_not_used" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: collect_evidence_not_used" }, { "description": "Workspace diagnosis report is present.", "id": "required_report_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched execution contract facts: diagnosis_report_present" }, { "description": "At least one query or metric evidence channel is observed.", "id": "query_or_metric_channel_observed", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched execution contract facts: query_or_metric" } ], "status": "pass", "workflow_facts": { "collect_evidence_not_used": true, "final_answer_emitted": true } }, "execution_contract_verdict": "pass", "overall_verdict": "pass", "quality_verdict": "pass", "semantic_shadow": { "available": false, "status": "not_available" }, "source": "deterministic_judge", "strategy": "deterministic_trace_heavy_v1", "workflow_verdict": "pass" }, "case_run_id": "evalcase-e2f133c76f9a43689a1e011ae870a994", "facts": { "artifact_observations": { "analysis_patch_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "artifacts": { "analysis_patch_path": "", "diagnosis_report_path": "evidences/workspace_evidence_report.json", "evidence_dir": "evidences", "logs_dir": "", "manifest_path": "", "manifest_path_mismatch": false, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "source_root": "/data/eval_audits/evalrun-9ca3e149083040cfa6e5a554382fc030/evalcase-e2f133c76f9a43689a1e011ae870a994", "source_scope": "audit_dir", "summary_txt_path": "", "telemetry_path": ".opencode/telemetry.log" }, "case_id": "map-selected-tikv-pods-to-worker-nodes-in-saas01", "cluster": { "actual_cluster_id": "10989049060142230334", "allowed_cluster_ids": [ "10989049060142230334" ], "expected_cluster_id": "10989049060142230334", "expected_parent_id": "", "matched": true }, "contract": { "actual_cluster_id": "10989049060142230334", "cluster_scope_match": true, "expected_allowed_window_id": "snapshot-20260721T100400Z", "expected_cluster_id": "10989049060142230334", "expected_parent_id": "", "time_window_match": false }, "critical_evidence": { "configured": true, "matched_check_ids": [ "kube_pod_info_mapping_evidence_present", "output_cluster_id_matches_reference", "output_snapshot_time_matches_reference", "evidence_sufficient_matches_reference", "pod_mappings_present", "pod_mappings_have_expected_unique_pod_count", "pod_names_match_reference", "worker_nodes_match_reference" ] }, "evidence": { "analysis_patch_path": "", "core_evidence_bundle_present": false, "decoded_plan_indexes_used": null, "events_attempted_implies_artifact_present": true, "events_channel_observed": true, "events_collection_attempted": false, "events_output_file_present": false, "hotspot_detected": false, "infra_clear": false, "kill_query_signal_present": false, "log_error_counts": {}, "logs_channel_observed": false, "logs_dir_present": false, "memory_tracker_panic_present": false, "metric_evidence": [ { "chart_data_ref": { "cluster_id": "10989049060142230334", "end": 1784628300, "kind": "promql_range", "promql": "kube_pod_info{pod=~\"db-tikv-0|db-tikv-35|db-tikv-69|db-tikv-144\"}", "start": 1784628180, "step": "60" }, "group": "tikv_pod_node_mapping", "id": "tikv_pod_node_mapping", "metric_group_id": "tikv_pod_node_mapping", "name": "tikv_pod_node_mapping", "promql": "kube_pod_info{pod=~\"db-tikv-0|db-tikv-35|db-tikv-69|db-tikv-144\"}", "ref": { "metric_group_id": "tikv_pod_node_mapping", "type": "metric" }, "series_summaries": [ { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.11.236", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10989049060142230334", "node": "ip-172-26-11-236.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-35", "pod_ip": "172.26.11.236", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "5c038e17-4c2b-4da6-b86e-d7373e5e23e2" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.139.120", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10989049060142230334", "node": "ip-172-26-139-120.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-0", "pod_ip": "172.26.139.120", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "e0a1c42c-9b75-474e-bd3d-d4aa91e98a79" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.154.209", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10324983984131567830", "node": "ip-172-26-154-209.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-0", "pod_ip": "172.26.154.209", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "438853ff-c5bd-45df-ad97-e054b0645649" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.155.247", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10698648422638660035", "node": "ip-172-26-155-247.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-0", "pod_ip": "172.26.155.247", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "e26a771a-d38d-4c5d-8853-a2936e6e519c" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.157.232", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10754652599988043412", "node": "ip-172-26-157-232.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-35", "pod_ip": "172.26.157.232", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "0d387b14-aebb-4d90-bde7-2db61baee646" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.25.44", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10324983984131567830", "node": "ip-172-26-25-44.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-35", "pod_ip": "172.26.25.44", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "ac2412b8-02bc-4f97-a187-a983bd81a50e" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.5.110", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb1379661944646416096", "node": "ip-172-26-5-110.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-0", "pod_ip": "172.26.5.110", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "e7ed7063-7266-4f27-bac9-485305c97be1" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.6.168", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10518506906906971664", "node": "ip-172-26-6-168.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-0", "pod_ip": "172.26.6.168", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "b7175198-6329-4a59-a0cd-2e868b6966c7" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.68.211", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10989049060142230334", "node": "ip-172-26-68-211.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-69", "pod_ip": "172.26.68.211", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "91233303-1032-4ff9-8cc9-ebf230647ad4" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.72.246", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10518506906906971664", "node": "ip-172-26-72-246.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-35", "pod_ip": "172.26.72.246", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "a2f91297-33ae-4bf3-9f4a-c1471bde9636" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.77.187", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10989049060142230334", "node": "ip-172-26-77-187.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-144", "pod_ip": "172.26.77.187", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "4dd0156e-7538-44e1-b0b4-971ad3175a1f" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.84.40", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10754652599988043412", "node": "ip-172-26-84-40.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-0", "pod_ip": "172.26.84.40", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "a88da5e8-e350-4672-8bd1-689363125839" }, "max": 1.0, "min": 1.0, "point_count": 3 }, { "avg": 1.0, "labels": { "__name__": "kube_pod_info", "cluster": "prod-1372813089454544954-eks-us-east-1-17f21136", "control_plane_info": "seed/aws-us-east-1", "created_by_kind": "StatefulSet", "created_by_name": "db-tikv", "host_ip": "172.26.89.206", "host_network": "true", "instance": "kube-state-metrics", "job": "kube-state-metrics", "k8s_cluster_info": "prod-1372813089454544954-eks-us-east-1-17f21136", "namespace": "tidb10754652599988043412", "node": "ip-172-26-89-206.ec2.internal", "o11y_cluster_info": "seed/aws/us-east-1/0c252928-4d3c-4029-b157-8b866adb3d67", "o11y_id": "0c252928-4d3c-4029-b157-8b866adb3d67", "pod": "db-tikv-69", "pod_ip": "172.26.89.206", "priority_class": "tidb-component-critical", "prometheus": "monitoring/o11y-vmagent", "seed_provider": "aws", "seed_region": "us-east-1", "shoot_infra": "aws", "tenant": "1372813089209061633", "uid": "dd3085f7-039d-4f8f-aa18-cfb0bab70a24" }, "max": 1.0, "min": 1.0, "point_count": 3 } ], "summary": { "avg": 1.0, "max": 1.0, "min": 1.0, "point_count": 39, "series_count": 13 }, "time_window": { "end": 1784628300, "start": 1784628180, "step": "60" }, "unit": "" } ], "panic_signal_artifact_present": false, "query_evidence": { "slow_query": [], "topsql": [] }, "query_or_metric_channel_observed": true, "slow_query_top_digest": "", "ticket_context_channel_observed": false, "topsql_collection_error_recorded": false }, "execution_contract": { "artifact_facts": { "analysis_patch_present": false, "core_evidence_bundle_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "channel_facts": { "events": true, "logs": false, "query_or_metric": true, "ticket_context": false }, "checks": [ { "description": "The minimal ad-hoc workflow is present.", "id": "required_workflow_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "subset_observed", "path": "", "paths": [ "final_answer_emitted" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "id": "collect_evidence_not_used", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_not_used" ], "matcher": "subset_observed", "path": "", "paths": [ "collect_evidence_not_used" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: collect_evidence_not_used" }, { "description": "Workspace diagnosis report is present.", "id": "required_report_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched execution contract facts: diagnosis_report_present" }, { "description": "At least one query or metric evidence channel is observed.", "id": "query_or_metric_channel_observed", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched execution contract facts: query_or_metric" } ], "status": "pass", "workflow_facts": { "collect_evidence_not_used": true, "final_answer_emitted": true } }, "output": { "analyze_primary_culprit_asserted": false, "claims_panic_not_confirmable": false, "diagnosis_cpu_hotspot_asserted": false, "diagnosis_kill_query_chain_asserted": false, "diagnosis_memory_tracker_panic_asserted": false, "diagnosis_plan_stats_issue_asserted": false, "diagnosis_single_instance_panic_restart_asserted": false, "fake_alarm_primary_asserted": false, "focuses_on_missing_index": false, "mentions_cpu_saturation": false, "mentions_est_act_divergence": false, "mentions_idle_transaction": false, "mentions_intermediate_result_explosion": false, "mentions_kill_query": false, "mentions_memory_tracker_consume": false, "mentions_p99_latency_spike": false, "mentions_rollback_cleanup": false, "mentions_statistics_uninitialized": false, "mentions_v8_5_2": false, "mentions_v8_5_3_or_fixed_release": false, "mentions_without_fix": false, "missing_index_primary_root_cause_asserted": false, "puller_or_source_side_primary_root_cause_asserted": false, "query_only_primary_root_cause_asserted": false, "wide_scan_primary_root_cause_asserted": false }, "routing": { "analysis_event_items_present": false, "analysis_has_key_evidence": false, "analysis_has_supporting_evidence": false, "analysis_items_shape_valid": false, "analysis_log_items_present": false, "analysis_log_or_event_channel_resolvable": true, "analysis_metric_refs_required": false, "analysis_metric_refs_resolvable": true, "analysis_patch_present": false, "analysis_patch_valid_json": false, "analysis_query_refs_required": false, "analysis_query_refs_resolvable": true, "classification_contract_valid": false, "diagnosis_analyst_contract_present": false, "fault_classification_contract_valid": false, "primary_fault_class_component_restart_or_panic": false, "primary_fault_class_resource_control_or_capacity": false, "primary_fault_class_service_unavailability_or_worker_failure": false, "primary_fault_class_sql_plan_or_query": false, "primary_fault_class_storage_or_infra_io": false, "primary_fault_class_tidb_cpu_gc_or_memory": false, "primary_fault_class_tikv_read_hotspot": false, "primary_fault_class_unknown_insufficient_evidence": false, "selected_diagnosis_skills_present": false, "signal_tags_contract_valid": false }, "runtime_context_present": true, "window": { "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "expected_allowed_window_id": "snapshot-20260721T100400Z", "expected_end_ts": 1784628240, "expected_start_ts": 1784627940, "matched_allowed_window": false }, "workflow": { "architecture_context_loaded": false, "collect_evidence_invoked": false, "diagnosis_run_initialized": true, "direct_conclusion_by_investigator": false, "evidence_collection_completed": true, "evidence_collection_started": false, "final_answer_emitted": true, "misdiagnosis_guardrails_loaded": false, "premise_verification_recorded": false, "summary_read_completed": false, "targeted_panic_log_query_completed": false } }, "id": "evaljudge-351b03431f1b4de4868059b77fd98d71", "invalid_subtype": null, "judged_at": "2026-07-21T10:14:46.796952+00:00", "judgment_verdict": "pass", "packet": { "artifact_resolution": { "candidate_run_dirs": [ "evidences" ], "manifest_path_mismatch": false, "primary_failure_code": null, "resolution_mode": "workspace_evidence_root", "scanned_scopes": [ { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "resolved", "selected": true, "source_kind": "audit_dir", "source_root": "/data/eval_audits/evalrun-9ca3e149083040cfa6e5a554382fc030/evalcase-e2f133c76f9a43689a1e011ae870a994", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 }, { "candidate_count": 1, "candidates": [ { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true } ], "decision": "skipped_lower_priority", "selected": false, "source_kind": "workspace_dir", "source_root": "/data/workspaces/ws-eval-1784628840145-28bfed9c", "telemetry_path": ".opencode/telemetry.log", "valid_candidate_count": 1 } ], "selected_candidate": { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "selected_scope": "audit_dir", "selected_trace_candidate": { "actual_cluster_id": "10989049060142230334", "actual_end_ts": 1784628300, "actual_start_ts": 1784628180, "analysis_patch_path": "", "analysis_patch_present": false, "diagnosis_report_path": "evidences/workspace_evidence_report.json", "diagnosis_report_present": true, "evidence_dir": "evidences", "evidence_dir_present": true, "invalid_reasons": [], "logs_dir": "", "logs_dir_present": false, "manifest_path": "", "manifest_path_mismatch": false, "manifest_path_mismatch_fields": [], "manifest_present": false, "manifest_valid_json": true, "resolution_mode": "workspace_evidence_root", "run_dir": "evidences", "summary_json_path": "", "summary_json_present": false, "summary_txt_path": "", "summary_txt_present": false, "valid_bundle": true }, "status": "resolved" }, "contract": { "checks": [ { "description": "Actual diagnosis cluster scope matches the expected cluster id.", "id": "cluster_scope_match", "match": "all_of", "passed": true, "required": true, "rules": [ { "allowed": [], "matched": true, "matched_keys": [ "cluster_scope_match" ], "matcher": "bool_true", "path": "cluster_scope_match", "source": "trace_fact", "value": "" } ], "snippet": "matched trace facts: cluster_scope_match" } ], "status": "pass" }, "critical_evidence": { "checks": [ { "artifact_paths": [ "/data/eval_audits/evalrun-9ca3e149083040cfa6e5a554382fc030/evalcase-e2f133c76f9a43689a1e011ae870a994/evidences/workspace_evidence_report.json" ], "description": "Evidence shows a kube_pod_info-based TiKV pod mapping query.", "earned_score": 6, "id": "kube_pod_info_mapping_evidence_present", "match": "any_of", "max_score": 6, "passed": true, "required": true, "rules": [ { "artifact_path": "/data/eval_audits/evalrun-9ca3e149083040cfa6e5a554382fc030/evalcase-e2f133c76f9a43689a1e011ae870a994/evidences/workspace_evidence_report.json", "artifacts": [ "workspace_report" ], "matched": true, "matched_keyword": "kube_pod_info", "matcher": "json_text_contains_any", "snippet": "group_id\": \"tikv_pod_node_mapping\", \"name\": \"tikv_pod_node_mapping\", \"promql\": \"kube_pod_info{pod=~\\\"db-tikv-0|db-tikv-35|db-tikv-69|db-tikv-144\\\"}\", \"unit\": \"\", \"summary\":", "source": "artifact_json" } ], "snippet": "group_id\": \"tikv_pod_node_mapping\", \"name\": \"tikv_pod_node_mapping\", \"promql\": \"kube_pod_info{pod=~\\\"db-tikv-0|db-tikv-35|db-tikv-69|db-tikv-144\\\"}\", \"unit\": \"\", \"summary\":" }, { "artifact_paths": [], "description": "Output JSON cluster_id matches the runtime reference.", "earned_score": 5, "id": "output_cluster_id_matches_reference", "match": "all_of", "max_score": 5, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "\"10989049060142230334\"", "source": "output_json" } ], "snippet": "\"10989049060142230334\"" }, { "artifact_paths": [], "description": "Output JSON snapshot_time matches the runtime reference.", "earned_score": 7, "id": "output_snapshot_time_matches_reference", "match": "all_of", "max_score": 7, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "\"2026-07-21T10:04:00Z\"", "source": "output_json" } ], "snippet": "\"2026-07-21T10:04:00Z\"" }, { "artifact_paths": [], "description": "Output JSON evidence_sufficient matches the runtime reference.", "earned_score": 4, "id": "evidence_sufficient_matches_reference", "match": "all_of", "max_score": 4, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_equals_reference", "snippet": "true", "source": "output_json" } ], "snippet": "true" }, { "artifact_paths": [ "output_json" ], "description": "Output JSON contains a non-empty TiKV pod mapping list.", "earned_score": 4, "id": "pod_mappings_present", "match": "all_of", "max_score": 4, "passed": true, "required": true, "rules": [ { "artifact_path": "output_json", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_list_nonempty", "snippet": "{\"tikv_pod\": \"db-tikv-0\", \"worker_node\": \"ip-172-26-139-120.ec2.internal\"}", "source": "output_json" } ], "snippet": "{\"tikv_pod\": \"db-tikv-0\", \"worker_node\": \"ip-172-26-139-120.ec2.internal\"}" }, { "artifact_paths": [ "output_json" ], "description": "Output JSON contains the expected number of unique TiKV pods.", "earned_score": 4, "id": "pod_mappings_have_expected_unique_pod_count", "match": "all_of", "max_score": 4, "passed": true, "required": true, "rules": [ { "artifact_path": "output_json", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_path_unique_count_gte", "snippet": "unique_count=4 values=['db-tikv-0', 'db-tikv-144', 'db-tikv-35', 'db-tikv-69']", "source": "output_json" } ], "snippet": "unique_count=4 values=['db-tikv-0', 'db-tikv-144', 'db-tikv-35', 'db-tikv-69']" }, { "artifact_paths": [], "description": "Output JSON TiKV pod ordering matches the runtime reference.", "earned_score": 15, "id": "pod_names_match_reference", "match": "all_of", "max_score": 15, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_array_matches_reference", "snippet": "[{\"tikv_pod\": \"db-tikv-0\", \"worker_node\": \"ip-172-26-139-120.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-35\", \"worker_node\": \"ip-172-26-11-236.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-69\", \"worker_node\": \"ip-172-26-68-211.ec2.internal\"}, {\"tikv_pod\"", "source": "output_json" } ], "snippet": "[{\"tikv_pod\": \"db-tikv-0\", \"worker_node\": \"ip-172-26-139-120.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-35\", \"worker_node\": \"ip-172-26-11-236.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-69\", \"worker_node\": \"ip-172-26-68-211.ec2.internal\"}, {\"tikv_pod\"" }, { "artifact_paths": [], "description": "Output JSON worker_node values match the runtime reference.", "earned_score": 24, "id": "worker_nodes_match_reference", "match": "all_of", "max_score": 24, "passed": true, "required": true, "rules": [ { "artifact_path": "", "artifacts": [], "matched": true, "matched_keyword": null, "matcher": "json_array_matches_reference", "snippet": "[{\"tikv_pod\": \"db-tikv-0\", \"worker_node\": \"ip-172-26-139-120.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-35\", \"worker_node\": \"ip-172-26-11-236.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-69\", \"worker_node\": \"ip-172-26-68-211.ec2.internal\"}, {\"tikv_pod\"", "source": "output_json" } ], "snippet": "[{\"tikv_pod\": \"db-tikv-0\", \"worker_node\": \"ip-172-26-139-120.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-35\", \"worker_node\": \"ip-172-26-11-236.ec2.internal\"}, {\"tikv_pod\": \"db-tikv-69\", \"worker_node\": \"ip-172-26-68-211.ec2.internal\"}, {\"tikv_pod\"" } ], "max_score": 80, "raw_max_score": 69, "raw_score": 69, "score": 80, "status": "pass" }, "critical_evidence_verdict": "pass", "diagnosis": { "checks": [ { "description": "identify_known_bug_63956", "id": "identify_known_bug_63956", "passed": false, "snippet": null }, { "description": "identify_kill_query_idle_txn_chain", "id": "identify_kill_query_idle_txn_chain", "passed": false, "snippet": null }, { "description": "identify_version_without_fix", "id": "identify_version_without_fix", "passed": false, "snippet": null }, { "description": "recommend_fixed_version_upgrade", "id": "recommend_fixed_version_upgrade", "passed": false, "snippet": null } ], "hard_gate_passed": true, "max_score": 0, "raw_score": 0, "score": 0, "semantic_shadow": { "available": false, "checks": [], "packet_path": "/data/eval_audits/evalrun-9ca3e149083040cfa6e5a554382fc030/evalcase-e2f133c76f9a43689a1e011ae870a994/semantic_packet_v1.json", "reason": "semantic_packet_missing", "status": "not_available", "summary": { "disagreement_flags": [], "shadow_max_score": 0, "shadow_score": 0 } }, "status": "pass" }, "evidence_collection": { "checks": [ { "description": "Workspace diagnosis report is present and readable.", "earned_score": 2, "id": "required_report_present", "match": "all_of", "max_score": 2, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched collection facts: diagnosis_report_present" }, { "description": "Query or metric evidence is observed for this ad-hoc request.", "earned_score": 2, "id": "query_or_metric_channel_observed", "match": "all_of", "max_score": 2, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched collection facts: query_or_metric" }, { "description": "If an events collection query is attempted, an events output file is persisted even when the result set is empty.", "earned_score": 2, "id": "attempted_events_collection_persisted", "match": "all_of", "max_score": 2, "passed": true, "required": false, "rules": [ { "matched": true, "matched_keys": [ "events_attempted_implies_artifact_present" ], "matcher": "bool_true", "path": "events_attempted_implies_artifact_present", "paths": [], "source": "evidence_fact", "threshold": null } ], "snippet": "matched collection facts: events_attempted_implies_artifact_present" } ], "max_score": 5, "raw_max_score": 6, "raw_score": 6, "score": 5, "status": "pass" }, "evidence_collection_verdict": "pass", "evidence_routing": { "checks": [ { "description": "Analysis patch is only required to be valid when it exists.", "earned_score": 0, "id": "analysis_patch_valid_when_present", "match": "any_of", "max_score": 0, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "analysis_patch_present" ], "matcher": "bool_false", "path": "analysis_patch_present", "paths": [], "source": "routing_fact" }, { "matched": false, "matched_keys": [], "matcher": "bool_true", "path": "analysis_patch_valid_json", "paths": [], "source": "routing_fact" } ], "snippet": "matched routing facts: analysis_patch_present" } ], "max_score": 0, "raw_max_score": 0, "raw_score": 0, "score": 0, "status": "pass" }, "evidence_routing_verdict": "pass", "execution_contract": { "artifact_facts": { "analysis_patch_present": false, "core_evidence_bundle_present": false, "diagnosis_report_present": true, "summary_artifact_present": false, "supplemental_loki_present": false }, "channel_facts": { "events": true, "logs": false, "query_or_metric": true, "ticket_context": false }, "checks": [ { "description": "The minimal ad-hoc workflow is present.", "id": "required_workflow_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "subset_observed", "path": "", "paths": [ "final_answer_emitted" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "id": "collect_evidence_not_used", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_not_used" ], "matcher": "subset_observed", "path": "", "paths": [ "collect_evidence_not_used" ], "source": "workflow_check", "threshold": null } ], "snippet": "matched execution contract facts: collect_evidence_not_used" }, { "description": "Workspace diagnosis report is present.", "id": "required_report_present", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "diagnosis_report_present" ], "matcher": "subset_observed", "path": "", "paths": [ "diagnosis_report_present" ], "source": "artifact_fact", "threshold": null } ], "snippet": "matched execution contract facts: diagnosis_report_present" }, { "description": "At least one query or metric evidence channel is observed.", "id": "query_or_metric_channel_observed", "match": "all_of", "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "query_or_metric" ], "matcher": "count_gte", "path": "", "paths": [ "query_or_metric" ], "source": "channel_fact", "threshold": 1.0 } ], "snippet": "matched execution contract facts: query_or_metric" } ], "status": "pass", "workflow_facts": { "collect_evidence_not_used": true, "final_answer_emitted": true } }, "execution_contract_verdict": "pass", "ground_truth_root_cause": "Correctly answering this ad-hoc case requires resolving live TiKV pod-to-worker-node mappings from current kube pod inventory for the exact cluster and recent snapshot, without confusing one TiKV pod with another worker host.", "overall_verdict": "pass", "quality_verdict": "pass", "scoring": { "failing_hard_gate_score_cap": 69, "final_score": 100, "hard_gate_passed": true, "normalized_total_score": 100, "passing_hard_gate_score_floor": 70, "raw_total_score": 85, "total_max_score": 85 }, "strategy": "deterministic_trace_heavy_v1", "workflow": { "checks": [ { "description": "Final answer is emitted for the ad-hoc query.", "earned_score": 0, "id": "final_answer_emitted", "match": "all_of", "max_score": 0, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "final_answer_emitted" ], "matcher": "bool_true", "path": "final_answer_emitted", "paths": [], "source": "workflow_fact", "threshold": null } ], "snippet": "matched workflow facts: final_answer_emitted" }, { "description": "This ad-hoc case must not invoke collect_evidence.py.", "earned_score": 0, "id": "collect_evidence_not_used", "match": "all_of", "max_score": 0, "passed": true, "required": true, "rules": [ { "matched": true, "matched_keys": [ "collect_evidence_invoked" ], "matcher": "bool_false", "path": "collect_evidence_invoked", "paths": [], "source": "workflow_fact", "threshold": null } ], "snippet": "matched workflow facts: collect_evidence_invoked" } ], "max_score": 0, "raw_max_score": 0, "raw_score": 0, "score": 0, "status": "pass" }, "workflow_verdict": "pass" }, "passed_checks": 0, "primary_failure_code": null, "score": 100, "total_checks": 0 }