from __future__ import annotations import re from lib.models.evaluator import ( CaseReview, ComparisonContract, FinalHumanConclusion, ReplayReadyCase, SkillConclusion, WindowReview, ) def build_comparison_contract( *, profile: str, match_levels: tuple[str, ...], mismatch_types: tuple[str, ...], human_conclusion_status_values: tuple[str, ...], skill_conclusion_fields: tuple[str, ...], ) -> ComparisonContract: return ComparisonContract( profile=profile, allowed_match_levels=list(match_levels), allowed_mismatch_types=list(mismatch_types), allowed_human_conclusion_statuses=list(human_conclusion_status_values), skill_conclusion_fields=list(skill_conclusion_fields), ) def build_case_review_stub(case: ReplayReadyCase) -> CaseReview: return CaseReview( incident_id=case.incident_id, source=case.source, initial_symptom=case.initial_symptom, canonical_incident=case.canonical_incident, replay_inputs=case.replay_inputs, skill_conclusion=SkillConclusion(), final_human_conclusion=FinalHumanConclusion(), match_level='', mismatch_type=[], comparison_notes=[ 'Replay not executed yet; this case review is a structured scaffold prepared from replay-ready inputs only.', 'Final human conclusion extraction has not been executed yet.', ], improvement_suggestions=[], window_reviews=[], ) def build_window_reviews(review: CaseReview, inferred: dict) -> list[WindowReview]: windows = list(review.final_human_conclusion.analysis_windows or []) if not windows: return [] skill = review.skill_conclusion human_path = str(inferred.get('path') or '') skill_path = str(skill.path or '') skill_bottleneck = str(skill.earliest_proven_bottleneck or '') window_reviews: list[WindowReview] = [] for window in windows: notes: list[str] = [] assessment = 'window_context_recorded' inferred_path = str(window.inferred_path or '') inferred_bottleneck = str(window.inferred_bottleneck or '') if inferred_path and skill_path and inferred_path == skill_path: notes.append(f'Skill path `{skill_path}` is consistent with this window.') assessment = 'window_path_matched' elif inferred_path and skill_path and inferred_path != skill_path: notes.append(f'Skill path `{skill_path}` does not match window path `{inferred_path}`.') assessment = 'window_path_mismatch' else: notes.append('This window is recorded as maintainer evidence, but path-level compare is still limited.') if inferred_bottleneck and skill_bottleneck and skill_bottleneck != inferred_bottleneck: notes.append( f'Window bottleneck expectation `{inferred_bottleneck}` is more specific than skill bottleneck `{skill_bottleneck}`.' ) elif inferred_bottleneck and not skill_bottleneck: notes.append( f'Window bottleneck expectation `{inferred_bottleneck}` was not surfaced in the current skill conclusion.' ) window_reviews.append( WindowReview( label=str(window.label or ''), start_time=str(window.start_time or ''), end_time=str(window.end_time or ''), human_summary=str(window.summary or ''), skill_summary=str(skill.provisional_conclusion or ''), assessment=assessment, notes=notes, ) ) return window_reviews def infer_human_conclusion_signals(final_human_conclusion: FinalHumanConclusion) -> dict: text = ' '.join( part for part in [ str(final_human_conclusion.owner_final_comment or '').strip(), str(final_human_conclusion.structured_conclusion or '').strip(), str(final_human_conclusion.all_comment_summary or '').strip(), ] if part ) lowered = text.lower() entry_mode = '' path = '' scope = '' is_tikv_side = '' if not text: return { 'entry_mode': '', 'path': '', 'scope': '', 'is_tikv_side': '', 'notes': ['No human conclusion text is available for structured compare.'], } if re.search(r'leader drop|leader missing|re-election|split vote|heartbeat', text, re.I): entry_mode = 'leader_drop' path = 'write_path' elif re.search( r'unified read pool|read pool 满|read pool满|cop duration|coprocessor|running task|running tasks|handle duration|time used by level|读压力|慢查询', text, re.I, ): entry_mode = 'generic_tikv_slow' path = 'read_path' elif re.search(r'disk spike|disk latency|wal sync|fdatasync|raft engine|io jitter|i/o jitter|sync_log|rocksdb background compaction|compaction jobs', text, re.I): entry_mode = 'io_jitter' path = 'store_local_path' elif re.search(r'tikvrpc|batch receive|wait connection establish|rpc layer latency|unreachable|network|packet loss|网络', text, re.I): entry_mode = 'network_issues' path = 'network_path' elif re.search(r'serverisbusy|server is busy|deadline exceeded|flow control|throttle|discard ratio|pending write|compaction pending|l0|memtable|流控|限流', text, re.I): entry_mode = 'flow_control' path = 'read_or_write_path' elif re.search(r'snapshot|scatter|rebalance|hot leader|delete-range', text, re.I): entry_mode = 'snapshot_backlog' path = 'snapshot_path' elif re.search(r'tikv|rocksdb|write hotspot|prewrite|commit|apply', text, re.I): entry_mode = 'generic_tikv_slow' path = 'write_path' if re.search(r'one of the two tikv|tikv-\d|single store|single tikv|one store|subset|hotspot|underutilized', lowered): scope = 'store_subset_or_hotspot' elif re.search(r'cluster|all queries|cluster-wide|many stores|all stores', lowered): scope = 'cluster_wide_or_unknown' if 'tikv' in lowered or entry_mode: is_tikv_side = 'True' notes = [] if entry_mode: notes.append(f'Inferred human entry_mode={entry_mode}.') if path: notes.append(f'Inferred human path={path}.') if scope: notes.append(f'Inferred human scope={scope}.') if not notes: notes.append('Human conclusion text is present but did not expose stable route/path/scope signals for compare.') return { 'entry_mode': entry_mode, 'path': path, 'scope': scope, 'is_tikv_side': is_tikv_side, 'notes': notes, } def classify_stopped_too_early(review: CaseReview) -> str: canonical = review.canonical_incident has_monitoring_inputs = bool( canonical.cluster_id or canonical.time_window or (canonical.clinic_links or []) or (canonical.grafana_links or []) ) if not has_monitoring_inputs: return 'replay_boundary_insufficient' return 'should_have_progressed_from_initial_inputs' def compare_case_review(review: CaseReview) -> CaseReview: if review.final_human_conclusion.status == 'human_conclusion_unclear': if review.match_level != 'human_conclusion_unclear': review.match_level = 'human_conclusion_unclear' if not any('blocked' in note.lower() for note in review.comparison_notes): review.comparison_notes.append('Comparison is blocked because the final owner conclusion is not explicit enough yet.') return review inferred = infer_human_conclusion_signals(review.final_human_conclusion) review.comparison_notes.extend(note for note in inferred.get('notes') or [] if note not in review.comparison_notes) review.window_reviews = build_window_reviews(review, inferred) mismatches: list[str] = [] skill = review.skill_conclusion human_entry_mode = str(inferred.get('entry_mode') or '') human_path = str(inferred.get('path') or '') human_scope = str(inferred.get('scope') or '') if human_entry_mode and skill.entry_mode and human_entry_mode != skill.entry_mode: mismatches.append('wrong_routing') if human_path and skill.path and human_path != skill.path: mismatches.append('wrong_path') if human_scope and skill.scope and human_scope != skill.scope: mismatches.append('wrong_scope') stopped_too_early_subtype = '' if ( not any(item in mismatches for item in ('wrong_routing', 'wrong_path')) and review.final_human_conclusion.status == 'clear' and skill.missing_evidence ): stopped_too_early_subtype = classify_stopped_too_early(review) if stopped_too_early_subtype == 'should_have_progressed_from_initial_inputs': mismatches.append('stopped_too_early') else: note = 'Replay boundary was likely insufficient for deeper proof, so the provisional stop is not treated as a direct skill mismatch.' if note not in review.comparison_notes: review.comparison_notes.append(note) review.mismatch_type = mismatches if not mismatches: review.match_level = 'matched' elif 'wrong_routing' in mismatches: review.match_level = 'mismatched' elif any(item in mismatches for item in ('wrong_path', 'wrong_scope')): review.match_level = 'partially_matched' elif 'stopped_too_early' in mismatches: review.match_level = 'partially_matched' else: review.match_level = 'partially_matched' suggestions = [] if 'wrong_routing' in mismatches: suggestions.append( f'Adjust entry routing so replay selects `{human_entry_mode}` instead of `{skill.entry_mode}` when the initial text shows those cues.' ) if 'wrong_path' in mismatches: suggestions.append( f'Refine path split so replay prefers `{human_path}` instead of `{skill.path}` for similar cases.' ) if 'wrong_scope' in mismatches: suggestions.append( f'Improve scope classification so replay distinguishes `{human_scope}` from `{skill.scope}` more reliably.' ) if 'stopped_too_early' in mismatches: suggestions.append( 'Tighten proof-threshold and output rules so replay can continue beyond a provisional conclusion when the eventual owner conclusion is clear.' ) if review.window_reviews: assessments = {item.assessment for item in review.window_reviews} if assessments - {'window_path_matched'}: suggestions.append( 'Add repeated-spike / multi-window review guidance so evaluator output can preserve per-window evidence instead of flattening the case into one blended conclusion.' ) review.improvement_suggestions = suggestions if mismatches: summary_note = f'Structured compare found mismatch types: {", ".join(mismatches)}.' else: summary_note = 'Structured compare found no routing/path/scope mismatch against the latest owner conclusion.' if summary_note not in review.comparison_notes: review.comparison_notes.append(summary_note) if review.window_reviews: note = f'Multi-window review captured {len(review.window_reviews)} window(s) for maintainer-facing comparison.' if note not in review.comparison_notes: review.comparison_notes.append(note) return review