from typing import List from slowlogs.ruler import Rule, TimeRange, DeltaLakeStore from dataclasses import dataclass import time from app.logcli.utils.logger import log_info, log_warning, log_error class RuleLargeIndexFullScan(Rule): @dataclass class Result: rule: str message: str issues_count: int metadata: "RuleLargeIndexFullScan.MetaData" issues: "List[RuleLargeIndexFullScan.Issue]" @dataclass class MetaData: analysis_time: float time_window: TimeRange comparison_window: TimeRange = None @dataclass class Issue: type: str digest: str query: str plan_digest: str query_time: float est_rows: float store_type: str level: str # 'critical' or 'major' @property def name(self): return "large_index_full_scan" @property def description(self): return "Detect queries with large index full scans (estRows > 100,000 and not TiFlash)" def _has_large_index_full_scan(self, plan_node: dict) -> tuple[bool, float, str]: """Check if the execution plan node contains a large full table scan.""" if not isinstance(plan_node, dict): return False, 0, "" name = plan_node.get("name", "") if "IndexFullScan_" in name: store_type = plan_node.get("storeType", "") est_rows = plan_node.get("estRows") if est_rows is not None: try: if isinstance(est_rows, str): est_rows = float(est_rows) if est_rows >= 10000 and store_type != "tiflash": level = 'critical' if est_rows >= 100000 else 'major' return True, est_rows, store_type, level except (ValueError, TypeError): pass # Check children recursively for child in plan_node.get("children", []): found, rows, st, level = self._has_large_index_full_scan(child) if found: return found, rows, st, level return False, 0, "", "" def check(self, store: DeltaLakeStore, window: TimeRange, comparison: TimeRange = None) -> Result: # First, get all slow queries with their plans query = """ SELECT digest, first(query) as query, plan_digest, max(query_time) as query_time, first(decoded_plan) as decoded_plan FROM slowlogs WHERE time >= ? AND time < ? AND decoded_plan IS NOT NULL AND decoded_plan like '%IndexFullScan_%' GROUP BY digest, plan_digest """ try: import json rows = store.execute( query, (window.start_ts, window.end_ts)).fetchall() issues = [] for row in rows: plan_str = row[4] # Skip if plan is None or empty if not plan_str: log_info(f"Skipping empty plan for digest: {row[0]}") continue try: # Try to parse the plan JSON plan = json.loads(plan_str) # Skip if plan is not a dictionary if not isinstance(plan, dict): log_info( f"Plan is not a dictionary for digest: {row[0]}") continue plan = plan.get("main", plan) has_scan, est_rows, store_type, level = self._has_large_index_full_scan( plan) if has_scan: issues.append(RuleLargeIndexFullScan.Issue( type=self.name, digest=row[0], query=row[1], plan_digest=row[2], query_time=row[3], est_rows=est_rows, store_type=store_type, level=level )) except json.JSONDecodeError as e: log_warning( f"Invalid JSON in plan for digest {row[0]}: {str(e)[:100]}") except Exception as e: log_warning( f"Unexpected error processing plan for digest {row[0]}: {str(e)}") return RuleLargeIndexFullScan.Result( rule=RuleLargeIndexFullScan.__name__, message=f"Found {len(issues)} queries with large full table scans (estRows > 100,000 and not TiFlash)", issues_count=len(issues), issues=issues, metadata=RuleLargeIndexFullScan.MetaData( analysis_time=time.time(), time_window=window, comparison_window=comparison ) ) except Exception as e: log_error(f"Error checking for large full scans: {str(e)}") return None