from app.logcli.utils.logger_utils import log_info, log_warning, log_error from slowlogs.ruler import TimeRange import json import os import sys import sqlite3 import csv from datetime import datetime from pathlib import Path from dataclasses import asdict, dataclass, field from typing import Dict, List, Any import requests @dataclass class SummaryResult: rule: str message: str issues_count: int metadata: Any issues: List[Any] new_issues_count: int = 0 new_issues: List[Dict[str, Any]] = field(default_factory=list) # Add the project root to the Python path project_root = str(Path(__file__).parent.parent.parent) if project_root not in sys.path: sys.path.append(project_root) FEISHU_MAX_CHARS_TEXT = 2000 FEISHU_MAX_REQUEST_BODY_BYTES_TEXT = 150 * 1024 # For general text messages # For Feishu message cards, the request body limit is stricter FEISHU_MAX_REQUEST_BODY_BYTES_CARD = 30 * 1024 # For message cards def save_result(detect_results: Dict[str, List[Any]], destination_path: str, window: 'TimeRange', comparison: 'TimeRange'): results_dir = os.path.join(destination_path, "slowlogs", "detector") os.makedirs(results_dir, exist_ok=True) base_filename = f"{window.start_ts}-{window.end_ts}.json" file_path = os.path.join(results_dir, base_filename) sequence = 0 original_file_path = file_path while os.path.exists(file_path): sequence += 1 name, ext = os.path.splitext(original_file_path) file_path = f"{name}({sequence}){ext}" def convert_to_serializable(obj): if hasattr(obj, '__dataclass__'): return asdict(obj) elif isinstance(obj, (list, tuple)): return [convert_to_serializable(item) for item in obj] elif isinstance(obj, dict): return {k: convert_to_serializable(v) for k, v in obj.items()} else: return str(obj) if not isinstance(obj, (str, int, float, bool, type(None))) else obj try: # Convert all results to serializable format serializable_results = {} for k, v in detect_results.items(): if v is None: continue try: serializable_results[k] = [ convert_to_serializable(item) for item in v] except Exception as e: log_error(f"Error serializing result for {k}: {e}") serializable_results[k] = str(v) with open(file_path, "w") as f: json.dump(serializable_results, f, indent=4, default=str) log_info(f"\nAll detection results saved to {file_path}") except IOError as e: log_error(f"Error saving results to {file_path}: {e}") def summarize_results_v2(detect_results: Dict[str, List[Any]], feishu_message_type: str = "text", feishu_page: str = None, issue_history_days: int = 30) -> str: report_lines = [] total_anomalies_across_all_clusters = 0 # Determine the effective max bytes based on message type if feishu_message_type == "card": max_bytes_limit = FEISHU_MAX_REQUEST_BODY_BYTES_CARD else: # Default to text max_bytes_limit = FEISHU_MAX_REQUEST_BODY_BYTES_TEXT # Initialize SQLite database db_path = os.path.expanduser('~/.slowlog_issues.db') os.makedirs(os.path.dirname(db_path), exist_ok=True) conn = sqlite3.connect(db_path) cursor = conn.cursor() # Create issues table if not exists cursor.execute(''' CREATE TABLE IF NOT EXISTS issues ( digest TEXT PRIMARY KEY, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) ''') # Create index on created_at for faster cleanup cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_issues_created_at ON issues(created_at) ''') # Clean up old records (older than 30 days) cursor.execute( 'DELETE FROM issues WHERE created_at < datetime("now", "-30 days")') conn.commit() # Prepare CSV file for new issues csv_path = os.path.expanduser('~/slowlog_issues.csv') csv_exists = os.path.exists(csv_path) new_issues = [] # Process each cluster and its results cluster_issue_counts = {} summary_results = {} for cluster_id, results in detect_results.items(): current_cluster_issues = 0 # Skip if results is not iterable if not hasattr(results, '__iter__') or isinstance(results, (str, bytes)): log_warning( f"Skipping non-iterable results for cluster {cluster_id}") continue summary_results[cluster_id] = [] for result in results.values(): # Skip if result is not an object with issues_count attribute if not hasattr(result, 'issues_count'): continue current_cluster_issues += result.issues_count rule_new_issues_count = 0 rule_new_issues = [] for issue in getattr(result, 'issues', []): # Check if digest exists in database digest = getattr(issue, 'digest', None) if not digest: continue cursor.execute( 'SELECT 1 FROM issues WHERE digest = ? and created_at >= datetime("now", ? || " days")', (digest, 0 - issue_history_days)) if not cursor.fetchone(): rule_new_issues_count += 1 rule_new_issues.append(issue) # New issue, add to database and CSV cursor.execute( 'INSERT OR IGNORE INTO issues (digest) VALUES (?)', (digest,)) new_issues.append({ 'digest': digest, 'query': getattr(issue, 'query', ''), 'message': getattr(issue, 'message', ''), 'type': getattr(issue, 'type', ''), 'cluster_id': cluster_id, 'timestamp': datetime.now().isoformat() }) # Create a new SummaryResult with the additional fields result = SummaryResult( rule=result.rule, message=result.message, issues_count=result.issues_count, metadata=result.metadata, issues=result.issues, new_issues_count=rule_new_issues_count, new_issues=rule_new_issues ) summary_results[cluster_id].append(result) cluster_issue_counts[cluster_id] = current_cluster_issues total_anomalies_across_all_clusters += current_cluster_issues # Write new issues to CSV and prepare report new_issues_report = "" if new_issues: # Prepare CSV content for the report from io import StringIO output = StringIO() csv_writer = csv.DictWriter( output, fieldnames=['digest', 'type', 'cluster_id', 'message', 'query', 'link']) if not csv_exists: csv_writer.writeheader() for issue in new_issues: # Construct the URL with the digest link = f"https://clinic.pingcap.com/portal/dashboard/cloud/ngm.html?provider=aws®ion=us-east-1&orgName=Buffet%27s%20Org&orgId=1372813089209061633&projectId=1372813089454544954&clusterId={issue['cluster_id']}&clusterName=prod-us-saas-region01-tidb-v75&deployType=dedicated#/slow_query?digest={issue['digest']}&from=1751760000&row=0&to=1751846399" csv_writer.writerow({ 'digest': issue['digest'], 'type': issue['type'], 'cluster_id': issue['cluster_id'], 'message': issue['message'].replace('\n', ' ').replace('\r', ''), 'query': issue['query'].replace('\n', ' ').replace('\r', ''), 'link': link }) # Join CSV lines new_issues_report = output.getvalue() # Write to file with open(csv_path, 'a', newline='') as f: f.write(new_issues_report + '\n') # Add new issues to Feishu document if feishu_page is provided if feishu_page and new_issues: try: # Also create a simplified markdown version for better readability table_header = "| Digest | Type | Cluster ID | Message (truncated) |\n|--------|------|------------|---------------------|\n" table_rows = [] for issue in new_issues: short_digest = issue['digest'][:8] short_message = ( issue['message'][:50] + '...') if len(issue['message']) > 50 else issue['message'] table_rows.append( f"| {short_digest} | {issue['type']} | {issue['cluster_id']} | {short_message} |") table_content = table_header + '\n'.join(table_rows) # Append to the existing Feishu document headers = {"Content-Type": "application/json"} payload = { "content": { "elements": [ { "tag": "markdown", "content": "## New Slow Query Issues\n" + table_content } ] } } # Make the API call to update the Feishu document response = requests.post( feishu_page, headers=headers, json=payload) response.raise_for_status() log_info( f"Successfully added {len(new_issues)} new issues to Feishu document") except Exception as e: log_error( f"Failed to update Feishu document with new issues: {str(e)}") conn.commit() conn.close() # ----------------------------------------------------- # Part 1: Overall Summary - Issues count per cluster # ----------------------------------------------------- report_lines.append("### Slow Query Detection Report ###\n") report_lines.append("#### Overall Cluster Summary (Issues Count) ####") if not cluster_issue_counts: report_lines.append("No slow query detection results available.") final_report = "\n".join(report_lines) log_info(final_report) return final_report # Sort clusters by issue count in descending order sorted_clusters = sorted(cluster_issue_counts.items(), key=lambda item: item[1], reverse=True) for cluster_id, count in sorted_clusters: report_lines.append( f"Cluster ID: {cluster_id} - Total Issues: {count}") report_lines.append("\n" + "-" * 40 + "\n") # Separator # ----------------------------------------------------- # Part 2: Detailed Issues for Top Clusters # ----------------------------------------------------- # Get the top 10 clusters with the most issues top_n_clusters_for_detail = 10 detailed_clusters_to_process = sorted_clusters[:top_n_clusters_for_detail] if detailed_clusters_to_process: report_lines.append( f"#### Detailed Issues for Top {len(detailed_clusters_to_process)} Clusters ####") initial_max_issues_per_rule = 1 if len(detailed_clusters_to_process) <= 1: initial_max_issues_per_rule = 15 elif len(detailed_clusters_to_process) <= 3: initial_max_issues_per_rule = 5 elif len(detailed_clusters_to_process) <= 5: initial_max_issues_per_rule = 3 detailed_report_content_lines = [] for cluster_id, _ in detailed_clusters_to_process: cluster_results = summary_results[cluster_id] cluster_has_issues = False cluster_detail_summary_lines = [] cluster_detail_summary_lines.append(f"Cluster ID: {cluster_id}") for result in cluster_results: if result.issues_count > 0: cluster_has_issues = True cluster_detail_summary_lines.append( f" Rule: {result.rule} - {result.message}") cluster_detail_summary_lines.append( f" Issues Found: {result.issues_count}, New Issues: {result.new_issues_count}") # Limit issues to display based on dynamic N issues_to_display = result.new_issues[:initial_max_issues_per_rule] for issue in issues_to_display: issue_details_parts = [f" -Digest: {issue.digest}"] # Conditionally add other unique attributes if hasattr(issue, 'query_time'): issue_details_parts.append( f"Query Time: {issue.query_time:.2f}") # Format for brevity if hasattr(issue, 'query'): issue_details_parts.append( f"Query: {issue.query[:60]}...") if hasattr(issue, 'level'): issue_details_parts.append(f"Level: {issue.level}") if hasattr(issue, 'variability_pct'): issue_details_parts.append( f"Variability: {issue.variability_pct:.2f}%") if hasattr(issue, 'plan_count'): issue_details_parts.append( f"Plan Count: {issue.plan_count}") cluster_detail_summary_lines.append( ", ".join(issue_details_parts)) if len(result.new_issues) > initial_max_issues_per_rule: cluster_detail_summary_lines.append( f" ... {len(result.new_issues) - initial_max_issues_per_rule} more issues not shown for this rule.") cluster_detail_summary_lines.append( "") # Blank line for readability if cluster_has_issues: detailed_report_content_lines.extend( cluster_detail_summary_lines) else: detailed_report_content_lines.append( f"Cluster ID: {cluster_id}\n No new slow query issues detected.") detailed_report_content_lines.append("") current_report_bytes = len("\n".join(report_lines).encode('utf-8')) for line in detailed_report_content_lines: line_bytes = len(line.encode('utf-8')) + \ 1 # +1 for newline character if current_report_bytes + line_bytes <= max_bytes_limit: report_lines.append(line) current_report_bytes += line_bytes else: # If adding this line exceeds the limit, add a truncation message and break if not report_lines or not report_lines[-1].startswith("... Detailed report truncated"): report_lines.append( f"\n... Detailed report truncated due to message size limit ({max_bytes_limit / 1024:.1f} KB).") break else: report_lines.append( "No top clusters with issues to display details for.") # ----------------------------------------------------- # Final Touches # ----------------------------------------------------- report_lines.append( f"\nTotal Anomalies Across All Clusters: {total_anomalies_across_all_clusters}") report_lines.append("-- End of Report --") final_report = "\n".join(report_lines) # Final byte-size check for safety final_report_bytes = len(final_report.encode('utf-8')) if final_report_bytes > max_bytes_limit: print( f"Warning: Final report is still too large ({final_report_bytes / 1024:.1f} KB) for Feishu API ({max_bytes_limit / 1024:.1f} KB limit). It might be truncated by Feishu.") else: print( f"Final report size: {final_report_bytes / 1024:.1f} KB / {max_bytes_limit / 1024:.1f} KB") print(final_report) return final_report, new_issues_report def feishu_notification(message, feishu_api: str): feishu_message = message headers = { "Content-Type": "application/json" } payload = { "msg_type": "text", "content": { "text": feishu_message } } try: response = requests.post( feishu_api, headers=headers, data=json.dumps(payload)) response.raise_for_status() print("Message successfully sent to Feishu!") print(f"Feishu API response: {response.json()}") except requests.exceptions.HTTPError as e: print(f"Failed to send message to Feishu, HTTP error: {e}") print(f"Response content: {e.response.text}") except requests.exceptions.RequestException as e: print(f"Failed to send message to Feishu, network error: {e}") except Exception as e: print(f"An unknown error occurred: {e}")