from slowlogs.result import save_result, summarize_results_v2, feishu_notification from slowlogs.ruler import check_all_rules, list_all_rules, TimeRange from app.logcli.utils.logger_utils import log_info, log_error import concurrent.futures from auth.client_auth import get_credentials import requests import os import sys from pathlib import Path from datetime import datetime, timezone # Add the project root to the Python path project_root = str(Path(__file__).parent.parent.parent) if project_root not in sys.path: sys.path.append(project_root) def list_slowlog_rules(custom_rules_dir: str = None): """ List all available slowlog detection rules, including custom rules. Args: custom_rules_dir: Optional path to a directory containing custom rule modules """ rules = list_all_rules(custom_rules_dir) if not rules: print("\nNo rules found!") return [] print("\nAvailable slowlog detection rules:") print("-" * 80) for i, rule in enumerate(rules, 1): rule_type = "[Custom] " if rule.get( 'is_custom', False) else "[Built-in]" print(f"{i:>3}. {rule_type} {rule.get('name', 'Unnamed')}") print( f" Description: {rule.get('description', 'No description')}") print("-" * 80) print( f"\nTotal: {len(rules)} rules ({len([r for r in rules if r.get('is_custom', False)])} custom)") return rules def slowlog_detector(vendor: str, region: str, tenant_id: str, cluster_id: str, start_ts: int, end_ts: int, comparison_start_ts, comparison_end_ts, destination_path, feishu_webhook, custom_rules_dir: str = None, rules_filter: str = None, issue_history_days: int = 30): creds = get_credentials() auth0_token = creds.get('auth0_token') os.environ['AWS_REGION'] = region os.environ['AWS_ACCESS_KEY_ID'] = creds['access_key_id'] os.environ['AWS_SECRET_ACCESS_KEY'] = creds['secret_access_key'] os.environ['AWS_SESSION_TOKEN'] = creds['session_token'] o11y_clusters = query_clusters( vendor, region, tenant_id, cluster_id, auth0_token) detect_results = {} # Convert timestamps to human-readable format def format_ts(ts): return datetime.fromtimestamp(ts, tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S %Z') # Log cluster information log_info(f"Detecting slowlogs for {len(o11y_clusters)} clusters:") for idx, cluster in enumerate(o11y_clusters, 1): log_info( f" {idx}. Cluster ID: {cluster.get('id')}, Name: {cluster.get('name')}, Region: {cluster.get('region')}") # Log time ranges in both timestamp and human-readable format log_info("Time ranges for detection:") log_info(" Detection window:") log_info(f" Start: {start_ts} ({format_ts(start_ts)})") log_info(f" End: {end_ts} ({format_ts(end_ts)})") log_info(" Comparison window:") log_info( f" Start: {comparison_start_ts} ({format_ts(comparison_start_ts)})") log_info( f" End: {comparison_end_ts} ({format_ts(comparison_end_ts)})") window = TimeRange(start_ts, end_ts) comparison = TimeRange(comparison_start_ts, comparison_end_ts) failed_list = [] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: future_to_cluster = {executor.submit(detect_cluster, o11y_cluster, window, comparison, custom_rules_dir, rules_filter): o11y_cluster for o11y_cluster in o11y_clusters} for future in concurrent.futures.as_completed(future_to_cluster): o11y_cluster = future_to_cluster[future] try: result = future.result() if result: detect_results[o11y_cluster['applicationID']] = result log_info(f"Received result for {o11y_cluster['id']}") except Exception as exc: failed_list.append(o11y_cluster['id']) log_error( f"Detect cluster {o11y_cluster['id']} generated an exception: {exc}") log_info("All cluster detect have completed.") save_result(detect_results, destination_path, window, comparison) report, new_issues_report = summarize_results_v2( detect_results, issue_history_days=issue_history_days) if new_issues_report and feishu_webhook: feishu_notification(message=new_issues_report, feishu_api=feishu_webhook) if len(failed_list) > 0: print(f"Failed cluster list: {failed_list}") def get_auth0_token(): return "token" def query_clusters(vendor, region, tenant_id, tidb_cluster_id, auth0_token): if vendor == "local": return [{ "id": "local", "name": "local", "region": region, "vendor": vendor, "tenantID": "local", "applicationID": "local" }] url = f"http://www.gs.us-west-2.aws.observability.tidbcloud.com/api/v1/clusters/?bizType=tidbcloud&vendor={vendor}®ion={region}" cluster_filter = "" if tenant_id: cluster_filter += f"&tenantID={tenant_id}" if tidb_cluster_id: cluster_filter += f"&&applicationID={tidb_cluster_id}" url += cluster_filter headers = { 'authorization': 'Bearer {}'.format(auth0_token) } resp = requests.get(url, headers=headers) info = resp.json() clusters = info["data"]["clusters"] # Define the conditions for removal exclude_application_ids = ['k8s-infra', 'base', 'seed'] exclude_project_ids = ['seed', "base"] filtered_data = [] for item in clusters: if not (item.get('applicationID') in exclude_application_ids or item.get('projectID') in exclude_project_ids): filtered_data.append(item) return filtered_data def detect_cluster(o11y_cluster, window, comparison, custom_rules_dir=None, rules_filter=None): try: log_info( f"Detect cluster {o11y_cluster['id']} of {o11y_cluster['tenantID']} in {o11y_cluster['region']} with timerange from {window.start_ts} to {window.end_ts}") results = check_all_rules( vendor=o11y_cluster["vendor"], region=o11y_cluster["region"], tenant_id=o11y_cluster["tenantID"], o11y_id=o11y_cluster["id"], window=window, comparison=comparison, custom_rules_dir=custom_rules_dir, rules_filter=rules_filter ) log_info(f"check_all_rules finished for {o11y_cluster['id']}") return results except Exception as e: log_error( f"An unknown error occurred in detect_cluster {o11y_cluster['id']}: {e}") import traceback traceback.print_exc() raise