""" Report generator. Generates performance test reports in various formats. """ from pathlib import Path from typing import Dict, List, Optional from .data_aggregator import aggregate_spu_data, compare_spu_data from .models import ( PerformanceCurveData, SPUComparisonReport, SPUReportData, WorkloadRunData, ) def generate_spu_report(spu_identifier: str) -> Optional[SPUReportData]: """Generate report data for a single SPU. Args: spu_identifier: SPU identifier string Returns: SPUReportData or None if no data found """ return aggregate_spu_data(spu_identifier) def generate_comparison_report( spu_id_1: str, spu_id_2: str, workflow_types: Optional[List[str]] = None ) -> Optional[SPUComparisonReport]: """Generate comparison report between two SPUs. Args: spu_id_1: First SPU identifier spu_id_2: Second SPU identifier workflow_types: List of workflow types to compare (None = all) Returns: SPUComparisonReport or None if data not found """ return compare_spu_data(spu_id_1, spu_id_2, workflow_types) def export_to_markdown( report_data: SPUReportData, output_path: Optional[str] = None ) -> str: """Export SPU report to Markdown format. Args: report_data: SPU report data output_path: Optional output file path Returns: Markdown content string """ lines = [] # Title lines.append(f"# {report_data.spu_identifier} Performance Test Report\n") # Cluster configuration lines.append("## Test Cluster Configuration\n") lines.append(_format_cluster_config_table(report_data.cluster_config)) lines.append("") # Workload client resource usage lines.append("## Workload Client Resource Usage\n") client_metrics_content = _format_client_metrics_section(report_data) if client_metrics_content: lines.append(client_metrics_content) lines.append("") else: lines.append("*No client resource metrics available*\n") # Test results for each workflow - group by workflow type lines.append("## Test Results\n") # Group workflows by type workflow_groups = {} for workflow in report_data.workflows: wf_type = workflow.workflow_type if wf_type not in workflow_groups: workflow_groups[wf_type] = [] workflow_groups[wf_type].append(workflow) # Display each workflow type for wf_type, workflows in sorted(workflow_groups.items()): lines.append(f"### {wf_type}\n") # Show test metadata for each task lines.append("**Test Executions:**\n") for workflow in workflows: lines.append( f"- Task ID: {workflow.task_id}, " f"Status: {workflow.status}, " f"Workflow Execution ID: {workflow.workflow_execution_id}" ) lines.append("") # Collect all runs from all tasks of this workflow type all_runs = [] for workflow in workflows: all_runs.extend(workflow.runs) if all_runs: lines.append("#### Performance Summary\n") lines.append(_format_performance_summary_table(all_runs)) lines.append("") # Raw data section lines.append("## Raw Data\n") for wf_type, workflows in sorted(workflow_groups.items()): # Collect all runs from all tasks of this workflow type all_runs = [] for workflow in workflows: all_runs.extend(workflow.runs) if all_runs: lines.append(f"### {wf_type} - Detailed Results\n") lines.append(_format_detailed_results_table(all_runs)) lines.append("") markdown_content = "\n".join(lines) # Write to file if path provided if output_path: Path(output_path).write_text(markdown_content, encoding="utf-8") return markdown_content def _format_comparison_summary_table(comparison) -> str: """Format comparison data using the hierarchical layout (方案 3). Structure: 1. Summary Statistics table (Thread × Metric with avg/median/std for both configs) 2. Detailed Runs - Configuration 1 (one table per task, grouped by thread count) 3. Detailed Runs - Configuration 2 (one table per task, grouped by thread count) """ lines = [] if not comparison.thread_data: return "" # Section 1: Summary Statistics Table lines.append("#### Summary Statistics\n") lines.append( _format_summary_statistics_table( comparison.thread_data, comparison.spu_1_identifier, comparison.spu_2_identifier, ) ) lines.append("") lines.append( "*\\* Improvement is calculated based on average (avg) values: " "`(Config2_avg - Config1_avg) / Config1_avg * 100%`. " "Positive values indicate Config2 performs better.*\n" ) lines.append("") # Section 2: Detailed Runs - Configuration 1 lines.append( f"#### Detailed Runs - Configuration 1 ({comparison.spu_1_identifier})\n" ) lines.append(_format_detailed_runs_by_config(comparison.thread_data, config_num=1)) lines.append("") # Section 3: Detailed Runs - Configuration 2 lines.append( f"#### Detailed Runs - Configuration 2 ({comparison.spu_2_identifier})\n" ) lines.append(_format_detailed_runs_by_config(comparison.thread_data, config_num=2)) return "\n".join(lines) def _format_summary_statistics_table( thread_data: Dict, config1_name: str, config2_name: str ) -> str: """Format summary statistics table. | Thread | Metric | Config1 Avg | Config1 Median | Config1 Std | Config2 Avg | Config2 Median | Config2 Std | Improvement | """ lines = [] # Build header lines.append( "| Thread | Metric | Config1 Avg | Config1 Median | Config1 Std | " "Config2 Avg | Config2 Median | Config2 Std | Improvement* |" ) lines.append( "|--------|--------|-------------|----------------|-------------|" "-------------|----------------|-------------|--------------|" ) # Process each thread count for thread_count in sorted(thread_data.keys()): thread_comp = thread_data[thread_count] # Get all metrics for this thread count metrics = sorted(thread_comp.summary_stats.keys()) for metric_name in metrics: stats1, stats2, improvement = thread_comp.summary_stats[metric_name] # Format metric name for display display_metric = ( metric_name.replace("latency.", "").replace("_", " ").title() ) # Format values based on metric type if "latency" in metric_name.lower(): # Latency in ms - show 2 decimal places lines.append( f"| {thread_count} | {display_metric} | " f"{stats1.avg:.2f} | {stats1.median:.2f} | {stats1.std:.2f} | " f"{stats2.avg:.2f} | {stats2.median:.2f} | {stats2.std:.2f} | " f"{improvement:+.2f}% |" ) else: # Throughput metrics - show as integers lines.append( f"| {thread_count} | {display_metric} | " f"{stats1.avg:,.0f} | {stats1.median:,.0f} | {stats1.std:,.0f} | " f"{stats2.avg:,.0f} | {stats2.median:,.0f} | {stats2.std:,.0f} | " f"{improvement:+.2f}% |" ) return "\n".join(lines) def _format_detailed_runs_by_config(thread_data: Dict, config_num: int) -> str: """Format detailed runs for one configuration. Shows one table per task, grouped by thread count. Args: thread_data: Dictionary of ThreadComparisonData keyed by thread count config_num: 1 for config1, 2 for config2 """ lines = [] # Collect all tasks across all thread counts all_tasks = {} # task_id -> {thread_count -> runs} for thread_count in sorted(thread_data.keys()): thread_comp = thread_data[thread_count] runs = thread_comp.config1_runs if config_num == 1 else thread_comp.config2_runs for task_run in runs: task_id = task_run.task_id if task_id not in all_tasks: all_tasks[task_id] = {} if thread_count not in all_tasks[task_id]: all_tasks[task_id][thread_count] = [] all_tasks[task_id][thread_count].append(task_run.run) if not all_tasks: lines.append("*No runs available for this configuration*\n") return "\n".join(lines) # Generate one table per task for task_id in sorted(all_tasks.keys()): lines.append(f"**Task {task_id}**\n") # Group by thread count for thread_count in sorted(all_tasks[task_id].keys()): runs = all_tasks[task_id][thread_count] if len(runs) > 0: lines.append(f"*Thread Count: {thread_count}*\n") lines.append(_format_task_runs_table(runs)) lines.append("") return "\n".join(lines) def _format_task_runs_table(runs: List[WorkloadRunData]) -> str: """Format a table showing all metrics for runs of a single task at a single thread count. Columns: Run # | All workload metrics | Client metrics (CPU/Memory) """ if not runs: return "" lines = [] # Determine which metrics are present all_metrics = set() has_client_metrics = False for run in runs: # Collect workload metrics for key in run.metrics: if isinstance(run.metrics[key], (int, float)): all_metrics.add(key) elif key == "latency" and isinstance(run.metrics[key], dict): for latency_key in run.metrics[key]: all_metrics.add(f"latency.{latency_key}") # Check for client metrics if run.client_metrics: has_client_metrics = True sorted_metrics = sorted(all_metrics) # Build header header_parts = ["| Run # |"] divider_parts = ["|-------|"] for metric in sorted_metrics: display_name = metric.replace("latency.", "").replace("_", " ").title() header_parts.append(f" {display_name} |") divider_parts.append("----------|") # Add client metrics columns if available if has_client_metrics: header_parts.extend( [ " CPU Max% |", " CPU Avg% |", " CPU P80% |", " CPU P95% |", " CPU P99% |", " Mem Max (MiB) |", " Mem Avg (MiB) |", " Mem P80 (MiB) |", " Mem P95 (MiB) |", " Mem P99 (MiB) |", " Net Max (ms) |", " Net Avg (ms) |", " Net P80 (ms) |", " Net P95 (ms) |", " Net P99 (ms) |", ] ) divider_parts.extend( [ "-----------|", "-----------|", "-----------|", "-----------|", "-----------|", "---------------|", "---------------|", "---------------|", "---------------|", "---------------|", "---------------|", "---------------|", "---------------|", "---------------|", "---------------|", ] ) lines.append("".join(header_parts)) lines.append("".join(divider_parts)) # Build data rows for idx, run in enumerate(runs, start=1): row_parts = [f"| {idx} |"] # Add workload metrics for metric in sorted_metrics: if "." in metric: # Nested metric (e.g., latency.p95) parts = metric.split(".") value = run.metrics for part in parts: value = value.get(part, {}) if isinstance(value, dict) else None else: value = run.metrics.get(metric) if value is not None: if "latency" in metric.lower(): row_parts.append(f" {value:.2f} |") else: row_parts.append(f" {value:,.0f} |") else: row_parts.append(" - |") # Add client metrics if available if has_client_metrics: if run.client_metrics: cpu_max = "-" cpu_avg = "-" cpu_p80 = "-" cpu_p95 = "-" cpu_p99 = "-" mem_max = "-" mem_avg = "-" mem_p80 = "-" mem_p95 = "-" mem_p99 = "-" net_max = "-" net_avg = "-" net_p80 = "-" net_p95 = "-" net_p99 = "-" stats = run.client_metrics.get("Stats", []) for stat in stats: index = stat.get("Index", "") metrics_list = stat.get("Metrics", []) if index == "cpu": for metric in metrics_list: method = metric.get("Method", "") value = metric.get("Value", 0) if method == "max": cpu_max = f"{value:.1f}" elif method == "avg": cpu_avg = f"{value:.1f}" elif method == "p80": cpu_p80 = f"{value:.1f}" elif method == "p95": cpu_p95 = f"{value:.1f}" elif method == "p99": cpu_p99 = f"{value:.1f}" elif index == "memory": for metric in metrics_list: method = metric.get("Method", "") value = metric.get("Value", 0) if method == "max": mem_max = f"{value:.1f}" elif method == "avg": mem_avg = f"{value:.1f}" elif method == "p80": mem_p80 = f"{value:.1f}" elif method == "p95": mem_p95 = f"{value:.1f}" elif method == "p99": mem_p99 = f"{value:.1f}" elif index == "network/srtt": for metric in metrics_list: method = metric.get("Method", "") value = metric.get("Value", 0) if method == "max": net_max = f"{value:.2f}" elif method == "avg": net_avg = f"{value:.2f}" elif method == "p80": net_p80 = f"{value:.2f}" elif method == "p95": net_p95 = f"{value:.2f}" elif method == "p99": net_p99 = f"{value:.2f}" row_parts.extend( [ f" {cpu_max} |", f" {cpu_avg} |", f" {cpu_p80} |", f" {cpu_p95} |", f" {cpu_p99} |", f" {mem_max} |", f" {mem_avg} |", f" {mem_p80} |", f" {mem_p95} |", f" {mem_p99} |", f" {net_max} |", f" {net_avg} |", f" {net_p80} |", f" {net_p95} |", f" {net_p99} |", ] ) else: row_parts.extend([" - |"] * 15) # 15 client metric columns lines.append("".join(row_parts)) return "\n".join(lines) def _format_metric_display_name(metric_name: str) -> str: """Format metric name for display in summary. Args: metric_name: Raw metric name (e.g., 'latency.p95', 'tpmC', 'tps') Returns: Formatted display name with units (e.g., 'P95 Latency (ms)', 'tpmC', 'TPS') """ if metric_name.startswith("latency."): # Extract the latency type (e.g., 'p95', 'avg', 'max') latency_type = metric_name.replace("latency.", "") # Format as "P95 Latency (ms)", "Avg Latency (ms)", etc. if latency_type.startswith("p"): # Percentile metrics: p50, p95, p99 -> P50, P95, P99 display_name = latency_type.upper() + " Latency (ms)" else: # Other metrics: avg, max, min -> Avg, Max, Min display_name = latency_type.title() + " Latency (ms)" return display_name elif metric_name.lower() in ["tpmc", "tps", "qps", "qphh"]: # Throughput metrics - use specific formatting metric_lower = metric_name.lower() if metric_lower == "tpmc": return "tpmC" elif metric_lower == "tps": return "TPS" elif metric_lower == "qps": return "QPS" elif metric_lower == "qphh": return "QphH" return metric_name.upper() elif metric_name == "tpmTotal": return "tpmTotal" elif metric_name == "efficiency": return "Efficiency (%)" else: # Other metrics - just title case with spaces return metric_name.replace("_", " ").title() def export_comparison_to_markdown( comparison_report: SPUComparisonReport, output_path: Optional[str] = None ) -> str: """Export comparison report to Markdown format. Args: comparison_report: Comparison report data output_path: Optional output file path Returns: Markdown content string """ lines = [] # Title lines.append( f"# Performance Comparison Report: {comparison_report.spu_1.spu_identifier} vs {comparison_report.spu_2.spu_identifier}\n" ) # Cluster configuration comparison lines.append("## Cluster Configuration Comparison\n") lines.append( _format_cluster_config_comparison_table( comparison_report.spu_1.cluster_config, comparison_report.spu_2.cluster_config, ) ) lines.append("") # Summary lines.append("## Summary\n") lines.append( "*Note: Improvements are calculated relative to Configuration 1 as baseline. " "Positive values indicate Configuration 2 performs better.*\n\n" ) lines.append("**Calculation Formula:**\n\n") lines.append("For throughput metrics (TPS, tpmC, etc.):\n\n") lines.append( "$$\\text{Improvement} = \\frac{\\text{Config}_2 - \\text{Config}_1}" "{\\text{Config}_1} \\times 100\\%$$\n\n" ) lines.append("For latency metrics (P95, P99, etc.):\n\n") lines.append( "$$\\text{Improvement} = -\\frac{\\text{Config}_2 - \\text{Config}_1}" "{\\text{Config}_1} \\times 100\\%$$\n\n" ) lines.append("**Average Improvement per Metric:**\n\n") lines.append( "For each metric, the improvement is averaged across all thread counts.\n\n" ) # Display per-metric improvements for each workflow for comparison in comparison_report.comparisons: lines.append(f"### {comparison.workflow_type}\n") if comparison.metric_improvements: # Sort metrics by name for consistent display sorted_metrics = sorted(comparison.metric_improvements.items()) for metric_name, avg_improvement in sorted_metrics: # Format metric name for display using unified formatter display_metric = _format_metric_display_name(metric_name) # Format improvement with appropriate sign lines.append(f"- **{display_metric}**: {avg_improvement:+.2f}%\n") else: lines.append("*No metrics available for comparison.*\n") lines.append("") lines.append("") # Detailed comparisons lines.append("## Detailed Comparisons\n") for comparison in comparison_report.comparisons: lines.append(f"### {comparison.workflow_type}\n") lines.append(_format_comparison_summary_table(comparison)) lines.append("") markdown_content = "\n".join(lines) # Write to file if path provided if output_path: Path(output_path).write_text(markdown_content, encoding="utf-8") return markdown_content def _format_cluster_config_table(cluster_config) -> str: """Format cluster configuration as Markdown table.""" lines = ["| Component | Specification | Count | Storage |"] lines.append("|-----------|---------------|-------|---------|") # TiDB lines.append( f"| TiDB | {cluster_config.tidb_spu} | {cluster_config.tidb_count} | - |" ) # TiKV storage_info = "" if cluster_config.tikv_storage_size_gi and cluster_config.tikv_storage_type: storage_info = f"{cluster_config.tikv_storage_size_gi} GiB ({cluster_config.tikv_storage_type})" lines.append( f"| TiKV | {cluster_config.tikv_spu} | {cluster_config.tikv_count} | {storage_info} |" ) # PD (if configured) if cluster_config.pd_spu and cluster_config.pd_count: lines.append( f"| PD | {cluster_config.pd_spu} | {cluster_config.pd_count} | - |" ) # TiFlash (if configured) if cluster_config.tiflash_spu and cluster_config.tiflash_count: storage_info = "" if ( cluster_config.tiflash_storage_size_gi and cluster_config.tiflash_storage_type ): storage_info = f"{cluster_config.tiflash_storage_size_gi} GiB ({cluster_config.tiflash_storage_type})" lines.append( f"| TiFlash | {cluster_config.tiflash_spu} | {cluster_config.tiflash_count} | {storage_info} |" ) lines.append("") lines.append(f"**Cloud Provider**: {cluster_config.cloud_provider} ") lines.append(f"**Region**: {cluster_config.region} ") lines.append(f"**TiDB Version**: {cluster_config.cluster_version}") return "\n".join(lines) def _format_cluster_config_comparison_table(config1, config2) -> str: """Format cluster configuration comparison as side-by-side table.""" lines = ["| Component | Configuration 1 | Configuration 2 |"] lines.append("|-----------|-----------------|-----------------|") # TiDB lines.append( f"| **TiDB** | {config1.tidb_spu} × {config1.tidb_count} | " f"{config2.tidb_spu} × {config2.tidb_count} |" ) # TiKV tikv1_storage = "" if config1.tikv_storage_size_gi and config1.tikv_storage_type: tikv1_storage = ( f" ({config1.tikv_storage_size_gi} GiB {config1.tikv_storage_type})" ) tikv2_storage = "" if config2.tikv_storage_size_gi and config2.tikv_storage_type: tikv2_storage = ( f" ({config2.tikv_storage_size_gi} GiB {config2.tikv_storage_type})" ) lines.append( f"| **TiKV** | {config1.tikv_spu} × {config1.tikv_count}{tikv1_storage} | " f"{config2.tikv_spu} × {config2.tikv_count}{tikv2_storage} |" ) # PD (if either has it) if (config1.pd_spu and config1.pd_count) or (config2.pd_spu and config2.pd_count): pd1 = ( f"{config1.pd_spu} × {config1.pd_count}" if config1.pd_spu and config1.pd_count else "-" ) pd2 = ( f"{config2.pd_spu} × {config2.pd_count}" if config2.pd_spu and config2.pd_count else "-" ) lines.append(f"| **PD** | {pd1} | {pd2} |") # TiFlash (if either has it) if (config1.tiflash_spu and config1.tiflash_count) or ( config2.tiflash_spu and config2.tiflash_count ): tiflash1 = ( f"{config1.tiflash_spu} × {config1.tiflash_count}" if config1.tiflash_spu and config1.tiflash_count else "-" ) tiflash2 = ( f"{config2.tiflash_spu} × {config2.tiflash_count}" if config2.tiflash_spu and config2.tiflash_count else "-" ) lines.append(f"| **TiFlash** | {tiflash1} | {tiflash2} |") # TiProxy (if either has it) tiproxy1 = "Enabled" if config1.tiproxy_enabled else "Disabled" tiproxy2 = "Enabled" if config2.tiproxy_enabled else "Disabled" lines.append(f"| **TiProxy** | {tiproxy1} | {tiproxy2} |") # Cloud and region lines.append(f"| **Cloud** | {config1.cloud_provider} | {config2.cloud_provider} |") lines.append(f"| **Region** | {config1.region} | {config2.region} |") lines.append( f"| **Version** | {config1.cluster_version} | {config2.cluster_version} |" ) return "\n".join(lines) def _format_client_metrics_section(report_data: SPUReportData) -> str: """Format workload client resource metrics section. Args: report_data: SPU report data containing workflows with client metrics Returns: Formatted markdown string with client metrics tables by workflow type """ lines = [] # Group workflows by type workflow_groups = {} for workflow in report_data.workflows: wf_type = workflow.workflow_type if wf_type not in workflow_groups: workflow_groups[wf_type] = [] workflow_groups[wf_type].append(workflow) # Process each workflow type for wf_type, workflows in sorted(workflow_groups.items()): # Collect all runs with client metrics runs_with_metrics = [] for workflow in workflows: for run in workflow.runs: if run.client_metrics: runs_with_metrics.append(run) if not runs_with_metrics: continue lines.append(f"### {wf_type}\n") lines.append(_format_client_metrics_table(runs_with_metrics)) lines.append("") return "\n".join(lines) if lines else "" def _format_client_metrics_table(runs: List[WorkloadRunData]) -> str: """Format client metrics as markdown table. Args: runs: List of workload runs with client_metrics data Returns: Formatted markdown table string """ if not runs: return "" # Build table header lines = [ "| Task ID | Node Name | CPU Max (%) | CPU Avg (%) | CPU P80 (%) | CPU P95 (%) | CPU P99 (%) | Memory Max (MiB) | Memory Avg (MiB) | Memory P80 (MiB) | Memory P95 (MiB) | Memory P99 (MiB) | Net Max (ms) | Net Avg (ms) | Net P80 (ms) | Net P95 (ms) | Net P99 (ms) |", "|---------|-----------|-------------|-------------|-------------|-------------|-------------|------------------|------------------|------------------|------------------|------------------|--------------|--------------|--------------|--------------|--------------|", ] # Sort by task_id sorted_runs = sorted( runs, key=lambda r: (r.task_id if r.task_id else 0, r.node_name) ) for run in sorted_runs: if not run.client_metrics: continue task_id = run.task_id if run.task_id else "N/A" node_name = run.node_name # Extract metrics from client_metrics cpu_max = "-" cpu_avg = "-" cpu_p80 = "-" cpu_p95 = "-" cpu_p99 = "-" mem_max = "-" mem_avg = "-" mem_p80 = "-" mem_p95 = "-" mem_p99 = "-" net_max = "-" net_avg = "-" net_p80 = "-" net_p95 = "-" net_p99 = "-" stats = run.client_metrics.get("Stats", []) for stat in stats: index = stat.get("Index", "") metrics = stat.get("Metrics", []) if index == "cpu": for metric in metrics: method = metric.get("Method", "") value = metric.get("Value", 0) if method == "max": cpu_max = f"{value:.1f}" elif method == "avg": cpu_avg = f"{value:.1f}" elif method == "p80": cpu_p80 = f"{value:.1f}" elif method == "p95": cpu_p95 = f"{value:.1f}" elif method == "p99": cpu_p99 = f"{value:.1f}" elif index == "memory": for metric in metrics: method = metric.get("Method", "") value = metric.get("Value", 0) if method == "max": mem_max = f"{value:.1f}" elif method == "avg": mem_avg = f"{value:.1f}" elif method == "p80": mem_p80 = f"{value:.1f}" elif method == "p95": mem_p95 = f"{value:.1f}" elif method == "p99": mem_p99 = f"{value:.1f}" elif index == "network/srtt": for metric in metrics: method = metric.get("Method", "") value = metric.get("Value", 0) if method == "max": net_max = f"{value:.2f}" elif method == "avg": net_avg = f"{value:.2f}" elif method == "p80": net_p80 = f"{value:.2f}" elif method == "p95": net_p95 = f"{value:.2f}" elif method == "p99": net_p99 = f"{value:.2f}" lines.append( f"| {task_id} | {node_name} | {cpu_max} | {cpu_avg} | {cpu_p80} | {cpu_p95} | {cpu_p99} | " f"{mem_max} | {mem_avg} | {mem_p80} | {mem_p95} | {mem_p99} | " f"{net_max} | {net_avg} | {net_p80} | {net_p95} | {net_p99} |" ) return "\n".join(lines) def _format_performance_summary_table(runs: List[WorkloadRunData]) -> str: """Format performance data as a unified table with all metrics. Creates a single table with all throughput and latency metrics: | Thread Count | tpmC | TPS | Avg Latency (ms) | P50 Latency (ms) | ... | """ if not runs: return "" # Group runs by thread count runs_by_thread = {} for run in runs: thread = run.params.get("thread") if thread: if thread not in runs_by_thread: runs_by_thread[thread] = [] runs_by_thread[thread].append(run) if not runs_by_thread: return "" threads = sorted(runs_by_thread.keys()) # Determine which metrics are available has_tpmC = any("tpmC" in run.metrics for run in runs) has_tps = any("tps" in run.metrics for run in runs) has_qps = any("qps" in run.metrics for run in runs) has_qphh = any("QphH" in run.metrics for run in runs) has_latency = any("latency" in run.metrics for run in runs) # Collect all available latency metrics latency_metrics = set() if has_latency: for run in runs: latency = run.metrics.get("latency", {}) latency_metrics.update(latency.keys()) # Define the order and display names for latency metrics latency_order = [ ("avg", "Avg Latency (ms)"), ("p50", "P50 Latency (ms)"), ("p90", "P90 Latency (ms)"), ("p95", "P95 Latency (ms)"), ("p99", "P99 Latency (ms)"), ("max", "Max Latency (ms)"), ("min", "Min Latency (ms)"), ] # Build column headers headers = ["Thread Count"] metric_columns = [] # Add throughput columns if has_tpmC: headers.append("tpmC") metric_columns.append(("tpmC", "tpmC", False)) if has_tps: headers.append("TPS") metric_columns.append(("tps", "TPS", False)) if has_qps: headers.append("QPS") metric_columns.append(("qps", "QPS", False)) if has_qphh: headers.append("QphH") metric_columns.append(("QphH", "QphH", False)) # Add latency columns for latency_key, display_name in latency_order: if latency_key in latency_metrics: headers.append(display_name) metric_columns.append((f"latency.{latency_key}", display_name, True)) # Format table lines = [] # Header header_parts = ["| " + " | ".join(headers) + " |"] divider_parts = ["|" + "|".join(["-" * (len(h) + 2) for h in headers]) + "|"] lines.append(header_parts[0]) lines.append(divider_parts[0]) # Data rows for thread in threads: thread_runs = runs_by_thread[thread] # Calculate average values across all runs for this thread count row_values = [str(thread)] for metric_key, display_name, is_latency in metric_columns: values = [] for run in thread_runs: if is_latency: # Extract latency metric latency_type = metric_key.replace("latency.", "") latency = run.metrics.get("latency", {}) value = latency.get(latency_type) else: # Extract throughput metric value = run.metrics.get(metric_key) if value is not None: values.append(value) # Calculate average if values: avg_value = sum(values) / len(values) if is_latency or "latency" in metric_key.lower(): # Format latency with 2 decimal places row_values.append(f"{avg_value:.2f}") else: # Format throughput as integer row_values.append(f"{avg_value:,.0f}") else: row_values.append("-") lines.append("| " + " | ".join(row_values) + " |") return "\n".join(lines) def _format_performance_table(runs: List[WorkloadRunData]) -> str: """Format performance data as Markdown table.""" if not runs: return "" # Determine which metrics to show has_tpmC = any("tpmC" in run.metrics for run in runs) has_tps = any("tps" in run.metrics for run in runs) has_qps = any("qps" in run.metrics for run in runs) has_latency = any("latency" in run.metrics for run in runs) # Build header - add Task ID column header_parts = ["| Task ID | Thread Count |"] divider_parts = ["|---------|--------------|"] if has_tpmC: header_parts.append(" tpmC |") divider_parts.append("------|") if has_tps: header_parts.append(" TPS |") divider_parts.append("-----|") if has_qps: header_parts.append(" QPS |") divider_parts.append("-----|") if has_latency: header_parts.append(" P95 Latency (ms) |") divider_parts.append("------------------|") header_parts.append(" P99 Latency (ms) |") divider_parts.append("------------------|") lines = ["".join(header_parts), "".join(divider_parts)] # Sort runs by task_id first, then by thread count sorted_runs = sorted( runs, key=lambda r: ( r.task_id if r.task_id else 0, r.params.get("thread", 0) if r.params.get("thread") else 0, ), ) # Build rows for run in sorted_runs: task_id = run.task_id if run.task_id else "N/A" thread_count = run.params.get("thread", "N/A") row_parts = [f"| {task_id} | {thread_count} |"] if has_tpmC: tpmC = run.metrics.get("tpmC", 0) row_parts.append(f" {tpmC:.2f} |" if tpmC else " - |") if has_tps: tps = run.metrics.get("tps", 0) row_parts.append(f" {tps:.2f} |" if tps else " - |") if has_qps: qps = run.metrics.get("qps", 0) row_parts.append(f" {qps:.2f} |" if qps else " - |") if has_latency: latency = run.metrics.get("latency", {}) p95 = latency.get("p95", 0) p99 = latency.get("p99", 0) row_parts.append(f" {p95:.2f} |" if p95 else " - |") row_parts.append(f" {p99:.2f} |" if p99 else " - |") lines.append("".join(row_parts)) return "\n".join(lines) def _format_detailed_results_table(runs: List[WorkloadRunData]) -> str: """Format detailed results as Markdown table.""" if not runs: return "" lines = [ "| Task ID | Node Name | Thread Count | Duration (min) | Status | Metrics |", "|---------|-----------|--------------|----------------|--------|---------|", ] # Sort by task_id first sorted_runs = sorted( runs, key=lambda r: (r.task_id if r.task_id else 0, r.node_name) ) for run in sorted_runs: task_id = run.task_id if run.task_id else "N/A" thread_count = run.params.get("thread", "N/A") duration = run.params.get("duration", "N/A") status = run.status # Format metrics summary - dynamically include all available metrics metrics_summary = [] # Add throughput metrics (check each one as they differ by workload) if "tpmC" in run.metrics: metrics_summary.append(f"tpmC: {run.metrics['tpmC']:.2f}") if "tps" in run.metrics: metrics_summary.append(f"TPS: {run.metrics['tps']:.2f}") if "qps" in run.metrics: metrics_summary.append(f"QPS: {run.metrics['qps']:.2f}") if "QphH" in run.metrics: metrics_summary.append(f"QphH: {run.metrics['QphH']:.2f}") # Add latency metrics (dynamically check which ones are available) if "latency" in run.metrics: latency = run.metrics["latency"] # Define preferred order for display latency_order = ["avg", "p50", "p90", "p95", "p99", "max", "min"] for key in latency_order: if key in latency: # Format label: "p95" -> "P95", "avg" -> "Avg" label = key.upper() if key.startswith("p") else key.title() metrics_summary.append(f"{label}: {latency[key]:.2f}ms") metrics_str = ", ".join(metrics_summary) if metrics_summary else "N/A" lines.append( f"| {task_id} | {run.node_name} | {thread_count} | {duration} | {status} | {metrics_str} |" ) return "\n".join(lines) def _generate_curve_data(runs: List[WorkloadRunData]) -> PerformanceCurveData: """Generate performance curve data from runs.""" curve = PerformanceCurveData() # Sort runs by thread count sorted_runs = sorted( runs, key=lambda r: (r.params.get("thread", 0) if r.params.get("thread") else 0), ) for run in sorted_runs: thread_count = run.params.get("thread") if not thread_count: continue tps = run.metrics.get("tps") or run.metrics.get("tpmC") p95 = run.metrics.get("latency", {}).get("p95") p99 = run.metrics.get("latency", {}).get("p99") curve.add_point(thread_count=thread_count, tps=tps, p95=p95, p99=p99) return curve def prepare_chart_data(report_data: SPUReportData) -> Dict: """Prepare data for Streamlit charts. Returns a dictionary with chart data organized by workflow type: { "bench-tpcc": { "tasks": [5, 8, 10], "tps_data": { 5: {"threads": [100, 200, ...], "values": [...]}, 8: {"threads": [100, 200, ...], "values": [...]}, }, "latency_p95_data": { 5: {"threads": [100, 200, ...], "values": [...]}, }, "latency_p99_data": {...}, "metric_type": "tpmC" # or "tps", "qps" } } """ chart_data = {} # Group workflows by type workflow_groups = {} for workflow in report_data.workflows: wf_type = workflow.workflow_type if wf_type not in workflow_groups: workflow_groups[wf_type] = [] workflow_groups[wf_type].append(workflow) # Process each workflow type for wf_type, workflows in workflow_groups.items(): # Collect all runs all_runs = [] task_ids = [] for workflow in workflows: all_runs.extend(workflow.runs) if workflow.task_id not in task_ids: task_ids.append(workflow.task_id) if not all_runs: continue # Determine primary metric (tpmC, tps, or qps) has_tpmC = any("tpmC" in run.metrics for run in all_runs) has_tps = any("tps" in run.metrics for run in all_runs) has_qps = any("qps" in run.metrics for run in all_runs) has_latency = any("latency" in run.metrics for run in all_runs) if has_tpmC: metric_type = "tpmC" metric_key = "tpmC" elif has_tps: metric_type = "tps" metric_key = "tps" elif has_qps: metric_type = "qps" metric_key = "qps" else: continue # No throughput metric # Organize data by task_id tps_data = {} latency_p95_data = {} latency_p99_data = {} for task_id in sorted(task_ids): task_runs = [r for r in all_runs if r.task_id == task_id] if not task_runs: continue # Sort by thread count task_runs.sort( key=lambda r: ( r.params.get("thread", 0) if r.params.get("thread") else 0 ) ) threads = [] tps_values = [] p95_values = [] p99_values = [] for run in task_runs: thread = run.params.get("thread") if not thread: continue threads.append(thread) # Get throughput metric tps_val = run.metrics.get(metric_key, 0) tps_values.append(tps_val if tps_val else None) # Get latency metrics if has_latency: latency = run.metrics.get("latency", {}) p95 = latency.get("p95") p99 = latency.get("p99") p95_values.append(p95 if p95 else None) p99_values.append(p99 if p99 else None) if threads: tps_data[task_id] = { "threads": threads, "values": tps_values, } if has_latency and p95_values: latency_p95_data[task_id] = { "threads": threads, "values": p95_values, } if has_latency and p99_values: latency_p99_data[task_id] = { "threads": threads, "values": p99_values, } chart_data[wf_type] = { "tasks": sorted(task_ids), "tps_data": tps_data, "latency_p95_data": latency_p95_data, "latency_p99_data": latency_p99_data, "metric_type": metric_type, } return chart_data def prepare_comparison_chart_data( comparison_report: SPUComparisonReport, ) -> Dict: """Prepare data for comparison charts. Returns a dictionary with comparison data organized by workflow type: { "bench-tpcc": { "threads": [100, 200, 400, ...], "metrics": { "tpmC": { "spu1_values": [1000, 2000, ...], "spu2_values": [1100, 2200, ...], "change_pct": [10.0, 10.0, ...], "is_latency": False, "display_name": "tpmC" }, "latency.p95": { "spu1_values": [50, 60, ...], "spu2_values": [45, 55, ...], "change_pct": [-10.0, -8.33, ...], "is_latency": True, "display_name": "P95 Latency" }, ... }, "spu1_name": "...", "spu2_name": "..." } } """ chart_data = {} for comparison in comparison_report.comparisons: wf_type = comparison.workflow_type # Get all threads threads = sorted(comparison.thread_data.keys()) # Collect all metrics from thread_data metrics_data = {} for thread in threads: thread_comp = comparison.thread_data[thread] # Process each metric in summary_stats for metric_name, ( stats1, stats2, improvement, ) in thread_comp.summary_stats.items(): if metric_name not in metrics_data: # Initialize metric data structure is_latency = "latency" in metric_name.lower() display_name = ( metric_name.replace("latency.", "").replace("_", " ").title() ) metrics_data[metric_name] = { "spu1_values": [], "spu2_values": [], "change_pct": [], "is_latency": is_latency, "display_name": display_name, } # Add values for this thread count metrics_data[metric_name]["spu1_values"].append(stats1.avg) metrics_data[metric_name]["spu2_values"].append(stats2.avg) metrics_data[metric_name]["change_pct"].append(improvement) chart_data[wf_type] = { "threads": threads, "metrics": metrics_data, "spu1_name": comparison.spu_1_identifier, "spu2_name": comparison.spu_2_identifier, } return chart_data