import pandas as pd from jinja2 import Template from pathlib import Path def generate_report(cluster_id, date, data, output_html): sum_total = sum(item['total'] for item in data) level_component = {} for item in data: level = item['level'] component = item['component'] total = item['total'] if level not in level_component: level_component[level] = {} level_component[level][component] = level_component[level].get(component, 0) + total sunburst_labels = ['Total'] sunburst_parents = [''] sunburst_values = [sum_total] for level in level_component: level_total = sum(level_component[level].values()) sunburst_labels.append(level) sunburst_parents.append('Total') sunburst_values.append(level_total) for component in level_component[level]: sunburst_labels.append(f"{component}") sunburst_parents.append(level) sunburst_values.append(level_component[level][component]) # 各level前20数据 level_top20 = {} for level in set(item['level'] for item in data): items = [item for item in data if item['level'] == level] level_top20[level] = sorted(items, key=lambda x: x['total'], reverse=True)[:20] error_components = {} for item in data: if item['level'] == 'ERROR': error_components[item['component']] = error_components.get(item['component'], 0) + item['total'] error_total = sum(error_components.values()) error_data = [{'component': k, 'total': v} for k, v in error_components.items()] new_data = [item for item in data if 'new' in item.get('tags', [])] new_levels = list(set(item['level'] for item in new_data)) template = Template(''' Log Analysis Report

Log Analysis Report

Cluster id: {{ cluster_id }}

Date: {{ date }}

Total Logs: {{ sum_total }}

New Logs (first appeared in the last week)

{% for item in new_data %} {% endfor %}
LevelComponentPositionTotalLog
{{ item.level }} {{ item.component }} {{ item.position }} {{ item.total }}
{{ item.log_content }}

ERROR Distribution (Total: {{ error_total }})

Top 20 per Level

''') html = template.render( cluster_id=cluster_id, date=date, sum_total=sum_total, sunburst_labels=sunburst_labels, sunburst_parents=sunburst_parents, sunburst_values=sunburst_values, level_top20=level_top20, error_data=error_data, error_total=error_total, new_data=new_data, new_levels=new_levels ) with open(output_html, 'w') as f: f.write(html) def update_statistic_csv(date, data, csv_path): Path(csv_path).parent.mkdir(parents=True, exist_ok=True) rows = [] all_total = 0 for level in data: for component, count in data[level].items(): rows.append({ "Level": level, "Component": component, date: count }) all_total = all_total + count rows.append({ "Level": "ALL_LEVEL", "Component": "ALL_COMPONENTS", date: all_total }) if not rows: return df_new = pd.DataFrame(rows) try: # 读取现有CSV文件 df_existing = pd.read_csv(csv_path) # 转换为多层索引 df_existing = df_existing.set_index(['Level', 'Component']) except FileNotFoundError: # 初始化空DataFrame df_existing = pd.DataFrame() df_existing.index = pd.MultiIndex.from_tuples([], names=['Level', 'Component']) # 合并新旧数据 if not df_existing.empty: # 合并现有数据 df_combined = df_existing.join( df_new.set_index(['Level', 'Component'])[date].to_frame(), how='outer' ) else: # 首次创建 df_combined = df_new.set_index(['Level', 'Component'])[date].to_frame() # 处理缺失值并转换类型 df_combined = df_combined.fillna(0).astype(int) # 按日期排序列 columns = sorted(df_combined.columns.tolist(), key=lambda x: x, reverse=True) df_combined = df_combined[columns] # 重置索引并排序 df_combined = df_combined.reset_index() df_combined = df_combined.sort_values(['Level', 'Component']) # 保存新CSV文件 df_combined.to_csv(csv_path, index=False)