#!/usr/bin/env python3 """One-off helper to backfill existing GCP CSV exports into MySQL.""" from __future__ import annotations import argparse import csv import json from datetime import datetime from pathlib import Path from typing import Dict, List, Optional, Sequence import sys GCP_DIR = Path(__file__).resolve().parents[1] from gcp.mysql_writer import GCPMySQLWriter DATE_FORMAT = "%Y-%m-%d" def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Backfill GCP billing CSV exports into MySQL", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--config", default=str((GCP_DIR / "config.json").resolve()), help="Path to gcp/config.json (or equivalent)", ) parser.add_argument( "--exports-root", default=str((GCP_DIR / "gcp_billing_exports").resolve()), help="Root directory containing environment folders with CSV exports", ) parser.add_argument( "--env", dest="environments", nargs="+", help="Limit to specific environments (e.g. prod nonprod)", ) parser.add_argument( "--project", dest="project_ids", action="append", help="Restrict to specific project IDs (repeatable)", ) parser.add_argument( "--limit", type=int, help="Maximum number of files to process", ) parser.add_argument( "--dry-run", action="store_true", help="Print actions without writing to MySQL", ) parser.add_argument( "--include-inactive", action="store_true", help="Include projects not marked active in gcp_account_details.csv", ) return parser.parse_args() def load_config(path: str) -> Dict: config_path = Path(path) if not config_path.exists(): print(f"未找到配置文件: {config_path}") sys.exit(1) with config_path.open("r", encoding="utf-8") as handle: return json.load(handle) def discover_env_dirs(root: Path) -> Sequence[Path]: if not root.exists(): return [] return sorted(path for path in root.iterdir() if path.is_dir()) def load_rows(csv_path: Path) -> List[Dict[str, str]]: with csv_path.open("r", encoding="utf-8") as handle: reader = csv.DictReader(handle) return [dict(row) for row in reader] def parse_dates(rows: List[Dict[str, str]]) -> List[datetime.date]: dates: List[datetime.date] = [] for row in rows: raw = (row.get("date") or "").strip() if not raw: continue try: dates.append(datetime.strptime(raw, DATE_FORMAT).date()) except ValueError: continue return dates def main() -> None: args = parse_args() exports_root = Path(args.exports_root).expanduser().resolve() config = load_config(args.config) mysql_writer = GCPMySQLWriter(config) if not mysql_writer.enabled: print("MySQL 同步未启用,请在 gcp/config.json 中设置 mysqlConfig.enabled = true 并提供连接信息。") sys.exit(1) env_whitelist = set(args.environments or []) project_filter = set(args.project_ids or []) active_projects = None if args.include_inactive else mysql_writer.active_projects total_files = 0 synced_files = 0 skipped_files = 0 env_dirs = discover_env_dirs(exports_root) if not env_dirs: print(f"目录 {exports_root} 未找到任何环境子目录") sys.exit(1) for env_dir in env_dirs: if args.limit and synced_files >= args.limit: break env_name = env_dir.name if env_whitelist and env_name not in env_whitelist: continue csv_files = sorted(path for path in env_dir.glob("*.csv") if path.is_file()) if not csv_files: print(f"环境 {env_name} 未找到 CSV 文件 (目录: {env_dir})") continue print(f"处理环境 {env_name} ({len(csv_files)} 个文件)") for csv_path in csv_files: if args.limit and synced_files >= args.limit: break rows = load_rows(csv_path) if not rows: skipped_files += 1 print(f" - 跳过 {csv_path.name} (空文件)") continue project_id = (rows[0].get("project_id") or "").strip() if not project_id: parts = csv_path.stem.split("_") if len(parts) >= 3: project_id = parts[-2] if not project_id: skipped_files += 1 print(f" - 跳过 {csv_path.name} (无法确定项目ID)") continue if project_filter and project_id not in project_filter: continue if active_projects is not None and project_id not in active_projects: skipped_files += 1 print(f" - 跳过 {csv_path.name} (项目 {project_id} 未在活跃账户列表中)") continue dates = parse_dates(rows) if not dates: skipped_files += 1 print(f" - 跳过 {csv_path.name} (缺少有效日期)") continue project_name = rows[0].get("project_name") or project_id start_date = min(dates) end_date = max(dates) total_files += 1 summary = ( f" - {csv_path.name}: {len(rows)} 行, {start_date} 至 {end_date}, 项目 {project_name}" ) if args.dry_run: print(f"{summary} [DRY RUN]") continue try: mysql_writer.sync( project_id=project_id, project_name=project_name, environment=env_name, records=rows, start_date=start_date, end_date=end_date, source_file=csv_path, metadata={"notes": "gcp/scripts/backfill_mysql.py"}, ) synced_files += 1 print(f"{summary} -> 已同步") except Exception as exc: # pylint: disable=broad-except skipped_files += 1 print(f" ✗ {csv_path.name} 同步失败: {exc}") print("\n完成掃描") print(f"总文件: {total_files}") print(f"已同步: {synced_files}") print(f"已跳过: {skipped_files}") if __name__ == "__main__": main()