"""Alibaba Cloud billing exporter built on BigQuery daily tables.""" from __future__ import annotations import json from collections import defaultdict from dataclasses import dataclass from datetime import date, datetime, timedelta from pathlib import Path from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple from google.cloud import bigquery from ..data.csv_handler import CSVHandler from .environments import EnvironmentManager ALIBABA_DIR = Path(__file__).resolve().parents[2] @dataclass class ExportStats: """Summary of a single export run.""" queried_days: int rows_returned: int files_written: int start_date: str end_date: str class AlibabaCloudBillingExporter: """Fetch billing data from BigQuery and persist monthly CSV snapshots.""" def __init__( self, config_path: Optional[str] = None, output_dir: Optional[Path] = None, ) -> None: self.config = self._load_config(config_path) self.dataset_name: str = self.config.get("datasetName", "gcp-digital-bi.alibaba_cloud") self.daily_prefix: str = self.config.get("dailyTablePrefix", "daily_") self.monthly_prefix: str = self.config.get("monthlyTablePrefix", "monthly_") self.daily_prefix_en: str = self.config.get("dailyTablePrefixEn", "daily_en_") self.monthly_prefix_en: str = self.config.get("monthlyTablePrefixEn", "monthly_en_") self.english_start_date: date = self._parse_config_date( self.config.get("englishBillingStartDate"), fallback=date(2025, 12, 1), ) self.exclude_recent_days: int = int( self.config.get("settings", {}).get("excludeRecentDays", 1) ) self.default_days: int = int(self.config.get("settings", {}).get("defaultDays", 7)) resolved_output = self._resolve_path( output_dir or self.config.get("dataDir") or "alibaba_cloud_billing_exports" ) resolved_output.mkdir(parents=True, exist_ok=True) self.output_dir = resolved_output self.csv_handler = CSVHandler() self.bq_client = bigquery.Client() self._daily_tables_cache: Optional[Set[str]] = None default_env = self.config.get("defaultEnvironment", "unassigned") self.env_manager = EnvironmentManager( environments_config=self.config.get("environments", {}), output_dir=self.output_dir, default_environment=default_env, cloud_webhook_url=self.config.get("cloudWebhookUrl") or self.config.get("cloud_webhook_url"), ) # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ def export( self, start_date: date, end_date: date, environments: Optional[Sequence[str]] = None, account_ids: Optional[Sequence[str]] = None, merge_existing: bool = True, dry_run: bool = False, ) -> ExportStats: """Export billing data for the given window.""" if start_date > end_date: raise ValueError("start_date cannot be after end_date") # Respect the configured freshness window. safe_end_date = self._get_safe_end_date(end_date) if safe_end_date < start_date: raise ValueError( "Date range falls entirely within the excluded recent-days window; " "please choose an earlier end date." ) environment_filter = self._normalise_environments(environments) account_filter = self._resolve_account_filters(environment_filter, account_ids) rows = self._query_billing_rows(start_date, safe_end_date, account_filter) queried_days = (safe_end_date - start_date).days + 1 unassigned_accounts = self._find_unassigned_accounts(rows, environment_filter) if unassigned_accounts: self._print_unassigned_notice(unassigned_accounts) if dry_run: print("Dry-run mode: previewing first 5 rows") for sample in rows[:5]: print(sample) return ExportStats( queried_days=queried_days, rows_returned=len(rows), files_written=0, start_date=start_date.isoformat(), end_date=safe_end_date.isoformat(), ) files_written = self._write_monthly_csvs(rows, merge_existing, environment_filter) return ExportStats( queried_days=queried_days, rows_returned=len(rows), files_written=files_written, start_date=start_date.isoformat(), end_date=safe_end_date.isoformat(), ) def resolve_date_range( self, start_date: Optional[str] = None, end_date: Optional[str] = None, days: Optional[int] = None, last_week: bool = False, ) -> Tuple[date, date]: """Normalise CLI date parameters into concrete boundaries.""" today = datetime.now().date() if last_week: days = 7 if days is not None: if days < 1: raise ValueError("days must be >= 1") end_date = today start_date = today - timedelta(days=days - 1) if start_date is None: start = today - timedelta(days=self.default_days - 1) elif isinstance(start_date, str): start = datetime.strptime(start_date, "%Y-%m-%d").date() else: start = start_date if end_date is None: end = today elif isinstance(end_date, str): end = datetime.strptime(end_date, "%Y-%m-%d").date() else: end = end_date return (start, end) # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ def _resolve_path(self, candidate: Optional[Path]) -> Path: if candidate is None: return ALIBABA_DIR if isinstance(candidate, (str, bytes)): candidate = Path(candidate) if candidate.is_absolute(): return candidate return ALIBABA_DIR / candidate def _load_config(self, config_path: Optional[str]) -> Dict: if config_path: path = Path(config_path) if not path.is_absolute(): candidate = Path.cwd() / path if candidate.exists(): path = candidate else: path = ALIBABA_DIR / path else: path = ALIBABA_DIR / "config.json" if not path.exists(): print(f"Warning: Alibaba Cloud config not found at {path}, using defaults.") return {} with path.open("r", encoding="utf-8") as handle: return json.load(handle) def _parse_config_date(self, value: Optional[str], fallback: date) -> date: if not value: return fallback try: return datetime.strptime(str(value), "%Y-%m-%d").date() except ValueError: print( "Warning: invalid englishBillingStartDate value " f"'{value}', expected YYYY-MM-DD; using {fallback.isoformat()}" ) return fallback def _get_safe_end_date(self, requested_end: date) -> date: """Enforce freshness buffer to minimise partial data exports.""" today = datetime.now().date() safe_date = today - timedelta(days=self.exclude_recent_days) return min(requested_end, safe_date) def _normalise_environments(self, environments: Optional[Sequence[str]]) -> List[str]: if not environments or "all" in environments: return self.env_manager.list_environments() existing_envs = set(self.env_manager.list_environments()) resolved = [env for env in environments if env in existing_envs] if not resolved: raise ValueError( f"No valid environments found in {environments}. " f"Available: {', '.join(sorted(existing_envs))}" ) return resolved def _resolve_account_filters( self, environments: Sequence[str], explicit_account_ids: Optional[Sequence[str]], ) -> List[str]: account_ids: List[str] = [] for env in environments: account_ids.extend(self.env_manager.accounts_for_environment(env)) if explicit_account_ids: account_ids.extend([str(account) for account in explicit_account_ids]) # De-duplicate while retaining order for predictability. seen = set() filtered: List[str] = [] for account in account_ids: account = str(account).strip() if account and account not in seen: filtered.append(account) seen.add(account) return filtered def _query_billing_rows( self, start_date: date, end_date: date, account_ids: Optional[Sequence[str]], ) -> List[Dict[str, str]]: """Query BigQuery and shape results into dictionaries.""" start_suffix = start_date.strftime("%Y%m%d") end_suffix = end_date.strftime("%Y%m%d") account_filter = "" if account_ids: literals = ", ".join(f"'{account}'" for account in account_ids) account_filter = ( "AND CAST(IFNULL(owner_account_id, account_id) AS STRING) " f"IN UNNEST([{literals}])" ) source_queries = [] for prefix, range_start, range_end, use_english in self._daily_table_ranges( start_date, end_date ): range_start_suffix = range_start.strftime("%Y%m%d") range_end_suffix = range_end.strftime("%Y%m%d") table_ids = self._resolve_daily_tables(range_start, range_end, use_english) for table_id in table_ids: if use_english: source_queries.append( f""" SELECT DATE(partition_date) AS day, CAST(IFNULL(owner_account_id, account_id) AS STRING) AS owner_account_id, IFNULL(owner_account_name, account_name) AS owner_account_name, COALESCE(cost_center, resource_group) AS financial_unit, product_code, product_name AS product, NULL AS product_detail_code, product_detail, business_type, region, zone AS availability_zone, amount_after_discount, pretax_gross_amount AS list_price, invoice_discount AS discount_amount, deducted_by_coupons AS coupon_deduction, currency AS pricing_currency FROM `{self.dataset_name}.{table_id}` WHERE 1=1 {account_filter} """ ) else: source_queries.append( f""" SELECT DATE(partition_date) AS day, CAST(IFNULL(owner_account_id, account_id) AS STRING) AS owner_account_id, IFNULL(owner_account, account) AS owner_account_name, financial_unit, product_code, product, product_detail_code, product_detail, business_type, region, availability_zone, amount_after_discount, list_price, discount_amount, coupon_deduction, pricing_currency FROM `{self.dataset_name}.{table_id}` WHERE 1=1 {account_filter} """ ) if not source_queries: return [] query = f""" WITH source AS ( {" UNION ALL ".join(source_queries)} ) SELECT day, owner_account_id AS account_id, ANY_VALUE(owner_account_name) AS account_name, ANY_VALUE(financial_unit) AS financial_unit, product_code, product, ANY_VALUE(product_detail_code) AS product_detail_code, product_detail, ANY_VALUE(business_type) AS business_type, region, availability_zone, SUM(amount_after_discount) AS net_cost, SUM(list_price) AS list_price, SUM(discount_amount) AS discount_amount, SUM(coupon_deduction) AS coupon_deduction, ANY_VALUE(pricing_currency) AS currency FROM source GROUP BY day, owner_account_id, product_code, product, product_detail, region, availability_zone HAVING net_cost IS NOT NULL AND ABS(net_cost) > 0 ORDER BY day ASC, owner_account_id ASC, product ASC """ job = self.bq_client.query(query) results = job.result() rows: List[Dict[str, str]] = [] for record in results: rows.append( { "date": record["day"].isoformat(), "account_id": record["account_id"], "account_name": record["account_name"] or "", "financial_unit": record["financial_unit"] or "", "product_code": record["product_code"] or "", "product": record["product"] or "", "product_detail": record["product_detail"] or "", "business_type": record["business_type"] or "", "region": record["region"] or "", "availability_zone": record["availability_zone"] or "", "cost": f"{float(record['net_cost'] or 0):.6f}", "list_price": f"{float(record['list_price'] or 0):.6f}", "discount_amount": f"{float(record['discount_amount'] or 0):.6f}", "coupon_deduction": f"{float(record['coupon_deduction'] or 0):.6f}", "currency": record["currency"] or "", } ) return rows def _find_unassigned_accounts( self, rows: Iterable[Dict[str, str]], environments: Sequence[str], ) -> List[str]: """Identify accounts routed to the default environment.""" default_env = self.env_manager.default_environment unassigned: List[str] = [] for row in rows: account_id = row.get("account_id") if not account_id: continue environment = self.env_manager.resolve_environment_for_account(account_id) if environment == default_env and account_id not in unassigned: unassigned.append(account_id) # Only warn if the default environment was filtered out, because rows for those # accounts will be skipped entirely. When the default environment is included, # those accounts are being properly processed (no warning needed). if environments and default_env not in environments: return unassigned return [] def _daily_table_ranges( self, start_date: date, end_date: date, ) -> List[Tuple[str, date, date, bool]]: if end_date < self.english_start_date: return [(self.daily_prefix, start_date, end_date, False)] if start_date >= self.english_start_date: return [(self.daily_prefix_en, start_date, end_date, True)] legacy_end = self.english_start_date - timedelta(days=1) return [ (self.daily_prefix, start_date, legacy_end, False), (self.daily_prefix_en, self.english_start_date, end_date, True), ] def _resolve_daily_tables( self, start_date: date, end_date: date, use_english: bool, ) -> List[str]: prefix = self.daily_prefix_en if use_english else self.daily_prefix available = self._available_daily_tables() resolved: List[str] = [] cursor = start_date while cursor <= end_date: table_id = f"{prefix}{cursor.strftime('%Y%m%d')}" if table_id in available: resolved.append(table_id) cursor += timedelta(days=1) return resolved def _available_daily_tables(self) -> Set[str]: if self._daily_tables_cache is not None: return self._daily_tables_cache try: tables = list(self.bq_client.list_tables(self.dataset_name)) self._daily_tables_cache = { tbl.table_id for tbl in tables if tbl.table_id.startswith(self.daily_prefix) or tbl.table_id.startswith(self.daily_prefix_en) } except Exception as exc: # pragma: no cover - diagnostics only print(f"Debug: failed to list tables for cache: {exc}") self._daily_tables_cache = set() return self._daily_tables_cache def _print_unassigned_notice(self, accounts: Sequence[str]) -> None: if not accounts: return sample = ", ".join(accounts[:5]) suffix = " ..." if len(accounts) > 5 else "" print( "Notice: accounts without environment mapping were assigned to " f"'{self.env_manager.default_environment}' ({sample}{suffix}). " "Update config.environments to avoid this." ) def _write_monthly_csvs( self, rows: Iterable[Dict[str, str]], merge_existing: bool, environments: Sequence[str], ) -> int: """Persist records grouped by account/month into environment directories.""" grouped: Dict[Tuple[str, str, str], List[Dict[str, str]]] = defaultdict(list) for row in rows: month = row["date"][:7] account_id = row["account_id"] environment = self.env_manager.resolve_environment_for_account(account_id) if environment not in environments: continue grouped[(environment, account_id, month)].append(row) files_written = 0 for (environment, account_id, month), records in grouped.items(): env_dir = self.output_dir / environment env_dir.mkdir(parents=True, exist_ok=True) filename = f"alibaba_cloud_{account_id}_{month}.csv" path = env_dir / filename if merge_existing: existing_rows = self.csv_handler.load_rows(path) merged = self.csv_handler.merge_rows(existing_rows, records) else: merged = sorted( records, key=lambda row: ( row.get("date", ""), row.get("product", ""), row.get("region", ""), ), ) self.csv_handler.write_rows(path, merged) files_written += 1 return files_written