version: 1

# Data Foundation — finance "datahub" extract source (ADR 0017): the source-system axis of
# ingestion, keyed on TiDB Cloud. The `finance` database on the finance Essential cluster is
# PingCAP's internal finance data hub (revenue recognition, billing, orders, collections,
# AR) — it even carries a `datahub_created_time` column. This stage introspects
# information_schema over SQL and `ticloud serverless export create`s each kept table to
# Parquet in an S3 bucket we own, owning the S3-snapshot `source` contract for the objects
# it writes (ADR 0015).
#
# NOTE: this cluster lives in a DIFFERENT TiDB Cloud org than nexuscrm, so it needs its own
# `ticloud` CLI profile (tidb_cloud_cli_profile below) authenticated with that org's API key.

source:
  # Logical source name (folder, contracts, prefixes). The underlying TiDB database is just
  # `finance` — keep `database` as-is; introspection and the export filter key on it.
  name: data_foundation_finance
  database: finance
  domain: finance
  owner: data-platform
  connection_env: TIDB_SOURCE_DSN
  secrets_file: secrets.env
  # Columns the engine treats as change watermarks when present on a table.
  # Source-specific convention — declared here, not in engine code. Ordered most- to
  # least-authoritative; observed across the finance schema's datetime/timestamp columns.
  watermark_candidates:
    - system_modstamp
    - last_modified_date
    - updated_at
    - updated_time
    - sync_time
    - created_at
  tidb_cloud:
    # Project "Data Platform" / org 1372813089208341658. The SQL DSN (secrets.env) and the
    # export --cluster-id both target this cluster's main branch — no branch override.
    project_id: "1372813089454565467"
    cluster_id: "10321274501953903777"
    # API key env-var names the ticloud profile is bootstrapped from. Values live in the
    # ignored secrets.env (per-source, so they travel with the source on a cloud runner —
    # not bound to ~/.ticloud). Defaults shown; only the names are tracked.
    credentials:
      public_key_env: TIDB_CLOUD_PUBLIC_KEY
      private_key_env: TIDB_CLOUD_PRIVATE_KEY

# Name of the ticloud profile for this source's org (1372813089208341658), distinct from the
# nexuscrm `pingcap-datalake` profile. `ticloud` reads keys only from a local config file, not
# the environment, so the engine bootstraps this profile from the secrets above at runtime
# (`ensure-profile`, also called automatically before start-export / write-manifest). No
# manual `ticloud config create`, no dependency on a specific machine.
tidb_cloud_cli_profile: data-foundation-finance

tables:
  # Extract every base table in the finance database except the exclusions below.
  include:
    - "*"
  # fnmatch patterns to drop. *_bak* are manual backup copies (marketplace_collection_bak,
  # bq_account_bak, monthly_cloud_*_bak, ...) — not the system of record. health_test is a
  # 1-column heartbeat table (also dropped by require_primary_key).
  exclude:
    - "*_bak*"
    - "health_test"
  # Skip tables without a primary key (not snapshot-diffable / not contract-worthy).
  # Drops ps_man_days (no PK); kept tables all key on `id`.
  require_primary_key: true

aws:
  profile: pingcap-datalake
  region: us-west-2

s3:
  # Co-located with the loader + Cloud Lake in us-west-2 (cross-region read from the
  # ap-northeast-1 cluster on export). Two buckets, split by sensitivity + lifecycle; these
  # names/prefixes/retention are the single source of truth — setup/grunt reads this block.
  region: us-west-2
  data:
    # Transient Parquet snapshots (sensitive financial data). Short lifecycle — raw Cloud
    # Lake is the durable copy. Bucket token abbreviated to `df-finance`: S3 bucket names
    # forbid underscores and cap at 63 chars, so the full source name does not fit.
    bucket: pingcap-datalake-df-finance-export-247906201378-us-west-2
    snapshot_prefix: tidb-essential/snapshots/source=data_foundation_finance
  metadata:
    # Durable manifests + runtime logs (low-sensitivity provenance).
    bucket: pingcap-datalake-df-finance-meta-247906201378-us-west-2
    manifest_prefix: tidb-essential/manifests/source=data_foundation_finance
    runtime_log_prefix: tidb-essential/logs/source=data_foundation_finance
  # Env var names holding the S3 credentials TiDB Cloud export issues to write the data
  # bucket. Values stay in the ignored secrets.env; only the names are tracked.
  export_credentials:
    access_key_id_env: TIDB_EXPORT_AWS_ACCESS_KEY_ID
    secret_access_key_env: TIDB_EXPORT_AWS_SECRET_ACCESS_KEY

export:
  file_type: parquet
  parquet_compression: ZSTD

# Extract runs on the 1st, 3rd, and 5th of every month (finance close cycle), not daily.
# No orchestrator exists yet; this is declared intent that a future timetable reads —
# `cron` is the canonical form.
schedule:
  cadence: monthly
  days_of_month: [1, 3, 5]
  cron: "0 6 1,3,5 * *"
  timezone: UTC

freshness:
  # Cadence is declared once, canonically, in the `schedule` block above. These thresholds
  # only need the resulting gap: runs on days 1/3/5, so the longest planned gap is day 5 ->
  # day 1 of the next month (~24-27 days). Thresholds sit above that gap so a normal
  # end-of-month quiet period is not flagged; they catch a fully missed close cycle.
  # (PT720H = 30d, PT792H = 33d.)
  degraded_after_hours: 720
  blocked_after_hours: 792

retention:
  # S3 Parquet is transient — landed, COPY INTO raw, then expired; raw Cloud Lake is the
  # durable copy. Sized to the monthly cadence: keep a couple of close cycles for delta
  # (needs prev snapshot) + reprocessing slack.
  s3_snapshots_days: 35
  manifests_days: 365
  raw_successful_snapshots: 12

governance:
  # Baseline classifications applied to every table in this source. Finance data hub:
  # revenue/billing/AR plus Salesforce account/org records.
  classifications:
    - restricted
    - financial
    - customer_data
  # Column/table-name regex → classification. Source-specific governance rules; the engine
  # applies them generically, it does not hardcode them. Every label must be in the fixed
  # ADR-0011 governance set.
  classification_patterns:
    pii: "email|phone|mobile|fax|first.?name|last.?name|street|address|city|state|postal|country|birth"
    financial: "amount|revenue|arr|mrr|acv|cost|budget|price|currency|payment|billing|invoice|forecast|collection|receivable|revrec|fee|charge|tax"
    operational_sensitive: "token|secret|password|credential|endpoint|url|uri|login|auth|session|sourceip|ip_address"
  notes:
    - Raw finance snapshots are restricted + financial by default.

contracts:
  output_dir: contracts/generated