version: 1

source:
  name: nexuscrm
  database: nexuscrm
  domain: crm
  owner: data-platform
  connection_env: TIDB_SOURCE_DSN
  secrets_file: secrets.env
  # Columns the engine treats as change watermarks when present on a table.
  # Source-specific (NexusCRM/Salesforce convention) — declared here, not in engine code.
  watermark_candidates:
    - SystemModstamp
    - LastModifiedDate
  tidb_cloud:
    # Prod (main branch). The SQL DSN (secrets.env) and the export --cluster-id both
    # target this cluster's main branch — no branch override.
    project_id: "1372813089454630337"
    cluster_id: "10996361222721502359"

tidb_cloud_cli_profile: pingcap-datalake

tables:
  # Extract every base table in the source database except the exclusions below.
  # Includes _System_* tables (owner-approved for prod onboarding — see governance).
  include:
    - "*"
  # fnmatch patterns to drop. _ops_* are leftover baseline/reference tables in prod;
  # EmailMessage is excluded (very large, ~74% of export volume — defer to a
  # watermark/incremental path if it's needed later).
  exclude:
    - "_ops_*"
    - "EmailMessage"
  # Skip tables without a primary key (not snapshot-diffable / not contract-worthy).
  require_primary_key: true

aws:
  profile: pingcap-datalake
  region: us-west-2

s3:
  region: us-west-2
  # Two buckets, split by sensitivity + lifecycle. These names/prefixes/retention are
  # the single source of truth: setup/grunt reads this block to provision the buckets.
  data:
    # Transient Parquet snapshots (sensitive). Short lifecycle — raw Cloud Lake is durable.
    bucket: pingcap-datalake-nexuscrm-export-247906201378-us-west-2
    snapshot_prefix: tidb-essential/snapshots/source=nexuscrm
  metadata:
    # Durable manifests + runtime logs (low-sensitivity provenance).
    bucket: pingcap-datalake-nexuscrm-meta-247906201378-us-west-2
    manifest_prefix: tidb-essential/manifests/source=nexuscrm
    runtime_log_prefix: tidb-essential/logs/source=nexuscrm
  # Env var names holding the S3 credentials TiDB Cloud export issues to write the
  # data bucket. Values stay in the ignored secrets.env; only the names are tracked.
  export_credentials:
    access_key_id_env: TIDB_EXPORT_AWS_ACCESS_KEY_ID
    secret_access_key_env: TIDB_EXPORT_AWS_SECRET_ACCESS_KEY

export:
  file_type: parquet
  parquet_compression: ZSTD

freshness:
  cadence: daily
  degraded_after_hours: 30
  blocked_after_hours: 48

retention:
  # S3 Parquet is transient — landed, COPY INTO raw, then expired. A week gives delta
  # (needs prev snapshot) + reprocessing slack; raw Cloud Lake is the durable copy.
  s3_snapshots_days: 7
  manifests_days: 365
  raw_successful_snapshots: 30

governance:
  # Baseline classifications applied to every table in this source.
  classifications:
    - restricted
    - customer_data
  # Column/table-name regex → classification. Source-specific governance rules;
  # the engine applies them generically, it does not hardcode them.
  classification_patterns:
    pii: "email|phone|mobile|fax|first.?name|last.?name|street|address|city|state|postal|country|birth"
    financial: "amount|revenue|arr|mrr|acv|cost|budget|price|currency|payment|billing|invoice|forecast"
    operational_sensitive: "token|secret|password|credential|endpoint|url|uri|login|auth|session|sourceip|ip_address"
  classification_name_rules:
    - name_prefix: _System_
      add: operational_sensitive
  notes:
    - Raw NexusCRM snapshots are restricted by default.
    - Full-schema extraction including _System_* tables is owner-approved (data-platform)
      for prod onboarding; _System_* tables are tagged operational_sensitive on extract.

contracts:
  output_dir: contracts/generated