version: 1

source:
  name: data_foundation_finance
  type: tidbcloud
  database: finance
  domain: finance
  owner: data-platform
  connection_env: TIDB_SOURCE_DSN
  tidb_cloud_cli_profile: data-foundation-finance
  watermark_candidates:
    - system_modstamp
    - last_modified_date
    - updated_at
    - updated_time
    - sync_time
    - created_at
  tidb_cloud:
    cluster_id: "10321274501953903777"
    public_key_env: TIDB_CLOUD_PUBLIC_KEY
    private_key_env: TIDB_CLOUD_PRIVATE_KEY
    gcs_service_account_key_env: TIDB_EXPORT_GCS_SERVICE_ACCOUNT_KEY

target:
  project: gcp-digital-bi
  location: us-west2
  datasets:
    raw: data_foundation_finance_raw
    final: data_foundation_finance_final
    control: data_foundation_finance_control

tables:
  include:
    - "*"
  exclude:
    - "*_bak*"
    - health_test
  require_primary_key: true

gcs:
  bucket: gcp-digital-bi-pingcap-df-finance-ingestion
  location: us-west2
  snapshot_prefix: snapshots
  manifest_prefix: manifests
  runtime_log_prefix: logs

export:
  tidb_materialize_empty_tables: true
  tidb_max_attempts: 3
  parquet_compression: ZSTD
  tidb_filter_max_bytes: 400
  poll_interval_seconds: 30
  timeout_seconds: 7200

load:
  full_snapshot_delete: true

retention:
  gcs_snapshots_days: 35
  manifests_days: 365

schedule:
  extract_cron: "0 6 1,3,5 * *"
  load_cron: "0 8 1,3,5 * *"
  timezone: UTC