version: 1

source:
  name: nexuscrm
  type: tidbcloud
  database: nexuscrm
  domain: crm
  owner: data-platform
  connection_env: TIDB_SOURCE_DSN
  tidb_cloud_cli_profile: pingcap-datalake
  watermark_candidates:
    - SystemModstamp
    - LastModifiedDate
  tidb_cloud:
    cluster_id: "10809370594661512983"
    public_key_env: TIDB_CLOUD_PUBLIC_KEY
    private_key_env: TIDB_CLOUD_PRIVATE_KEY
    gcs_service_account_key_env: TIDB_EXPORT_GCS_SERVICE_ACCOUNT_KEY

target:
  project: gcp-digital-bi
  location: us-west2
  datasets:
    raw: nexuscrm_raw
    final: nexuscrm_final
    control: nexuscrm_control

tables:
  include:
    - "*"
  exclude:
    - "_ops_*"
    - EmailMessage
  require_primary_key: true

gcs:
  bucket: gcp-digital-bi-pingcap-nexuscrm-ingestion
  location: us-west2
  snapshot_prefix: snapshots
  manifest_prefix: manifests
  runtime_log_prefix: logs

export:
  tidb_materialize_empty_tables: true
  tidb_max_attempts: 3
  parquet_compression: ZSTD
  tidb_filter_max_bytes: 10000
  tidb_fault_domains:
    - name: business
      include:
        - "*"
      exclude:
        - "_System_*"
    - name: system
      include:
        - "_System_*"
  poll_interval_seconds: 30
  timeout_seconds: 7200

load:
  full_snapshot_delete: true
  manifest_freshness_max_age_hours: 30

retention:
  gcs_snapshots_days: 7
  manifests_days: 365

schedule:
  extract_cron: "0 1 * * *"
  load_cron: "30 1 * * *"
  timezone: UTC