version: 1

# Demo of the external-incremental mode on S3 with the `last_modified` strategy (ADR 0018):
# a bucket filled by a pipeline we do not extract, where files arrive over time and the
# incremental cursor is each object's S3 last-modified time (a dltHub filesystem-incremental
# shape). Fill in a real bucket + S3 read credentials to run:
#   infer-source -> generate-contracts -> load-run

load:
  name: events
  engine: databend
  secrets_file: secrets.env

cloud_lake:
  dsn_env: DATABEND_DSN
  databases:
    # Load provisions only the raw layer (ADR 0016).
    raw: events_raw

external:
  uri: s3://REPLACE-WITH-EXTERNAL-BUCKET/events/
  incremental:
    # last_modified strategy: each object is its own increment, loaded when its S3
    # LastModified is newer than max(_source_last_modified_ms) already in the raw table.
    # Each row is stamped with _source_file + _source_last_modified(_ms).
    strategy: last_modified
  delta: true
  table: events
  domain: telemetry
  owner: data-platform
  system: external-s3
  # The external producer does not enforce a key; declare it for raw dedup detection.
  primary_key: [event_id]

s3:
  region: us-west-2
  # S3 read credentials for the Cloud Lake COPY + listing. The bucket is externally owned,
  # so its owner grants this principal read; values live in the ignored secrets.env.
  credentials:
    access_key_id_env: EVENTS_S3_ACCESS_KEY_ID
    secret_access_key_env: EVENTS_S3_SECRET_ACCESS_KEY
    # Optional: set for temporary/STS/SSO credentials (adds SESSION_TOKEN to the COPY).
    session_token_env: EVENTS_S3_SESSION_TOKEN

governance:
  classifications:
    - internal

setup:
  databend:
    files:
      - ingestion/load/s3_bulk_load/setup/databend/events/001_databases.sql

source_contracts:
  - ref: ingestion/load/s3_bulk_load/sources/events/contracts/upstream/*.external.source.contract.yaml