version: 1

load:
  name: metadb
  engine: databend
  secrets_file: secrets.env

cloud_lake:
  dsn_env: DATABEND_DSN
  databases:
    # Load provisions only the raw layer. Staging and above are transform-layer
    # concerns owned by the transformation tool, not the load (ADR 0016).
    raw: metadb_raw

# Externally-managed source (ADR 0018): a pre-existing bucket produced by another pipeline.
# No extract stage or run manifest — the loader lists the bucket, infers the schema from a
# sample object, and loads incrementally per the chosen strategy. This block is storage-
# agnostic; the storage credential lives under `gcs` below.
#
# This is the external counterpart to ingestion/extract/bigquery/sources/metadb (ADR 0017):
# the same logical billing2_invoices table, but here the export is run by a pipeline we do
# NOT own, so the source contract is *provided* (inferred + committed under contracts/upstream/),
# not owned by an extract stage.
external:
  uri: gs://metadb/metadb/billing/billing2_invoices/
  incremental:
    # filename strategy: an epoch-ms extraction timestamp captured from each object name,
    # e.g. run-1778601958447-part-block-0-r-00000-snappy.parquet. Objects sharing it are one
    # snapshot, loaded together and stamped with _extraction_ts / _extraction_at.
    strategy: filename
    pattern: 'run-(?P<ts>\d+)-'
  delta: true
  table: billing2_invoices
  domain: billing
  owner: data-platform
  system: bigquery
  # The external producer does not enforce a primary key; declare it for raw dedup detection.
  primary_key: [id]

gcs:
  # Read credential for the COPY. Databend reads gcs:// via an external stage created with
  # this base64 service-account key (value in the ignored secrets.env).
  credential_env: METADB_GCS_CREDENTIAL
  # Read credential is a load-stage concern (no extract stage owns it): the reader SA + key
  # + objectViewer binding on the external bucket are provisioned by setup/grunt here.
  reader:
    project: gcp-digital-bi-dev

governance:
  classifications:
    - restricted
    - financial
    - customer_data
  classification_patterns:
    financial: "price|cost|invoice|payment|loss|amount|revenue|currency|billing|fee|charge"
    operational_sensitive: "token|secret|password|credential|payment_meta|payment_method|payment_code"

setup:
  databend:
    files:
      - ingestion/load/gcs_bulk_load/setup/databend/metadb/001_databases.sql

# Externally-managed source: its `source` contract is provided (bootstrapped by inference via
# `infer-source`, committed under contracts/upstream/). The load owns only the raw-snapshot
# product contract it generates from that upstream (ADR 0015/0016).
source_contracts:
  - ref: ingestion/load/gcs_bulk_load/sources/metadb/contracts/upstream/*.external.source.contract.yaml