name: ingestion-load-assets
purpose: Local Airflow registration for manifest-driven and external-incremental load assets.
load_asset_units:
  - name: data-foundation-finance-manifest-load
    mode: manifest
    dag_id: orchestration_load_data_foundation_finance
    task_id: load_data_foundation_finance_manifest
    owner: data-platform
    start_date: "2026-01-01T00:00:00Z"
    catchup: false
    max_active_runs: 1
    tags:
      - orchestration
      - assets
      - ingestion
      - load
      - manifest
    runner:
      executable_path: orchestration/airflow/bin/load-runner
      uv_project_environment: /tmp/pingcap-datalake-load-venv
    loader_module: ingestion.load.s3_bulk_load.engine.cli
    produced_by_ref: ingestion.load.s3_bulk_load.engine.load
    source_path: ingestion/load/s3_bulk_load/sources/data_foundation_finance/source.yaml
    manifest:
      asset:
        name: ingestion.extract.tidbcloud.data_foundation_finance.manifest
        uri: x-datalake://ingestion/manifest/tidbcloud/data_foundation_finance
        group: manifest
        extra:
          handoff: run_manifest
          extract_source_path: ingestion/extract/tidbcloud/sources/data_foundation_finance/source.yaml
      path_env: DATA_FOUNDATION_FINANCE_LOAD_MANIFEST
      event_extra_keys:
        - manifest_uri
        - manifest_path
  - name: nexuscrm-manifest-load
    mode: manifest
    dag_id: orchestration_load_nexuscrm
    task_id: load_nexuscrm_manifest
    owner: data-platform
    start_date: "2026-01-01T00:00:00Z"
    catchup: false
    max_active_runs: 1
    tags:
      - orchestration
      - assets
      - ingestion
      - load
      - manifest
      - nexuscrm
    runner:
      executable_path: orchestration/airflow/bin/load-runner
      uv_project_environment: /tmp/pingcap-datalake-load-venv
    loader_module: ingestion.load.s3_bulk_load.engine.cli
    produced_by_ref: ingestion.load.s3_bulk_load.engine.load
    source_path: ingestion/load/s3_bulk_load/sources/nexuscrm/source.yaml
    manifest:
      asset:
        name: ingestion.extract.tidbcloud.nexuscrm.manifest
        uri: x-datalake://ingestion/manifest/tidbcloud/nexuscrm
        group: manifest
        extra:
          handoff: run_manifest
          extract_source_path: ingestion/extract/tidbcloud/sources/nexuscrm/source.yaml
      path_env: NEXUSCRM_LOAD_MANIFEST
      event_extra_keys:
        - manifest_uri
        - manifest_path
  - name: metadb-external-incremental-poll
    mode: external_incremental
    dag_id: orchestration_load_metadb_external_incremental
    task_id: poll_metadb_external_incremental
    owner: data-platform
    start_date: "2026-01-01T00:00:00Z"
    catchup: false
    max_active_runs: 1
    poll_schedule: "17 * * * *"
    tags:
      - orchestration
      - assets
      - ingestion
      - load
      - external-incremental
    runner:
      executable_path: orchestration/airflow/bin/load-runner
      uv_project_environment: /tmp/pingcap-datalake-load-venv
    loader_module: ingestion.load.gcs_bulk_load.engine.cli
    produced_by_ref: ingestion.load.gcs_bulk_load.engine.load
    source_path: ingestion/load/gcs_bulk_load/sources/metadb/source.yaml
notes:
  - Manifest-driven loads are scheduled by a manifest handoff asset; the task resolves the concrete manifest path from asset event metadata or the configured path_env.
  - External-incremental loads use a cron poll because the loader self-watermarks and drift-gates against the committed source contract.