version: 1

source:
  name: helix
  type: mysql_query
  database: helix
  domain: cloud-governance
  owner: it
  connection_env: TIDB_SOURCE_DSN
  watermark_candidates:
    - source_updated_at
    - deleted_at
  query:
    tables:
      - table: cloud_accounts
        sql_file: ../queries/helix/cloud_accounts.sql
        primary_key:
          - scope_id
        classifications:
          - internal
          - cloud-account-inventory
          - contains-owner-email
        columns:
          - {name: management_id, data_type: string, nullable: false}
          - {name: account_id, data_type: string, nullable: false}
          - {name: class_type, data_type: string, nullable: false}
          - {name: owner, data_type: string, nullable: true}
          - {name: name, data_type: string, nullable: true}
          - {name: owner_email, data_type: string, nullable: true}
          - {name: description, data_type: string, nullable: true}
          - {name: provider, data_type: string, nullable: false}
          - {name: department_lark_id, data_type: string, nullable: true}
          - {name: etl_loaded_at, data_type: timestamp, nullable: false}
          - {name: department, data_type: string, nullable: true}
          - {name: management_type, data_type: string, nullable: true}
          - {name: auth_type, data_type: string, nullable: false}
          - {name: scope_id, data_type: string, nullable: false}
          - {name: scope_type, data_type: string, nullable: false}
          - {name: source_provider, data_type: string, nullable: false}
          - {name: account_status, data_type: string, nullable: false}
          - {name: environment, data_type: string, nullable: false}
          - {name: integration_id, data_type: string, nullable: false}
          - {name: integration_slug, data_type: string, nullable: false}
          - {name: owner_person_id, data_type: string, nullable: true}
          - {name: department_path, data_type: string, nullable: true}
          - {name: first_seen_at, data_type: timestamp, nullable: false}
          - {name: last_synced_at, data_type: timestamp, nullable: false}
          - {name: source_created_at, data_type: timestamp, nullable: false}
          - {name: source_updated_at, data_type: timestamp, nullable: false}
          - {name: decommissioned_at, data_type: timestamp, nullable: true}
          - {name: is_deleted, data_type: boolean, nullable: false}
          - {name: deleted_at, data_type: timestamp, nullable: true}
      - table: users
        sql_file: ../queries/helix/users.sql
        primary_key:
          - user_id
        classifications:
          - internal
          - employee-directory
          - contains-work-email
          - excludes-sensitive-personal-fields
        columns:
          - {name: user_id, data_type: string, nullable: false}
          - {name: source_system, data_type: string, nullable: false}
          - {name: hris_instance_id, data_type: string, nullable: true}
          - {name: hris_person_id, data_type: string, nullable: false}
          - {name: employee_no, data_type: string, nullable: true}
          - {name: work_email, data_type: string, nullable: false}
          - {name: display_name, data_type: string, nullable: true}
          - {name: first_name, data_type: string, nullable: true}
          - {name: last_name, data_type: string, nullable: true}
          - {name: avatar_url, data_type: string, nullable: true}
          - {name: employment_status, data_type: string, nullable: false}
          - {name: hris_record_state, data_type: string, nullable: false}
          - {name: is_active, data_type: boolean, nullable: false}
          - {name: is_deleted, data_type: boolean, nullable: false}
          - {name: is_china_employee, data_type: boolean, nullable: false}
          - {name: job_title, data_type: string, nullable: true}
          - {name: employee_type_code, data_type: string, nullable: true}
          - {name: manager_user_id, data_type: string, nullable: true}
          - {name: manager_external_id, data_type: string, nullable: true}
          - {name: manager_display_name, data_type: string, nullable: true}
          - {name: manager_work_email, data_type: string, nullable: true}
          - {name: department_code, data_type: string, nullable: true}
          - {name: department_name, data_type: string, nullable: true}
          - {name: department_path, data_type: string, nullable: true}
          - {name: department_lark_id, data_type: string, nullable: true}
          - {name: work_location_code, data_type: string, nullable: true}
          - {name: work_location_name, data_type: string, nullable: true}
          - {name: office_code, data_type: string, nullable: true}
          - {name: country, data_type: string, nullable: true}
          - {name: city, data_type: string, nullable: true}
          - {name: residence_country, data_type: string, nullable: true}
          - {name: company_code, data_type: string, nullable: true}
          - {name: company_name, data_type: string, nullable: true}
          - {name: cost_center, data_type: string, nullable: true}
          - {name: job_function, data_type: string, nullable: true}
          - {name: job_family, data_type: string, nullable: true}
          - {name: hire_date, data_type: date, nullable: true}
          - {name: termination_date, data_type: date, nullable: true}
          - {name: work_start_date, data_type: date, nullable: true}
          - {name: seniority_date, data_type: date, nullable: true}
          - {name: probation_end_date, data_type: date, nullable: true}
          - {name: fte, data_type: float64, nullable: true}
          - {name: last_login_at, data_type: timestamp, nullable: true}
          - {name: hris_source_updated_at, data_type: timestamp, nullable: true}
          - {name: last_synced_at, data_type: timestamp, nullable: false}
          - {name: source_created_at, data_type: timestamp, nullable: false}
          - {name: source_updated_at, data_type: timestamp, nullable: false}
          - {name: hris_absent_since, data_type: timestamp, nullable: true}
          - {name: hris_purged_at, data_type: timestamp, nullable: true}
          - {name: hris_purged_reason, data_type: string, nullable: true}
          - {name: deleted_at, data_type: timestamp, nullable: true}

target:
  project: gcp-digital-bi
  location: us-west2
  datasets:
    raw: helix_raw
    final: helix
    control: helix_control

tables:
  include:
    - cloud_accounts
    - users
  exclude: []
  require_primary_key: true

gcs:
  bucket: gcp-digital-bi-pingcap-helix-ingestion
  location: us-west2
  snapshot_prefix: snapshots
  manifest_prefix: manifests
  runtime_log_prefix: logs

export:
  parquet_compression: ZSTD

load:
  full_snapshot_delete: true
  final_table_names:
    cloud_accounts: cloud_accounts
    users: pingcap_users

retention:
  gcs_snapshots_days: 35
  manifests_days: 365

schedule:
  pipeline_cron: "15 23 * * *"
  timezone: UTC

governance:
  classifications:
    - internal