[project]
name = "pingcap-datalake"
version = "0.1.0"
description = "Local tooling for TiDB Cloud Lake design and ingestion exploration."
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
    "databend-driver>=0.20.0",
    "pymysql>=1.1.1",
    "python-dotenv>=1.0.1",
    "pyyaml>=6.0.2",
]

[project.optional-dependencies]
dlt-probe = [
    "databend-sqlalchemy>=0.5.5",
    "dlt[sqlalchemy]>=1.27.0",
]
# Transform layer (transform/): dbt-on-Databend. Optional, like dlt-probe — ingestion
# runtime does not need dbt installed. Install with: uv sync --extra transform
transform = [
    "dbt-core>=1.8,<1.9",
    # Pinned until gateway warehouse + seed support lands upstream.
    "dbt-databend-cloud @ git+https://github.com/ywang-official/dbt-databend.git@codex/databend-gateway-seed-support",
    # The adapter pins databend-sqlalchemy 0.3.x, which still imports pkg_resources;
    # setuptools>=81 drops it, so keep setuptools old enough to provide it.
    "setuptools<81",
]
# Python transform jobs (transform/python/): pandas/numpy revrec logic ported from the
# BigQuery dbt python model. Deliberately dbt-free so an orchestrator worker (e.g. Airflow)
# can run a job with just the driver + pandas. Install with: uv sync --extra pyjobs
pyjobs = [
    "pandas>=2.0",
    "numpy>=1.26",
]
# Local Airflow orchestration plane. Airflow itself runs from the Docker image in
# orchestration/airflow/setup; this extra is for local import/parse checks.
orchestration = [
    "apache-airflow==3.2.1",
    "astronomer-cosmos>=1.14,<2",
]

[tool.uv]
package = false
# dlt-probe (extract-side schema inference) and transform (dbt-on-Databend) pin
# incompatible databend-sqlalchemy ranges (>=0.5.5 vs the adapter's <0.4). They are
# never installed together, so declare them conflicting and let uv resolve each alone.
conflicts = [
    [
        { extra = "dlt-probe" },
        { extra = "transform" },
    ],
    [
        { extra = "orchestration" },
        { extra = "transform" },
    ],
]