"""Pipeline to load slack into duckdb."""

from typing import List

import dlt
import pendulum
from slack import slack_source

dlt.config["runtime.log_level"] = "DEBUG"
dlt.config["destination.bigquery.location"] = "us-west2"

def load_all_resources(replies: bool = False) -> None:
    """Load all resources from slack without any selection of channels."""

    pipeline = dlt.pipeline(
        pipeline_name="slack",
        destination="bigquery",
        dataset_name="raw_slack_data"
    )

    source = slack_source(
        page_size=1000,
        start_date=pendulum.datetime(2025, 3, 1),
        end_date=pendulum.now().subtract(days=1),
        replies=replies,
    )

    source.schema.update_preferred_types(
        {
            "re:.+": "text"
        }
    )

    # Uncomment the following line to load only the access_logs resource. It is not selected
    # by default because it is a resource just available on paid accounts.
    # source.access_logs.selected = True

    load_info = pipeline.run(
        source,
    )
    print(load_info)


def select_resource(selected_channels: List[str], replies: bool = False) -> None:
    """Execute a pipeline that will load the given Slack list of channels with the selected
    channels incrementally beginning at the given start date."""

    pipeline = dlt.pipeline(
        pipeline_name="slack",
        destination="bigquery",
        dataset_name="raw_slack_data"
    )

    source = slack_source(
        page_size=20,
        selected_channels=selected_channels,
        replies=replies,
        start_date=pendulum.datetime(2025, 3, 1),
        end_date=pendulum.now().subtract(days=1)
    )

    load_info = pipeline.run(
        source,
    )
    print(load_info)


if __name__ == "__main__":
    # Add your desired resources to the list...
    resources = ["access_logs", "conversations", "conversations_history"]

    select_resource(selected_channels=["help-forum"], replies=True)