#!/bin/bash
# Log parsing script - Parse log files in statement_logs directory and write to Delta Lake table

# Configure Python environment
if [ ! -d ".venv" ]; then
    echo "Creating Python virtual environment..."
    python3 -m venv .venv
fi

echo "Activating virtual environment..."
source .venv/bin/activate

echo "Installing dependencies..."
pip install -r requirements.txt
pip install -r requirements_parser.txt

echo "Starting to parse log files..."
echo "Input directory: ./statement_logs"
echo "Output directory: ./statement_delta"
echo ""

# Run parser
python parse_logs_to_delta.py \
    --input-dir "./statement_logs" \
    --output-dir "./statement_delta" \
    --table-name "statement_logs"

echo ""
echo "Parsing completed!"
echo "Results saved in ./statement_delta/statement_logs directory"
echo "You can use Delta Lake tools to query this table"