#!/bin/bash
SOURCE_PROJECT="gcp-digital-bi"
TARGET_PROJECT="gcp-rd-research-dev"
TARGET_DATASET_PREFIX="mirror_"

# 创建日志文件
LOG_FILE="mirror_log_$(date +%Y%m%d_%H%M%S).txt"
echo "Mirroring process started at $(date)" > $LOG_FILE

datasets=$(bq ls --project_id=$SOURCE_PROJECT --format=json | jq -r '.[].datasetReference.datasetId')

for dataset in $datasets; do
  [[ "$dataset" =~ ^(INFORMATION_SCHEMA|_|aws_nonprod_billing) ]] && continue
  
  target_dataset="${TARGET_DATASET_PREFIX}${dataset}"
  echo "Processing dataset: $dataset" | tee -a $LOG_FILE
  
  # 创建目标数据集
  bq mk --project_id=$TARGET_PROJECT --dataset --description "Mirror of ${SOURCE_PROJECT}.${dataset}" "${target_dataset}" 2>&1 | tee -a $LOG_FILE

  tables=$(bq ls --max_results 1000 --project_id=$SOURCE_PROJECT --format=json $dataset | jq -r '.[]?.tableReference?.tableId? // empty')
  
  for table in $tables; do
    [[ -z "$table" ]] && continue
    
    # 测试是否有权限访问该表
    if bq query --use_legacy_sql=false --project_id=$TARGET_PROJECT \
       --format=none "SELECT 1 FROM \`${SOURCE_PROJECT}.${dataset}.${table}\` LIMIT 1" 2>/dev/null; then
      
      echo "Creating view for ${dataset}.${table}" | tee -a $LOG_FILE
      bq mk --project_id=$TARGET_PROJECT \
            --use_legacy_sql=false \
            --view="SELECT * FROM \`${SOURCE_PROJECT}.${dataset}.${table}\`" \
            "${target_dataset}.${table}" 2>&1 | tee -a $LOG_FILE
    else
      echo "SKIPPING (no access): ${dataset}.${table}" | tee -a $LOG_FILE
    fi
  done
done