#!/usr/bin/env bash
set -euo pipefail

# Verify that:
# 1) empty windows keep retrying (requeue path)
# 2) once upstream data appears, downstream_ts keeps progressing
#
# Usage examples:
#   ./verify-requeue-progress.sh --unit-only
#   ./verify-requeue-progress.sh \
#     --namespace metrics-integration \
#     --selector 'app.kubernetes.io/name=metrics-integration-pipeline' \
#     --duration 300
#
# Notes:
# - Requires kubectl configured to target the cluster where pipeline is running.
# - ConfigMap defaults: downstream-ts, key: downstream_ts.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PIPELINE_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)"

NAMESPACE=""
SELECTOR=""
DURATION=300
INTERVAL=15
CONFIGMAP_NAME="downstream-ts"
CM_KEY="downstream_ts"
UNIT_ONLY=0
SINCE_LOG="20m"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --namespace)
      NAMESPACE="$2"; shift 2 ;;
    --selector)
      SELECTOR="$2"; shift 2 ;;
    --duration)
      DURATION="$2"; shift 2 ;;
    --interval)
      INTERVAL="$2"; shift 2 ;;
    --configmap)
      CONFIGMAP_NAME="$2"; shift 2 ;;
    --cm-key)
      CM_KEY="$2"; shift 2 ;;
    --since-log)
      SINCE_LOG="$2"; shift 2 ;;
    --unit-only)
      UNIT_ONLY=1; shift 1 ;;
    -h|--help)
      sed -n '1,40p' "$0"
      exit 0 ;;
    *)
      echo "Unknown arg: $1" >&2
      exit 2 ;;
  esac
done

echo "[1/3] Running focused unit tests..."
(
  cd "$PIPELINE_DIR"
  go test ./internal/otel_collector/processor/thanos \
    -run 'TestDoQuery_NoDatapoint_SwitchesToRequeueStrategy|TestDoQuery_NoTaskInContext_DoesNotNack' -v
  go test ./internal/otel_collector/extension/taskqueue \
    -run 'TestAckTask|TestNackAndRequeueTask|TestDuplicateAck' -v
)

echo "[PASS] Unit tests passed."

if [[ "$UNIT_ONLY" -eq 1 ]]; then
  echo "[DONE] unit-only mode completed."
  exit 0
fi

if [[ -z "$NAMESPACE" || -z "$SELECTOR" ]]; then
  echo "[ERROR] integration mode requires --namespace and --selector" >&2
  echo "        or use --unit-only" >&2
  exit 2
fi

if ! command -v kubectl >/dev/null 2>&1; then
  echo "[ERROR] kubectl not found in PATH" >&2
  exit 2
fi

get_downstream_ts() {
  kubectl -n "$NAMESPACE" get configmap "$CONFIGMAP_NAME" -o "jsonpath={.data.${CM_KEY}}" 2>/dev/null || true
}

pick_pod() {
  kubectl -n "$NAMESPACE" get pods -l "$SELECTOR" \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | head -n1
}

echo "[2/3] Watching runtime behavior..."
POD="$(pick_pod)"
if [[ -z "$POD" ]]; then
  echo "[ERROR] no pod found by selector: $SELECTOR" >&2
  exit 2
fi

echo "Using pod: $POD"

START_TS="$(get_downstream_ts)"
if [[ -z "$START_TS" ]]; then
  echo "[WARN] Could not read ${CONFIGMAP_NAME}.${CM_KEY} at start (continuing)."
else
  echo "Start downstream_ts: $START_TS"
fi

REQUEUE_LOG_COUNT=0
ACK_LOG_COUNT=0
PROGRESS_OBSERVED=0

END_EPOCH=$(( $(date +%s) + DURATION ))
while [[ $(date +%s) -lt $END_EPOCH ]]; do
  CUR_POD="$(pick_pod)"
  if [[ -n "$CUR_POD" && "$CUR_POD" != "$POD" ]]; then
    echo "[INFO] Pod rotated: $POD -> $CUR_POD"
    POD="$CUR_POD"
  fi

  LOG_SAMPLE="$(kubectl -n "$NAMESPACE" logs "$POD" --since="$SINCE_LOG" 2>/dev/null || true)"

  REQUEUE_LOG_COUNT=$(printf '%s' "$LOG_SAMPLE" | grep -c 'no datapoint exists, should requeue and wait\.' || true)
  ACK_LOG_COUNT=$(printf '%s' "$LOG_SAMPLE" | grep -c 'timestamp acked\|exporter send succeed, should ack' || true)

  CUR_TS="$(get_downstream_ts)"
  if [[ -n "$START_TS" && -n "$CUR_TS" ]]; then
    if [[ "$CUR_TS" =~ ^[0-9]+$ && "$START_TS" =~ ^[0-9]+$ ]]; then
      if (( CUR_TS > START_TS )); then
        PROGRESS_OBSERVED=1
      fi
    fi
  fi

  echo "tick: requeue_logs=$REQUEUE_LOG_COUNT ack_logs=$ACK_LOG_COUNT downstream_ts=${CUR_TS:-N/A}"
  sleep "$INTERVAL"
done

echo "[3/3] Verdict"

if (( REQUEUE_LOG_COUNT > 0 )); then
  echo "[OK] Observed requeue-on-empty logs."
else
  echo "[WARN] No requeue-on-empty logs observed in window."
fi

if (( ACK_LOG_COUNT > 0 )); then
  echo "[OK] Observed ack-related logs."
else
  echo "[WARN] No ack-related logs observed in window."
fi

if (( PROGRESS_OBSERVED == 1 )); then
  echo "[OK] downstream_ts progressed during watch window."
  echo "[PASS] Behavior matches: empty retries + progress after data arrives."
  exit 0
fi

echo "[WARN] downstream_ts did not progress in this window."
echo "       This can still happen if upstream remained empty the whole time."
echo "       Re-run with longer --duration or during known backfill window."
exit 1