#!/usr/bin/env python3 from __future__ import annotations import argparse import json import os import socket import subprocess from pathlib import Path from common import DeployError, log, run DEPLOYMENT_ROOT = (Path.home() / "nexuscrm-deployment").resolve() NGINX_LOG_DIR = Path("/var/log/nginx") STATE_DIR = DEPLOYMENT_ROOT / "offline" / "envs" / "staging" / ".guard-state" STATE_FILE = STATE_DIR / "offline-host-guard.json" SERVICE_SPECS = [ { "environment": "staging", "service": "nexuscrm-staging-backend.service", "health_url": "http://127.0.0.1:3101/health", "index_url": "http://127.0.0.1/index.html", "index_host": "nexuscrm-new-staging.pingcap.com", "access_log": NGINX_LOG_DIR / "nexuscrm-staging.access.log", }, { "environment": "exp", "service": "nexuscrm-exp-backend.service", "health_url": "http://127.0.0.1:3103/health", "index_url": "http://127.0.0.1/index.html", "index_host": "nexuscrm-new-exp.pingcap.com", "access_log": NGINX_LOG_DIR / "nexuscrm-exp.access.log", }, ] def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--min-disk-free-gb", type=int, default=5) parser.add_argument("--scan-lines", type=int, default=200) parser.add_argument("--notify-on-failure", action="store_true") return parser.parse_args() def assert_disk_free(min_free_gb: int) -> None: result = subprocess.run( ["df", "-Pk", str(DEPLOYMENT_ROOT)], text=True, capture_output=True, check=False, ) if result.returncode != 0 or not result.stdout.strip(): raise DeployError("failed to read disk usage") lines = result.stdout.strip().splitlines() if len(lines) < 2: raise DeployError("unexpected df output") available_kb = int(lines[1].split()[3]) if available_kb < min_free_gb * 1024 * 1024: raise DeployError(f"disk free space below threshold: {available_kb // 1024 // 1024} GiB available") def service_is_active(service: str) -> bool: return subprocess.run(["systemctl", "is-active", "--quiet", service], check=False).returncode == 0 def check_service(spec: dict[str, str]) -> None: service = spec["service"] if not service_is_active(service): raise DeployError(f"service not active: {service}") run(["curl", "-fsS", "-o", "/dev/null", spec["health_url"]]) index_cmd = ["curl", "-fsS", "-o", "/dev/null"] index_host = spec.get("index_host") if index_host: index_cmd.extend(["-H", f"Host: {index_host}"]) index_cmd.append(spec["index_url"]) run(index_cmd) def detect_recent_5xx(log_path: Path, scan_lines: int) -> str: if not log_path.is_file(): return "" result = subprocess.run( ["tail", "-n", str(scan_lines), str(log_path)], text=True, capture_output=True, check=False, ) if result.returncode != 0: return "" matches = [] for line in result.stdout.splitlines(): fields = line.split() if len(fields) < 9: continue status = fields[8] if len(status) == 3 and status.startswith("5"): matches.append(line) if not matches: return "" return " || ".join(matches[-3:]) def load_state() -> dict[str, str]: if not STATE_FILE.is_file(): return {} try: return json.loads(STATE_FILE.read_text()) except Exception: return {} def save_state(state: dict[str, str]) -> None: STATE_DIR.mkdir(parents=True, exist_ok=True) STATE_FILE.write_text(json.dumps(state, sort_keys=True) + "\n") def notify_failure(environment: str, check_name: str, failure: str, details: str = "") -> None: app_id = os.environ.get("FEISHU_APP_ID", "").strip() app_secret = os.environ.get("FEISHU_APP_SECRET", "").strip() chat_id = os.environ.get("FEISHU_CHAT_ID", "").strip() if not app_id or not app_secret or not chat_id: log("offline host guard notification skipped: FEISHU_APP_ID / FEISHU_APP_SECRET / FEISHU_CHAT_ID not fully configured") return run( [ "python3", str(DEPLOYMENT_ROOT / "scripts/notify_feishu_offline_guard.py"), "--app-id", app_id, "--app-secret", app_secret, "--chat-id", chat_id, "--host", socket.gethostname(), "--environment", environment, "--check", check_name, "--failure", failure, "--details", details, ] ) def main() -> int: args = parse_args() state = load_state() failures: list[tuple[str, str, str, str]] = [] try: assert_disk_free(args.min_disk_free_gb) except DeployError as exc: failures.append(("host", "disk", str(exc), "")) for spec in SERVICE_SPECS: env_name = spec["environment"] try: check_service(spec) except DeployError as exc: failures.append((env_name, "service-health", str(exc), "")) except Exception as exc: failures.append((env_name, "service-health", str(exc), "")) recent_5xx = detect_recent_5xx(Path(spec["access_log"]), args.scan_lines) if recent_5xx: failures.append((env_name, "nginx-5xx", "recent 5xx detected", recent_5xx)) if failures: if args.notify_on_failure: for env_name, check_name, failure, details in failures: fingerprint = json.dumps( { "environment": env_name, "check": check_name, "failure": failure, "details": details, }, sort_keys=True, ) state_key = f"{env_name}:{check_name}" if state.get(state_key) == fingerprint: continue try: notify_failure(env_name, check_name, failure, details) state[state_key] = fingerprint except Exception as exc: log(f"offline host guard notification failed: {exc}") save_state(state) lines = [f"{env_name}:{check_name}:{failure}" for env_name, check_name, failure, _ in failures] raise DeployError("\n".join(lines)) if state: save_state({}) log("offline host guard ok") return 0 if __name__ == "__main__": try: raise SystemExit(main()) except DeployError as exc: log(f"failed: {exc}") raise SystemExit(1)