#!/usr/bin/env python3 """Bounded reader for raw evidence archived under evidences/raw/. Raw payload files are single-line JSON and can be large; this tool lets the diagnosis analyst inspect them without loading whole payloads into the chat context. Output is always bounded (series/points/rows caps). Run from the workspace root (the directory containing evidences/). Usage: # Index view: list archived payloads (optionally by kind) python3 read_raw_evidence.py --list [--kind metrics] # Extract one payload, bounded python3 read_raw_evidence.py --file raw/metrics/0001_TiDB_CPU.json \ [--series-contains db-tidb-0] [--start 1774951200 --end 1774959060] \ [--max-series 5] [--max-points 120] # Or select by manifest filters (first match wins; ambiguity lists matches) python3 read_raw_evidence.py --kind metrics --label-contains cpu """ from __future__ import annotations import argparse import json import sys from datetime import datetime, timezone from pathlib import Path RAW_ROOT = Path("evidences") / "raw" def _iso(ts: float) -> str: try: return datetime.fromtimestamp(float(ts), tz=timezone.utc).strftime("%m-%dT%H:%M:%SZ") except (OverflowError, OSError, ValueError): return str(ts) def _load_manifest() -> list[dict]: path = RAW_ROOT / "manifest.jsonl" if not path.is_file(): print(f"error: {path} not found; run the standard collector first", file=sys.stderr) raise SystemExit(1) entries = [] for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: entries.append(json.loads(line)) except json.JSONDecodeError: continue return entries def _list_entries(entries: list[dict], kind: str | None) -> None: shown = 0 for e in entries: if kind and e.get("kind") != kind: continue window = f"{e.get('start', '')}..{e.get('end', '')}".strip(".") print(f"{e.get('file','')}\t{e.get('kind','')}\t{e.get('label','')}\t" f"{window}\t{e.get('bytes', 0)}B\t{str(e.get('query_preview',''))[:80]}") shown += 1 if shown >= 200: print(f"... truncated at 200 entries ({len(entries)} total)") break if shown == 0: print("no matching manifest entries") def _select_file(entries: list[dict], args: argparse.Namespace) -> Path: if args.file: path = Path("evidences") / args.file if not str(args.file).startswith("evidences") else Path(args.file) if not path.is_file(): print(f"error: {path} not found", file=sys.stderr) raise SystemExit(1) return path matches = [ e for e in entries if (not args.kind or e.get("kind") == args.kind) and (not args.label_contains or args.label_contains.lower() in str(e.get("label", "")).lower()) and (not args.query_contains or args.query_contains.lower() in str(e.get("query_preview", "")).lower()) ] if not matches: print("error: no manifest entry matches the filters; use --list to inspect", file=sys.stderr) raise SystemExit(1) if len(matches) > 1: print(f"{len(matches)} entries match; picking the first. Candidates:", file=sys.stderr) for e in matches[:10]: print(f" {e.get('file','')}\t{e.get('label','')}", file=sys.stderr) return Path("evidences") / matches[0]["file"] def _series_matches(labels: dict, needle: str | None) -> bool: if not needle: return True hay = " ".join(str(v) for v in labels.values()).lower() return needle.lower() in hay def _print_prom_series(result: list, args: argparse.Namespace) -> None: shown = 0 for series in result: labels = series.get("metric") or series.get("stream") or {} if not _series_matches(labels, args.series_contains): continue values = series.get("values") or ([series["value"]] if series.get("value") else []) picked = [] for pair in values: try: ts = float(pair[0]) except (TypeError, ValueError, IndexError): continue if ts > 1e17: # loki nanoseconds ts /= 1e9 if args.start and ts < float(args.start): continue if args.end and ts > float(args.end): continue picked.append((ts, pair[1])) step = max(1, len(picked) // args.max_points) label_str = ",".join(f"{k}={v}" for k, v in sorted(labels.items())) or "(no labels)" print(f"# series: {label_str} points={len(picked)} (showing every {step})") for ts, val in picked[::step][: args.max_points]: text = str(val) print(f"{_iso(ts)}\t{text[:200]}") shown += 1 if shown >= args.max_series: print(f"# ... series truncated at {args.max_series} (total {len(result)})") break if shown == 0: print("no series matched the filters") def _print_rows(payload: dict, args: argparse.Namespace) -> None: columns = [ str(c.get("name") if isinstance(c, dict) else c) for c in payload.get("columns") or [] ] if columns: print("\t".join(columns)) rows = payload.get("rows") or [] for row in rows[: args.max_rows]: if isinstance(row, dict): print("\t".join(str(row.get(c, ""))[:200] for c in columns) if columns else json.dumps(row, ensure_ascii=False)[:1000]) else: print("\t".join(str(cell)[:200] for cell in row)) if len(rows) > args.max_rows: print(f"... rows truncated at {args.max_rows} (total {len(rows)})") def _print_fallback(payload: object, args: argparse.Namespace) -> None: if isinstance(payload, dict): for key, value in payload.items(): size = len(value) if isinstance(value, (list, dict, str)) else 1 print(f"# key: {key} ({type(value).__name__}, size={size})") text = json.dumps(payload, ensure_ascii=False, default=str) print(text[: args.max_chars]) if len(text) > args.max_chars: print(f"... truncated at {args.max_chars} chars (total {len(text)})") def main() -> int: parser = argparse.ArgumentParser(description="Bounded reader for evidences/raw/ payloads") parser.add_argument("--list", action="store_true", help="List manifest entries") parser.add_argument("--kind", help="Filter by kind: metrics/logs/queries/events/meta") parser.add_argument("--label-contains", help="Manifest label substring filter") parser.add_argument("--query-contains", help="Manifest query substring filter") parser.add_argument("--file", help="Raw file path relative to evidences/ (e.g. raw/metrics/0001_x.json)") parser.add_argument("--series-contains", help="Substring matched against series label values") parser.add_argument("--start", help="Window start (epoch seconds)") parser.add_argument("--end", help="Window end (epoch seconds)") parser.add_argument("--max-series", type=int, default=5) parser.add_argument("--max-points", type=int, default=120) parser.add_argument("--max-rows", type=int, default=50) parser.add_argument("--max-chars", type=int, default=4000) args = parser.parse_args() entries = _load_manifest() if args.list: _list_entries(entries, args.kind) return 0 path = _select_file(entries, args) record = json.loads(path.read_text(encoding="utf-8")) payload = record.get("payload", record) print(f"# file: {path}") print(f"# label: {record.get('label','')} kind: {record.get('kind','')} " f"window: {record.get('start','')}..{record.get('end','')}") query = str(record.get("query", "")) if query: print(f"# query: {query[:200]}") result = None if isinstance(payload, dict): data = payload.get("data") if isinstance(data, dict) and isinstance(data.get("result"), list): result = data["result"] elif isinstance(payload.get("result"), list): result = payload["result"] if result is not None: _print_prom_series(result, args) elif isinstance(payload, dict) and payload.get("rows") is not None: _print_rows(payload, args) else: _print_fallback(payload, args) return 0 if __name__ == "__main__": raise SystemExit(main())