"""``POST /api/v1/observability/client-errors`` — frontend crash reports. ## Why this exists ``MobileErrorBoundary`` caught render crashes and logged them to ``console.error`` **in DEV only**. In production a white screen inside the Feishu webview left no trace anywhere: no console to open, no log to grep, no way to tell whether one person hit it once or two hundred people hit it all morning. Every mobile white-screen report was therefore unreproducible by construction. This endpoint is the missing half. It is deliberately minimal — one structured log line per report plus one privacy-safe daily aggregate. The aggregate deliberately stores only day, surface and count: no person, route, message, component, stack or user agent. This gives the Intelligence Center a durable platform-health signal without turning crash reporting into a second error-tracking product or a shadow store of user-visible data. Log collection continues to ship the forensic record to the cloud collector. ## The three things that keep it from becoming a liability 1. **Rate limited per user** (Redis fixed window, fail-open). A render loop can fire an error boundary dozens of times a second; without a cap, one looping client could flood the log budget for everyone. Fail-open matches the other limiters here: a Redis outage must not silence crash reports. 2. **Bounded payload.** Every field is length-capped server-side, not just in the client, and the whole model forbids extra keys. A stack trace is the only large field and it is truncated hard. 3. **No free-form user data.** There is deliberately no `extra` / `context` dict: a page's local state is exactly where PII lives (an employee's email, a salary field, a token in a URL), and an error reporter that accepts arbitrary state will eventually log all of it. The fields here are the ones needed to FIND the bug — component, route, message, stack — and nothing else. Authenticated on purpose: an anonymous crash sink on a public origin is an open log-injection endpoint. The caller's identity also answers "is this one person or everyone?", which is the first question worth asking. """ from __future__ import annotations import asyncio import json import re import uuid from datetime import UTC, date, datetime from typing import Literal from fastapi import APIRouter, Depends, Request from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import Date, DateTime, Index, Integer, String, UniqueConstraint, func from sqlalchemy.dialects.mysql import insert as mysql_insert from sqlalchemy.orm import Mapped, mapped_column from app.auth.models import AuthUser from app.core import database as database_module from app.core.database import Base from app.core.dependencies import get_current_user from app.core.exceptions import AppException from app.core.redis_client import get_redis from app.core.response import success_response from app.core.sql_types import GUID as Uuid router = APIRouter(prefix="/api/v1/observability", tags=["observability"]) from app.core.safe_logging import get_safe_logger _log = get_safe_logger("helix.client_error") # Per-user cap. Generous enough for a burst of genuinely distinct crashes on # one page load, tight enough that a render loop cannot flood the log budget. _RATE_LIMIT_MAX = 20 _RATE_LIMIT_WINDOW_SECONDS = 60 _MAX_MESSAGE = 500 _MAX_STACK = 4_000 _MAX_SHORT = 200 # Control characters are stripped from every field before it is logged. # Without this, a newline inside `message` writes what looks like a second # log line — and since the fallback handler emits a bare `%(message)s` # with no timestamp or level prefix, a forged line is indistinguishable # from a real one in the collector. Any authenticated user could then bury # a real incident in plausible noise, or trip alert rules at will. _CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") class ClientErrorDailyAggregate(Base): """PII-free durable count used for health scoring and trend charts. The uniqueness key intentionally has no route or fingerprint. Even a route can contain an employee or cloud-object identifier, while a hash of a low-entropy error message is dictionary-reversible. Detailed diagnosis remains in the access-controlled log collector; Helix persists only the minimum count needed to tell whether client stability is improving. """ __tablename__ = "client_error_daily_aggregates" id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4) day: Mapped[date] = mapped_column(Date, nullable=False) surface: Mapped[str] = mapped_column(String(16), nullable=False) report_count: Mapped[int] = mapped_column( Integer, nullable=False, server_default="0", ) first_seen_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now(), ) last_seen_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now(), ) __table_args__ = ( UniqueConstraint( "day", "surface", name="uq_client_error_daily_day_surface", ), Index("ix_client_error_daily_day", "day"), ) async def _increment_daily_aggregate( *, surface: str, now: datetime, ) -> None: """Atomically increment telemetry without making crash intake depend on it. A dedicated session plus a 500ms bound keeps the write durable in the normal path without allowing a locked telemetry row or exhausted pool to hold the mobile white-screen safety response open. MySQL/TiDB's atomic upsert prevents lost increments when many clients report concurrently. """ try: async with asyncio.timeout(0.5): async with database_module.async_session_factory() as db: stmt = mysql_insert(ClientErrorDailyAggregate).values( id=uuid.uuid4(), day=now.date(), surface=surface, report_count=1, first_seen_at=now, last_seen_at=now, ) await db.execute( stmt.on_duplicate_key_update( report_count=ClientErrorDailyAggregate.report_count + 1, last_seen_at=now, ) ) await db.commit() except Exception as exc: # pragma: no cover - DB outage safety path _log.warning("client-error aggregate unavailable: %s", exc) def _clean(value: str | None) -> str | None: return None if value is None else _CONTROL_CHARS.sub(" ", value) class ClientErrorReport(BaseModel): """One frontend crash. Every field is capped; unknown fields are refused. ``model_config`` forbids extras so a future client cannot start smuggling a state dump through an undeclared key — the whole point of not having an ``extra`` field is defeated if arbitrary keys are accepted silently. """ model_config = ConfigDict(extra="forbid") # Where it broke. surface: Literal["mobile", "desktop"] = Field( ..., description="Which shell caught it.", ) route: str = Field( ..., max_length=_MAX_SHORT, # Enforced, not merely documented. The client strips the query # string because it can hold a live action token # (`/cloud/grants/renew?token=…`), but that made the invariant a # convention one careless call site could break. A pattern makes # the server the place it holds. pattern=r"^/[^?#]*$", description="Router path at crash time (path only — never the query " "string, which can carry action tokens).", ) component: str | None = Field( None, max_length=_MAX_SHORT, description="Nearest component name from the React error info.", ) # What broke. message: str = Field(..., max_length=_MAX_MESSAGE) stack: str | None = Field(None, max_length=_MAX_STACK) component_stack: str | None = Field(None, max_length=_MAX_STACK) # Context needed to reproduce. app_version: str | None = Field(None, max_length=_MAX_SHORT) user_agent: str | None = Field(None, max_length=_MAX_SHORT) async def _rate_limit(user_id: str) -> None: """Fixed-window per-user cap. Fail-open on Redis trouble. Fail-open is the right call here even though it is the wrong call for a login limiter: the downside of an unlimited crash report is log volume, while the downside of dropping one is an invisible production crash. """ key = f"client_error_ratelimit:{user_id}" try: redis = get_redis() async with redis.pipeline(transaction=False) as pipe: pipe.incr(key) pipe.expire(key, _RATE_LIMIT_WINDOW_SECONDS, nx=True) results = await pipe.execute() count = int(results[0]) if results else 0 if count > _RATE_LIMIT_MAX: ttl = await redis.ttl(key) if ttl < 0: ttl = _RATE_LIMIT_WINDOW_SECONDS raise AppException( code="CLIENT_ERROR_RATE_LIMITED", message="Too many client error reports; slow down.", status_code=429, headers={"Retry-After": str(ttl)}, ) except AppException: raise except Exception as exc: # pragma: no cover - Redis outage path _log.warning("client-error rate limiter unavailable: %s", exc) @router.post("/client-errors", summary="Report a frontend crash") async def report_client_error( payload: ClientErrorReport, request: Request, user: AuthUser = Depends(get_current_user), ): """Record one frontend crash as a structured log line. Returns 202-shaped success with the trace id so the client can show it — a user who can read back "trace abc123" turns an unreproducible white screen into a searchable incident. """ await _rate_limit(str(user.id)) trace_id = getattr(request.state, "trace_id", None) record = { "surface": _clean(payload.surface), "route": _clean(payload.route), "component": _clean(payload.component), "message": _clean(payload.message), # Stacks keep their newlines: `json.dumps` below escapes every # control character anyway (that is where the injection protection # actually comes from), so flattening them here bought nothing and # cost the one thing a stack trace is for — the collector could no # longer split it back into frames. "stack": payload.stack, "component_stack": payload.component_stack, "app_version": _clean(payload.app_version), "user_agent": _clean(payload.user_agent), "person_id": str(user.id), "trace_id": trace_id, } # The payload goes in the MESSAGE, serialised, not in `extra`. # # `extra=` only reaches the output if a formatter is configured to # render it, and this process configures none — uvicorn's default # logging config defines no root handler, so `helix.client_error` # falls through to `logging.lastResort`, whose format is bare # `%(message)s`. Everything passed via `extra` was therefore being # dropped, including `person_id` and `trace_id` — the two fields that # answer "who hit this, and can I correlate it?", which this endpoint # exists to answer. One JSON blob in the message survives any # formatter, and stays greppable and machine-parseable either way. _log.error("client_error %s", json.dumps(record, ensure_ascii=False)) # Persist after the forensic log exists, but before returning, so a worker # restart cannot create an acknowledged-yet-never-counted gap. The helper # is fail-open and time-bounded; the controlled log remains authoritative. await _increment_daily_aggregate( surface=payload.surface, now=datetime.now(UTC), ) return success_response(data={"recorded": True, "trace_id": trace_id}, request=request)