#!/usr/bin/env python3 """Check first-party code identifiers against reviewed terminology rules.""" from __future__ import annotations import argparse import re import sys from dataclasses import dataclass from pathlib import Path from typing import Iterable SOURCE_ROOTS = ( "internal", "desktop", "web", "shared", "miniapp", "x", "kits", "guardrails", "cmd", ) SOURCE_SUFFIXES = { ".go", ".js", ".mjs", ".cjs", ".mts", ".ts", ".tsx", ".vue", } SKIPPED_PARTS = { "node_modules", ".data", "agentteam-worktrees", ".worktrees", ".claude", "dist", } @dataclass(frozen=True) class Rule: pattern: re.Pattern[str] preferred: str def is_identifier_start(char: str) -> bool: return char == "_" or char.isascii() and char.isalpha() def is_identifier_continue(char: str) -> bool: return char == "_" or char.isascii() and char.isalnum() VUE_DYNAMIC_ATTRIBUTE_RE = re.compile( r"(?:^|\s)(?::[\w.:-]+|@[\w.:-]+|#[\w.:-]+|v-[\w:.-]+)\s*=\s*$" ) def is_vue_dynamic_attribute(text: str, quote_index: int) -> bool: """Return whether a quote starts a Vue directive expression value.""" tag_start = text.rfind("<", 0, quote_index) tag_end = text.rfind(">", 0, quote_index) if tag_start <= tag_end: return False return VUE_DYNAMIC_ATTRIBUTE_RE.search(text[tag_start + 1 : quote_index]) is not None def scan_vue_expression_value(text: str, start: int, quote: str) -> tuple[str, int]: """Extract a quoted Vue directive expression and its closing quote index.""" index = start while index < len(text): char = text[index] if char == "\\": index += 2 continue if char == quote: return text[start:index], index index += 1 return text[start:], len(text) def mask_vue_static_text(text: str) -> str: """Keep Vue tags/expressions while masking rendered prose and styles.""" chars = list(text) index = 0 in_tag = False tag_start = -1 quote = "" interpolation = False script = False style = False def mask(position: int) -> None: if chars[position] != "\n": chars[position] = " " while index < len(text): if script: closing = re.match(r"", index + 4) if end < 0: end = len(text) for position in range(index, min(end + 3, len(text))): mask(position) index = min(end + 3, len(text)) continue if interpolation: if text.startswith("}}", index): interpolation = False index += 1 continue if text.startswith("{{", index): interpolation = True index += 2 continue if text[index] == "<": in_tag = True tag_start = index quote = "" index += 1 continue mask(index) index += 1 continue if quote: if text[index] == "\\": index += min(2, len(text) - index) continue if text[index] == quote: quote = "" index += 1 continue if text[index] in {"'", '"'}: quote = text[index] index += 1 continue if text[index] == ">": tag = text[tag_start : index + 1] match = re.match(r"<\s*(/?)\s*(script|style)\b", tag, re.IGNORECASE) if match and not match.group(1): script = match.group(2).lower() == "script" style = match.group(2).lower() == "style" in_tag = False index += 1 return "".join(chars) def scan_text( text: str, *, language: str = "javascript", vue: bool = False ) -> list[tuple[int, str]]: """Return code identifiers while skipping comments and literal text.""" if vue: text = mask_vue_static_text(text) identifiers: list[tuple[int, str]] = [] index = 0 line = 1 state = "code" quote = "" template_expression_depths: list[int] = [] while index < len(text): char = text[index] following = text[index + 1] if index + 1 < len(text) else "" if state == "code": if char == "/" and following == "/": state = "line_comment" index += 2 continue if char == "/" and following == "*": state = "block_comment" index += 2 continue if char in {"'", '"'}: if vue and is_vue_dynamic_attribute(text, index): expression, end = scan_vue_expression_value(text, index + 1, char) for expression_line, identifier in scan_text(expression): identifiers.append((line + expression_line - 1, identifier)) line += expression.count("\n") index = end + 1 if end < len(text) else end continue state = "string" quote = char index += 1 continue if char == "`": if language == "go": state = "raw_string" else: state = "template" index += 1 continue if template_expression_depths: if char == "{": template_expression_depths[-1] += 1 index += 1 continue if char == "}": template_expression_depths[-1] -= 1 index += 1 if template_expression_depths[-1] == 0: template_expression_depths.pop() state = "template" continue if is_identifier_start(char): end = index + 1 while end < len(text) and is_identifier_continue(text[end]): end += 1 identifiers.append((line, text[index:end])) index = end continue if char == "\n": line += 1 index += 1 continue if state == "line_comment": if char == "\n": line += 1 state = "code" index += 1 continue if state == "block_comment": if char == "*" and following == "/": state = "code" index += 2 continue if char == "\n": line += 1 index += 1 continue if state == "template": if char == "\\": if following == "\n": line += 1 index += 2 continue if char == "`": state = "code" index += 1 continue if char == "$" and following == "{": template_expression_depths.append(1) state = "code" index += 2 continue if char == "\n": line += 1 index += 1 continue if state == "raw_string": if char == "`": state = "code" index += 1 continue if char == "\n": line += 1 index += 1 continue if char == "\\": if following == "\n": line += 1 index += 2 continue if char == quote: state = "code" index += 1 continue if char == "\n": line += 1 index += 1 return identifiers def source_paths(repo_root: Path) -> Iterable[Path]: for source_root in SOURCE_ROOTS: root = repo_root / source_root if not root.is_dir(): continue for path in root.rglob("*"): if not path.is_file() or path.suffix not in SOURCE_SUFFIXES: continue relative = path.relative_to(repo_root) if any(part in SKIPPED_PARTS for part in relative.parts): continue if relative.parts[:3] == ("internal", "tidb", "migrations"): continue yield path def load_rules(path: Path) -> list[Rule]: rules: list[Rule] = [] for number, raw_line in enumerate(path.read_text().splitlines(), start=1): if not raw_line or raw_line.startswith("#"): continue try: pattern, preferred = raw_line.split("\t", maxsplit=1) except ValueError as error: raise ValueError(f"{path}:{number}: expected patternpreferred") from error if not preferred: raise ValueError(f"{path}:{number}: preferred terminology is empty") rules.append(Rule(re.compile(pattern), preferred)) if not rules: raise ValueError(f"{path}: no terminology rules") return rules def load_exceptions(path: Path) -> set[tuple[str, str]]: exceptions: set[tuple[str, str]] = set() for number, raw_line in enumerate(path.read_text().splitlines(), start=1): if not raw_line or raw_line.startswith("#"): continue try: source_path, identifier = raw_line.split("::", maxsplit=1) except ValueError as error: raise ValueError(f"{path}:{number}: expected path::identifier") from error if not source_path or not identifier: raise ValueError(f"{path}:{number}: exception path or identifier is empty") exceptions.add((source_path, identifier)) return exceptions def matching_rule(identifier: str, rules: list[Rule]) -> Rule | None: return next((rule for rule in rules if rule.pattern.fullmatch(identifier)), None) def scan_repository( repo_root: Path, rules: list[Rule], exceptions: set[tuple[str, str]] ) -> tuple[list[str], set[tuple[str, str]]]: hits: list[str] = [] used_exceptions: set[tuple[str, str]] = set() for path in source_paths(repo_root): relative = path.relative_to(repo_root).as_posix() text = path.read_text(encoding="utf-8", errors="replace") if path.suffix == ".go": language = "go" elif path.suffix == ".vue": language = "vue" else: language = "javascript" for line, identifier in scan_text( text, language=language, vue=language == "vue" ): rule = matching_rule(identifier, rules) if rule is None: continue key = (relative, identifier) if key in exceptions: used_exceptions.add(key) continue hits.append(f"{relative}:{line}:{identifier} -> {rule.preferred}") return hits, used_exceptions def self_test() -> None: sample = """ surfaceNotificationPanel := runtimeSurface RepoRoleCanonical := canonicalJSON // ignoredSurfaceNotification and RepoRoleCanonical message := "SurfaceNotificationPanel RepoRoleCanonical" template := `ignored SurfaceNotificationPanel ${surfaceNotificationPanel}` /* ignoredCanonicalFailure */ """ assert scan_text(sample) == [ (2, "surfaceNotificationPanel"), (2, "runtimeSurface"), (3, "RepoRoleCanonical"), (3, "canonicalJSON"), (5, "message"), (6, "template"), (6, "surfaceNotificationPanel"), ] template_identifiers = { identifier for _, identifier in scan_text( "template := `${runtimeMessageSurface} ${surfaceNotificationPanel}`", language="javascript", ) } assert template_identifiers == { "template", "runtimeMessageSurface", "surfaceNotificationPanel", } go_raw_identifiers = scan_text( "const frozen = `runtime payload ${SurfaceNotificationPanel}`", language="go", ) assert go_raw_identifiers == [(1, "const"), (1, "frozen")] vue_identifiers = { identifier for _, identifier in scan_text( """ """, vue=True, ) } assert "SurfaceNotificationPanel" in vue_identifiers assert "runtimeMessageSurface" in vue_identifiers assert "PromptSurfaceLabel" not in vue_identifiers assert "CanonicalJSONObject" in vue_identifiers assert "SurfaceAgentsBulkWakeActions" in vue_identifiers rules = [ Rule(re.compile(r"[A-Za-z0-9_]*[Ss]urfaceNotification[A-Za-z0-9_]*"), "scope"), Rule( re.compile(r"[A-Za-z0-9_]*[Rr]epoRoleCanonical[A-Za-z0-9_]*"), "RepoRolePrimary", ), Rule(re.compile(r"[A-Za-z0-9_]*[Rr]untimeMessageSurface[A-Za-z0-9_]*"), "delivery"), Rule(re.compile(r"[A-Za-z0-9_]*[Rr]untimeSurfaceReceipts[A-Za-z0-9_]*"), "receipts"), Rule(re.compile(r"[A-Za-z0-9_]*[Ss]urfaceAgentsBulkWakeActions[A-Za-z0-9_]*"), "actions"), Rule(re.compile(r"[A-Za-z0-9_]*(?:[Cc]anonicalJSONObject)[A-Za-z0-9_]*"), "deterministic JSON"), Rule(re.compile(r"[A-Za-z0-9_]*[Pp]romptSurface[A-Za-z0-9_]*"), "prompt slot"), ] assert matching_rule("surfaceNotificationPanel", rules) == rules[0] assert matching_rule("SurfaceNotificationPanel", rules) == rules[0] assert matching_rule("RepoRoleCanonical", rules) == rules[1] assert matching_rule("repoRoleCanonical", rules) == rules[1] assert matching_rule("legacyRepoRoleCanonical", rules) == rules[1] assert matching_rule("TestRepoRoleCanonical", rules) == rules[1] assert matching_rule("runtimeMessageSurface", rules) == rules[2] assert matching_rule("runtimeSurfaceReceipts", rules) == rules[3] assert matching_rule("surfaceAgentsBulkWakeActions", rules) == rules[4] assert matching_rule("CanonicalJSONObject", rules) == rules[5] assert matching_rule("canonicalJSONObject", rules) == rules[5] assert matching_rule("PromptSurfaceDeliveryText", rules) == rules[6] assert matching_rule("promptSurfaceDeliveryText", rules) == rules[6] assert matching_rule("runtimeSurface", rules) is None assert matching_rule("canonicalJSON", rules) is None print("semantic terminology scanner self-test passed") def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--self-test", action="store_true") parser.add_argument("--root", type=Path) parser.add_argument("--rules", type=Path) parser.add_argument("--exceptions", type=Path) args = parser.parse_args() if args.self_test: self_test() return 0 if args.rules is None or args.exceptions is None: parser.error("--rules and --exceptions are required") repo_root = (args.root or Path(__file__).resolve().parents[2]).resolve() try: rules = load_rules(args.rules) exceptions = load_exceptions(args.exceptions) except (OSError, ValueError, re.error) as error: print(f"FAIL: {error}", file=sys.stderr) return 2 hits, used_exceptions = scan_repository(repo_root, rules, exceptions) for hit in hits: print(hit) stale = exceptions - used_exceptions if stale: for source_path, identifier in sorted(stale): print(f"STALE EXCEPTION: {source_path}::{identifier}", file=sys.stderr) return 2 return 1 if hits else 0 if __name__ == "__main__": raise SystemExit(main())