package memorysearch

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"strconv"
	"strings"
	"sync"
	"unicode/utf8"

	"github.com/yuin/goldmark"
	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/extension"
	gast "github.com/yuin/goldmark/extension/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/text"
)

// Chunk targets. The chunker aims for chunks of roughly chunkTargetTokens
// model-token estimate, never exceeding chunkMaxTokens unless a single
// atomic AST block (fenced code, table, list) is itself larger. Paragraphs
// that would exceed chunkMaxTokens are split by sentence/word so the chunk
// stays within budget. Overlap between adjacent chunks under the same
// heading copies the previous chunk's last block to preserve search context.
const (
	chunkTargetTokens = 600
	chunkMaxTokens    = 900
	chunkOverlapMin   = 80
	chunkOverlapMax   = 120
)

// ChunkInput is the per-file input to Chunk. Kind is the path-domain Kind
// (mirrors SourceFile.Kind); Format records the on-disk source format and
// drives the chunker dispatcher (markdown vs. plain text).
type ChunkInput struct {
	Path           string
	Kind           string
	Format         string
	Archived       bool
	ContentHash    string
	SourceRevision string
	Data           []byte
}

var goldmarkParser = sync.OnceValue(func() parser.Parser {
	md := goldmark.New(goldmark.WithExtensions(extension.GFM))
	return md.Parser()
})

// Chunk splits a single source file into stable SearchChunks. The
// stable ID is sha256(path + NUL + headingAnchor + NUL + decimal(ordinal)),
// so the same source content always yields the same chunk IDs. The chunker
// dispatches on ChunkInput.Format (markdown vs. plain text).
func Chunk(in ChunkInput) []SearchChunk {
	if len(in.Data) == 0 {
		return nil
	}
	blocks := extractBlocks(in.Format, in.Data)
	if len(blocks) == 0 {
		return nil
	}
	return assembleChunks(in, blocks)
}

// mdBlock is one AST-level block with its resolved heading context and line
// range. The chunker groups blocks into chunks. Atomic blocks (fenced code,
// tables, lists) are never split mid-construct even if they exceed
// chunkMaxTokens; paragraphs and text blocks may be split into multiple
// mdBlocks to respect the size budget. Atomic blocks also force a chunk
// boundary before and after themselves so that search results can target a
// code block or table without carrying unrelated paragraph prose.
type mdBlock struct {
	HeadingPath   []string
	HeadingText   string
	HeadingAnchor string
	Body          string
	Atomic        bool
	StartLine     int
	EndLine       int
}

func extractBlocks(format string, data []byte) []mdBlock {
	switch format {
	case SourceFormatMarkdown, "markdown-long-ext":
		return extractMarkdownBlocks(data)
	default:
		return extractTextBlocks(data)
	}
}

func extractMarkdownBlocks(data []byte) []mdBlock {
	reader := text.NewReader(data)
	doc := goldmarkParser().Parse(reader)
	lineStarts := computeLineStarts(data)

	var blocks []mdBlock
	headingStack := []string{}

	var visit func(parent ast.Node)
	visit = func(parent ast.Node) {
		for c := parent.FirstChild(); c != nil; c = c.NextSibling() {
			switch c.Kind() {
			case ast.KindHeading:
				// Headings establish context for subsequent blocks but do not
				// emit a block themselves; the heading text is captured in
				// HeadingText/HeadingAnchor of the following blocks, and
				// emitting a separate body chunk for the bare marker line
				// would only duplicate that text.
				h := c.(*ast.Heading)
				heading := string(c.Text(data))
				pushHeading(&headingStack, h.Level, heading)
			case ast.KindParagraph:
				body, startLine, endLine := nodeBodyAndLines(c, data, lineStarts)
				if strings.TrimSpace(body) == "" {
					continue
				}
				para := mdBlock{
					HeadingPath:   copyStrings(headingStack),
					HeadingText:   lastNonEmpty(headingStack),
					HeadingAnchor: anchorize(lastNonEmpty(headingStack)),
					Body:          body,
					StartLine:     startLine,
					EndLine:       endLine,
				}
				// Paragraphs are splittable: a paragraph that exceeds the
				// token budget is broken into multiple mdBlocks by sentence
				// so the resulting chunks stay within chunkMaxTokens.
				blocks = append(blocks, splitSplittableBlock(para, chunkMaxTokens)...)
			case ast.KindFencedCodeBlock, ast.KindCodeBlock, ast.KindList, gast.KindTable:
				// Atomic blocks: never split mid-construct even if they
				// exceed chunkMaxTokens. The body preserves the original
				// source syntax (code fences, list markers, table pipes).
				// Atomic also forces a chunk boundary before and after so
				// search results can target the construct in isolation.
				body, startLine, endLine := nodeBodyAndLines(c, data, lineStarts)
				if strings.TrimSpace(body) == "" {
					continue
				}
				blocks = append(blocks, mdBlock{
					HeadingPath:   copyStrings(headingStack),
					HeadingText:   lastNonEmpty(headingStack),
					HeadingAnchor: anchorize(lastNonEmpty(headingStack)),
					Body:          body,
					Atomic:        true,
					StartLine:     startLine,
					EndLine:       endLine,
				})
			case ast.KindBlockquote:
				visit(c)
			default:
				// Default covers HTMLBlock, ThematicBreak, and other
				// text-leaning block kinds. They are splittable just like
				// paragraphs: route through splitSplittableBlock so the
				// 900-token ceiling holds for oversized HTML or text dumps.
				body, startLine, endLine := nodeBodyAndLines(c, data, lineStarts)
				if strings.TrimSpace(body) == "" {
					continue
				}
				raw := mdBlock{
					HeadingPath:   copyStrings(headingStack),
					HeadingText:   lastNonEmpty(headingStack),
					HeadingAnchor: anchorize(lastNonEmpty(headingStack)),
					Body:          body,
					StartLine:     startLine,
					EndLine:       endLine,
				}
				blocks = append(blocks, splitSplittableBlock(raw, chunkMaxTokens)...)
			}
		}
	}
	visit(doc)
	return blocks
}

// extractTextBlocks handles non-markdown text content (.txt, .json, .yaml) by
// grouping consecutive non-blank lines into paragraphs. Paragraphs that
// exceed chunkMaxTokens are split by sentence via splitSplittableBlock.
func extractTextBlocks(data []byte) []mdBlock {
	lines := strings.Split(string(data), "\n")
	var blocks []mdBlock
	var cur []string
	startLine := 0
	flush := func(endLine int) {
		if len(cur) == 0 {
			return
		}
		body := strings.Join(cur, "\n")
		raw := mdBlock{
			HeadingPath:   nil,
			HeadingText:   "",
			HeadingAnchor: "",
			Body:          body,
			StartLine:     startLine,
			EndLine:       endLine,
		}
		blocks = append(blocks, splitSplittableBlock(raw, chunkMaxTokens)...)
		cur = nil
	}
	for i, line := range lines {
		if strings.TrimSpace(line) == "" {
			flush(i)
			continue
		}
		if len(cur) == 0 {
			startLine = i + 1
		}
		cur = append(cur, line)
	}
	flush(len(lines))
	return blocks
}

// nodeBodyAndLines returns the raw source bytes for an AST block, the
// 1-based line of its first byte, and the 1-based line of its last byte.
//
// The byte range is computed by walking the subtree and taking the min/max
// of descendant line segments, then extending the range back to the start
// of the first line and forward through the trailing newline of the last
// line. The extension is necessary because Goldmark's line segments start
// AFTER the block's leading syntax (heading `#`, list `-`, table `|`), and
// the original source syntax must be preserved in the chunk Body.
//
// Fenced code blocks additionally include their opening and closing fence
// lines, which Goldmark excludes from any descendant segment.
func nodeBodyAndLines(n ast.Node, data []byte, lineStarts []int) (string, int, int) {
	start, stop, ok := nodeByteRange(n)
	if !ok {
		return string(n.Text(data)), 0, 0
	}
	start, stop = extendToLineBoundaries(data, start, stop)
	if n.Kind() == ast.KindFencedCodeBlock {
		start, stop = extendFencedCodeRange(data, start, stop)
	}
	if stop <= start || stop > len(data) {
		return string(n.Text(data)), 0, 0
	}
	body := string(data[start:stop])
	startLine := lineOf(lineStarts, start)
	endLine := lineOf(lineStarts, maxOffset(stop-1, start))
	return body, startLine, endLine
}

func nodeByteRange(n ast.Node) (start, stop int, ok bool) {
	first := -1
	last := -1
	ast.Walk(n, func(nn ast.Node, entering bool) (ast.WalkStatus, error) {
		if !entering {
			return ast.WalkContinue, nil
		}
		// Inline nodes panic when Lines() is called; only block-level nodes
		// carry source segments in the Goldmark AST.
		if nn.Type() != ast.TypeBlock {
			return ast.WalkContinue, nil
		}
		lines := nn.Lines()
		for i := 0; i < lines.Len(); i++ {
			seg := lines.At(i)
			if first < 0 || seg.Start < first {
				first = int(seg.Start)
			}
			if int(seg.Stop) > last {
				last = int(seg.Stop)
			}
		}
		return ast.WalkContinue, nil
	})
	if first < 0 {
		return 0, 0, false
	}
	return first, last, true
}

// extendToLineBoundaries grows the [start, stop) byte range to cover the
// full first and last lines of the block. The first byte is moved back to
// the byte after the preceding newline (or 0); the stop byte is moved
// forward to include the trailing newline of the last line.
func extendToLineBoundaries(data []byte, start, stop int) (int, int) {
	for start > 0 && data[start-1] != '\n' {
		start--
	}
	for stop < len(data) && data[stop] != '\n' {
		stop++
	}
	if stop < len(data) {
		stop++ // include the trailing '\n'
	}
	if stop > len(data) {
		stop = len(data)
	}
	return start, stop
}

// extendFencedCodeRange grows the [start, stop) byte range to include the
// opening and closing fence lines of a FencedCodeBlock. Goldmark excludes
// the fence lines from descendant line segments, so they must be recovered
// by scanning the surrounding source.
func extendFencedCodeRange(data []byte, start, stop int) (int, int) {
	// Opening fence: the line immediately preceding 'start' (which is at the
	// start of the first inner content line).
	if start > 0 && data[start-1] == '\n' {
		fenceEnd := start - 1 // position of the '\n'
		fenceStart := 0
		for i := fenceEnd - 1; i >= 0; i-- {
			if data[i] == '\n' {
				fenceStart = i + 1
				break
			}
		}
		if isFenceLine(data[fenceStart:fenceEnd]) {
			start = fenceStart
		}
	}
	// Closing fence: the line immediately following 'stop' (which is just
	// past the trailing '\n' of the last inner content line).
	if stop < len(data) {
		fenceStart := stop
		fenceEnd := stop
		for fenceEnd < len(data) && data[fenceEnd] != '\n' {
			fenceEnd++
		}
		if isFenceLine(data[fenceStart:fenceEnd]) {
			stop = fenceEnd
			if stop < len(data) {
				stop++ // include the '\n'
			}
		}
	}
	if stop > len(data) {
		stop = len(data)
	}
	return start, stop
}

// isFenceLine reports whether a line (without its trailing newline) is a
// Markdown fenced-code opening or closing marker.
func isFenceLine(line []byte) bool {
	trimmed := bytes.TrimLeft(line, " \t")
	if len(trimmed) >= 3 && bytes.HasPrefix(trimmed, []byte("```")) {
		return true
	}
	if len(trimmed) >= 3 && bytes.HasPrefix(trimmed, []byte("~~~")) {
		return true
	}
	return false
}

func computeLineStarts(data []byte) []int {
	starts := make([]int, 1, bytes.Count(data, []byte{'\n'})+1)
	for i, b := range data {
		if b == '\n' {
			starts = append(starts, i+1)
		}
	}
	return starts
}

func lineOf(lineStarts []int, offset int) int {
	lo, hi := 0, len(lineStarts)-1
	for lo < hi {
		mid := (lo + hi + 1) / 2
		if lineStarts[mid] <= offset {
			lo = mid
		} else {
			hi = mid - 1
		}
	}
	return lo + 1
}

func maxOffset(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func pushHeading(stack *[]string, level int, text string) {
	if level < 1 {
		level = 1
	}
	if level-1 < len(*stack) {
		*stack = (*stack)[:level-1]
	}
	for len(*stack) < level-1 {
		*stack = append(*stack, "")
	}
	*stack = append(*stack, text)
}

func copyStrings(in []string) []string {
	if len(in) == 0 {
		return nil
	}
	out := make([]string, len(in))
	copy(out, in)
	return out
}

func lastNonEmpty(stack []string) string {
	for i := len(stack) - 1; i >= 0; i-- {
		if stack[i] != "" {
			return stack[i]
		}
	}
	return ""
}

// anchorize produces a stable heading anchor by lowercasing, replacing
// whitespace runs with single hyphens, and dropping punctuation that is not
// a hyphen or path-safe. CJK and other non-ASCII letters are preserved.
func anchorize(text string) string {
	text = strings.TrimSpace(strings.ToLower(text))
	var out strings.Builder
	out.Grow(len(text))
	inSpace := false
	for _, r := range text {
		switch {
		case r == ' ' || r == '\t' || r == '\n' || r == '\r':
			inSpace = true
		case r == '-' || r == '_':
			if inSpace {
				out.WriteRune('-')
				inSpace = false
			}
			out.WriteRune(r)
		case r >= '0' && r <= '9', r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z':
			if inSpace {
				out.WriteRune('-')
				inSpace = false
			}
			out.WriteRune(r)
		case r >= 0x80:
			if inSpace {
				out.WriteRune('-')
				inSpace = false
			}
			out.WriteRune(r)
		default:
			inSpace = true
		}
	}
	return strings.Trim(out.String(), "-")
}

// chunkTitle returns a display-friendly chunk title. It prefers the deepest
// available heading text, falling back to the file path basename.
func chunkTitle(path string, headingPath []string, headingText string) string {
	if headingText != "" {
		return headingText
	}
	if len(headingPath) > 0 {
		return headingPath[len(headingPath)-1]
	}
	if idx := strings.LastIndexByte(path, '/'); idx >= 0 {
		return path[idx+1:]
	}
	return path
}

// splitSplittableBlock applies sentence-level splitting to a paragraph or
// text block whose body exceeds maxTokens. Atomic blocks (fenced code,
// table, list) bypass this path. The split operates on byte spans of the
// original body so that line ranges are derived from real newlines, not
// from post-split whitespace counting.
//
// The splitter is strict: every emitted piece (except atomic blocks, which
// bypass splitting entirely) must satisfy estimateTokens(piece) <= maxTokens.
// Sentences that exceed maxTokens are recursively split by word, then by
// rune; the rune fallback guarantees the hard ceiling even for contiguous
// CJK runs or over-long ASCII tokens that contain no whitespace.
func splitSplittableBlock(b mdBlock, maxTokens int) []mdBlock {
	if maxTokens <= 0 {
		return []mdBlock{b}
	}
	if estimateTokens(b.Body) <= maxTokens {
		return []mdBlock{b}
	}
	spans := splitBodySpans(b.Body, maxTokens)
	if len(spans) <= 1 {
		return []mdBlock{b}
	}
	out := make([]mdBlock, 0, len(spans))
	for _, sp := range spans {
		if sp.stop <= sp.start {
			continue
		}
		piece := b.Body[sp.start:sp.stop]
		out = append(out, mdBlock{
			HeadingPath:   b.HeadingPath,
			HeadingText:   b.HeadingText,
			HeadingAnchor: b.HeadingAnchor,
			Body:          piece,
			StartLine:     b.StartLine + countNewlines(b.Body[:sp.start]),
			EndLine:       b.StartLine + countNewlines(b.Body[:sp.stop-1]),
			Atomic:        false,
		})
	}
	if len(out) == 0 {
		return []mdBlock{b}
	}
	return out
}

// splitBodySpans walks body and returns ordered, contiguous, non-overlapping
// byte spans such that:
//   - the concatenation of body[span.start:span.stop] for all spans equals body
//     (no whitespace is invented or dropped);
//   - estimateTokens(body[span.start:span.stop]) <= maxTokens for every span.
//
// It first splits on sentence boundaries (ASCII + CJK terminators). A
// sentence whose body itself exceeds maxTokens is recursively split on word
// boundaries; a word whose body still exceeds maxTokens is split on rune
// boundaries. The rune fallback guarantees the hard ceiling for contiguous
// CJK runs and for over-long ASCII tokens that have no internal whitespace.
func splitBodySpans(body string, maxTokens int) []span {
	atoms := sentenceAtoms(body)
	// Expand any oversized atoms into smaller ones.
	var expanded []span
	for _, a := range atoms {
		if estimateTokens(body[a.start:a.stop]) <= maxTokens {
			expanded = append(expanded, a)
			continue
		}
		expanded = append(expanded, wordAtoms(body, a.start, a.stop, maxTokens)...)
	}
	// Greedily group atoms into pieces under maxTokens. Because every atom
	// individually satisfies the ceiling, the greedy walk always makes
	// progress and produces at least one atom per piece.
	var out []span
	pieceStart := -1
	pieceEnd := 0
	pieceTokens := 0
	for _, a := range expanded {
		st := estimateTokens(body[a.start:a.stop])
		if pieceStart >= 0 && pieceTokens+st > maxTokens {
			out = append(out, span{pieceStart, pieceEnd})
			pieceStart = -1
			pieceTokens = 0
		}
		if pieceStart < 0 {
			pieceStart = a.start
		}
		pieceEnd = a.stop
		pieceTokens += st
	}
	if pieceStart >= 0 {
		out = append(out, span{pieceStart, pieceEnd})
	}
	return out
}

type span struct{ start, stop int }

// sentenceAtoms returns byte spans that partition body at sentence
// terminators (ASCII '.', '!', '?' and CJK '。', '！', '？'). Each terminator
// is kept with the preceding atom, along with any trailing whitespace, so
// atoms are contiguous and cover the entire body.
func sentenceAtoms(body string) []span {
	var out []span
	atomStart := 0
	i := 0
	for i < len(body) {
		r, size := utf8DecodeRune(body[i:])
		if size == 0 {
			break
		}
		i += size
		if isSentenceTerminator(r) {
			for i < len(body) {
				r2, size2 := utf8DecodeRune(body[i:])
				if size2 == 0 || !isSpaceRune(r2) {
					break
				}
				i += size2
			}
			out = append(out, span{atomStart, i})
			atomStart = i
		}
	}
	if atomStart < len(body) {
		out = append(out, span{atomStart, len(body)})
	}
	if len(out) == 0 && len(body) > 0 {
		out = append(out, span{0, len(body)})
	}
	return out
}

// wordAtoms splits body[start:stop] on whitespace into atoms, each carrying
// its leading whitespace so atoms stay contiguous. A word that itself
// exceeds maxTokens is recursively split on rune boundaries via runeAtoms.
func wordAtoms(body string, start, stop, maxTokens int) []span {
	var atoms []span
	wordStart := -1
	for i := start; i < stop; {
		r, size := utf8DecodeRune(body[i:])
		if size == 0 {
			break
		}
		if isSpaceRune(r) {
			if wordStart >= 0 {
				if estimateTokens(body[wordStart:i]) > maxTokens {
					atoms = append(atoms, runeAtoms(body, wordStart, i, maxTokens)...)
				} else {
					atoms = append(atoms, span{wordStart, i})
				}
				wordStart = -1
			}
			atoms = append(atoms, span{i, i + size})
			i += size
			continue
		}
		if wordStart < 0 {
			wordStart = i
		}
		i += size
	}
	if wordStart >= 0 {
		if estimateTokens(body[wordStart:stop]) > maxTokens {
			atoms = append(atoms, runeAtoms(body, wordStart, stop, maxTokens)...)
		} else {
			atoms = append(atoms, span{wordStart, stop})
		}
	}
	if len(atoms) == 0 {
		atoms = runeAtoms(body, start, stop, maxTokens)
	}
	return atoms
}

// runeAtoms splits body[start:stop] into contiguous rune-count chunks each
// within maxTokens. The estimateTokens heuristic counts every non-whitespace
// rune as one token, so capping the rune count caps the token count.
func runeAtoms(body string, start, stop, maxTokens int) []span {
	var out []span
	s := start
	count := 0
	prevS := s
	for i := s; i < stop; {
		_, size := utf8DecodeRune(body[i:])
		if size == 0 {
			break
		}
		i += size
		count++
		if count >= maxTokens {
			out = append(out, span{prevS, i})
			prevS = i
			count = 0
		}
	}
	if prevS < stop {
		out = append(out, span{prevS, stop})
	}
	if len(out) == 0 {
		out = []span{{start, stop}}
	}
	return out
}

func isSpaceRune(r rune) bool {
	switch r {
	case ' ', '\t', '\n', '\r':
		return true
	}
	return false
}

// utf8DecodeRune is a thin wrapper over utf8.DecodeRuneInString that returns
// rune 0 and size 0 on empty input. It is broken out so the splitter can be
// read without dragging in another inline DecodeRune call site.
func utf8DecodeRune(s string) (rune, int) {
	if len(s) == 0 {
		return 0, 0
	}
	r, size := utf8.DecodeRuneInString(s)
	if size == 0 {
		return 0, 0
	}
	return r, size
}

func countNewlines(s string) int {
	return strings.Count(s, "\n")
}

func isSentenceTerminator(r rune) bool {
	switch r {
	case '.', '!', '?', '。', '！', '？':
		return true
	}
	return false
}

func assembleChunks(in ChunkInput, blocks []mdBlock) []SearchChunk {
	var chunks []SearchChunk
	var pending []mdBlock
	pendingTokens := 0
	ordinal := 0

	emit := func() {
		if len(pending) == 0 {
			return
		}
		body := strings.Join(blockBodies(pending), "\n\n")
		ordinal++
		head := pending[0]
		chunks = append(chunks, SearchChunk{
			ID:              stableChunkID(in.Path, head.HeadingAnchor, ordinal),
			Path:            in.Path,
			Title:           chunkTitle(in.Path, head.HeadingPath, head.HeadingText),
			HeadingPath:     copyStrings(head.HeadingPath),
			HeadingText:     head.HeadingText,
			HeadingAnchor:   head.HeadingAnchor,
			Body:            body,
			Kind:            in.Kind,
			Format:          in.Format,
			Archived:        in.Archived,
			Identifiers:     ExtractIdentifiers(body),
			IdentifierParts: ExtractIdentifierParts(body),
			Ordinal:         ordinal,
			StartLine:       pending[0].StartLine,
			EndLine:         pending[len(pending)-1].EndLine,
			ContentHash:     in.ContentHash,
			SourceRevision:  in.SourceRevision,
		})
		pending = nil
		pendingTokens = 0
	}

	for i, b := range blocks {
		blockTokens := estimateTokens(b.Body)
		headingChanged := len(pending) > 0 && !equalStrings(pending[0].HeadingPath, b.HeadingPath)
		wouldOverflow := pendingTokens+blockTokens > chunkMaxTokens
		// Atomic blocks (fenced code, tables, lists) must land in their own
		// chunk so search results can target the construct in isolation. Flush
		// any pending splittable blocks both before and after the atomic one;
		// do not carry overlap across an atomic boundary because the fence or
		// table syntax would leak into the adjacent chunk.
		if len(pending) > 0 && (headingChanged || wouldOverflow || b.Atomic) {
			emit()
			if i > 0 && !headingChanged && !b.Atomic {
				overlap := blocks[i-1]
				overlapTokens := estimateTokens(overlap.Body)
				// Only carry the overlap if it leaves enough budget for the
				// next block to land in the same chunk without re-crossing
				// the ceiling. Otherwise the carried context would defeat
				// the splittable ceiling guarantee for the next chunk.
				if overlapTokens >= chunkOverlapMin &&
					overlapTokens <= chunkOverlapMax &&
					overlapTokens+blockTokens <= chunkMaxTokens {
					pending = append(pending, overlap)
					pendingTokens += overlapTokens
				}
			}
		}
		pending = append(pending, b)
		pendingTokens += blockTokens
		if b.Atomic {
			// Flush immediately so the atomic block is its own chunk and the
			// next splittable block starts fresh.
			emit()
		}
	}
	emit()
	return chunks
}

func blockBodies(blocks []mdBlock) []string {
	out := make([]string, 0, len(blocks))
	for _, b := range blocks {
		out = append(out, b.Body)
	}
	return out
}

func equalStrings(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

// stableChunkID returns sha256(path + NUL + headingAnchor + NUL + ordinal)
// where ordinal is written as normalized decimal via strconv.Itoa. The
// decimal form is part of the frozen contract: a chunk that re-appears at
// the same path + heading + ordinal must produce the same ID across
// implementations.
func stableChunkID(path, headingAnchor string, ordinal int) string {
	h := sha256.New()
	h.Write([]byte(path))
	h.Write([]byte{0})
	h.Write([]byte(headingAnchor))
	h.Write([]byte{0})
	h.Write([]byte(strconv.Itoa(ordinal)))
	return hex.EncodeToString(h.Sum(nil))
}

// estimateTokens is a deterministic rune/ASCII-word/CJK heuristic. It is not
// a model tokenizer; it exists only to make chunk sizes roughly match what a
// real model tokenizer would produce, so chunks do not blow past context
// budgets.
//
//   - ASCII alphanumeric words (including underscores) count as 1 token each.
//   - CJK and other non-ASCII runes count as 1 token each.
//   - ASCII punctuation counts as 1 token when it stands alone.
//   - Whitespace counts as 0 tokens.
func estimateTokens(s string) int {
	tokens := 0
	inWord := false
	for _, r := range s {
		switch {
		case r == ' ' || r == '\t' || r == '\n' || r == '\r':
			inWord = false
		case r < 0x80 && (r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r == '_'):
			if !inWord {
				tokens++
				inWord = true
			}
		case r < 0x80:
			tokens++
			inWord = false
		default:
			tokens++
			inWord = false
		}
	}
	return tokens
}