package memorysearch

import (
	"crypto/sha256"
	"fmt"
	"strconv"
	"strings"
	"testing"
)

func chunkBody(s string) []SearchChunk {
	return Chunk(ChunkInput{
		Path:           "test.md",
		Kind:           SourceKindRoot,
		Format:         SourceFormatMarkdown,
		ContentHash:    "h",
		SourceRevision: "r",
		Data:           []byte(s),
	})
}

func TestChunkEmpty(t *testing.T) {
	if got := chunkBody(""); len(got) != 0 {
		t.Fatalf("expected no chunks for empty input, got %d", len(got))
	}
}

func TestChunkMarkdownHeadingPath(t *testing.T) {
	body := "# Top\n\nParagraph under top.\n\n## Sub\n\nParagraph under sub.\n"
	chunks := chunkBody(body)
	if len(chunks) == 0 {
		t.Fatalf("no chunks emitted")
	}
	wantPaths := [][]string{{"Top"}, {"Top", "Sub"}}
	if len(chunks) != len(wantPaths) {
		t.Fatalf("expected %d chunks (one per heading section), got %d", len(wantPaths), len(chunks))
	}
	for i, want := range wantPaths {
		if !equalStrings(chunks[i].HeadingPath, want) {
			t.Errorf("chunk[%d] HeadingPath = %v, want %v", i, chunks[i].HeadingPath, want)
		}
	}
	if chunks[0].HeadingText != "Top" {
		t.Errorf("chunk[0] HeadingText = %q, want %q", chunks[0].HeadingText, "Top")
	}
	if chunks[1].HeadingText != "Sub" {
		t.Errorf("chunk[1] HeadingText = %q, want %q", chunks[1].HeadingText, "Sub")
	}
}

func TestChunkMarkdownChineseHeading(t *testing.T) {
	body := "# 迁移审计\n\nPR #2731 在 merge tree 中出现 migration 254 重号。\n"
	chunks := chunkBody(body)
	if len(chunks) != 1 {
		t.Fatalf("expected 1 chunk, got %d", len(chunks))
	}
	if chunks[0].HeadingText != "迁移审计" {
		t.Errorf("HeadingText = %q", chunks[0].HeadingText)
	}
	if !strings.Contains(chunks[0].Body, "PR #2731") {
		t.Errorf("Body lost content: %q", chunks[0].Body)
	}
	if chunks[0].StartLine < 1 {
		t.Errorf("StartLine should be 1-based positive, got %d", chunks[0].StartLine)
	}
	if chunks[0].EndLine < chunks[0].StartLine {
		t.Errorf("EndLine %d < StartLine %d", chunks[0].EndLine, chunks[0].StartLine)
	}
}

func TestChunkMarkdownMixedCJKAscii(t *testing.T) {
	body := "# Review 流程\n\nOpen PR, request review from @alice, then merge after approval.\n"
	chunks := chunkBody(body)
	if len(chunks) != 1 {
		t.Fatalf("expected 1 chunk, got %d", len(chunks))
	}
	if chunks[0].HeadingText != "Review 流程" {
		t.Errorf("HeadingText = %q", chunks[0].HeadingText)
	}
}

func TestChunkMarkdownFencedCodeIsAtomic(t *testing.T) {
	body := "# Code\n\nIntro.\n\n```go\nfunc main() {\n    println(\"hi\")\n}\n```\n\nAfter.\n"
	chunks := chunkBody(body)
	for _, c := range chunks {
		if strings.Contains(c.Body, "```go") && strings.Contains(c.Body, "After.") {
			t.Errorf("fenced code block leaked into adjacent block: %q", c.Body)
		}
	}
	// The fenced code block must live in a chunk that preserves the full
	// source syntax: opening fence (with language), inner lines, closing fence.
	var fenceChunk *SearchChunk
	for i := range chunks {
		if strings.Contains(chunks[i].Body, "func main") && strings.Contains(chunks[i].Body, "println") {
			fenceChunk = &chunks[i]
			break
		}
	}
	if fenceChunk == nil {
		t.Fatalf("fenced code body lost across chunks")
	}
	for _, want := range []string{"```go", "func main() {", "println(\"hi\")", "```"} {
		if !strings.Contains(fenceChunk.Body, want) {
			t.Errorf("fence source %q missing from chunk body: %q", want, fenceChunk.Body)
		}
	}
}

func TestChunkMarkdownListAtomic(t *testing.T) {
	body := "# Tasks\n\n- item one\n- item two\n- item three\n"
	chunks := chunkBody(body)
	if len(chunks) != 1 {
		t.Fatalf("expected single chunk for tight list, got %d", len(chunks))
	}
	// Original list marker syntax (`- `) must be preserved in the chunk body,
	// not flattened to bare text by the AST walk.
	for _, want := range []string{"- item one", "- item two", "- item three"} {
		if !strings.Contains(chunks[0].Body, want) {
			t.Errorf("list marker %q missing from chunk body: %q", want, chunks[0].Body)
		}
	}
}

func TestChunkMarkdownGFMTableAtomic(t *testing.T) {
	body := "# Matrix\n\n| Kind | Count |\n| --- | --- |\n| a | 1 |\n| b | 2 |\n"
	chunks := chunkBody(body)
	if len(chunks) == 0 {
		t.Fatalf("no chunks emitted")
	}
	// Find the chunk that contains the table header.
	var tableChunk *SearchChunk
	for i := range chunks {
		if strings.Contains(chunks[i].Body, "Kind") && strings.Contains(chunks[i].Body, "Count") {
			tableChunk = &chunks[i]
			break
		}
	}
	if tableChunk == nil {
		t.Fatalf("table body not found in any chunk")
	}
	// Full table syntax must be preserved: header row with pipes, delimiter
	// row, and each data row with leading/trailing pipes.
	for _, want := range []string{"| Kind | Count |", "| --- | --- |", "| a | 1 |", "| b | 2 |"} {
		if !strings.Contains(tableChunk.Body, want) {
			t.Errorf("table source %q missing from chunk body: %q", want, tableChunk.Body)
		}
	}
	// No chunk should split the table across the header/data boundary.
	for _, c := range chunks {
		if strings.Contains(c.Body, "| Kind |") && !strings.Contains(c.Body, "| a | 1 |") {
			t.Errorf("table header and data rows split across chunks: %q", c.Body)
		}
	}
}

func TestChunkMarkdownLongDocumentSplits(t *testing.T) {
	// Build a document with many distinct paragraphs, each above the
	// overlap floor. The chunker must split across paragraphs (not within
	// one) once accumulated size exceeds the target.
	var b strings.Builder
	b.WriteString("# Long\n\n")
	for i := 0; i < 60; i++ {
		b.WriteString("Paragraph ")
		b.WriteString(strings.Repeat("迁移 ", 20))
		b.WriteString("\n\n")
	}
	chunks := chunkBody(b.String())
	if len(chunks) < 2 {
		t.Fatalf("expected long document to split into multiple chunks, got %d", len(chunks))
	}
	// Splittable paragraphs must respect the hard 900-token ceiling. Atomic
	// blocks (fenced code, tables, lists) are exempt because they cannot be
	// split without destroying the source construct; this corpus has none,
	// so every chunk must be within budget.
	for i, c := range chunks {
		if got := estimateTokens(c.Body); got > chunkMaxTokens {
			t.Errorf("chunk[%d] exceeds %d-token ceiling: %d tokens (body preview: %q)",
				i, chunkMaxTokens, got, previewBody(c.Body))
		}
	}
}

func TestChunkMarkdownAtomicBlockMayExceedCeiling(t *testing.T) {
	// A single fenced code block that exceeds chunkMaxTokens must be emitted
	// as one chunk that preserves the full block; the chunker must not split
	// it mid-construct. The chunk is allowed to exceed the 900-token ceiling
	// because splitting a fenced block destroys the source syntax.
	var b strings.Builder
	b.WriteString("# Code\n\n```go\n")
	for i := 0; i < 400; i++ {
		b.WriteString("    println(\"line\", ")
		b.WriteString(strconv.Itoa(i))
		b.WriteString(")\n")
	}
	b.WriteString("```\n")
	chunks := chunkBody(b.String())
	var codeChunk *SearchChunk
	for i := range chunks {
		if strings.Contains(chunks[i].Body, "```go") {
			codeChunk = &chunks[i]
			break
		}
	}
	if codeChunk == nil {
		t.Fatalf("atomic fenced-code chunk not emitted")
	}
	if got := estimateTokens(codeChunk.Body); got <= chunkMaxTokens {
		t.Errorf("atomic code chunk below ceiling (expected exempt): %d tokens", got)
	}
	if !strings.HasPrefix(strings.TrimSpace(codeChunk.Body), "```go") {
		t.Errorf("atomic code chunk lost opening fence: %q", previewBody(codeChunk.Body))
	}
	if !strings.HasSuffix(strings.TrimSpace(codeChunk.Body), "```") {
		t.Errorf("atomic code chunk lost closing fence: %q", previewBody(codeChunk.Body))
	}
}

func TestChunkSplittableHardCeilingCJKContiguous(t *testing.T) {
	// 901 contiguous CJK runes have no internal whitespace, so the word
	// splitter cannot break them. The rune fallback must kick in so every
	// emitted piece is at most chunkMaxTokens.
	body := "# Audit\n\n" + strings.Repeat("迁", 901) + "\n"
	chunks := chunkBody(body)
	if len(chunks) < 2 {
		t.Fatalf("expected 901 CJK runes to split, got %d chunk(s)", len(chunks))
	}
	for i, c := range chunks {
		if got := estimateTokens(c.Body); got > chunkMaxTokens {
			t.Errorf("chunk[%d] exceeds %d-token ceiling: %d tokens (body preview: %q)",
				i, chunkMaxTokens, got, previewBody(c.Body))
		}
	}
}

func TestChunkSplittableHardCeilingOverlongASCIIToken(t *testing.T) {
	// A single ASCII alphanumeric token counts as 1 token under the
	// estimateTokens heuristic, so a 901-char run is not actually over
	// budget. The chunker should emit it as one chunk and the ceiling
	// invariant still holds. The point of this test is to document that
	// the heuristic deliberately treats ASCII words as atomic and that the
	// rune fallback only fires when estimateTokens actually crosses the
	// ceiling (e.g. contiguous CJK).
	body := "# Audit\n\n" + strings.Repeat("a", 901) + "\n"
	chunks := chunkBody(body)
	if len(chunks) != 1 {
		t.Fatalf("expected single chunk for 1-token ASCII run, got %d", len(chunks))
	}
	for i, c := range chunks {
		if got := estimateTokens(c.Body); got > chunkMaxTokens {
			t.Errorf("chunk[%d] exceeds %d-token ceiling: %d tokens",
				i, chunkMaxTokens, got)
		}
	}
}

func TestChunkSplittableHardCeilingOverlongASCIIPhrase(t *testing.T) {
	// 1000 whitespace-separated ASCII words really do produce 1000 tokens
	// and must be split. Each piece must respect the ceiling.
	var b strings.Builder
	b.WriteString("# Audit\n\n")
	for i := 0; i < 1000; i++ {
		b.WriteString("word")
		b.WriteString(strconv.Itoa(i + 1))
		b.WriteString(" ")
	}
	b.WriteString("\n")
	chunks := chunkBody(b.String())
	if len(chunks) < 2 {
		t.Fatalf("expected 1000-word ASCII phrase to split, got %d chunks", len(chunks))
	}
	for i, c := range chunks {
		if got := estimateTokens(c.Body); got > chunkMaxTokens {
			t.Errorf("chunk[%d] exceeds %d-token ceiling: %d tokens",
				i, chunkMaxTokens, got)
		}
	}
}

func TestChunkSplittableLineBoundsCoverOriginalLines(t *testing.T) {
	// A 1000-line paragraph of `word\n` lines. The chunker must split into
	// multiple chunks, and each chunk's line range must cover exactly the
	// original lines whose text appears in the chunk body. Previously the
	// splitter computed ranges from post-split newline counts and produced
	// nonsense like 3-3 / 4-4 while the actual content spanned lines 3-902.
	var b strings.Builder
	b.WriteString("# Long\n\n")
	for i := 0; i < 1000; i++ {
		b.WriteString("word")
		b.WriteString(strconv.Itoa(i + 1))
		b.WriteString("\n")
	}
	chunks := chunkBody(b.String())
	if len(chunks) < 2 {
		t.Fatalf("expected 1000-line paragraph to split, got %d chunks", len(chunks))
	}
	// Reconstruct expected 1-based line numbers in the source document.
	// Heading "# Long" is line 1; blank is line 2; word1..word1000 occupy
	// lines 3..1002.
	const wordBaseLine = 3
	for _, c := range chunks {
		if c.StartLine < 1 || c.EndLine < c.StartLine {
			t.Errorf("chunk has invalid line range: start=%d end=%d", c.StartLine, c.EndLine)
			continue
		}
		// Recover which wordN lines this chunk claims to cover by reading
		// the chunk body line-by-line (NOT via strings.Contains, which
		// would match word1 inside word1001).
		lowest, highest := -1, -1
		for bodyLineIdx, bodyLine := range strings.Split(strings.TrimRight(c.Body, "\n"), "\n") {
			lineNum := c.StartLine + bodyLineIdx
			// Verify the line really exists in the source document.
			var sourceIdx int
			if _, err := fmt.Sscanf(bodyLine, "word%d", &sourceIdx); err != nil {
				// Skip lines that aren't pure `wordN` (e.g., fence markers or
				// heading residue). The 1000-line corpus never triggers this.
				continue
			}
			expectedLine := wordBaseLine + sourceIdx - 1
			if lineNum != expectedLine {
				t.Errorf("chunk body line %q landed at chunk line %d but source places it at %d",
					bodyLine, lineNum, expectedLine)
				continue
			}
			if lowest < 0 || lineNum < lowest {
				lowest = lineNum
			}
			if lineNum > highest {
				highest = lineNum
			}
		}
		if lowest < 0 {
			continue
		}
		// Tighter bound: the chunk's declared range should hug the carried
		// word range within +/- 1 line (allowing for trailing newline of
		// the last word). Anything wider means line bounds drifted.
		if c.StartLine < lowest-1 || c.StartLine > lowest {
			t.Errorf("chunk StartLine %d drifted from word range start %d (body preview: %q)",
				c.StartLine, lowest, previewBody(c.Body))
		}
		if c.EndLine > highest+1 || c.EndLine < highest {
			t.Errorf("chunk EndLine %d drifted from word range end %d (body preview: %q)",
				c.EndLine, highest, previewBody(c.Body))
		}
	}
}

func previewBody(body string) string {
	const max = 80
	if len(body) <= max {
		return body
	}
	return body[:max] + "..."
}

func TestChunkOverlapDoesNotPushNextChunkOverCeiling(t *testing.T) {
	// Build a paragraph large enough to split into >= 2 chunks, then a
	// small paragraph, then a near-ceiling paragraph. Without the
	// overlap-budget check, the carried overlap (80-120 tokens) plus the
	// near-ceiling block (e.g. 850 tokens) would land in the same chunk
	// and exceed the 900-token ceiling.
	var big strings.Builder
	big.WriteString("# Audit\n\n")
	// First paragraph: 1500 tokens forces a split and primes the overlap.
	for i := 0; i < 1500; i++ {
		big.WriteString("word")
		big.WriteString(strconv.Itoa(i))
		big.WriteString(" ")
	}
	big.WriteString("\n\n")
	// Small paragraph that the chunker will combine with whatever it can.
	big.WriteString("Small bridge sentence.\n\n")
	// Near-ceiling paragraph: 850 tokens. With overlap carried (120), the
	// naive sum is 970 — over the ceiling. The fix drops the overlap.
	for i := 0; i < 850; i++ {
		big.WriteString("term")
		big.WriteString(strconv.Itoa(i))
		big.WriteString(" ")
	}
	big.WriteString("\n")
	chunks := chunkBody(big.String())
	if len(chunks) < 2 {
		t.Fatalf("expected multiple chunks, got %d", len(chunks))
	}
	for i, c := range chunks {
		if got := estimateTokens(c.Body); got > chunkMaxTokens {
			t.Errorf("chunk[%d] exceeds %d-token ceiling (overlap pushed it over): %d tokens (body preview: %q)",
				i, chunkMaxTokens, got, previewBody(c.Body))
		}
	}
}

func TestChunkHTMLBlockIsSplittable(t *testing.T) {
	// A raw HTML block with many lines must split like a paragraph; the
	// default case in the markdown dispatcher routes through the same
	// splitter instead of treating the whole HTML block as atomic.
	var b strings.Builder
	b.WriteString("# Notes\n\n")
	b.WriteString("<div>\n")
	for i := 0; i < 1500; i++ {
		b.WriteString("<p>")
		b.WriteString(strconv.Itoa(i))
		b.WriteString("</p>\n")
	}
	b.WriteString("</div>\n")
	chunks := chunkBody(b.String())
	if len(chunks) < 2 {
		t.Fatalf("expected HTML block to split, got %d chunk(s)", len(chunks))
	}
	for i, c := range chunks {
		if got := estimateTokens(c.Body); got > chunkMaxTokens {
			t.Errorf("chunk[%d] (HTML block) exceeds %d-token ceiling: %d tokens",
				i, chunkMaxTokens, got)
		}
	}
}

func TestChunkLineBoundsAreMonotonic(t *testing.T) {
	body := "# A\n\ntext one\n\n# B\n\ntext two\n\n# C\n\ntext three\n"
	chunks := chunkBody(body)
	for i, c := range chunks {
		if c.StartLine < 1 {
			t.Errorf("chunk[%d] StartLine %d < 1", i, c.StartLine)
		}
		if c.EndLine < c.StartLine {
			t.Errorf("chunk[%d] EndLine %d < StartLine %d", i, c.EndLine, c.StartLine)
		}
		if i > 0 && chunks[i-1].EndLine > c.StartLine {
			t.Errorf("chunk[%d] overlaps chunk[%d] line range", i, i-1)
		}
	}
}

func TestChunkStableID(t *testing.T) {
	body := "# A\n\nhello\n"
	first := chunkBody(body)
	second := chunkBody(body)
	if len(first) != len(second) {
		t.Fatalf("chunk count changed across runs: %d vs %d", len(first), len(second))
	}
	for i := range first {
		if first[i].ID != second[i].ID {
			t.Errorf("chunk[%d] ID not stable: %q vs %q", i, first[i].ID, second[i].ID)
		}
		if first[i].Ordinal != second[i].Ordinal {
			t.Errorf("chunk[%d] Ordinal not stable", i)
		}
	}
}

func TestChunkOrdinalIncrementsPerFile(t *testing.T) {
	body := "# A\n\ntext\n\n# B\n\ntext\n\n# C\n\ntext\n"
	chunks := chunkBody(body)
	for i, c := range chunks {
		if c.Ordinal != i+1 {
			t.Errorf("chunk[%d] Ordinal = %d, want %d", i, c.Ordinal, i+1)
		}
	}
}

func TestChunkIDChangesOnHeadingChange(t *testing.T) {
	base := "# Alpha\n\nbody\n"
	moved := "# Beta\n\nbody\n"
	a := chunkBody(base)
	b := chunkBody(moved)
	if len(a) != 1 || len(b) != 1 {
		t.Fatalf("expected 1 chunk each, got %d/%d", len(a), len(b))
	}
	if a[0].ID == b[0].ID {
		t.Errorf("chunk ID did not change when heading changed")
	}
}

// TestStableChunkIDKnownVector pins the stableChunkID contract to a frozen
// SHA256 vector. The contract is: sha256(path + NUL + headingAnchor + NUL +
// decimal(ordinal)). Changing the algorithm, the field order, or the ordinal
// encoding breaks every chunk ID and forces a clean derivative rebuild —
// this test forces an explicit acknowledgement of that breakage.
func TestStableChunkIDKnownVector(t *testing.T) {
	cases := []struct {
		name          string
		path          string
		headingAnchor string
		ordinal       int
		wantHex       string
	}{
		{
			name:          "root markdown first chunk",
			path:          "test.md",
			headingAnchor: "title",
			ordinal:       1,
			wantHex:       "f514b4aef1b5b3148d054a3f919d85c7a8aad7778e915965a69ab2901e791e65",
		},
		{
			name:          "root markdown second chunk",
			path:          "test.md",
			headingAnchor: "title",
			ordinal:       2,
			wantHex:       "e06f79f67e2c3690abc17cd12386e7f42444bd3e485dd9062373e46fcb925c30",
		},
		{
			name:          "concept path with multi-segment anchor",
			path:          "knowledge/concepts/migration.md",
			headingAnchor: "migration-audit",
			ordinal:       3,
			wantHex:       "daee4390cbd3a64e7db4564981ab152c92fb72f752c84cae4dec868a17567e84",
		},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := stableChunkID(c.path, c.headingAnchor, c.ordinal); got != c.wantHex {
				t.Errorf("stableChunkID(%q, %q, %d)\n  got  %s\n  want %s",
					c.path, c.headingAnchor, c.ordinal, got, c.wantHex)
			}
			// Cross-check against an independent reimplementation of the
			// frozen contract so a future edit to stableChunkID cannot
			// silently drift past both the vector and this mirror.
			h := sha256.New()
			h.Write([]byte(c.path))
			h.Write([]byte{0})
			h.Write([]byte(c.headingAnchor))
			h.Write([]byte{0})
			h.Write([]byte(strconv.Itoa(c.ordinal)))
			independent := hexEncodeToString(h.Sum(nil))
			if independent != c.wantHex {
				t.Errorf("independent reimplementation disagrees\n  got  %s\n  want %s",
					independent, c.wantHex)
			}
		})
	}
}

func hexEncodeToString(b []byte) string {
	const hexDigits = "0123456789abcdef"
	out := make([]byte, len(b)*2)
	for i, v := range b {
		out[i*2] = hexDigits[v>>4]
		out[i*2+1] = hexDigits[v&0x0f]
	}
	return string(out)
}

func TestChunkTextFileParagraphs(t *testing.T) {
	body := "first paragraph line one\nfirst paragraph line two\n\nsecond paragraph\n"
	chunks := Chunk(ChunkInput{
		Path:           "notes.txt",
		Kind:           SourceKindOther,
		Format:         SourceFormatText,
		ContentHash:    "h",
		SourceRevision: "r",
		Data:           []byte(body),
	})
	if len(chunks) == 0 {
		t.Fatalf("no chunks emitted")
	}
	combined := strings.Join(chunkBodies(chunks), "\n")
	if !strings.Contains(combined, "first paragraph line one") {
		t.Errorf("first paragraph lost: %q", combined)
	}
	if !strings.Contains(combined, "second paragraph") {
		t.Errorf("second paragraph lost: %q", combined)
	}
	if chunks[0].StartLine != 1 {
		t.Errorf("chunk[0] StartLine = %d, want 1", chunks[0].StartLine)
	}
}

func TestChunkPropagatesKindAndFormat(t *testing.T) {
	body := "# Heading\n\nbody.\n"
	chunks := Chunk(ChunkInput{
		Path:           "knowledge/concepts/migration.md",
		Kind:           SourceKindConcept,
		Format:         SourceFormatMarkdown,
		Archived:       true,
		ContentHash:    "h",
		SourceRevision: "r",
		Data:           []byte(body),
	})
	if len(chunks) != 1 {
		t.Fatalf("expected 1 chunk, got %d", len(chunks))
	}
	if chunks[0].Kind != SourceKindConcept {
		t.Errorf("Kind = %q, want %q", chunks[0].Kind, SourceKindConcept)
	}
	if chunks[0].Format != SourceFormatMarkdown {
		t.Errorf("Format = %q, want %q", chunks[0].Format, SourceFormatMarkdown)
	}
	if !chunks[0].Archived {
		t.Errorf("Archived lost")
	}
	if chunks[0].ContentHash != "h" {
		t.Errorf("ContentHash lost: %q", chunks[0].ContentHash)
	}
	if chunks[0].SourceRevision != "r" {
		t.Errorf("SourceRevision lost: %q", chunks[0].SourceRevision)
	}
}

func chunkBodies(cs []SearchChunk) []string {
	out := make([]string, len(cs))
	for i, c := range cs {
		out[i] = c.Body
	}
	return out
}

func TestChunkIdentifierTextExtracted(t *testing.T) {
	body := strings.Join([]string{
		"# Project",
		"",
		"Track PR #2731 and task #t1835. The fix landed in commit abc1234.",
		"See https://example.com/docs/pr-2731 for context. Ping @alice for review.",
		"Implementation lives in internal/memorysearch/source.go.",
		"The snake_case_func and camelCaseFunc helpers are exported.",
		"",
	}, "\n")
	chunks := chunkBody(body)
	if len(chunks) == 0 {
		t.Fatalf("no chunks")
	}
	ids := chunks[0].Identifiers
	// The frozen contract: URLs and @handles are kept as their full text
	// (scheme+host+path, leading '@'). Bare host names and bare handles
	// without their sigil are NOT first-class identifiers.
	expect := []string{
		"#2731", "#t1835", "abc1234",
		"https://example.com/docs/pr-2731",
		"@alice",
		"internal/memorysearch/source.go",
		"snake_case_func", "camelCaseFunc",
	}
	for _, want := range expect {
		if !contains(ids, want) {
			t.Errorf("identifier %q missing from %v", want, ids)
		}
	}
	for _, reject := range []string{"example.com", "alice"} {
		if contains(ids, reject) {
			t.Errorf("legacy bare text %q leaked as identifier: %v", reject, ids)
		}
	}
}

func TestExtractIdentifiersFullURLText(t *testing.T) {
	// Full URL text must be preserved (scheme+host+path, plus
	// optional query and fragment) with trailing sentence punctuation
	// stripped. Each URL is one identifier; the masking layer keeps its
	// internal slashes out of the path extractor.
	text := strings.Join([]string{
		"Open https://example.com/docs/pr-2731 for the spec.",
		"Query param case: https://api.example.com/v1/search?q=memory&lang=en.",
		"Fragment case: https://example.com/agent/#toc-routing.",
		"Port case: http://localhost:8080/healthz.",
	}, "\n")
	ids := ExtractIdentifiers(text)
	for _, want := range []string{
		"https://example.com/docs/pr-2731",
		"https://api.example.com/v1/search?q=memory&lang=en",
		"https://example.com/agent/#toc-routing",
		"http://localhost:8080/healthz",
	} {
		if !contains(ids, want) {
			t.Errorf("full URL text %q missing from %v", want, ids)
		}
	}
	// The path inside a URL must NOT be re-extracted as a standalone path
	// identifier, because URL spans are masked before the path regex runs.
	for _, leaked := range []string{"docs/pr-2731", "v1/search", "agent/#toc-routing"} {
		if contains(ids, leaked) {
			t.Errorf("URL-internal path leaked as standalone identifier: %q (ids=%v)", leaked, ids)
		}
	}
}

func TestExtractIdentifiersHandleTextWithSigil(t *testing.T) {
	ids := ExtractIdentifiers("Ping @alice and @bob_smith about @CamelCase.")
	for _, want := range []string{"@alice", "@bob_smith", "@CamelCase"} {
		if !contains(ids, want) {
			t.Errorf("handle text %q missing from %v", want, ids)
		}
	}
	// The body of a handle must NOT leak as a standalone bare identifier:
	// handles are masked before snake_case / path extraction runs.
	for _, reject := range []string{"alice", "bob_smith", "CamelCase"} {
		if contains(ids, reject) {
			t.Errorf("bare handle without sigil leaked: %q (ids=%v)", reject, ids)
		}
	}
}

func TestExtractIdentifierPartsDeriveFromURLAndHandle(t *testing.T) {
	parts := ExtractIdentifierParts("see https://example.com/docs/pr-2731 and ping @alice_exp")
	// URL parts: scheme-side segments split on punctuation.
	for _, want := range []string{"https", "example", "com", "docs", "pr", "2731"} {
		if !contains(parts, want) {
			t.Errorf("url-derived part %q missing from %v", want, parts)
		}
	}
	// Handle parts: sigil stripped, snake_case split.
	for _, want := range []string{"alice", "exp"} {
		if !contains(parts, want) {
			t.Errorf("handle-derived part %q missing from %v", want, parts)
		}
	}
}

func TestChunkIdentifierPartsIncludeNumericAndStructural(t *testing.T) {
	body := "PR #2731 lives at internal/memorysearch/source.go.\n"
	chunks := chunkBody(body)
	if len(chunks) == 0 {
		t.Fatalf("no chunks")
	}
	parts := chunks[0].IdentifierParts
	expect := []string{"2731", "internal", "memorysearch", "source", "go"}
	for _, want := range expect {
		if !contains(parts, want) {
			t.Errorf("identifier part %q missing from %v", want, parts)
		}
	}
}

func TestExtractIdentifiersDedup(t *testing.T) {
	ids := ExtractIdentifiers("See #2731 and #2731 again, plus #2731.")
	count := 0
	for _, id := range ids {
		if id == "#2731" {
			count++
		}
	}
	if count != 1 {
		t.Errorf("expected #2731 once, got %d (ids=%v)", count, ids)
	}
}

func TestExtractIdentifiersRejectsBareNumbers(t *testing.T) {
	ids := ExtractIdentifiers("the answer is 42 and 12345")
	for _, id := range ids {
		if id == "42" || id == "12345" {
			t.Errorf("bare number leaked as identifier: %q", id)
		}
	}
}

func TestExtractIdentifierPartsSnakeCaseSplits(t *testing.T) {
	parts := ExtractIdentifierParts("see foo_bar_baz helper")
	for _, want := range []string{"foo", "bar", "baz"} {
		if !contains(parts, want) {
			t.Errorf("snake_case part %q missing from %v", want, parts)
		}
	}
}

func TestExtractIdentifierPartsCamelCaseSplits(t *testing.T) {
	parts := ExtractIdentifierParts("see fooBarBaz helper")
	for _, want := range []string{"foo", "bar", "baz"} {
		if !contains(parts, want) {
			t.Errorf("camelCase part %q missing from %v", want, parts)
		}
	}
}

func TestExtractIdentifiersIgnoresChineseOnlyText(t *testing.T) {
	ids := ExtractIdentifiers("迁移 重号 审计")
	if len(ids) != 0 {
		t.Errorf("expected no identifiers from CJK-only text, got %v", ids)
	}
}

func TestExtractIdentifiersMasksURLBeforeSHAAndHandle(t *testing.T) {
	// A URL's interior must not leak SHA-like hex runs or @handle identifiers.
	// Each input has exactly one identifier: the URL itself.
	cases := []struct {
		name string
		text string
		want string
	}{
		{
			name: "sha-like hex inside URL path",
			text: "see https://example.com/abcdef1/docs for context",
			want: "https://example.com/abcdef1/docs",
		},
		{
			name: "handle inside URL query",
			text: "review https://example.com/page?owner=@bob_smith",
			want: "https://example.com/page?owner=@bob_smith",
		},
		{
			name: "sha inside URL query",
			text: "log https://example.com/commits?rev=abcdef1234567",
			want: "https://example.com/commits?rev=abcdef1234567",
		},
		{
			name: "numbered ref inside URL fragment",
			text: "track https://example.com/ticket#2731",
			want: "https://example.com/ticket#2731",
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			ids := ExtractIdentifiers(tc.text)
			// The URL must always be present.
			if !contains(ids, tc.want) {
				t.Errorf("URL %q missing from %v", tc.want, ids)
			}
			// Anything else extracted from inside the URL is a leak.
			for _, id := range ids {
				if id == tc.want {
					continue
				}
				t.Errorf("URL-internal text leaked as separate identifier: %q (ids=%v)", id, ids)
			}
		})
	}
}

func TestExtractIdentifiersPreservesFirstOccurrenceOrder(t *testing.T) {
	// The output order must follow the source text's byte order regardless
	// of which category produced each identifier. The previous batch-by-
	// category implementation produced [#123, foo_bar] for this input because
	// refs ran before snake_case; the contract is the opposite.
	cases := []struct {
		name string
		text string
		want []string
	}{
		{
			name: "snake before ref",
			text: "foo_bar then #123",
			want: []string{"foo_bar", "#123"},
		},
		{
			name: "handle before path before ref",
			text: "@zed then path/to/file.md then #42",
			want: []string{"@zed", "path/to/file.md", "#42"},
		},
		{
			name: "url before sha after",
			text: "open https://example.com/x and commit abcdef1",
			want: []string{"https://example.com/x", "abcdef1"},
		},
		{
			name: "camel before snake",
			text: "fooBar then baz_qux",
			want: []string{"fooBar", "baz_qux"},
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			ids := ExtractIdentifiers(tc.text)
			// Order-sensitive compare on the leading prefix. Extra tail items
			// are allowed (e.g. `baz_qux` also extracts `foo` / `bar` parts
			// via the parts path, but ExtractIdentifiers itself does not split
			// so only the camel/snake bodies appear here).
			if len(ids) < len(tc.want) {
				t.Fatalf("expected at least %d identifiers, got %d (%v)", len(tc.want), len(ids), ids)
			}
			for i, w := range tc.want {
				if ids[i] != w {
					t.Errorf("position %d: got %q, want %q (full ids=%v)", i, ids[i], w, ids)
					break
				}
			}
		})
	}
}

func TestEstimateTokensHeuristic(t *testing.T) {
	cases := []struct {
		text string
		want int
	}{
		{"", 0},
		{"hello", 1},
		{"hello world", 2},
		{"hello_world", 1}, // underscore does not split words in the heuristic
		{"迁移", 2},          // 2 CJK runes = 2 tokens (1 per non-ASCII rune)
		{"迁移 重号", 4},       // 4 CJK runes = 4 tokens
		{"a 1", 2},         // 2 alphanumeric words
	}
	for _, c := range cases {
		if got := estimateTokens(c.text); got != c.want {
			t.Errorf("estimateTokens(%q) = %d, want %d", c.text, got, c.want)
		}
	}
}

// FuzzChunk exercises the chunker against random Markdown-ish input to catch
// panics and invariant violations without asserting on Goldmark-private node
// ordering.
func FuzzChunk(f *testing.F) {
	f.Add("# Hello\n\nworld\n")
	f.Add("中文标题\n\nPR #2731 review\n")
	f.Add("```\ncode block\n```\n")
	f.Add("| a | b |\n| - | - |\n| 1 | 2 |\n")
	f.Fuzz(func(t *testing.T, in string) {
		defer func() {
			if r := recover(); r != nil {
				t.Fatalf("chunker panicked: %v", r)
			}
		}()
		chunks := Chunk(ChunkInput{
			Path:           "fuzz.md",
			Kind:           SourceKindRoot,
			Format:         SourceFormatMarkdown,
			ContentHash:    "h",
			SourceRevision: "r",
			Data:           []byte(in),
		})
		seenIDs := map[string]struct{}{}
		for _, c := range chunks {
			if c.ID == "" {
				t.Fatalf("chunk with empty ID")
			}
			if _, dup := seenIDs[c.ID]; dup {
				t.Fatalf("duplicate chunk ID %q", c.ID)
			}
			seenIDs[c.ID] = struct{}{}
			if c.StartLine < 0 || c.EndLine < 0 {
				t.Fatalf("negative line bound: start=%d end=%d", c.StartLine, c.EndLine)
			}
			if c.EndLine < c.StartLine {
				t.Fatalf("end < start: %d < %d", c.EndLine, c.StartLine)
			}
			if c.Path != "fuzz.md" {
				t.Fatalf("chunk path mutated: %q", c.Path)
			}
		}
	})
}