// Copyright 2026 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::VecDeque;

use anyhow::{Result, bail};
use kvenginepb::fts::NgramOption;
use tantivy::tokenizer::{
    LowerCaser, RawTokenizer, SimpleTokenizer, TextAnalyzer, Token, TokenFilter, TokenStream,
    Tokenizer,
};

// NGRAM_V1 tokenizer design
//
// `NgramV1Spec` turns protobuf `NgramOption` into a Tantivy `TextAnalyzer`. The
// analyzer always has the shape
// `inner tokenizer [-> LowerCaser] -> NgramTokenizer`, where the inner
// tokenizer decides how input text is segmented before optional lowercasing and
// n-gram expansion:
//
// - WORD: `SimpleTokenizer` splits on non-alphanumeric characters. Each
//   alphanumeric run is expanded independently; grams never include or cross
//   separators.
// - CHAR: `RawTokenizer` keeps the whole field as one token at position 0.
//   Grams may include whitespace and punctuation and can span semantic word
//   boundaries.
//
// `NgramTokenizer` is a tokenizer wrapper, not a Tantivy `TokenFilter`, because
// it expands one inner token into many overlapping tokens with independent
// positions.
//
// For each inner token, `NgramSpanState` walks Unicode scalar boundaries (not
// UTF-16 code units or grapheme clusters) and emits overlapping grams from
// `min_gram` through `max_gram` for each start position before sliding the
// window forward. Positions are derived from a character-base cursor plus
// `gram_char_start`, so all grams that share the same start character also
// share the same position. The cursor reserves the full character length of
// every inner token plus one separator position, including for tokens too short
// to emit a gram. Inner positions act as a lower bound, preserving explicit
// larger position gaps. Every gram retains its parent token's original offsets.
//
// Optional lowercasing is applied once per inner token before expansion, so
// gram boundaries and positions are based on normalized Unicode scalars. Since
// Unicode lowercasing can change scalar counts and UTF-8 byte boundaries, gram
// offsets are not derived from the normalized byte ranges.

const MIN_GRAM_LIMIT: u64 = 2;
const MAX_GRAM_LIMIT: u64 = 5;

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum NgramV1Granularity {
    Word,
    Char,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct GramRange {
    min: usize,
    max: usize,
}

impl GramRange {
    fn new(min: usize, max: usize) -> Self {
        Self { min, max }
    }

    fn longest_query_range(self, input_char_count: usize) -> Self {
        let gram = input_char_count.clamp(self.min, self.max);
        Self::new(gram, gram)
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum NgramAnalyzerKind {
    Index,
    Query,
}

impl NgramAnalyzerKind {
    fn active_gram_range(
        self,
        configured_gram_range: GramRange,
        input_char_count: usize,
    ) -> GramRange {
        match self {
            NgramAnalyzerKind::Index => configured_gram_range,
            NgramAnalyzerKind::Query => configured_gram_range.longest_query_range(input_char_count),
        }
    }
}

/// Parsed NGRAM_V1 analyzer configuration from protobuf `NgramOption`.
///
/// Use [`Self::index_analyzer`] to build the full Tantivy pipeline. WORD and
/// CHAR modes differ only in the inner tokenizer passed to [`NgramTokenizer`];
/// expansion logic is shared.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct NgramV1Spec {
    gram_range: GramRange,
    granularity: NgramV1Granularity,
    lower_case: bool,
}

impl NgramV1Spec {
    /// Parses and validates an NGRAM_V1 analyzer specification from protobuf
    /// metadata.
    ///
    /// # Errors
    ///
    /// Returns an error unless `2 <= min_gram <= max_gram <= 5` and
    /// `granularity` is `WORD` or `CHAR` (case-insensitive).
    pub fn from_option(option: &NgramOption) -> Result<Self> {
        let min_gram = option.get_min_gram();
        let max_gram = option.get_max_gram();
        if min_gram < MIN_GRAM_LIMIT {
            bail!("min_gram {min_gram} must be at least {MIN_GRAM_LIMIT}");
        }
        if min_gram > max_gram {
            bail!("min_gram {min_gram} must be less than or equal to max_gram {max_gram}");
        }
        if max_gram > MAX_GRAM_LIMIT {
            bail!("max_gram {max_gram} must be at most {MAX_GRAM_LIMIT}");
        }

        let granularity = option.get_granularity();
        let granularity = if granularity.eq_ignore_ascii_case("WORD") {
            NgramV1Granularity::Word
        } else if granularity.eq_ignore_ascii_case("CHAR") {
            NgramV1Granularity::Char
        } else {
            bail!("granularity {granularity:?} must be WORD or CHAR");
        };

        Ok(Self {
            gram_range: GramRange::new(usize::try_from(min_gram)?, usize::try_from(max_gram)?),
            granularity,
            lower_case: option.get_lower_case(),
        })
    }

    /// Builds the Tantivy analyzer described by this specification.
    ///
    /// `WORD` mode splits on non-alphanumeric characters before expanding each
    /// token independently. `CHAR` mode expands the entire input, including
    /// separators. When enabled, lowercasing is applied to each inner token
    /// before Ngram expansion.
    pub fn index_analyzer(self) -> TextAnalyzer {
        match self.granularity {
            NgramV1Granularity::Word => {
                self.analyzer_with(SimpleTokenizer::default(), NgramAnalyzerKind::Index)
            }
            NgramV1Granularity::Char => {
                self.analyzer_with(RawTokenizer::default(), NgramAnalyzerKind::Index)
            }
        }
    }

    pub fn query_analyzer(self) -> TextAnalyzer {
        match self.granularity {
            NgramV1Granularity::Word => {
                self.analyzer_with(SimpleTokenizer::default(), NgramAnalyzerKind::Query)
            }
            NgramV1Granularity::Char => {
                self.analyzer_with(RawTokenizer::default(), NgramAnalyzerKind::Query)
            }
        }
    }

    fn analyzer_with<T: Tokenizer>(
        self,
        inner: T,
        analyzer_kind: NgramAnalyzerKind,
    ) -> TextAnalyzer {
        if self.lower_case {
            TextAnalyzer::from(NgramTokenizer::new(
                LowerCaser.transform(inner),
                self.gram_range,
                analyzer_kind,
            ))
        } else {
            TextAnalyzer::from(NgramTokenizer::new(inner, self.gram_range, analyzer_kind))
        }
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct NgramSpan {
    byte_start: usize,
    byte_end: usize,
    char_start: usize,
}

/// Sliding-window state for one inner token's character n-grams.
///
/// `boundaries` stores UTF-8 byte offsets of consecutive Unicode scalars,
/// always relative to the current inner token text. The deque never grows
/// beyond `max_gram + 1` entries.
///
/// Emission order for `"hello"` with `min_gram = 2`, `max_gram = 3`:
/// `he`, `hel`, `el`, `ell`, `ll`, `llo`, `lo`.
#[derive(Clone)]
struct NgramSpanState {
    configured_gram_range: GramRange,
    active_gram_range: GramRange,
    boundaries: VecDeque<usize>,
    char_start: usize,
    next_gram_len: usize,
    analyzer_kind: NgramAnalyzerKind,
}

impl NgramSpanState {
    fn new(configured_gram_range: GramRange, analyzer_kind: NgramAnalyzerKind) -> Self {
        let mut boundaries = VecDeque::with_capacity(configured_gram_range.max + 1);
        boundaries.push_back(0);
        Self {
            configured_gram_range,
            active_gram_range: configured_gram_range,
            boundaries,
            char_start: 0,
            next_gram_len: configured_gram_range.min,
            analyzer_kind,
        }
    }

    fn reset(&mut self, input_char_count: usize) {
        self.boundaries.clear();
        self.boundaries.push_back(0);
        self.char_start = 0;
        self.active_gram_range = self
            .analyzer_kind
            .active_gram_range(self.configured_gram_range, input_char_count);
        self.next_gram_len = self.active_gram_range.min;
    }

    fn fill_boundaries(&mut self, text: &str) {
        while self.boundaries.len() <= self.active_gram_range.max {
            let byte_start = *self.boundaries.back().unwrap();
            let Some(character) = text[byte_start..].chars().next() else {
                break;
            };
            self.boundaries.push_back(byte_start + character.len_utf8());
        }
    }

    fn next(&mut self, text: &str) -> Option<NgramSpan> {
        loop {
            self.fill_boundaries(text);
            let available_chars = self.boundaries.len() - 1;
            if self.next_gram_len <= self.active_gram_range.max
                && self.next_gram_len <= available_chars
            {
                let span = NgramSpan {
                    byte_start: self.boundaries[0],
                    byte_end: self.boundaries[self.next_gram_len],
                    char_start: self.char_start,
                };
                self.next_gram_len += 1;
                return Some(span);
            }

            if available_chars == 0 {
                return None;
            }
            self.boundaries.pop_front();
            self.char_start += 1;
            self.next_gram_len = self.active_gram_range.min;
        }
    }
}

/// Expands each token from `inner` into overlapping character n-grams.
///
/// `NgramTokenizer` sits at the core of the NGRAM_V1 analyzer. It delegates
/// segmentation to `inner` and only performs n-gram expansion inside each
/// emitted token:
///
/// ```text
/// field text -> inner tokenizer -> [word/raw token, ...]
///                                   |
///                                   v
///                         NgramSpanState per token
///                                   |
///                                   v
///                         overlapping min..=max grams
/// ```
///
/// Token metadata is derived from the inner token:
/// - `text`: slice `[byte_start..byte_end)` of the inner token text
/// - `offset_from` / `offset_to`: the parent token's original offsets
/// - `position`: character-base position plus the gram's zero-based char index
///   within the token
/// - `position_length`: always `1`
///
/// See the module-level design note above for WORD vs CHAR inner tokenizers and
/// why this type is implemented as a tokenizer wrapper instead of a
/// `TokenFilter`.
#[derive(Clone)]
struct NgramTokenizer<T> {
    inner: T,
    spans: NgramSpanState,
    token: Token,
}

impl<T: Tokenizer> NgramTokenizer<T> {
    fn new(inner: T, gram_range: GramRange, analyzer_kind: NgramAnalyzerKind) -> Self {
        debug_assert!(gram_range.min > 0);
        debug_assert!(gram_range.min <= gram_range.max);
        Self {
            inner,
            spans: NgramSpanState::new(gram_range, analyzer_kind),
            token: Token::default(),
        }
    }
}

/// Streams n-grams for the current inner token before advancing to the next
/// one.
///
/// `has_current_token` tracks whether `inner` already produced a token whose
/// grams are still being drained by `spans`. `current_position_base` is the
/// base position of that token. `next_position_base` prevents consecutive
/// inner tokens from producing overlapping or decreasing positions.
struct NgramTokenStream<'a, T> {
    inner: T,
    spans: &'a mut NgramSpanState,
    token: &'a mut Token,
    has_current_token: bool,
    current_position_base: usize,
    next_position_base: usize,
}

impl<T: Tokenizer> Tokenizer for NgramTokenizer<T> {
    type TokenStream<'a> = NgramTokenStream<'a, T::TokenStream<'a>>;

    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
        self.spans.reset(0);
        self.token.reset();
        NgramTokenStream {
            inner: self.inner.token_stream(text),
            spans: &mut self.spans,
            token: &mut self.token,
            has_current_token: false,
            current_position_base: 0,
            next_position_base: 0,
        }
    }
}

impl<T: TokenStream> TokenStream for NgramTokenStream<'_, T> {
    fn advance(&mut self) -> bool {
        loop {
            if self.has_current_token {
                let input = self.inner.token();
                if let Some(span) = self.spans.next(&input.text) {
                    self.token.offset_from = input.offset_from;
                    self.token.offset_to = input.offset_to;
                    self.token.position = self.current_position_base + span.char_start;
                    self.token.position_length = 1;
                    self.token.text.clear();
                    self.token
                        .text
                        .push_str(&input.text[span.byte_start..span.byte_end]);
                    return true;
                }
                self.has_current_token = false;
            }

            if !self.inner.advance() {
                return false;
            }
            let input = self.inner.token();
            let char_count = input.text.chars().count();
            // SimpleTokenizer positions tokens by word ordinal, which is too
            // small to follow the character positions emitted for prior words.
            self.current_position_base = self.next_position_base.max(input.position);
            self.next_position_base = self.current_position_base + char_count + 1;
            self.spans.reset(char_count);
            self.has_current_token = true;
        }
    }

    fn token(&self) -> &Token {
        self.token
    }

    fn token_mut(&mut self) -> &mut Token {
        self.token
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[derive(Clone)]
    struct FixedTokenizer {
        tokens: Vec<Token>,
        token: Token,
    }

    impl FixedTokenizer {
        fn new(tokens: Vec<Token>) -> Self {
            Self {
                tokens,
                token: Token::default(),
            }
        }
    }

    struct FixedTokenStream<'a> {
        tokens: &'a [Token],
        next_token: usize,
        token: &'a mut Token,
    }

    impl Tokenizer for FixedTokenizer {
        type TokenStream<'a> = FixedTokenStream<'a>;

        fn token_stream<'a>(&'a mut self, _text: &'a str) -> Self::TokenStream<'a> {
            self.token.reset();
            FixedTokenStream {
                tokens: &self.tokens,
                next_token: 0,
                token: &mut self.token,
            }
        }
    }

    impl TokenStream for FixedTokenStream<'_> {
        fn advance(&mut self) -> bool {
            let Some(token) = self.tokens.get(self.next_token) else {
                return false;
            };
            self.token.clone_from(token);
            self.next_token += 1;
            true
        }

        fn token(&self) -> &Token {
            self.token
        }

        fn token_mut(&mut self) -> &mut Token {
            self.token
        }
    }

    fn option(min_gram: u64, max_gram: u64, granularity: &str, lower_case: bool) -> NgramOption {
        let mut option = NgramOption::new();
        option.set_min_gram(min_gram);
        option.set_max_gram(max_gram);
        option.set_granularity(granularity.to_string());
        option.set_lower_case(lower_case);
        option
    }

    fn char_tokens(text: &str, spec: NgramV1Spec) -> Vec<Token> {
        assert_eq!(spec.granularity, NgramV1Granularity::Char);
        tokens(text, spec)
    }

    fn word_tokens(text: &str, spec: NgramV1Spec) -> Vec<Token> {
        assert_eq!(spec.granularity, NgramV1Granularity::Word);
        tokens(text, spec)
    }

    fn tokens(text: &str, spec: NgramV1Spec) -> Vec<Token> {
        let mut analyzer = spec.index_analyzer();
        let mut stream = analyzer.token_stream(text);
        let mut tokens = Vec::new();
        stream.process(&mut |token| tokens.push(token.clone()));
        tokens
    }

    fn query_tokens(text: &str, spec: NgramV1Spec) -> Vec<Token> {
        let mut analyzer = spec.query_analyzer();
        let mut stream = analyzer.token_stream(text);
        let mut tokens = Vec::new();
        stream.process(&mut |token| tokens.push(token.clone()));
        tokens
    }

    fn fixed_token(text: &str, offset_from: usize, position: usize) -> Token {
        Token {
            offset_from,
            offset_to: offset_from + text.len(),
            position,
            text: text.to_string(),
            position_length: 1,
        }
    }

    #[test]
    fn test_ngram_v1_spec_accepts_supported_range() {
        let spec = NgramV1Spec::from_option(&option(2, 5, "CHAR", true)).unwrap();
        assert_eq!(spec.gram_range.min, 2);
        assert_eq!(spec.gram_range.max, 5);
        assert_eq!(spec.granularity, NgramV1Granularity::Char);
        assert!(spec.lower_case);

        assert_eq!(
            NgramV1Spec::from_option(&option(3, 3, "WORD", false))
                .unwrap()
                .granularity,
            NgramV1Granularity::Word
        );
    }

    #[test]
    fn test_ngram_v1_spec_accepts_case_insensitive_granularity() {
        assert_eq!(
            NgramV1Spec::from_option(&option(3, 3, "word", false))
                .unwrap()
                .granularity,
            NgramV1Granularity::Word
        );
        assert_eq!(
            NgramV1Spec::from_option(&option(3, 3, "cHaR", false))
                .unwrap()
                .granularity,
            NgramV1Granularity::Char
        );
    }

    #[test]
    fn test_ngram_v1_spec_rejects_invalid_range_and_granularity() {
        assert!(NgramV1Spec::from_option(&option(1, 3, "CHAR", false)).is_err());
        assert!(NgramV1Spec::from_option(&option(4, 3, "CHAR", false)).is_err());
        assert!(NgramV1Spec::from_option(&option(2, 6, "CHAR", false)).is_err());
        assert!(NgramV1Spec::from_option(&option(2, 3, "TOKEN", false)).is_err());
    }

    #[test]
    fn test_char_query_analyzer_emits_only_longest_grams() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "CHAR", false)).unwrap();
        let tokens = query_tokens("hello", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("hel", 0), ("ell", 1), ("llo", 2)]
        );
    }

    #[test]
    fn test_char_query_analyzer_clamps_gram_to_normalized_input_length() {
        let spec = NgramV1Spec::from_option(&option(2, 4, "CHAR", true)).unwrap();
        let cases: &[(&str, &[&str])] = &[
            ("a", &[]),
            ("ab", &["ab"]),
            ("abc", &["abc"]),
            ("abcd", &["abcd"]),
            ("abcde", &["abcd", "bcde"]),
            ("é界a", &["é界a"]),
            ("İ", &["i\u{307}"]),
        ];

        for (text, expected) in cases {
            let tokens = query_tokens(text, spec);
            assert_eq!(
                tokens
                    .iter()
                    .map(|token| token.text.as_str())
                    .collect::<Vec<_>>(),
                *expected,
                "input: {text:?}"
            );
        }
    }

    #[test]
    fn test_word_query_analyzer_handles_each_word_independently() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "WORD", true)).unwrap();
        let tokens = query_tokens("Hello A BC 世界你好", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["hel", "ell", "llo", "bc", "世界你", "界你好"]
        );
    }

    #[test]
    fn test_ngram_wrapper_adds_positions_and_inherits_offsets() {
        let inner = FixedTokenizer::new(vec![
            fixed_token("hello", 10, 6),
            fixed_token("a", 20, 12),
            fixed_token("world", 22, 14),
        ]);
        let mut tokenizer =
            NgramTokenizer::new(inner, GramRange::new(3, 3), NgramAnalyzerKind::Index);
        let mut stream = tokenizer.token_stream("xxxxxxxxxxhello.....a.world");
        let mut tokens = Vec::new();
        stream.process(&mut |token| tokens.push(token.clone()));

        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [
                ("hel", 6),
                ("ell", 7),
                ("llo", 8),
                ("wor", 14),
                ("orl", 15),
                ("rld", 16),
            ]
        );
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.offset_from, token.offset_to))
                .collect::<Vec<_>>(),
            [(10, 15), (10, 15), (10, 15), (22, 27), (22, 27), (22, 27)]
        );
    }

    #[test]
    fn test_ngram_wrapper_empty_input_emits_no_tokens() {
        let mut tokenizer = NgramTokenizer::new(
            RawTokenizer::default(),
            GramRange::new(2, 3),
            NgramAnalyzerKind::Index,
        );
        assert!(!tokenizer.token_stream("").advance());
    }

    #[test]
    fn test_char_ngram_includes_separators_and_crosses_words() {
        let spec = NgramV1Spec::from_option(&option(3, 3, "CHAR", false)).unwrap();
        let tokens = char_tokens("ab cd", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["ab ", "b c", " cd"]
        );
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.position)
                .collect::<Vec<_>>(),
            [0, 1, 2]
        );
    }

    #[test]
    fn test_char_ngram_uses_unicode_scalar_boundaries() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "CHAR", false)).unwrap();
        let tokens = char_tokens("é界a", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["é界", "界a"]
        );
    }

    #[test]
    fn test_char_ngram_shares_position_for_same_start() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "CHAR", false)).unwrap();
        let tokens = char_tokens("hello", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [
                ("he", 0),
                ("hel", 0),
                ("el", 1),
                ("ell", 1),
                ("ll", 2),
                ("llo", 2),
                ("lo", 3),
            ]
        );
    }

    #[test]
    fn test_char_ngram_lowercases_before_slicing() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "CHAR", true)).unwrap();
        let tokens = char_tokens("Thé", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["th", "thé", "hé"]
        );
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.position)
                .collect::<Vec<_>>(),
            [0, 0, 1]
        );
    }

    #[test]
    fn test_char_ngram_lowercase_uses_normalized_scalar_boundaries() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "CHAR", true)).unwrap();

        // `İ` lowercases to two Unicode scalars and therefore forms one bigram.
        let tokens = char_tokens("İ", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("i\u{307}", 0)]
        );

        let tokens = char_tokens("İA", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("i\u{307}", 0), ("\u{307}a", 1)]
        );
    }

    #[test]
    fn test_word_ngram_does_not_cross_words() {
        let spec = NgramV1Spec::from_option(&option(3, 3, "WORD", false)).unwrap();
        let tokens = word_tokens("hello world", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [
                ("hel", 0),
                ("ell", 1),
                ("llo", 2),
                ("wor", 6),
                ("orl", 7),
                ("rld", 8),
            ]
        );
    }

    #[test]
    fn test_word_ngram_preserves_original_case_and_accents() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "WORD", false)).unwrap();
        let tokens = word_tokens("Thé", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("Th", 0), ("Thé", 0), ("hé", 1)]
        );
    }

    #[test]
    fn test_word_ngram_lowercase_does_not_remove_accents() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "WORD", true)).unwrap();
        let tokens = word_tokens("Thé", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["th", "thé", "hé"]
        );
    }

    #[test]
    fn test_word_ngram_tokenizes_before_lowercase() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "WORD", true)).unwrap();
        let tokens = word_tokens("İA", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("i\u{307}", 0), ("\u{307}a", 1)]
        );
    }

    #[test]
    fn test_word_ngram_treats_separators_as_one_boundary() {
        let spec = NgramV1Spec::from_option(&option(3, 3, "WORD", false)).unwrap();
        for text in [
            "hello world",
            "hello    world",
            "hello, world",
            "hello—world",
        ] {
            let tokens = word_tokens(text, spec);
            let world_start = tokens
                .iter()
                .find(|token| token.text == "wor")
                .map(|token| token.position);
            assert_eq!(world_start, Some(6), "input: {text:?}");
        }
    }

    #[test]
    fn test_word_ngram_short_word_still_advances_next_word_position() {
        let spec = NgramV1Spec::from_option(&option(3, 3, "WORD", false)).unwrap();
        let tokens = word_tokens("a hello", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("hel", 2), ("ell", 3), ("llo", 4)]
        );
    }

    #[test]
    fn test_word_ngram_positions_do_not_overlap_between_words() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "WORD", false)).unwrap();
        let tokens = word_tokens("abcde dexxx", spec);
        assert!(
            tokens
                .windows(2)
                .all(|pair| pair[0].position <= pair[1].position)
        );
        assert_eq!(
            tokens
                .iter()
                .filter(|token| token.text == "de")
                .map(|token| token.position)
                .collect::<Vec<_>>(),
            [3, 6]
        );
    }

    #[test]
    fn test_word_ngram_uses_unicode_scalar_boundaries() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "WORD", false)).unwrap();
        let tokens = word_tokens("éclair", spec);
        assert_eq!(tokens[0].text, "éc");
        assert_eq!(tokens[1].text, "cl");
    }

    #[test]
    fn test_char_ngram_chinese_overlapping_bigrams() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "CHAR", false)).unwrap();
        let tokens = char_tokens("你好世界", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("你好", 0), ("好世", 1), ("世界", 2)]
        );
    }

    #[test]
    fn test_char_ngram_chinese_includes_whitespace() {
        let spec = NgramV1Spec::from_option(&option(3, 3, "CHAR", false)).unwrap();
        let tokens = char_tokens("你好 世界", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["你好 ", "好 世", " 世界"]
        );
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.position)
                .collect::<Vec<_>>(),
            [0, 1, 2]
        );
    }

    #[test]
    fn test_char_ngram_chinese_mixed_ascii_utf8_boundaries() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "CHAR", false)).unwrap();
        let tokens = char_tokens("ab你好", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("ab", 0), ("b你", 1), ("你好", 2)]
        );
    }

    #[test]
    fn test_char_ngram_chinese_lowercase_preserves_cjk() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "CHAR", true)).unwrap();
        let tokens = char_tokens("Hello世界", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| token.text.as_str())
                .collect::<Vec<_>>(),
            ["he", "el", "ll", "lo", "o世", "世界"]
        );
    }

    #[test]
    fn test_word_ngram_chinese_uses_alphanumeric_run() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "WORD", false)).unwrap();
        let tokens = word_tokens("社会物流总额", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [
                ("社会", 0),
                ("会物", 1),
                ("物流", 2),
                ("流总", 3),
                ("总额", 4),
            ]
        );
    }

    #[test]
    fn test_word_ngram_chinese_mixed_english() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "WORD", false)).unwrap();
        let tokens = word_tokens("hello 世界", spec);
        assert_eq!(
            tokens
                .iter()
                .filter(|token| token.text.chars().any(|ch| ch > '\u{7f}'))
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [("世界", 6)]
        );
        assert!(!tokens.iter().any(|token| token.text == "o世"));
        assert_eq!(
            tokens
                .iter()
                .find(|token| token.text == "世界")
                .map(|token| (token.offset_from, token.offset_to)),
            Some((6, 12))
        );
    }

    #[test]
    fn test_word_ngram_chinese_contiguous_text_emits_all_ngrams() {
        let spec = NgramV1Spec::from_option(&option(2, 3, "WORD", false)).unwrap();
        let tokens = word_tokens("数据库系统", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [
                ("数据", 0),
                ("数据库", 0),
                ("据库", 1),
                ("据库系", 1),
                ("库系", 2),
                ("库系统", 2),
                ("系统", 3),
            ]
        );
    }

    #[test]
    fn test_word_ngram_mixed_script_uses_one_alphanumeric_run() {
        let spec = NgramV1Spec::from_option(&option(2, 2, "WORD", false)).unwrap();
        let tokens = word_tokens("Hello世界", spec);
        assert_eq!(
            tokens
                .iter()
                .map(|token| (token.text.as_str(), token.position))
                .collect::<Vec<_>>(),
            [
                ("He", 0),
                ("el", 1),
                ("ll", 2),
                ("lo", 3),
                ("o世", 4),
                ("世界", 5),
            ]
        );
    }
}