pub mod analyzer;
pub mod token_filter;
pub mod tokenizer;

// There are three concepts here:
// 1. Tokenizer: Responsible for converting (splitting) text into tokens.
// 2. Token Filter: Responsible for filtering tokens (e.g., stopword filtering)
//    or transforming them (e.g., case conversion).
// 3. Analyzer: A tokenizer plus zero or more filters. Note: The order of
//    multiple filters may affect the final tokenization result.

#[cfg(test)]
mod tests {
    use tantivy::tokenizer::Token;

    pub fn assert_token(token: &Token, position: usize, text: &str, from: usize, to: usize) {
        assert_eq!(
            token.position, position,
            "expected position {position} but {token:?}"
        );
        assert_eq!(token.text, text, "expected text {text} but {token:?}");
        assert_eq!(
            token.offset_from, from,
            "expected offset_from {from} but {token:?}"
        );
        assert_eq!(token.offset_to, to, "expected offset_to {to} but {token:?}");
    }
}