use anyhow::Result;
use serde::{Deserialize, Serialize};
use tantivy::tokenizer::{RawTokenizer, SimpleTokenizer, TextAnalyzer, TextAnalyzerBuilder};

mod preserve_underscore_tokenizer;
use preserve_underscore_tokenizer::PreserveUnderscoreTokenizer;

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum TokenizerType {
    Simple,
    Keyword,
    PreserveUnderscore,
}

impl TokenizerType {
    pub fn build(&self) -> Result<TextAnalyzerBuilder> {
        match self {
            TokenizerType::Simple => {
                Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic())
            }
            TokenizerType::Keyword => Ok(TextAnalyzer::builder(RawTokenizer::default()).dynamic()),
            TokenizerType::PreserveUnderscore => {
                Ok(TextAnalyzer::builder(PreserveUnderscoreTokenizer::default()).dynamic())
            }
        }
    }
}