Raise matching floor: boundary tokenizer, stemming/folding/stopwords, boosts, snippets, engine seam

src/tokenizer.rs

@@ -0,0 +1,131 @@
+use tantivy::tokenizer::{
+    AsciiFoldingFilter, Language, LowerCaser, RemoveLongFilter, Stemmer, StopWordFilter,
+    TextAnalyzer, Token, TokenStream, Tokenizer,
+};
+
+#[derive(Clone, Default)]
+pub struct BoundaryTokenizer {
+    token: Token,
+}
+
+impl BoundaryTokenizer {
+    pub fn new() -> Self {
+        Self {
+            token: Token::default(),
+        }
+    }
+}
+
+impl Tokenizer for BoundaryTokenizer {
+    type TokenStream<'a> = BoundaryTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        BoundaryTokenStream {
+            text,
+            chars: text.char_indices().peekable(),
+            token: &mut self.token,
+            position: 0,
+        }
+    }
+}
+
+pub struct BoundaryTokenStream<'a> {
+    text: &'a str,
+    chars: std::iter::Peekable<std::str::CharIndices<'a>>,
+    token: &'a mut Token,
+    position: usize,
+}
+
+impl TokenStream for BoundaryTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        let mut start = None;
+        let mut alphabetic = None;
+        while let Some(&(index, ch)) = self.chars.peek() {
+            if !ch.is_alphanumeric() {
+                if start.is_none() {
+                    self.chars.next();
+                    continue;
+                }
+                break;
+            }
+            match alphabetic {
+                None => {
+                    start = Some(index);
+                    alphabetic = Some(ch.is_alphabetic());
+                    self.chars.next();
+                }
+                Some(prev) if prev == ch.is_alphabetic() => {
+                    self.chars.next();
+                }
+                Some(_) => break,
+            }
+        }
+        let Some(start) = start else {
+            return false;
+        };
+        let end = self
+            .chars
+            .peek()
+            .map(|&(index, _)| index)
+            .unwrap_or(self.text.len());
+        self.token.text = self.text[start..end].to_string();
+        self.token.offset_from = start;
+        self.token.offset_to = end;
+        self.token.position = self.position;
+        self.position += 1;
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
+
+pub fn analyzer() -> TextAnalyzer {
+    TextAnalyzer::builder(BoundaryTokenizer::new())
+        .filter(LowerCaser)
+        .filter(AsciiFoldingFilter)
+        .filter(StopWordFilter::new(Language::English).expect("english stopwords"))
+        .filter(Stemmer::new(Language::English))
+        .filter(RemoveLongFilter::limit(255))
+        .build()
+}
+
+pub const TOKENIZER_NAME: &str = "frx";
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tokens(text: &str) -> Vec<String> {
+        let mut analyzer = analyzer();
+        let mut stream = analyzer.token_stream(text);
+        let mut out = Vec::new();
+        while let Some(token) = stream.next() {
+            out.push(token.text.clone());
+        }
+        out
+    }
+
+    #[test]
+    fn splits_model_numbers_and_folds_case() {
+        let tokens = tokens("DLEX5555 Café");
+        assert!(tokens.contains(&"dlex".to_string()), "{tokens:?}");
+        assert!(tokens.contains(&"5555".to_string()), "{tokens:?}");
+        assert!(tokens.contains(&"cafe".to_string()), "{tokens:?}");
+    }
+
+    #[test]
+    fn stems_and_drops_stopwords() {
+        let tokens = tokens("the dryers are searching");
+        assert!(!tokens.contains(&"the".to_string()), "{tokens:?}");
+        assert!(!tokens.contains(&"are".to_string()), "{tokens:?}");
+        assert!(tokens.iter().any(|t| t.starts_with("dryer")), "{tokens:?}");
+        assert!(tokens.iter().any(|t| t.starts_with("search")), "{tokens:?}");
+    }
+}