132 lines
3.7 KiB
Rust
132 lines
3.7 KiB
Rust
use tantivy::tokenizer::{
|
|
AsciiFoldingFilter, Language, LowerCaser, RemoveLongFilter, Stemmer, StopWordFilter,
|
|
TextAnalyzer, Token, TokenStream, Tokenizer,
|
|
};
|
|
|
|
#[derive(Clone, Default)]
|
|
pub struct BoundaryTokenizer {
|
|
token: Token,
|
|
}
|
|
|
|
impl BoundaryTokenizer {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
token: Token::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Tokenizer for BoundaryTokenizer {
|
|
type TokenStream<'a> = BoundaryTokenStream<'a>;
|
|
|
|
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
|
|
self.token.reset();
|
|
BoundaryTokenStream {
|
|
text,
|
|
chars: text.char_indices().peekable(),
|
|
token: &mut self.token,
|
|
position: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub struct BoundaryTokenStream<'a> {
|
|
text: &'a str,
|
|
chars: std::iter::Peekable<std::str::CharIndices<'a>>,
|
|
token: &'a mut Token,
|
|
position: usize,
|
|
}
|
|
|
|
impl TokenStream for BoundaryTokenStream<'_> {
|
|
fn advance(&mut self) -> bool {
|
|
let mut start = None;
|
|
let mut alphabetic = None;
|
|
while let Some(&(index, ch)) = self.chars.peek() {
|
|
if !ch.is_alphanumeric() {
|
|
if start.is_none() {
|
|
self.chars.next();
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
match alphabetic {
|
|
None => {
|
|
start = Some(index);
|
|
alphabetic = Some(ch.is_alphabetic());
|
|
self.chars.next();
|
|
}
|
|
Some(prev) if prev == ch.is_alphabetic() => {
|
|
self.chars.next();
|
|
}
|
|
Some(_) => break,
|
|
}
|
|
}
|
|
let Some(start) = start else {
|
|
return false;
|
|
};
|
|
let end = self
|
|
.chars
|
|
.peek()
|
|
.map(|&(index, _)| index)
|
|
.unwrap_or(self.text.len());
|
|
self.token.text = self.text[start..end].to_string();
|
|
self.token.offset_from = start;
|
|
self.token.offset_to = end;
|
|
self.token.position = self.position;
|
|
self.position += 1;
|
|
true
|
|
}
|
|
|
|
fn token(&self) -> &Token {
|
|
self.token
|
|
}
|
|
|
|
fn token_mut(&mut self) -> &mut Token {
|
|
self.token
|
|
}
|
|
}
|
|
|
|
pub fn analyzer() -> TextAnalyzer {
|
|
TextAnalyzer::builder(BoundaryTokenizer::new())
|
|
.filter(LowerCaser)
|
|
.filter(AsciiFoldingFilter)
|
|
.filter(StopWordFilter::new(Language::English).expect("english stopwords"))
|
|
.filter(Stemmer::new(Language::English))
|
|
.filter(RemoveLongFilter::limit(255))
|
|
.build()
|
|
}
|
|
|
|
pub const TOKENIZER_NAME: &str = "frx";
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn tokens(text: &str) -> Vec<String> {
|
|
let mut analyzer = analyzer();
|
|
let mut stream = analyzer.token_stream(text);
|
|
let mut out = Vec::new();
|
|
while let Some(token) = stream.next() {
|
|
out.push(token.text.clone());
|
|
}
|
|
out
|
|
}
|
|
|
|
#[test]
|
|
fn splits_model_numbers_and_folds_case() {
|
|
let tokens = tokens("DLEX5555 Café");
|
|
assert!(tokens.contains(&"dlex".to_string()), "{tokens:?}");
|
|
assert!(tokens.contains(&"5555".to_string()), "{tokens:?}");
|
|
assert!(tokens.contains(&"cafe".to_string()), "{tokens:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn stems_and_drops_stopwords() {
|
|
let tokens = tokens("the dryers are searching");
|
|
assert!(!tokens.contains(&"the".to_string()), "{tokens:?}");
|
|
assert!(!tokens.contains(&"are".to_string()), "{tokens:?}");
|
|
assert!(tokens.iter().any(|t| t.starts_with("dryer")), "{tokens:?}");
|
|
assert!(tokens.iter().any(|t| t.starts_with("search")), "{tokens:?}");
|
|
}
|
|
}
|