Raise matching floor: boundary tokenizer, stemming/folding/stopwords, boosts, snippets, engine seam

src/index.rs

@@ -7,13 +7,19 @@ use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use tantivy::collector::{Count, TopDocs};
 use tantivy::directory::MmapDirectory;
-use tantivy::query::{AllQuery, BooleanQuery, Occur, Query, TermQuery};
-use tantivy::schema::{Field, IndexRecordOption, STORED, STRING, Schema, TEXT, Value};
+use tantivy::query::{
+    AllQuery, BooleanQuery, BoostQuery, EmptyQuery, Occur, PhraseQuery, Query, TermQuery,
+};
+use tantivy::schema::{
+    Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value,
+};
+use tantivy::snippet::SnippetGenerator;
 use tantivy::tokenizer::TokenStream;
 use tantivy::{Index, IndexReader, IndexWriter, TantivyDocument, Term, doc};
 
 use crate::extract::{extract_file, summary_of, supported};
 use crate::message::{EXPOSURE_FULL, ResponseItem};
+use crate::tokenizer::{TOKENIZER_NAME, analyzer};
 
 const WRITER_BUDGET: usize = 50_000_000;
 const MAX_QUERY_TERMS: usize = 10;
@@ -107,6 +113,7 @@ impl LocalIndex {
         let schema = build_schema();
         let index = Index::open_or_create(MmapDirectory::open(dir)?, schema)
             .context("opening tantivy index")?;
+        index.tokenizers().register(TOKENIZER_NAME, analyzer());
         let schema = index.schema();
         let fields = Fields {
             url: schema.get_field("url")?,
@@ -198,10 +205,28 @@ impl LocalIndex {
         self.reader.reload()?;
         let searcher = self.reader.searcher();
         let terms = self.query_terms(text);
-        let user_query: Box<dyn Query> = if terms.is_empty() {
+        let user_query: Box<dyn Query> = if text.trim().is_empty() {
             Box::new(AllQuery)
+        } else if terms.is_empty() {
+            Box::new(EmptyQuery)
         } else {
-            self.coverage_query(&terms, min_coverage)
+            let gate = self.coverage_query(&terms, min_coverage);
+            let mut parts: Vec<(Occur, Box<dyn Query>)> = vec![(Occur::Must, gate)];
+            if terms.len() >= 2 {
+                let phrase_terms: Vec<(usize, Term)> = terms
+                    .iter()
+                    .enumerate()
+                    .map(|(index, term)| (index, Term::from_field_text(self.fields.body, term)))
+                    .collect();
+                parts.push((
+                    Occur::Should,
+                    Box::new(BoostQuery::new(
+                        Box::new(PhraseQuery::new_with_offset_and_slop(phrase_terms, 1)),
+                        3.0,
+                    )),
+                ));
+            }
+            Box::new(BooleanQuery::new(parts))
         };
         let query: Box<dyn Query> = if only_shared {
             let shared_term = TermQuery::new(
@@ -217,13 +242,18 @@ impl LocalIndex {
         };
         let total = searcher.search(&*query, &Count)? as u64;
         let top = searcher.search(&*query, &TopDocs::with_limit(limit).order_by_score())?;
+        let snippet_generator = SnippetGenerator::create(&searcher, &*query, self.fields.body).ok();
         let mut hits = Vec::with_capacity(top.len());
         for (_score, address) in top {
             let document: TantivyDocument = searcher.doc(address)?;
+            let snippet = snippet_generator
+                .as_ref()
+                .map(|generator| generator.snippet_from_doc(&document).fragment().to_string())
+                .filter(|fragment| !fragment.trim().is_empty());
             hits.push(SearchHit {
                 url: text_value(&document, self.fields.url),
                 title: text_value(&document, self.fields.title),
-                summary: text_value(&document, self.fields.summary),
+                summary: snippet.unwrap_or_else(|| text_value(&document, self.fields.summary)),
                 published: text_value(&document, self.fields.published),
                 exposure: text_value(&document, self.fields.exposure),
                 collection: text_value(&document, self.fields.collection),
@@ -237,7 +267,7 @@ impl LocalIndex {
     fn query_terms(&self, text: &str) -> Vec<String> {
         let mut terms = Vec::new();
         let mut seen = HashSet::new();
-        if let Some(mut tokenizer) = self.index.tokenizers().get("default") {
+        if let Some(mut tokenizer) = self.index.tokenizers().get(TOKENIZER_NAME) {
             let mut stream = tokenizer.token_stream(text);
             while let Some(token) = stream.next() {
                 let term = token.text.to_string();
@@ -270,9 +300,12 @@ impl LocalIndex {
                     let term_query: Box<dyn Query> = Box::new(BooleanQuery::new(vec![
                         (
                             Occur::Should,
-                            Box::new(TermQuery::new(
-                                Term::from_field_text(self.fields.title, term),
-                                IndexRecordOption::WithFreqs,
+                            Box::new(BoostQuery::new(
+                                Box::new(TermQuery::new(
+                                    Term::from_field_text(self.fields.title, term),
+                                    IndexRecordOption::WithFreqs,
+                                )),
+                                2.0,
                             )),
                         ),
                         (
@@ -345,9 +378,16 @@ fn text_value(document: &TantivyDocument, field: Field) -> String {
 
 fn build_schema() -> Schema {
     let mut builder = Schema::builder();
+    let frx_options = TextOptions::default()
+        .set_indexing_options(
+            TextFieldIndexing::default()
+                .set_tokenizer(TOKENIZER_NAME)
+                .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+        )
+        .set_stored();
     builder.add_text_field("url", STRING | STORED);
-    builder.add_text_field("title", TEXT | STORED);
-    builder.add_text_field("body", TEXT | STORED);
+    builder.add_text_field("title", frx_options.clone());
+    builder.add_text_field("body", frx_options);
     builder.add_text_field("summary", STORED);
     builder.add_text_field("published", STORED);
     builder.add_text_field("exposure", STRING | STORED);