Raise matching floor: boundary tokenizer, stemming/folding/stopwords, boosts, snippets, engine seam

src/node.rs

@@ -18,6 +18,7 @@ use tokio::task::JoinHandle;
 
 use crate::config::{CLASS_ENRICHMENT, Config, Member, load_members};
 use crate::crypto::{Keypair, now_ts, poll_signing_bytes};
+use crate::engine::{SearchEngine, TantivyEngine};
 use crate::index::{LocalIndex, SearchHit, response_items};
 use crate::message::{
     AggregateBody, Envelope, QueryBody, ResponseBody, ResponseItem, TYPE_AGGREGATE, TYPE_QUERY,
@@ -34,7 +35,7 @@ struct Aggregates {
 pub struct Node {
     pub config: Config,
     pub key: Keypair,
-    index: LocalIndex,
+    engine: Arc<dyn SearchEngine>,
     members: RwLock<Vec<Member>>,
     members_mtime: Mutex<Option<SystemTime>>,
     registry: Option<Arc<Watcher>>,
@@ -113,7 +114,10 @@ pub struct LocalPart {
 impl Node {
     pub fn open(config: Config) -> Result<Arc<Self>> {
         let key = config.load_key()?;
-        let index = LocalIndex::open(&config.index_dir())?;
+        let engine: Arc<dyn SearchEngine> = Arc::new(TantivyEngine {
+            index: LocalIndex::open(&config.index_dir())?,
+            min_coverage: config.matching.min_coverage,
+        });
         let members_path = config.members_path();
         let members = load_members(&members_path)?;
         let members_mtime = fs::metadata(&members_path)
@@ -155,10 +159,10 @@ impl Node {
         Ok(Arc::new(Self {
             config,
             key,
-            index,
             members: RwLock::new(members),
             members_mtime: Mutex::new(members_mtime),
             registry,
+            engine,
             enc_secret,
             enc_public,
             aggregates: Mutex::new(Aggregates::default()),
@@ -229,7 +233,7 @@ impl Node {
     }
 
     pub fn doc_count(&self) -> u64 {
-        self.index.doc_count()
+        self.engine.doc_count()
     }
 
     pub fn identifier(&self) -> String {
@@ -249,7 +253,8 @@ impl Node {
     }
 
     pub fn local_search(&self, text: &str, limit: usize) -> Result<(Vec<SearchHit>, u64)> {
-        self.index.search(text, limit, false)
+        let output = self.engine.search(text, limit, false)?;
+        Ok((output.hits, output.total.unwrap_or(0)))
     }
 
     pub async fn start(config: Config) -> Result<NodeHandle> {
@@ -310,9 +315,9 @@ impl Node {
         let max = max_results.unwrap_or(self.config.query.max_results).max(1);
         let query = QueryBody::new(text, max);
         let qid = query.qid.clone();
-        let (local_hits, local_total) =
-            self.index
-                .search_with_coverage(text, max, false, self.config.matching.min_coverage)?;
+        let output = self.engine.search(text, max, false)?;
+        let local_hits = output.hits;
+        let local_total = output.total.unwrap_or(0);
         let local_items = response_items(&local_hits);
         let mut responses = Vec::new();
         if network {
@@ -562,16 +567,11 @@ impl Node {
 
     async fn respond(&self, query: &QueryBody, querier: &str, relay: &str) -> Result<()> {
         let max = query.budget.max_results.clamp(1, 1000);
-        let (hits, total) = self.index.search_with_coverage(
-            &query.text,
-            max,
-            true,
-            self.config.matching.min_coverage,
-        )?;
-        if hits.is_empty() {
+        let output = self.engine.search(&query.text, max, true)?;
+        if output.hits.is_empty() {
             return Ok(());
         }
-        let body = build_response(&query.qid, response_items(&hits), total, max);
+        let body = build_response(&query.qid, response_items(&output.hits), output.total, max);
         self.send_payload(querier, TYPE_RESPONSE, serde_json::to_value(&body)?, relay)
             .await
     }