Lexical coverage gate for matching; fix demo false positive

src/index.rs

@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
@@ -6,14 +7,18 @@ use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use tantivy::collector::{Count, TopDocs};
 use tantivy::directory::MmapDirectory;
-use tantivy::query::{AllQuery, BooleanQuery, Occur, Query, QueryParser, TermQuery};
+use tantivy::query::{AllQuery, BooleanQuery, Occur, Query, TermQuery};
 use tantivy::schema::{Field, IndexRecordOption, STORED, STRING, Schema, TEXT, Value};
+use tantivy::tokenizer::TokenStream;
 use tantivy::{Index, IndexReader, IndexWriter, TantivyDocument, Term, doc};
 
 use crate::extract::{extract_file, summary_of, supported};
 use crate::message::{EXPOSURE_FULL, ResponseItem};
 
 const WRITER_BUDGET: usize = 50_000_000;
+const MAX_QUERY_TERMS: usize = 10;
+const MAX_COMBOS: usize = 256;
+const DEFAULT_MIN_COVERAGE: f64 = 0.4;
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Collection {
@@ -178,17 +183,25 @@ impl LocalIndex {
         text: &str,
         limit: usize,
         only_shared: bool,
+    ) -> Result<(Vec<SearchHit>, u64)> {
+        self.search_with_coverage(text, limit, only_shared, DEFAULT_MIN_COVERAGE)
+    }
+
+    pub fn search_with_coverage(
+        &self,
+        text: &str,
+        limit: usize,
+        only_shared: bool,
+        min_coverage: f64,
     ) -> Result<(Vec<SearchHit>, u64)> {
         let limit = limit.max(1);
         self.reader.reload()?;
         let searcher = self.reader.searcher();
-        let user_query: Box<dyn Query> = if text.trim().is_empty() {
+        let terms = self.query_terms(text);
+        let user_query: Box<dyn Query> = if terms.is_empty() {
             Box::new(AllQuery)
         } else {
-            let parser =
-                QueryParser::for_index(&self.index, vec![self.fields.title, self.fields.body]);
-            let (query, _errors) = parser.parse_query_lenient(text);
-            query
+            self.coverage_query(&terms, min_coverage)
         };
         let query: Box<dyn Query> = if only_shared {
             let shared_term = TermQuery::new(
@@ -221,11 +234,107 @@ impl LocalIndex {
         Ok((hits, total))
     }
 
+    fn query_terms(&self, text: &str) -> Vec<String> {
+        let mut terms = Vec::new();
+        let mut seen = HashSet::new();
+        if let Some(mut tokenizer) = self.index.tokenizers().get("default") {
+            let mut stream = tokenizer.token_stream(text);
+            while let Some(token) = stream.next() {
+                let term = token.text.to_string();
+                if seen.insert(term.clone()) {
+                    terms.push(term);
+                }
+                if terms.len() >= MAX_QUERY_TERMS {
+                    break;
+                }
+            }
+        }
+        terms
+    }
+
+    fn coverage_query(&self, terms: &[String], min_coverage: f64) -> Box<dyn Query> {
+        let n = terms.len();
+        let mut required = if n <= 2 {
+            n
+        } else {
+            ((n as f64) * min_coverage).ceil() as usize
+        };
+        required = required.clamp(1, n);
+        let combos = combinations(n, required, MAX_COMBOS);
+        let mut shoulds: Vec<(Occur, Box<dyn Query>)> = Vec::with_capacity(combos.len());
+        for combo in combos {
+            let musts: Vec<(Occur, Box<dyn Query>)> = combo
+                .iter()
+                .map(|index| {
+                    let term = terms[*index].as_str();
+                    let term_query: Box<dyn Query> = Box::new(BooleanQuery::new(vec![
+                        (
+                            Occur::Should,
+                            Box::new(TermQuery::new(
+                                Term::from_field_text(self.fields.title, term),
+                                IndexRecordOption::WithFreqs,
+                            )),
+                        ),
+                        (
+                            Occur::Should,
+                            Box::new(TermQuery::new(
+                                Term::from_field_text(self.fields.body, term),
+                                IndexRecordOption::WithFreqs,
+                            )),
+                        ),
+                    ]));
+                    (Occur::Must, term_query)
+                })
+                .collect();
+            shoulds.push((Occur::Should, Box::new(BooleanQuery::new(musts))));
+        }
+        Box::new(BooleanQuery::new(shoulds))
+    }
+
     pub fn doc_count(&self) -> u64 {
         self.reader.searcher().num_docs()
     }
 }
 
+fn binomial(n: usize, k: usize) -> usize {
+    if k > n {
+        return 0;
+    }
+    let k = k.min(n - k);
+    let mut result = 1usize;
+    for index in 0..k {
+        result = result * (n - index) / (index + 1);
+    }
+    result
+}
+
+fn combinations(n: usize, start_required: usize, cap: usize) -> Vec<Vec<usize>> {
+    let mut required = start_required;
+    while required < n && binomial(n, required) > cap {
+        required += 1;
+    }
+    fn walk(
+        start: usize,
+        remaining: usize,
+        n: usize,
+        current: &mut Vec<usize>,
+        out: &mut Vec<Vec<usize>>,
+    ) {
+        if remaining == 0 {
+            out.push(current.clone());
+            return;
+        }
+        for index in start..=n - remaining {
+            current.push(index);
+            walk(index + 1, remaining - 1, n, current, out);
+            current.pop();
+        }
+    }
+    let mut out = Vec::new();
+    walk(0, required, n, &mut Vec::new(), &mut out);
+    out
+}
+
 fn text_value(document: &TantivyDocument, field: Field) -> String {
     document
         .get_first(field)
@@ -321,6 +430,44 @@ mod tests {
         assert!(items[0].content.is_some());
     }
 
+    #[test]
+    fn coverage_gate_rejects_weak_single_term_overlap() {
+        let temp = tempfile::tempdir().unwrap();
+        let dir = temp.path().join("corpus");
+        fs::create_dir_all(&dir).unwrap();
+        fs::write(dir.join("errors.txt"), "error codes and diagnostics").unwrap();
+        let index = LocalIndex::open(&temp.path().join("index")).unwrap();
+        index
+            .add_collection(&collection("test", &dir, true, EXPOSURE_FULL))
+            .unwrap();
+
+        let (hits, total) = index.search("launch codes", 10, true).unwrap();
+        assert_eq!(total, 0, "two-term queries require both terms");
+        assert!(hits.is_empty());
+    }
+
+    #[test]
+    fn coverage_gate_allows_partial_long_queries() {
+        let temp = tempfile::tempdir().unwrap();
+        let dir = temp.path().join("corpus");
+        fs::create_dir_all(&dir).unwrap();
+        fs::write(
+            dir.join("manual.txt"),
+            "LG DLEX5555 dryer circuit diagram and wiring",
+        )
+        .unwrap();
+        let index = LocalIndex::open(&temp.path().join("index")).unwrap();
+        index
+            .add_collection(&collection("test", &dir, true, EXPOSURE_FULL))
+            .unwrap();
+
+        let (hits, total) = index
+            .search("LG dryer 5555 circuit diagram", 10, true)
+            .unwrap();
+        assert_eq!(total, 1, "four of five terms is enough");
+        assert_eq!(hits.len(), 1);
+    }
+
     #[test]
     fn reindex_removes_deleted_files() {
         let temp = tempfile::tempdir().unwrap();