Lexical coverage gate for matching; fix demo false positive

2026-09-15 07:14:06 -04:00
parent 67112c2af3
commit 4d4fdbe104
5 changed files with 186 additions and 10 deletions
@@ -4,7 +4,7 @@
 - `rfc.txt` (FRX — Federated Retrieval Exchange, Draft 0.5) is the normative spec; `src/` is the Phase 1 `frxd` implementation (single crate, two binaries).
 - `frxd` is the member node (init/add/index/serve/relay/query/status); `frx` is the thin client (search/query/status). Relay and node roles are separate subcommands.
 - `DEPLOY.md` documents the TLS/deployment story: members need no TLS (outbound HTTPS), relays terminate TLS with Caddy or a tunnel, `ca_cert` adds private CAs, `allow_insecure` opts into plain http on private networks, and non-loopback `http://` is refused by default.
- Commands: `cargo build`, `cargo test` (97 tests: unit in `src/`; e2e `tests/phase1.rs`; conformance `tests/conformance.rs`; aggregates + member directory `tests/aggregates.rs`; registry `tests/registry.rs`; federation/isolation/admission `tests/federation.rs`; SSE `tests/streaming.rs`; encrypted unicast `tests/encryption.rs`; concurrency/restart `tests/concurrency.rs`; real subprocess CLI `tests/cli.rs`; 1000-doc `tests/scale.rs`; purge-log absence `tests/purges.rs`; shared fixtures `tests/common/mod.rs`). No CI/lint config.
+- Commands: `cargo build`, `cargo test` (99 tests: unit in `src/`; e2e `tests/phase1.rs`; conformance `tests/conformance.rs`; aggregates + member directory `tests/aggregates.rs`; registry `tests/registry.rs`; federation/isolation/admission `tests/federation.rs`; SSE `tests/streaming.rs`; encrypted unicast `tests/encryption.rs`; concurrency/restart `tests/concurrency.rs`; real subprocess CLI `tests/cli.rs`; 1000-doc `tests/scale.rs`; purge-log absence `tests/purges.rs`; shared fixtures `tests/common/mod.rs`). No CI/lint config.
 - E2E pattern: relay + nodes in-process on ephemeral ports with tempdir corpora; use `tests/common/mod.rs` helpers (`spawn_relay*`, `query_envelope`, `poll_messages`, `register`) for new coverage. Raw relay polls return envelopes (payload under `body`), not response bodies.

 ## Editing the spec
@@ -47,5 +47,5 @@
 - Language: Rust (settled, matches §7). Decided by the engine requirement, not preference: Tantivy gives in-process Lucene-class BM25 + incremental indexing; C/C++ embedded alternatives are worse (Xapian GPL-2+, CLucene unmaintained, SQLite FTS5 thin), plus single static musl binaries for the install story and memory safety on the untrusted network/crypto path. Don't re-litigate.
 - frxd modes (one binary, config toggles, no code required of publishers): querier (broadcast/local-first search), responder (match incoming queries against shared collections, sign), local index (watch dirs, extract text, explicit shared marking per I9). Use RFC terms querier/responder, not "subscriber/publisher".
 - Roles are not exclusive: a single node may issue queries and answer them concurrently (I5, §3 "any member"). Implement querier/responder as independent enable flags — never an exclusive mode enum or fixed deployment role.
- Matching accuracy is a project-health concern: start lexical (Tantivy), plan a hybrid cheap lexical gate + optional local embedding rerank (two-stage ingestion, Appendix A); embedding model stays local and replaceable (I2/I5).
+- Matching accuracy is a project-health concern: the cheap lexical gate is implemented (`[match] min_coverage`, default 0.4; queries of one or two terms require all of them, longer queries require a coverage fraction via OR-of-ANDs) after a demo false positive ("internal launch codes" matching "error codes"). Next stage is an optional local embedding rerank behind the gate; the model stays local and replaceable (I2/I5). Re-measure precision with a real corpus before tuning the threshold.
 - Identity/registry (RFC Draft 0.5 §4/§6): MA-hosted FQDN identifiers first (`<label>.frx.<ma-domain>`, no DNS needed by users), signed versioned registry snapshot with the MA key pinned; envelope `from` = identifier, `key` = pubkey; registry outage fails static. Member-hosted identities, MA anchor rollover, and unicast confidentiality are §10 open. Implementation phases: A (signed registry snapshot) and B (identifier + `key` + JCS on the wire) are built and tested. Prioritize frictionless onboarding (users may be department-level and cannot create DNS).
@@ -13,6 +13,26 @@ pub struct Config {
    pub query: QuerySection,
    #[serde(default)]
    pub index: IndexSection,
+    #[serde(default, rename = "match")]
+    pub matching: MatchSection,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct MatchSection {
+    #[serde(default = "default_min_coverage")]
+    pub min_coverage: f64,
+}
+
+impl Default for MatchSection {
+    fn default() -> Self {
+        Self {
+            min_coverage: default_min_coverage(),
+        }
+    }
+}
+
+fn default_min_coverage() -> f64 {
+    0.4
 }

 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -145,6 +165,7 @@ impl Config {
            index: IndexSection {
                data_dir: data_dir.to_string(),
            },
+            matching: MatchSection::default(),
        }
    }

@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
@@ -6,14 +7,18 @@ use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use tantivy::collector::{Count, TopDocs};
 use tantivy::directory::MmapDirectory;
-use tantivy::query::{AllQuery, BooleanQuery, Occur, Query, QueryParser, TermQuery};
+use tantivy::query::{AllQuery, BooleanQuery, Occur, Query, TermQuery};
 use tantivy::schema::{Field, IndexRecordOption, STORED, STRING, Schema, TEXT, Value};
+use tantivy::tokenizer::TokenStream;
 use tantivy::{Index, IndexReader, IndexWriter, TantivyDocument, Term, doc};

 use crate::extract::{extract_file, summary_of, supported};
 use crate::message::{EXPOSURE_FULL, ResponseItem};

 const WRITER_BUDGET: usize = 50_000_000;
+const MAX_QUERY_TERMS: usize = 10;
+const MAX_COMBOS: usize = 256;
+const DEFAULT_MIN_COVERAGE: f64 = 0.4;

 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Collection {
@@ -178,17 +183,25 @@ impl LocalIndex {
        text: &str,
        limit: usize,
        only_shared: bool,
+    ) -> Result<(Vec<SearchHit>, u64)> {
+        self.search_with_coverage(text, limit, only_shared, DEFAULT_MIN_COVERAGE)
+    }
+
+    pub fn search_with_coverage(
+        &self,
+        text: &str,
+        limit: usize,
+        only_shared: bool,
+        min_coverage: f64,
    ) -> Result<(Vec<SearchHit>, u64)> {
        let limit = limit.max(1);
        self.reader.reload()?;
        let searcher = self.reader.searcher();
-        let user_query: Box<dyn Query> = if text.trim().is_empty() {
+        let terms = self.query_terms(text);
+        let user_query: Box<dyn Query> = if terms.is_empty() {
            Box::new(AllQuery)
        } else {
-            let parser =
-                QueryParser::for_index(&self.index, vec![self.fields.title, self.fields.body]);
-            let (query, _errors) = parser.parse_query_lenient(text);
-            query
+            self.coverage_query(&terms, min_coverage)
        };
        let query: Box<dyn Query> = if only_shared {
            let shared_term = TermQuery::new(
@@ -221,11 +234,107 @@ impl LocalIndex {
        Ok((hits, total))
    }

+    fn query_terms(&self, text: &str) -> Vec<String> {
+        let mut terms = Vec::new();
+        let mut seen = HashSet::new();
+        if let Some(mut tokenizer) = self.index.tokenizers().get("default") {
+            let mut stream = tokenizer.token_stream(text);
+            while let Some(token) = stream.next() {
+                let term = token.text.to_string();
+                if seen.insert(term.clone()) {
+                    terms.push(term);
+                }
+                if terms.len() >= MAX_QUERY_TERMS {
+                    break;
+                }
+            }
+        }
+        terms
+    }
+
+    fn coverage_query(&self, terms: &[String], min_coverage: f64) -> Box<dyn Query> {
+        let n = terms.len();
+        let mut required = if n <= 2 {
+            n
+        } else {
+            ((n as f64) * min_coverage).ceil() as usize
+        };
+        required = required.clamp(1, n);
+        let combos = combinations(n, required, MAX_COMBOS);
+        let mut shoulds: Vec<(Occur, Box<dyn Query>)> = Vec::with_capacity(combos.len());
+        for combo in combos {
+            let musts: Vec<(Occur, Box<dyn Query>)> = combo
+                .iter()
+                .map(|index| {
+                    let term = terms[*index].as_str();
+                    let term_query: Box<dyn Query> = Box::new(BooleanQuery::new(vec![
+                        (
+                            Occur::Should,
+                            Box::new(TermQuery::new(
+                                Term::from_field_text(self.fields.title, term),
+                                IndexRecordOption::WithFreqs,
+                            )),
+                        ),
+                        (
+                            Occur::Should,
+                            Box::new(TermQuery::new(
+                                Term::from_field_text(self.fields.body, term),
+                                IndexRecordOption::WithFreqs,
+                            )),
+                        ),
+                    ]));
+                    (Occur::Must, term_query)
+                })
+                .collect();
+            shoulds.push((Occur::Should, Box::new(BooleanQuery::new(musts))));
+        }
+        Box::new(BooleanQuery::new(shoulds))
+    }
+
    pub fn doc_count(&self) -> u64 {
        self.reader.searcher().num_docs()
    }
 }

+fn binomial(n: usize, k: usize) -> usize {
+    if k > n {
+        return 0;
+    }
+    let k = k.min(n - k);
+    let mut result = 1usize;
+    for index in 0..k {
+        result = result * (n - index) / (index + 1);
+    }
+    result
+}
+
+fn combinations(n: usize, start_required: usize, cap: usize) -> Vec<Vec<usize>> {
+    let mut required = start_required;
+    while required < n && binomial(n, required) > cap {
+        required += 1;
+    }
+    fn walk(
+        start: usize,
+        remaining: usize,
+        n: usize,
+        current: &mut Vec<usize>,
+        out: &mut Vec<Vec<usize>>,
+    ) {
+        if remaining == 0 {
+            out.push(current.clone());
+            return;
+        }
+        for index in start..=n - remaining {
+            current.push(index);
+            walk(index + 1, remaining - 1, n, current, out);
+            current.pop();
+        }
+    }
+    let mut out = Vec::new();
+    walk(0, required, n, &mut Vec::new(), &mut out);
+    out
+}
+
 fn text_value(document: &TantivyDocument, field: Field) -> String {
    document
        .get_first(field)
@@ -321,6 +430,44 @@ mod tests {
        assert!(items[0].content.is_some());
    }

+    #[test]
+    fn coverage_gate_rejects_weak_single_term_overlap() {
+        let temp = tempfile::tempdir().unwrap();
+        let dir = temp.path().join("corpus");
+        fs::create_dir_all(&dir).unwrap();
+        fs::write(dir.join("errors.txt"), "error codes and diagnostics").unwrap();
+        let index = LocalIndex::open(&temp.path().join("index")).unwrap();
+        index
+            .add_collection(&collection("test", &dir, true, EXPOSURE_FULL))
+            .unwrap();
+
+        let (hits, total) = index.search("launch codes", 10, true).unwrap();
+        assert_eq!(total, 0, "two-term queries require both terms");
+        assert!(hits.is_empty());
+    }
+
+    #[test]
+    fn coverage_gate_allows_partial_long_queries() {
+        let temp = tempfile::tempdir().unwrap();
+        let dir = temp.path().join("corpus");
+        fs::create_dir_all(&dir).unwrap();
+        fs::write(
+            dir.join("manual.txt"),
+            "LG DLEX5555 dryer circuit diagram and wiring",
+        )
+        .unwrap();
+        let index = LocalIndex::open(&temp.path().join("index")).unwrap();
+        index
+            .add_collection(&collection("test", &dir, true, EXPOSURE_FULL))
+            .unwrap();
+
+        let (hits, total) = index
+            .search("LG dryer 5555 circuit diagram", 10, true)
+            .unwrap();
+        assert_eq!(total, 1, "four of five terms is enough");
+        assert_eq!(hits.len(), 1);
+    }
+
    #[test]
    fn reindex_removes_deleted_files() {
        let temp = tempfile::tempdir().unwrap();
@@ -310,7 +310,9 @@ impl Node {
        let max = max_results.unwrap_or(self.config.query.max_results).max(1);
        let query = QueryBody::new(text, max);
        let qid = query.qid.clone();
-        let (local_hits, local_total) = self.index.search(text, max, false)?;
+        let (local_hits, local_total) =
+            self.index
+                .search_with_coverage(text, max, false, self.config.matching.min_coverage)?;
        let local_items = response_items(&local_hits);
        let mut responses = Vec::new();
        if network {
@@ -560,7 +562,12 @@ impl Node {

    async fn respond(&self, query: &QueryBody, querier: &str, relay: &str) -> Result<()> {
        let max = query.budget.max_results.clamp(1, 1000);
-        let (hits, total) = self.index.search(&query.text, max, true)?;
+        let (hits, total) = self.index.search_with_coverage(
+            &query.text,
+            max,
+            true,
+            self.config.matching.min_coverage,
+        )?;
        if hits.is_empty() {
            return Ok(());
        }
@@ -30,6 +30,7 @@ pub fn config_for(dir: &Path, name: &str, relay_url: &str) -> Config {
        index: IndexSection {
            data_dir: dir.join("data").display().to_string(),
        },
+        matching: Default::default(),
    };
    config.save_key(&Keypair::generate()).unwrap();
    config