fix(core,fuse): compact the search indexes and stop pruning after partial scans

FTS5 keeps deleted entries until their segments are merged, which with
these indexes is close to never. On one machine local_fts held 317 MiB for
40,000 paths, against 20 MiB rebuilt, and search ran four times slower. The
indexes are now compacted once a day after a local scan. A scan that hit the
entry cap or lost a batch no longer prunes the paths it never reached, which
the next scan used to add back (B217).

crates/pdfs-core/src/db/local.rs

@@ -111,9 +111,80 @@ impl Db {
         )?;
         let count: i64 = tx.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?;
         tx.commit()?;
+        drop(conn);
+        self.compact_search_indexes_if_due(finished_at)?;
         Ok(count)
     }
 
+    /// Close a scan that did not see everything: it stopped at the entry cap,
+    /// or a batch failed to write. Nothing is pruned, since a path it missed is
+    /// not a path that is gone; the next complete scan prunes. Returns the
+    /// number of indexed entries.
+    ///
+    /// Pruning after such a scan dropped every entry it had not reached, and
+    /// the next scan put them back. That churn is most of what grew
+    /// `local_fts` to 16 times its size (`docs/BUGS.md` B217).
+    pub fn local_finish_partial_scan(&self, finished_at: i64) -> Result<i64> {
+        let conn = self.conn.lock();
+        conn.execute(
+            "INSERT INTO sync_state (key, value) VALUES ('local_indexed_at', ?1)
+             ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+            [finished_at.to_string()],
+        )?;
+        let count: i64 = conn.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?;
+        Ok(count)
+    }
+
+    /// Compact both full-text indexes, at most once a day.
+    ///
+    /// FTS5 keeps a deleted entry's data until the segments holding it are
+    /// merged, and with the few large segments these indexes settle into that
+    /// is close to never. Every file the local scan sees disappear or appear
+    /// leaves some behind, and every node row the tree replaces does too: on
+    /// one machine `local_fts` held 317 MiB for 40,000 paths, against 20 MiB
+    /// rebuilt, and a search took four times as long (`docs/BUGS.md` B217).
+    ///
+    /// `local_fts` is rebuilt from `local_files`, which also drops anything
+    /// left over, and `nodes_fts`, which has no content table to rebuild from,
+    /// is merged into one segment. Each runs in its own transaction on the
+    /// write connection, about half a second on an index that size; once a
+    /// day keeps the leftovers small without paying that at every scan.
+    pub fn compact_search_indexes_if_due(&self, now: i64) -> Result<()> {
+        const EVERY_SECS: i64 = 24 * 60 * 60;
+        let last: i64 = self
+            .read()
+            .query_row(
+                "SELECT value FROM sync_state WHERE key = 'fts_compacted_at'",
+                [],
+                |row| row.get::<_, String>(0),
+            )
+            .optional()?
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(0);
+        if now - last < EVERY_SECS {
+            return Ok(());
+        }
+        let started = std::time::Instant::now();
+        {
+            let conn = self.conn.lock();
+            conn.execute("INSERT INTO local_fts(local_fts) VALUES('rebuild')", [])?;
+        }
+        {
+            let conn = self.conn.lock();
+            conn.execute("INSERT INTO nodes_fts(nodes_fts) VALUES('optimize')", [])?;
+            conn.execute(
+                "INSERT INTO sync_state (key, value) VALUES ('fts_compacted_at', ?1)
+                 ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+                [now.to_string()],
+            )?;
+        }
+        tracing::info!(
+            elapsed_ms = started.elapsed().as_millis() as u64,
+            "compacted the search indexes"
+        );
+        Ok(())
+    }
+
     /// When the last local scan completed (epoch seconds), or `None` if the index
     /// has never been built. The daemon uses this to decide whether a fresh mount
     /// needs an immediate rescan or can serve the existing index.

crates/pdfs-core/src/db/tests.rs

@@ -1429,6 +1429,77 @@ fn rescanning_an_unchanged_file_keeps_exactly_one_index_entry() {
     }
 }
 
+/// A scan that stopped early or lost a batch did not see every file, so it
+/// must not prune the ones it missed (`docs/BUGS.md` B217).
+#[test]
+fn a_partial_local_scan_prunes_nothing() {
+    let db = Db::open_in_memory().unwrap();
+    let g = db.local_begin_scan().unwrap();
+    db.local_upsert_batch(
+        g,
+        &[
+            local("/home/u/a.txt", "a.txt", false),
+            local("/home/u/b.txt", "b.txt", false),
+        ],
+    )
+    .unwrap();
+    assert_eq!(db.local_finish_scan(g, 1).unwrap(), 2);
+
+    let g = db.local_begin_scan().unwrap();
+    db.local_upsert_batch(g, &[local("/home/u/a.txt", "a.txt", false)])
+        .unwrap();
+    assert_eq!(db.local_finish_partial_scan(2).unwrap(), 2);
+    assert_eq!(db.search_local("b.txt", 10).unwrap().len(), 1);
+    assert_eq!(db.local_indexed_at().unwrap(), Some(2));
+}
+
+/// Deleted entries pile up in an FTS5 index until it is compacted. The daily
+/// compaction must leave both indexes whole and searchable, and must not run
+/// again the same day (`docs/BUGS.md` B217).
+#[test]
+fn compacting_the_search_indexes_keeps_them_whole_and_runs_once_a_day() {
+    let db = Db::open_in_memory().unwrap();
+    db.upsert_node(&folder("root", None, "My Files")).unwrap();
+    db.upsert_node(&file("f1", "root", "quarterly-report.pdf", 1))
+        .unwrap();
+    for round in 0..20 {
+        let g = db.local_begin_scan().unwrap();
+        let name = format!("churn-{round}.txt");
+        db.local_upsert_batch(
+            g,
+            &[
+                local("/home/u/keep.txt", "keep.txt", false),
+                local(&format!("/home/u/{name}"), &name, false),
+            ],
+        )
+        .unwrap();
+        // Far in the past, so no scan compacts on its own.
+        db.local_finish_scan(g, 0).unwrap();
+    }
+
+    let day = 24 * 60 * 60;
+    db.compact_search_indexes_if_due(day).unwrap();
+    {
+        let conn = db.conn.lock();
+        for table in ["local_fts", "nodes_fts"] {
+            conn.execute(
+                &format!("INSERT INTO {table}({table}) VALUES('integrity-check')"),
+                [],
+            )
+            .unwrap();
+        }
+    }
+    assert_eq!(db.search_local("keep", 10).unwrap().len(), 1);
+    assert_eq!(db.search_local("churn-19", 10).unwrap().len(), 1);
+    assert!(db.search_local("churn-3.txt", 10).unwrap().is_empty());
+    assert_eq!(db.search("quarterly", 10).unwrap().len(), 1);
+    let stamp = |db: &Db| db.state_str("fts_compacted_at").unwrap();
+    assert_eq!(stamp(&db).as_deref(), Some(day.to_string().as_str()));
+
+    db.compact_search_indexes_if_due(day + 60).unwrap();
+    assert_eq!(stamp(&db).as_deref(), Some(day.to_string().as_str()));
+}
+
 /// Queries below the trigram minimum still match, via the `LIKE` fallback.
 #[test]
 fn local_search_short_query_like_fallback() {

crates/pdfs-core/src/localindex.rs

@@ -42,7 +42,7 @@ const SKIP_PATH_SUFFIXES: &[&str] = &["go/pkg/mod"];
 
 /// Hard cap on indexed entries, so a pathological home directory cannot grow the
 /// database without bound. Reaching it stops the walk early.
-const MAX_ENTRIES: usize = 400_000;
+pub const MAX_ENTRIES: usize = 400_000;
 
 /// Rows handed to the sink at a time. Large enough that the per-transaction cost
 /// amortises, small enough that the writer lock is released often (FUSE

crates/pdfs-fuse/src/background.rs

@@ -723,16 +723,24 @@ fn scan_local_once(
     // has seen and stays indeterminate.
     let job = transfers.begin_background_job("Indexing this computer");
     job.detail("Scanning your files");
+    let mut failed = false;
     let walked = localindex::scan(&[home], &excludes, |batch| {
         if let Err(e) = db.local_upsert_batch(generation, &batch) {
             warn!(error = %e, "local index: batch write failed");
+            failed = true;
         }
     });
 
-    // Prune what this scan did not see and rebuild the FTS index over the rest,
-    // even if some batches failed — a partial index still beats none.
+    // Prune what this scan did not see, unless it did not see everything:
+    // a path it never reached, or one whose batch failed to write, is not a
+    // path that is gone (`docs/BUGS.md` B217).
     job.detail("Building the search index");
-    match db.local_finish_scan(generation, now_secs()) {
+    let finished = if failed || walked >= localindex::MAX_ENTRIES {
+        db.local_finish_partial_scan(now_secs())
+    } else {
+        db.local_finish_scan(generation, now_secs())
+    };
+    match finished {
         Ok(indexed) => info!(
             walked,
             indexed,