diff --git a/crates/pdfs-core/src/db/local.rs b/crates/pdfs-core/src/db/local.rs index f5f4ae2..f15eb22 100644 --- a/crates/pdfs-core/src/db/local.rs +++ b/crates/pdfs-core/src/db/local.rs @@ -111,9 +111,80 @@ impl Db { )?; let count: i64 = tx.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?; tx.commit()?; + drop(conn); + self.compact_search_indexes_if_due(finished_at)?; Ok(count) } + /// Close a scan that did not see everything: it stopped at the entry cap, + /// or a batch failed to write. Nothing is pruned, since a path it missed is + /// not a path that is gone; the next complete scan prunes. Returns the + /// number of indexed entries. + /// + /// Pruning after such a scan dropped every entry it had not reached, and + /// the next scan put them back. That churn is most of what grew + /// `local_fts` to 16 times its size (`docs/BUGS.md` B217). + pub fn local_finish_partial_scan(&self, finished_at: i64) -> Result { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO sync_state (key, value) VALUES ('local_indexed_at', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [finished_at.to_string()], + )?; + let count: i64 = conn.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?; + Ok(count) + } + + /// Compact both full-text indexes, at most once a day. + /// + /// FTS5 keeps a deleted entry's data until the segments holding it are + /// merged, and with the few large segments these indexes settle into that + /// is close to never. Every file the local scan sees disappear or appear + /// leaves some behind, and every node row the tree replaces does too: on + /// one machine `local_fts` held 317 MiB for 40,000 paths, against 20 MiB + /// rebuilt, and a search took four times as long (`docs/BUGS.md` B217). + /// + /// `local_fts` is rebuilt from `local_files`, which also drops anything + /// left over, and `nodes_fts`, which has no content table to rebuild from, + /// is merged into one segment. Each runs in its own transaction on the + /// write connection, about half a second on an index that size; once a + /// day keeps the leftovers small without paying that at every scan. + pub fn compact_search_indexes_if_due(&self, now: i64) -> Result<()> { + const EVERY_SECS: i64 = 24 * 60 * 60; + let last: i64 = self + .read() + .query_row( + "SELECT value FROM sync_state WHERE key = 'fts_compacted_at'", + [], + |row| row.get::<_, String>(0), + ) + .optional()? + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + if now - last < EVERY_SECS { + return Ok(()); + } + let started = std::time::Instant::now(); + { + let conn = self.conn.lock(); + conn.execute("INSERT INTO local_fts(local_fts) VALUES('rebuild')", [])?; + } + { + let conn = self.conn.lock(); + conn.execute("INSERT INTO nodes_fts(nodes_fts) VALUES('optimize')", [])?; + conn.execute( + "INSERT INTO sync_state (key, value) VALUES ('fts_compacted_at', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [now.to_string()], + )?; + } + tracing::info!( + elapsed_ms = started.elapsed().as_millis() as u64, + "compacted the search indexes" + ); + Ok(()) + } + /// When the last local scan completed (epoch seconds), or `None` if the index /// has never been built. The daemon uses this to decide whether a fresh mount /// needs an immediate rescan or can serve the existing index. diff --git a/crates/pdfs-core/src/db/tests.rs b/crates/pdfs-core/src/db/tests.rs index 8be06bc..55a586c 100644 --- a/crates/pdfs-core/src/db/tests.rs +++ b/crates/pdfs-core/src/db/tests.rs @@ -1429,6 +1429,77 @@ fn rescanning_an_unchanged_file_keeps_exactly_one_index_entry() { } } +/// A scan that stopped early or lost a batch did not see every file, so it +/// must not prune the ones it missed (`docs/BUGS.md` B217). +#[test] +fn a_partial_local_scan_prunes_nothing() { + let db = Db::open_in_memory().unwrap(); + let g = db.local_begin_scan().unwrap(); + db.local_upsert_batch( + g, + &[ + local("/home/u/a.txt", "a.txt", false), + local("/home/u/b.txt", "b.txt", false), + ], + ) + .unwrap(); + assert_eq!(db.local_finish_scan(g, 1).unwrap(), 2); + + let g = db.local_begin_scan().unwrap(); + db.local_upsert_batch(g, &[local("/home/u/a.txt", "a.txt", false)]) + .unwrap(); + assert_eq!(db.local_finish_partial_scan(2).unwrap(), 2); + assert_eq!(db.search_local("b.txt", 10).unwrap().len(), 1); + assert_eq!(db.local_indexed_at().unwrap(), Some(2)); +} + +/// Deleted entries pile up in an FTS5 index until it is compacted. The daily +/// compaction must leave both indexes whole and searchable, and must not run +/// again the same day (`docs/BUGS.md` B217). +#[test] +fn compacting_the_search_indexes_keeps_them_whole_and_runs_once_a_day() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&file("f1", "root", "quarterly-report.pdf", 1)) + .unwrap(); + for round in 0..20 { + let g = db.local_begin_scan().unwrap(); + let name = format!("churn-{round}.txt"); + db.local_upsert_batch( + g, + &[ + local("/home/u/keep.txt", "keep.txt", false), + local(&format!("/home/u/{name}"), &name, false), + ], + ) + .unwrap(); + // Far in the past, so no scan compacts on its own. + db.local_finish_scan(g, 0).unwrap(); + } + + let day = 24 * 60 * 60; + db.compact_search_indexes_if_due(day).unwrap(); + { + let conn = db.conn.lock(); + for table in ["local_fts", "nodes_fts"] { + conn.execute( + &format!("INSERT INTO {table}({table}) VALUES('integrity-check')"), + [], + ) + .unwrap(); + } + } + assert_eq!(db.search_local("keep", 10).unwrap().len(), 1); + assert_eq!(db.search_local("churn-19", 10).unwrap().len(), 1); + assert!(db.search_local("churn-3.txt", 10).unwrap().is_empty()); + assert_eq!(db.search("quarterly", 10).unwrap().len(), 1); + let stamp = |db: &Db| db.state_str("fts_compacted_at").unwrap(); + assert_eq!(stamp(&db).as_deref(), Some(day.to_string().as_str())); + + db.compact_search_indexes_if_due(day + 60).unwrap(); + assert_eq!(stamp(&db).as_deref(), Some(day.to_string().as_str())); +} + /// Queries below the trigram minimum still match, via the `LIKE` fallback. #[test] fn local_search_short_query_like_fallback() { diff --git a/crates/pdfs-core/src/localindex.rs b/crates/pdfs-core/src/localindex.rs index 8335ea8..c77996f 100644 --- a/crates/pdfs-core/src/localindex.rs +++ b/crates/pdfs-core/src/localindex.rs @@ -42,7 +42,7 @@ const SKIP_PATH_SUFFIXES: &[&str] = &["go/pkg/mod"]; /// Hard cap on indexed entries, so a pathological home directory cannot grow the /// database without bound. Reaching it stops the walk early. -const MAX_ENTRIES: usize = 400_000; +pub const MAX_ENTRIES: usize = 400_000; /// Rows handed to the sink at a time. Large enough that the per-transaction cost /// amortises, small enough that the writer lock is released often (FUSE diff --git a/crates/pdfs-fuse/src/background.rs b/crates/pdfs-fuse/src/background.rs index 64afdd2..9026a4c 100644 --- a/crates/pdfs-fuse/src/background.rs +++ b/crates/pdfs-fuse/src/background.rs @@ -723,16 +723,24 @@ fn scan_local_once( // has seen and stays indeterminate. let job = transfers.begin_background_job("Indexing this computer"); job.detail("Scanning your files"); + let mut failed = false; let walked = localindex::scan(&[home], &excludes, |batch| { if let Err(e) = db.local_upsert_batch(generation, &batch) { warn!(error = %e, "local index: batch write failed"); + failed = true; } }); - // Prune what this scan did not see and rebuild the FTS index over the rest, - // even if some batches failed — a partial index still beats none. + // Prune what this scan did not see, unless it did not see everything: + // a path it never reached, or one whose batch failed to write, is not a + // path that is gone (`docs/BUGS.md` B217). job.detail("Building the search index"); - match db.local_finish_scan(generation, now_secs()) { + let finished = if failed || walked >= localindex::MAX_ENTRIES { + db.local_finish_partial_scan(now_secs()) + } else { + db.local_finish_scan(generation, now_secs()) + }; + match finished { Ok(indexed) => info!( walked, indexed, diff --git a/docs/BUGS.md b/docs/BUGS.md index 201193b..0a470f8 100644 --- a/docs/BUGS.md +++ b/docs/BUGS.md @@ -100,6 +100,7 @@ a live account since. | Entry | Title | |---|---| +| [B217](#b217--the-search-indexes-grow-to-many-times-their-size-and-slow-search-down) | The search indexes grow to many times their size and slow search down | | [B216](#b216--a-damaged-database-is-rebuilt-without-the-writes-still-queued-in-it) | A damaged database is rebuilt without the writes still queued in it | | [B215](#b215--a-remote-name-holding--or--becomes-a-path-outside-the-sync-folder) | A remote name holding `/` or `..` becomes a path outside the sync folder | | [B214](#b214--deleting-a-mirror-folder-here-trashes-another-devices-new-files-in-it) | Deleting a mirror folder here trashes another device's new files in it | @@ -319,6 +320,33 @@ a live account since. Newest first. +### B217 — The search indexes grow to many times their size and slow search down + +**Status:** Fixed, not verified live +**Found:** 2026-10-16, architecture audit (`FINDINGS.md` 2.2.1 and 2.2.3). On a live 567 MiB +`cache.db`, `local_fts` held 317 MiB for 40,000 indexed paths. Rebuilt, it was 20 MiB, and a +search candidate query went from 27 ms to 7 ms. `nodes_fts` shrank from 21 to 16 MiB. +**Where:** `Db::local_finish_scan`, `Db::compact_search_indexes_if_due` in +`crates/pdfs-core/src/db/local.rs`; `index_local_files` in `crates/pdfs-fuse/src/background.rs` + +**Repro:** Run the daemon for weeks with a home directory where files come and go, then compare +the size of `local_fts_data` with what a `rebuild` leaves. + +**Cause.** FTS5 keeps a deleted entry's data until the segments holding it are merged, and with +the few large segments these indexes settle into, that is close to never. The client never ran +`optimize` or `rebuild`. The local scan made it worse: a scan that stopped at the entry cap, or +lost a batch, still pruned every path it had not reached, and the next scan put them all back. + +**Fix.** The search indexes are compacted at most once a day, at the end of a local scan: +`local_fts` is rebuilt from `local_files`, and `nodes_fts` is merged into one segment. Each takes +about half a second at that size. A scan that did not see everything closes with +`local_finish_partial_scan`, which prunes nothing. + +**Test:** `db::tests::compacting_the_search_indexes_keeps_them_whole_and_runs_once_a_day`, +`db::tests::a_partial_local_scan_prunes_nothing`. + +--- + ### B216 — A damaged database is rebuilt without the writes still queued in it **Status:** Fixed, not verified live