fixed long getattr load times with ls -l

crates/pdfs-core/src/db/migrations.rs

@@ -9,7 +9,7 @@ use super::Db;
 use crate::Result;
 
 /// Current schema version. Bump on every forward migration added below.
-pub(super) const SCHEMA_VERSION: i64 = 14;
+pub(super) const SCHEMA_VERSION: i64 = 15;
 
 impl Db {
     pub(super) fn migrate(&self) -> Result<()> {
@@ -78,6 +78,9 @@ impl Db {
         if current < 14 {
             tx.execute_batch(MIGRATION_V14)?;
         }
+        if current < 15 {
+            tx.execute_batch(MIGRATION_V15)?;
+        }
         tx.execute(
             "INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
              ON CONFLICT(key) DO UPDATE SET value = excluded.value",
@@ -340,3 +343,29 @@ ALTER TABLE photos ADD COLUMN kind INTEGER NOT NULL DEFAULT 0;
 const MIGRATION_V14: &str = "
 CREATE INDEX cache_entries_lru ON cache_entries(kind, last_accessed);
 ";
+
+/// Schema v15: key the search index by rowid so a row can be replaced without
+/// scanning the whole index (B12).
+///
+/// v3 built `nodes_fts` with the node's `uid` as an `UNINDEXED` column and
+/// `upsert_nodes` refreshed a row with `DELETE FROM nodes_fts WHERE uid = ?`.
+/// `UNINDEXED` means exactly what it says — the column is retrievable but not
+/// searchable — so that predicate could only ever be a full scan of the index
+/// (`EXPLAIN QUERY PLAN`: `SCAN nodes_fts VIRTUAL TABLE`). Every node written
+/// scanned every node indexed: measured at 6.5 ms per node against a 17k-node
+/// index, about half the cost of a cold listing, and *growing with the size of
+/// the account* rather than the size of the listing.
+///
+/// FTS5 deletes by `rowid` efficiently, and `nodes` is an ordinary rowid table
+/// (`uid` is its `TEXT PRIMARY KEY`, not `WITHOUT ROWID`), so its rowid is a
+/// stable per-node key that `ON CONFLICT DO UPDATE` preserves. Rebuild the index
+/// keyed by it and drop the `uid` column: searches now join on `rowid`, which is
+/// also a cheaper join than the old one on a TEXT uid.
+const MIGRATION_V15: &str = "
+DROP TABLE IF EXISTS nodes_fts;
+CREATE VIRTUAL TABLE nodes_fts USING fts5(
+  name, tokenize='trigram'
+);
+INSERT INTO nodes_fts (rowid, name)
+  SELECT rowid, name FROM nodes WHERE trashed = 0;
+";

crates/pdfs-core/src/db/nodes.rs

@@ -70,11 +70,22 @@ impl Db {
             // FTS5 has no UPSERT, so refresh the row by delete-then-insert.
             // Trashed nodes are kept out of the index entirely so they never
             // surface in search results.
-            tx.execute("DELETE FROM nodes_fts WHERE uid = ?1", params![uid])?;
+            //
+            // Keyed by the node's rowid, never by its uid: the uid is not an
+            // indexed FTS column, so deleting by it scans the entire index once
+            // per node written (B12). The rowid is stable across the upsert
+            // above — `ON CONFLICT DO UPDATE` keeps the existing row — so it is
+            // the same key before and after.
+            let rowid: i64 = tx.query_row(
+                "SELECT rowid FROM nodes WHERE uid = ?1",
+                params![uid],
+                |row| row.get(0),
+            )?;
+            tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![rowid])?;
             if !node.trashed {
                 tx.execute(
-                    "INSERT INTO nodes_fts (uid, name) VALUES (?1, ?2)",
-                    params![uid, node.name],
+                    "INSERT INTO nodes_fts (rowid, name) VALUES (?1, ?2)",
+                    params![rowid, node.name],
                 )?;
             }
         }
@@ -88,8 +99,21 @@ impl Db {
         let uid = uid.to_string();
         let mut conn = self.conn.lock();
         let tx = conn.transaction()?;
+        // Read the rowid before dropping the node: it is the search index's key
+        // (B12), and it is gone once the row is. A node with no row leaves
+        // nothing to unindex — the index is keyed off `nodes`, so it cannot
+        // hold an entry the table never had.
+        let rowid: Option<i64> = tx
+            .query_row(
+                "SELECT rowid FROM nodes WHERE uid = ?1",
+                params![uid],
+                |row| row.get(0),
+            )
+            .optional()?;
         tx.execute("DELETE FROM nodes WHERE uid = ?1", params![uid])?;
-        tx.execute("DELETE FROM nodes_fts WHERE uid = ?1", params![uid])?;
+        if let Some(rowid) = rowid {
+            tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![rowid])?;
+        }
         tx.commit()?;
         Ok(())
     }
@@ -123,7 +147,7 @@ impl Db {
                 .join(" AND ");
             let mut stmt = conn.prepare(
                 "SELECT n.node_json, n.uid
-                 FROM nodes_fts f JOIN nodes n ON n.uid = f.uid
+                 FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
                  WHERE f.name MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
                  ORDER BY f.rank LIMIT ?2",
             )?;

crates/pdfs-core/src/db/tests.rs

@@ -564,6 +564,78 @@ fn search_drops_fts_row_on_delete_and_trash() {
     assert_eq!(db.search("unique", 10).unwrap().len(), 0);
 }
 
+/// Renaming re-indexes the node, which is the case delete-by-rowid has to get
+/// right: the *old* name must leave the index. A stale row here would be
+/// invisible in every other test — the new name is findable either way.
+#[test]
+fn renaming_a_node_replaces_the_name_in_the_search_index() {
+    let db = Db::open_in_memory().unwrap();
+    db.upsert_node(&folder("root", None, "My Files")).unwrap();
+    db.upsert_node(&file("f1", "root", "beforename.txt", 1))
+        .unwrap();
+    assert_eq!(db.search("beforename", 10).unwrap().len(), 1);
+
+    db.upsert_node(&file("f1", "root", "aftername.txt", 1))
+        .unwrap();
+
+    assert_eq!(
+        db.search("aftername", 10).unwrap().len(),
+        1,
+        "the new name must be searchable"
+    );
+    assert_eq!(
+        db.search("beforename", 10).unwrap().len(),
+        0,
+        "the old name must have left the index, not merely been added alongside"
+    );
+}
+
+/// B12: the cost of writing a listing must not scale with how much is already
+/// indexed. The old `DELETE FROM nodes_fts WHERE uid = ?` scanned the whole
+/// index once per node written, so a large account paid more to write the *same*
+/// folder — measured at 6.5 ms per node against a 17k-node index on the live DB.
+///
+/// Written as a ratio between two index sizes rather than an absolute time, so
+/// it fails on the scaling regression and not on a slow machine.
+#[test]
+fn writing_nodes_does_not_scale_with_the_size_of_the_index() {
+    use std::time::Instant;
+
+    /// Fill `db` with `n` indexed nodes, then time re-writing a fixed batch.
+    fn cost_against_index_of(n: usize) -> std::time::Duration {
+        let db = Db::open_in_memory().unwrap();
+        db.upsert_node(&folder("root", None, "My Files")).unwrap();
+        let filler: Vec<_> = (0..n)
+            .map(|i| file(&format!("bg{i}"), "root", &format!("background{i}.txt"), 1))
+            .collect();
+        db.upsert_nodes(&filler).unwrap();
+
+        // The batch under test is identical in both runs; only the amount of
+        // already-indexed data around it differs.
+        let batch: Vec<_> = (0..200)
+            .map(|i| file(&format!("m{i}"), "root", &format!("measured{i}.txt"), 1))
+            .collect();
+        db.upsert_nodes(&batch).unwrap();
+
+        let t = Instant::now();
+        db.upsert_nodes(&batch).unwrap();
+        t.elapsed()
+    }
+
+    let small = cost_against_index_of(200);
+    let large = cost_against_index_of(5000);
+
+    // Calibration, measured while writing this: over the same 25× growth in
+    // index size, the old full-scan path cost 6.4× more (46 ms → 294 ms) while
+    // the rowid path costs 1.04× (39 ms → 41 ms). 3× sits well clear of both.
+    println!("B12: 200-node write — against 200 indexed {small:?}, against 5000 {large:?}");
+    assert!(
+        large < small * 3,
+        "writing a listing got dramatically more expensive as the index grew, \
+         which is the B12 full-scan regression; small={small:?} large={large:?}"
+    );
+}
+
 fn activity(target: &str, kind: ActivityKind, ok: bool) -> ActivityEntry {
     ActivityEntry {
         time: 1700,