fix(fs) performance, locks

crates/pdfs-core/src/db/cache.rs

@@ -42,6 +42,30 @@ impl Db {
         Ok(())
     }
 
+    /// Bump `last_accessed` for many keys in one transaction.
+    ///
+    /// The read path touches a key per 128 KiB the kernel asks for — 32 times
+    /// per 4 MiB block — and each of those was its own write transaction under
+    /// the single connection mutex, contending with every `lookup` and
+    /// `readdir`. Callers accumulate touches in memory and flush them here.
+    pub fn cache_accessed_batch(&self, touches: &[(String, i64)]) -> Result<()> {
+        if touches.is_empty() {
+            return Ok(());
+        }
+        let mut conn = self.conn.lock();
+        let tx = conn.transaction()?;
+        {
+            let mut stmt = tx.prepare_cached(
+                "UPDATE cache_entries SET last_accessed = ?2 WHERE cache_key = ?1",
+            )?;
+            for (key, at) in touches {
+                stmt.execute(params![key, at])?;
+            }
+        }
+        tx.commit()?;
+        Ok(())
+    }
+
     /// Drop a single cache-entry row (one evicted blob or block).
     pub fn cache_remove(&self, key: &str) -> Result<()> {
         let conn = self.conn.lock();

crates/pdfs-core/src/db/migrations.rs

@@ -9,7 +9,7 @@ use super::Db;
 use crate::Result;
 
 /// Current schema version. Bump on every forward migration added below.
-pub(super) const SCHEMA_VERSION: i64 = 21;
+pub(super) const SCHEMA_VERSION: i64 = 22;
 
 impl Db {
     pub(super) fn migrate(&self) -> Result<()> {
@@ -113,6 +113,24 @@ impl Db {
                 tx.execute_batch(MIGRATION_V21)?;
             }
         }
+        if current < 22 {
+            // Older fixtures (and a database that never grew a table because
+            // its feature was never used) may be missing any of these, and an
+            // index on an absent table is a hard error rather than a no-op.
+            for (table, sql) in MIGRATION_V22 {
+                let present: bool = tx.query_row(
+                    "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
+                    [table],
+                    |row| row.get::<_, i64>(0),
+                )? > 0;
+                if present {
+                    tx.execute_batch(sql)?;
+                }
+            }
+            // `activity_time` indexes `time DESC` while both activity queries
+            // order by `id`: it has only ever cost writes.
+            tx.execute_batch("DROP INDEX IF EXISTS activity_time;")?;
+        }
         tx.execute(
             "INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
              ON CONFLICT(key) DO UPDATE SET value = excluded.value",
@@ -583,3 +601,35 @@ SELECT n.rowid, n.name, paths.path
   FROM nodes n JOIN paths ON paths.rowid = n.rowid
  WHERE paths.trashed = 0;
 ";
+
+/// V22: indexes for the lookups that were scanning, each paired with the table
+/// it needs to exist for.
+///
+/// Each backs a query on a hot path that had no index to use:
+///
+/// - `sync_entry(remote_uid)` — the reconcile resolves a remote uid to its
+///   local entry once per node, per pass.
+/// - `local_files(scan_gen)` — the sweep that retires the previous generation
+///   after a local index scan.
+/// - `nodes(parent_uid) WHERE trashed = 0` — every listing, with the partial
+///   index keeping it to the live rows a listing actually wants.
+///
+/// Deliberately *not* here: an index on `pending_op(next_attempt_at, id)`.
+/// `next_due_op` orders by `id`, which is the rowid, so it already stops at the
+/// first due row; offering the planner a `next_attempt_at` index makes it scan
+/// that index and then sort the matches by `id` instead —
+/// `next_due_op_does_not_scale_with_queue_length` measures the regression.
+const MIGRATION_V22: [(&str, &str); 3] = [
+    (
+        "sync_entry",
+        "CREATE INDEX IF NOT EXISTS idx_sync_entry_remote_uid ON sync_entry(remote_uid);",
+    ),
+    (
+        "local_files",
+        "CREATE INDEX IF NOT EXISTS idx_local_files_scan_gen ON local_files(scan_gen);",
+    ),
+    (
+        "nodes",
+        "CREATE INDEX IF NOT EXISTS idx_nodes_parent_live ON nodes(parent_uid) WHERE trashed = 0;",
+    ),
+];

crates/pdfs-core/src/db/mod.rs

@@ -66,6 +66,20 @@ const WAL_SIZE_LIMIT: i64 = 64 * 1024 * 1024;
 /// genuinely stuck holder surfaces as an error rather than an apparent hang.
 const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
 
+/// Page cache, in KiB (applied negated, which is how SQLite reads a size in KiB
+/// rather than in pages). 32 MiB against a database that is typically tens of
+/// MiB: large enough to hold the working set of a listing-heavy session, small
+/// enough to be unremarkable in a desktop daemon's RSS.
+const CACHE_SIZE_KIB: i64 = 32 * 1024;
+
+/// Ceiling on the memory-mapped window over the database file. 256 MiB covers
+/// any realistic index while keeping the mapping bounded.
+const MMAP_SIZE: i64 = 256 * 1024 * 1024;
+
+/// WAL pages between automatic checkpoints (default 1000, i.e. ~4 MiB). Halved
+/// so the synchronous checkpoint a committing thread inherits stays short.
+const WAL_AUTOCHECKPOINT_PAGES: i64 = 500;
+
 /// Handle to the unified metadata database.
 ///
 /// Cheap to wrap in an `Arc`; clone the `Arc`, not this. All access goes through
@@ -102,6 +116,24 @@ impl Db {
         // (Bare SQLite does default to 0, which is where the belief that this
         // was unset came from. That default is not what we get.)
         conn.busy_timeout(BUSY_TIMEOUT)?;
+        // Page cache. The default is 2 MiB, against a schema whose hot table
+        // stores a JSON blob per node — a listing of a few thousand files
+        // evicts itself while it is being read.
+        conn.pragma_update(None, "cache_size", -CACHE_SIZE_KIB)?;
+        // Sorts and recursive CTEs materialise their intermediate results.
+        // Every `ORDER BY … COLLATE NOCASE` listing and every ancestor walk
+        // spills those to a file in `/tmp` by default; they are small and
+        // short-lived, so memory is both faster and less surprising.
+        conn.pragma_update(None, "temp_store", "MEMORY")?;
+        // Read the database through the page cache of the OS rather than
+        // `pread` per page. Bounded rather than unlimited so the daemon's RSS
+        // does not track the database size.
+        conn.pragma_update(None, "mmap_size", MMAP_SIZE)?;
+        // Checkpointing is synchronous in whichever thread happens to commit
+        // the transaction that crosses the threshold — often a FUSE callback.
+        // A smaller autocheckpoint makes that debt smaller and more frequent
+        // rather than rare and multi-second.
+        conn.pragma_update(None, "wal_autocheckpoint", WAL_AUTOCHECKPOINT_PAGES)?;
 
         let db = Self {
             conn: Mutex::new(conn),

crates/pdfs-core/src/db/pins.rs

@@ -4,6 +4,7 @@
 use rusqlite::{OptionalExtension, params};
 
 use super::Db;
+use super::utils::MAX_PATH_DEPTH;
 use crate::Result;
 
 /// One row of [`Db::pin_list`]. `recursive` is pin *policy* — whether the whole
@@ -66,6 +67,11 @@ impl Db {
     /// is pinned recursively. The ancestor check walks `parent_uid` to the root
     /// via a CTE; a direct pin is honoured even when the node has no `nodes` row
     /// yet (e.g. a CLI that never hydrates the node cache).
+    ///
+    /// The walk is depth-capped like [`super::utils::path_of`]: a `parent_uid`
+    /// cycle is corrupt data the API can hand us, and `UNION ALL` over one never
+    /// terminates — while holding the daemon's only SQLite connection, on a path
+    /// that runs per cached read.
     pub fn is_pinned(&self, uid: &str) -> Result<bool> {
         let conn = self.conn.lock();
         let direct: Option<i64> = conn
@@ -78,14 +84,18 @@ impl Db {
         }
         let anc: Option<i64> = conn
             .query_row(
-                "WITH RECURSIVE anc(uid, parent_uid) AS (
-                   SELECT uid, parent_uid FROM nodes WHERE uid = ?1
-                   UNION ALL
-                   SELECT n.uid, n.parent_uid FROM nodes n JOIN anc ON n.uid = anc.parent_uid
-                 )
-                 SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid
-                 WHERE a.uid != ?1 AND p.recursive = 1
-                 LIMIT 1",
+                &format!(
+                    "WITH RECURSIVE anc(uid, parent_uid, depth) AS (
+                       SELECT uid, parent_uid, 0 FROM nodes WHERE uid = ?1
+                       UNION ALL
+                       SELECT n.uid, n.parent_uid, anc.depth + 1
+                       FROM nodes n JOIN anc ON n.uid = anc.parent_uid
+                       WHERE anc.depth < {MAX_PATH_DEPTH}
+                     )
+                     SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid
+                     WHERE a.uid != ?1 AND p.recursive = 1
+                     LIMIT 1"
+                ),
                 params![uid],
                 |r| r.get(0),
             )

crates/pdfs-core/src/db/tests.rs

@@ -2628,6 +2628,26 @@ fn non_recursive_folder_pin_does_not_cover_children() {
     assert!(!db.is_pinned(&du("rep")).unwrap());
 }
 
+/// A `parent_uid` cycle is corrupt data the API can hand us, and `is_pinned`'s
+/// ancestor walk is `UNION ALL` — uncapped it never terminates, while holding
+/// the daemon's only SQLite connection, on a path that runs per cached read.
+/// The answer here matters less than the fact that there is one.
+#[test]
+fn is_pinned_terminates_on_a_parent_cycle() {
+    let db = Db::open_in_memory().unwrap();
+    let du = |l: &str| uid(l).to_string();
+    // a -> b -> a, with nothing pinned above either.
+    db.upsert_node(&folder("a", Some("b"), "A")).unwrap();
+    db.upsert_node(&folder("b", Some("a"), "B")).unwrap();
+    db.upsert_node(&file("leaf", "a", "leaf.txt", 1)).unwrap();
+
+    assert!(!db.is_pinned(&du("leaf")).unwrap());
+
+    // And a pin *inside* the cycle is still found rather than walked past.
+    db.pin_add(&du("a"), "A", true).unwrap();
+    assert!(db.is_pinned(&du("leaf")).unwrap());
+}
+
 #[test]
 fn schema_objects_exist() {
     let db = Db::open_in_memory().unwrap();

crates/pdfs-core/src/db/utils.rs

@@ -35,7 +35,7 @@ pub(super) fn like_escape(s: &str) -> String {
 /// nowhere near this; a parent *cycle* is unbounded, and this query is on the
 /// path of every search result, including the `LIKE` lane that reads `nodes`
 /// directly and so never passed through the index's cycle check.
-const MAX_PATH_DEPTH: usize = 256;
+pub(super) const MAX_PATH_DEPTH: usize = 256;
 
 /// Resolve a node's mountpoint-relative path by walking `parent_uid` to the
 /// root via a recursive CTE. The root (the node with no parent) is excluded, so