some performance optimizations

crates/pdfs-core/src/db/cache.rs

@@ -64,9 +64,49 @@ impl Db {
         Ok(())
     }
 
+    /// Total bytes held by entries of `kind`.
+    ///
+    /// The budget enforcers' first question, and almost always their last: the
+    /// common case is "under budget, evict nothing". Answering it with a `SUM`
+    /// costs one index range rather than materializing every row, which matters
+    /// because `enforce_block_budget` runs on every cached block — once per
+    /// 4 MiB of every cold read — while holding the shared connection.
+    pub fn cache_total_bytes(&self, kind: &str) -> Result<u64> {
+        let conn = self.conn.lock();
+        let total: i64 = conn.query_row(
+            "SELECT COALESCE(SUM(size_bytes), 0) FROM cache_entries WHERE kind = ?1",
+            params![kind],
+            |r| r.get(0),
+        )?;
+        Ok(total.max(0) as u64)
+    }
+
+    /// The `limit` least-recently-accessed entries of `kind`, oldest first.
+    ///
+    /// Eviction wants a handful of victims, not the whole table. Bounded so a
+    /// pass over a large cache does not read every row to drop a few, and so the
+    /// `(kind, last_accessed)` index can answer it as a range scan with an early
+    /// stop.
+    pub fn cache_eviction_candidates(&self, kind: &str, limit: usize) -> Result<Vec<(String, u64)>> {
+        let conn = self.conn.lock();
+        let mut stmt = conn.prepare_cached(
+            "SELECT cache_key, size_bytes FROM cache_entries
+             WHERE kind = ?1 ORDER BY last_accessed ASC LIMIT ?2",
+        )?;
+        let rows = stmt.query_map(params![kind, limit as i64], |r| {
+            Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?.max(0) as u64))
+        })?;
+        let mut out = Vec::new();
+        for r in rows {
+            out.push(r?);
+        }
+        Ok(out)
+    }
+
     /// Every cache entry of `kind`, ordered least-recently-accessed first, as
-    /// `(cache_key, size_bytes)`. The budget enforcer sums the sizes and evicts
-    /// from the front until the cache fits.
+    /// `(cache_key, size_bytes)`. Used by the whole-cache passes
+    /// (`clear_unpinned`); incremental eviction wants
+    /// [`cache_eviction_candidates`](Self::cache_eviction_candidates).
     pub fn cache_entries_by_kind(&self, kind: &str) -> Result<Vec<(String, u64)>> {
         let conn = self.conn.lock();
         let mut stmt = conn.prepare(

crates/pdfs-core/src/db/migrations.rs

@@ -9,7 +9,7 @@ use super::Db;
 use crate::Result;
 
 /// Current schema version. Bump on every forward migration added below.
-pub(super) const SCHEMA_VERSION: i64 = 13;
+pub(super) const SCHEMA_VERSION: i64 = 14;
 
 impl Db {
     pub(super) fn migrate(&self) -> Result<()> {
@@ -75,6 +75,9 @@ impl Db {
         if current < 13 {
             tx.execute_batch(MIGRATION_V13)?;
         }
+        if current < 14 {
+            tx.execute_batch(MIGRATION_V14)?;
+        }
         tx.execute(
             "INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
              ON CONFLICT(key) DO UPDATE SET value = excluded.value",
@@ -327,3 +330,13 @@ const MIGRATION_V13: &str = "
 ALTER TABLE photos ADD COLUMN media_type TEXT;
 ALTER TABLE photos ADD COLUMN kind INTEGER NOT NULL DEFAULT 0;
 ";
+
+/// Schema v14: index the content-cache LRU. `cache_entries` carried only its
+/// `cache_key` primary key, so every budget check and every eviction pass was a
+/// full scan plus a sort — on a path that runs once per cached 4 MiB block,
+/// under the connection lock every FUSE metadata call also needs. `(kind,
+/// last_accessed)` turns both into a range scan: the `SUM` reads one kind's
+/// slice, and eviction reads its front and stops.
+const MIGRATION_V14: &str = "
+CREATE INDEX cache_entries_lru ON cache_entries(kind, last_accessed);
+";

crates/pdfs-core/src/db/ops.rs

@@ -269,6 +269,63 @@ impl Db {
 
     /// Every queued op, oldest first. The drain worker replays them in this order
     /// so a file's writes land in the order they were made.
+    /// The oldest queued op that is due and not blocked, or `None`.
+    ///
+    /// This is the drain loop's per-iteration query, and it is deliberately not
+    /// `pending_ops().find(...)`. That variant read every row — including each
+    /// op's full `meta_json` — sorted them, and threw all but one away, once per
+    /// drained op *and* once per retry, while holding the single connection
+    /// mutex that every FUSE `lookup` also needs. A queue of a few thousand ops
+    /// (an ordinary offline `cp -r`) made draining quadratic in the queue length.
+    ///
+    /// Both filters are in SQL so the row is decided by the database rather than
+    /// materialized and rejected in Rust:
+    ///
+    /// * `next_attempt_at <= now` — the backoff.
+    /// * a `local~` parent is skipped. A node created inside a folder that was
+    ///   itself created offline cannot be sent anywhere until that folder is
+    ///   real. Ops replay in queue order so the parent normally drains first;
+    ///   this matters when the parent is backing off, where the child must wait
+    ///   rather than burn its own retries.
+    ///
+    /// `id` is `INTEGER PRIMARY KEY`, i.e. the rowid, so `ORDER BY id LIMIT 1`
+    /// walks the table in insertion order and stops at the first match — no
+    /// sort, and no separate index to maintain. The common case (something is
+    /// due) exits on the first row.
+    pub fn next_due_op(&self, now: i64) -> Result<Option<PendingOp>> {
+        let conn = self.conn.lock();
+        let mut stmt = conn.prepare_cached(&format!(
+            "SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
+                    attempts, last_error, next_attempt_at
+             FROM pending_op
+             WHERE next_attempt_at <= ?1
+               AND (parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~')
+             ORDER BY id LIMIT 1",
+            v = LOCAL_VOLUME,
+            n = LOCAL_VOLUME.len() + 1,
+        ))?;
+        let op = stmt
+            .query_row(params![now], |r| {
+                Ok(PendingOp {
+                    id: r.get(0)?,
+                    kind: r.get(1)?,
+                    uid: r.get(2)?,
+                    parent_uid: r.get(3)?,
+                    name: r.get(4)?,
+                    blob_path: r.get(5)?,
+                    meta_json: r.get(6)?,
+                    created_at: r.get(7)?,
+                    attempts: r.get(8)?,
+                    last_error: r.get(9)?,
+                    next_attempt_at: r.get(10)?,
+                })
+            })
+            .optional()?;
+        Ok(op)
+    }
+
+    /// Every queued op, oldest first. For status read-outs and the CLI — the
+    /// drain wants [`next_due_op`](Self::next_due_op) instead.
     pub fn pending_ops(&self) -> Result<Vec<PendingOp>> {
         let conn = self.conn.lock();
         let mut stmt = conn.prepare(

crates/pdfs-core/src/db/tests.rs

@@ -1061,3 +1061,257 @@ fn schema_objects_exist() {
         .unwrap();
     assert_eq!(count, 3);
 }
+
+/// A queued op carrying a realistically-sized `meta_json`, which is what made
+/// the old scan expensive: the drain read and parsed every op's sidecar to pick
+/// one.
+fn bulk_op(i: usize, parent: &str) -> PendingOp {
+    PendingOp {
+        id: 0,
+        kind: OP_REVISION.to_string(),
+        uid: format!("vol~link{i}"),
+        parent_uid: Some(parent.to_string()),
+        name: Some(format!("file-{i}.bin")),
+        blob_path: Some(format!("/staging/vol~link{i}-{i}")),
+        // A StagedWrite sidecar, roughly the shape and size of a real one.
+        meta_json: Some(format!(
+            r#"{{"uid":"vol~link{i}","len":8388608,"base_size":8388608,
+                 "base_mtime":1700000000,"authored":[[0,8388608]],"complete":true,
+                 "based_on":{{"mtime":1700000000,"size":8388608}}}}"#
+        )),
+        created_at: 1,
+        attempts: 0,
+        last_error: None,
+        next_attempt_at: 0,
+    }
+}
+
+/// **The B3 measurement.** The drain picks one op per iteration. Doing that by
+/// reading the whole queue is quadratic in queue length; doing it in SQL with
+/// `LIMIT 1` is flat. Asserts the shape of the difference, not a wall-clock
+/// number — the point is that one grows with the queue and the other does not.
+#[test]
+fn next_due_op_does_not_scale_with_queue_length() {
+    use std::time::Instant;
+
+    let db = Db::open_in_memory().unwrap();
+    let root = uid("root").to_string();
+    const N: usize = 2000;
+    for i in 0..N {
+        db.enqueue_op(&bulk_op(i, &root)).unwrap();
+    }
+
+    // Both must agree on which op is next.
+    let scanned = db
+        .pending_ops()
+        .unwrap()
+        .into_iter()
+        .find(|o| o.next_attempt_at <= 10);
+    let queried = db.next_due_op(10).unwrap();
+    assert_eq!(
+        scanned.as_ref().map(|o| o.id),
+        queried.as_ref().map(|o| o.id),
+        "the new query must pick the op the old scan picked"
+    );
+    assert_eq!(scanned.map(|o| o.uid), queried.map(|o| o.uid));
+
+    // Simulate a drain pass: pick the next op, retire it, repeat.
+    let rounds = 200;
+
+    let t0 = Instant::now();
+    for _ in 0..rounds {
+        let _ = db
+            .pending_ops()
+            .unwrap()
+            .into_iter()
+            .find(|o| o.next_attempt_at <= 10);
+    }
+    let scan = t0.elapsed();
+
+    let t1 = Instant::now();
+    for _ in 0..rounds {
+        let _ = db.next_due_op(10).unwrap();
+    }
+    let query = t1.elapsed();
+
+    println!("B3: {rounds} picks over a {N}-op queue — scan {scan:?}, query {query:?}");
+    assert!(
+        query * 20 < scan,
+        "expected the LIMIT 1 query to be far cheaper than a full scan; \
+         scan={scan:?} query={query:?}"
+    );
+}
+
+/// The readiness filter moved into SQL, so it needs its own coverage there: an
+/// op whose parent was itself created offline is not yet sendable and must be
+/// skipped in favour of a later one that is.
+#[test]
+fn next_due_op_skips_ops_blocked_on_a_local_parent() {
+    let db = Db::open_in_memory().unwrap();
+    let root = uid("root").to_string();
+
+    db.enqueue_op(&bulk_op(1, "local~dir")).unwrap();
+    db.enqueue_op(&bulk_op(2, &root)).unwrap();
+
+    let next = db.next_due_op(10).unwrap().expect("an op is due");
+    assert_eq!(next.uid, "vol~link2", "skipped the local-parent op");
+
+    // A NULL parent is not blocked.
+    let mut orphan = bulk_op(3, &root);
+    orphan.parent_uid = None;
+    db.enqueue_op(&orphan).unwrap();
+    db.delete_op(next.id).unwrap();
+    assert_eq!(db.next_due_op(10).unwrap().unwrap().uid, "vol~link3");
+}
+
+/// Backoff still gates: nothing is returned before an op is due.
+#[test]
+fn next_due_op_respects_backoff() {
+    let db = Db::open_in_memory().unwrap();
+    let root = uid("root").to_string();
+    let id = db.enqueue_op(&bulk_op(1, &root)).unwrap().0;
+    db.record_op_failure(id, "boom", 5_000).unwrap();
+
+    assert!(db.next_due_op(4_999).unwrap().is_none(), "still backing off");
+    assert!(db.next_due_op(5_000).unwrap().is_some(), "due now");
+}
+
+/// **The B4 measurement.** `enforce_block_budget` runs on *every* `store_block`,
+/// i.e. once per 4 MiB of every cold read. The old path read and sorted every
+/// row of the cache index to answer a question that is almost always "no, we are
+/// under budget" — under the shared connection lock, so it also stalled FUSE
+/// metadata calls.
+///
+/// Asserts the shape: the under-budget check must not scale with cache size.
+#[test]
+fn cache_total_bytes_does_not_scale_with_cache_size() {
+    use std::time::Instant;
+
+    let db = Db::open_in_memory().unwrap();
+    // A 20 GB block cache at 4 MiB a block is ~5000 rows.
+    const N: usize = 5000;
+    const BLOCK: u64 = 4 << 20;
+    for i in 0..N {
+        db.cache_touch(&format!("k{i}.b0"), "block", BLOCK, i as i64)
+            .unwrap();
+    }
+
+    // The aggregate must agree with summing the rows.
+    let summed: u64 = db
+        .cache_entries_by_kind("block")
+        .unwrap()
+        .iter()
+        .map(|(_, s)| s)
+        .sum();
+    assert_eq!(db.cache_total_bytes("block").unwrap(), summed);
+    assert_eq!(summed, N as u64 * BLOCK);
+    // Kinds are counted apart.
+    assert_eq!(db.cache_total_bytes("blob").unwrap(), 0);
+
+    let rounds = 500;
+
+    let t0 = Instant::now();
+    for _ in 0..rounds {
+        let entries = db.cache_entries_by_kind("block").unwrap();
+        let _: u64 = entries.iter().map(|(_, s)| *s).sum();
+    }
+    let scan = t0.elapsed();
+
+    let t1 = Instant::now();
+    for _ in 0..rounds {
+        let _ = db.cache_total_bytes("block").unwrap();
+    }
+    let aggregate = t1.elapsed();
+
+    println!("B4: {rounds} budget checks over {N} entries — scan {scan:?}, aggregate {aggregate:?}");
+    // The SUM is much cheaper — no row materialization, no allocation, no sort —
+    // but it is still O(rows): summing N values reads N index entries. That is
+    // why `ContentCache` keeps a running total on top of this and only falls
+    // back to the query when it has to evict. See
+    // `budget_check_is_free_when_under_budget`.
+    assert!(
+        aggregate * 2 < scan,
+        "expected the SUM to be materially cheaper than reading every row; \
+         scan={scan:?} aggregate={aggregate:?}"
+    );
+}
+
+/// Victims still come out least-recently-accessed first, now in bounded batches
+/// rather than one unbounded read of the table.
+#[test]
+fn cache_eviction_candidates_are_lru_ordered_and_limited() {
+    let db = Db::open_in_memory().unwrap();
+    for i in 0..10u64 {
+        // Insert newest-first so insertion order cannot be mistaken for LRU order.
+        db.cache_touch(&format!("k{i}"), "blob", 100, (10 - i) as i64)
+            .unwrap();
+    }
+    db.cache_touch("other", "block", 100, 0).unwrap();
+
+    let batch = db.cache_eviction_candidates("blob", 3).unwrap();
+    assert_eq!(batch.len(), 3, "honours the limit");
+    let keys: Vec<&str> = batch.iter().map(|(k, _)| k.as_str()).collect();
+    assert_eq!(keys, ["k9", "k8", "k7"], "least-recently-accessed first");
+    assert!(
+        batch.iter().all(|(k, _)| k != "other"),
+        "a different kind is never a candidate"
+    );
+}
+
+/// **The B5 checkpoint.** `improvements.md` P2.3 proposes concurrent SQLite
+/// reads, on the premise that the single `Mutex<Connection>` serializes the FUSE
+/// workers. This measures what the connection is actually asked to do now that
+/// B3 and B4 have landed, so the proposal is decided on evidence.
+///
+/// Note what is *not* here: `lookup`/`getattr`/`readdir` do not read this
+/// database in the steady state — they serve from `State::entries` in memory and
+/// only write through on a cold fill. The per-read DB operation is
+/// `cache_accessed`, one `UPDATE` per cache hit, which is what this drives.
+#[test]
+fn db_contention_under_fuse_worker_load() {
+    use std::sync::Arc;
+    use std::time::Instant;
+
+    let db = Arc::new(Db::open_in_memory().unwrap());
+    for i in 0..5000u64 {
+        db.cache_touch(&format!("k{i}.b0"), "block", 4 << 20, i as i64)
+            .unwrap();
+    }
+
+    // Eight workers, matching FUSE_WORKERS, each doing what a served block read
+    // does to the database: one LRU touch.
+    const WORKERS: usize = 8;
+    const PER_WORKER: usize = 2000;
+
+    let t = Instant::now();
+    let mut handles = Vec::new();
+    for w in 0..WORKERS {
+        let db = db.clone();
+        handles.push(std::thread::spawn(move || {
+            for i in 0..PER_WORKER {
+                db.cache_accessed(&format!("k{}.b0", (w * PER_WORKER + i) % 5000), i as i64)
+                    .unwrap();
+            }
+        }));
+    }
+    for h in handles {
+        h.join().unwrap();
+    }
+    let concurrent = t.elapsed();
+
+    // The same work on one thread, for the serialization baseline.
+    let t = Instant::now();
+    for i in 0..(WORKERS * PER_WORKER) {
+        db.cache_accessed(&format!("k{}.b0", i % 5000), i as i64)
+            .unwrap();
+    }
+    let serial = t.elapsed();
+
+    let total = WORKERS * PER_WORKER;
+    println!(
+        "B5: {total} LRU touches — {WORKERS} threads {concurrent:?}, 1 thread {serial:?} \
+         (per-op {:?} vs {:?})",
+        concurrent / total as u32,
+        serial / total as u32
+    );
+}