From 232b3b53488a831c2458bd1d1493db36c8198a74 Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Sat, 18 Jul 2026 14:20:37 +0200 Subject: [PATCH] db.rs refactor --- crates/pdfs-core/src/db.rs | 3179 ------------------------ crates/pdfs-core/src/db/activity.rs | 67 + crates/pdfs-core/src/db/cache.rs | 126 + crates/pdfs-core/src/db/devices.rs | 48 + crates/pdfs-core/src/db/local.rs | 163 ++ crates/pdfs-core/src/db/migrations.rs | 332 +++ crates/pdfs-core/src/db/mod.rs | 109 + crates/pdfs-core/src/db/nodes.rs | 241 ++ crates/pdfs-core/src/db/ops.rs | 344 +++ crates/pdfs-core/src/db/photos.rs | 244 ++ crates/pdfs-core/src/db/pins.rs | 124 + crates/pdfs-core/src/db/state.rs | 98 + crates/pdfs-core/src/db/sync.rs | 226 ++ crates/pdfs-core/src/db/tests.rs | 1061 ++++++++ crates/pdfs-core/src/db/trash.rs | 65 + crates/pdfs-core/src/db/utils.rs | 59 + crates/pdfs-fuse/src/control.rs | 717 ++++++ crates/pdfs-fuse/src/devices.rs | 698 ++++++ crates/pdfs-fuse/src/drain.rs | 769 ++++++ crates/pdfs-fuse/src/lib.rs | 3191 +------------------------ crates/pdfs-fuse/src/photos.rs | 363 +++ crates/pdfs-fuse/src/sharing.rs | 410 ++++ crates/pdfs-fuse/src/state.rs | 300 +++ crates/pdfs-fuse/src/workers.rs | 78 + 24 files changed, 6658 insertions(+), 6354 deletions(-) delete mode 100644 crates/pdfs-core/src/db.rs create mode 100644 crates/pdfs-core/src/db/activity.rs create mode 100644 crates/pdfs-core/src/db/cache.rs create mode 100644 crates/pdfs-core/src/db/devices.rs create mode 100644 crates/pdfs-core/src/db/local.rs create mode 100644 crates/pdfs-core/src/db/migrations.rs create mode 100644 crates/pdfs-core/src/db/mod.rs create mode 100644 crates/pdfs-core/src/db/nodes.rs create mode 100644 crates/pdfs-core/src/db/ops.rs create mode 100644 crates/pdfs-core/src/db/photos.rs create mode 100644 crates/pdfs-core/src/db/pins.rs create mode 100644 crates/pdfs-core/src/db/state.rs create mode 100644 crates/pdfs-core/src/db/sync.rs create mode 100644 crates/pdfs-core/src/db/tests.rs create mode 100644 crates/pdfs-core/src/db/trash.rs create mode 100644 crates/pdfs-core/src/db/utils.rs create mode 100644 crates/pdfs-fuse/src/control.rs create mode 100644 crates/pdfs-fuse/src/devices.rs create mode 100644 crates/pdfs-fuse/src/drain.rs create mode 100644 crates/pdfs-fuse/src/photos.rs create mode 100644 crates/pdfs-fuse/src/sharing.rs create mode 100644 crates/pdfs-fuse/src/state.rs create mode 100644 crates/pdfs-fuse/src/workers.rs diff --git a/crates/pdfs-core/src/db.rs b/crates/pdfs-core/src/db.rs deleted file mode 100644 index bb77352..0000000 --- a/crates/pdfs-core/src/db.rs +++ /dev/null @@ -1,3179 +0,0 @@ -//! Unified SQLite metadata cache — the single persistence layer behind FUSE -//! inode bookkeeping, full-text search, content-cache LRU tracking, and pins. -//! -//! Only the daemon (`pdfs-fuse`) opens this for writes; the GUI and CLI reach -//! the same data through the control socket. The connection is wrapped in a -//! `Mutex` because the FUSE callbacks are synchronous and already serialize -//! behind the `State` lock, so a connection pool would be overkill. -//! -//! This module is the P0 foundation: it opens the database, enables WAL, and -//! applies the forward-only schema migrations. Write-through of nodes, the -//! event cursor, FTS, and the cache index land in later phases on this schema. - -use std::collections::HashMap; -use std::path::Path; -use parking_lot::Mutex; - -use proton_drive_rs::proton_sdk::ids::NodeUid; -use proton_drive_rs::{Node, NodeKind}; -use rusqlite::{Connection, OptionalExtension, params}; - -use crate::Result; -use crate::control::{ActivityEntry, ActivityKind}; -use crate::localindex::LocalEntry; - -/// Current schema version. Bump on every forward migration added below. -const SCHEMA_VERSION: i64 = 13; - -/// How many activity rows to keep. Older rows are pruned on insert, so the feed -/// stays a bounded "recent history" rather than growing without limit. -const ACTIVITY_KEEP: i64 = 2000; - -/// The `kind` of a [`PendingOp`] that uploads a staged file as a new revision. -pub const OP_REVISION: &str = "revision"; - -/// The `kind` of a [`PendingOp`] that creates a file that so far exists only -/// locally, under a `local:` placeholder uid (offline.md Phase 3b). -/// -/// The written bytes ride along on the same row rather than as a follow-on -/// [`OP_REVISION`]: draining the create mints the node's real uid, which would -/// leave a separate revision op addressed to a uid that no longer exists. -pub const OP_CREATE: &str = "create"; - -/// The `kind` of a [`PendingOp`] that creates a folder that so far exists only -/// locally. Ordering matters: a child's op is queued after its parent's and -/// [`Db::pending_ops`] replays by row id, so the parent has a real uid by the -/// time the child drains. -pub const OP_MKDIR: &str = "mkdir"; - -/// The `kind` of a [`PendingOp`] that gives a node a new name, a new parent, or -/// both — the queued form of `mv` (offline.md Phase 3b). -/// -/// `parent_uid` and `name` hold the node's *desired end state*, not a delta, so -/// a second rename simply replaces the row (see [`Db::enqueue_op`]) and the -/// drain can compare them against the remote and skip whichever half already -/// matches. The parent may be a `local~` placeholder — moving a file into a -/// folder that was itself created offline — in which case it is rewritten by -/// that folder's own drain, exactly as for [`OP_CREATE`]. -pub const OP_RENAME: &str = "rename"; - -/// The `kind` of a [`PendingOp`] that trashes a node the server knows about -/// (offline.md Phase 3b). -/// -/// A node that only ever existed locally never gets one of these: there is -/// nothing to trash remotely, so deleting it just drops its queued ops -/// ([`Db::delete_ops_for_uid`]). -pub const OP_TRASH: &str = "trash"; - -/// Whether at most one op of this `kind` may be queued per node, so that a newer -/// one replaces the older rather than queueing behind it. -/// -/// True exactly of the kinds that describe a node's desired *end state* rather -/// than a step towards it: the newest revision already contains every earlier -/// one's bytes, the newest name is the only name wanted, and a trash subsumes -/// anything queued before it. A `create`/`mkdir` is the opposite — it is the one -/// thing that will ever make the node exist, so it must never be replaced. -pub fn op_supersedes(kind: &str) -> bool { - matches!(kind, OP_REVISION | OP_RENAME | OP_TRASH) -} - -/// The volume id given to a node that exists only on this machine, so far. A -/// real [`NodeUid`] is `{volume}~{link}`, so a placeholder is `local~` and -/// round-trips through the same `Display`/parse path as any other uid. -/// -/// Nothing bearing this volume may be handed to the API — it would 404. The -/// drain replaces it with the uid the server assigns. -pub const LOCAL_VOLUME: &str = "local"; - -/// A mutation that has been accepted locally but not yet performed against the -/// API — the durable half of the write-back queue (offline.md Phase 3). -/// -/// The daemon answers the FUSE call the moment this row and its staged blob are -/// on disk, so a `cp` into the mount runs at disk speed and the upload happens -/// behind it. That also makes an offline write succeed rather than EIO: the row -/// simply waits for the network. -#[derive(Debug, Clone)] -pub struct PendingOp { - /// Row id, `0` on a value being inserted. - pub id: i64, - /// See [`OP_REVISION`], [`OP_CREATE`], [`OP_MKDIR`]. - pub kind: String, - /// Node this op targets. For [`OP_CREATE`]/[`OP_MKDIR`] this is the - /// `local~` placeholder the node is known by until it drains. - pub uid: String, - /// Where the new node goes; only set for [`OP_CREATE`]/[`OP_MKDIR`]. May - /// itself be a placeholder when the parent folder is also still queued, in - /// which case it is rewritten when the parent drains. - pub parent_uid: Option, - /// The new node's name; only set for [`OP_CREATE`]/[`OP_MKDIR`]. - pub name: Option, - /// Staged blob holding the bytes to upload. - pub blob_path: Option, - /// Serialized [`StagedWrite`](crate::cache::StagedWrite). - pub meta_json: Option, - /// When the op was queued (ms since epoch). - pub created_at: i64, - pub attempts: i64, - pub last_error: Option, - /// Earliest ms at which to retry, for backoff. - pub next_attempt_at: i64, -} - -/// How much work the queue owes the server, by whether it carries bytes. -#[derive(Debug, Clone, Copy, Default)] -pub struct PendingCounts { - /// Queued `create`/`revision` ops: files whose content is not on the remote. - pub uploads: i64, - /// Queued `mkdir`/`rename`/`trash` ops: metadata the remote has not been - /// told about. - pub changes: i64, -} - -/// One cached blob or block as [`Db::cache_rebuild`] wants it: borrowed, because -/// the caller builds the whole list from a directory walk before handing it over. -#[derive(Debug, Clone, Copy)] -pub struct CacheEntryInput<'a> { - /// On-disk filename of the blob or block. - pub key: &'a str, - /// `'blob'` or `'block'` — the two byte budgets are counted separately. - pub kind: &'a str, - pub size: u64, - /// Initial LRU key, in unix milliseconds. - pub last_accessed: i64, -} - -/// The outcome of folding freshly written bytes into a queued create. -#[derive(Debug, Clone)] -pub struct AttachedBlob { - /// Row id of the create the bytes were attached to. - pub id: i64, - /// Blob the create held before, now orphaned. - pub superseded: Option, -} - -/// Size the WAL is truncated back to after a checkpoint. Comfortably above the -/// steady-state working set (a few MB), so the truncation only claws back the -/// outliers rather than fighting the normal write path for disk. -const WAL_SIZE_LIMIT: i64 = 64 * 1024 * 1024; - -/// A photo whose thumbnail state is not known yet: it has never been asked for. -pub const THUMB_UNKNOWN: i64 = 0; -/// A thumbnail exists for this photo — served by the server, or generated locally -/// from the full file when the server had none. -pub const THUMB_HAVE: i64 = 1; -/// This photo can never be given a thumbnail: the server has none and the bytes -/// could not be decoded locally either. Never retried. -pub const THUMB_NONE: i64 = 2; - -/// Below this length the trigram tokenizer indexes nothing (it needs 3-char -/// grams), so short queries fall back to a `LIKE` scan over `nodes.name`. -const TRIGRAM_MIN: usize = 3; - -/// A node loaded back from the database, paired with whether its directory -/// listing was complete (`listed`) when last persisted. Only meaningful for -/// folders; always `false` for files. -pub struct StoredNode { - pub node: Node, - pub listed: bool, -} - -/// One full-text search match: the stored [`Node`] plus its mountpoint-relative -/// path (`/`-joined, root excluded) so the front-end can navigate to or open it. -pub struct SearchHit { - pub node: Node, - pub path: String, -} - -/// One photo of the persisted timeline. The timeline itself is server-ordered -/// (newest first) and stored with that order in `seq`; `ratio` and `thumb_state` -/// are locally learned and survive a refresh of the timeline. -#[derive(Clone, Debug, PartialEq)] -pub struct StoredPhoto { - pub uid: String, - pub capture_time: i64, - pub name: Option, - /// Aspect ratio (w/h), known once a thumbnail has been decoded. - pub ratio: Option, - /// One of [`THUMB_UNKNOWN`] / [`THUMB_HAVE`] / [`THUMB_NONE`]. - pub thumb_state: i64, - /// Which Photos-page tab this entry belongs to, derived from its name and - /// media type when the timeline was last replaced. - pub kind: crate::control::PhotoKind, -} - -/// One trashed node, as persisted for the Trash page. Trashed nodes live outside -/// the mounted tree, so a uid is their only handle. -#[derive(Clone, Debug, PartialEq)] -pub struct StoredTrash { - pub uid: String, - pub name: String, - pub is_dir: bool, - pub size: i64, - pub mtime: i64, -} - -/// The one Proton Drive Device this machine is registered as (devices.md). -/// Cached so restarts reuse the same device rather than creating a new one. -#[derive(Clone, Debug, PartialEq)] -pub struct StoredDevice { - pub uid: String, - pub share_id: String, - pub root_uid: String, - pub name: String, - pub created: i64, -} - -/// One local folder the user added to this device's sync set. -#[derive(Clone, Debug, PartialEq)] -pub struct StoredSyncFolder { - pub id: i64, - pub local_path: String, - pub remote_uid: String, - pub remote_share_id: String, - /// `mirror` (full local copy, two-way synced) or `ondemand` (FUSE mount). - pub mode: String, - /// A mode the user asked for that could not be applied on the spot — the - /// folder was mid-pass, or had un-uploaded changes. The engine applies it - /// once the folder is safe to switch, so the request is queued rather than - /// rejected. `None` when the folder is where the user wants it. - pub pending_mode: Option, - /// `idle` | `syncing` | `error` | `conflict`. - pub state: String, - pub last_sync: i64, -} - -/// One per-file sync baseline row: what a path looked like on both sides at the -/// last successful sync, so the next reconcile can tell which side changed -/// (devices.md Phase 2). `remote_rev`/`remote_hash` hold the remote signature — -/// its modification time and size as strings — since no cheap content hash is -/// exposed; change detection is `(mtime, size)` on each side. -#[derive(Clone, Debug, PartialEq)] -pub struct StoredSyncEntry { - pub rel_path: String, - pub remote_uid: Option, - pub local_mtime: i64, - pub local_size: i64, - pub remote_rev: Option, - pub remote_hash: Option, -} - -/// Handle to the unified metadata database. -/// -/// Cheap to wrap in an `Arc`; clone the `Arc`, not this. All access goes through -/// the inner `Mutex`. -pub struct Db { - conn: Mutex, -} - -impl Db { - /// Open (creating if absent) the database at `path`, enable WAL, and bring - /// the schema up to [`SCHEMA_VERSION`]. - pub fn open(path: &Path) -> Result { - let conn = Connection::open(path)?; - // WAL: readers never block the single writer. NORMAL sync is the - // standard durability/throughput tradeoff for WAL. - conn.pragma_update(None, "journal_mode", "WAL")?; - conn.pragma_update(None, "synchronous", "NORMAL")?; - conn.pragma_update(None, "foreign_keys", "ON")?; - // Hand the WAL's disk back after a checkpoint. Without a limit SQLite - // reuses the file in place but never shrinks it, so a single large - // transaction (a full-index FTS rebuild in `local_finish_scan` is the - // one that reaches this size) leaves the WAL at its high-water mark - // forever — a multi-GB file next to a database two orders of magnitude - // smaller. Checkpointing is unaffected; only the file is truncated back. - conn.pragma_update(None, "journal_size_limit", WAL_SIZE_LIMIT)?; - - let db = Self { - conn: Mutex::new(conn), - }; - db.migrate()?; - Ok(db) - } - - /// Open an in-memory database. For tests. - #[cfg(test)] - pub fn open_in_memory() -> Result { - let conn = Connection::open_in_memory()?; - let db = Self { - conn: Mutex::new(conn), - }; - db.migrate()?; - Ok(db) - } - - /// Run forward-only migrations from the stored `schema_version` up to - /// [`SCHEMA_VERSION`]. Each step is wrapped in its own transaction. - fn migrate(&self) -> Result<()> { - let mut conn = self.conn.lock(); - - // `sync_state` is the key/value table holding `schema_version` and the - // event cursor (later). Create it first so we can read the version. - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS sync_state (key TEXT PRIMARY KEY, value TEXT);", - )?; - - let current: i64 = conn - .query_row( - "SELECT value FROM sync_state WHERE key = 'schema_version'", - [], - |row| row.get::<_, String>(0), - ) - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(0); - - if current >= SCHEMA_VERSION { - return Ok(()); - } - - let tx = conn.transaction()?; - if current < 1 { - tx.execute_batch(MIGRATION_V1)?; - } - if current < 2 { - tx.execute_batch(MIGRATION_V2)?; - } - if current < 3 { - tx.execute_batch(MIGRATION_V3)?; - } - if current < 4 { - tx.execute_batch(MIGRATION_V4)?; - } - if current < 5 { - tx.execute_batch(MIGRATION_V5)?; - } - if current < 6 { - tx.execute_batch(MIGRATION_V6)?; - } - if current < 7 { - tx.execute_batch(MIGRATION_V7)?; - } - if current < 8 { - tx.execute_batch(MIGRATION_V8)?; - } - if current < 9 { - tx.execute_batch(MIGRATION_V9)?; - } - if current < 10 { - tx.execute_batch(MIGRATION_V10)?; - } - if current < 11 { - tx.execute_batch(MIGRATION_V11)?; - } - if current < 12 { - tx.execute_batch(MIGRATION_V12)?; - } - if current < 13 { - tx.execute_batch(MIGRATION_V13)?; - } - tx.execute( - "INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1) - ON CONFLICT(key) DO UPDATE SET value = excluded.value", - [SCHEMA_VERSION.to_string()], - )?; - tx.commit()?; - Ok(()) - } - - /// Run a closure with the locked connection. Escape hatch for callers in - /// later phases until typed query methods are added. - pub fn with_conn(&self, f: impl FnOnce(&Connection) -> Result) -> Result { - let conn = self.conn.lock(); - f(&conn) - } - - /// Write-through a node from the in-memory cache. Indexed columns are kept - /// for query/search/eviction; the full [`Node`] is stored as JSON so the - /// hot-cache maps rehydrate losslessly on the next mount. `listed` is never - /// changed here — it is owned by [`set_listed`](Self::set_listed). - pub fn upsert_node(&self, node: &Node) -> Result<()> { - self.upsert_nodes(std::slice::from_ref(node)) - } - - /// Write-through a batch of nodes as one transaction — a whole directory - /// listing, typically. Otherwise identical to [`upsert_node`](Self::upsert_node), - /// which is the single-node case of it. - /// - /// The commit count is the point: SQLite autocommits every statement that is - /// not in an explicit transaction, so interning a folder of a thousand - /// children row-by-row cost a thousand fsyncs, and `ls` waited for all of - /// them. - pub fn upsert_nodes(&self, nodes: &[Node]) -> Result<()> { - if nodes.is_empty() { - return Ok(()); - } - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - for node in nodes { - let json = serde_json::to_string(node)?; - let uid = node.uid.to_string(); - tx.execute( - "INSERT INTO nodes - (uid, parent_uid, name, is_dir, size, mtime, trashed, node_json) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) - ON CONFLICT(uid) DO UPDATE SET - parent_uid = excluded.parent_uid, - name = excluded.name, - is_dir = excluded.is_dir, - size = excluded.size, - mtime = excluded.mtime, - trashed = excluded.trashed, - node_json = excluded.node_json", - params![ - uid, - node.parent_uid.as_ref().map(|u| u.to_string()), - node.name, - node.is_folder() as i64, - node_size(node), - node.modification_time, - node.trashed as i64, - json, - ], - )?; - // FTS5 has no UPSERT, so refresh the row by delete-then-insert. - // Trashed nodes are kept out of the index entirely so they never - // surface in search results. - tx.execute("DELETE FROM nodes_fts WHERE uid = ?1", params![uid])?; - if !node.trashed { - tx.execute( - "INSERT INTO nodes_fts (uid, name) VALUES (?1, ?2)", - params![uid, node.name], - )?; - } - } - tx.commit()?; - Ok(()) - } - - /// Drop a node row (delete or trash from the hot cache). Children rows are - /// not cascaded here; the daemon forgets a whole subtree node-by-node. - pub fn delete_node(&self, uid: &NodeUid) -> Result<()> { - let uid = uid.to_string(); - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute("DELETE FROM nodes WHERE uid = ?1", params![uid])?; - tx.execute("DELETE FROM nodes_fts WHERE uid = ?1", params![uid])?; - tx.commit()?; - Ok(()) - } - - /// Full-text search over node names, newest schema's trigram index giving - /// substring (not just prefix) matches. Returns up to `limit` non-trashed - /// hits, each with its mountpoint-relative path resolved. Queries shorter - /// than [`TRIGRAM_MIN`] fall back to a `LIKE` scan since trigram indexes - /// nothing below 3 chars. - pub fn search(&self, query: &str, limit: usize) -> Result> { - let query = query.trim(); - if query.is_empty() { - return Ok(Vec::new()); - } - let conn = self.conn.lock(); - let rows: Vec<(String, String)> = if query.chars().count() < TRIGRAM_MIN { - let pat = format!("%{}%", like_escape(query)); - let mut stmt = conn.prepare( - "SELECT node_json, uid FROM nodes - WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL - ORDER BY name LIMIT ?2", - )?; - collect_pairs(stmt.query_map(params![pat, limit as i64], pair)?)? - } else { - // Escape double quotes and quote each term, then combine with AND so - // all terms must match but can appear in any order or position. - let phrase = query - .split_whitespace() - .map(|word| format!("\"{}\"", word.replace('"', "\"\""))) - .collect::>() - .join(" AND "); - let mut stmt = conn.prepare( - "SELECT n.node_json, n.uid - FROM nodes_fts f JOIN nodes n ON n.uid = f.uid - WHERE f.name MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL - ORDER BY f.rank LIMIT ?2", - )?; - collect_pairs(stmt.query_map(params![phrase, limit as i64], pair)?)? - }; - - let mut hits = Vec::with_capacity(rows.len()); - for (json, uid) in rows { - let node: Node = serde_json::from_str(&json)?; - let path = path_of(&conn, &uid)?; - hits.push(SearchHit { node, path }); - } - Ok(hits) - } - - /// Mark (or unmark) a folder's child listing as complete. A listed folder - /// rehydrates its `children` map on mount even when empty; an unlisted one - /// re-enumerates from the remote on next access. - pub fn set_listed(&self, uid: &NodeUid, listed: bool) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE nodes SET listed = ?2 WHERE uid = ?1", - params![uid.to_string(), listed as i64], - )?; - Ok(()) - } - - /// Load every persisted node for cold-start hydration of the `State` maps. - pub fn load_all(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = - conn.prepare("SELECT node_json, listed FROM nodes WHERE node_json IS NOT NULL")?; - let rows = stmt.query_map([], |row| { - let json: String = row.get(0)?; - let listed: i64 = row.get(1)?; - Ok((json, listed != 0)) - })?; - let mut out = Vec::new(); - for row in rows { - let (json, listed) = row?; - let node: Node = serde_json::from_str(&json)?; - out.push(StoredNode { node, listed }); - } - Ok(out) - } - - /// Load one persisted node back by uid. Used to recover the My Files root - /// when the API is unreachable, so the mount can still serve the cached tree - /// (offline.md Phase 1). - pub fn node_by_uid(&self, uid: &str) -> Result> { - let conn = self.conn.lock(); - let json: Option = conn - .query_row( - "SELECT node_json FROM nodes WHERE uid = ?1 AND node_json IS NOT NULL", - params![uid], - |r| r.get(0), - ) - .optional()?; - match json { - Some(json) => Ok(Some(serde_json::from_str(&json)?)), - None => Ok(None), - } - } - - /// Read the persisted incremental-sync cursor (a `DriveEventId`), if any. - /// The daemon resumes from this on restart instead of reseeding to the - /// server head, so changes made while unmounted are still applied (P2). - pub fn get_event_cursor(&self) -> Result> { - let conn = self.conn.lock(); - let v = conn - .query_row( - "SELECT value FROM sync_state WHERE key = 'event_cursor'", - [], - |r| r.get::<_, String>(0), - ) - .optional()?; - Ok(v) - } - - /// Persist the incremental-sync cursor after a batch of events is applied. - pub fn set_event_cursor(&self, cursor: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO sync_state (key, value) VALUES ('event_cursor', ?1) - ON CONFLICT(key) DO UPDATE SET value = excluded.value", - params![cursor], - )?; - Ok(()) - } - - /// Read a `sync_state` value as a string. - pub fn state_str(&self, key: &str) -> Result> { - let conn = self.conn.lock(); - let v = conn - .query_row( - "SELECT value FROM sync_state WHERE key = ?1", - params![key], - |r| r.get::<_, String>(0), - ) - .optional()?; - Ok(v) - } - - /// Write a `sync_state` string value. - pub fn set_state_str(&self, key: &str, value: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO sync_state (key, value) VALUES (?1, ?2) - ON CONFLICT(key) DO UPDATE SET value = excluded.value", - params![key, value], - )?; - Ok(()) - } - - /// Queue an op to be performed by the drain worker, returning its row id. - /// - /// One *revision* op per node: a second write to the same file before the - /// first has drained replaces it, since the newer blob already contains - /// everything the older one did. The superseded blob's path is returned so - /// the caller can delete it. - /// - /// Rows supersede only their own kind, and only the kinds - /// [`op_supersedes`] names. A create op for the same uid must survive — - /// dropping it would leave a file that exists nowhere but this machine with - /// nothing left to create it. (Writes to a node that is itself still queued - /// fold into the create row instead; see [`Db::attach_blob_to_create`].) - pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option)> { - let conn = self.conn.lock(); - let superseded: Option = if op_supersedes(&op.kind) { - let blob: Option = conn - .query_row( - "SELECT blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2", - params![op.uid, op.kind], - |r| r.get(0), - ) - .optional()? - .flatten(); - conn.execute( - "DELETE FROM pending_op WHERE uid = ?1 AND kind = ?2", - params![op.uid, op.kind], - )?; - blob - } else { - None - }; - conn.execute( - "INSERT INTO pending_op - (kind, uid, parent_uid, name, blob_path, meta_json, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - op.kind, - op.uid, - op.parent_uid, - op.name, - op.blob_path, - op.meta_json, - op.created_at - ], - )?; - Ok((conn.last_insert_rowid(), superseded)) - } - - /// Point a queued create at the bytes that were just written to it, returning - /// any blob it previously held so the caller can discard it. - /// - /// This is what `release` does for a file that only exists locally: the node - /// has no uid to hang a revision op on yet, so the bytes ride on the create. - /// Repeated writes before the drain simply replace the blob. - /// - /// Returns `Ok(None)` and touches nothing if the create has already drained — - /// the caller must then queue an ordinary revision against the real uid. - pub fn attach_blob_to_create( - &self, - uid: &str, - blob_path: &str, - meta_json: &str, - ) -> Result> { - let conn = self.conn.lock(); - let existing: Option<(i64, Option)> = conn - .query_row( - "SELECT id, blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2", - params![uid, OP_CREATE], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional()?; - let Some((id, superseded)) = existing else { - return Ok(None); - }; - conn.execute( - "UPDATE pending_op - SET blob_path = ?2, meta_json = ?3, attempts = 0, next_attempt_at = 0 - WHERE id = ?1", - params![id, blob_path, meta_json], - )?; - Ok(Some(AttachedBlob { id, superseded })) - } - - /// Point a queued create at a new parent and name, for a node renamed or - /// moved before it ever reached the server. - /// - /// A `local~` uid means nothing to the API, so there is no rename call to - /// make — the node is still only a queued intent, and rewriting that intent - /// *is* the rename. Returns false when the create has already drained, in - /// which case the node has a real uid and the caller must rename it there - /// instead (offline.md Phase 3b). - pub fn rewrite_op_target(&self, uid: &str, parent_uid: &str, name: &str) -> Result { - let conn = self.conn.lock(); - let n = conn.execute( - "UPDATE pending_op SET parent_uid = ?2, name = ?3 - WHERE uid = ?1 AND kind IN (?4, ?5)", - params![uid, parent_uid, name, OP_CREATE, OP_MKDIR], - )?; - Ok(n > 0) - } - - /// Drop every op targeting a node **or anything queued beneath it**, - /// returning the staged blobs they held so the caller can delete them. - /// - /// Used when a node that only ever existed locally is deleted: there is - /// nothing on the server to trash, so the queued work simply stops being - /// wanted. - /// - /// The descent is what keeps the queue alive. Deleting a folder created - /// offline drops the `mkdir` that would have given it a real uid, and any - /// op still queued under that placeholder is then unreachable forever: it - /// can never be attempted (its parent is a `local~` uid, so `op_is_ready` - /// refuses it) and nothing is left to remap it. It would sit in the queue, - /// and in the user's pending count, for the life of the database. - /// - /// Only `create`/`mkdir` ops carry a `parent_uid`, so for a file this - /// recursion finds nothing and costs one query. - pub fn delete_ops_for_uid(&self, uid: &str) -> Result> { - const SUBTREE: &str = " - WITH RECURSIVE doomed(uid) AS ( - SELECT ?1 - UNION - SELECT p.uid FROM pending_op p JOIN doomed d ON p.parent_uid = d.uid - )"; - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - let blobs: Vec = { - let mut stmt = tx.prepare(&format!( - "{SUBTREE} - SELECT blob_path FROM pending_op - WHERE uid IN (SELECT uid FROM doomed) AND blob_path IS NOT NULL" - ))?; - let rows = stmt.query_map(params![uid], |r| r.get::<_, String>(0))?; - rows.collect::, _>>()? - }; - tx.execute( - &format!("{SUBTREE} DELETE FROM pending_op WHERE uid IN (SELECT uid FROM doomed)"), - params![uid], - )?; - tx.commit()?; - Ok(blobs) - } - - /// Rewrite every queued op that points at a placeholder parent, once that - /// parent has drained and has a real uid. Also moves the node rows whose - /// parent column still names the placeholder, so listings keep resolving. - pub fn remap_local_uid(&self, local: &str, real: &str) -> Result<()> { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute( - "UPDATE pending_op SET parent_uid = ?2 WHERE parent_uid = ?1", - params![local, real], - )?; - tx.execute( - "UPDATE nodes SET parent_uid = ?2 WHERE parent_uid = ?1", - params![local, real], - )?; - tx.execute("DELETE FROM nodes WHERE uid = ?1", params![local])?; - tx.commit()?; - Ok(()) - } - - /// Every queued op, oldest first. The drain worker replays them in this order - /// so a file's writes land in the order they were made. - pub fn pending_ops(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at, - attempts, last_error, next_attempt_at - FROM pending_op ORDER BY id", - )?; - let rows = stmt - .query_map([], |r| { - Ok(PendingOp { - id: r.get(0)?, - kind: r.get(1)?, - uid: r.get(2)?, - parent_uid: r.get(3)?, - name: r.get(4)?, - blob_path: r.get(5)?, - meta_json: r.get(6)?, - created_at: r.get(7)?, - attempts: r.get(8)?, - last_error: r.get(9)?, - next_attempt_at: r.get(10)?, - }) - })? - .collect::, _>>()?; - Ok(rows) - } - - /// How many queued ops there are of each kind, for `Response::Status`. - /// - /// Split because "3 uploads queued" has to mean three files whose bytes are - /// not on the remote yet. A queued `mkdir`/`rename`/`trash` is also work the - /// mount owes the server, but it carries no bytes and reporting it as an - /// upload is simply untrue. - pub fn pending_op_counts(&self) -> Result { - let conn = self.conn.lock(); - let uploads = conn.query_row( - "SELECT COUNT(*) FROM pending_op WHERE kind IN (?1, ?2)", - params![OP_REVISION, OP_CREATE], - |r| r.get(0), - )?; - let changes = conn.query_row( - "SELECT COUNT(*) FROM pending_op WHERE kind NOT IN (?1, ?2)", - params![OP_REVISION, OP_CREATE], - |r| r.get(0), - )?; - Ok(PendingCounts { uploads, changes }) - } - - /// Drop a queued op, once its upload has actually landed. - pub fn delete_op(&self, id: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute("DELETE FROM pending_op WHERE id = ?1", params![id])?; - Ok(()) - } - - /// Record a failed attempt and when to next try. Leaves the row in place — - /// the staged bytes are still the only copy of the user's write. - pub fn record_op_failure(&self, id: i64, error: &str, next_attempt_at: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE pending_op - SET attempts = attempts + 1, last_error = ?2, next_attempt_at = ?3 - WHERE id = ?1", - params![id, error, next_attempt_at], - )?; - Ok(()) - } - - /// Read a `sync_state` value as an integer (the freshness stamps of the - /// photos timeline and the trash listing are kept there). - pub fn state_i64(&self, key: &str) -> Result> { - let conn = self.conn.lock(); - let v: Option = conn - .query_row( - "SELECT value FROM sync_state WHERE key = ?1", - params![key], - |r| r.get(0), - ) - .optional()?; - Ok(v.and_then(|v| v.parse().ok())) - } - - /// Write a `sync_state` integer value. - pub fn set_state_i64(&self, key: &str, value: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO sync_state (key, value) VALUES (?1, ?2) - ON CONFLICT(key) DO UPDATE SET value = excluded.value", - params![key, value.to_string()], - )?; - Ok(()) - } - - /// Drop a `sync_state` key, so whatever it stamped counts as never fetched. - pub fn clear_state(&self, key: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute("DELETE FROM sync_state WHERE key = ?1", params![key])?; - Ok(()) - } - - /// Replace the persisted photos timeline with `items` (newest first), keeping - /// what was locally learned about the photos that are still there: a ratio or - /// a thumbnail verdict costs a download to rediscover, while the server order - /// and capture times are cheap and authoritative. - /// - /// Photos no longer in the timeline are dropped, so a deletion on another - /// client doesn't leave a ghost tile behind. - pub fn photos_replace( - &self, - items: &[(String, i64, Option, Option)], - ) -> Result<()> { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - // `media_type` is learned-and-kept like the ratio and thumb verdict: the - // timeline DTO carries only the uid and capture time, so the daemon may - // not know a photo's media type yet when it replaces the timeline. Keep - // any previously learned value so the Photos/Videos/Raw split survives a - // refresh instead of collapsing back to name-extension guesses. - let learned: HashMap, i64, Option)> = { - let mut stmt = - tx.prepare("SELECT uid, ratio, thumb_state, media_type FROM photos")?; - let rows = stmt.query_map([], |r| { - Ok((r.get::<_, String>(0)?, (r.get(1)?, r.get(2)?, r.get(3)?))) - })?; - rows.collect::>()? - }; - - tx.execute("DELETE FROM photos", [])?; - { - let mut stmt = tx.prepare( - "INSERT INTO photos - (uid, capture_time, name, ratio, thumb_state, seq, media_type, kind) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - )?; - for (seq, (uid, capture_time, name, media_type)) in items.iter().enumerate() { - let (ratio, thumb_state, learned_media) = learned - .get(uid) - .cloned() - .unwrap_or((None, THUMB_UNKNOWN, None)); - let media_type = media_type.clone().or(learned_media); - // The tab this photo lands in is derived here, once, so a page or - // count query is a plain indexed `WHERE kind = ?` rather than a - // reclassification of every row. - let kind = crate::control::PhotoKind::classify( - name.as_deref(), - media_type.as_deref(), - ); - stmt.execute(params![ - uid, - capture_time, - name, - ratio, - thumb_state, - seq as i64, - media_type, - kind.as_i64(), - ])?; - } - } - tx.commit()?; - Ok(()) - } - - /// One page of the persisted timeline, newest first. `kind`, when set, - /// restricts the page to one tab (Photos / Videos / Raw); `range`, when set, - /// restricts it to a `[from, to)` capture-time window (epoch seconds) — the - /// date scrubber's jump. `offset` is relative to whatever the filters leave. - pub fn photos_page( - &self, - offset: usize, - limit: usize, - kind: Option, - range: Option<(i64, i64)>, - ) -> Result> { - let conn = self.conn.lock(); - // Built up so any combination of the optional filters is one indexed - // query rather than a statement per case. - let mut sql = - String::from("SELECT uid, capture_time, name, ratio, thumb_state, kind FROM photos"); - let mut binds: Vec = Vec::new(); - let mut conds: Vec = Vec::new(); - if let Some(k) = kind { - binds.push(k.as_i64()); - conds.push(format!("kind = ?{}", binds.len())); - } - if let Some((from, to)) = range { - binds.push(from); - conds.push(format!("capture_time >= ?{}", binds.len())); - binds.push(to); - conds.push(format!("capture_time < ?{}", binds.len())); - } - if !conds.is_empty() { - sql.push_str(" WHERE "); - sql.push_str(&conds.join(" AND ")); - } - binds.push(limit as i64); - let limit_pos = binds.len(); - binds.push(offset as i64); - let offset_pos = binds.len(); - sql.push_str(&format!(" ORDER BY seq LIMIT ?{limit_pos} OFFSET ?{offset_pos}")); - - let mut stmt = conn.prepare(&sql)?; - let rows = stmt - .query_map(rusqlite::params_from_iter(binds), |r| { - Ok(StoredPhoto { - uid: r.get(0)?, - capture_time: r.get(1)?, - name: r.get(2)?, - ratio: r.get(3)?, - thumb_state: r.get(4)?, - kind: crate::control::PhotoKind::from_i64(r.get(5)?), - }) - })? - .collect::>()?; - Ok(rows) - } - - /// The months the timeline spans, newest first, each with how many photos it - /// holds — the data behind the date scrubber. Buckets are local-time - /// `(year, month)` so they line up with the day headings the gallery draws - /// and with the boundaries a front-end computes when it jumps to one. `kind` - /// scopes the counts to one tab when set. - pub fn photos_months( - &self, - kind: Option, - ) -> Result> { - let conn = self.conn.lock(); - let (filter, binds): (&str, Vec) = match kind { - Some(k) => (" WHERE kind = ?1", vec![k.as_i64()]), - None => ("", Vec::new()), - }; - let sql = format!( - "SELECT CAST(strftime('%Y', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS y, \ - CAST(strftime('%m', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS m, \ - COUNT(*) \ - FROM photos{filter} GROUP BY y, m ORDER BY y DESC, m DESC" - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt - .query_map(rusqlite::params_from_iter(binds), |r| { - Ok((r.get::<_, i32>(0)?, r.get::<_, i32>(1)?, r.get::<_, i64>(2)? as usize)) - })? - .collect::>()?; - Ok(rows) - } - - /// Per-tab counts for the Photos page subtitle: `(photos, videos, raw)`. - pub fn photos_counts(&self) -> Result<(usize, usize, usize)> { - use crate::control::PhotoKind; - let conn = self.conn.lock(); - let mut stmt = conn.prepare("SELECT kind, COUNT(*) FROM photos GROUP BY kind")?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?)))?; - let (mut photos, mut videos, mut raw) = (0usize, 0usize, 0usize); - for row in rows { - let (kind, n) = row?; - match PhotoKind::from_i64(kind) { - PhotoKind::Photo => photos = n as usize, - PhotoKind::Video => videos = n as usize, - PhotoKind::Raw => raw = n as usize, - } - } - Ok((photos, videos, raw)) - } - - /// The stored photos for `uids`, in no particular order. Used by the thumbnail - /// path, which needs each photo's capture time (the cache validity tag) and - /// its thumbnail verdict. - pub fn photos_by_uid(&self, uids: &[String]) -> Result> { - if uids.is_empty() { - return Ok(Vec::new()); - } - let conn = self.conn.lock(); - let placeholders = vec!["?"; uids.len()].join(","); - let mut stmt = conn.prepare(&format!( - "SELECT uid, capture_time, name, ratio, thumb_state, kind FROM photos - WHERE uid IN ({placeholders})" - ))?; - let rows = stmt.query_map(rusqlite::params_from_iter(uids), |r| { - Ok(StoredPhoto { - uid: r.get(0)?, - capture_time: r.get(1)?, - name: r.get(2)?, - ratio: r.get(3)?, - thumb_state: r.get(4)?, - kind: crate::control::PhotoKind::from_i64(r.get(5)?), - }) - })?; - let mut photos = Vec::new(); - for row in rows { - photos.push(row?); - } - Ok(photos) - } - - /// Record what a thumbnail attempt learned: whether the photo now has one - /// ([`THUMB_HAVE`] / [`THUMB_NONE`]), and its aspect ratio if the pixels were - /// seen. A `None` ratio leaves any previously learned one alone. - pub fn photo_set_thumb(&self, uid: &str, state: i64, ratio: Option) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE photos SET thumb_state = ?2, ratio = COALESCE(?3, ratio) WHERE uid = ?1", - params![uid, state, ratio], - )?; - Ok(()) - } - - /// Number of photos in the persisted timeline. - pub fn photos_count(&self) -> Result { - let conn = self.conn.lock(); - let n: i64 = conn.query_row("SELECT COUNT(*) FROM photos", [], |r| r.get(0))?; - Ok(n.max(0) as usize) - } - - /// Replace the persisted trash listing. - pub fn trash_replace(&self, items: &[StoredTrash]) -> Result<()> { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute("DELETE FROM trash", [])?; - { - let mut stmt = tx.prepare( - "INSERT INTO trash (uid, name, is_dir, size, mtime) VALUES (?1, ?2, ?3, ?4, ?5)", - )?; - for item in items { - stmt.execute(params![ - item.uid, - item.name, - item.is_dir as i64, - item.size, - item.mtime - ])?; - } - } - tx.commit()?; - Ok(()) - } - - /// The persisted trash listing, folders first then by name — the order the - /// Trash page shows it in. - pub fn trash_list(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT uid, name, is_dir, size, mtime FROM trash - ORDER BY is_dir DESC, name COLLATE NOCASE", - )?; - let rows = stmt.query_map([], |r| { - Ok(StoredTrash { - uid: r.get(0)?, - name: r.get(1)?, - is_dir: r.get::<_, i64>(2)? != 0, - size: r.get(3)?, - mtime: r.get(4)?, - }) - })?; - let mut items = Vec::new(); - for row in rows { - items.push(row?); - } - Ok(items) - } - - // ---- device sync (devices.md) ----------------------------------------- - - /// The registered device for this machine, if one has been created/cached. - pub fn device_get(&self) -> Result> { - let conn = self.conn.lock(); - conn.query_row( - "SELECT uid, share_id, root_uid, name, created FROM device LIMIT 1", - [], - |r| { - Ok(StoredDevice { - uid: r.get(0)?, - share_id: r.get(1)?, - root_uid: r.get(2)?, - name: r.get(3)?, - created: r.get(4)?, - }) - }, - ) - .optional() - .map_err(Into::into) - } - - /// Persist (or replace) this machine's device. The table holds a single row. - pub fn device_set(&self, dev: &StoredDevice) -> Result<()> { - let conn = self.conn.lock(); - conn.execute("DELETE FROM device", [])?; - conn.execute( - "INSERT INTO device (uid, share_id, root_uid, name, created) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![dev.uid, dev.share_id, dev.root_uid, dev.name, dev.created], - )?; - Ok(()) - } - - /// Add a synced folder, returning its new row id. - pub fn sync_folder_add( - &self, - local_path: &str, - remote_uid: &str, - remote_share_id: &str, - ) -> Result { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO sync_folder (local_path, remote_uid, remote_share_id) - VALUES (?1, ?2, ?3)", - params![local_path, remote_uid, remote_share_id], - )?; - Ok(conn.last_insert_rowid()) - } - - /// Every synced folder, oldest first. - pub fn sync_folder_list(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync - FROM sync_folder ORDER BY id", - )?; - let rows = stmt.query_map([], |r| { - Ok(StoredSyncFolder { - id: r.get(0)?, - local_path: r.get(1)?, - remote_uid: r.get(2)?, - remote_share_id: r.get(3)?, - mode: r.get(4)?, - pending_mode: r.get(5)?, - state: r.get(6)?, - last_sync: r.get(7)?, - }) - })?; - let mut items = Vec::new(); - for row in rows { - items.push(row?); - } - Ok(items) - } - - /// Look up one synced folder by id. - pub fn sync_folder_get(&self, id: i64) -> Result> { - let conn = self.conn.lock(); - conn.query_row( - "SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync - FROM sync_folder WHERE id = ?1", - params![id], - |r| { - Ok(StoredSyncFolder { - id: r.get(0)?, - local_path: r.get(1)?, - remote_uid: r.get(2)?, - remote_share_id: r.get(3)?, - mode: r.get(4)?, - pending_mode: r.get(5)?, - state: r.get(6)?, - last_sync: r.get(7)?, - }) - }, - ) - .optional() - .map_err(Into::into) - } - - /// Remove a synced folder and its per-file baseline. - pub fn sync_folder_remove(&self, id: i64) -> Result { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute("DELETE FROM sync_entry WHERE folder_id = ?1", params![id])?; - let n = tx.execute("DELETE FROM sync_folder WHERE id = ?1", params![id])?; - tx.commit()?; - Ok(n > 0) - } - - /// Update a synced folder's mode (`mirror`/`ondemand`). The folder has - /// reached the mode it was asked for, so any queued request is satisfied and - /// cleared in the same write — a `pending_mode` outliving the switch it asked - /// for would have the engine try to apply it again on the next pass. - pub fn sync_folder_set_mode(&self, id: i64, mode: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE sync_folder SET mode = ?2, pending_mode = NULL WHERE id = ?1", - params![id, mode], - )?; - Ok(()) - } - - /// Queue (or, with `None`, withdraw) a mode the folder should move to once it - /// is safe to switch. - pub fn sync_folder_set_pending_mode(&self, id: i64, mode: Option<&str>) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE sync_folder SET pending_mode = ?2 WHERE id = ?1", - params![id, mode], - )?; - Ok(()) - } - - /// Update a synced folder's state and stamp `last_sync` to now. - pub fn sync_folder_set_state(&self, id: i64, state: &str, last_sync: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE sync_folder SET state = ?2, last_sync = ?3 WHERE id = ?1", - params![id, state, last_sync], - )?; - Ok(()) - } - - /// The whole per-file sync baseline for a folder, keyed by relative path. - pub fn sync_entries(&self, folder_id: i64) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash - FROM sync_entry WHERE folder_id = ?1", - )?; - let rows = stmt.query_map(params![folder_id], |r| { - Ok(StoredSyncEntry { - rel_path: r.get(0)?, - remote_uid: r.get(1)?, - local_mtime: r.get(2)?, - local_size: r.get(3)?, - remote_rev: r.get(4)?, - remote_hash: r.get(5)?, - }) - })?; - let mut map = HashMap::new(); - for row in rows { - let e = row?; - map.insert(e.rel_path.clone(), e); - } - Ok(map) - } - - /// Insert or replace one baseline row. - pub fn sync_entry_upsert(&self, folder_id: i64, e: &StoredSyncEntry) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO sync_entry - (folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) - ON CONFLICT(folder_id, rel_path) DO UPDATE SET - remote_uid = excluded.remote_uid, - local_mtime = excluded.local_mtime, - local_size = excluded.local_size, - remote_rev = excluded.remote_rev, - remote_hash = excluded.remote_hash", - params![ - folder_id, - e.rel_path, - e.remote_uid, - e.local_mtime, - e.local_size, - e.remote_rev, - e.remote_hash, - ], - )?; - Ok(()) - } - - /// Drop one baseline row (a path that left the sync set on both sides). - pub fn sync_entry_remove(&self, folder_id: i64, rel_path: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "DELETE FROM sync_entry WHERE folder_id = ?1 AND rel_path = ?2", - params![folder_id, rel_path], - )?; - Ok(()) - } - - // ---- activity --------------------------------------------------------- - - /// Append one entry to the activity log, pruning back to [`ACTIVITY_KEEP`] - /// rows. `kind` round-trips through serde rather than a hand-written string - /// table, so adding an [`ActivityKind`] variant needs no change here. - pub fn activity_add(&self, entry: &ActivityEntry) -> Result<()> { - let kind = serde_json::to_string(&entry.kind)?; - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute( - "INSERT INTO activity (time, kind, target, detail, ok) VALUES (?1, ?2, ?3, ?4, ?5)", - params![entry.time, kind, entry.target, entry.detail, entry.ok], - )?; - tx.execute( - "DELETE FROM activity WHERE id <= ( - SELECT id FROM activity ORDER BY id DESC LIMIT 1 OFFSET ?1 - )", - params![ACTIVITY_KEEP], - )?; - tx.commit()?; - Ok(()) - } - - /// The most recent activity, newest first, capped at `limit` entries. Rows - /// whose stored `kind` no longer parses (written by an older build) are - /// skipped rather than failing the whole read. - pub fn activity_list(&self, limit: usize) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT time, kind, target, detail, ok FROM activity - ORDER BY id DESC LIMIT ?1", - )?; - let rows = stmt.query_map(params![limit as i64], |r| { - Ok(( - r.get::<_, i64>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - r.get::<_, String>(3)?, - r.get::<_, bool>(4)?, - )) - })?; - let mut out = Vec::new(); - for row in rows { - let (time, kind, target, detail, ok) = row?; - let Ok(kind) = serde_json::from_str::(&kind) else { - continue; - }; - out.push(ActivityEntry { - time, - kind, - target, - detail, - ok, - }); - } - Ok(out) - } - - /// Drop the entire baseline for a folder. Used when flipping ondemand→mirror: - /// the local tree was evicted, so the old baseline is stale and would make the - /// next reconcile mistake "locally deleted" for "must re-download". Clearing it - /// leaves an empty baseline + full remote = pure download (devices.md P3). - pub fn sync_entries_clear(&self, folder_id: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "DELETE FROM sync_entry WHERE folder_id = ?1", - params![folder_id], - )?; - Ok(()) - } - - /// Children of `parent`, for the offline `ensure_children` fast path when a - /// listed folder's `children` map entry was trimmed mid-run. Returns `None` - /// when the folder is not marked `listed` (listing unknown, must re-fetch). - pub fn children_if_listed(&self, parent: &NodeUid) -> Result>> { - let conn = self.conn.lock(); - let listed: Option = conn - .query_row( - "SELECT listed FROM nodes WHERE uid = ?1", - params![parent.to_string()], - |r| r.get(0), - ) - .optional()?; - if listed != Some(1) { - return Ok(None); - } - let mut stmt = conn.prepare( - "SELECT node_json FROM nodes - WHERE parent_uid = ?1 AND node_json IS NOT NULL AND trashed = 0", - )?; - let rows = stmt.query_map(params![parent.to_string()], |r| r.get::<_, String>(0))?; - let mut out = Vec::new(); - for json in rows { - out.push(serde_json::from_str(&json?)?); - } - Ok(Some(out)) - } - - // --- Content-cache LRU index (P4) ------------------------------------- - // - // Replaces the per-eviction `read_dir` scans in `ContentCache`. Each cached - // blob/block carries one row keyed by its on-disk filename (`cache_key`), - // tagged with `kind` ('blob' | 'block') so the two byte budgets stay - // separate. `last_accessed` (unix seconds) is the LRU key. The daemon owns - // the on-disk cache and rebuilds this index from disk on open, then keeps it - // in sync on every store/read/evict, so it is authoritative for eviction. - - /// Insert or refresh a cache entry: set its on-disk `size` and bump - /// `last_accessed` to `now`. Called on every store of a blob or block. - pub fn cache_touch(&self, key: &str, kind: &str, size: u64, now: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO cache_entries (cache_key, kind, size_bytes, last_accessed) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(cache_key) DO UPDATE SET - kind = excluded.kind, - size_bytes = excluded.size_bytes, - last_accessed = excluded.last_accessed", - params![key, kind, size as i64, now], - )?; - Ok(()) - } - - /// Bump only `last_accessed` on a cache hit (LRU ordering). A missing row is - /// a no-op — best effort, mirroring the old best-effort mtime touch. - pub fn cache_accessed(&self, key: &str, now: i64) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "UPDATE cache_entries SET last_accessed = ?2 WHERE cache_key = ?1", - params![key, now], - )?; - Ok(()) - } - - /// Drop a single cache-entry row (one evicted blob or block). - pub fn cache_remove(&self, key: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "DELETE FROM cache_entries WHERE cache_key = ?1", - params![key], - )?; - Ok(()) - } - - /// Drop the whole-file blob row for `key` plus every block row for the same - /// uid (`.b`). Called by `ContentCache::evict`, which removes the - /// blob and all of a uid's cached blocks at once. - pub fn cache_remove_all(&self, key: &str) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "DELETE FROM cache_entries WHERE cache_key = ?1 OR cache_key LIKE ?2", - params![key, format!("{key}.b%")], - )?; - Ok(()) - } - - /// Every cache entry of `kind`, ordered least-recently-accessed first, as - /// `(cache_key, size_bytes)`. The budget enforcer sums the sizes and evicts - /// from the front until the cache fits. - pub fn cache_entries_by_kind(&self, kind: &str) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "SELECT cache_key, size_bytes FROM cache_entries - WHERE kind = ?1 ORDER BY last_accessed ASC", - )?; - let rows = stmt.query_map(params![kind], |r| { - Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?.max(0) as u64)) - })?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) - } - - /// Replace the whole cache index with `entries`, as one transaction. The - /// daemon calls this on open, so a crash or an external file deletion can - /// never leave a phantom row inflating the budget total. - /// - /// The clear and the refill are the same commit deliberately: they are one - /// operation — "the index now describes what is on disk" — and a crash - /// between them would otherwise leave an empty index against a full cache - /// directory, i.e. a budget total of zero and nothing eviction can see. - /// - /// One commit also matters for speed. A mount is the only caller and it runs - /// on the open path, so a cache of thousands of blobs used to pay thousands - /// of autocommit fsyncs before the mountpoint appeared. - pub fn cache_rebuild(&self, entries: &[CacheEntryInput<'_>]) -> Result<()> { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute("DELETE FROM cache_entries", [])?; - { - let mut stmt = tx.prepare( - "INSERT INTO cache_entries (cache_key, kind, size_bytes, last_accessed) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(cache_key) DO UPDATE SET - kind = excluded.kind, - size_bytes = excluded.size_bytes, - last_accessed = excluded.last_accessed", - )?; - for e in entries { - stmt.execute(params![e.key, e.kind, e.size as i64, e.last_accessed])?; - } - } - tx.commit()?; - Ok(()) - } - - // --- Pins (P5) -------------------------------------------------------- - // - // The pin registry, formerly `pins.json`. One row per directly-pinned node, - // keyed by uid display string. `recursive` marks a folder pin: its whole - // subtree counts as pinned, resolved against `nodes` via a CTE so a new - // descendant is covered the moment it lands in the node cache. - - /// Record `uid` as pinned under display `path`. `recursive` keeps the whole - /// subtree of a folder. Idempotent — re-pinning refreshes the path/flag. - pub fn pin_add(&self, uid: &str, path: &str, recursive: bool) -> Result<()> { - let conn = self.conn.lock(); - conn.execute( - "INSERT INTO pins (uid, path, recursive) VALUES (?1, ?2, ?3) - ON CONFLICT(uid) DO UPDATE SET - path = excluded.path, recursive = excluded.recursive", - params![uid, path, recursive as i64], - )?; - Ok(()) - } - - /// Drop `uid`'s pin row. Returns whether a row existed. - pub fn pin_remove(&self, uid: &str) -> Result { - let conn = self.conn.lock(); - let n = conn.execute("DELETE FROM pins WHERE uid = ?1", params![uid])?; - Ok(n > 0) - } - - /// Every directly-pinned entry `(uid, path, recursive)`, ordered by uid. - pub fn pin_list(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare("SELECT uid, path, recursive FROM pins ORDER BY uid")?; - let rows = stmt.query_map([], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, String>(1)?, - r.get::<_, i64>(2)? != 0, - )) - })?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) - } - - /// Whether `uid` is pinned — directly, or because a strict ancestor folder - /// is pinned recursively. The ancestor check walks `parent_uid` to the root - /// via a CTE; a direct pin is honoured even when the node has no `nodes` row - /// yet (e.g. a CLI that never hydrates the node cache). - pub fn is_pinned(&self, uid: &str) -> Result { - let conn = self.conn.lock(); - let direct: Option = conn - .query_row("SELECT 1 FROM pins WHERE uid = ?1", params![uid], |r| { - r.get(0) - }) - .optional()?; - if direct.is_some() { - return Ok(true); - } - let anc: Option = conn - .query_row( - "WITH RECURSIVE anc(uid, parent_uid) AS ( - SELECT uid, parent_uid FROM nodes WHERE uid = ?1 - UNION ALL - SELECT n.uid, n.parent_uid FROM nodes n JOIN anc ON n.uid = anc.parent_uid - ) - SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid - WHERE a.uid != ?1 AND p.recursive = 1 - LIMIT 1", - params![uid], - |r| r.get(0), - ) - .optional()?; - Ok(anc.is_some()) - } - - /// Every uid that is pinned, directly or transitively: all direct pins plus - /// every descendant of a recursively-pinned folder. Used to build the - /// eviction-exempt set (the budget enforcer hashes these into cache keys). - pub fn pinned_uids(&self) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "WITH RECURSIVE sub(uid) AS ( - SELECT uid FROM pins WHERE recursive = 1 - UNION - SELECT n.uid FROM nodes n JOIN sub ON n.parent_uid = sub.uid - ) - SELECT uid FROM sub - UNION - SELECT uid FROM pins", - )?; - let rows = stmt.query_map([], |r| r.get::<_, String>(0))?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) - } - - /// Every descendant uid of `folder` (all depths), via a `parent_uid` CTE. - /// Used to evict a recursively-pinned subtree's cached blobs on unpin. - pub fn descendants(&self, folder: &str) -> Result> { - let conn = self.conn.lock(); - let mut stmt = conn.prepare( - "WITH RECURSIVE sub(uid) AS ( - SELECT uid FROM nodes WHERE parent_uid = ?1 - UNION - SELECT n.uid FROM nodes n JOIN sub ON n.parent_uid = sub.uid - ) - SELECT uid FROM sub", - )?; - let rows = stmt.query_map(params![folder], |r| r.get::<_, String>(0))?; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) - } - - // ---- local (non-Drive) file index (schema v6) ---- - - /// Open a new local-index scan generation. Rows written under it survive - /// [`local_finish_scan`](Self::local_finish_scan); rows still carrying an - /// older generation are pruned there as "no longer on disk". - pub fn local_begin_scan(&self) -> Result { - let conn = self.conn.lock(); - let current: i64 = conn - .query_row( - "SELECT value FROM sync_state WHERE key = 'local_scan_gen'", - [], - |row| row.get::<_, String>(0), - ) - .optional()? - .and_then(|v| v.parse().ok()) - .unwrap_or(0); - let next = current + 1; - conn.execute( - "INSERT INTO sync_state (key, value) VALUES ('local_scan_gen', ?1) - ON CONFLICT(key) DO UPDATE SET value = excluded.value", - [next.to_string()], - )?; - Ok(next) - } - - /// Write one batch of walked entries under scan generation `generation`. The - /// FTS index is *not* touched here — it is rebuilt once in - /// [`local_finish_scan`](Self::local_finish_scan). - pub fn local_upsert_batch(&self, generation: i64, entries: &[LocalEntry]) -> Result<()> { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - { - let mut stmt = tx.prepare_cached( - "INSERT INTO local_files (path, name, is_dir, size, mtime, scan_gen) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(path) DO UPDATE SET - name = excluded.name, - is_dir = excluded.is_dir, - size = excluded.size, - mtime = excluded.mtime, - scan_gen = excluded.scan_gen", - )?; - for e in entries { - stmt.execute(params![ - e.path, - e.name, - e.is_dir as i64, - e.size, - e.mtime, - generation - ])?; - } - } - tx.commit()?; - Ok(()) - } - - /// Close scan generation `generation`: drop every row an older scan wrote - /// (those paths are gone from disk), rebuild the FTS index over what remains, - /// and stamp the completion time. Returns the number of indexed entries. - pub fn local_finish_scan(&self, generation: i64, finished_at: i64) -> Result { - let mut conn = self.conn.lock(); - let tx = conn.transaction()?; - tx.execute( - "DELETE FROM local_files WHERE scan_gen != ?1", - params![generation], - )?; - tx.execute_batch("INSERT INTO local_fts(local_fts) VALUES('rebuild');")?; - tx.execute( - "INSERT INTO sync_state (key, value) VALUES ('local_indexed_at', ?1) - ON CONFLICT(key) DO UPDATE SET value = excluded.value", - [finished_at.to_string()], - )?; - let count: i64 = tx.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?; - tx.commit()?; - Ok(count) - } - - /// When the last local scan completed (epoch seconds), or `None` if the index - /// has never been built. The daemon uses this to decide whether a fresh mount - /// needs an immediate rescan or can serve the existing index. - pub fn local_indexed_at(&self) -> Result> { - let conn = self.conn.lock(); - Ok(conn - .query_row( - "SELECT value FROM sync_state WHERE key = 'local_indexed_at'", - [], - |row| row.get::<_, String>(0), - ) - .optional()? - .and_then(|v| v.parse().ok())) - } - - /// Substring search over indexed local file names, newest-modified first - /// within a relevance tier. Mirrors [`search`](Self::search): the trigram - /// index handles queries of at least [`TRIGRAM_MIN`] chars, shorter ones fall - /// back to a `LIKE` scan. - pub fn search_local(&self, query: &str, limit: usize) -> Result> { - let query = query.trim(); - if query.is_empty() { - return Ok(Vec::new()); - } - let conn = self.conn.lock(); - let mut stmt; - let rows = if query.chars().count() < TRIGRAM_MIN { - let pat = format!("%{}%", like_escape(query)); - stmt = conn.prepare( - "SELECT path, name, is_dir, size, mtime FROM local_files - WHERE name LIKE ?1 ESCAPE '\\' - ORDER BY mtime DESC LIMIT ?2", - )?; - stmt.query_map(params![pat, limit as i64], local_hit)? - } else { - let phrase = query - .split_whitespace() - .map(|word| format!("\"{}\"", word.replace('"', "\"\""))) - .collect::>() - .join(" AND "); - stmt = conn.prepare( - "SELECT f.path, f.name, f.is_dir, f.size, f.mtime - FROM local_fts x JOIN local_files f ON f.id = x.rowid - WHERE x.name MATCH ?1 - ORDER BY x.rank LIMIT ?2", - )?; - stmt.query_map(params![phrase, limit as i64], local_hit)? - }; - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) - } -} - -/// One hit from [`Db::search_local`]: an indexed file on the machine itself, not -/// in Drive. `path` is absolute. -pub struct LocalFileHit { - pub path: String, - pub name: String, - pub is_dir: bool, - pub size: i64, - pub mtime: i64, -} - -/// Row mapper for [`Db::search_local`]'s two query paths, which select the same -/// columns in the same order. -fn local_hit(row: &rusqlite::Row) -> rusqlite::Result { - Ok(LocalFileHit { - path: row.get(0)?, - name: row.get(1)?, - is_dir: row.get::<_, i64>(2)? != 0, - size: row.get(3)?, - mtime: row.get(4)?, - }) -} - -/// Row mapper for the `(node_json, uid)` pairs both search paths return. -fn pair(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> { - Ok((row.get(0)?, row.get(1)?)) -} - -/// Drain a `query_map` of [`pair`] rows into a `Vec`, propagating row errors. -fn collect_pairs( - rows: impl Iterator>, -) -> Result> { - let mut out = Vec::new(); - for r in rows { - out.push(r?); - } - Ok(out) -} - -/// Escape `LIKE` wildcards in a user query so `%` and `_` match literally -/// (paired with `ESCAPE '\'` in the statement). -fn like_escape(s: &str) -> String { - s.replace('\\', "\\\\") - .replace('%', "\\%") - .replace('_', "\\_") -} - -/// Resolve a node's mountpoint-relative path by walking `parent_uid` to the -/// root via a recursive CTE. The root (the node with no parent) is excluded, so -/// a top-level file `report.pdf` yields `"report.pdf"`, not `"My Files/report.pdf"`. -fn path_of(conn: &Connection, uid: &str) -> Result { - let mut stmt = conn.prepare( - "WITH RECURSIVE anc(uid, parent_uid, name, depth) AS ( - SELECT uid, parent_uid, name, 0 FROM nodes WHERE uid = ?1 - UNION ALL - SELECT n.uid, n.parent_uid, n.name, anc.depth + 1 - FROM nodes n JOIN anc ON n.uid = anc.parent_uid - ) - SELECT name, parent_uid FROM anc ORDER BY depth DESC", - )?; - let rows = stmt.query_map(params![uid], |r| { - Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)) - })?; - let mut parts = Vec::new(); - for r in rows { - let (name, parent_uid) = r?; - // Skip the root node (no parent); its name is the mount itself. - if parent_uid.is_some() { - parts.push(name); - } - } - Ok(parts.join("/")) -} - -/// Effective plaintext size of a node for the indexed `size` column: the -/// claimed size when known, else the on-storage size; folders are 0. -fn node_size(node: &Node) -> i64 { - match &node.kind { - NodeKind::Folder => 0, - NodeKind::File { - total_size_on_storage, - claimed_size, - .. - } => claimed_size.unwrap_or(*total_size_on_storage).max(0), - } -} - -/// Schema v1: nodes + FTS5 trigram index + cache LRU. `sync_state` is created -/// in [`Db::migrate`] before this runs. -const MIGRATION_V1: &str = " -CREATE TABLE nodes ( - uid TEXT PRIMARY KEY, - parent_uid TEXT, - name TEXT NOT NULL, - is_dir INTEGER NOT NULL, - size INTEGER, - mtime INTEGER NOT NULL, - revision_hash TEXT, - trashed INTEGER NOT NULL DEFAULT 0 -); -CREATE INDEX idx_nodes_parent ON nodes(parent_uid); - -CREATE VIRTUAL TABLE nodes_fts USING fts5( - name, path, content='', tokenize='trigram' -); - -CREATE TABLE cache_entries ( - cache_key TEXT PRIMARY KEY, - size_bytes INTEGER, - last_accessed INTEGER, - is_pinned INTEGER NOT NULL DEFAULT 0 -); -"; - -/// Schema v2: node write-through (P1). `node_json` stores the full [`Node`] so -/// the hot-cache maps rehydrate losslessly; `listed` records whether a folder's -/// child listing was complete when persisted. -const MIGRATION_V2: &str = " -ALTER TABLE nodes ADD COLUMN node_json TEXT; -ALTER TABLE nodes ADD COLUMN listed INTEGER NOT NULL DEFAULT 0; -"; - -/// Schema v3: FTS search (P3). The v1 `nodes_fts` was contentless (`content=''`) -/// and never populated — drop it and recreate as a self-contained trigram index -/// carrying the `uid` (UNINDEXED, so retrievable but not tokenized) alongside the -/// indexed `name`. Backfill from the nodes already persisted by P1. -const MIGRATION_V3: &str = " -DROP TABLE IF EXISTS nodes_fts; -CREATE VIRTUAL TABLE nodes_fts USING fts5( - uid UNINDEXED, name, tokenize='trigram' -); -INSERT INTO nodes_fts (uid, name) - SELECT uid, name FROM nodes WHERE trashed = 0; -"; - -/// Schema v4: content-cache LRU index (P4). The v1 `cache_entries` keyed only by -/// `cache_key`; add a `kind` discriminator so the whole-file blob pool and the -/// block pool — which carry separate byte budgets — can be summed and evicted -/// independently. Existing rows (none in practice, the table was never written) -/// default to `'blob'`. The daemon rebuilds the index from disk on open, so no -/// backfill is needed here. -const MIGRATION_V4: &str = " -ALTER TABLE cache_entries ADD COLUMN kind TEXT NOT NULL DEFAULT 'blob'; -"; - -/// Schema v5: pins move out of `pins.json` into the DB (P5). One row per -/// directly-pinned node, keyed by uid display string so a pin survives -/// renames/moves. `recursive` marks a folder pin whose whole subtree is kept; -/// descendants are resolved on demand against `nodes` via a CTE rather than -/// being expanded into rows here. `ContentCache::open` imports any legacy -/// `pins.json` into this table once, then deletes the file. -const MIGRATION_V5: &str = " -CREATE TABLE pins ( - uid TEXT PRIMARY KEY, - path TEXT NOT NULL, - recursive INTEGER NOT NULL DEFAULT 0 -); -"; - -/// Schema v6: the index of *local* (non-Drive) files, so the launcher prompt can -/// search the machine alongside Drive. `local_files` is keyed by absolute path; -/// `scan_gen` stamps the scan that last saw a row, so a rescan prunes vanished -/// files with one `DELETE` instead of diffing. -/// -/// `local_fts` is an *external-content* FTS5 index over `local_files.name`: the -/// text lives once in the base table and the index is rebuilt in bulk at the end -/// of a scan (`INSERT INTO local_fts(local_fts) VALUES('rebuild')`). That is far -/// cheaper than the delete-then-insert-per-row dance `nodes_fts` needs, because -/// a scan rewrites most rows at once rather than trickling single updates. -const MIGRATION_V6: &str = " -CREATE TABLE local_files ( - id INTEGER PRIMARY KEY, - path TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - is_dir INTEGER NOT NULL, - size INTEGER NOT NULL DEFAULT 0, - mtime INTEGER NOT NULL DEFAULT 0, - scan_gen INTEGER NOT NULL DEFAULT 0 -); - -CREATE VIRTUAL TABLE local_fts USING fts5( - name, content='local_files', content_rowid='id', tokenize='trigram' -); -"; - -/// Schema v7: the photos timeline and the trash listing become persistent, so -/// opening the app paints them from disk instead of re-fetching the world (both -/// were memory-only: the timeline behind a 60 s TTL, the trash not cached at all). -/// -/// `photos.seq` preserves the server's newest-first order, which is the only order -/// the timeline has — `capture_time` ties are common (a burst of shots) and would -/// otherwise shuffle between refreshes. `ratio` and `thumb_state` are *locally -/// learned*: they cost a download to rediscover, so `photos_replace` carries them -/// across a refresh while capture times and order come from the server. -const MIGRATION_V7: &str = " -CREATE TABLE photos ( - uid TEXT PRIMARY KEY, - capture_time INTEGER NOT NULL, - name TEXT, - ratio REAL, - thumb_state INTEGER NOT NULL DEFAULT 0, - seq INTEGER NOT NULL -); -CREATE INDEX idx_photos_seq ON photos(seq); - -CREATE TABLE trash ( - uid TEXT PRIMARY KEY, - name TEXT NOT NULL, - is_dir INTEGER NOT NULL, - size INTEGER NOT NULL DEFAULT 0, - mtime INTEGER NOT NULL DEFAULT 0 -); -"; - -/// Schema v8: device sync (devices.md). This machine registers as one Proton Drive -/// **Device** (a share + root folder on the main volume); `device` is a singleton -/// row cached so we reuse the same device across restarts instead of creating a -/// new one each run. `sync_folder` is one row per local folder the user added, -/// each mapped to a folder under the device root; `mode` is `mirror` (full local -/// copy, two-way synced) or `ondemand` (FUSE mount at `local_path`, no local -/// storage). `sync_entry` is the per-file baseline for three-way merge — added in -/// this migration so Phase 2 has the table, though Phase 1 leaves it empty. -const MIGRATION_V8: &str = " -CREATE TABLE device ( - uid TEXT PRIMARY KEY, - share_id TEXT NOT NULL, - root_uid TEXT NOT NULL, - name TEXT NOT NULL, - created INTEGER NOT NULL DEFAULT 0 -); - -CREATE TABLE sync_folder ( - id INTEGER PRIMARY KEY, - local_path TEXT NOT NULL UNIQUE, - remote_uid TEXT NOT NULL, - remote_share_id TEXT NOT NULL, - mode TEXT NOT NULL DEFAULT 'mirror', - state TEXT NOT NULL DEFAULT 'idle', - last_sync INTEGER NOT NULL DEFAULT 0 -); - -CREATE TABLE sync_entry ( - folder_id INTEGER NOT NULL, - rel_path TEXT NOT NULL, - remote_uid TEXT, - local_mtime INTEGER NOT NULL DEFAULT 0, - local_size INTEGER NOT NULL DEFAULT 0, - remote_hash TEXT, - remote_rev TEXT, - PRIMARY KEY (folder_id, rel_path) -); -"; - -const MIGRATION_V9: &str = " -CREATE TABLE activity ( - id INTEGER PRIMARY KEY, - time INTEGER NOT NULL, - kind TEXT NOT NULL, - target TEXT NOT NULL, - detail TEXT NOT NULL DEFAULT '', - ok INTEGER NOT NULL DEFAULT 1 -); - -CREATE INDEX activity_time ON activity(time DESC); -"; - -/// Schema v10: a mode switch the user asked for is queued rather than rejected -/// when the folder is mid-pass or has un-uploaded changes, so `pending_mode` -/// records the intent until the engine can act on it. NULL is the resting state: -/// the folder is already where the user wants it. -const MIGRATION_V10: &str = " -ALTER TABLE sync_folder ADD COLUMN pending_mode TEXT; -"; - -/// Schema v11: writes no longer upload inside the FUSE handler. `release` stages -/// the bytes and records the intended upload here, and a drain worker performs it -/// (offline.md Phase 3). The row outlives the process, so a write survives both a -/// dead network and a restart. -/// -/// `blob_path` points into the content cache's `staging/` dir and `meta_json` is -/// the [`StagedWrite`] sidecar describing which of its bytes are real — an -/// incomplete blob must be gap-filled from the remote base before it can be -/// uploaded. `next_attempt_at` is a ms deadline implementing retry backoff. -/// -/// [`StagedWrite`]: crate::cache::StagedWrite -const MIGRATION_V11: &str = " -CREATE TABLE pending_op ( - id INTEGER PRIMARY KEY, - kind TEXT NOT NULL, - uid TEXT NOT NULL, - blob_path TEXT, - meta_json TEXT, - created_at INTEGER NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - next_attempt_at INTEGER NOT NULL DEFAULT 0 -); - -CREATE INDEX pending_op_uid ON pending_op(uid); -"; - -/// Schema v12: `pending_op` also carries mutations that *create* a node, which a -/// revision op never had to describe — it always addressed a node the server had -/// already minted a uid for (offline.md Phase 3b). -/// -/// An offline `create`/`mkdir` cannot get a real uid, so the node is invented -/// locally under a `local~` placeholder and the op records where it goes -/// (`parent_uid`) and what it is called (`name`). Both are nullable because a -/// revision op sets neither. -/// -/// `parent_uid` is indexed: draining a folder rewrites every queued child that -/// points at its placeholder. -const MIGRATION_V12: &str = " -ALTER TABLE pending_op ADD COLUMN parent_uid TEXT; -ALTER TABLE pending_op ADD COLUMN name TEXT; - -CREATE INDEX pending_op_parent ON pending_op(parent_uid); -"; - -/// Schema v13: photos record their media type and a derived `kind` so the -/// timeline can be split into Photos / Videos / Raw tabs. `media_type` is -/// nullable — it is filled once the daemon resolves a photo's node — while -/// `kind` (see [`crate::control::PhotoKind`]) defaults to `0` (still photo), the -/// classification anything unresolved falls back to. Existing rows are -/// reclassified for free on the next timeline refresh, so no backfill is needed. -const MIGRATION_V13: &str = " -ALTER TABLE photos ADD COLUMN media_type TEXT; -ALTER TABLE photos ADD COLUMN kind INTEGER NOT NULL DEFAULT 0; -"; - -#[cfg(test)] -mod tests { - use super::*; - - /// `Db::open` applies the pragmas that `open_in_memory` skips, so the WAL - /// settings can only be checked against a real file. - #[test] - fn open_bounds_the_wal_size() { - let path = std::env::temp_dir().join(format!("pdfs-db-wal-{}.db", std::process::id())); - let _ = std::fs::remove_file(&path); - let db = Db::open(&path).unwrap(); - let conn = db.conn.lock(); - - let mode: String = conn - .query_row("PRAGMA journal_mode", [], |r| r.get(0)) - .unwrap(); - assert_eq!(mode, "wal"); - // Without this the WAL is reused in place but never shrinks, so one - // oversized transaction strands its high-water mark on disk forever. - let limit: i64 = conn - .query_row("PRAGMA journal_size_limit", [], |r| r.get(0)) - .unwrap(); - assert_eq!(limit, WAL_SIZE_LIMIT); - - drop(conn); - drop(db); - let _ = std::fs::remove_file(&path); - } - use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId}; - - fn uid(link: &str) -> NodeUid { - NodeUid::new(VolumeId::from("vol"), LinkId::from(link)) - } - - // `NodeVerification` is not re-exported, so build test nodes by - // deserializing JSON (the field has a serde default and can be omitted). - fn node_from( - parent: serde_json::Value, - link: &str, - name: &str, - kind: serde_json::Value, - ) -> Node { - let v = serde_json::json!({ - "uid": {"volume_id": "vol", "link_id": link}, - "parent_uid": parent, - "kind": kind, - "name": name, - "creation_time": 100, - "modification_time": 200, - "trashed": false, - "signature_email": null, - }); - serde_json::from_value(v).unwrap() - } - - #[test] - fn photos_replace_keeps_what_was_learned_and_drops_what_left() { - let db = Db::open_in_memory().unwrap(); - db.photos_replace(&[ - ("p1".into(), 300, None, Some("video/mp4".into())), - ("p2".into(), 200, None, None), - ("p3".into(), 100, None, None), - ]) - .unwrap(); - - // A thumbnail attempt teaches us p1's ratio and that p2 can never have one. - db.photo_set_thumb("p1", THUMB_HAVE, Some(1.5)).unwrap(); - db.photo_set_thumb("p2", THUMB_NONE, None).unwrap(); - - // The next refresh brings a new photo, keeps p1 and p2, and loses p3. - // p1's media type arrives as `None` this time and must not be forgotten. - db.photos_replace(&[ - ("p0".into(), 400, Some("new.jpg".into()), None), - ("p1".into(), 300, None, None), - ("p2".into(), 200, None, None), - ]) - .unwrap(); - - let page = db.photos_page(0, 10, None, None).unwrap(); - assert_eq!( - page.iter().map(|p| p.uid.as_str()).collect::>(), - ["p0", "p1", "p2"], - "server order is preserved, and the dropped photo is gone" - ); - assert_eq!(page[0].name.as_deref(), Some("new.jpg")); - // Ratios and verdicts cost a download to rediscover: they survive a refresh. - assert_eq!(page[1].ratio, Some(1.5)); - assert_eq!(page[1].thumb_state, THUMB_HAVE); - assert_eq!(page[2].thumb_state, THUMB_NONE); - // A photo we know nothing about yet starts blank. - assert_eq!(page[0].ratio, None); - assert_eq!(page[0].thumb_state, THUMB_UNKNOWN); - // Media type is learned-and-kept like the ratio: p1 keeps the video type - // it was first seen with even though the later refresh carried `None`, so - // it stays classified as a video. - assert_eq!(page[1].kind, crate::control::PhotoKind::Video); - - assert_eq!(db.photos_count().unwrap(), 3); - // The counts break down by tab, and a filtered page returns only its tab. - assert_eq!(db.photos_counts().unwrap(), (2, 1, 0)); - let videos = db - .photos_page(0, 10, Some(crate::control::PhotoKind::Video), None) - .unwrap(); - assert_eq!( - videos.iter().map(|p| p.uid.as_str()).collect::>(), - ["p1"] - ); - let by_uid = db.photos_by_uid(&["p2".into()]).unwrap(); - assert_eq!(by_uid.len(), 1); - assert_eq!(by_uid[0].capture_time, 200); - - // A date-range page keeps only the window's photos: [150, 350) is p1+p2, - // not p0 at 400. Combined with a kind filter both conditions apply. - let ranged = db.photos_page(0, 10, None, Some((150, 350))).unwrap(); - assert_eq!( - ranged.iter().map(|p| p.uid.as_str()).collect::>(), - ["p1", "p2"] - ); - let ranged_video = db - .photos_page(0, 10, Some(crate::control::PhotoKind::Video), Some((150, 350))) - .unwrap(); - assert_eq!( - ranged_video.iter().map(|p| p.uid.as_str()).collect::>(), - ["p1"] - ); - // All three surviving photos sit in the same (1970-01) local month. - let months = db.photos_months(None).unwrap(); - assert_eq!(months.len(), 1); - assert_eq!(months[0].2, 3); - } - - #[test] - fn photos_page_slices_the_timeline_in_order() { - let db = Db::open_in_memory().unwrap(); - let items: Vec<_> = (0..5) - .map(|i| (format!("p{i}"), 500 - i as i64, None, None::)) - .collect(); - db.photos_replace(&items).unwrap(); - - let page = db.photos_page(2, 2, None, None).unwrap(); - assert_eq!( - page.iter().map(|p| p.uid.as_str()).collect::>(), - ["p2", "p3"] - ); - assert!(db.photos_page(9, 2, None, None).unwrap().is_empty()); - } - - #[test] - fn trash_replace_lists_folders_first() { - let db = Db::open_in_memory().unwrap(); - db.trash_replace(&[ - StoredTrash { - uid: "t1".into(), - name: "zeta.txt".into(), - is_dir: false, - size: 10, - mtime: 1, - }, - StoredTrash { - uid: "t2".into(), - name: "Alpha".into(), - is_dir: true, - size: 0, - mtime: 2, - }, - ]) - .unwrap(); - - let items = db.trash_list().unwrap(); - assert_eq!( - items.iter().map(|i| i.name.as_str()).collect::>(), - ["Alpha", "zeta.txt"] - ); - - // A replace is a replace: emptying the trash on the server empties it here. - db.trash_replace(&[]).unwrap(); - assert!(db.trash_list().unwrap().is_empty()); - } - - #[test] - fn state_stamps_round_trip_and_clear() { - let db = Db::open_in_memory().unwrap(); - assert_eq!(db.state_i64("photos_synced_ms").unwrap(), None); - db.set_state_i64("photos_synced_ms", 1234).unwrap(); - assert_eq!(db.state_i64("photos_synced_ms").unwrap(), Some(1234)); - db.clear_state("photos_synced_ms").unwrap(); - assert_eq!( - db.state_i64("photos_synced_ms").unwrap(), - None, - "a cleared stamp reads as never fetched, so the next request blocks on a refresh" - ); - } - - fn folder(link: &str, parent: Option<&str>, name: &str) -> Node { - let parent = match parent { - Some(p) => serde_json::json!({"volume_id": "vol", "link_id": p}), - None => serde_json::Value::Null, - }; - node_from(parent, link, name, serde_json::json!("Folder")) - } - - fn file(link: &str, parent: &str, name: &str, size: i64) -> Node { - let kind = serde_json::json!({ - "File": { - "media_type": "text/plain", - "total_size_on_storage": size + 10, - "claimed_size": size, - "claimed_modification_time": null, - } - }); - node_from( - serde_json::json!({"volume_id": "vol", "link_id": parent}), - link, - name, - kind, - ) - } - - /// Recovering the root by uid is what lets the daemon mount offline - /// (offline.md Phase 1): the uid is remembered in `sync_state`, the node - /// itself comes back out of `nodes`. - #[test] - fn node_by_uid_recovers_a_stored_node() { - let db = Db::open_in_memory().unwrap(); - let root = folder("root", None, "My Files"); - db.upsert_node(&root).unwrap(); - db.set_state_str("root_uid", &root.uid.to_string()).unwrap(); - - let key = db.state_str("root_uid").unwrap().unwrap(); - let got = db.node_by_uid(&key).unwrap().expect("root recovered"); - assert_eq!(got.uid, root.uid); - assert_eq!(got.name, "My Files"); - assert!(got.is_folder()); - - assert!(db.node_by_uid("vol~nope").unwrap().is_none()); - assert!(db.state_str("never_written").unwrap().is_none()); - } - - #[test] - fn upsert_and_load_all_roundtrip() { - let db = Db::open_in_memory().unwrap(); - let root = folder("root", None, "My Files"); - let child = file("f1", "root", "hello.txt", 42); - db.upsert_node(&root).unwrap(); - db.upsert_node(&child).unwrap(); - - let loaded = db.load_all().unwrap(); - assert_eq!(loaded.len(), 2); - let f = loaded.iter().find(|s| s.node.uid == uid("f1")).unwrap(); - assert_eq!(f.node.name, "hello.txt"); - assert!(!f.listed); - match &f.node.kind { - NodeKind::File { claimed_size, .. } => assert_eq!(*claimed_size, Some(42)), - _ => panic!("expected file"), - } - } - - #[test] - fn upsert_nodes_and_load_all_roundtrip() { - let db = Db::open_in_memory().unwrap(); - let root = folder("root", None, "My Files"); - let child1 = file("f1", "root", "hello.txt", 42); - let child2 = file("f2", "root", "world.txt", 100); - db.upsert_nodes(&[root, child1, child2]).unwrap(); - - let loaded = db.load_all().unwrap(); - assert_eq!(loaded.len(), 3); - let f1 = loaded.iter().find(|s| s.node.uid == uid("f1")).unwrap(); - assert_eq!(f1.node.name, "hello.txt"); - let f2 = loaded.iter().find(|s| s.node.uid == uid("f2")).unwrap(); - assert_eq!(f2.node.name, "world.txt"); - } - - #[test] - fn upsert_is_idempotent_update() { - let db = Db::open_in_memory().unwrap(); - let mut n = folder("root", None, "My Files"); - db.upsert_node(&n).unwrap(); - n.name = "Renamed".into(); - db.upsert_node(&n).unwrap(); - let loaded = db.load_all().unwrap(); - assert_eq!(loaded.len(), 1); - assert_eq!(loaded[0].node.name, "Renamed"); - } - - #[test] - fn delete_node_removes_row() { - let db = Db::open_in_memory().unwrap(); - let n = folder("root", None, "My Files"); - db.upsert_node(&n).unwrap(); - db.delete_node(&uid("root")).unwrap(); - assert!(db.load_all().unwrap().is_empty()); - } - - #[test] - fn children_if_listed_gated_on_flag() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&file("f1", "root", "a.txt", 1)).unwrap(); - db.upsert_node(&file("f2", "root", "b.txt", 2)).unwrap(); - - // Not listed yet → unknown, force a re-fetch. - assert!(db.children_if_listed(&uid("root")).unwrap().is_none()); - - db.set_listed(&uid("root"), true).unwrap(); - let kids = db.children_if_listed(&uid("root")).unwrap().unwrap(); - assert_eq!(kids.len(), 2); - } - - #[test] - fn children_if_listed_excludes_trashed() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - let mut trashed = file("f1", "root", "a.txt", 1); - trashed.trashed = true; - db.upsert_node(&trashed).unwrap(); - db.set_listed(&uid("root"), true).unwrap(); - assert_eq!( - db.children_if_listed(&uid("root")).unwrap().unwrap().len(), - 0 - ); - } - - fn local(path: &str, name: &str, is_dir: bool) -> LocalEntry { - LocalEntry { - path: path.into(), - name: name.into(), - is_dir, - size: 10, - mtime: 5, - } - } - - /// A local scan is searchable by substring, and a *later* scan prunes the - /// paths it no longer sees — including out of the FTS index, so a deleted - /// file cannot keep surfacing in the prompt. - #[test] - fn local_scan_indexes_then_prunes_stale_paths() { - let db = Db::open_in_memory().unwrap(); - - let gen1 = db.local_begin_scan().unwrap(); - db.local_upsert_batch( - gen1, - &[ - local("/home/u/docs/report.pdf", "report.pdf", false), - local("/home/u/docs/notes.md", "notes.md", false), - ], - ) - .unwrap(); - assert_eq!(db.local_finish_scan(gen1, 1_000).unwrap(), 2); - - // Trigram index gives substring (not just prefix) matches. - let hits = db.search_local("port", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].path, "/home/u/docs/report.pdf"); - assert!(!hits[0].is_dir); - assert_eq!(db.local_indexed_at().unwrap(), Some(1_000)); - - // Second scan sees only notes.md → report.pdf is gone from disk. - let gen2 = db.local_begin_scan().unwrap(); - db.local_upsert_batch(gen2, &[local("/home/u/docs/notes.md", "notes.md", false)]) - .unwrap(); - assert_eq!(db.local_finish_scan(gen2, 2_000).unwrap(), 1); - assert!(db.search_local("report", 10).unwrap().is_empty()); - assert_eq!(db.search_local("notes", 10).unwrap().len(), 1); - } - - /// Queries below the trigram minimum still match, via the `LIKE` fallback. - #[test] - fn local_search_short_query_like_fallback() { - let db = Db::open_in_memory().unwrap(); - let g = db.local_begin_scan().unwrap(); - db.local_upsert_batch(g, &[local("/home/u/a.txt", "a.txt", false)]) - .unwrap(); - db.local_finish_scan(g, 1).unwrap(); - assert_eq!(db.search_local("a", 10).unwrap().len(), 1); - assert!(db.search_local("", 10).unwrap().is_empty()); - } - - #[test] - fn event_cursor_roundtrip() { - let db = Db::open_in_memory().unwrap(); - // Absent before first write → seed from server head on first mount. - assert!(db.get_event_cursor().unwrap().is_none()); - db.set_event_cursor("evt-1").unwrap(); - assert_eq!(db.get_event_cursor().unwrap().as_deref(), Some("evt-1")); - // Overwrites, not appends. - db.set_event_cursor("evt-2").unwrap(); - assert_eq!(db.get_event_cursor().unwrap().as_deref(), Some("evt-2")); - } - - #[test] - fn search_trigram_substring_and_path() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&folder("docs", Some("root"), "Documents")) - .unwrap(); - db.upsert_node(&file("f1", "docs", "report.pdf", 1)) - .unwrap(); - db.upsert_node(&file("f2", "root", "notes.txt", 2)).unwrap(); - db.upsert_node(&file( - "f3", - "root", - "Rampage Open Air 2026 - order 166765244.pdf", - 3, - )) - .unwrap(); - - // Substring match (trigram), not just prefix. - let hits = db.search("port", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].node.name, "report.pdf"); - // Path is mountpoint-relative, root excluded. - assert_eq!(hits[0].path, "Documents/report.pdf"); - - // Top-level file → bare name. - let hits = db.search("notes", 10).unwrap(); - assert_eq!(hits[0].path, "notes.txt"); - - // Multi-term FTS5 query matching (out of order, separated terms) - let hits = db.search("rampage 2026", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!( - hits[0].node.name, - "Rampage Open Air 2026 - order 166765244.pdf" - ); - } - - #[test] - fn search_excludes_trashed_and_respects_limit() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&file("f1", "root", "alpha.txt", 1)).unwrap(); - let mut gone = file("f2", "root", "alphb.txt", 1); - gone.trashed = true; - db.upsert_node(&gone).unwrap(); - - let hits = db.search("alph", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].node.name, "alpha.txt"); - - db.upsert_node(&file("f3", "root", "alphc.txt", 1)).unwrap(); - assert_eq!(db.search("alph", 1).unwrap().len(), 1); - } - - #[test] - fn search_short_query_like_fallback() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&file("f1", "root", "ab.txt", 1)).unwrap(); - // Under trigram min length → LIKE path still finds it. - let hits = db.search("ab", 10).unwrap(); - assert_eq!(hits.len(), 1); - assert_eq!(hits[0].node.name, "ab.txt"); - } - - #[test] - fn search_drops_fts_row_on_delete_and_trash() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&file("f1", "root", "unique.txt", 1)) - .unwrap(); - assert_eq!(db.search("unique", 10).unwrap().len(), 1); - - // Re-upsert as trashed → leaves the index. - let mut t = file("f1", "root", "unique.txt", 1); - t.trashed = true; - db.upsert_node(&t).unwrap(); - assert_eq!(db.search("unique", 10).unwrap().len(), 0); - - // Resurrect, then hard-delete. - db.upsert_node(&file("f1", "root", "unique.txt", 1)) - .unwrap(); - assert_eq!(db.search("unique", 10).unwrap().len(), 1); - db.delete_node(&uid("f1")).unwrap(); - assert_eq!(db.search("unique", 10).unwrap().len(), 0); - } - - fn activity(target: &str, kind: ActivityKind, ok: bool) -> ActivityEntry { - ActivityEntry { - time: 1700, - kind, - target: target.into(), - detail: "detail".into(), - ok, - } - } - - #[test] - fn activity_reads_back_newest_first() { - let db = Db::open_in_memory().unwrap(); - db.activity_add(&activity("a.txt", ActivityKind::Upload, true)) - .unwrap(); - db.activity_add(&activity("b.txt", ActivityKind::Download, false)) - .unwrap(); - - let items = db.activity_list(10).unwrap(); - assert_eq!(items.len(), 2); - assert_eq!(items[0].target, "b.txt"); - assert_eq!(items[0].kind, ActivityKind::Download); - assert!(!items[0].ok); - assert_eq!(items[0].detail, "detail"); - assert_eq!(items[0].time, 1700); - assert_eq!(items[1].target, "a.txt"); - - assert_eq!(db.activity_list(1).unwrap().len(), 1); - } - - #[test] - fn activity_prunes_to_the_keep_limit() { - let db = Db::open_in_memory().unwrap(); - for i in 0..(ACTIVITY_KEEP + 10) { - db.activity_add(&activity(&format!("f{i}"), ActivityKind::Upload, true)) - .unwrap(); - } - let items = db.activity_list(ACTIVITY_KEEP as usize * 2).unwrap(); - assert_eq!(items.len(), ACTIVITY_KEEP as usize); - // The newest survive; the oldest are the ones dropped. - assert_eq!(items[0].target, format!("f{}", ACTIVITY_KEEP + 9)); - } - - #[test] - fn opens_and_migrates() { - let db = Db::open_in_memory().unwrap(); - let version: String = db - .with_conn(|c| { - Ok(c.query_row( - "SELECT value FROM sync_state WHERE key = 'schema_version'", - [], - |r| r.get(0), - )?) - }) - .unwrap(); - assert_eq!(version, SCHEMA_VERSION.to_string()); - } - - #[test] - fn a_second_write_supersedes_the_first_pending_op() { - let db = Db::open_in_memory().unwrap(); - let op = |blob: &str| PendingOp { - id: 0, - kind: OP_REVISION.to_string(), - uid: uid("a").to_string(), - parent_uid: None, - name: None, - blob_path: Some(blob.to_string()), - meta_json: Some("{}".to_string()), - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }; - - let (_, superseded) = db.enqueue_op(&op("/staging/first")).unwrap(); - assert_eq!(superseded, None, "nothing to supersede on the first write"); - - // The newer blob already contains everything the older one did, so the - // older op must go — and its blob must be reported so it can be deleted - // rather than leaked. - let (id2, superseded) = db.enqueue_op(&op("/staging/second")).unwrap(); - assert_eq!(superseded.as_deref(), Some("/staging/first")); - - let ops = db.pending_ops().unwrap(); - assert_eq!(ops.len(), 1, "one queued upload per node"); - assert_eq!(ops[0].id, id2); - assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/second")); - assert_eq!(db.pending_op_counts().unwrap().uploads, 1); - } - - /// Deleting a folder that was created offline must take the ops queued - /// underneath it with it: they name a placeholder parent that will now never - /// become real, so nothing could ever drain them and nothing is left to - /// rewrite them. - #[test] - fn deleting_a_queued_folder_takes_its_queued_children_with_it() { - let db = Db::open_in_memory().unwrap(); - let op = |kind: &str, uid: &str, parent: &str, blob: Option<&str>| PendingOp { - id: 0, - kind: kind.to_string(), - uid: uid.to_string(), - parent_uid: Some(parent.to_string()), - name: Some("n".to_string()), - blob_path: blob.map(str::to_string), - meta_json: None, - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }; - let root = uid("root").to_string(); - db.enqueue_op(&op(OP_MKDIR, "local~dir", &root, None)) - .unwrap(); - db.enqueue_op(&op(OP_MKDIR, "local~sub", "local~dir", None)) - .unwrap(); - db.enqueue_op(&op( - OP_CREATE, - "local~deep", - "local~sub", - Some("/staging/deep"), - )) - .unwrap(); - // A sibling outside the doomed subtree must survive. - db.enqueue_op(&op(OP_CREATE, "local~other", &root, Some("/staging/other"))) - .unwrap(); - - let blobs = db.delete_ops_for_uid("local~dir").unwrap(); - assert_eq!( - blobs, - vec!["/staging/deep"], - "the subtree's bytes come back" - ); - - let left: Vec = db - .pending_ops() - .unwrap() - .into_iter() - .map(|o| o.uid) - .collect(); - assert_eq!(left, vec!["local~other"]); - } - - /// A rename is the node's desired end state, so the newest one is the only - /// one worth performing — but it must not disturb the queued *upload* of the - /// same node, which is unrelated work. - #[test] - fn a_second_rename_supersedes_the_first_but_leaves_the_upload_alone() { - let db = Db::open_in_memory().unwrap(); - let rename = |name: &str| PendingOp { - id: 0, - kind: OP_RENAME.to_string(), - uid: uid("a").to_string(), - parent_uid: Some(uid("parent").to_string()), - name: Some(name.to_string()), - blob_path: None, - meta_json: None, - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }; - db.enqueue_op(&PendingOp { - id: 0, - kind: OP_REVISION.to_string(), - uid: uid("a").to_string(), - parent_uid: None, - name: None, - blob_path: Some("/staging/blob".to_string()), - meta_json: Some("{}".to_string()), - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }) - .unwrap(); - - db.enqueue_op(&rename("first")).unwrap(); - let (_, superseded) = db.enqueue_op(&rename("second")).unwrap(); - assert_eq!(superseded, None, "a rename owns no blob to clean up"); - - let ops = db.pending_ops().unwrap(); - assert_eq!(ops.len(), 2, "the queued upload survives the rename"); - let renames: Vec<_> = ops.iter().filter(|o| o.kind == OP_RENAME).collect(); - assert_eq!(renames.len(), 1, "one rename per node"); - assert_eq!(renames[0].name.as_deref(), Some("second")); - - let counts = db.pending_op_counts().unwrap(); - assert_eq!(counts.uploads, 1, "the revision is the only upload"); - assert_eq!(counts.changes, 1, "the rename carries no bytes"); - } - - /// Renaming a node whose create has not drained rewrites the intent rather - /// than queueing a rename against a uid the server has never issued. - #[test] - fn renaming_a_queued_create_rewrites_its_target() { - let db = Db::open_in_memory().unwrap(); - let local = "local~abc"; - db.enqueue_op(&PendingOp { - id: 0, - kind: OP_CREATE.to_string(), - uid: local.to_string(), - parent_uid: Some(uid("old").to_string()), - name: Some("draft.txt".to_string()), - blob_path: Some("/staging/blob".to_string()), - meta_json: Some("{}".to_string()), - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }) - .unwrap(); - - let rewritten = db - .rewrite_op_target(local, &uid("new").to_string(), "final.txt") - .unwrap(); - assert!(rewritten); - - let ops = db.pending_ops().unwrap(); - assert_eq!(ops.len(), 1, "a rewrite is not a second op"); - assert_eq!(ops[0].name.as_deref(), Some("final.txt")); - assert_eq!( - ops[0].parent_uid.as_deref(), - Some(uid("new").to_string()).as_deref() - ); - assert_eq!( - ops[0].blob_path.as_deref(), - Some("/staging/blob"), - "the bytes riding on the create are untouched" - ); - - // Once the create has drained there is no intent left to rewrite, and the - // caller has to rename the real node instead. - db.delete_op(ops[0].id).unwrap(); - assert!( - !db.rewrite_op_target(local, &uid("new").to_string(), "final.txt") - .unwrap() - ); - } - - #[test] - fn a_write_folds_into_a_queued_create_instead_of_superseding_it() { - let db = Db::open_in_memory().unwrap(); - let local = "local~abc"; - db.enqueue_op(&PendingOp { - id: 0, - kind: OP_CREATE.to_string(), - uid: local.to_string(), - parent_uid: Some(uid("parent").to_string()), - name: Some("new.txt".to_string()), - blob_path: None, - meta_json: None, - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }) - .unwrap(); - - let first = db - .attach_blob_to_create(local, "/staging/first", "{}") - .unwrap() - .expect("create is still queued"); - assert_eq!(first.superseded, None); - - // Rewriting the file before it drains replaces the bytes but must leave - // the create itself alone: it is the only thing that will ever bring this - // file into existence remotely. - let second = db - .attach_blob_to_create(local, "/staging/second", "{}") - .unwrap() - .expect("create is still queued"); - assert_eq!(second.superseded.as_deref(), Some("/staging/first")); - - let ops = db.pending_ops().unwrap(); - assert_eq!(ops.len(), 1, "still exactly one create"); - assert_eq!(ops[0].kind, OP_CREATE); - assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/second")); - assert_eq!(ops[0].name.as_deref(), Some("new.txt")); - } - - #[test] - fn attaching_to_an_already_drained_create_reports_it_is_gone() { - let db = Db::open_in_memory().unwrap(); - // No create queued: the caller must fall back to a revision op rather - // than silently dropping the bytes. - let out = db - .attach_blob_to_create("local~gone", "/staging/x", "{}") - .unwrap(); - assert!(out.is_none()); - } - - #[test] - fn draining_a_folder_repoints_its_queued_children() { - let db = Db::open_in_memory().unwrap(); - let local_dir = "local~dir"; - let real_dir = uid("realdir").to_string(); - db.enqueue_op(&PendingOp { - id: 0, - kind: OP_CREATE.to_string(), - uid: "local~child".to_string(), - parent_uid: Some(local_dir.to_string()), - name: Some("inside.txt".to_string()), - blob_path: Some("/staging/child".to_string()), - meta_json: Some("{}".to_string()), - created_at: 2, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }) - .unwrap(); - - db.remap_local_uid(local_dir, &real_dir).unwrap(); - - // The child was queued against a folder that did not exist yet. Once the - // folder is real, the child must target the server's uid — otherwise the - // upload would address `local~dir` and 404. - let ops = db.pending_ops().unwrap(); - assert_eq!(ops[0].parent_uid.as_deref(), Some(real_dir.as_str())); - } - - #[test] - fn a_failed_op_stays_queued_with_backoff() { - let db = Db::open_in_memory().unwrap(); - db.enqueue_op(&PendingOp { - id: 0, - kind: OP_REVISION.to_string(), - uid: uid("a").to_string(), - parent_uid: None, - name: None, - blob_path: Some("/staging/blob".to_string()), - meta_json: Some("{}".to_string()), - created_at: 1, - attempts: 0, - last_error: None, - next_attempt_at: 0, - }) - .unwrap(); - let id = db.pending_ops().unwrap()[0].id; - - db.record_op_failure(id, "network unreachable", 5_000) - .unwrap(); - - // The staged blob is the only copy of the user's bytes: a failure must - // never drop the row, only defer it. - let ops = db.pending_ops().unwrap(); - assert_eq!(ops.len(), 1); - assert_eq!(ops[0].attempts, 1); - assert_eq!(ops[0].last_error.as_deref(), Some("network unreachable")); - assert_eq!(ops[0].next_attempt_at, 5_000); - - db.record_op_failure(id, "still down", 9_000).unwrap(); - assert_eq!(db.pending_ops().unwrap()[0].attempts, 2); - - db.delete_op(id).unwrap(); - assert_eq!(db.pending_op_counts().unwrap().uploads, 0); - } - - #[test] - fn migrate_is_idempotent() { - let db = Db::open_in_memory().unwrap(); - // Second migrate is a no-op (already at head) and must not error. - db.migrate().unwrap(); - } - - /// A queued mode switch is a promise the daemon has to keep across a restart, - /// so it lives in the row, and reaching the mode is what retires it. - #[test] - fn pending_mode_is_queued_until_the_mode_is_reached() { - let db = Db::open_in_memory().unwrap(); - let id = db - .sync_folder_add("/home/me/Downloads", "v~l", "s") - .unwrap(); - assert_eq!(db.sync_folder_get(id).unwrap().unwrap().pending_mode, None); - - db.sync_folder_set_pending_mode(id, Some("ondemand")) - .unwrap(); - assert_eq!( - db.sync_folder_get(id) - .unwrap() - .unwrap() - .pending_mode - .as_deref(), - Some("ondemand") - ); - // The listing carries it too — it is what the front-ends paint from. - assert_eq!( - db.sync_folder_list().unwrap()[0].pending_mode.as_deref(), - Some("ondemand") - ); - - // Landing the switch satisfies the request: a `pending_mode` outliving it - // would have the engine try to apply the same switch on every later pass. - db.sync_folder_set_mode(id, "ondemand").unwrap(); - let folder = db.sync_folder_get(id).unwrap().unwrap(); - assert_eq!(folder.mode, "ondemand"); - assert_eq!(folder.pending_mode, None); - - // And the user can withdraw a request that hasn't landed yet. - db.sync_folder_set_pending_mode(id, Some("mirror")).unwrap(); - db.sync_folder_set_pending_mode(id, None).unwrap(); - let folder = db.sync_folder_get(id).unwrap().unwrap(); - assert_eq!(folder.mode, "ondemand"); - assert_eq!(folder.pending_mode, None); - } - - #[test] - fn cache_index_touch_access_and_lru_order() { - let db = Db::open_in_memory().unwrap(); - db.cache_touch("k1", "blob", 100, 10).unwrap(); - db.cache_touch("k2", "blob", 200, 20).unwrap(); - // LRU-first: k1 (older access) before k2. - let rows = db.cache_entries_by_kind("blob").unwrap(); - assert_eq!(rows, vec![("k1".into(), 100), ("k2".into(), 200)]); - - // Accessing k1 moves it to the back (most recent). - db.cache_accessed("k1", 30).unwrap(); - let rows = db.cache_entries_by_kind("blob").unwrap(); - assert_eq!(rows[0].0, "k2"); - assert_eq!(rows[1].0, "k1"); - - // Re-touch updates size, not just time. - db.cache_touch("k1", "blob", 150, 40).unwrap(); - let rows = db.cache_entries_by_kind("blob").unwrap(); - assert_eq!(rows.iter().find(|(k, _)| k == "k1").unwrap().1, 150); - } - - #[test] - fn cache_index_kinds_are_separate() { - let db = Db::open_in_memory().unwrap(); - db.cache_touch("blob1", "blob", 100, 1).unwrap(); - db.cache_touch("blk1.b0", "block", 50, 1).unwrap(); - assert_eq!(db.cache_entries_by_kind("blob").unwrap().len(), 1); - assert_eq!(db.cache_entries_by_kind("block").unwrap().len(), 1); - } - - #[test] - fn cache_index_remove_and_remove_all() { - let db = Db::open_in_memory().unwrap(); - // A blob plus two of its blocks (key prefix shared). - db.cache_touch("abc", "blob", 1, 1).unwrap(); - db.cache_touch("abc.b0", "block", 1, 1).unwrap(); - db.cache_touch("abc.b1", "block", 1, 1).unwrap(); - // An unrelated entry that must survive. - db.cache_touch("xyz", "blob", 1, 1).unwrap(); - - db.cache_remove("abc.b0").unwrap(); - assert_eq!(db.cache_entries_by_kind("block").unwrap().len(), 1); - - // remove_all drops the blob row and every remaining block of that uid. - db.cache_remove_all("abc").unwrap(); - assert!(db.cache_entries_by_kind("block").unwrap().is_empty()); - let blobs = db.cache_entries_by_kind("blob").unwrap(); - assert_eq!(blobs, vec![("xyz".into(), 1)]); - } - - /// A rebuild is a replacement, not a merge: whatever the index said before - /// is what a stale or externally-deleted cache file would leave behind. - #[test] - fn cache_index_rebuild_replaces_every_row() { - let db = Db::open_in_memory().unwrap(); - db.cache_touch("gone", "blob", 1, 1).unwrap(); - db.cache_touch("also-gone", "block", 1, 1).unwrap(); - - db.cache_rebuild(&[CacheEntryInput { - key: "kept", - kind: "blob", - size: 7, - last_accessed: 42, - }]) - .unwrap(); - - assert_eq!( - db.cache_entries_by_kind("blob").unwrap(), - vec![("kept".to_string(), 7)] - ); - assert!(db.cache_entries_by_kind("block").unwrap().is_empty()); - - // An empty rebuild is how a cache directory that vanished reports itself. - db.cache_rebuild(&[]).unwrap(); - assert!(db.cache_entries_by_kind("blob").unwrap().is_empty()); - } - - #[test] - fn pin_add_list_remove_roundtrip() { - let db = Db::open_in_memory().unwrap(); - db.pin_add("vol~a", "docs/a.txt", false).unwrap(); - db.pin_add("vol~d", "docs", true).unwrap(); - let list = db.pin_list().unwrap(); - assert_eq!(list.len(), 2); - assert_eq!(list[0], ("vol~a".into(), "docs/a.txt".into(), false)); - assert_eq!(list[1], ("vol~d".into(), "docs".into(), true)); - - // Re-pin refreshes path/flag, not a duplicate row. - db.pin_add("vol~a", "moved/a.txt", false).unwrap(); - let list = db.pin_list().unwrap(); - assert_eq!(list.len(), 2); - assert_eq!(list[0].1, "moved/a.txt"); - - assert!(db.pin_remove("vol~a").unwrap()); - assert!(!db.pin_remove("vol~a").unwrap()); - assert_eq!(db.pin_list().unwrap().len(), 1); - } - - #[test] - fn is_pinned_direct_without_node_row() { - // A direct pin counts even when the node was never hydrated into `nodes`. - let db = Db::open_in_memory().unwrap(); - db.pin_add("vol~a", "a.txt", false).unwrap(); - assert!(db.is_pinned("vol~a").unwrap()); - assert!(!db.is_pinned("vol~b").unwrap()); - } - - #[test] - fn recursive_folder_pin_covers_subtree() { - let db = Db::open_in_memory().unwrap(); - // root/docs/{report.pdf, sub/deep.txt}, root/loose.txt - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&folder("docs", Some("root"), "Documents")) - .unwrap(); - db.upsert_node(&file("rep", "docs", "report.pdf", 1)) - .unwrap(); - db.upsert_node(&folder("sub", Some("docs"), "Sub")).unwrap(); - db.upsert_node(&file("deep", "sub", "deep.txt", 1)).unwrap(); - db.upsert_node(&file("loose", "root", "loose.txt", 1)) - .unwrap(); - - // Pin the Documents folder recursively (uids are `vol~link` display form). - let du = |l: &str| uid(l).to_string(); - db.pin_add(&du("docs"), "Documents", true).unwrap(); - - // Everything under docs (any depth) is pinned; loose.txt is not. - assert!(db.is_pinned(&du("docs")).unwrap()); - assert!(db.is_pinned(&du("rep")).unwrap()); - assert!(db.is_pinned(&du("deep")).unwrap()); - assert!(!db.is_pinned(&du("loose")).unwrap()); - - // pinned_uids expands the subtree (folder + descendants), no loose.txt. - let mut got = db.pinned_uids().unwrap(); - got.sort(); - assert_eq!(got, vec![du("deep"), du("docs"), du("rep"), du("sub")]); - - // descendants() lists the subtree of a folder (excludes the folder). - let mut desc = db.descendants(&du("docs")).unwrap(); - desc.sort(); - assert_eq!(desc, vec![du("deep"), du("rep"), du("sub")]); - } - - #[test] - fn non_recursive_folder_pin_does_not_cover_children() { - let db = Db::open_in_memory().unwrap(); - db.upsert_node(&folder("root", None, "My Files")).unwrap(); - db.upsert_node(&folder("docs", Some("root"), "Documents")) - .unwrap(); - db.upsert_node(&file("rep", "docs", "report.pdf", 1)) - .unwrap(); - // A non-recursive pin on the folder covers only the folder itself. - let du = |l: &str| uid(l).to_string(); - db.pin_add(&du("docs"), "Documents", false).unwrap(); - assert!(db.is_pinned(&du("docs")).unwrap()); - assert!(!db.is_pinned(&du("rep")).unwrap()); - } - - #[test] - fn schema_objects_exist() { - let db = Db::open_in_memory().unwrap(); - let count: i64 = db - .with_conn(|c| { - Ok(c.query_row( - "SELECT count(*) FROM sqlite_master - WHERE name IN ('nodes', 'nodes_fts', 'cache_entries')", - [], - |r| r.get(0), - )?) - }) - .unwrap(); - assert_eq!(count, 3); - } -} diff --git a/crates/pdfs-core/src/db/activity.rs b/crates/pdfs-core/src/db/activity.rs new file mode 100644 index 0000000..cbab0d5 --- /dev/null +++ b/crates/pdfs-core/src/db/activity.rs @@ -0,0 +1,67 @@ +//! The activity feed: a bounded, newest-first log of what the daemon did. + +use rusqlite::params; + +use super::Db; +use crate::Result; +use crate::control::{ActivityEntry, ActivityKind}; + +/// How many activity rows to keep. Older rows are pruned on insert, so the feed +/// stays a bounded "recent history" rather than growing without limit. +pub(super) const ACTIVITY_KEEP: i64 = 2000; + +impl Db { + pub fn activity_add(&self, entry: &ActivityEntry) -> Result<()> { + let kind = serde_json::to_string(&entry.kind)?; + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute( + "INSERT INTO activity (time, kind, target, detail, ok) VALUES (?1, ?2, ?3, ?4, ?5)", + params![entry.time, kind, entry.target, entry.detail, entry.ok], + )?; + tx.execute( + "DELETE FROM activity WHERE id <= ( + SELECT id FROM activity ORDER BY id DESC LIMIT 1 OFFSET ?1 + )", + params![ACTIVITY_KEEP], + )?; + tx.commit()?; + Ok(()) + } + + /// The most recent activity, newest first, capped at `limit` entries. Rows + /// whose stored `kind` no longer parses (written by an older build) are + /// skipped rather than failing the whole read. + pub fn activity_list(&self, limit: usize) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT time, kind, target, detail, ok FROM activity + ORDER BY id DESC LIMIT ?1", + )?; + let rows = stmt.query_map(params![limit as i64], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + r.get::<_, bool>(4)?, + )) + })?; + let mut out = Vec::new(); + for row in rows { + let (time, kind, target, detail, ok) = row?; + let Ok(kind) = serde_json::from_str::(&kind) else { + continue; + }; + out.push(ActivityEntry { + time, + kind, + target, + detail, + ok, + }); + } + Ok(out) + } + +} diff --git a/crates/pdfs-core/src/db/cache.rs b/crates/pdfs-core/src/db/cache.rs new file mode 100644 index 0000000..18f8d01 --- /dev/null +++ b/crates/pdfs-core/src/db/cache.rs @@ -0,0 +1,126 @@ +//! The content-cache index: one row per cached blob, carrying the access stamp +//! the LRU eviction pass sorts on. + +use rusqlite::params; + +use super::Db; +use crate::Result; + +pub struct CacheEntryInput<'a> { + /// On-disk filename of the blob or block. + pub key: &'a str, + /// `'blob'` or `'block'` — the two byte budgets are counted separately. + pub kind: &'a str, + pub size: u64, + /// Initial LRU key, in unix milliseconds. + pub last_accessed: i64, +} + +impl Db { + pub fn cache_touch(&self, key: &str, kind: &str, size: u64, now: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO cache_entries (cache_key, kind, size_bytes, last_accessed) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(cache_key) DO UPDATE SET + kind = excluded.kind, + size_bytes = excluded.size_bytes, + last_accessed = excluded.last_accessed", + params![key, kind, size as i64, now], + )?; + Ok(()) + } + + /// Bump only `last_accessed` on a cache hit (LRU ordering). A missing row is + /// a no-op — best effort, mirroring the old best-effort mtime touch. + pub fn cache_accessed(&self, key: &str, now: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE cache_entries SET last_accessed = ?2 WHERE cache_key = ?1", + params![key, now], + )?; + Ok(()) + } + + /// Drop a single cache-entry row (one evicted blob or block). + pub fn cache_remove(&self, key: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "DELETE FROM cache_entries WHERE cache_key = ?1", + params![key], + )?; + Ok(()) + } + + /// Drop the whole-file blob row for `key` plus every block row for the same + /// uid (`.b`). Called by `ContentCache::evict`, which removes the + /// blob and all of a uid's cached blocks at once. + pub fn cache_remove_all(&self, key: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "DELETE FROM cache_entries WHERE cache_key = ?1 OR cache_key LIKE ?2", + params![key, format!("{key}.b%")], + )?; + Ok(()) + } + + /// Every cache entry of `kind`, ordered least-recently-accessed first, as + /// `(cache_key, size_bytes)`. The budget enforcer sums the sizes and evicts + /// from the front until the cache fits. + pub fn cache_entries_by_kind(&self, kind: &str) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT cache_key, size_bytes FROM cache_entries + WHERE kind = ?1 ORDER BY last_accessed ASC", + )?; + let rows = stmt.query_map(params![kind], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?.max(0) as u64)) + })?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) + } + + /// Replace the whole cache index with `entries`, as one transaction. The + /// daemon calls this on open, so a crash or an external file deletion can + /// never leave a phantom row inflating the budget total. + /// + /// The clear and the refill are the same commit deliberately: they are one + /// operation — "the index now describes what is on disk" — and a crash + /// between them would otherwise leave an empty index against a full cache + /// directory, i.e. a budget total of zero and nothing eviction can see. + /// + /// One commit also matters for speed. A mount is the only caller and it runs + /// on the open path, so a cache of thousands of blobs used to pay thousands + /// of autocommit fsyncs before the mountpoint appeared. + pub fn cache_rebuild(&self, entries: &[CacheEntryInput<'_>]) -> Result<()> { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute("DELETE FROM cache_entries", [])?; + { + let mut stmt = tx.prepare( + "INSERT INTO cache_entries (cache_key, kind, size_bytes, last_accessed) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(cache_key) DO UPDATE SET + kind = excluded.kind, + size_bytes = excluded.size_bytes, + last_accessed = excluded.last_accessed", + )?; + for e in entries { + stmt.execute(params![e.key, e.kind, e.size as i64, e.last_accessed])?; + } + } + tx.commit()?; + Ok(()) + } + + // --- Pins (P5) -------------------------------------------------------- + // + // The pin registry, formerly `pins.json`. One row per directly-pinned node, + // keyed by uid display string. `recursive` marks a folder pin: its whole + // subtree counts as pinned, resolved against `nodes` via a CTE so a new + // descendant is covered the moment it lands in the node cache. + +} diff --git a/crates/pdfs-core/src/db/devices.rs b/crates/pdfs-core/src/db/devices.rs new file mode 100644 index 0000000..1dae831 --- /dev/null +++ b/crates/pdfs-core/src/db/devices.rs @@ -0,0 +1,48 @@ +//! This machine as a registered Proton Drive device. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; + +pub struct StoredDevice { + pub uid: String, + pub share_id: String, + pub root_uid: String, + pub name: String, + pub created: i64, +} + +impl Db { + pub fn device_get(&self) -> Result> { + let conn = self.conn.lock(); + conn.query_row( + "SELECT uid, share_id, root_uid, name, created FROM device LIMIT 1", + [], + |r| { + Ok(StoredDevice { + uid: r.get(0)?, + share_id: r.get(1)?, + root_uid: r.get(2)?, + name: r.get(3)?, + created: r.get(4)?, + }) + }, + ) + .optional() + .map_err(Into::into) + } + + /// Persist (or replace) this machine's device. The table holds a single row. + pub fn device_set(&self, dev: &StoredDevice) -> Result<()> { + let conn = self.conn.lock(); + conn.execute("DELETE FROM device", [])?; + conn.execute( + "INSERT INTO device (uid, share_id, root_uid, name, created) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![dev.uid, dev.share_id, dev.root_uid, dev.name, dev.created], + )?; + Ok(()) + } + +} diff --git a/crates/pdfs-core/src/db/local.rs b/crates/pdfs-core/src/db/local.rs new file mode 100644 index 0000000..c37cc29 --- /dev/null +++ b/crates/pdfs-core/src/db/local.rs @@ -0,0 +1,163 @@ +//! The local-file index: a generation-stamped scan of the mount, so search can +//! answer over files the daemon has on disk as well as the remote tree. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; +use crate::localindex::LocalEntry; + +use super::utils::{TRIGRAM_MIN, like_escape}; + +/// One hit from [`Db::search_local`]: an indexed file on the machine itself, not +/// in Drive. `path` is absolute. +pub struct LocalFileHit { + pub path: String, + pub name: String, + pub is_dir: bool, + pub size: i64, + pub mtime: i64, +} + +impl Db { + pub fn local_begin_scan(&self) -> Result { + let conn = self.conn.lock(); + let current: i64 = conn + .query_row( + "SELECT value FROM sync_state WHERE key = 'local_scan_gen'", + [], + |row| row.get::<_, String>(0), + ) + .optional()? + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + let next = current + 1; + conn.execute( + "INSERT INTO sync_state (key, value) VALUES ('local_scan_gen', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [next.to_string()], + )?; + Ok(next) + } + + /// Write one batch of walked entries under scan generation `generation`. The + /// FTS index is *not* touched here — it is rebuilt once in + /// [`local_finish_scan`](Self::local_finish_scan). + pub fn local_upsert_batch(&self, generation: i64, entries: &[LocalEntry]) -> Result<()> { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + { + let mut stmt = tx.prepare_cached( + "INSERT INTO local_files (path, name, is_dir, size, mtime, scan_gen) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(path) DO UPDATE SET + name = excluded.name, + is_dir = excluded.is_dir, + size = excluded.size, + mtime = excluded.mtime, + scan_gen = excluded.scan_gen", + )?; + for e in entries { + stmt.execute(params![ + e.path, + e.name, + e.is_dir as i64, + e.size, + e.mtime, + generation + ])?; + } + } + tx.commit()?; + Ok(()) + } + + /// Close scan generation `generation`: drop every row an older scan wrote + /// (those paths are gone from disk), rebuild the FTS index over what remains, + /// and stamp the completion time. Returns the number of indexed entries. + pub fn local_finish_scan(&self, generation: i64, finished_at: i64) -> Result { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute( + "DELETE FROM local_files WHERE scan_gen != ?1", + params![generation], + )?; + tx.execute_batch("INSERT INTO local_fts(local_fts) VALUES('rebuild');")?; + tx.execute( + "INSERT INTO sync_state (key, value) VALUES ('local_indexed_at', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [finished_at.to_string()], + )?; + let count: i64 = tx.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?; + tx.commit()?; + Ok(count) + } + + /// When the last local scan completed (epoch seconds), or `None` if the index + /// has never been built. The daemon uses this to decide whether a fresh mount + /// needs an immediate rescan or can serve the existing index. + pub fn local_indexed_at(&self) -> Result> { + let conn = self.conn.lock(); + Ok(conn + .query_row( + "SELECT value FROM sync_state WHERE key = 'local_indexed_at'", + [], + |row| row.get::<_, String>(0), + ) + .optional()? + .and_then(|v| v.parse().ok())) + } + + /// Substring search over indexed local file names, newest-modified first + /// within a relevance tier. Mirrors [`search`](Self::search): the trigram + /// index handles queries of at least [`TRIGRAM_MIN`] chars, shorter ones fall + /// back to a `LIKE` scan. + pub fn search_local(&self, query: &str, limit: usize) -> Result> { + let query = query.trim(); + if query.is_empty() { + return Ok(Vec::new()); + } + let conn = self.conn.lock(); + let mut stmt; + let rows = if query.chars().count() < TRIGRAM_MIN { + let pat = format!("%{}%", like_escape(query)); + stmt = conn.prepare( + "SELECT path, name, is_dir, size, mtime FROM local_files + WHERE name LIKE ?1 ESCAPE '\\' + ORDER BY mtime DESC LIMIT ?2", + )?; + stmt.query_map(params![pat, limit as i64], local_hit)? + } else { + let phrase = query + .split_whitespace() + .map(|word| format!("\"{}\"", word.replace('"', "\"\""))) + .collect::>() + .join(" AND "); + stmt = conn.prepare( + "SELECT f.path, f.name, f.is_dir, f.size, f.mtime + FROM local_fts x JOIN local_files f ON f.id = x.rowid + WHERE x.name MATCH ?1 + ORDER BY x.rank LIMIT ?2", + )?; + stmt.query_map(params![phrase, limit as i64], local_hit)? + }; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) + } +} + +/// Row → [`LocalFileHit`]. Every local-search query selects the same +/// columns in the same order. +pub(super) fn local_hit(row: &rusqlite::Row) -> rusqlite::Result { + Ok(LocalFileHit { + path: row.get(0)?, + name: row.get(1)?, + is_dir: row.get::<_, i64>(2)? != 0, + size: row.get(3)?, + mtime: row.get(4)?, + }) +} + diff --git a/crates/pdfs-core/src/db/migrations.rs b/crates/pdfs-core/src/db/migrations.rs new file mode 100644 index 0000000..7b66243 --- /dev/null +++ b/crates/pdfs-core/src/db/migrations.rs @@ -0,0 +1,332 @@ +//! Forward-only schema migrations. +//! +//! Each `MIGRATION_V*` is the SQL that moves the schema from version N-1 to N. +//! They are applied in order and never edited once shipped: a released version +//! is already on disk somewhere, so a change here would silently diverge from +//! what an existing install actually has. Add a new one instead. + + +use super::Db; +use crate::Result; + +/// Current schema version. Bump on every forward migration added below. +pub(super) const SCHEMA_VERSION: i64 = 13; + +impl Db { + pub(super) fn migrate(&self) -> Result<()> { + let mut conn = self.conn.lock(); + + // `sync_state` is the key/value table holding `schema_version` and the + // event cursor (later). Create it first so we can read the version. + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS sync_state (key TEXT PRIMARY KEY, value TEXT);", + )?; + + let current: i64 = conn + .query_row( + "SELECT value FROM sync_state WHERE key = 'schema_version'", + [], + |row| row.get::<_, String>(0), + ) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + + if current >= SCHEMA_VERSION { + return Ok(()); + } + + let tx = conn.transaction()?; + if current < 1 { + tx.execute_batch(MIGRATION_V1)?; + } + if current < 2 { + tx.execute_batch(MIGRATION_V2)?; + } + if current < 3 { + tx.execute_batch(MIGRATION_V3)?; + } + if current < 4 { + tx.execute_batch(MIGRATION_V4)?; + } + if current < 5 { + tx.execute_batch(MIGRATION_V5)?; + } + if current < 6 { + tx.execute_batch(MIGRATION_V6)?; + } + if current < 7 { + tx.execute_batch(MIGRATION_V7)?; + } + if current < 8 { + tx.execute_batch(MIGRATION_V8)?; + } + if current < 9 { + tx.execute_batch(MIGRATION_V9)?; + } + if current < 10 { + tx.execute_batch(MIGRATION_V10)?; + } + if current < 11 { + tx.execute_batch(MIGRATION_V11)?; + } + if current < 12 { + tx.execute_batch(MIGRATION_V12)?; + } + if current < 13 { + tx.execute_batch(MIGRATION_V13)?; + } + tx.execute( + "INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + [SCHEMA_VERSION.to_string()], + )?; + tx.commit()?; + Ok(()) + } + +} + +/// Schema v1: nodes + FTS5 trigram index + cache LRU. `sync_state` is created +/// in [`Db::migrate`] before this runs. +const MIGRATION_V1: &str = " +CREATE TABLE nodes ( + uid TEXT PRIMARY KEY, + parent_uid TEXT, + name TEXT NOT NULL, + is_dir INTEGER NOT NULL, + size INTEGER, + mtime INTEGER NOT NULL, + revision_hash TEXT, + trashed INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX idx_nodes_parent ON nodes(parent_uid); + +CREATE VIRTUAL TABLE nodes_fts USING fts5( + name, path, content='', tokenize='trigram' +); + +CREATE TABLE cache_entries ( + cache_key TEXT PRIMARY KEY, + size_bytes INTEGER, + last_accessed INTEGER, + is_pinned INTEGER NOT NULL DEFAULT 0 +); +"; + +/// Schema v2: node write-through (P1). `node_json` stores the full [`Node`] so +/// the hot-cache maps rehydrate losslessly; `listed` records whether a folder's +/// child listing was complete when persisted. +const MIGRATION_V2: &str = " +ALTER TABLE nodes ADD COLUMN node_json TEXT; +ALTER TABLE nodes ADD COLUMN listed INTEGER NOT NULL DEFAULT 0; +"; + +/// Schema v3: FTS search (P3). The v1 `nodes_fts` was contentless (`content=''`) +/// and never populated — drop it and recreate as a self-contained trigram index +/// carrying the `uid` (UNINDEXED, so retrievable but not tokenized) alongside the +/// indexed `name`. Backfill from the nodes already persisted by P1. +const MIGRATION_V3: &str = " +DROP TABLE IF EXISTS nodes_fts; +CREATE VIRTUAL TABLE nodes_fts USING fts5( + uid UNINDEXED, name, tokenize='trigram' +); +INSERT INTO nodes_fts (uid, name) + SELECT uid, name FROM nodes WHERE trashed = 0; +"; + +/// Schema v4: content-cache LRU index (P4). The v1 `cache_entries` keyed only by +/// `cache_key`; add a `kind` discriminator so the whole-file blob pool and the +/// block pool — which carry separate byte budgets — can be summed and evicted +/// independently. Existing rows (none in practice, the table was never written) +/// default to `'blob'`. The daemon rebuilds the index from disk on open, so no +/// backfill is needed here. +const MIGRATION_V4: &str = " +ALTER TABLE cache_entries ADD COLUMN kind TEXT NOT NULL DEFAULT 'blob'; +"; + +/// Schema v5: pins move out of `pins.json` into the DB (P5). One row per +/// directly-pinned node, keyed by uid display string so a pin survives +/// renames/moves. `recursive` marks a folder pin whose whole subtree is kept; +/// descendants are resolved on demand against `nodes` via a CTE rather than +/// being expanded into rows here. `ContentCache::open` imports any legacy +/// `pins.json` into this table once, then deletes the file. +const MIGRATION_V5: &str = " +CREATE TABLE pins ( + uid TEXT PRIMARY KEY, + path TEXT NOT NULL, + recursive INTEGER NOT NULL DEFAULT 0 +); +"; + +/// Schema v6: the index of *local* (non-Drive) files, so the launcher prompt can +/// search the machine alongside Drive. `local_files` is keyed by absolute path; +/// `scan_gen` stamps the scan that last saw a row, so a rescan prunes vanished +/// files with one `DELETE` instead of diffing. +/// +/// `local_fts` is an *external-content* FTS5 index over `local_files.name`: the +/// text lives once in the base table and the index is rebuilt in bulk at the end +/// of a scan (`INSERT INTO local_fts(local_fts) VALUES('rebuild')`). That is far +/// cheaper than the delete-then-insert-per-row dance `nodes_fts` needs, because +/// a scan rewrites most rows at once rather than trickling single updates. +const MIGRATION_V6: &str = " +CREATE TABLE local_files ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + is_dir INTEGER NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + mtime INTEGER NOT NULL DEFAULT 0, + scan_gen INTEGER NOT NULL DEFAULT 0 +); + +CREATE VIRTUAL TABLE local_fts USING fts5( + name, content='local_files', content_rowid='id', tokenize='trigram' +); +"; + +/// Schema v7: the photos timeline and the trash listing become persistent, so +/// opening the app paints them from disk instead of re-fetching the world (both +/// were memory-only: the timeline behind a 60 s TTL, the trash not cached at all). +/// +/// `photos.seq` preserves the server's newest-first order, which is the only order +/// the timeline has — `capture_time` ties are common (a burst of shots) and would +/// otherwise shuffle between refreshes. `ratio` and `thumb_state` are *locally +/// learned*: they cost a download to rediscover, so `photos_replace` carries them +/// across a refresh while capture times and order come from the server. +const MIGRATION_V7: &str = " +CREATE TABLE photos ( + uid TEXT PRIMARY KEY, + capture_time INTEGER NOT NULL, + name TEXT, + ratio REAL, + thumb_state INTEGER NOT NULL DEFAULT 0, + seq INTEGER NOT NULL +); +CREATE INDEX idx_photos_seq ON photos(seq); + +CREATE TABLE trash ( + uid TEXT PRIMARY KEY, + name TEXT NOT NULL, + is_dir INTEGER NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + mtime INTEGER NOT NULL DEFAULT 0 +); +"; + +/// Schema v8: device sync (devices.md). This machine registers as one Proton Drive +/// **Device** (a share + root folder on the main volume); `device` is a singleton +/// row cached so we reuse the same device across restarts instead of creating a +/// new one each run. `sync_folder` is one row per local folder the user added, +/// each mapped to a folder under the device root; `mode` is `mirror` (full local +/// copy, two-way synced) or `ondemand` (FUSE mount at `local_path`, no local +/// storage). `sync_entry` is the per-file baseline for three-way merge — added in +/// this migration so Phase 2 has the table, though Phase 1 leaves it empty. +const MIGRATION_V8: &str = " +CREATE TABLE device ( + uid TEXT PRIMARY KEY, + share_id TEXT NOT NULL, + root_uid TEXT NOT NULL, + name TEXT NOT NULL, + created INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE sync_folder ( + id INTEGER PRIMARY KEY, + local_path TEXT NOT NULL UNIQUE, + remote_uid TEXT NOT NULL, + remote_share_id TEXT NOT NULL, + mode TEXT NOT NULL DEFAULT 'mirror', + state TEXT NOT NULL DEFAULT 'idle', + last_sync INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE sync_entry ( + folder_id INTEGER NOT NULL, + rel_path TEXT NOT NULL, + remote_uid TEXT, + local_mtime INTEGER NOT NULL DEFAULT 0, + local_size INTEGER NOT NULL DEFAULT 0, + remote_hash TEXT, + remote_rev TEXT, + PRIMARY KEY (folder_id, rel_path) +); +"; + +const MIGRATION_V9: &str = " +CREATE TABLE activity ( + id INTEGER PRIMARY KEY, + time INTEGER NOT NULL, + kind TEXT NOT NULL, + target TEXT NOT NULL, + detail TEXT NOT NULL DEFAULT '', + ok INTEGER NOT NULL DEFAULT 1 +); + +CREATE INDEX activity_time ON activity(time DESC); +"; + +/// Schema v10: a mode switch the user asked for is queued rather than rejected +/// when the folder is mid-pass or has un-uploaded changes, so `pending_mode` +/// records the intent until the engine can act on it. NULL is the resting state: +/// the folder is already where the user wants it. +const MIGRATION_V10: &str = " +ALTER TABLE sync_folder ADD COLUMN pending_mode TEXT; +"; + +/// Schema v11: writes no longer upload inside the FUSE handler. `release` stages +/// the bytes and records the intended upload here, and a drain worker performs it +/// (offline.md Phase 3). The row outlives the process, so a write survives both a +/// dead network and a restart. +/// +/// `blob_path` points into the content cache's `staging/` dir and `meta_json` is +/// the [`StagedWrite`] sidecar describing which of its bytes are real — an +/// incomplete blob must be gap-filled from the remote base before it can be +/// uploaded. `next_attempt_at` is a ms deadline implementing retry backoff. +/// +/// [`StagedWrite`]: crate::cache::StagedWrite +const MIGRATION_V11: &str = " +CREATE TABLE pending_op ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + uid TEXT NOT NULL, + blob_path TEXT, + meta_json TEXT, + created_at INTEGER NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + next_attempt_at INTEGER NOT NULL DEFAULT 0 +); + +CREATE INDEX pending_op_uid ON pending_op(uid); +"; + +/// Schema v12: `pending_op` also carries mutations that *create* a node, which a +/// revision op never had to describe — it always addressed a node the server had +/// already minted a uid for (offline.md Phase 3b). +/// +/// An offline `create`/`mkdir` cannot get a real uid, so the node is invented +/// locally under a `local~` placeholder and the op records where it goes +/// (`parent_uid`) and what it is called (`name`). Both are nullable because a +/// revision op sets neither. +/// +/// `parent_uid` is indexed: draining a folder rewrites every queued child that +/// points at its placeholder. +const MIGRATION_V12: &str = " +ALTER TABLE pending_op ADD COLUMN parent_uid TEXT; +ALTER TABLE pending_op ADD COLUMN name TEXT; + +CREATE INDEX pending_op_parent ON pending_op(parent_uid); +"; + +/// Schema v13: photos record their media type and a derived `kind` so the +/// timeline can be split into Photos / Videos / Raw tabs. `media_type` is +/// nullable — it is filled once the daemon resolves a photo's node — while +/// `kind` (see [`crate::control::PhotoKind`]) defaults to `0` (still photo), the +/// classification anything unresolved falls back to. Existing rows are +/// reclassified for free on the next timeline refresh, so no backfill is needed. +const MIGRATION_V13: &str = " +ALTER TABLE photos ADD COLUMN media_type TEXT; +ALTER TABLE photos ADD COLUMN kind INTEGER NOT NULL DEFAULT 0; +"; + diff --git a/crates/pdfs-core/src/db/mod.rs b/crates/pdfs-core/src/db/mod.rs new file mode 100644 index 0000000..b4b51ee --- /dev/null +++ b/crates/pdfs-core/src/db/mod.rs @@ -0,0 +1,109 @@ +//! Unified SQLite metadata cache — the single persistence layer behind FUSE +//! inode bookkeeping, full-text search, content-cache LRU tracking, and pins. +//! +//! Only the daemon (`pdfs-fuse`) opens this for writes; the GUI and CLI reach +//! the same data through the control socket. The connection is wrapped in a +//! `Mutex` because the FUSE callbacks are synchronous and already serialize +//! behind the `State` lock, so a connection pool would be overkill. +//! +//! This module is the P0 foundation: it opens the database, enables WAL, and +//! applies the forward-only schema migrations. Write-through of nodes, the +//! event cursor, FTS, and the cache index land in later phases on this schema. +//! +//! The surface is one `Db` type; its methods live in a submodule per table group +//! (`nodes`, `photos`, `pins`, …) as separate `impl Db` blocks. Everything those +//! modules define is re-exported here, so callers keep saying `db::StoredPhoto` +//! and never name a submodule. + +use parking_lot::Mutex; +use rusqlite::Connection; + +use crate::Result; +use std::path::Path; + +mod activity; +mod cache; +mod devices; +mod local; +mod migrations; +mod nodes; +mod ops; +mod photos; +mod pins; +mod state; +mod sync; +mod trash; +mod utils; + +pub use cache::CacheEntryInput; +pub use devices::StoredDevice; +pub use local::LocalFileHit; +pub use nodes::{SearchHit, StoredNode}; +pub use ops::{ + LOCAL_VOLUME, + AttachedBlob, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingCounts, + PendingOp, op_supersedes, +}; +pub use photos::{StoredPhoto, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN}; +pub use sync::{StoredSyncEntry, StoredSyncFolder}; +pub use trash::StoredTrash; + +/// Size the WAL is truncated back to after a checkpoint. Comfortably above the +/// steady-state working set (a few MB), so the truncation only claws back the +/// outliers rather than fighting the normal write path for disk. +const WAL_SIZE_LIMIT: i64 = 64 * 1024 * 1024; + +/// Handle to the unified metadata database. +/// +/// Cheap to wrap in an `Arc`; clone the `Arc`, not this. All access goes through +/// the inner `Mutex`. +pub struct Db { + conn: Mutex, +} + +impl Db { + /// Open (creating if absent) the database at `path`, enable WAL, and bring + /// the schema up to [`SCHEMA_VERSION`]. + pub fn open(path: &Path) -> Result { + let conn = Connection::open(path)?; + // WAL: readers never block the single writer. NORMAL sync is the + // standard durability/throughput tradeoff for WAL. + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "synchronous", "NORMAL")?; + conn.pragma_update(None, "foreign_keys", "ON")?; + // Hand the WAL's disk back after a checkpoint. Without a limit SQLite + // reuses the file in place but never shrinks it, so a single large + // transaction (a full-index FTS rebuild in `local_finish_scan` is the + // one that reaches this size) leaves the WAL at its high-water mark + // forever — a multi-GB file next to a database two orders of magnitude + // smaller. Checkpointing is unaffected; only the file is truncated back. + conn.pragma_update(None, "journal_size_limit", WAL_SIZE_LIMIT)?; + + let db = Self { + conn: Mutex::new(conn), + }; + db.migrate()?; + Ok(db) + } + + /// Open an in-memory database. For tests. + #[cfg(test)] + pub fn open_in_memory() -> Result { + let conn = Connection::open_in_memory()?; + let db = Self { + conn: Mutex::new(conn), + }; + db.migrate()?; + Ok(db) + } + + /// Run a closure with the locked connection. Escape hatch for callers that + /// need a query no typed method covers yet. + pub fn with_conn(&self, f: impl FnOnce(&Connection) -> Result) -> Result { + let conn = self.conn.lock(); + f(&conn) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/pdfs-core/src/db/nodes.rs b/crates/pdfs-core/src/db/nodes.rs new file mode 100644 index 0000000..23a3307 --- /dev/null +++ b/crates/pdfs-core/src/db/nodes.rs @@ -0,0 +1,241 @@ +//! Node rows: the persisted mirror of the remote tree, plus the trigram +//! full-text index over names and paths that backs `Request::Search`. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use proton_drive_rs::{Node, NodeKind}; + +use super::utils::{TRIGRAM_MIN, collect_pairs, like_escape, pair, path_of}; + +pub struct StoredNode { + pub node: Node, + pub listed: bool, +} + +/// One full-text search match: the stored [`Node`] plus its mountpoint-relative +/// path (`/`-joined, root excluded) so the front-end can navigate to or open it. +pub struct SearchHit { + pub node: Node, + pub path: String, +} + +impl Db { + pub fn upsert_node(&self, node: &Node) -> Result<()> { + self.upsert_nodes(std::slice::from_ref(node)) + } + + /// Write-through a batch of nodes as one transaction — a whole directory + /// listing, typically. Otherwise identical to [`upsert_node`](Self::upsert_node), + /// which is the single-node case of it. + /// + /// The commit count is the point: SQLite autocommits every statement that is + /// not in an explicit transaction, so interning a folder of a thousand + /// children row-by-row cost a thousand fsyncs, and `ls` waited for all of + /// them. + pub fn upsert_nodes(&self, nodes: &[Node]) -> Result<()> { + if nodes.is_empty() { + return Ok(()); + } + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + for node in nodes { + let json = serde_json::to_string(node)?; + let uid = node.uid.to_string(); + tx.execute( + "INSERT INTO nodes + (uid, parent_uid, name, is_dir, size, mtime, trashed, node_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(uid) DO UPDATE SET + parent_uid = excluded.parent_uid, + name = excluded.name, + is_dir = excluded.is_dir, + size = excluded.size, + mtime = excluded.mtime, + trashed = excluded.trashed, + node_json = excluded.node_json", + params![ + uid, + node.parent_uid.as_ref().map(|u| u.to_string()), + node.name, + node.is_folder() as i64, + node_size(node), + node.modification_time, + node.trashed as i64, + json, + ], + )?; + // FTS5 has no UPSERT, so refresh the row by delete-then-insert. + // Trashed nodes are kept out of the index entirely so they never + // surface in search results. + tx.execute("DELETE FROM nodes_fts WHERE uid = ?1", params![uid])?; + if !node.trashed { + tx.execute( + "INSERT INTO nodes_fts (uid, name) VALUES (?1, ?2)", + params![uid, node.name], + )?; + } + } + tx.commit()?; + Ok(()) + } + + /// Drop a node row (delete or trash from the hot cache). Children rows are + /// not cascaded here; the daemon forgets a whole subtree node-by-node. + pub fn delete_node(&self, uid: &NodeUid) -> Result<()> { + let uid = uid.to_string(); + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute("DELETE FROM nodes WHERE uid = ?1", params![uid])?; + tx.execute("DELETE FROM nodes_fts WHERE uid = ?1", params![uid])?; + tx.commit()?; + Ok(()) + } + + /// Full-text search over node names, newest schema's trigram index giving + /// substring (not just prefix) matches. Returns up to `limit` non-trashed + /// hits, each with its mountpoint-relative path resolved. Queries shorter + /// than [`TRIGRAM_MIN`] fall back to a `LIKE` scan since trigram indexes + /// nothing below 3 chars. + pub fn search(&self, query: &str, limit: usize) -> Result> { + let query = query.trim(); + if query.is_empty() { + return Ok(Vec::new()); + } + let conn = self.conn.lock(); + let rows: Vec<(String, String)> = if query.chars().count() < TRIGRAM_MIN { + let pat = format!("%{}%", like_escape(query)); + let mut stmt = conn.prepare( + "SELECT node_json, uid FROM nodes + WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL + ORDER BY name LIMIT ?2", + )?; + collect_pairs(stmt.query_map(params![pat, limit as i64], pair)?)? + } else { + // Escape double quotes and quote each term, then combine with AND so + // all terms must match but can appear in any order or position. + let phrase = query + .split_whitespace() + .map(|word| format!("\"{}\"", word.replace('"', "\"\""))) + .collect::>() + .join(" AND "); + let mut stmt = conn.prepare( + "SELECT n.node_json, n.uid + FROM nodes_fts f JOIN nodes n ON n.uid = f.uid + WHERE f.name MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL + ORDER BY f.rank LIMIT ?2", + )?; + collect_pairs(stmt.query_map(params![phrase, limit as i64], pair)?)? + }; + + let mut hits = Vec::with_capacity(rows.len()); + for (json, uid) in rows { + let node: Node = serde_json::from_str(&json)?; + let path = path_of(&conn, &uid)?; + hits.push(SearchHit { node, path }); + } + Ok(hits) + } + + /// Mark (or unmark) a folder's child listing as complete. A listed folder + /// rehydrates its `children` map on mount even when empty; an unlisted one + /// re-enumerates from the remote on next access. + pub fn set_listed(&self, uid: &NodeUid, listed: bool) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE nodes SET listed = ?2 WHERE uid = ?1", + params![uid.to_string(), listed as i64], + )?; + Ok(()) + } + + /// Load every persisted node for cold-start hydration of the `State` maps. + pub fn load_all(&self) -> Result> { + let conn = self.conn.lock(); + let mut stmt = + conn.prepare("SELECT node_json, listed FROM nodes WHERE node_json IS NOT NULL")?; + let rows = stmt.query_map([], |row| { + let json: String = row.get(0)?; + let listed: i64 = row.get(1)?; + Ok((json, listed != 0)) + })?; + let mut out = Vec::new(); + for row in rows { + let (json, listed) = row?; + let node: Node = serde_json::from_str(&json)?; + out.push(StoredNode { node, listed }); + } + Ok(out) + } + + /// Load one persisted node back by uid. Used to recover the My Files root + /// when the API is unreachable, so the mount can still serve the cached tree + /// (offline.md Phase 1). + pub fn node_by_uid(&self, uid: &str) -> Result> { + let conn = self.conn.lock(); + let json: Option = conn + .query_row( + "SELECT node_json FROM nodes WHERE uid = ?1 AND node_json IS NOT NULL", + params![uid], + |r| r.get(0), + ) + .optional()?; + match json { + Some(json) => Ok(Some(serde_json::from_str(&json)?)), + None => Ok(None), + } + } + + /// Read the persisted incremental-sync cursor (a `DriveEventId`), if any. + /// The daemon resumes from this on restart instead of reseeding to the + /// server head, so changes made while unmounted are still applied (P2). + pub fn children_if_listed(&self, parent: &NodeUid) -> Result>> { + let conn = self.conn.lock(); + let listed: Option = conn + .query_row( + "SELECT listed FROM nodes WHERE uid = ?1", + params![parent.to_string()], + |r| r.get(0), + ) + .optional()?; + if listed != Some(1) { + return Ok(None); + } + let mut stmt = conn.prepare( + "SELECT node_json FROM nodes + WHERE parent_uid = ?1 AND node_json IS NOT NULL AND trashed = 0", + )?; + let rows = stmt.query_map(params![parent.to_string()], |r| r.get::<_, String>(0))?; + let mut out = Vec::new(); + for json in rows { + out.push(serde_json::from_str(&json?)?); + } + Ok(Some(out)) + } + + // --- Content-cache LRU index (P4) ------------------------------------- + // + // Replaces the per-eviction `read_dir` scans in `ContentCache`. Each cached + // blob/block carries one row keyed by its on-disk filename (`cache_key`), + // tagged with `kind` ('blob' | 'block') so the two byte budgets stay + // separate. `last_accessed` (unix seconds) is the LRU key. The daemon owns + // the on-disk cache and rebuilds this index from disk on open, then keeps it + // in sync on every store/read/evict, so it is authoritative for eviction. + +} + + +/// Effective plaintext size of a node for the indexed `size` column: the +/// claimed size when known, else the on-storage size; folders are 0. +pub(super) fn node_size(node: &Node) -> i64 { + match &node.kind { + NodeKind::Folder => 0, + NodeKind::File { + total_size_on_storage, + claimed_size, + .. + } => claimed_size.unwrap_or(*total_size_on_storage).max(0), + } +} diff --git a/crates/pdfs-core/src/db/ops.rs b/crates/pdfs-core/src/db/ops.rs new file mode 100644 index 0000000..8e438f8 --- /dev/null +++ b/crates/pdfs-core/src/db/ops.rs @@ -0,0 +1,344 @@ +//! The offline mutation queue (`pending_op`): writes the kernel has accepted but +//! the remote has not yet seen. Rebuilt into memory on mount and drained in row +//! order, so a child never drains before the parent that gives it a real uid. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; + +/// The `kind` of a [`PendingOp`] that uploads a staged file as a new revision. +pub const OP_REVISION: &str = "revision"; + +/// The `kind` of a [`PendingOp`] that creates a file that so far exists only +/// locally, under a `local:` placeholder uid (offline.md Phase 3b). +/// +/// The written bytes ride along on the same row rather than as a follow-on +/// [`OP_REVISION`]: draining the create mints the node's real uid, which would +/// leave a separate revision op addressed to a uid that no longer exists. +pub const OP_CREATE: &str = "create"; + +/// The `kind` of a [`PendingOp`] that creates a folder that so far exists only +/// locally. Ordering matters: a child's op is queued after its parent's and +/// [`Db::pending_ops`] replays by row id, so the parent has a real uid by the +/// time the child drains. +pub const OP_MKDIR: &str = "mkdir"; + +/// The `kind` of a [`PendingOp`] that gives a node a new name, a new parent, or +/// both — the queued form of `mv` (offline.md Phase 3b). +/// +/// `parent_uid` and `name` hold the node's *desired end state*, not a delta, so +/// a second rename simply replaces the row (see [`Db::enqueue_op`]) and the +/// drain can compare them against the remote and skip whichever half already +/// matches. The parent may be a `local~` placeholder — moving a file into a +/// folder that was itself created offline — in which case it is rewritten by +/// that folder's own drain, exactly as for [`OP_CREATE`]. +pub const OP_RENAME: &str = "rename"; + +/// The `kind` of a [`PendingOp`] that trashes a node the server knows about +/// (offline.md Phase 3b). +/// +/// A node that only ever existed locally never gets one of these: there is +/// nothing to trash remotely, so deleting it just drops its queued ops +/// ([`Db::delete_ops_for_uid`]). +pub const OP_TRASH: &str = "trash"; + +/// Whether at most one op of this `kind` may be queued per node, so that a newer +/// one replaces the older rather than queueing behind it. +/// +/// True exactly of the kinds that describe a node's desired *end state* rather +/// than a step towards it: the newest revision already contains every earlier +/// one's bytes, the newest name is the only name wanted, and a trash subsumes +/// anything queued before it. A `create`/`mkdir` is the opposite — it is the one +/// thing that will ever make the node exist, so it must never be replaced. +pub fn op_supersedes(kind: &str) -> bool { + matches!(kind, OP_REVISION | OP_RENAME | OP_TRASH) +} + +/// The volume id given to a node that exists only on this machine, so far. A +/// real [`NodeUid`] is `{volume}~{link}`, so a placeholder is `local~` and +/// round-trips through the same `Display`/parse path as any other uid. +/// +/// Nothing bearing this volume may be handed to the API — it would 404. The +/// drain replaces it with the uid the server assigns. +pub const LOCAL_VOLUME: &str = "local"; + +/// A mutation that has been accepted locally but not yet performed against the +/// API — the durable half of the write-back queue (offline.md Phase 3). +/// +/// The daemon answers the FUSE call the moment this row and its staged blob are +/// on disk, so a `cp` into the mount runs at disk speed and the upload happens +/// behind it. That also makes an offline write succeed rather than EIO: the row +/// simply waits for the network. +#[derive(Debug, Clone)] +pub struct PendingOp { + /// Row id, `0` on a value being inserted. + pub id: i64, + /// See [`OP_REVISION`], [`OP_CREATE`], [`OP_MKDIR`]. + pub kind: String, + /// Node this op targets. For [`OP_CREATE`]/[`OP_MKDIR`] this is the + /// `local~` placeholder the node is known by until it drains. + pub uid: String, + /// Where the new node goes; only set for [`OP_CREATE`]/[`OP_MKDIR`]. May + /// itself be a placeholder when the parent folder is also still queued, in + /// which case it is rewritten when the parent drains. + pub parent_uid: Option, + /// The new node's name; only set for [`OP_CREATE`]/[`OP_MKDIR`]. + pub name: Option, + /// Staged blob holding the bytes to upload. + pub blob_path: Option, + /// Serialized [`StagedWrite`](crate::cache::StagedWrite). + pub meta_json: Option, + /// When the op was queued (ms since epoch). + pub created_at: i64, + pub attempts: i64, + pub last_error: Option, + /// Earliest ms at which to retry, for backoff. + pub next_attempt_at: i64, +} + +/// How much work the queue owes the server, by whether it carries bytes. +#[derive(Debug, Clone, Copy, Default)] +pub struct PendingCounts { + /// Queued `create`/`revision` ops: files whose content is not on the remote. + pub uploads: i64, + /// Queued `mkdir`/`rename`/`trash` ops: metadata the remote has not been + /// told about. + pub changes: i64, +} + +/// The outcome of folding freshly written bytes into a queued create. +#[derive(Debug, Clone)] +pub struct AttachedBlob { + /// Row id of the create the bytes were attached to. + pub id: i64, + /// Blob the create held before, now orphaned. + pub superseded: Option, +} + +impl Db { + pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option)> { + let conn = self.conn.lock(); + let superseded: Option = if op_supersedes(&op.kind) { + let blob: Option = conn + .query_row( + "SELECT blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2", + params![op.uid, op.kind], + |r| r.get(0), + ) + .optional()? + .flatten(); + conn.execute( + "DELETE FROM pending_op WHERE uid = ?1 AND kind = ?2", + params![op.uid, op.kind], + )?; + blob + } else { + None + }; + conn.execute( + "INSERT INTO pending_op + (kind, uid, parent_uid, name, blob_path, meta_json, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + op.kind, + op.uid, + op.parent_uid, + op.name, + op.blob_path, + op.meta_json, + op.created_at + ], + )?; + Ok((conn.last_insert_rowid(), superseded)) + } + + /// Point a queued create at the bytes that were just written to it, returning + /// any blob it previously held so the caller can discard it. + /// + /// This is what `release` does for a file that only exists locally: the node + /// has no uid to hang a revision op on yet, so the bytes ride on the create. + /// Repeated writes before the drain simply replace the blob. + /// + /// Returns `Ok(None)` and touches nothing if the create has already drained — + /// the caller must then queue an ordinary revision against the real uid. + pub fn attach_blob_to_create( + &self, + uid: &str, + blob_path: &str, + meta_json: &str, + ) -> Result> { + let conn = self.conn.lock(); + let existing: Option<(i64, Option)> = conn + .query_row( + "SELECT id, blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2", + params![uid, OP_CREATE], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?; + let Some((id, superseded)) = existing else { + return Ok(None); + }; + conn.execute( + "UPDATE pending_op + SET blob_path = ?2, meta_json = ?3, attempts = 0, next_attempt_at = 0 + WHERE id = ?1", + params![id, blob_path, meta_json], + )?; + Ok(Some(AttachedBlob { id, superseded })) + } + + /// Point a queued create at a new parent and name, for a node renamed or + /// moved before it ever reached the server. + /// + /// A `local~` uid means nothing to the API, so there is no rename call to + /// make — the node is still only a queued intent, and rewriting that intent + /// *is* the rename. Returns false when the create has already drained, in + /// which case the node has a real uid and the caller must rename it there + /// instead (offline.md Phase 3b). + pub fn rewrite_op_target(&self, uid: &str, parent_uid: &str, name: &str) -> Result { + let conn = self.conn.lock(); + let n = conn.execute( + "UPDATE pending_op SET parent_uid = ?2, name = ?3 + WHERE uid = ?1 AND kind IN (?4, ?5)", + params![uid, parent_uid, name, OP_CREATE, OP_MKDIR], + )?; + Ok(n > 0) + } + + /// Drop every op targeting a node **or anything queued beneath it**, + /// returning the staged blobs they held so the caller can delete them. + /// + /// Used when a node that only ever existed locally is deleted: there is + /// nothing on the server to trash, so the queued work simply stops being + /// wanted. + /// + /// The descent is what keeps the queue alive. Deleting a folder created + /// offline drops the `mkdir` that would have given it a real uid, and any + /// op still queued under that placeholder is then unreachable forever: it + /// can never be attempted (its parent is a `local~` uid, so `op_is_ready` + /// refuses it) and nothing is left to remap it. It would sit in the queue, + /// and in the user's pending count, for the life of the database. + /// + /// Only `create`/`mkdir` ops carry a `parent_uid`, so for a file this + /// recursion finds nothing and costs one query. + pub fn delete_ops_for_uid(&self, uid: &str) -> Result> { + const SUBTREE: &str = " + WITH RECURSIVE doomed(uid) AS ( + SELECT ?1 + UNION + SELECT p.uid FROM pending_op p JOIN doomed d ON p.parent_uid = d.uid + )"; + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + let blobs: Vec = { + let mut stmt = tx.prepare(&format!( + "{SUBTREE} + SELECT blob_path FROM pending_op + WHERE uid IN (SELECT uid FROM doomed) AND blob_path IS NOT NULL" + ))?; + let rows = stmt.query_map(params![uid], |r| r.get::<_, String>(0))?; + rows.collect::, _>>()? + }; + tx.execute( + &format!("{SUBTREE} DELETE FROM pending_op WHERE uid IN (SELECT uid FROM doomed)"), + params![uid], + )?; + tx.commit()?; + Ok(blobs) + } + + /// Rewrite every queued op that points at a placeholder parent, once that + /// parent has drained and has a real uid. Also moves the node rows whose + /// parent column still names the placeholder, so listings keep resolving. + pub fn remap_local_uid(&self, local: &str, real: &str) -> Result<()> { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute( + "UPDATE pending_op SET parent_uid = ?2 WHERE parent_uid = ?1", + params![local, real], + )?; + tx.execute( + "UPDATE nodes SET parent_uid = ?2 WHERE parent_uid = ?1", + params![local, real], + )?; + tx.execute("DELETE FROM nodes WHERE uid = ?1", params![local])?; + tx.commit()?; + Ok(()) + } + + /// Every queued op, oldest first. The drain worker replays them in this order + /// so a file's writes land in the order they were made. + pub fn pending_ops(&self) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at, + attempts, last_error, next_attempt_at + FROM pending_op ORDER BY id", + )?; + let rows = stmt + .query_map([], |r| { + Ok(PendingOp { + id: r.get(0)?, + kind: r.get(1)?, + uid: r.get(2)?, + parent_uid: r.get(3)?, + name: r.get(4)?, + blob_path: r.get(5)?, + meta_json: r.get(6)?, + created_at: r.get(7)?, + attempts: r.get(8)?, + last_error: r.get(9)?, + next_attempt_at: r.get(10)?, + }) + })? + .collect::, _>>()?; + Ok(rows) + } + + /// How many queued ops there are of each kind, for `Response::Status`. + /// + /// Split because "3 uploads queued" has to mean three files whose bytes are + /// not on the remote yet. A queued `mkdir`/`rename`/`trash` is also work the + /// mount owes the server, but it carries no bytes and reporting it as an + /// upload is simply untrue. + pub fn pending_op_counts(&self) -> Result { + let conn = self.conn.lock(); + let uploads = conn.query_row( + "SELECT COUNT(*) FROM pending_op WHERE kind IN (?1, ?2)", + params![OP_REVISION, OP_CREATE], + |r| r.get(0), + )?; + let changes = conn.query_row( + "SELECT COUNT(*) FROM pending_op WHERE kind NOT IN (?1, ?2)", + params![OP_REVISION, OP_CREATE], + |r| r.get(0), + )?; + Ok(PendingCounts { uploads, changes }) + } + +/// Nothing bearing this volume may be handed to the API — it would 404. The +/// drain replaces it with the uid the server assigns. +pub const LOCAL_VOLUME: &str = "local"; + + /// Drop a queued op, once its upload has actually landed. + pub fn delete_op(&self, id: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute("DELETE FROM pending_op WHERE id = ?1", params![id])?; + Ok(()) + } + + /// Record a failed attempt and when to next try. Leaves the row in place — + /// the staged bytes are still the only copy of the user's write. + pub fn record_op_failure(&self, id: i64, error: &str, next_attempt_at: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE pending_op + SET attempts = attempts + 1, last_error = ?2, next_attempt_at = ?3 + WHERE id = ?1", + params![id, error, next_attempt_at], + )?; + Ok(()) + } + +} diff --git a/crates/pdfs-core/src/db/photos.rs b/crates/pdfs-core/src/db/photos.rs new file mode 100644 index 0000000..79a3fdd --- /dev/null +++ b/crates/pdfs-core/src/db/photos.rs @@ -0,0 +1,244 @@ +//! The photos timeline: a flat, date-ordered projection of the photo share, +//! persisted so the gallery opens instantly instead of re-enumerating on launch. + +use rusqlite::params; + +use std::collections::HashMap; + +use super::Db; +use crate::Result; + +/// A photo whose thumbnail state is not known yet: it has never been asked for. +pub const THUMB_UNKNOWN: i64 = 0; +/// A thumbnail exists for this photo — served by the server, or generated locally +/// from the full file when the server had none. +pub const THUMB_HAVE: i64 = 1; +/// This photo can never be given a thumbnail: the server has none and the bytes +/// could not be decoded locally either. Never retried. +pub const THUMB_NONE: i64 = 2; + +/// One photo of the persisted timeline. The timeline itself is server-ordered +/// (newest first) and stored with that order in `seq`; `ratio` and `thumb_state` +/// are locally learned and survive a refresh of the timeline. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredPhoto { + pub uid: String, + pub capture_time: i64, + pub name: Option, + /// Aspect ratio (w/h), known once a thumbnail has been decoded. + pub ratio: Option, + /// One of [`THUMB_UNKNOWN`] / [`THUMB_HAVE`] / [`THUMB_NONE`]. + pub thumb_state: i64, + /// Which Photos-page tab this entry belongs to, derived from its name and + /// media type when the timeline was last replaced. + pub kind: crate::control::PhotoKind, +} + +impl Db { + pub fn photos_replace( + &self, + items: &[(String, i64, Option, Option)], + ) -> Result<()> { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + // `media_type` is learned-and-kept like the ratio and thumb verdict: the + // timeline DTO carries only the uid and capture time, so the daemon may + // not know a photo's media type yet when it replaces the timeline. Keep + // any previously learned value so the Photos/Videos/Raw split survives a + // refresh instead of collapsing back to name-extension guesses. + let learned: HashMap, i64, Option)> = { + let mut stmt = + tx.prepare("SELECT uid, ratio, thumb_state, media_type FROM photos")?; + let rows = stmt.query_map([], |r| { + Ok((r.get::<_, String>(0)?, (r.get(1)?, r.get(2)?, r.get(3)?))) + })?; + rows.collect::>()? + }; + + tx.execute("DELETE FROM photos", [])?; + { + let mut stmt = tx.prepare( + "INSERT INTO photos + (uid, capture_time, name, ratio, thumb_state, seq, media_type, kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + )?; + for (seq, (uid, capture_time, name, media_type)) in items.iter().enumerate() { + let (ratio, thumb_state, learned_media) = learned + .get(uid) + .cloned() + .unwrap_or((None, THUMB_UNKNOWN, None)); + let media_type = media_type.clone().or(learned_media); + // The tab this photo lands in is derived here, once, so a page or + // count query is a plain indexed `WHERE kind = ?` rather than a + // reclassification of every row. + let kind = crate::control::PhotoKind::classify( + name.as_deref(), + media_type.as_deref(), + ); + stmt.execute(params![ + uid, + capture_time, + name, + ratio, + thumb_state, + seq as i64, + media_type, + kind.as_i64(), + ])?; + } + } + tx.commit()?; + Ok(()) + } + + /// One page of the persisted timeline, newest first. `kind`, when set, + /// restricts the page to one tab (Photos / Videos / Raw); `range`, when set, + /// restricts it to a `[from, to)` capture-time window (epoch seconds) — the + /// date scrubber's jump. `offset` is relative to whatever the filters leave. + pub fn photos_page( + &self, + offset: usize, + limit: usize, + kind: Option, + range: Option<(i64, i64)>, + ) -> Result> { + let conn = self.conn.lock(); + // Built up so any combination of the optional filters is one indexed + // query rather than a statement per case. + let mut sql = + String::from("SELECT uid, capture_time, name, ratio, thumb_state, kind FROM photos"); + let mut binds: Vec = Vec::new(); + let mut conds: Vec = Vec::new(); + if let Some(k) = kind { + binds.push(k.as_i64()); + conds.push(format!("kind = ?{}", binds.len())); + } + if let Some((from, to)) = range { + binds.push(from); + conds.push(format!("capture_time >= ?{}", binds.len())); + binds.push(to); + conds.push(format!("capture_time < ?{}", binds.len())); + } + if !conds.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conds.join(" AND ")); + } + binds.push(limit as i64); + let limit_pos = binds.len(); + binds.push(offset as i64); + let offset_pos = binds.len(); + sql.push_str(&format!(" ORDER BY seq LIMIT ?{limit_pos} OFFSET ?{offset_pos}")); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(binds), |r| { + Ok(StoredPhoto { + uid: r.get(0)?, + capture_time: r.get(1)?, + name: r.get(2)?, + ratio: r.get(3)?, + thumb_state: r.get(4)?, + kind: crate::control::PhotoKind::from_i64(r.get(5)?), + }) + })? + .collect::>()?; + Ok(rows) + } + + /// The months the timeline spans, newest first, each with how many photos it + /// holds — the data behind the date scrubber. Buckets are local-time + /// `(year, month)` so they line up with the day headings the gallery draws + /// and with the boundaries a front-end computes when it jumps to one. `kind` + /// scopes the counts to one tab when set. + pub fn photos_months( + &self, + kind: Option, + ) -> Result> { + let conn = self.conn.lock(); + let (filter, binds): (&str, Vec) = match kind { + Some(k) => (" WHERE kind = ?1", vec![k.as_i64()]), + None => ("", Vec::new()), + }; + let sql = format!( + "SELECT CAST(strftime('%Y', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS y, \ + CAST(strftime('%m', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS m, \ + COUNT(*) \ + FROM photos{filter} GROUP BY y, m ORDER BY y DESC, m DESC" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(binds), |r| { + Ok((r.get::<_, i32>(0)?, r.get::<_, i32>(1)?, r.get::<_, i64>(2)? as usize)) + })? + .collect::>()?; + Ok(rows) + } + + /// Per-tab counts for the Photos page subtitle: `(photos, videos, raw)`. + pub fn photos_counts(&self) -> Result<(usize, usize, usize)> { + use crate::control::PhotoKind; + let conn = self.conn.lock(); + let mut stmt = conn.prepare("SELECT kind, COUNT(*) FROM photos GROUP BY kind")?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?)))?; + let (mut photos, mut videos, mut raw) = (0usize, 0usize, 0usize); + for row in rows { + let (kind, n) = row?; + match PhotoKind::from_i64(kind) { + PhotoKind::Photo => photos = n as usize, + PhotoKind::Video => videos = n as usize, + PhotoKind::Raw => raw = n as usize, + } + } + Ok((photos, videos, raw)) + } + + /// The stored photos for `uids`, in no particular order. Used by the thumbnail + /// path, which needs each photo's capture time (the cache validity tag) and + /// its thumbnail verdict. + pub fn photos_by_uid(&self, uids: &[String]) -> Result> { + if uids.is_empty() { + return Ok(Vec::new()); + } + let conn = self.conn.lock(); + let placeholders = vec!["?"; uids.len()].join(","); + let mut stmt = conn.prepare(&format!( + "SELECT uid, capture_time, name, ratio, thumb_state, kind FROM photos + WHERE uid IN ({placeholders})" + ))?; + let rows = stmt.query_map(rusqlite::params_from_iter(uids), |r| { + Ok(StoredPhoto { + uid: r.get(0)?, + capture_time: r.get(1)?, + name: r.get(2)?, + ratio: r.get(3)?, + thumb_state: r.get(4)?, + kind: crate::control::PhotoKind::from_i64(r.get(5)?), + }) + })?; + let mut photos = Vec::new(); + for row in rows { + photos.push(row?); + } + Ok(photos) + } + + /// Record what a thumbnail attempt learned: whether the photo now has one + /// ([`THUMB_HAVE`] / [`THUMB_NONE`]), and its aspect ratio if the pixels were + /// seen. A `None` ratio leaves any previously learned one alone. + pub fn photo_set_thumb(&self, uid: &str, state: i64, ratio: Option) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE photos SET thumb_state = ?2, ratio = COALESCE(?3, ratio) WHERE uid = ?1", + params![uid, state, ratio], + )?; + Ok(()) + } + + /// Number of photos in the persisted timeline. + pub fn photos_count(&self) -> Result { + let conn = self.conn.lock(); + let n: i64 = conn.query_row("SELECT COUNT(*) FROM photos", [], |r| r.get(0))?; + Ok(n.max(0) as usize) + } + +} diff --git a/crates/pdfs-core/src/db/pins.rs b/crates/pdfs-core/src/db/pins.rs new file mode 100644 index 0000000..f4d37bc --- /dev/null +++ b/crates/pdfs-core/src/db/pins.rs @@ -0,0 +1,124 @@ +//! Pinned nodes: files and folders the user has asked to keep on disk, exempt +//! from LRU eviction. A recursive pin covers a whole subtree. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; + + + +impl Db { + pub fn pin_add(&self, uid: &str, path: &str, recursive: bool) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO pins (uid, path, recursive) VALUES (?1, ?2, ?3) + ON CONFLICT(uid) DO UPDATE SET + path = excluded.path, recursive = excluded.recursive", + params![uid, path, recursive as i64], + )?; + Ok(()) + } + + /// Drop `uid`'s pin row. Returns whether a row existed. + pub fn pin_remove(&self, uid: &str) -> Result { + let conn = self.conn.lock(); + let n = conn.execute("DELETE FROM pins WHERE uid = ?1", params![uid])?; + Ok(n > 0) + } + + /// Every directly-pinned entry `(uid, path, recursive)`, ordered by uid. + pub fn pin_list(&self) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare("SELECT uid, path, recursive FROM pins ORDER BY uid")?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, i64>(2)? != 0, + )) + })?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) + } + + /// Whether `uid` is pinned — directly, or because a strict ancestor folder + /// is pinned recursively. The ancestor check walks `parent_uid` to the root + /// via a CTE; a direct pin is honoured even when the node has no `nodes` row + /// yet (e.g. a CLI that never hydrates the node cache). + pub fn is_pinned(&self, uid: &str) -> Result { + let conn = self.conn.lock(); + let direct: Option = conn + .query_row("SELECT 1 FROM pins WHERE uid = ?1", params![uid], |r| { + r.get(0) + }) + .optional()?; + if direct.is_some() { + return Ok(true); + } + let anc: Option = conn + .query_row( + "WITH RECURSIVE anc(uid, parent_uid) AS ( + SELECT uid, parent_uid FROM nodes WHERE uid = ?1 + UNION ALL + SELECT n.uid, n.parent_uid FROM nodes n JOIN anc ON n.uid = anc.parent_uid + ) + SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid + WHERE a.uid != ?1 AND p.recursive = 1 + LIMIT 1", + params![uid], + |r| r.get(0), + ) + .optional()?; + Ok(anc.is_some()) + } + + /// Every uid that is pinned, directly or transitively: all direct pins plus + /// every descendant of a recursively-pinned folder. Used to build the + /// eviction-exempt set (the budget enforcer hashes these into cache keys). + pub fn pinned_uids(&self) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "WITH RECURSIVE sub(uid) AS ( + SELECT uid FROM pins WHERE recursive = 1 + UNION + SELECT n.uid FROM nodes n JOIN sub ON n.parent_uid = sub.uid + ) + SELECT uid FROM sub + UNION + SELECT uid FROM pins", + )?; + let rows = stmt.query_map([], |r| r.get::<_, String>(0))?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) + } + + /// Every descendant uid of `folder` (all depths), via a `parent_uid` CTE. + /// Used to evict a recursively-pinned subtree's cached blobs on unpin. + pub fn descendants(&self, folder: &str) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "WITH RECURSIVE sub(uid) AS ( + SELECT uid FROM nodes WHERE parent_uid = ?1 + UNION + SELECT n.uid FROM nodes n JOIN sub ON n.parent_uid = sub.uid + ) + SELECT uid FROM sub", + )?; + let rows = stmt.query_map(params![folder], |r| r.get::<_, String>(0))?; + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) + } + + // ---- local (non-Drive) file index (schema v6) ---- + +} diff --git a/crates/pdfs-core/src/db/state.rs b/crates/pdfs-core/src/db/state.rs new file mode 100644 index 0000000..02a22d9 --- /dev/null +++ b/crates/pdfs-core/src/db/state.rs @@ -0,0 +1,98 @@ +//! The `state` key/value table: small scalars that outlive a mount — the volume +//! event cursor, listing freshness stamps, the local-index generation. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; + +impl Db { + pub fn get_event_cursor(&self) -> Result> { + let conn = self.conn.lock(); + let v = conn + .query_row( + "SELECT value FROM sync_state WHERE key = 'event_cursor'", + [], + |r| r.get::<_, String>(0), + ) + .optional()?; + Ok(v) + } + + /// Persist the incremental-sync cursor after a batch of events is applied. + pub fn set_event_cursor(&self, cursor: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO sync_state (key, value) VALUES ('event_cursor', ?1) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![cursor], + )?; + Ok(()) + } + + /// Read a `sync_state` value as a string. + pub fn state_str(&self, key: &str) -> Result> { + let conn = self.conn.lock(); + let v = conn + .query_row( + "SELECT value FROM sync_state WHERE key = ?1", + params![key], + |r| r.get::<_, String>(0), + ) + .optional()?; + Ok(v) + } + + /// Write a `sync_state` string value. + pub fn set_state_str(&self, key: &str, value: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO sync_state (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![key, value], + )?; + Ok(()) + } + + /// Queue an op to be performed by the drain worker, returning its row id. + /// + /// One *revision* op per node: a second write to the same file before the + /// first has drained replaces it, since the newer blob already contains + /// everything the older one did. The superseded blob's path is returned so + /// the caller can delete it. + /// + /// Rows supersede only their own kind, and only the kinds + /// [`op_supersedes`] names. A create op for the same uid must survive — + /// dropping it would leave a file that exists nowhere but this machine with + /// nothing left to create it. (Writes to a node that is itself still queued + pub fn state_i64(&self, key: &str) -> Result> { + let conn = self.conn.lock(); + let v: Option = conn + .query_row( + "SELECT value FROM sync_state WHERE key = ?1", + params![key], + |r| r.get(0), + ) + .optional()?; + Ok(v.and_then(|v| v.parse().ok())) + } + + /// Write a `sync_state` integer value. + pub fn set_state_i64(&self, key: &str, value: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO sync_state (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![key, value.to_string()], + )?; + Ok(()) + } + + /// Drop a `sync_state` key, so whatever it stamped counts as never fetched. + pub fn clear_state(&self, key: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute("DELETE FROM sync_state WHERE key = ?1", params![key])?; + Ok(()) + } + +} diff --git a/crates/pdfs-core/src/db/sync.rs b/crates/pdfs-core/src/db/sync.rs new file mode 100644 index 0000000..ea9844b --- /dev/null +++ b/crates/pdfs-core/src/db/sync.rs @@ -0,0 +1,226 @@ +//! Folder-sync bookkeeping: the folders this device mirrors, and the per-file +//! state each pass reconciles against. + +use rusqlite::{OptionalExtension, params}; + +use super::Db; +use crate::Result; +use std::collections::HashMap; + +pub struct StoredSyncFolder { + pub id: i64, + pub local_path: String, + pub remote_uid: String, + pub remote_share_id: String, + /// `mirror` (full local copy, two-way synced) or `ondemand` (FUSE mount). + pub mode: String, + /// A mode the user asked for that could not be applied on the spot — the + /// folder was mid-pass, or had un-uploaded changes. The engine applies it + /// once the folder is safe to switch, so the request is queued rather than + /// rejected. `None` when the folder is where the user wants it. + pub pending_mode: Option, + /// `idle` | `syncing` | `error` | `conflict`. + pub state: String, + pub last_sync: i64, +} + +/// One per-file sync baseline row: what a path looked like on both sides at the +/// last successful sync, so the next reconcile can tell which side changed +/// (devices.md Phase 2). `remote_rev`/`remote_hash` hold the remote signature — +/// its modification time and size as strings — since no cheap content hash is +/// exposed; change detection is `(mtime, size)` on each side. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredSyncEntry { + pub rel_path: String, + pub remote_uid: Option, + pub local_mtime: i64, + pub local_size: i64, + pub remote_rev: Option, + pub remote_hash: Option, +} + +impl Db { + pub fn sync_folder_add( + &self, + local_path: &str, + remote_uid: &str, + remote_share_id: &str, + ) -> Result { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO sync_folder (local_path, remote_uid, remote_share_id) + VALUES (?1, ?2, ?3)", + params![local_path, remote_uid, remote_share_id], + )?; + Ok(conn.last_insert_rowid()) + } + + /// Every synced folder, oldest first. + pub fn sync_folder_list(&self) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync + FROM sync_folder ORDER BY id", + )?; + let rows = stmt.query_map([], |r| { + Ok(StoredSyncFolder { + id: r.get(0)?, + local_path: r.get(1)?, + remote_uid: r.get(2)?, + remote_share_id: r.get(3)?, + mode: r.get(4)?, + pending_mode: r.get(5)?, + state: r.get(6)?, + last_sync: r.get(7)?, + }) + })?; + let mut items = Vec::new(); + for row in rows { + items.push(row?); + } + Ok(items) + } + + /// Look up one synced folder by id. + pub fn sync_folder_get(&self, id: i64) -> Result> { + let conn = self.conn.lock(); + conn.query_row( + "SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync + FROM sync_folder WHERE id = ?1", + params![id], + |r| { + Ok(StoredSyncFolder { + id: r.get(0)?, + local_path: r.get(1)?, + remote_uid: r.get(2)?, + remote_share_id: r.get(3)?, + mode: r.get(4)?, + pending_mode: r.get(5)?, + state: r.get(6)?, + last_sync: r.get(7)?, + }) + }, + ) + .optional() + .map_err(Into::into) + } + + /// Remove a synced folder and its per-file baseline. + pub fn sync_folder_remove(&self, id: i64) -> Result { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute("DELETE FROM sync_entry WHERE folder_id = ?1", params![id])?; + let n = tx.execute("DELETE FROM sync_folder WHERE id = ?1", params![id])?; + tx.commit()?; + Ok(n > 0) + } + + /// Update a synced folder's mode (`mirror`/`ondemand`). The folder has + /// reached the mode it was asked for, so any queued request is satisfied and + /// cleared in the same write — a `pending_mode` outliving the switch it asked + /// for would have the engine try to apply it again on the next pass. + pub fn sync_folder_set_mode(&self, id: i64, mode: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE sync_folder SET mode = ?2, pending_mode = NULL WHERE id = ?1", + params![id, mode], + )?; + Ok(()) + } + + /// Queue (or, with `None`, withdraw) a mode the folder should move to once it + /// is safe to switch. + pub fn sync_folder_set_pending_mode(&self, id: i64, mode: Option<&str>) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE sync_folder SET pending_mode = ?2 WHERE id = ?1", + params![id, mode], + )?; + Ok(()) + } + + /// Update a synced folder's state and stamp `last_sync` to now. + pub fn sync_folder_set_state(&self, id: i64, state: &str, last_sync: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "UPDATE sync_folder SET state = ?2, last_sync = ?3 WHERE id = ?1", + params![id, state, last_sync], + )?; + Ok(()) + } + + /// The whole per-file sync baseline for a folder, keyed by relative path. + pub fn sync_entries(&self, folder_id: i64) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash + FROM sync_entry WHERE folder_id = ?1", + )?; + let rows = stmt.query_map(params![folder_id], |r| { + Ok(StoredSyncEntry { + rel_path: r.get(0)?, + remote_uid: r.get(1)?, + local_mtime: r.get(2)?, + local_size: r.get(3)?, + remote_rev: r.get(4)?, + remote_hash: r.get(5)?, + }) + })?; + let mut map = HashMap::new(); + for row in rows { + let e = row?; + map.insert(e.rel_path.clone(), e); + } + Ok(map) + } + + /// Insert or replace one baseline row. + pub fn sync_entry_upsert(&self, folder_id: i64, e: &StoredSyncEntry) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "INSERT INTO sync_entry + (folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(folder_id, rel_path) DO UPDATE SET + remote_uid = excluded.remote_uid, + local_mtime = excluded.local_mtime, + local_size = excluded.local_size, + remote_rev = excluded.remote_rev, + remote_hash = excluded.remote_hash", + params![ + folder_id, + e.rel_path, + e.remote_uid, + e.local_mtime, + e.local_size, + e.remote_rev, + e.remote_hash, + ], + )?; + Ok(()) + } + + /// Drop one baseline row (a path that left the sync set on both sides). + pub fn sync_entry_remove(&self, folder_id: i64, rel_path: &str) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "DELETE FROM sync_entry WHERE folder_id = ?1 AND rel_path = ?2", + params![folder_id, rel_path], + )?; + Ok(()) + } + + /// Drop the entire baseline for a folder. Used when flipping ondemand→mirror: + /// the local tree was evicted, so the old baseline is stale and would make the + /// next reconcile mistake "locally deleted" for "must re-download". Clearing it + /// leaves an empty baseline + full remote = pure download (devices.md P3). + pub fn sync_entries_clear(&self, folder_id: i64) -> Result<()> { + let conn = self.conn.lock(); + conn.execute( + "DELETE FROM sync_entry WHERE folder_id = ?1", + params![folder_id], + )?; + Ok(()) + } + +} diff --git a/crates/pdfs-core/src/db/tests.rs b/crates/pdfs-core/src/db/tests.rs new file mode 100644 index 0000000..91a5881 --- /dev/null +++ b/crates/pdfs-core/src/db/tests.rs @@ -0,0 +1,1061 @@ +use super::*; +use super::activity::ACTIVITY_KEEP; +use super::migrations::SCHEMA_VERSION; +use crate::control::{ActivityEntry, ActivityKind}; +use crate::localindex::LocalEntry; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use proton_drive_rs::{Node, NodeKind}; + + + /// `Db::open` applies the pragmas that `open_in_memory` skips, so the WAL + /// settings can only be checked against a real file. + #[test] + fn open_bounds_the_wal_size() { + let path = std::env::temp_dir().join(format!("pdfs-db-wal-{}.db", std::process::id())); + let _ = std::fs::remove_file(&path); + let db = Db::open(&path).unwrap(); + let conn = db.conn.lock(); + + let mode: String = conn + .query_row("PRAGMA journal_mode", [], |r| r.get(0)) + .unwrap(); + assert_eq!(mode, "wal"); + // Without this the WAL is reused in place but never shrinks, so one + // oversized transaction strands its high-water mark on disk forever. + let limit: i64 = conn + .query_row("PRAGMA journal_size_limit", [], |r| r.get(0)) + .unwrap(); + assert_eq!(limit, WAL_SIZE_LIMIT); + + drop(conn); + drop(db); + let _ = std::fs::remove_file(&path); + } + use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId}; + + fn uid(link: &str) -> NodeUid { + NodeUid::new(VolumeId::from("vol"), LinkId::from(link)) + } + + // `NodeVerification` is not re-exported, so build test nodes by + // deserializing JSON (the field has a serde default and can be omitted). + fn node_from( + parent: serde_json::Value, + link: &str, + name: &str, + kind: serde_json::Value, + ) -> Node { + let v = serde_json::json!({ + "uid": {"volume_id": "vol", "link_id": link}, + "parent_uid": parent, + "kind": kind, + "name": name, + "creation_time": 100, + "modification_time": 200, + "trashed": false, + "signature_email": null, + }); + serde_json::from_value(v).unwrap() + } + + #[test] + fn photos_replace_keeps_what_was_learned_and_drops_what_left() { + let db = Db::open_in_memory().unwrap(); + db.photos_replace(&[ + ("p1".into(), 300, None, Some("video/mp4".into())), + ("p2".into(), 200, None, None), + ("p3".into(), 100, None, None), + ]) + .unwrap(); + + // A thumbnail attempt teaches us p1's ratio and that p2 can never have one. + db.photo_set_thumb("p1", THUMB_HAVE, Some(1.5)).unwrap(); + db.photo_set_thumb("p2", THUMB_NONE, None).unwrap(); + + // The next refresh brings a new photo, keeps p1 and p2, and loses p3. + // p1's media type arrives as `None` this time and must not be forgotten. + db.photos_replace(&[ + ("p0".into(), 400, Some("new.jpg".into()), None), + ("p1".into(), 300, None, None), + ("p2".into(), 200, None, None), + ]) + .unwrap(); + + let page = db.photos_page(0, 10, None, None).unwrap(); + assert_eq!( + page.iter().map(|p| p.uid.as_str()).collect::>(), + ["p0", "p1", "p2"], + "server order is preserved, and the dropped photo is gone" + ); + assert_eq!(page[0].name.as_deref(), Some("new.jpg")); + // Ratios and verdicts cost a download to rediscover: they survive a refresh. + assert_eq!(page[1].ratio, Some(1.5)); + assert_eq!(page[1].thumb_state, THUMB_HAVE); + assert_eq!(page[2].thumb_state, THUMB_NONE); + // A photo we know nothing about yet starts blank. + assert_eq!(page[0].ratio, None); + assert_eq!(page[0].thumb_state, THUMB_UNKNOWN); + // Media type is learned-and-kept like the ratio: p1 keeps the video type + // it was first seen with even though the later refresh carried `None`, so + // it stays classified as a video. + assert_eq!(page[1].kind, crate::control::PhotoKind::Video); + + assert_eq!(db.photos_count().unwrap(), 3); + // The counts break down by tab, and a filtered page returns only its tab. + assert_eq!(db.photos_counts().unwrap(), (2, 1, 0)); + let videos = db + .photos_page(0, 10, Some(crate::control::PhotoKind::Video), None) + .unwrap(); + assert_eq!( + videos.iter().map(|p| p.uid.as_str()).collect::>(), + ["p1"] + ); + let by_uid = db.photos_by_uid(&["p2".into()]).unwrap(); + assert_eq!(by_uid.len(), 1); + assert_eq!(by_uid[0].capture_time, 200); + + // A date-range page keeps only the window's photos: [150, 350) is p1+p2, + // not p0 at 400. Combined with a kind filter both conditions apply. + let ranged = db.photos_page(0, 10, None, Some((150, 350))).unwrap(); + assert_eq!( + ranged.iter().map(|p| p.uid.as_str()).collect::>(), + ["p1", "p2"] + ); + let ranged_video = db + .photos_page(0, 10, Some(crate::control::PhotoKind::Video), Some((150, 350))) + .unwrap(); + assert_eq!( + ranged_video.iter().map(|p| p.uid.as_str()).collect::>(), + ["p1"] + ); + // All three surviving photos sit in the same (1970-01) local month. + let months = db.photos_months(None).unwrap(); + assert_eq!(months.len(), 1); + assert_eq!(months[0].2, 3); + } + + #[test] + fn photos_page_slices_the_timeline_in_order() { + let db = Db::open_in_memory().unwrap(); + let items: Vec<_> = (0..5) + .map(|i| (format!("p{i}"), 500 - i as i64, None, None::)) + .collect(); + db.photos_replace(&items).unwrap(); + + let page = db.photos_page(2, 2, None, None).unwrap(); + assert_eq!( + page.iter().map(|p| p.uid.as_str()).collect::>(), + ["p2", "p3"] + ); + assert!(db.photos_page(9, 2, None, None).unwrap().is_empty()); + } + + #[test] + fn trash_replace_lists_folders_first() { + let db = Db::open_in_memory().unwrap(); + db.trash_replace(&[ + StoredTrash { + uid: "t1".into(), + name: "zeta.txt".into(), + is_dir: false, + size: 10, + mtime: 1, + }, + StoredTrash { + uid: "t2".into(), + name: "Alpha".into(), + is_dir: true, + size: 0, + mtime: 2, + }, + ]) + .unwrap(); + + let items = db.trash_list().unwrap(); + assert_eq!( + items.iter().map(|i| i.name.as_str()).collect::>(), + ["Alpha", "zeta.txt"] + ); + + // A replace is a replace: emptying the trash on the server empties it here. + db.trash_replace(&[]).unwrap(); + assert!(db.trash_list().unwrap().is_empty()); + } + + #[test] + fn state_stamps_round_trip_and_clear() { + let db = Db::open_in_memory().unwrap(); + assert_eq!(db.state_i64("photos_synced_ms").unwrap(), None); + db.set_state_i64("photos_synced_ms", 1234).unwrap(); + assert_eq!(db.state_i64("photos_synced_ms").unwrap(), Some(1234)); + db.clear_state("photos_synced_ms").unwrap(); + assert_eq!( + db.state_i64("photos_synced_ms").unwrap(), + None, + "a cleared stamp reads as never fetched, so the next request blocks on a refresh" + ); + } + + fn folder(link: &str, parent: Option<&str>, name: &str) -> Node { + let parent = match parent { + Some(p) => serde_json::json!({"volume_id": "vol", "link_id": p}), + None => serde_json::Value::Null, + }; + node_from(parent, link, name, serde_json::json!("Folder")) + } + + fn file(link: &str, parent: &str, name: &str, size: i64) -> Node { + let kind = serde_json::json!({ + "File": { + "media_type": "text/plain", + "total_size_on_storage": size + 10, + "claimed_size": size, + "claimed_modification_time": null, + } + }); + node_from( + serde_json::json!({"volume_id": "vol", "link_id": parent}), + link, + name, + kind, + ) + } + + /// Recovering the root by uid is what lets the daemon mount offline + /// (offline.md Phase 1): the uid is remembered in `sync_state`, the node + /// itself comes back out of `nodes`. + #[test] + fn node_by_uid_recovers_a_stored_node() { + let db = Db::open_in_memory().unwrap(); + let root = folder("root", None, "My Files"); + db.upsert_node(&root).unwrap(); + db.set_state_str("root_uid", &root.uid.to_string()).unwrap(); + + let key = db.state_str("root_uid").unwrap().unwrap(); + let got = db.node_by_uid(&key).unwrap().expect("root recovered"); + assert_eq!(got.uid, root.uid); + assert_eq!(got.name, "My Files"); + assert!(got.is_folder()); + + assert!(db.node_by_uid("vol~nope").unwrap().is_none()); + assert!(db.state_str("never_written").unwrap().is_none()); + } + + #[test] + fn upsert_and_load_all_roundtrip() { + let db = Db::open_in_memory().unwrap(); + let root = folder("root", None, "My Files"); + let child = file("f1", "root", "hello.txt", 42); + db.upsert_node(&root).unwrap(); + db.upsert_node(&child).unwrap(); + + let loaded = db.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + let f = loaded.iter().find(|s| s.node.uid == uid("f1")).unwrap(); + assert_eq!(f.node.name, "hello.txt"); + assert!(!f.listed); + match &f.node.kind { + NodeKind::File { claimed_size, .. } => assert_eq!(*claimed_size, Some(42)), + _ => panic!("expected file"), + } + } + + #[test] + fn upsert_nodes_and_load_all_roundtrip() { + let db = Db::open_in_memory().unwrap(); + let root = folder("root", None, "My Files"); + let child1 = file("f1", "root", "hello.txt", 42); + let child2 = file("f2", "root", "world.txt", 100); + db.upsert_nodes(&[root, child1, child2]).unwrap(); + + let loaded = db.load_all().unwrap(); + assert_eq!(loaded.len(), 3); + let f1 = loaded.iter().find(|s| s.node.uid == uid("f1")).unwrap(); + assert_eq!(f1.node.name, "hello.txt"); + let f2 = loaded.iter().find(|s| s.node.uid == uid("f2")).unwrap(); + assert_eq!(f2.node.name, "world.txt"); + } + + #[test] + fn upsert_is_idempotent_update() { + let db = Db::open_in_memory().unwrap(); + let mut n = folder("root", None, "My Files"); + db.upsert_node(&n).unwrap(); + n.name = "Renamed".into(); + db.upsert_node(&n).unwrap(); + let loaded = db.load_all().unwrap(); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].node.name, "Renamed"); + } + + #[test] + fn delete_node_removes_row() { + let db = Db::open_in_memory().unwrap(); + let n = folder("root", None, "My Files"); + db.upsert_node(&n).unwrap(); + db.delete_node(&uid("root")).unwrap(); + assert!(db.load_all().unwrap().is_empty()); + } + + #[test] + fn children_if_listed_gated_on_flag() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&file("f1", "root", "a.txt", 1)).unwrap(); + db.upsert_node(&file("f2", "root", "b.txt", 2)).unwrap(); + + // Not listed yet → unknown, force a re-fetch. + assert!(db.children_if_listed(&uid("root")).unwrap().is_none()); + + db.set_listed(&uid("root"), true).unwrap(); + let kids = db.children_if_listed(&uid("root")).unwrap().unwrap(); + assert_eq!(kids.len(), 2); + } + + #[test] + fn children_if_listed_excludes_trashed() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + let mut trashed = file("f1", "root", "a.txt", 1); + trashed.trashed = true; + db.upsert_node(&trashed).unwrap(); + db.set_listed(&uid("root"), true).unwrap(); + assert_eq!( + db.children_if_listed(&uid("root")).unwrap().unwrap().len(), + 0 + ); + } + + fn local(path: &str, name: &str, is_dir: bool) -> LocalEntry { + LocalEntry { + path: path.into(), + name: name.into(), + is_dir, + size: 10, + mtime: 5, + } + } + + /// A local scan is searchable by substring, and a *later* scan prunes the + /// paths it no longer sees — including out of the FTS index, so a deleted + /// file cannot keep surfacing in the prompt. + #[test] + fn local_scan_indexes_then_prunes_stale_paths() { + let db = Db::open_in_memory().unwrap(); + + let gen1 = db.local_begin_scan().unwrap(); + db.local_upsert_batch( + gen1, + &[ + local("/home/u/docs/report.pdf", "report.pdf", false), + local("/home/u/docs/notes.md", "notes.md", false), + ], + ) + .unwrap(); + assert_eq!(db.local_finish_scan(gen1, 1_000).unwrap(), 2); + + // Trigram index gives substring (not just prefix) matches. + let hits = db.search_local("port", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].path, "/home/u/docs/report.pdf"); + assert!(!hits[0].is_dir); + assert_eq!(db.local_indexed_at().unwrap(), Some(1_000)); + + // Second scan sees only notes.md → report.pdf is gone from disk. + let gen2 = db.local_begin_scan().unwrap(); + db.local_upsert_batch(gen2, &[local("/home/u/docs/notes.md", "notes.md", false)]) + .unwrap(); + assert_eq!(db.local_finish_scan(gen2, 2_000).unwrap(), 1); + assert!(db.search_local("report", 10).unwrap().is_empty()); + assert_eq!(db.search_local("notes", 10).unwrap().len(), 1); + } + + /// Queries below the trigram minimum still match, via the `LIKE` fallback. + #[test] + fn local_search_short_query_like_fallback() { + let db = Db::open_in_memory().unwrap(); + let g = db.local_begin_scan().unwrap(); + db.local_upsert_batch(g, &[local("/home/u/a.txt", "a.txt", false)]) + .unwrap(); + db.local_finish_scan(g, 1).unwrap(); + assert_eq!(db.search_local("a", 10).unwrap().len(), 1); + assert!(db.search_local("", 10).unwrap().is_empty()); + } + + #[test] + fn event_cursor_roundtrip() { + let db = Db::open_in_memory().unwrap(); + // Absent before first write → seed from server head on first mount. + assert!(db.get_event_cursor().unwrap().is_none()); + db.set_event_cursor("evt-1").unwrap(); + assert_eq!(db.get_event_cursor().unwrap().as_deref(), Some("evt-1")); + // Overwrites, not appends. + db.set_event_cursor("evt-2").unwrap(); + assert_eq!(db.get_event_cursor().unwrap().as_deref(), Some("evt-2")); + } + + #[test] + fn search_trigram_substring_and_path() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&folder("docs", Some("root"), "Documents")) + .unwrap(); + db.upsert_node(&file("f1", "docs", "report.pdf", 1)) + .unwrap(); + db.upsert_node(&file("f2", "root", "notes.txt", 2)).unwrap(); + db.upsert_node(&file( + "f3", + "root", + "Rampage Open Air 2026 - order 166765244.pdf", + 3, + )) + .unwrap(); + + // Substring match (trigram), not just prefix. + let hits = db.search("port", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].node.name, "report.pdf"); + // Path is mountpoint-relative, root excluded. + assert_eq!(hits[0].path, "Documents/report.pdf"); + + // Top-level file → bare name. + let hits = db.search("notes", 10).unwrap(); + assert_eq!(hits[0].path, "notes.txt"); + + // Multi-term FTS5 query matching (out of order, separated terms) + let hits = db.search("rampage 2026", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0].node.name, + "Rampage Open Air 2026 - order 166765244.pdf" + ); + } + + #[test] + fn search_excludes_trashed_and_respects_limit() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&file("f1", "root", "alpha.txt", 1)).unwrap(); + let mut gone = file("f2", "root", "alphb.txt", 1); + gone.trashed = true; + db.upsert_node(&gone).unwrap(); + + let hits = db.search("alph", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].node.name, "alpha.txt"); + + db.upsert_node(&file("f3", "root", "alphc.txt", 1)).unwrap(); + assert_eq!(db.search("alph", 1).unwrap().len(), 1); + } + + #[test] + fn search_short_query_like_fallback() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&file("f1", "root", "ab.txt", 1)).unwrap(); + // Under trigram min length → LIKE path still finds it. + let hits = db.search("ab", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].node.name, "ab.txt"); + } + + #[test] + fn search_drops_fts_row_on_delete_and_trash() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&file("f1", "root", "unique.txt", 1)) + .unwrap(); + assert_eq!(db.search("unique", 10).unwrap().len(), 1); + + // Re-upsert as trashed → leaves the index. + let mut t = file("f1", "root", "unique.txt", 1); + t.trashed = true; + db.upsert_node(&t).unwrap(); + assert_eq!(db.search("unique", 10).unwrap().len(), 0); + + // Resurrect, then hard-delete. + db.upsert_node(&file("f1", "root", "unique.txt", 1)) + .unwrap(); + assert_eq!(db.search("unique", 10).unwrap().len(), 1); + db.delete_node(&uid("f1")).unwrap(); + assert_eq!(db.search("unique", 10).unwrap().len(), 0); + } + + fn activity(target: &str, kind: ActivityKind, ok: bool) -> ActivityEntry { + ActivityEntry { + time: 1700, + kind, + target: target.into(), + detail: "detail".into(), + ok, + } + } + + #[test] + fn activity_reads_back_newest_first() { + let db = Db::open_in_memory().unwrap(); + db.activity_add(&activity("a.txt", ActivityKind::Upload, true)) + .unwrap(); + db.activity_add(&activity("b.txt", ActivityKind::Download, false)) + .unwrap(); + + let items = db.activity_list(10).unwrap(); + assert_eq!(items.len(), 2); + assert_eq!(items[0].target, "b.txt"); + assert_eq!(items[0].kind, ActivityKind::Download); + assert!(!items[0].ok); + assert_eq!(items[0].detail, "detail"); + assert_eq!(items[0].time, 1700); + assert_eq!(items[1].target, "a.txt"); + + assert_eq!(db.activity_list(1).unwrap().len(), 1); + } + + #[test] + fn activity_prunes_to_the_keep_limit() { + let db = Db::open_in_memory().unwrap(); + for i in 0..(ACTIVITY_KEEP + 10) { + db.activity_add(&activity(&format!("f{i}"), ActivityKind::Upload, true)) + .unwrap(); + } + let items = db.activity_list(ACTIVITY_KEEP as usize * 2).unwrap(); + assert_eq!(items.len(), ACTIVITY_KEEP as usize); + // The newest survive; the oldest are the ones dropped. + assert_eq!(items[0].target, format!("f{}", ACTIVITY_KEEP + 9)); + } + + #[test] + fn opens_and_migrates() { + let db = Db::open_in_memory().unwrap(); + let version: String = db + .with_conn(|c| { + Ok(c.query_row( + "SELECT value FROM sync_state WHERE key = 'schema_version'", + [], + |r| r.get(0), + )?) + }) + .unwrap(); + assert_eq!(version, SCHEMA_VERSION.to_string()); + } + + #[test] + fn a_second_write_supersedes_the_first_pending_op() { + let db = Db::open_in_memory().unwrap(); + let op = |blob: &str| PendingOp { + id: 0, + kind: OP_REVISION.to_string(), + uid: uid("a").to_string(), + parent_uid: None, + name: None, + blob_path: Some(blob.to_string()), + meta_json: Some("{}".to_string()), + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }; + + let (_, superseded) = db.enqueue_op(&op("/staging/first")).unwrap(); + assert_eq!(superseded, None, "nothing to supersede on the first write"); + + // The newer blob already contains everything the older one did, so the + // older op must go — and its blob must be reported so it can be deleted + // rather than leaked. + let (id2, superseded) = db.enqueue_op(&op("/staging/second")).unwrap(); + assert_eq!(superseded.as_deref(), Some("/staging/first")); + + let ops = db.pending_ops().unwrap(); + assert_eq!(ops.len(), 1, "one queued upload per node"); + assert_eq!(ops[0].id, id2); + assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/second")); + assert_eq!(db.pending_op_counts().unwrap().uploads, 1); + } + + /// Deleting a folder that was created offline must take the ops queued + /// underneath it with it: they name a placeholder parent that will now never + /// become real, so nothing could ever drain them and nothing is left to + /// rewrite them. + #[test] + fn deleting_a_queued_folder_takes_its_queued_children_with_it() { + let db = Db::open_in_memory().unwrap(); + let op = |kind: &str, uid: &str, parent: &str, blob: Option<&str>| PendingOp { + id: 0, + kind: kind.to_string(), + uid: uid.to_string(), + parent_uid: Some(parent.to_string()), + name: Some("n".to_string()), + blob_path: blob.map(str::to_string), + meta_json: None, + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }; + let root = uid("root").to_string(); + db.enqueue_op(&op(OP_MKDIR, "local~dir", &root, None)) + .unwrap(); + db.enqueue_op(&op(OP_MKDIR, "local~sub", "local~dir", None)) + .unwrap(); + db.enqueue_op(&op( + OP_CREATE, + "local~deep", + "local~sub", + Some("/staging/deep"), + )) + .unwrap(); + // A sibling outside the doomed subtree must survive. + db.enqueue_op(&op(OP_CREATE, "local~other", &root, Some("/staging/other"))) + .unwrap(); + + let blobs = db.delete_ops_for_uid("local~dir").unwrap(); + assert_eq!( + blobs, + vec!["/staging/deep"], + "the subtree's bytes come back" + ); + + let left: Vec = db + .pending_ops() + .unwrap() + .into_iter() + .map(|o| o.uid) + .collect(); + assert_eq!(left, vec!["local~other"]); + } + + /// A rename is the node's desired end state, so the newest one is the only + /// one worth performing — but it must not disturb the queued *upload* of the + /// same node, which is unrelated work. + #[test] + fn a_second_rename_supersedes_the_first_but_leaves_the_upload_alone() { + let db = Db::open_in_memory().unwrap(); + let rename = |name: &str| PendingOp { + id: 0, + kind: OP_RENAME.to_string(), + uid: uid("a").to_string(), + parent_uid: Some(uid("parent").to_string()), + name: Some(name.to_string()), + blob_path: None, + meta_json: None, + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }; + db.enqueue_op(&PendingOp { + id: 0, + kind: OP_REVISION.to_string(), + uid: uid("a").to_string(), + parent_uid: None, + name: None, + blob_path: Some("/staging/blob".to_string()), + meta_json: Some("{}".to_string()), + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }) + .unwrap(); + + db.enqueue_op(&rename("first")).unwrap(); + let (_, superseded) = db.enqueue_op(&rename("second")).unwrap(); + assert_eq!(superseded, None, "a rename owns no blob to clean up"); + + let ops = db.pending_ops().unwrap(); + assert_eq!(ops.len(), 2, "the queued upload survives the rename"); + let renames: Vec<_> = ops.iter().filter(|o| o.kind == OP_RENAME).collect(); + assert_eq!(renames.len(), 1, "one rename per node"); + assert_eq!(renames[0].name.as_deref(), Some("second")); + + let counts = db.pending_op_counts().unwrap(); + assert_eq!(counts.uploads, 1, "the revision is the only upload"); + assert_eq!(counts.changes, 1, "the rename carries no bytes"); + } + + /// Renaming a node whose create has not drained rewrites the intent rather + /// than queueing a rename against a uid the server has never issued. + #[test] + fn renaming_a_queued_create_rewrites_its_target() { + let db = Db::open_in_memory().unwrap(); + let local = "local~abc"; + db.enqueue_op(&PendingOp { + id: 0, + kind: OP_CREATE.to_string(), + uid: local.to_string(), + parent_uid: Some(uid("old").to_string()), + name: Some("draft.txt".to_string()), + blob_path: Some("/staging/blob".to_string()), + meta_json: Some("{}".to_string()), + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }) + .unwrap(); + + let rewritten = db + .rewrite_op_target(local, &uid("new").to_string(), "final.txt") + .unwrap(); + assert!(rewritten); + + let ops = db.pending_ops().unwrap(); + assert_eq!(ops.len(), 1, "a rewrite is not a second op"); + assert_eq!(ops[0].name.as_deref(), Some("final.txt")); + assert_eq!( + ops[0].parent_uid.as_deref(), + Some(uid("new").to_string()).as_deref() + ); + assert_eq!( + ops[0].blob_path.as_deref(), + Some("/staging/blob"), + "the bytes riding on the create are untouched" + ); + + // Once the create has drained there is no intent left to rewrite, and the + // caller has to rename the real node instead. + db.delete_op(ops[0].id).unwrap(); + assert!( + !db.rewrite_op_target(local, &uid("new").to_string(), "final.txt") + .unwrap() + ); + } + + #[test] + fn a_write_folds_into_a_queued_create_instead_of_superseding_it() { + let db = Db::open_in_memory().unwrap(); + let local = "local~abc"; + db.enqueue_op(&PendingOp { + id: 0, + kind: OP_CREATE.to_string(), + uid: local.to_string(), + parent_uid: Some(uid("parent").to_string()), + name: Some("new.txt".to_string()), + blob_path: None, + meta_json: None, + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }) + .unwrap(); + + let first = db + .attach_blob_to_create(local, "/staging/first", "{}") + .unwrap() + .expect("create is still queued"); + assert_eq!(first.superseded, None); + + // Rewriting the file before it drains replaces the bytes but must leave + // the create itself alone: it is the only thing that will ever bring this + // file into existence remotely. + let second = db + .attach_blob_to_create(local, "/staging/second", "{}") + .unwrap() + .expect("create is still queued"); + assert_eq!(second.superseded.as_deref(), Some("/staging/first")); + + let ops = db.pending_ops().unwrap(); + assert_eq!(ops.len(), 1, "still exactly one create"); + assert_eq!(ops[0].kind, OP_CREATE); + assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/second")); + assert_eq!(ops[0].name.as_deref(), Some("new.txt")); + } + + #[test] + fn attaching_to_an_already_drained_create_reports_it_is_gone() { + let db = Db::open_in_memory().unwrap(); + // No create queued: the caller must fall back to a revision op rather + // than silently dropping the bytes. + let out = db + .attach_blob_to_create("local~gone", "/staging/x", "{}") + .unwrap(); + assert!(out.is_none()); + } + + #[test] + fn draining_a_folder_repoints_its_queued_children() { + let db = Db::open_in_memory().unwrap(); + let local_dir = "local~dir"; + let real_dir = uid("realdir").to_string(); + db.enqueue_op(&PendingOp { + id: 0, + kind: OP_CREATE.to_string(), + uid: "local~child".to_string(), + parent_uid: Some(local_dir.to_string()), + name: Some("inside.txt".to_string()), + blob_path: Some("/staging/child".to_string()), + meta_json: Some("{}".to_string()), + created_at: 2, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }) + .unwrap(); + + db.remap_local_uid(local_dir, &real_dir).unwrap(); + + // The child was queued against a folder that did not exist yet. Once the + // folder is real, the child must target the server's uid — otherwise the + // upload would address `local~dir` and 404. + let ops = db.pending_ops().unwrap(); + assert_eq!(ops[0].parent_uid.as_deref(), Some(real_dir.as_str())); + } + + #[test] + fn a_failed_op_stays_queued_with_backoff() { + let db = Db::open_in_memory().unwrap(); + db.enqueue_op(&PendingOp { + id: 0, + kind: OP_REVISION.to_string(), + uid: uid("a").to_string(), + parent_uid: None, + name: None, + blob_path: Some("/staging/blob".to_string()), + meta_json: Some("{}".to_string()), + created_at: 1, + attempts: 0, + last_error: None, + next_attempt_at: 0, + }) + .unwrap(); + let id = db.pending_ops().unwrap()[0].id; + + db.record_op_failure(id, "network unreachable", 5_000) + .unwrap(); + + // The staged blob is the only copy of the user's bytes: a failure must + // never drop the row, only defer it. + let ops = db.pending_ops().unwrap(); + assert_eq!(ops.len(), 1); + assert_eq!(ops[0].attempts, 1); + assert_eq!(ops[0].last_error.as_deref(), Some("network unreachable")); + assert_eq!(ops[0].next_attempt_at, 5_000); + + db.record_op_failure(id, "still down", 9_000).unwrap(); + assert_eq!(db.pending_ops().unwrap()[0].attempts, 2); + + db.delete_op(id).unwrap(); + assert_eq!(db.pending_op_counts().unwrap().uploads, 0); + } + + #[test] + fn migrate_is_idempotent() { + let db = Db::open_in_memory().unwrap(); + // Second migrate is a no-op (already at head) and must not error. + db.migrate().unwrap(); + } + + /// A queued mode switch is a promise the daemon has to keep across a restart, + /// so it lives in the row, and reaching the mode is what retires it. + #[test] + fn pending_mode_is_queued_until_the_mode_is_reached() { + let db = Db::open_in_memory().unwrap(); + let id = db + .sync_folder_add("/home/me/Downloads", "v~l", "s") + .unwrap(); + assert_eq!(db.sync_folder_get(id).unwrap().unwrap().pending_mode, None); + + db.sync_folder_set_pending_mode(id, Some("ondemand")) + .unwrap(); + assert_eq!( + db.sync_folder_get(id) + .unwrap() + .unwrap() + .pending_mode + .as_deref(), + Some("ondemand") + ); + // The listing carries it too — it is what the front-ends paint from. + assert_eq!( + db.sync_folder_list().unwrap()[0].pending_mode.as_deref(), + Some("ondemand") + ); + + // Landing the switch satisfies the request: a `pending_mode` outliving it + // would have the engine try to apply the same switch on every later pass. + db.sync_folder_set_mode(id, "ondemand").unwrap(); + let folder = db.sync_folder_get(id).unwrap().unwrap(); + assert_eq!(folder.mode, "ondemand"); + assert_eq!(folder.pending_mode, None); + + // And the user can withdraw a request that hasn't landed yet. + db.sync_folder_set_pending_mode(id, Some("mirror")).unwrap(); + db.sync_folder_set_pending_mode(id, None).unwrap(); + let folder = db.sync_folder_get(id).unwrap().unwrap(); + assert_eq!(folder.mode, "ondemand"); + assert_eq!(folder.pending_mode, None); + } + + #[test] + fn cache_index_touch_access_and_lru_order() { + let db = Db::open_in_memory().unwrap(); + db.cache_touch("k1", "blob", 100, 10).unwrap(); + db.cache_touch("k2", "blob", 200, 20).unwrap(); + // LRU-first: k1 (older access) before k2. + let rows = db.cache_entries_by_kind("blob").unwrap(); + assert_eq!(rows, vec![("k1".into(), 100), ("k2".into(), 200)]); + + // Accessing k1 moves it to the back (most recent). + db.cache_accessed("k1", 30).unwrap(); + let rows = db.cache_entries_by_kind("blob").unwrap(); + assert_eq!(rows[0].0, "k2"); + assert_eq!(rows[1].0, "k1"); + + // Re-touch updates size, not just time. + db.cache_touch("k1", "blob", 150, 40).unwrap(); + let rows = db.cache_entries_by_kind("blob").unwrap(); + assert_eq!(rows.iter().find(|(k, _)| k == "k1").unwrap().1, 150); + } + + #[test] + fn cache_index_kinds_are_separate() { + let db = Db::open_in_memory().unwrap(); + db.cache_touch("blob1", "blob", 100, 1).unwrap(); + db.cache_touch("blk1.b0", "block", 50, 1).unwrap(); + assert_eq!(db.cache_entries_by_kind("blob").unwrap().len(), 1); + assert_eq!(db.cache_entries_by_kind("block").unwrap().len(), 1); + } + + #[test] + fn cache_index_remove_and_remove_all() { + let db = Db::open_in_memory().unwrap(); + // A blob plus two of its blocks (key prefix shared). + db.cache_touch("abc", "blob", 1, 1).unwrap(); + db.cache_touch("abc.b0", "block", 1, 1).unwrap(); + db.cache_touch("abc.b1", "block", 1, 1).unwrap(); + // An unrelated entry that must survive. + db.cache_touch("xyz", "blob", 1, 1).unwrap(); + + db.cache_remove("abc.b0").unwrap(); + assert_eq!(db.cache_entries_by_kind("block").unwrap().len(), 1); + + // remove_all drops the blob row and every remaining block of that uid. + db.cache_remove_all("abc").unwrap(); + assert!(db.cache_entries_by_kind("block").unwrap().is_empty()); + let blobs = db.cache_entries_by_kind("blob").unwrap(); + assert_eq!(blobs, vec![("xyz".into(), 1)]); + } + + /// A rebuild is a replacement, not a merge: whatever the index said before + /// is what a stale or externally-deleted cache file would leave behind. + #[test] + fn cache_index_rebuild_replaces_every_row() { + let db = Db::open_in_memory().unwrap(); + db.cache_touch("gone", "blob", 1, 1).unwrap(); + db.cache_touch("also-gone", "block", 1, 1).unwrap(); + + db.cache_rebuild(&[CacheEntryInput { + key: "kept", + kind: "blob", + size: 7, + last_accessed: 42, + }]) + .unwrap(); + + assert_eq!( + db.cache_entries_by_kind("blob").unwrap(), + vec![("kept".to_string(), 7)] + ); + assert!(db.cache_entries_by_kind("block").unwrap().is_empty()); + + // An empty rebuild is how a cache directory that vanished reports itself. + db.cache_rebuild(&[]).unwrap(); + assert!(db.cache_entries_by_kind("blob").unwrap().is_empty()); + } + + #[test] + fn pin_add_list_remove_roundtrip() { + let db = Db::open_in_memory().unwrap(); + db.pin_add("vol~a", "docs/a.txt", false).unwrap(); + db.pin_add("vol~d", "docs", true).unwrap(); + let list = db.pin_list().unwrap(); + assert_eq!(list.len(), 2); + assert_eq!(list[0], ("vol~a".into(), "docs/a.txt".into(), false)); + assert_eq!(list[1], ("vol~d".into(), "docs".into(), true)); + + // Re-pin refreshes path/flag, not a duplicate row. + db.pin_add("vol~a", "moved/a.txt", false).unwrap(); + let list = db.pin_list().unwrap(); + assert_eq!(list.len(), 2); + assert_eq!(list[0].1, "moved/a.txt"); + + assert!(db.pin_remove("vol~a").unwrap()); + assert!(!db.pin_remove("vol~a").unwrap()); + assert_eq!(db.pin_list().unwrap().len(), 1); + } + + #[test] + fn is_pinned_direct_without_node_row() { + // A direct pin counts even when the node was never hydrated into `nodes`. + let db = Db::open_in_memory().unwrap(); + db.pin_add("vol~a", "a.txt", false).unwrap(); + assert!(db.is_pinned("vol~a").unwrap()); + assert!(!db.is_pinned("vol~b").unwrap()); + } + + #[test] + fn recursive_folder_pin_covers_subtree() { + let db = Db::open_in_memory().unwrap(); + // root/docs/{report.pdf, sub/deep.txt}, root/loose.txt + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&folder("docs", Some("root"), "Documents")) + .unwrap(); + db.upsert_node(&file("rep", "docs", "report.pdf", 1)) + .unwrap(); + db.upsert_node(&folder("sub", Some("docs"), "Sub")).unwrap(); + db.upsert_node(&file("deep", "sub", "deep.txt", 1)).unwrap(); + db.upsert_node(&file("loose", "root", "loose.txt", 1)) + .unwrap(); + + // Pin the Documents folder recursively (uids are `vol~link` display form). + let du = |l: &str| uid(l).to_string(); + db.pin_add(&du("docs"), "Documents", true).unwrap(); + + // Everything under docs (any depth) is pinned; loose.txt is not. + assert!(db.is_pinned(&du("docs")).unwrap()); + assert!(db.is_pinned(&du("rep")).unwrap()); + assert!(db.is_pinned(&du("deep")).unwrap()); + assert!(!db.is_pinned(&du("loose")).unwrap()); + + // pinned_uids expands the subtree (folder + descendants), no loose.txt. + let mut got = db.pinned_uids().unwrap(); + got.sort(); + assert_eq!(got, vec![du("deep"), du("docs"), du("rep"), du("sub")]); + + // descendants() lists the subtree of a folder (excludes the folder). + let mut desc = db.descendants(&du("docs")).unwrap(); + desc.sort(); + assert_eq!(desc, vec![du("deep"), du("rep"), du("sub")]); + } + + #[test] + fn non_recursive_folder_pin_does_not_cover_children() { + let db = Db::open_in_memory().unwrap(); + db.upsert_node(&folder("root", None, "My Files")).unwrap(); + db.upsert_node(&folder("docs", Some("root"), "Documents")) + .unwrap(); + db.upsert_node(&file("rep", "docs", "report.pdf", 1)) + .unwrap(); + // A non-recursive pin on the folder covers only the folder itself. + let du = |l: &str| uid(l).to_string(); + db.pin_add(&du("docs"), "Documents", false).unwrap(); + assert!(db.is_pinned(&du("docs")).unwrap()); + assert!(!db.is_pinned(&du("rep")).unwrap()); + } + + #[test] + fn schema_objects_exist() { + let db = Db::open_in_memory().unwrap(); + let count: i64 = db + .with_conn(|c| { + Ok(c.query_row( + "SELECT count(*) FROM sqlite_master + WHERE name IN ('nodes', 'nodes_fts', 'cache_entries')", + [], + |r| r.get(0), + )?) + }) + .unwrap(); + assert_eq!(count, 3); + } diff --git a/crates/pdfs-core/src/db/trash.rs b/crates/pdfs-core/src/db/trash.rs new file mode 100644 index 0000000..e2f8ad7 --- /dev/null +++ b/crates/pdfs-core/src/db/trash.rs @@ -0,0 +1,65 @@ +//! Trashed nodes, persisted so the trash view opens without a round-trip. + +use rusqlite::params; + +use super::Db; +use crate::Result; + +pub struct StoredTrash { + pub uid: String, + pub name: String, + pub is_dir: bool, + pub size: i64, + pub mtime: i64, +} + +impl Db { + pub fn trash_replace(&self, items: &[StoredTrash]) -> Result<()> { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + tx.execute("DELETE FROM trash", [])?; + { + let mut stmt = tx.prepare( + "INSERT INTO trash (uid, name, is_dir, size, mtime) VALUES (?1, ?2, ?3, ?4, ?5)", + )?; + for item in items { + stmt.execute(params![ + item.uid, + item.name, + item.is_dir as i64, + item.size, + item.mtime + ])?; + } + } + tx.commit()?; + Ok(()) + } + + /// The persisted trash listing, folders first then by name — the order the + /// Trash page shows it in. + pub fn trash_list(&self) -> Result> { + let conn = self.conn.lock(); + let mut stmt = conn.prepare( + "SELECT uid, name, is_dir, size, mtime FROM trash + ORDER BY is_dir DESC, name COLLATE NOCASE", + )?; + let rows = stmt.query_map([], |r| { + Ok(StoredTrash { + uid: r.get(0)?, + name: r.get(1)?, + is_dir: r.get::<_, i64>(2)? != 0, + size: r.get(3)?, + mtime: r.get(4)?, + }) + })?; + let mut items = Vec::new(); + for row in rows { + items.push(row?); + } + Ok(items) + } + + // ---- device sync (devices.md) ----------------------------------------- + +} diff --git a/crates/pdfs-core/src/db/utils.rs b/crates/pdfs-core/src/db/utils.rs new file mode 100644 index 0000000..a60462c --- /dev/null +++ b/crates/pdfs-core/src/db/utils.rs @@ -0,0 +1,59 @@ +//! Small query helpers shared across the table modules. + +use rusqlite::{Connection, params}; + +use crate::Result; + +/// Below this length the trigram tokenizer indexes nothing (it needs 3-char +/// grams), so short queries fall back to a `LIKE` scan over `nodes.name`. +pub(super) const TRIGRAM_MIN: usize = 3; + +pub(super) fn pair(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> { + Ok((row.get(0)?, row.get(1)?)) +} + +/// Drain a `query_map` of [`pair`] rows into a `Vec`, propagating row errors. +pub(super) fn collect_pairs( + rows: impl Iterator>, +) -> Result> { + let mut out = Vec::new(); + for r in rows { + out.push(r?); + } + Ok(out) +} + +/// Escape `LIKE` wildcards in a user query so `%` and `_` match literally +/// (paired with `ESCAPE '\'` in the statement). +pub(super) fn like_escape(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// Resolve a node's mountpoint-relative path by walking `parent_uid` to the +/// root via a recursive CTE. The root (the node with no parent) is excluded, so +/// a top-level file `report.pdf` yields `"report.pdf"`, not `"My Files/report.pdf"`. +pub(super) fn path_of(conn: &Connection, uid: &str) -> Result { + let mut stmt = conn.prepare( + "WITH RECURSIVE anc(uid, parent_uid, name, depth) AS ( + SELECT uid, parent_uid, name, 0 FROM nodes WHERE uid = ?1 + UNION ALL + SELECT n.uid, n.parent_uid, n.name, anc.depth + 1 + FROM nodes n JOIN anc ON n.uid = anc.parent_uid + ) + SELECT name, parent_uid FROM anc ORDER BY depth DESC", + )?; + let rows = stmt.query_map(params![uid], |r| { + Ok((r.get::<_, String>(0)?, r.get::<_, Option>(1)?)) + })?; + let mut parts = Vec::new(); + for r in rows { + let (name, parent_uid) = r?; + // Skip the root node (no parent); its name is the mount itself. + if parent_uid.is_some() { + parts.push(name); + } + } + Ok(parts.join("/")) +} diff --git a/crates/pdfs-fuse/src/control.rs b/crates/pdfs-fuse/src/control.rs new file mode 100644 index 0000000..d131d73 --- /dev/null +++ b/crates/pdfs-fuse/src/control.rs @@ -0,0 +1,717 @@ +//! The control socket: the daemon`s IPC surface. +//! +//! The CLI and the GUI never touch the mount or the database directly — they +//! open this Unix socket, write one JSON [`CtlRequest`] line, and read one JSON +//! [`CtlResponse`] line back. That keeps the daemon the single writer of both +//! the FUSE state and `cache.db`. +//! +//! Connections are served concurrently, one thread each, because requests differ +//! wildly in cost: an `OpenPhoto` downloads a whole photo, and a serial loop +//! would stall the GUI`s 2s status poll behind it. + +use std::io::{BufRead, BufReader, Write as _}; +use std::sync::atomic::Ordering; +use std::time::Instant; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; + +use pdfs_core::config::AppDirs; +use pdfs_core::control::{ + ActivityKind, PhotoMonth, RefreshScope, Request as CtlRequest, Response as CtlResponse, + TransferDirection, +}; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use tracing::{info, warn}; + +use super::transfers::CountingReader; +use super::{Core, count_noun, human_bytes, human_duration, parse_uid}; + +/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path +/// must live under `mountpoint`; a relative path is taken as already relative to +/// the mount root. +fn rel_to_mount(mountpoint: &Path, path: &str) -> Result { + let p = Path::new(path); + if p.is_absolute() { + p.strip_prefix(mountpoint) + .map(Path::to_path_buf) + .map_err(|_| format!("{path} is not under the mountpoint")) + } else { + Ok(p.to_path_buf()) + } +} + +/// Handle one control-socket connection: read a single JSON request line, +/// dispatch it against `core`, and write back a JSON response line. +fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: UnixStream) { + let mut reader = BufReader::new(match stream.try_clone() { + Ok(s) => s, + Err(e) => { + warn!(error = %e, "control: clone stream failed"); + return; + } + }); + let mut line = String::new(); + if reader.read_line(&mut line).is_err() || line.trim().is_empty() { + return; + } + let response = match serde_json::from_str::(line.trim()) { + Ok(CtlRequest::Status) => { + let pins = core.cache.list_pins(); + let queued = core.db.pending_op_counts().unwrap_or_default(); + CtlResponse::Status { + username: username.to_string(), + mountpoint: mountpoint.display().to_string(), + pinned: pins.len(), + used: core.cache.usage(), + budget: core.cache.budget(), + pins, + online: core.online.load(Ordering::Relaxed), + pending_uploads: queued.uploads.max(0) as u64, + pending_changes: queued.changes.max(0) as u64, + } + } + Ok(CtlRequest::Pin { path }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.pin(&rel) { + Ok(name) => CtlResponse::Ok { + message: format!("pinned {name}"), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::Unpin { path }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.unpin(&rel) { + Ok(name) => CtlResponse::Ok { + message: format!("unpinned {name}"), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListPins) => CtlResponse::Pins { + pins: core.cache.list_pins(), + }, + Ok(CtlRequest::ListDir { path }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.list_dir(&rel) { + Ok(entries) => CtlResponse::Entries { entries }, + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::Refresh { scope }) => { + let result = match &scope { + RefreshScope::Dir { path } => { + rel_to_mount(mountpoint, path).and_then(|rel| core.refresh_dir(&rel)) + } + RefreshScope::Trash => { + core.invalidate_trash(); + Ok(()) + } + RefreshScope::Photos => { + core.invalidate_photos(); + Ok(()) + } + }; + match result { + Ok(()) => CtlResponse::Ok { + message: "refreshed".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::PhotosTimeline { + offset, + limit, + kind, + range, + }) => { + match core.photos_timeline(offset, limit, kind, range) { + Ok(Some(items)) => CtlResponse::Photos { + available: true, + items, + counts: core.db.photos_counts().ok(), + }, + Ok(None) => CtlResponse::Photos { + available: false, + items: Vec::new(), + counts: None, + }, + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::PhotoMonths { kind }) => match core.db.photos_months(kind) { + Ok(months) => CtlResponse::PhotoMonths { + months: months + .into_iter() + .map(|(year, month, count)| PhotoMonth { + year, + month, + count, + }) + .collect(), + }, + Err(e) => CtlResponse::Error { + message: e.to_string(), + }, + }, + Ok(CtlRequest::PhotoThumbs { uids }) => { + let parsed: Vec = uids.iter().filter_map(|u| parse_uid(u)).collect(); + CtlResponse::Thumbs { + items: core.photo_thumbs(&parsed), + } + } + Ok(CtlRequest::OpenPhoto { uid }) => match parse_uid(&uid) { + Some(u) => match core.open_photo(&u) { + Ok(p) => CtlResponse::FilePath { + path: p.display().to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + None => CtlResponse::Error { + message: format!("bad uid: {uid}"), + }, + }, + Ok(CtlRequest::UploadPhoto { + name, + media_type, + bytes, + capture_time, + }) => { + let photos = core.photos(); + let metadata = proton_drive_rs::PhotoUploadMetadata { + capture_time, + ..Default::default() + }; + let guard = core.transfers.begin( + name.clone(), + "", + TransferDirection::Upload, + bytes.len() as u64, + ); + let reader = CountingReader::new(std::io::Cursor::new(&bytes), &guard); + match core.rt.block_on(photos.upload_photo_from( + &name, + &media_type, + reader, + bytes.len() as i64, + Vec::new(), + metadata, + false, + )) { + Ok(uid) => { + // The photo we just uploaded belongs at the head of the + // timeline, and the GUI reloads the gallery the moment this + // reply lands — so refresh now rather than leaving it to a + // background pass that would land just after that reload. + if let Err(e) = core.rt.block_on(core.refresh_timeline()) { + warn!(error = %e, "timeline refresh after upload failed"); + } + CtlResponse::Ok { + message: format!("uploaded photo with uid {uid}"), + } + } + Err(e) => CtlResponse::Error { + message: format!("upload photo failed: {e}"), + }, + } + } + Ok(CtlRequest::OpenFile { path }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.open_file(&rel) { + Ok(p) => CtlResponse::FilePath { + path: p.display().to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::Search { query, limit }) => match core.search(&query, limit) { + Ok(hits) => CtlResponse::SearchResults { hits }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::SearchLocal { query, limit }) => match core.search_local(&query, limit) { + Ok(hits) => CtlResponse::LocalResults { + hits, + indexing: core.indexing.load(Ordering::Relaxed), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::Rename { path, new_name }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.rename(&rel, &new_name) { + Ok(name) => { + core.log_activity(ActivityKind::Rename, &name, format!("was {path}"), true); + CtlResponse::Ok { + message: format!("renamed to {name}"), + } + } + Err(e) => { + core.log_activity(ActivityKind::Rename, &path, &e, false); + CtlResponse::Error { message: e } + } + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::Move { path, new_parent }) => { + match ( + rel_to_mount(mountpoint, &path), + rel_to_mount(mountpoint, &new_parent), + ) { + (Ok(rel), Ok(parent_rel)) => match core.move_to(&rel, &parent_rel) { + Ok(name) => { + let dest = if new_parent.is_empty() { + "My files".to_string() + } else { + new_parent.clone() + }; + core.log_activity(ActivityKind::Move, &name, format!("to {dest}"), true); + CtlResponse::Ok { + message: format!("moved {name}"), + } + } + Err(e) => { + core.log_activity(ActivityKind::Move, &path, &e, false); + CtlResponse::Error { message: e } + } + }, + (Err(e), _) | (_, Err(e)) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::Delete { path }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.delete(&rel) { + Ok(name) => { + core.log_activity(ActivityKind::Trash, &name, "", true); + CtlResponse::Ok { + message: format!("trashed {name}"), + } + } + Err(e) => { + core.log_activity(ActivityKind::Trash, &path, &e, false); + CtlResponse::Error { message: e } + } + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::CreateFolder { parent, name }) => match rel_to_mount(mountpoint, &parent) { + Ok(parent_rel) => match core.create_folder(&parent_rel, &name) { + Ok(name) => { + core.log_activity(ActivityKind::CreateFolder, &name, "", true); + CtlResponse::Ok { + message: format!("created folder {name}"), + } + } + Err(e) => { + core.log_activity(ActivityKind::CreateFolder, &name, &e, false); + CtlResponse::Error { message: e } + } + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::UploadFile { + parent, + name, + bytes, + }) => match rel_to_mount(mountpoint, &parent) { + Ok(parent_rel) => match core.upload(&parent_rel, &name, &bytes) { + Ok(name) => { + core.log_activity(ActivityKind::Upload, &name, "", true); + CtlResponse::Ok { + message: format!("uploaded {name}"), + } + } + Err(e) => { + core.log_activity(ActivityKind::Upload, &name, &e, false); + CtlResponse::Error { message: e } + } + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::UploadPaths { parent, sources }) => { + match rel_to_mount(mountpoint, &parent) { + // Ack immediately and upload on a background thread: a directory tree + // far outlasts the socket read timeout. Progress and completion are + // observed through GetQueueStatus; the activity log gets the summary + // when the whole batch finishes. + Ok(parent_rel) => { + let core = core.clone(); + let paths: Vec = sources.into_iter().map(PathBuf::from).collect(); + let n = paths.len(); + std::thread::spawn(move || { + let started = Instant::now(); + match core.upload_paths(&parent_rel, &paths) { + Ok(stats) => { + // e.g. "Uploaded 700 files to Photos" — name the destination + // so the log reads like a sentence rather than a bare count. + let dest = parent_rel + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("your Drive"); + let target = format!( + "{} to {dest}", + count_noun(stats.uploaded, "file", "files") + ); + // Size · folders · elapsed, with a trailing failure count + // when some files didn't make it. + let mut parts = vec![human_bytes(stats.bytes)]; + if stats.folders > 0 { + parts.push(count_noun(stats.folders, "folder", "folders")); + } + parts.push(human_duration(started.elapsed())); + if stats.failed > 0 { + parts.push(format!("{} failed", stats.failed)); + } + core.log_activity( + ActivityKind::Upload, + target, + parts.join(" · "), + stats.failed == 0, + ); + } + Err(e) => { + warn!(error = %e, "bulk upload failed"); + core.log_activity(ActivityKind::Upload, "bulk upload", &e, false); + } + } + }); + CtlResponse::Ok { + message: format!("uploading {n} item(s)"), + } + } + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::ListTrash) => match core.list_trash() { + Ok(entries) => CtlResponse::Entries { entries }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::Restore { uids }) => match core.restore(&uids) { + Ok(n) => { + core.log_activity(ActivityKind::Restore, format!("{n} item(s)"), "", true); + CtlResponse::Ok { + message: format!("restored {n} item(s)"), + } + } + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::DeleteForever { uids }) => match core.delete_forever(&uids) { + Ok(n) => { + core.log_activity( + ActivityKind::DeleteForever, + format!("{n} item(s)"), + "", + true, + ); + CtlResponse::Ok { + message: format!("permanently deleted {n} item(s)"), + } + } + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::EmptyTrash) => match core.empty_trash() { + Ok(n) => { + core.log_activity(ActivityKind::EmptyTrash, format!("{n} item(s)"), "", true); + CtlResponse::Ok { + message: format!("emptied trash ({n} item(s))"), + } + } + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::PurgeCache) => { + let freed = core.cache.clear_unpinned(); + CtlResponse::Ok { + message: format!( + "purged {:.1} MiB of unpinned cache", + freed as f64 / 1_048_576.0 + ), + } + } + Ok(CtlRequest::GetQueueStatus) => CtlResponse::Transfers { + items: core.transfers.snapshot(), + jobs: core.jobs_snapshot(), + }, + Ok(CtlRequest::SetCacheBudget { bytes }) => { + core.cache.set_budget(bytes); + // Persist so the next mount keeps the new cap. Best-effort: a config + // write failure is reported but the live cap is already applied. + match AppDirs::new().map(|dirs| { + let mut cfg = dirs.load_config(); + cfg.cache_budget = Some(bytes); + dirs.save_config(&cfg) + }) { + Ok(Ok(())) => CtlResponse::Ok { + message: format!("cache budget set to {bytes} bytes"), + }, + Ok(Err(e)) => CtlResponse::Error { + message: format!("budget applied but config write failed: {e}"), + }, + Err(e) => CtlResponse::Error { + message: format!("budget applied but config unavailable: {e}"), + }, + } + } + Ok(CtlRequest::ListDevices) => match core.list_devices() { + Ok(items) => CtlResponse::Devices { items }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::RenameDevice { uid, name }) => match core.rename_device(&uid, &name) { + Ok(()) => CtlResponse::Ok { + message: format!("renamed device to {name}"), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::DeleteDevice { uid }) => match core.delete_device(&uid) { + Ok(()) => CtlResponse::Ok { + message: "device deleted".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::AddSyncFolder { local_path }) => { + // Registering the device and uploading a folder tree far outlasts the + // socket read timeout, so ack immediately and work on a background + // thread. The folder appears in ListSyncFolders once the row lands; + // completion (and any failures) go to the activity log. + let core = core.clone(); + let path = PathBuf::from(&local_path); + std::thread::spawn(move || { + let started = Instant::now(); + match core.add_sync_folder(&path) { + Ok(folder) => { + let name = Path::new(&folder.local_path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(&folder.local_path) + .to_string(); + core.log_activity( + ActivityKind::Upload, + format!("synced {name}"), + human_duration(started.elapsed()), + folder.state != "error", + ); + } + Err(e) => { + warn!(error = %e, "add sync folder failed"); + core.log_activity(ActivityKind::Upload, "add sync folder", &e, false); + } + } + }); + CtlResponse::Ok { + message: format!("adding {local_path}"), + } + } + Ok(CtlRequest::ListSyncFolders) => match core.list_sync_folders() { + Ok(items) => CtlResponse::SyncFolders { items }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::RemoveSyncFolder { id, delete_remote }) => { + match core.remove_sync_folder(id, delete_remote) { + Ok(()) => CtlResponse::Ok { + message: "removed synced folder".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::SetSyncFolderMode { id, mode }) => { + match core.request_sync_folder_mode(id, &mode) { + Ok(message) => { + core.log_activity(ActivityKind::Upload, &message, "", true); + CtlResponse::Ok { message } + } + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::SyncNow { id }) => { + core.sync_now(id); + CtlResponse::Ok { + message: match id { + Some(id) => format!("reconciling folder {id}"), + None => "reconciling all folders".to_string(), + }, + } + } + Ok(CtlRequest::ShareNode { + path, + emails, + role, + message, + }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.share_node(&rel, &emails, &role, message.as_deref()) { + Ok((proton, external)) => { + core.log_activity( + ActivityKind::Share, + &path, + format!("{} recipient(s) as {role}", proton + external), + true, + ); + CtlResponse::Ok { + message: format!("invited {proton} Proton and {external} external user(s)"), + } + } + Err(e) => { + core.log_activity(ActivityKind::Share, &path, &e, false); + CtlResponse::Error { message: e } + } + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListShare { path }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.list_share(&rel) { + Ok((entries, link)) => CtlResponse::Share { entries, link }, + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::UpdateShareRole { + path, + id, + kind, + role, + }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.update_share_role(&rel, &id, kind, &role) { + Ok(()) => CtlResponse::Ok { + message: format!("role updated to {role}"), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::RemoveShareEntry { path, id, kind }) => { + match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.remove_share_entry(&rel, &id, kind) { + Ok(()) => { + core.log_activity(ActivityKind::Unshare, &path, "access removed", true); + CtlResponse::Ok { + message: "removed".to_string(), + } + } + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::CreatePublicLink { + path, + role, + password, + expires, + }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.create_public_link(&rel, &role, password.as_deref(), expires) { + Ok(link) => { + core.log_activity(ActivityKind::PublicLink, &path, "link created", true); + CtlResponse::PublicLink { link } + } + Err(e) => { + core.log_activity(ActivityKind::PublicLink, &path, &e, false); + CtlResponse::Error { message: e } + } + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::RemovePublicLink { path, id }) => match rel_to_mount(mountpoint, &path) { + Ok(rel) => match core.remove_public_link(&rel, &id) { + Ok(()) => { + core.log_activity(ActivityKind::Unshare, &path, "link removed", true); + CtlResponse::Ok { + message: "public link removed".to_string(), + } + } + Err(e) => CtlResponse::Error { message: e }, + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListSharedWithMe) => match core.list_shared_with_me() { + Ok(entries) => CtlResponse::Entries { entries }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::LeaveShared { uid }) => match core.leave_shared(&uid) { + Ok(()) => { + core.log_activity(ActivityKind::Unshare, "shared item", "left", true); + CtlResponse::Ok { + message: "left shared item".to_string(), + } + } + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListInvitations) => match core.list_invitations() { + Ok(items) => CtlResponse::Invitations { items }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::AcceptInvitation { id }) => match core.accept_invitation(&id) { + Ok(()) => CtlResponse::Ok { + message: "invitation accepted".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::RejectInvitation { id }) => match core.reject_invitation(&id) { + Ok(()) => CtlResponse::Ok { + message: "invitation rejected".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListBookmarks) => match core.list_bookmarks() { + Ok(items) => CtlResponse::Bookmarks { items }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::CreateBookmark { url, password }) => { + match core.create_bookmark(&url, password.as_deref()) { + Ok(()) => CtlResponse::Ok { + message: "bookmark saved".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::DeleteBookmark { token }) => match core.delete_bookmark(&token) { + Ok(()) => CtlResponse::Ok { + message: "bookmark removed".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListSharedByMe) => match core.list_shared_by_me() { + Ok(items) => CtlResponse::SharedByMe { items }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::ListActivity { limit }) => CtlResponse::Activity { + items: core.list_activity(limit), + }, + Err(e) => CtlResponse::Error { + message: format!("bad request: {e}"), + }, + }; + let mut out = match serde_json::to_vec(&response) { + Ok(v) => v, + Err(e) => { + warn!(error = %e, "control: serialize response failed"); + return; + } + }; + out.push(b'\n'); + let mut stream = stream; + let _ = stream.write_all(&out); +} + +/// Listen on the control socket, serving one command per connection, each on its +/// own thread. Runs on its own thread; returns only if the listener itself fails. +/// +/// Concurrent rather than serial because requests differ wildly in cost: an +/// `OpenPhoto` downloads a whole photo, and while it ran the accept loop used to +/// stall every other caller behind it — the GUI's 2s status poll, and the +/// thumbnail batches the gallery needs to paint. [`Core`] is a bundle of handles +/// (`Arc`/`Clone`), so each connection gets its own copy of it. +pub(crate) fn run_control_socket(core: Core, username: String, mountpoint: PathBuf, listener: UnixListener) { + info!("control socket listening"); + for conn in listener.incoming() { + match conn { + Ok(stream) => { + let core = core.clone(); + let username = username.clone(); + let mountpoint = mountpoint.clone(); + if let Err(e) = std::thread::Builder::new() + .name("pdfs-control".into()) + .spawn(move || handle_control_conn(&core, &username, &mountpoint, stream)) + { + warn!(error = %e, "control: spawn handler failed"); + } + } + Err(e) => { + warn!(error = %e, "control: accept failed"); + } + } + } +} diff --git a/crates/pdfs-fuse/src/devices.rs b/crates/pdfs-fuse/src/devices.rs new file mode 100644 index 0000000..465bd79 --- /dev/null +++ b/crates/pdfs-fuse/src/devices.rs @@ -0,0 +1,698 @@ +//! This machine as a Proton Drive Device, and the local folders it keeps in +//! step with the remote (devices.md). +//! +//! A sync folder runs in one of two modes. `mirror` keeps a real local tree that +//! the engine in [`super::sync`] reconciles both ways; `ondemand` evicts that +//! tree and mounts a second FUSE session over the same path, rooted at the +//! folder`s remote node. Flipping between them is the delicate part — see +//! [`Core::apply_sync_folder_mode`]. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use parking_lot::Mutex; +use pdfs_core::control::{ActivityKind, DeviceInfo, JobItem, SyncFolderInfo, SyncPhase, SyncProgress}; +use pdfs_core::db::{StoredDevice, StoredSyncFolder}; +use fuser::{Config, MountOption, Session}; +use proton_drive_rs::{DeviceType, Node}; +use proton_drive_rs::proton_sdk::ids::{DeviceUid, NodeUid}; +use tracing::{info, warn}; + +use super::{ + BackgroundSession, Core, ProtonFs, State, SwitchBlocked, clear_stale_mount, device_type_str, dir_is_empty, + evict_dir_contents, now_secs, parse_uid, sync_folder_info, this_hostname, +}; +use super::sync::{self, base_name}; + +impl Core { + // ---- devices ---------------------------------------------------------- + + /// List the account's registered devices, flagging the one *this* machine + /// syncs to so a front-end can treat it as more than another computer in the + /// list — deleting it takes this machine's synced folders down with it. + pub(crate) fn list_devices(&self) -> Result, String> { + let devices = self + .rt + .block_on(self.client.enumerate_devices()) + .map_err(|e| format!("list devices: {e}"))?; + // No cached device row yet means this machine syncs nothing, so none of + // the listed devices is ours. + let this_uid = self.db.device_get().ok().flatten().map(|d| d.uid); + Ok(devices + .into_iter() + .map(|d| { + let uid = d.uid.to_string(); + DeviceInfo { + this_device: this_uid.as_deref() == Some(uid.as_str()), + uid, + name: d.name.unwrap_or_else(|_| "(unnamed device)".to_string()), + device_type: device_type_str(d.device_type).to_string(), + last_sync: d.last_sync_time, + } + }) + .collect()) + } + + /// Rename a device by its uid. + pub(crate) fn rename_device(&self, uid: &str, name: &str) -> Result<(), String> { + if name.is_empty() { + return Err("device name must not be empty".to_string()); + } + let device_uid = DeviceUid::from(uid); + self.rt + .block_on(self.client.rename_device(&device_uid, name)) + .map_err(|e| format!("rename device: {e}"))?; + Ok(()) + } + + /// Delete (deregister) a device by its uid. + pub(crate) fn delete_device(&self, uid: &str) -> Result<(), String> { + let device_uid = DeviceUid::from(uid); + self.rt + .block_on(self.client.delete_device(&device_uid)) + .map_err(|e| format!("delete device: {e}"))?; + Ok(()) + } + + // ---- device folder sync (devices.md, Phase 1) ------------------------- + + /// Auto-register (or recover) this machine as a Proton Drive Device, caching + /// it so restarts reuse the same device. Recovery matches an existing remote + /// Linux device by name before creating a new one, so a lost local record + /// doesn't orphan the device's root folder. + pub(crate) fn ensure_device(&self) -> Result { + let name = this_hostname(); + // Enumerate the remote devices once: used both to validate any cached + // record and to recover an existing device by name. + let remote = self + .rt + .block_on(self.client.enumerate_devices()) + .map_err(|e| format!("enumerate devices: {e}"))?; + + // A cached device is only trustworthy if it still exists remotely. A + // device deleted from another client (or the web UI) leaves a stale row + // whose root folder is gone, so creating folders under it fails with + // "parent node is not a folder". Re-register in that case. + if let Some(dev) = self.db.device_get().map_err(|e| format!("db: {e:?}"))? { + if remote.iter().any(|d| d.uid.to_string() == dev.uid) { + return Ok(dev); + } + warn!(uid = %dev.uid, "cached device is gone remotely; re-registering"); + } + + // Recover: an existing remote Linux device with the same name is ours. + let existing = remote.into_iter().find(|d| { + d.device_type == DeviceType::Linux && d.name.as_deref().ok() == Some(name.as_str()) + }); + let dev = match existing { + Some(d) => StoredDevice { + uid: d.uid.to_string(), + share_id: d.share_id.to_string(), + root_uid: d.root_folder_uid.to_string(), + name, + created: d.creation_time, + }, + None => { + let d = self + .rt + .block_on(self.client.create_device(&name, DeviceType::Linux)) + .map_err(|e| format!("create device: {e}"))?; + StoredDevice { + uid: d.uid.to_string(), + share_id: d.share_id.to_string(), + root_uid: d.root_folder_uid.to_string(), + name, + created: d.creation_time, + } + } + }; + self.db.device_set(&dev).map_err(|e| format!("db: {e:?}"))?; + Ok(dev) + } + + /// An untrashed folder named `name` directly under the device root, if one + /// already exists. + pub(crate) fn find_device_child_folder( + &self, + root_uid: &NodeUid, + name: &str, + ) -> Result, String> { + let uids = self + .rt + .block_on(self.client.enumerate_folder_children_node_uids(root_uid)) + .map_err(|e| format!("list device root: {e}"))?; + if uids.is_empty() { + return Ok(None); + } + let nodes = self + .rt + .block_on(self.client.enumerate_nodes(&uids)) + .map_err(|e| format!("resolve device root children: {e}"))?; + Ok(nodes + .into_iter() + .find(|n| n.is_folder() && !n.trashed && n.name == name) + .map(|n| n.uid)) + } + + /// Add a local folder to this device's sync set: register the device if + /// needed, create a matching folder under the device root, upload the local + /// tree into it once, and record the mapping. Phase 1 is a one-shot upload — + /// the two-way engine (Phase 2) reconciles later changes. + pub(crate) fn add_sync_folder(&self, local: &Path) -> Result { + let meta = + std::fs::metadata(local).map_err(|e| format!("stat {}: {e}", local.display()))?; + if !meta.is_dir() { + return Err(format!("{} is not a directory", local.display())); + } + let local = local + .canonicalize() + .map_err(|e| format!("canonicalize {}: {e}", local.display()))?; + let local_str = local.to_string_lossy().to_string(); + + // Reject duplicates up front for a clear error (UNIQUE would also catch it). + if self + .db + .sync_folder_list() + .map_err(|e| format!("db: {e:?}"))? + .iter() + .any(|f| f.local_path == local_str) + { + return Err(format!("{} is already synced", local.display())); + } + + let name = local + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| format!("unusable folder name: {}", local.display()))? + .to_string(); + + let device = self.ensure_device()?; + let root_uid = parse_uid(&device.root_uid) + .ok_or_else(|| format!("bad device root uid: {}", device.root_uid))?; + + // The synced folder's remote root: the folder under the device root named + // after the local basename. Reuse an existing one rather than creating a + // second folder with the same name — re-adding a folder (after a removal, or + // after a failed add that had already created it) must land back on the + // original, not leave the user with two "Downloads" in their Drive. The + // reconcile treats an existing remote tree correctly: unmatched paths read as + // a conflict, not as data loss. + let remote_root = match self.find_device_child_folder(&root_uid, &name)? { + Some(uid) => { + info!(name, "reusing existing device folder"); + uid + } + None => self + .rt + .block_on( + self.client + .create_folder(&root_uid, &name, Some(now_secs())), + ) + .map_err(|e| format!("create device folder {name}: {e}"))?, + }; + + let id = self + .db + .sync_folder_add(&local_str, &remote_root.to_string(), &device.share_id) + .map_err(|e| format!("db: {e:?}"))?; + + // Hand the initial upload to the sync engine: an empty baseline against a + // full local tree reconciles as "upload everything", and the folder is + // added to the filesystem watch set in the same pass. + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); + + info!(local = %local.display(), id, "added sync folder"); + self.db + .sync_folder_get(id) + .map_err(|e| format!("db: {e:?}"))? + .ok_or_else(|| "sync folder vanished after insert".to_string()) + } + + /// List this device's synced folders for the front-ends, each carrying the + /// live progress of its pass when one is running. + pub(crate) fn list_sync_folders(&self) -> Result, String> { + let progress = self.sync_progress.lock(); + Ok(self + .db + .sync_folder_list() + .map_err(|e| format!("db: {e:?}"))? + .into_iter() + .map(|f| { + let live = progress.get(&f.id).cloned(); + sync_folder_info(f, live) + }) + .collect()) + } + + /// Everything the daemon is chewing on that isn't moving bytes, for + /// `GetQueueStatus`: the registered jobs (bulk-upload scans, the local index) + /// plus a synthesized job per running sync pass, so one Activity view answers + /// "is anything still happening?" without also polling `ListSyncFolders`. + /// + /// The sync passes are folded in here rather than tracked as registry jobs + /// because the Devices page needs them per folder anyway + /// ([`SyncFolderInfo::progress`]) — this keeps one source of truth and hits + /// the db only while a pass is actually running. + pub(crate) fn jobs_snapshot(&self) -> Vec { + let mut jobs = self.transfers.jobs_snapshot(); + let mut passes: Vec<(i64, SyncProgress)> = self + .sync_progress + .lock() + .iter() + .map(|(id, p)| (*id, p.clone())) + .collect(); + if passes.is_empty() { + return jobs; + } + passes.sort_by_key(|(id, _)| *id); + + let names: HashMap = self + .db + .sync_folder_list() + .unwrap_or_default() + .into_iter() + .map(|f| (f.id, f.local_path)) + .collect(); + for (id, p) in passes { + // The row is titled with the folder's own name; the full local path + // is what the Devices page shows, and is far too long for this line. + let folder = names + .get(&id) + .and_then(|path| Path::new(path).file_name()) + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| "folder".to_string()); + jobs.push(match p.phase { + // The scan's total is the last pass's baseline, so a folder that + // has never synced still reports indeterminate (`total: 0`) — but + // every later pass has a real bar. A grown folder can push `done` + // past the estimate; clamp so the row never reads "600 of 500". + SyncPhase::Scanning => JobItem { + title: format!("Checking {folder}"), + detail: "Looking for changes".to_string(), + done: p.done as u64, + total: if p.total == 0 { + 0 + } else { + p.total.max(p.done) as u64 + }, + }, + SyncPhase::Applying => JobItem { + title: format!("Syncing {folder}"), + detail: p.current.clone(), + done: p.done as u64, + total: p.total.max(p.done) as u64, + }, + }); + } + jobs + } + + /// The lock guarding sync-folder `id` against concurrent reconcile/mode-switch. + pub(crate) fn sync_lock(&self, id: i64) -> Arc> { + self.sync_locks + .lock() + .entry(id) + .or_default() + .clone() + } + + /// Remove a synced folder from the sync set. `delete_remote` also deletes its + /// folder under the device root; otherwise the cloud copy is left in place. + pub(crate) fn remove_sync_folder(&self, id: i64, delete_remote: bool) -> Result<(), String> { + let folder = self + .db + .sync_folder_get(id) + .map_err(|e| format!("db: {e:?}"))? + .ok_or_else(|| format!("no synced folder with id {id}"))?; + // An `ondemand` folder *is* a FUSE mount over its local path, so dropping + // only the row would strand the mount: the path would keep serving a + // folder the daemon no longer tracks, and nothing would ever unmount it. + // Tear it down first — including before trashing the remote tree it + // serves, which would otherwise leave it answering for deleted nodes. + if let Some(session) = self.mounts.lock().remove(&id) { + if let Err(e) = session.umount_and_join() { + warn!(id, error = %e, "unmount on-demand folder failed"); + } else { + info!(id, path = %folder.local_path, "unmounted on-demand folder"); + } + } + if delete_remote + && let Some(uid) = parse_uid(&folder.remote_uid) + && let Err(e) = self.rt.block_on(self.client.trash_nodes(&[uid])) + { + warn!(id, error = %e, "delete remote device folder failed"); + } + if !self + .db + .sync_folder_remove(id) + .map_err(|e| format!("db: {e:?}"))? + { + return Err(format!("no synced folder with id {id}")); + } + // Stop watching the folder we just dropped. + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + Ok(()) + } + + /// Trigger a reconcile: one folder by id, or every folder when `id` is `None`. + pub(crate) fn sync_now(&self, id: Option) { + let _ = match id { + Some(id) => self.sync_tx.send(sync::SyncMsg::Reconcile(id)), + None => self.sync_tx.send(sync::SyncMsg::ReconcileAll), + }; + } + + /// A sibling Core that shares this one's client/rt/cache/db (and transfer, + /// activity, mount registries) but gets a **fresh, empty `State`** — its own + /// inode space starting at [`ROOT_INO`]. Used to root a secondary FUSE session + /// at an `ondemand` sync folder without colliding with the main mount's inodes + /// (devices.md Phase 3). + pub(crate) fn fork_state(&self) -> Core { + let mut fork = self.clone(); + fork.state = Arc::new(Mutex::new(State { + entries: HashMap::new(), + by_uid: HashMap::new(), + children: HashMap::new(), + next_ino: 2, + handles: HashMap::new(), + next_fh: 1, + db: self.db.clone(), + })); + fork + } + + /// Ask for a synced folder to move to `mode`, applying it now if the folder is + /// free and safe to switch, and **queueing** it otherwise. Returns the human + /// message for the reply. + /// + /// Queueing rather than rejecting is what makes the toggle usable: a folder + /// that syncs continuously (a busy Downloads folder) is almost never caught in + /// the narrow window where it is both unlocked and `idle`, so a `try_lock` + /// rejection asks the user to keep retrying until they get lucky. Instead the + /// intent is recorded and the engine applies it at the end of the pass already + /// running — which, seeing a queued `ondemand`, also stops pulling down files + /// it is about to evict ([`Core::push_pass`]). + pub(crate) fn request_sync_folder_mode(&self, id: i64, mode: &str) -> Result { + if mode != "mirror" && mode != "ondemand" { + return Err(format!("unknown mode {mode:?} (want mirror|ondemand)")); + } + let folder = self + .db + .sync_folder_get(id) + .map_err(|e| format!("db: {e:?}"))? + .ok_or_else(|| format!("no synced folder with id {id}"))?; + if folder.mode == mode { + // Already there. A queued switch the other way is the user changing + // their mind back before it landed, so withdraw it. + if folder.pending_mode.is_some() { + self.db + .sync_folder_set_pending_mode(id, None) + .map_err(|e| format!("db: {e:?}"))?; + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + return Ok(format!("staying {mode}")); + } + return Ok(format!("already {mode}")); + } + + match self.apply_sync_folder_mode(id, mode) { + Ok(message) => Ok(message), + // Not switchable this instant — remember the intent instead of making + // the user retry, and kick a pass to clear whatever is in the way. + Err(SwitchBlocked::NotNow) => { + self.db + .sync_folder_set_pending_mode(id, Some(mode)) + .map_err(|e| format!("db: {e:?}"))?; + let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); + Ok(match mode { + "ondemand" => format!( + "{} will go on-demand once its local changes are uploaded", + base_name(&folder.local_path) + ), + _ => format!( + "{} will start mirroring once the current sync finishes", + base_name(&folder.local_path) + ), + }) + } + Err(SwitchBlocked::Failed(e)) => Err(e), + } + } + + /// Apply a queued mode switch if the folder has one and is now able to take it. + /// Called by the sync engine after every pass, so a switch the user asked for + /// mid-sync lands as soon as the pass that blocked it is done. A folder that is + /// still not ready (its push failed, so the local copy is not safe to evict) + /// keeps its `pending_mode` and is retried after the next pass. + pub(crate) fn settle_pending_mode(&self, id: i64) { + let Ok(Some(folder)) = self.db.sync_folder_get(id) else { + return; + }; + let Some(mode) = folder.pending_mode.as_deref() else { + return; + }; + if folder.mode == mode { + let _ = self.db.sync_folder_set_pending_mode(id, None); + return; + } + match self.apply_sync_folder_mode(id, mode) { + Ok(message) => { + info!(id, mode, "applied queued mode switch"); + self.log_activity(ActivityKind::Sync, &message, "", true); + } + // Still blocked: the pass could not get everything up, so the local copy + // is not safe to evict yet. Leave the request standing — the next pass + // (poll, or the retry the engine schedules) tries again. + Err(SwitchBlocked::NotNow) => { + info!(id, mode, "queued mode switch still waiting"); + } + Err(SwitchBlocked::Failed(e)) => { + warn!(id, mode, error = %e, "queued mode switch failed; withdrawing"); + let _ = self.db.sync_folder_set_pending_mode(id, None); + self.log_activity( + ActivityKind::Sync, + format!("couldn't switch {}", base_name(&folder.local_path)), + e, + false, + ); + } + } + } + + /// Flip a synced folder between `mirror` (full local copy + two-way sync) and + /// `ondemand` (a FUSE mount over the local path; no local storage). Returns a + /// human message on success. + /// + /// - **mirror→ondemand**: require the folder in-sync (`idle`), stop watching it, + /// evict the local files to reclaim disk, then mount a secondary `ProtonFs` + /// rooted at the folder's remote node over its local path. + /// - **ondemand→mirror**: unmount, clear the stale baseline (the local tree was + /// evicted), then hand the folder back to the engine, which re-downloads it. + /// + /// [`SwitchBlocked::NotNow`] means "not yet, try after a pass" and is never an + /// error the user needs to see — callers queue on it. + pub(crate) fn apply_sync_folder_mode(&self, id: i64, mode: &str) -> Result { + // Hold the folder's lock for the whole switch so no reconcile pass can be + // running over the tree we are about to evict (or start while we mount over + // it). A pass in flight holds the lock for its full duration, so `try_lock` + // failing is exactly "still syncing" — and it is the only reliable signal: + // the `state` column is still `idle` in the window between `add_sync_folder` + // inserting the row and the engine picking it up. + let lock = self.sync_lock(id); + let Some(_guard) = lock.try_lock() else { + return Err(SwitchBlocked::NotNow); + }; + // Re-read under the lock: a pass that finished while we waited may have + // changed `state`. + let folder = self + .db + .sync_folder_get(id) + .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))? + .ok_or_else(|| SwitchBlocked::Failed(format!("no synced folder with id {id}")))?; + if folder.mode == mode { + return Ok(format!("already {mode}")); + } + let local = PathBuf::from(&folder.local_path); + + match mode { + "ondemand" => { + // Only flip a folder that is fully in sync — a failed reconcile means + // local edits could still be un-uploaded, and we are about to delete + // the local copy. Not an error: a pass makes this true, and the queued + // request is applied once one does. + if folder.state != "idle" { + return Err(SwitchBlocked::NotNow); + } + let root_uid = parse_uid(&folder.remote_uid).ok_or_else(|| { + SwitchBlocked::Failed(format!("bad remote uid: {}", folder.remote_uid)) + })?; + let root = self + .rt + .block_on(self.client.enumerate_nodes(std::slice::from_ref(&root_uid))) + .map_err(|e| SwitchBlocked::Failed(format!("fetch remote root: {e}")))? + .into_iter() + .next() + .ok_or_else(|| SwitchBlocked::Failed("remote folder not found".to_string()))?; + + // Reclaim the disk: empty the local dir (keep it as the mountpoint). + evict_dir_contents(&local).map_err(|e| { + SwitchBlocked::Failed(format!("evict {}: {e}", local.display())) + })?; + + let session = self + .spawn_ondemand_mount(&local, root) + .map_err(SwitchBlocked::Failed)?; + self.mounts.lock().insert(id, session); + // Persist the mode only now that the mount is actually up. Writing it + // first would strand the folder on any failure below: the engine skips + // non-mirror folders, so an `ondemand` row with no mount is a folder + // that is neither mirrored nor browsable, and nothing retries it. + // Failing before this point leaves it `mirror`, and the next pass + // re-downloads whatever eviction removed. + self.db + .sync_folder_set_mode(id, "ondemand") + .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?; + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + self.db.sync_folder_set_state(id, "idle", now_secs()).ok(); + info!(id, path = %local.display(), "mounted sync folder on-demand"); + Ok(format!("{} is now on-demand", local.display())) + } + _ => { + // ondemand→mirror: tear down the secondary mount first. + if let Some(session) = self.mounts.lock().remove(&id) + && let Err(e) = session.umount_and_join() + { + warn!(id, error = %e, "unmount on-demand folder failed"); + } + // The evicted local tree makes the old baseline lie ("everything + // deleted locally"); clear it so the reconcile is a pure download. + self.db + .sync_entries_clear(id) + .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?; + self.db + .sync_folder_set_mode(id, "mirror") + .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?; + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); + info!(id, path = %local.display(), "restored sync folder to mirror"); + Ok(format!( + "{} is mirroring again; downloading", + local.display() + )) + } + } + } + + /// Spawn a secondary FUSE session rooted at `root` over `local` on a forked + /// inode space. Clears any stale kernel mount first (a crashed run can leave + /// one, which would fail the fresh mount with EBUSY). + pub(crate) fn spawn_ondemand_mount(&self, local: &Path, root: Node) -> Result { + clear_stale_mount(local); + let mut config = Config::default(); + config.mount_options = vec![ + MountOption::FSName("protondrive".to_string()), + MountOption::Subtype("protondrive".to_string()), + MountOption::DefaultPermissions, + ]; + let fs = ProtonFs::new(self.fork_state(), root); + Session::new(fs, local, &config) + .and_then(|s| s.spawn()) + .map_err(|e| format!("mount {}: {e}", local.display())) + } + + /// Re-establish FUSE mounts for folders left in `ondemand` mode across a daemon + /// restart (their local dirs are empty on disk — the files live in the cloud). + /// Best-effort per folder: a missing local path or a failed remote fetch marks + /// the folder `error` and moves on rather than aborting the rest. Runs on its + /// own thread from `mount` so the network fetches never block startup + /// (devices.md Phase 4). + pub(crate) fn restore_ondemand_mounts(&self) { + let folders = match self.db.sync_folder_list() { + Ok(f) => f, + Err(e) => { + warn!(error = ?e, "restore on-demand: cannot list folders"); + return; + } + }; + for folder in folders { + if folder.mode != "ondemand" { + continue; + } + let local = PathBuf::from(&folder.local_path); + if !local.is_dir() { + warn!(id = folder.id, path = %local.display(), "restore on-demand: local path missing"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + // An `ondemand` folder's local dir is empty by construction — the switch + // evicts it. Finding files there means the row is lying (a switch that + // died between persisting the mode and evicting), and mounting over them + // would hide real local data behind the remote tree. Leave the files + // alone and let the user resolve it. + match dir_is_empty(&local) { + Ok(true) => {} + Ok(false) => { + warn!( + id = folder.id, + path = %local.display(), + "restore on-demand: local dir is not empty; refusing to mount over it" + ); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + Err(e) => { + warn!(id = folder.id, path = %local.display(), error = %e, "restore on-demand: cannot read local dir"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + } + let Some(root_uid) = parse_uid(&folder.remote_uid) else { + warn!(id = folder.id, "restore on-demand: bad remote uid"); + continue; + }; + let root = match self + .rt + .block_on(self.client.enumerate_nodes(std::slice::from_ref(&root_uid))) + { + Ok(v) => match v.into_iter().next() { + Some(n) => n, + None => { + warn!(id = folder.id, "restore on-demand: remote folder gone"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + }, + Err(e) => { + warn!(id = folder.id, error = %e, "restore on-demand: fetch remote failed"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + }; + match self.spawn_ondemand_mount(&local, root) { + Ok(session) => { + self.mounts.lock().insert(folder.id, session); + let _ = self.db.sync_folder_set_state(folder.id, "idle", now_secs()); + info!(id = folder.id, path = %local.display(), "remounted on-demand folder"); + } + Err(e) => { + warn!(id = folder.id, error = %e, "restore on-demand: mount failed"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + } + } + } + } + +} diff --git a/crates/pdfs-fuse/src/drain.rs b/crates/pdfs-fuse/src/drain.rs new file mode 100644 index 0000000..579b3c7 --- /dev/null +++ b/crates/pdfs-fuse/src/drain.rs @@ -0,0 +1,769 @@ +//! The write-back drain: turning queued mutations into remote calls. +//! +//! Every write the kernel hands us is answered the moment its bytes and its +//! `pending_op` row are on disk (offline.md Phase 3), which is what lets a `cp` +//! into the mount run at disk speed and lets an offline write succeed at all. +//! This module is the other half: the worker that walks that queue and performs +//! the uploads, creates, renames and trashes it recorded. +//! +//! Two invariants matter more than anything else here, because between them +//! they are the only thing standing between a queued write and lost data: +//! +//! 1. A staged blob is the *only* copy of the user`s bytes. It is dropped only +//! after the op it belongs to has provably landed — never before, and never +//! on a failure path that might be retried. +//! 2. A failure never pauses the queue. Recording it pushes that op`s +//! `next_attempt_at` past now, so one file wedged against a vanished parent +//! cannot hold up an unrelated upload behind it. +//! +//! When the remote has moved on underneath a queued revision, the losing bytes +//! are kept as a conflict copy rather than discarded — see +//! [`Core::revision_conflict`]. + +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; + +use pdfs_core::cache::StagedWrite; +use pdfs_core::control::{ActivityKind, TransferDirection}; +use pdfs_core::db::{OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingOp}; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use tracing::{debug, error, info, warn}; + +use super::state::Intervals; +use super::transfers::CountingReader; +use super::{ + Core, DRAIN_BACKOFF_MAX, DRAIN_BACKOFF_MIN, DRAIN_IDLE_POLL, ROOT_INO, conflict_name, + is_already_exists, is_gone, is_local_uid_str, media_type_for, node_size, now_millis, now_secs, + op_is_ready, parse_node_uid, +}; + +impl Core { + /// Drain the pending-op queue: the background half of every write + /// (offline.md Phase 3). + /// + /// Runs for the life of the mount. Ops are replayed oldest-first, each + /// retried with doubling backoff and *never* dropped on failure — the staged + /// blob is the only copy of the user's bytes, so a failed op stays queued + /// until it lands or the user deletes the file. + /// + /// A failure does not pause the queue. Recording it pushes that op's + /// `next_attempt_at` past `now`, so the next pass simply picks the next op + /// that is due — one file wedged against a folder that no longer exists must + /// not hold up an unrelated upload behind it. The worker only blocks once + /// nothing is due at all. + pub(crate) fn run_pending_drain(&self) { + loop { + let now = now_millis(); + let due = self + .db + .pending_ops() + .unwrap_or_default() + .into_iter() + .find(|op| op.next_attempt_at <= now && op_is_ready(op)); + + let Some(op) = due.filter(|_| self.online.load(Ordering::Relaxed)) else { + self.wait_for_drain_work(); + continue; + }; + if let Err(e) = self.drain_op(&op) { + let attempts = op.attempts + 1; + let backoff = DRAIN_BACKOFF_MIN + .saturating_mul(1u32 << attempts.min(6)) + .min(DRAIN_BACKOFF_MAX); + warn!(uid = %op.uid, attempts, error = %e, "pending upload failed; will retry"); + if let Err(e) = self.db.record_op_failure( + op.id, + &e.to_string(), + now_millis() + backoff.as_millis() as i64, + ) { + // The backoff is the only thing keeping a failing op from + // being picked again immediately, so without it the loop + // would spin on this op as fast as the API can refuse it. + error!(uid = %op.uid, error = %e, "recording a drain failure failed"); + self.wait_for_drain_work(); + } + } + } + } + + /// Block until there is plausibly something to do: a new op, a reconnect, or + /// the shortest outstanding backoff elapsing. + pub(crate) fn wait_for_drain_work(&self) { + let (lock, cv) = &*self.drain_wake; + let mut woken = lock.lock(); + if !*woken { + cv.wait_for(&mut woken, DRAIN_IDLE_POLL); + } + *woken = false; + } + + /// Perform one queued op and retire it. + pub(crate) fn drain_op(&self, op: &PendingOp) -> Result<(), Box> { + match op.kind.as_str() { + OP_REVISION => self.drain_revision(op), + OP_CREATE | OP_MKDIR => self.drain_local_node(op), + OP_RENAME => self.drain_rename(op), + OP_TRASH => self.drain_trash(op), + other => Err(format!("unknown pending op kind {other:?}").into()), + } + } + + /// Apply a queued rename/move to the remote. + /// + /// The op is the desired end state, so the remote's current state decides + /// what actually has to be called: either half may already match (the event + /// sync saw someone else do it, or an earlier attempt got half way through + /// before failing). That also makes the whole thing idempotent, which a + /// retrying queue needs. + pub(crate) fn drain_rename(&self, op: &PendingOp) -> Result<(), Box> { + let uid = parse_node_uid(&op.uid).ok_or("rename op has an unparseable uid")?; + let parent_str = op.parent_uid.as_deref().ok_or("rename op has no parent")?; + if is_local_uid_str(parent_str) { + return Err(format!("destination {parent_str} has not been created yet").into()); + } + let parent = parse_node_uid(parent_str).ok_or("rename op has an unparseable parent")?; + let name = op.name.clone().ok_or("rename op has no name")?; + + // The node we were asked to rename may be gone or trashed by now. Either + // way there is nothing to rename and nothing to lose — a rename holds no + // bytes — so the op is satisfied rather than retried forever. + let node = match self.fetch_node_remote(&uid)? { + Some(n) if !n.trashed => n, + _ => { + warn!(%uid, name, "renamed node is gone or trashed remotely; dropping the rename"); + self.db.delete_op(op.id)?; + return Ok(()); + } + }; + // The name half goes first, so that a collision on the move half below is + // about the name the node will actually land under rather than the one it + // is about to lose. + let mut landed = node.name.clone(); + if landed != name { + match self.rt.block_on(self.client.rename_node(&uid, &name, None)) { + Ok(()) => landed = name.clone(), + // Someone took the name while we were offline. Renaming to a + // *different* name is the non-destructive resolution: it neither + // clobbers their file nor drops ours, and it is visible. + Err(e) if is_already_exists(&e) => { + let alt = conflict_name(&name, now_secs()); + warn!(%uid, name, alt, "rename target name is taken; using a conflict name"); + self.rt + .block_on(self.client.rename_node(&uid, &alt, None))?; + landed = alt.clone(); + self.adopt_drained_name(&uid, &alt); + self.log_activity( + ActivityKind::Rename, + &name, + format!("name was taken remotely; renamed to {alt}"), + false, + ); + } + Err(e) => return Err(e.into()), + } + } + if node.parent_uid.as_ref() != Some(&parent) { + match self.rt.block_on(self.client.move_node(&uid, &parent)) { + Ok(()) => {} + // The destination holds a `landed` of its own. Same resolution as + // a name collision, and it has to happen before the move: the API + // has no move-and-rename, so the node is renamed out of the way + // here and moved second. + Err(e) if is_already_exists(&e) => { + let alt = conflict_name(&landed, now_secs()); + warn!(%uid, name = %landed, alt, "destination already holds that name; using a conflict name"); + self.rt + .block_on(self.client.rename_node(&uid, &alt, None))?; + self.rt.block_on(self.client.move_node(&uid, &parent))?; + self.adopt_drained_name(&uid, &alt); + self.log_activity( + ActivityKind::Rename, + &landed, + format!("destination already had that name; moved as {alt}"), + false, + ); + } + // The destination folder is gone. Leaving the node in its current + // parent is the honest outcome: it is not where the user asked for + // it, but it exists, it is where it has always been, and the + // rename half above still applied. Retrying could only fail again + // — the folder is not coming back — and would wedge the queue. + Err(e) if is_gone(&e) => { + warn!(%uid, name = %landed, %parent, "move destination is gone; leaving the node where it is"); + self.log_activity( + ActivityKind::Rename, + &landed, + "destination folder no longer exists; the file was left in place" + .to_string(), + false, + ); + } + Err(e) => return Err(e.into()), + } + } + self.db.delete_op(op.id)?; + info!(%uid, name = %landed, "pending rename landed"); + Ok(()) + } + + /// Apply a queued trash to the remote. + /// + /// A node that is already gone is a success, not a failure: the outcome the + /// op asked for holds either way, and retrying forever against a node the + /// server has forgotten would wedge the queue. + pub(crate) fn drain_trash(&self, op: &PendingOp) -> Result<(), Box> { + let uid = parse_node_uid(&op.uid).ok_or("trash op has an unparseable uid")?; + let name = op.name.clone().unwrap_or_else(|| op.uid.clone()); + match self + .rt + .block_on(self.client.trash_nodes(std::slice::from_ref(&uid))) + { + Ok(()) => {} + Err(e) if is_gone(&e) => { + debug!(%uid, name, "node was already gone remotely; trash op satisfied"); + } + Err(e) => return Err(e.into()), + } + self.db.delete_op(op.id)?; + self.invalidate_trash(); + self.log_activity(ActivityKind::Trash, &name, "trashed", true); + info!(%uid, name, "pending trash landed"); + Ok(()) + } + + /// Record the name the remote actually gave a node, after a conflict forced + /// it away from the one the user asked for. Best effort: the event sync + /// would correct the tree anyway, but not before the user has looked at it. + pub(crate) fn adopt_drained_name(&self, uid: &NodeUid, name: &str) { + let mut st = self.state.lock(); + let Some(&ino) = st.by_uid.get(uid) else { + return; + }; + let Some(entry) = st.entries.get_mut(&ino) else { + return; + }; + entry.node.name = name.to_string(); + let node = entry.node.clone(); + if let Err(e) = st.db.upsert_node(&node) { + warn!(%uid, error = %e, "db upsert_node failed after a conflict rename"); + } + } + + /// Make a node that so far exists only on this machine real, and adopt the + /// uid the server gives it (offline.md Phase 3b). + pub(crate) fn drain_local_node(&self, op: &PendingOp) -> Result<(), Box> { + let local = parse_node_uid(&op.uid).ok_or("pending op has an unparseable uid")?; + let parent_str = op.parent_uid.as_deref().ok_or("create op has no parent")?; + // `run_pending_drain` will not offer an op whose parent is still a + // placeholder, so reaching here with one is a bug, not a wait. + if is_local_uid_str(parent_str) { + return Err(format!("parent {parent_str} has not been created yet").into()); + } + let parent = parse_node_uid(parent_str).ok_or("create op has an unparseable parent")?; + let wanted = op.name.clone().ok_or("create op has no name")?; + + // Someone else may have taken the name while this sat in the queue — + // reachable only because the op waited, which is the whole point of the + // queue. Never overwrite theirs and never drop ours: land under a + // conflict name, exactly as the sync engine does. + let mut name = wanted.clone(); + let mut real = self.create_drained_node(op, &parent, &name); + if real.as_ref().is_err_and(|e| is_already_exists(e.as_ref())) { + name = conflict_name(&wanted, now_secs()); + warn!(%local, wanted, name, "name is taken remotely; creating under a conflict name"); + real = self.create_drained_node(op, &parent, &name); + if real.is_ok() { + self.log_activity( + ActivityKind::Upload, + &wanted, + format!("name was taken remotely; created as {name}"), + false, + ); + } + } + // The folder this node was created in may have been trashed remotely + // while the op waited, in which case nothing will ever make the op + // succeed as written and retrying it forever wedges the queue behind a + // file that has bytes to save. Re-home it to the root: not where the + // user put it, but it exists, it is visible, and the bytes are intact. + if let Some(root) = self.root_uid() + && real.is_err() + && self.parent_is_gone(&parent) + { + warn!(%local, name, "parent folder is gone remotely; creating in the root instead"); + real = self.create_drained_node(op, &root, &name); + if real.as_ref().is_err_and(|e| is_already_exists(e.as_ref())) { + name = conflict_name(&wanted, now_secs()); + real = self.create_drained_node(op, &root, &name); + } + if real.is_ok() { + self.log_activity( + ActivityKind::Upload, + &wanted, + format!("its folder was trashed remotely; created in the root as {name}"), + false, + ); + } + } + let real = real?; + + // Retire the op before touching anything else: if we crash here the node + // exists remotely and the local placeholder is reconciled by the event + // sync, whereas a surviving op would create the file a second time. + self.db.delete_op(op.id)?; + self.adopt_real_uid(&local, &real)?; + if let Some(blob) = op.blob_path.as_deref() { + self.cache.discard_staged(Path::new(blob)); + } + self.pending.lock().remove(&local); + self.log_activity(ActivityKind::Upload, &name, "created", true); + info!(%local, %real, name, kind = %op.kind, "pending create landed"); + Ok(()) + } + + /// Make one queued `create`/`mkdir` real under a given name, and hand back + /// the API's own error so the caller can tell a name clash from a failure. + /// + /// Split out because the conflict path has to run it twice: the second time + /// under a different name. + pub(crate) fn create_drained_node( + &self, + op: &PendingOp, + parent: &NodeUid, + name: &str, + ) -> Result> { + match op.kind == OP_MKDIR { + true => { + Ok(self + .rt + .block_on(self.client.create_folder(parent, name, Some(now_secs())))?) + } + false => self.upload_created_file(op, parent, name), + } + } + + /// Upload the bytes a queued create accumulated, if any. A file that was + /// created but never written (`touch`) has no blob and uploads as empty. + pub(crate) fn upload_created_file( + &self, + op: &PendingOp, + parent: &NodeUid, + name: &str, + ) -> Result> { + let Some(blob) = op.blob_path.as_deref() else { + return Ok(self.rt.block_on(self.client.upload_file( + parent, + name, + media_type_for(name), + b"", + ))?); + }; + let meta: StagedWrite = serde_json::from_str(op.meta_json.as_deref().unwrap_or(""))?; + // An incomplete blob would be authored bytes over zeros, and there is no + // base to repair it from — the file has never existed remotely. Refusing + // to queue that is `queue_revision`'s job, so reaching here means the blob + // is whole. + if !meta.complete { + return Err("queued create holds an incomplete blob".into()); + } + let guard = self + .transfers + .begin(name, op.uid.clone(), TransferDirection::Upload, meta.len); + let reader = CountingReader::new(File::open(blob)?, &guard); + let uid = self.rt.block_on(self.client.upload_file_from( + parent, + name, + media_type_for(name), + reader, + meta.len as i64, + Vec::new(), + None, + false, + ))?; + Ok(uid) + } + + /// Swap a placeholder uid for the real one across everything that keyed off + /// it: queued children, the DB, the in-memory tree, and the caches. + /// + /// The inode is deliberately kept, so anything already holding the file open + /// keeps working across the drain. + pub(crate) fn adopt_real_uid( + &self, + local: &NodeUid, + real: &NodeUid, + ) -> Result<(), Box> { + let node = self.fetch_node(real).map_err(|e| format!("{e:?}"))?; + // Repoints queued children and node rows, and drops the placeholder row. + self.db + .remap_local_uid(&local.to_string(), &real.to_string())?; + + let mut st = self.state.lock(); + if let Some(ino) = st.by_uid.remove(local) { + st.by_uid.insert(real.clone(), ino); + if let Some(e) = st.entries.get_mut(&ino) { + e.uid = real.clone(); + e.node = node.clone(); + } + // Where the op said the node goes and where it actually landed can + // differ — a conflict re-homes it — so the tree follows the parent + // the server reports rather than the one we asked for. + if let Some(parent) = node.parent_uid.clone() + && let Some(&pino) = st.by_uid.get(&parent) + && st.entries.get(&ino).is_some_and(|e| e.parent != pino) + { + let old = st.entries.get(&ino).map(|e| e.parent); + if let Some(old) = old + && let Some(kids) = st.children.get_mut(&old) + { + kids.retain(|&k| k != ino); + } + if let Some(e) = st.entries.get_mut(&ino) { + e.parent = pino; + } + if let Some(kids) = st.children.get_mut(&pino) + && !kids.contains(&ino) + { + kids.push(ino); + } + } + } + drop(st); + // Write the real node through, now that nothing points at the old uid. + if let Err(e) = self.db.upsert_node(&node) { + warn!(%real, error = %e, "db upsert_node failed after remap"); + } + if node.is_folder() { + // It was recorded as listed while local (it was empty and had nothing + // to enumerate). That still holds: its queued children re-intern under + // the real uid as they drain. + if let Err(e) = self.db.set_listed(real, true) { + warn!(%real, error = %e, "db set_listed(true) failed after remap"); + } + } + Ok(()) + } + + /// Why a queued write must not be applied to its node, or `None` when it + /// still can be. + /// + /// A queued write is an edit of a specific revision. Time passes before it + /// drains — indefinitely, if that is how long the network is gone — and in + /// that window the node can be rewritten by another device, trashed, or + /// deleted outright. Sending the blob anyway would silently drop whatever + /// happened in between, which is exactly the thing the sync engine refuses + /// to do (offline.md Phase 3b). + /// + /// Only checkable against a recorded baseline: a write staged before + /// [`StagedWrite::based_on`] existed, or one against a node that has never + /// existed remotely, has nothing to compare and is applied as before. + pub(crate) fn revision_conflict( + &self, + uid: &NodeUid, + meta: &StagedWrite, + ) -> Result, Box> { + let Some(base) = meta.based_on else { + return Ok(None); + }; + let Some(node) = self.fetch_node_remote(uid)? else { + return Ok(Some("the file no longer exists remotely".into())); + }; + if node.trashed { + return Ok(Some("the file was trashed remotely".into())); + } + let (mtime, size) = (node.modification_time, node_size(&node)); + if mtime != base.mtime || size != base.size { + return Ok(Some(format!( + "the remote revision changed under the queued write \ + (expected {} bytes at mtime {}, found {size} at {mtime})", + base.size, base.mtime + ))); + } + Ok(None) + } + + /// Land a queued write that can no longer be applied to its own node as a + /// *new* file beside it, and retire the op. + /// + /// The non-destructive resolution, and the same one the sync engine reaches + /// for: the remote keeps whatever it has, the user keeps their bytes, and + /// the name says which is which. + /// + /// An incomplete blob is gap-filled from whatever the node holds *now* — + /// mixing revisions, which is only defensible because the result is + /// explicitly a conflict copy rather than anyone's file. When even that is + /// impossible the op is dropped but the staged bytes are deliberately left + /// on disk: unreachable through the mount, but not destroyed, and the + /// activity log says where they are. + pub(crate) fn keep_as_conflict_copy( + &self, + op: &PendingOp, + blob: &Path, + meta: &StagedWrite, + uid: &NodeUid, + reason: &str, + ) -> Result<(), Box> { + let place = self.node_place(uid); + let name = place + .as_ref() + .map(|(_, n)| n.clone()) + .unwrap_or_else(|| meta.uid.clone()); + // A node the tree has forgotten still has bytes worth keeping, so the + // copy falls back to the root rather than being abandoned. + let Some(parent) = place.map(|(p, _)| p).or_else(|| self.root_uid()) else { + return self.abandon_to_staging(op, blob, uid, &name, reason); + }; + warn!(%uid, name, reason, "queued write conflicts; keeping a conflict copy"); + + if !meta.complete { + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(blob)?; + let mut written = Intervals::default(); + for &(s, e) in &meta.authored { + written.add(s, e); + } + if let Err(e) = self.fill_gaps( + uid, + &file, + meta.len, + meta.base_mtime, + meta.base_size, + &written, + ) { + error!(%uid, name, error = ?e, "cannot complete a conflicted partial write"); + return self.abandon_to_staging(op, blob, uid, &name, reason); + } + } + + let alt = conflict_name(&name, now_secs()); + let guard = + self.transfers + .begin(&alt, meta.uid.clone(), TransferDirection::Upload, meta.len); + let reader = CountingReader::new(File::open(blob)?, &guard); + self.rt.block_on(self.client.upload_file_from( + &parent, + &alt, + media_type_for(&alt), + reader, + meta.len as i64, + Vec::new(), + None, + false, + ))?; + drop(guard); + + self.db.delete_op(op.id)?; + // Dropping the pending entry hands the node back to the remote's truth: + // reads stop coming from the staged blob, and the event sync stops + // skipping it as "ahead of the server" (offline.md Phase 3a). + self.pending.lock().remove(uid); + self.cache.discard_staged(blob); + self.cache.evict(uid); + self.evict_reader(uid); + // The conflict copy is a node the tree has never seen. + let mut st = self.state.lock(); + if let Some(&ino) = st.by_uid.get(&parent) { + st.invalidate_listing(ino); + } + drop(st); + self.log_activity( + ActivityKind::Upload, + &name, + format!("{reason}; local changes uploaded as {alt}"), + false, + ); + info!(%uid, name, alt, "queued write landed as a conflict copy"); + Ok(()) + } + + /// Give up on placing a queued write anywhere the mount can see, without + /// destroying it: the op goes (it could only fail forever) but the staged + /// blob deliberately stays on disk, and the activity log says where. + /// + /// The last resort of the conflict path, and the same bargain + /// [`Core::stage_orphaned_write`] strikes at the other end: bytes we cannot + /// place are still bytes we do not get to delete. + pub(crate) fn abandon_to_staging( + &self, + op: &PendingOp, + blob: &Path, + uid: &NodeUid, + name: &str, + reason: &str, + ) -> Result<(), Box> { + error!(%uid, name, reason, staged = %blob.display(), + "cannot place a conflicted write; bytes kept in staging"); + self.db.delete_op(op.id)?; + self.pending.lock().remove(uid); + self.cache.evict(uid); + self.evict_reader(uid); + self.log_activity( + ActivityKind::Upload, + name, + format!("{reason}; local changes kept at {}", blob.display()), + false, + ); + Ok(()) + } + + /// Whether a folder a queued op targets has stopped being a place a node can + /// go. A failure to ask is not an answer: only a definite "trashed" or "not + /// there" counts, so a network fault leaves the op to retry normally. + pub(crate) fn parent_is_gone(&self, parent: &NodeUid) -> bool { + match self.fetch_node_remote(parent) { + Ok(Some(node)) => node.trashed, + Ok(None) => true, + Err(_) => false, + } + } + + /// A node's parent uid and name, as the tree currently has them. + pub(crate) fn node_place(&self, uid: &NodeUid) -> Option<(NodeUid, String)> { + let st = self.state.lock(); + let ino = st.by_uid.get(uid)?; + let entry = st.entries.get(ino)?; + let parent = entry.node.parent_uid.clone()?; + Some((parent, entry.node.name.clone())) + } + + /// The uid of the My Files root, which every node in the mount descends + /// from — the last resort for placing a file whose own parent is unknown. + /// + /// `None` only for a [`Core::fork_state`] sibling that has not interned its + /// root yet, which is not where the drain runs; the drain must not panic + /// over it regardless, since that would stop the queue for good. + pub(crate) fn root_uid(&self) -> Option { + let st = self.state.lock(); + st.entries.get(&ROOT_INO).map(|e| e.uid.clone()) + } + + /// Upload a staged revision of a file the server already knows about. + pub(crate) fn drain_revision(&self, op: &PendingOp) -> Result<(), Box> { + let blob = op + .blob_path + .clone() + .ok_or("pending op has no staged blob")?; + let blob = PathBuf::from(blob); + let meta: StagedWrite = serde_json::from_str(op.meta_json.as_deref().unwrap_or(""))?; + let uid = parse_node_uid(&meta.uid).ok_or("staged write has an unparseable uid")?; + + if let Some(reason) = self.revision_conflict(&uid, &meta)? { + return self.keep_as_conflict_copy(op, &blob, &meta, &uid, &reason); + } + + // An incomplete blob is authored bytes over zeros; the untouched ranges + // have to be filled from the base before it can be sent. This is the case + // the write could not resolve at release time (it was offline), and the + // reason it is safe to do now is that we are not. + if !meta.complete { + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&blob)?; + let mut written = Intervals::default(); + for &(s, e) in &meta.authored { + written.add(s, e); + } + self.fill_gaps( + &uid, + &file, + meta.len, + meta.base_mtime, + meta.base_size, + &written, + ) + .map_err(|e| format!("gap-fill from base failed: {e:?}"))?; + } + + let name = { + let st = self.state.lock(); + st.by_uid + .get(&uid) + .and_then(|ino| st.entries.get(ino)) + .map(|e| e.node.name.clone()) + .unwrap_or_else(|| meta.uid.clone()) + }; + let guard = + self.transfers + .begin(&name, meta.uid.clone(), TransferDirection::Upload, meta.len); + let reader = CountingReader::new(File::open(&blob)?, &guard); + self.rt.block_on(self.client.upload_new_revision_from( + &uid, + reader, + meta.len as i64, + Vec::new(), + None, + ))?; + drop(guard); + + // Retire the op before dropping the blob: a crash between the two leaves + // an orphaned file (harmless), whereas the reverse would leave a queued + // op pointing at nothing. + self.db.delete_op(op.id)?; + self.pending.lock().remove(&uid); + + // The staged blob now matches the sealed revision, so a pinned file keeps + // it as its cached content rather than re-downloading what we just sent. + if self.cache.is_pinned(&uid) + && let Ok(bytes) = std::fs::read(&blob) + { + let _ = self.cache.store(&uid, now_secs(), meta.len, &bytes); + } + self.cache.discard_staged(&blob); + self.evict_reader(&uid); + self.refresh_after_upload(&uid); + self.log_activity(ActivityKind::Upload, &name, "uploaded", true); + info!(%uid, len = meta.len, "pending upload landed"); + Ok(()) + } + + /// Adopt the server's metadata for a node we have just uploaded a revision + /// for. + /// + /// [`State::record_pending_write`] deliberately stamps the node with the + /// moment we *accepted* the write, so `ls` reflects it before the upload + /// lands. The server stamps the sealed revision with its own time, and the + /// two differ by however long the upload took. That difference is not + /// cosmetic: [`Core::remote_baseline`] reads this node to build + /// [`StagedWrite::based_on`], and [`Core::revision_conflict`] compares that + /// baseline against the remote — so leaving our optimistic time in place + /// makes the *next* write to this file look like another device changed it + /// underneath us, and diverts it into a conflict copy over nothing. + /// + /// Best effort: a failure here costs a spurious conflict copy on the next + /// write, not this upload, which has already landed. + pub(crate) fn refresh_after_upload(&self, uid: &NodeUid) { + // A write queued while this one was on the wire is ahead of the remote + // again, and its optimistic size/mtime must win — the same reason + // `apply_event` refuses to overwrite a node with a queued write. + if self.pending.lock().contains_key(uid) { + return; + } + let node = match self.fetch_node_remote(uid) { + Ok(Some(node)) => node, + // Trashed or deleted under us: the tree will hear it from the event + // sync, which is better placed to unhook the inode than we are. + Ok(None) => return, + Err(e) => { + warn!(%uid, error = %e, + "refreshing metadata after an upload failed; \ + the next write to this file may conflict with itself"); + return; + } + }; + let mut st = self.state.lock(); + let Some(parent) = st + .by_uid + .get(uid) + .and_then(|ino| st.entries.get(ino)) + .map(|e| e.parent) + else { + return; + }; + st.intern(parent, node); + } + +} diff --git a/crates/pdfs-fuse/src/lib.rs b/crates/pdfs-fuse/src/lib.rs index c405274..03c92a8 100644 --- a/crates/pdfs-fuse/src/lib.rs +++ b/crates/pdfs-fuse/src/lib.rs @@ -34,9 +34,8 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::ffi::OsStr; use std::fs::File; -use std::io::{BufRead, BufReader, Write as _}; use std::os::unix::fs::FileExt; -use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::net::UnixListener; use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use parking_lot::{Condvar, Mutex}; @@ -53,29 +52,35 @@ use fuser::{ use pdfs_core::cache::{BLOCK_SIZE, Baseline, ContentCache, StagedWrite}; use pdfs_core::config::AppDirs; use pdfs_core::control::{ - ActivityEntry, ActivityKind, BookmarkInfo, DeviceInfo, DirEntry, InvitationInfo, JobItem, - LocalHit, PhotoItem, PhotoKind, PhotoMonth, PhotoThumb, PublicLinkInfo, RefreshScope, - Request as CtlRequest, - Response as CtlResponse, SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, + ActivityEntry, ActivityKind, DirEntry, + LocalHit, PhotoKind, PublicLinkInfo, SearchHit, SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, }; use pdfs_core::db::{ - self, Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingOp, - StoredDevice, StoredNode, StoredPhoto, StoredSyncFolder, StoredTrash, + Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingOp, StoredNode, StoredSyncFolder, StoredTrash, }; use pdfs_core::localindex; use proton_drive_rs::proton_sdk::api::ResponseCode; use proton_drive_rs::proton_sdk::error::ProtonError; -use proton_drive_rs::proton_sdk::ids::{DeviceUid, DriveEventId, LinkId, NodeUid, VolumeId}; +use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, VolumeId}; use proton_drive_rs::{ - DeviceType, DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, ProtonDriveClient, + DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, ProtonDriveClient, ProtonPhotosClient, RevisionReader, ThumbnailType, }; +mod control; +use control::run_control_socket; +mod devices; +mod drain; +mod photos; +mod sharing; +mod state; mod sync; mod transfers; -use sync::base_name; +mod workers; +use state::{Entry, Intervals, PendingRevision, State, WriteHandle}; use tracing::{debug, error, info, warn}; +use workers::{FUSE_WORKERS, Workers}; use transfers::{CountingReader, CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry}; /// Attribute/entry cache lifetime handed back to the kernel. Long because the @@ -178,290 +183,6 @@ const XATTR_THUMBNAIL: &str = "user.proton.thumbnail"; /// Extended attribute exposing the larger server-side preview image of a file. const XATTR_PREVIEW: &str = "user.proton.preview"; -/// A node known to the filesystem, addressed by its kernel inode. -struct Entry { - uid: NodeUid, - parent: u64, - node: Node, -} - -/// A set of non-overlapping `[start, end)` byte ranges, kept sorted and merged. -/// Tracks which bytes of a [`WriteHandle`]'s scratch file were authored locally -/// (vs. still living only in the remote base), so reads and the commit gap-fill -/// know which regions to pull from the network. -#[derive(Clone, Default)] -struct Intervals(Vec<(u64, u64)>); - -impl Intervals { - /// Mark `[start, end)` as authored, coalescing with any touching ranges. - fn add(&mut self, start: u64, end: u64) { - if start >= end { - return; - } - self.0.push((start, end)); - self.0.sort_by_key(|&(s, _)| s); - let mut merged: Vec<(u64, u64)> = Vec::with_capacity(self.0.len()); - for &(s, e) in &self.0 { - match merged.last_mut() { - Some(last) if s <= last.1 => last.1 = last.1.max(e), - _ => merged.push((s, e)), - } - } - self.0 = merged; - } - - /// Drop everything at or beyond `len` (a shrink/truncate). - fn clip(&mut self, len: u64) { - self.0.retain(|&(s, _)| s < len); - for iv in &mut self.0 { - iv.1 = iv.1.min(len); - } - } - - /// Split `[start, end)` into contiguous `(s, e, authored)` segments, in - /// order. `authored == true` means the bytes live in the scratch file; - /// `false` means they must come from the remote base (or are a hole). - fn segments(&self, start: u64, end: u64) -> Vec<(u64, u64, bool)> { - let mut out = Vec::new(); - let mut pos = start; - for &(s, e) in &self.0 { - if e <= start { - continue; - } - if s >= end { - break; - } - let ws = s.max(start); - let we = e.min(end); - if pos < ws { - out.push((pos, ws, false)); - } - out.push((ws, we, true)); - pos = we; - } - if pos < end { - out.push((pos, end, false)); - } - out - } -} - -/// State for a file opened for writing. Authored bytes are staged in an on-disk -/// scratch file (positional reads/writes) rather than RAM, so a multi-GiB write -/// never balloons the daemon. On flush/release the scratch file — gap-filled -/// from the remote base where untouched — is streamed up as one new revision, -/// since the SDK seals whole revisions rather than byte ranges. -struct WriteHandle { - ino: u64, - uid: NodeUid, - /// Disk-backed staging buffer. Shared (`Arc`) so reads can use it without - /// holding the state lock across I/O. Accessed positionally (`read_at`/ - /// `write_at`), so a clone never disturbs another's file offset. - file: Arc, - /// Scratch file path, removed on release. - path: PathBuf, - /// Byte ranges authored into `file`. Everything else in `[0, len)` is base. - written: Intervals, - /// Logical file size (may exceed authored bytes after a truncate-extend). - len: u64, - /// Size of the remote base at open, for serving untouched ranges. - base_size: u64, - /// Modification time of the remote base at open, validating its block cache. - base_mtime: i64, - /// Whether anything diverged from the remote and needs an upload. - dirty: bool, -} - -/// A released write whose upload has not happened yet (offline.md Phase 3). -/// -/// The bytes live in the content cache's staging dir and the intent lives in the -/// `pending_op` table; this pairs them in memory so a read can be served without -/// a database round trip. -#[derive(Clone)] -struct PendingRevision { - /// Staged blob holding the written bytes. - path: PathBuf, - /// Which of those bytes are real, and what base the gaps refer to. - meta: StagedWrite, -} - -/// Mutable inode bookkeeping, guarded by a mutex because fuser drives the -/// `Filesystem` trait through `&self`. -struct State { - /// inode -> node metadata. - entries: HashMap, - /// Dedupe inodes by node uid so a node keeps a stable inode across lookups. - by_uid: HashMap, - /// Cached directory listings: parent inode -> child inodes. Presence of a - /// key means the directory has been enumerated. - children: HashMap>, - next_ino: u64, - /// Open write handles keyed by file handle id. Read-only opens use fh 0 and - /// have no entry here. - handles: HashMap, - next_fh: u64, - /// Unified SQLite metadata cache. Every map mutation below writes through to - /// it inside the `State` lock so the DB stays the authoritative copy across - /// restarts (see plan.md P1). - db: Arc, -} - -impl State { - fn intern(&mut self, parent: u64, node: Node) -> u64 { - if let Err(e) = self.db.upsert_node(&node) { - warn!(uid = %node.uid, error = %e, "db upsert_node failed"); - } - self.intern_mem(parent, node) - } - - /// Allocate (or reuse) a stable inode for a node, updating the hot-cache maps - /// only. Every caller owes the DB a write-through — see the callers below; - /// the split exists so a batch can pay for one transaction instead of `n`. - fn intern_mem(&mut self, parent: u64, node: Node) -> u64 { - if let Some(&ino) = self.by_uid.get(&node.uid) { - if let Some(e) = self.entries.get_mut(&ino) { - e.node = node; - e.parent = parent; - } - return ino; - } - let ino = self.next_ino; - self.next_ino += 1; - self.by_uid.insert(node.uid.clone(), ino); - self.entries.insert( - ino, - Entry { - uid: node.uid.clone(), - parent, - node, - }, - ); - ino - } - - /// Allocate (or reuse) a stable inode for a node that came *from* the - /// database, which is why nothing is written back. - fn intern_from_db(&mut self, parent: u64, node: Node) -> u64 { - self.intern_mem(parent, node) - } - - /// Allocate (or reuse) stable inodes for a whole listing, writing every node - /// through in a single transaction. - /// - /// This is what keeps `ls` on a large folder quick: one commit for the - /// listing, rather than one autocommit — and one fsync — per child. - fn intern_batch(&mut self, parent: u64, nodes: Vec) -> Vec { - if let Err(e) = self.db.upsert_nodes(&nodes) { - warn!(error = %e, "db upsert_nodes failed"); - } - nodes - .into_iter() - .map(|node| self.intern_mem(parent, node)) - .collect() - } - - /// Forget a node entirely: drop its inode, its uid mapping, its own cached - /// listing, its slot in its parent's listing, and its DB row. Returns - /// `(parent_ino, name)` when the node was known, so the caller can notify - /// the kernel. - fn forget(&mut self, uid: &NodeUid) -> Option<(u64, String)> { - let ino = self.by_uid.remove(uid)?; - if let Err(e) = self.db.delete_node(uid) { - warn!(%uid, error = %e, "db delete_node failed"); - } - let entry = self.entries.remove(&ino)?; - self.children.remove(&ino); - if let Some(kids) = self.children.get_mut(&entry.parent) { - kids.retain(|&k| k != ino); - } - Some((entry.parent, entry.node.name)) - } - - /// Move a node to a new parent and/or name within the tree, writing it - /// through like any other mutation. - /// - /// The online rename instead forgets the node and lets the destination - /// re-enumerate, which is the cheaper way to stay honest about what the - /// server did. A queued rename cannot: re-enumerating needs the network, and - /// the server has not been told yet in any case — so this *is* the tree's - /// new truth until the op drains (offline.md Phase 3b). - fn rename_in_place(&mut self, ino: u64, new_parent: u64, new_parent_uid: &NodeUid, name: &str) { - let Some(entry) = self.entries.get_mut(&ino) else { - return; - }; - let old_parent = entry.parent; - entry.parent = new_parent; - entry.node.name = name.to_string(); - entry.node.parent_uid = Some(new_parent_uid.clone()); - let node = entry.node.clone(); - if old_parent != new_parent { - if let Some(kids) = self.children.get_mut(&old_parent) { - kids.retain(|&k| k != ino); - } - // Only if the destination is listed: pushing into a listing that was - // never enumerated would invent a one-child folder. - if let Some(kids) = self.children.get_mut(&new_parent) - && !kids.contains(&ino) - { - kids.push(ino); - } - } - if let Err(e) = self.db.upsert_node(&node) { - warn!(uid = %node.uid, error = %e, "db upsert_node failed for a queued rename"); - } - } - - /// Drop a directory's cached child listing and mark it unlisted in the DB, - /// so the next access re-enumerates instead of trusting a stale listing. - fn invalidate_listing(&mut self, ino: u64) { - if self.children.remove(&ino).is_none() { - return; - } - if let Some(e) = self.entries.get(&ino) { - let uid = e.uid.clone(); - if let Err(err) = self.db.set_listed(&uid, false) { - warn!(%uid, error = %err, "db set_listed(false) failed"); - } - } - } - - /// Update a file entry's recorded plaintext size so `getattr` reflects an - /// in-progress write before the new revision is sealed. - fn set_size(&mut self, ino: u64, size: u64) { - if let Some(e) = self.entries.get_mut(&ino) - && let NodeKind::File { claimed_size, .. } = &mut e.node.kind - { - *claimed_size = Some(size as i64); - } - } - - /// Update a file entry's modification time (epoch seconds). - fn touch_mtime(&mut self, ino: u64, secs: i64) { - if let Some(e) = self.entries.get_mut(&ino) { - e.node.modification_time = secs; - } - } - - /// Record the size and mtime of a write that has been accepted but not yet - /// uploaded, persisting them like any other node mutation. - /// - /// The write-through is what makes the new size outlive the process: until - /// the op drains, the remote still holds the old revision (or, for a node - /// created offline, nothing at all), so this row is the only record that the - /// file is as long as the caller was told it is. Without it a restart serves - /// the stale size and the file reads as truncated — or empty — while its - /// bytes sit safely in staging (offline.md Phase 3). - fn record_pending_write(&mut self, ino: u64, size: u64, mtime: i64) { - self.set_size(ino, size); - self.touch_mtime(ino, mtime); - if let Some(e) = self.entries.get(&ino) - && let Err(err) = self.db.upsert_node(&e.node) - { - warn!(uid = %e.uid, error = %err, "db upsert_node failed for a queued write"); - } - } -} - /// How many files the bulk uploader ships at once. Overlaps the per-file network /// round-trips without letting an unbounded number of block buffers pile up. const UPLOAD_CONCURRENCY: usize = 4; @@ -562,73 +283,6 @@ async fn run_uploads( /// (two API calls and a node-key unlock) the next time that file is read. const MAX_OPEN_READERS: usize = 64; -/// How many FUSE handlers may run off the dispatch loop at once. -/// -/// `fuser`'s session loop is non-concurrent: it reads one request, calls the -/// handler, and only then reads the next. A handler that touches the network -/// therefore stalls every `getattr`/`lookup` on the mount behind it. The slow -/// handlers hand their `Reply` to this pool instead and answer from a worker, -/// which frees the loop immediately. -/// -/// Bounded on purpose: one worker can hold a 4 MiB block in flight, and an -/// unbounded pool would let read-ahead on a big file spawn threads without -/// limit. Sized just above the SDK's in-flight block cap so the semaphore -/// there, not thread count, is what bounds download memory. -const FUSE_WORKERS: usize = 8; - -/// Bounded thread pool behind the network-touching FUSE handlers. -/// -/// Shared by every session forked off one [`Core`] (the main mount plus each -/// on-demand sync folder), so the bound is per daemon rather than per mount. -struct Workers { - tx: std::sync::mpsc::Sender>, -} - -impl Workers { - fn new(n: usize) -> std::io::Result { - let (tx, rx) = std::sync::mpsc::channel::>(); - let rx = Arc::new(Mutex::new(rx)); - for i in 0..n { - let rx = rx.clone(); - std::thread::Builder::new() - .name(format!("pdfs-fuse-{i}")) - .spawn(move || { - loop { - // Held only to take the next job, never across it. - let job = rx.lock().recv(); - match job { - // A panicking handler must not cost the pool a - // worker for the rest of the run. The dropped - // `Reply` answers EIO on its own, so the caller of - // the failed op is told; the next job is unaffected. - // - // This only holds because the shared state is behind - // `parking_lot` mutexes: a `std` mutex held at the - // point of the panic would come back poisoned, and - // the worker we just rescued would die on its next - // acquisition — as would every other thread. - Ok(job) => { - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job)); - } - // Every sender is gone: the daemon is shutting down. - Err(_) => break, - } - } - })?; - } - Ok(Self { tx }) - } - - /// Run `job` on a worker. Falls back to running it inline — the pre-pool - /// behaviour — if no worker is left to take it, so a shut-down pool degrades - /// to a slow mount rather than a mount that answers every read with EIO. - fn run(&self, job: impl FnOnce() + Send + 'static) { - if let Err(e) = self.tx.send(Box::new(job)) { - warn!("fuse worker pool is gone; serving inline"); - (e.0)(); - } - } -} /// An open reader plus the node metadata it was opened against. /// @@ -2113,733 +1767,6 @@ impl Core { } } - /// Drain the pending-op queue: the background half of every write - /// (offline.md Phase 3). - /// - /// Runs for the life of the mount. Ops are replayed oldest-first, each - /// retried with doubling backoff and *never* dropped on failure — the staged - /// blob is the only copy of the user's bytes, so a failed op stays queued - /// until it lands or the user deletes the file. - /// - /// A failure does not pause the queue. Recording it pushes that op's - /// `next_attempt_at` past `now`, so the next pass simply picks the next op - /// that is due — one file wedged against a folder that no longer exists must - /// not hold up an unrelated upload behind it. The worker only blocks once - /// nothing is due at all. - fn run_pending_drain(&self) { - loop { - let now = now_millis(); - let due = self - .db - .pending_ops() - .unwrap_or_default() - .into_iter() - .find(|op| op.next_attempt_at <= now && op_is_ready(op)); - - let Some(op) = due.filter(|_| self.online.load(Ordering::Relaxed)) else { - self.wait_for_drain_work(); - continue; - }; - if let Err(e) = self.drain_op(&op) { - let attempts = op.attempts + 1; - let backoff = DRAIN_BACKOFF_MIN - .saturating_mul(1u32 << attempts.min(6)) - .min(DRAIN_BACKOFF_MAX); - warn!(uid = %op.uid, attempts, error = %e, "pending upload failed; will retry"); - if let Err(e) = self.db.record_op_failure( - op.id, - &e.to_string(), - now_millis() + backoff.as_millis() as i64, - ) { - // The backoff is the only thing keeping a failing op from - // being picked again immediately, so without it the loop - // would spin on this op as fast as the API can refuse it. - error!(uid = %op.uid, error = %e, "recording a drain failure failed"); - self.wait_for_drain_work(); - } - } - } - } - - /// Block until there is plausibly something to do: a new op, a reconnect, or - /// the shortest outstanding backoff elapsing. - fn wait_for_drain_work(&self) { - let (lock, cv) = &*self.drain_wake; - let mut woken = lock.lock(); - if !*woken { - cv.wait_for(&mut woken, DRAIN_IDLE_POLL); - } - *woken = false; - } - - /// Perform one queued op and retire it. - fn drain_op(&self, op: &PendingOp) -> Result<(), Box> { - match op.kind.as_str() { - OP_REVISION => self.drain_revision(op), - OP_CREATE | OP_MKDIR => self.drain_local_node(op), - OP_RENAME => self.drain_rename(op), - OP_TRASH => self.drain_trash(op), - other => Err(format!("unknown pending op kind {other:?}").into()), - } - } - - /// Apply a queued rename/move to the remote. - /// - /// The op is the desired end state, so the remote's current state decides - /// what actually has to be called: either half may already match (the event - /// sync saw someone else do it, or an earlier attempt got half way through - /// before failing). That also makes the whole thing idempotent, which a - /// retrying queue needs. - fn drain_rename(&self, op: &PendingOp) -> Result<(), Box> { - let uid = parse_node_uid(&op.uid).ok_or("rename op has an unparseable uid")?; - let parent_str = op.parent_uid.as_deref().ok_or("rename op has no parent")?; - if is_local_uid_str(parent_str) { - return Err(format!("destination {parent_str} has not been created yet").into()); - } - let parent = parse_node_uid(parent_str).ok_or("rename op has an unparseable parent")?; - let name = op.name.clone().ok_or("rename op has no name")?; - - // The node we were asked to rename may be gone or trashed by now. Either - // way there is nothing to rename and nothing to lose — a rename holds no - // bytes — so the op is satisfied rather than retried forever. - let node = match self.fetch_node_remote(&uid)? { - Some(n) if !n.trashed => n, - _ => { - warn!(%uid, name, "renamed node is gone or trashed remotely; dropping the rename"); - self.db.delete_op(op.id)?; - return Ok(()); - } - }; - // The name half goes first, so that a collision on the move half below is - // about the name the node will actually land under rather than the one it - // is about to lose. - let mut landed = node.name.clone(); - if landed != name { - match self.rt.block_on(self.client.rename_node(&uid, &name, None)) { - Ok(()) => landed = name.clone(), - // Someone took the name while we were offline. Renaming to a - // *different* name is the non-destructive resolution: it neither - // clobbers their file nor drops ours, and it is visible. - Err(e) if is_already_exists(&e) => { - let alt = conflict_name(&name, now_secs()); - warn!(%uid, name, alt, "rename target name is taken; using a conflict name"); - self.rt - .block_on(self.client.rename_node(&uid, &alt, None))?; - landed = alt.clone(); - self.adopt_drained_name(&uid, &alt); - self.log_activity( - ActivityKind::Rename, - &name, - format!("name was taken remotely; renamed to {alt}"), - false, - ); - } - Err(e) => return Err(e.into()), - } - } - if node.parent_uid.as_ref() != Some(&parent) { - match self.rt.block_on(self.client.move_node(&uid, &parent)) { - Ok(()) => {} - // The destination holds a `landed` of its own. Same resolution as - // a name collision, and it has to happen before the move: the API - // has no move-and-rename, so the node is renamed out of the way - // here and moved second. - Err(e) if is_already_exists(&e) => { - let alt = conflict_name(&landed, now_secs()); - warn!(%uid, name = %landed, alt, "destination already holds that name; using a conflict name"); - self.rt - .block_on(self.client.rename_node(&uid, &alt, None))?; - self.rt.block_on(self.client.move_node(&uid, &parent))?; - self.adopt_drained_name(&uid, &alt); - self.log_activity( - ActivityKind::Rename, - &landed, - format!("destination already had that name; moved as {alt}"), - false, - ); - } - // The destination folder is gone. Leaving the node in its current - // parent is the honest outcome: it is not where the user asked for - // it, but it exists, it is where it has always been, and the - // rename half above still applied. Retrying could only fail again - // — the folder is not coming back — and would wedge the queue. - Err(e) if is_gone(&e) => { - warn!(%uid, name = %landed, %parent, "move destination is gone; leaving the node where it is"); - self.log_activity( - ActivityKind::Rename, - &landed, - "destination folder no longer exists; the file was left in place" - .to_string(), - false, - ); - } - Err(e) => return Err(e.into()), - } - } - self.db.delete_op(op.id)?; - info!(%uid, name = %landed, "pending rename landed"); - Ok(()) - } - - /// Apply a queued trash to the remote. - /// - /// A node that is already gone is a success, not a failure: the outcome the - /// op asked for holds either way, and retrying forever against a node the - /// server has forgotten would wedge the queue. - fn drain_trash(&self, op: &PendingOp) -> Result<(), Box> { - let uid = parse_node_uid(&op.uid).ok_or("trash op has an unparseable uid")?; - let name = op.name.clone().unwrap_or_else(|| op.uid.clone()); - match self - .rt - .block_on(self.client.trash_nodes(std::slice::from_ref(&uid))) - { - Ok(()) => {} - Err(e) if is_gone(&e) => { - debug!(%uid, name, "node was already gone remotely; trash op satisfied"); - } - Err(e) => return Err(e.into()), - } - self.db.delete_op(op.id)?; - self.invalidate_trash(); - self.log_activity(ActivityKind::Trash, &name, "trashed", true); - info!(%uid, name, "pending trash landed"); - Ok(()) - } - - /// Record the name the remote actually gave a node, after a conflict forced - /// it away from the one the user asked for. Best effort: the event sync - /// would correct the tree anyway, but not before the user has looked at it. - fn adopt_drained_name(&self, uid: &NodeUid, name: &str) { - let mut st = self.state.lock(); - let Some(&ino) = st.by_uid.get(uid) else { - return; - }; - let Some(entry) = st.entries.get_mut(&ino) else { - return; - }; - entry.node.name = name.to_string(); - let node = entry.node.clone(); - if let Err(e) = st.db.upsert_node(&node) { - warn!(%uid, error = %e, "db upsert_node failed after a conflict rename"); - } - } - - /// Make a node that so far exists only on this machine real, and adopt the - /// uid the server gives it (offline.md Phase 3b). - fn drain_local_node(&self, op: &PendingOp) -> Result<(), Box> { - let local = parse_node_uid(&op.uid).ok_or("pending op has an unparseable uid")?; - let parent_str = op.parent_uid.as_deref().ok_or("create op has no parent")?; - // `run_pending_drain` will not offer an op whose parent is still a - // placeholder, so reaching here with one is a bug, not a wait. - if is_local_uid_str(parent_str) { - return Err(format!("parent {parent_str} has not been created yet").into()); - } - let parent = parse_node_uid(parent_str).ok_or("create op has an unparseable parent")?; - let wanted = op.name.clone().ok_or("create op has no name")?; - - // Someone else may have taken the name while this sat in the queue — - // reachable only because the op waited, which is the whole point of the - // queue. Never overwrite theirs and never drop ours: land under a - // conflict name, exactly as the sync engine does. - let mut name = wanted.clone(); - let mut real = self.create_drained_node(op, &parent, &name); - if real.as_ref().is_err_and(|e| is_already_exists(e.as_ref())) { - name = conflict_name(&wanted, now_secs()); - warn!(%local, wanted, name, "name is taken remotely; creating under a conflict name"); - real = self.create_drained_node(op, &parent, &name); - if real.is_ok() { - self.log_activity( - ActivityKind::Upload, - &wanted, - format!("name was taken remotely; created as {name}"), - false, - ); - } - } - // The folder this node was created in may have been trashed remotely - // while the op waited, in which case nothing will ever make the op - // succeed as written and retrying it forever wedges the queue behind a - // file that has bytes to save. Re-home it to the root: not where the - // user put it, but it exists, it is visible, and the bytes are intact. - if let Some(root) = self.root_uid() - && real.is_err() - && self.parent_is_gone(&parent) - { - warn!(%local, name, "parent folder is gone remotely; creating in the root instead"); - real = self.create_drained_node(op, &root, &name); - if real.as_ref().is_err_and(|e| is_already_exists(e.as_ref())) { - name = conflict_name(&wanted, now_secs()); - real = self.create_drained_node(op, &root, &name); - } - if real.is_ok() { - self.log_activity( - ActivityKind::Upload, - &wanted, - format!("its folder was trashed remotely; created in the root as {name}"), - false, - ); - } - } - let real = real?; - - // Retire the op before touching anything else: if we crash here the node - // exists remotely and the local placeholder is reconciled by the event - // sync, whereas a surviving op would create the file a second time. - self.db.delete_op(op.id)?; - self.adopt_real_uid(&local, &real)?; - if let Some(blob) = op.blob_path.as_deref() { - self.cache.discard_staged(Path::new(blob)); - } - self.pending.lock().remove(&local); - self.log_activity(ActivityKind::Upload, &name, "created", true); - info!(%local, %real, name, kind = %op.kind, "pending create landed"); - Ok(()) - } - - /// Make one queued `create`/`mkdir` real under a given name, and hand back - /// the API's own error so the caller can tell a name clash from a failure. - /// - /// Split out because the conflict path has to run it twice: the second time - /// under a different name. - fn create_drained_node( - &self, - op: &PendingOp, - parent: &NodeUid, - name: &str, - ) -> Result> { - match op.kind == OP_MKDIR { - true => { - Ok(self - .rt - .block_on(self.client.create_folder(parent, name, Some(now_secs())))?) - } - false => self.upload_created_file(op, parent, name), - } - } - - /// Upload the bytes a queued create accumulated, if any. A file that was - /// created but never written (`touch`) has no blob and uploads as empty. - fn upload_created_file( - &self, - op: &PendingOp, - parent: &NodeUid, - name: &str, - ) -> Result> { - let Some(blob) = op.blob_path.as_deref() else { - return Ok(self.rt.block_on(self.client.upload_file( - parent, - name, - media_type_for(name), - b"", - ))?); - }; - let meta: StagedWrite = serde_json::from_str(op.meta_json.as_deref().unwrap_or(""))?; - // An incomplete blob would be authored bytes over zeros, and there is no - // base to repair it from — the file has never existed remotely. Refusing - // to queue that is `queue_revision`'s job, so reaching here means the blob - // is whole. - if !meta.complete { - return Err("queued create holds an incomplete blob".into()); - } - let guard = self - .transfers - .begin(name, op.uid.clone(), TransferDirection::Upload, meta.len); - let reader = CountingReader::new(File::open(blob)?, &guard); - let uid = self.rt.block_on(self.client.upload_file_from( - parent, - name, - media_type_for(name), - reader, - meta.len as i64, - Vec::new(), - None, - false, - ))?; - Ok(uid) - } - - /// Swap a placeholder uid for the real one across everything that keyed off - /// it: queued children, the DB, the in-memory tree, and the caches. - /// - /// The inode is deliberately kept, so anything already holding the file open - /// keeps working across the drain. - fn adopt_real_uid( - &self, - local: &NodeUid, - real: &NodeUid, - ) -> Result<(), Box> { - let node = self.fetch_node(real).map_err(|e| format!("{e:?}"))?; - // Repoints queued children and node rows, and drops the placeholder row. - self.db - .remap_local_uid(&local.to_string(), &real.to_string())?; - - let mut st = self.state.lock(); - if let Some(ino) = st.by_uid.remove(local) { - st.by_uid.insert(real.clone(), ino); - if let Some(e) = st.entries.get_mut(&ino) { - e.uid = real.clone(); - e.node = node.clone(); - } - // Where the op said the node goes and where it actually landed can - // differ — a conflict re-homes it — so the tree follows the parent - // the server reports rather than the one we asked for. - if let Some(parent) = node.parent_uid.clone() - && let Some(&pino) = st.by_uid.get(&parent) - && st.entries.get(&ino).is_some_and(|e| e.parent != pino) - { - let old = st.entries.get(&ino).map(|e| e.parent); - if let Some(old) = old - && let Some(kids) = st.children.get_mut(&old) - { - kids.retain(|&k| k != ino); - } - if let Some(e) = st.entries.get_mut(&ino) { - e.parent = pino; - } - if let Some(kids) = st.children.get_mut(&pino) - && !kids.contains(&ino) - { - kids.push(ino); - } - } - } - drop(st); - // Write the real node through, now that nothing points at the old uid. - if let Err(e) = self.db.upsert_node(&node) { - warn!(%real, error = %e, "db upsert_node failed after remap"); - } - if node.is_folder() { - // It was recorded as listed while local (it was empty and had nothing - // to enumerate). That still holds: its queued children re-intern under - // the real uid as they drain. - if let Err(e) = self.db.set_listed(real, true) { - warn!(%real, error = %e, "db set_listed(true) failed after remap"); - } - } - Ok(()) - } - - /// Why a queued write must not be applied to its node, or `None` when it - /// still can be. - /// - /// A queued write is an edit of a specific revision. Time passes before it - /// drains — indefinitely, if that is how long the network is gone — and in - /// that window the node can be rewritten by another device, trashed, or - /// deleted outright. Sending the blob anyway would silently drop whatever - /// happened in between, which is exactly the thing the sync engine refuses - /// to do (offline.md Phase 3b). - /// - /// Only checkable against a recorded baseline: a write staged before - /// [`StagedWrite::based_on`] existed, or one against a node that has never - /// existed remotely, has nothing to compare and is applied as before. - fn revision_conflict( - &self, - uid: &NodeUid, - meta: &StagedWrite, - ) -> Result, Box> { - let Some(base) = meta.based_on else { - return Ok(None); - }; - let Some(node) = self.fetch_node_remote(uid)? else { - return Ok(Some("the file no longer exists remotely".into())); - }; - if node.trashed { - return Ok(Some("the file was trashed remotely".into())); - } - let (mtime, size) = (node.modification_time, node_size(&node)); - if mtime != base.mtime || size != base.size { - return Ok(Some(format!( - "the remote revision changed under the queued write \ - (expected {} bytes at mtime {}, found {size} at {mtime})", - base.size, base.mtime - ))); - } - Ok(None) - } - - /// Land a queued write that can no longer be applied to its own node as a - /// *new* file beside it, and retire the op. - /// - /// The non-destructive resolution, and the same one the sync engine reaches - /// for: the remote keeps whatever it has, the user keeps their bytes, and - /// the name says which is which. - /// - /// An incomplete blob is gap-filled from whatever the node holds *now* — - /// mixing revisions, which is only defensible because the result is - /// explicitly a conflict copy rather than anyone's file. When even that is - /// impossible the op is dropped but the staged bytes are deliberately left - /// on disk: unreachable through the mount, but not destroyed, and the - /// activity log says where they are. - fn keep_as_conflict_copy( - &self, - op: &PendingOp, - blob: &Path, - meta: &StagedWrite, - uid: &NodeUid, - reason: &str, - ) -> Result<(), Box> { - let place = self.node_place(uid); - let name = place - .as_ref() - .map(|(_, n)| n.clone()) - .unwrap_or_else(|| meta.uid.clone()); - // A node the tree has forgotten still has bytes worth keeping, so the - // copy falls back to the root rather than being abandoned. - let Some(parent) = place.map(|(p, _)| p).or_else(|| self.root_uid()) else { - return self.abandon_to_staging(op, blob, uid, &name, reason); - }; - warn!(%uid, name, reason, "queued write conflicts; keeping a conflict copy"); - - if !meta.complete { - let file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(blob)?; - let mut written = Intervals::default(); - for &(s, e) in &meta.authored { - written.add(s, e); - } - if let Err(e) = self.fill_gaps( - uid, - &file, - meta.len, - meta.base_mtime, - meta.base_size, - &written, - ) { - error!(%uid, name, error = ?e, "cannot complete a conflicted partial write"); - return self.abandon_to_staging(op, blob, uid, &name, reason); - } - } - - let alt = conflict_name(&name, now_secs()); - let guard = - self.transfers - .begin(&alt, meta.uid.clone(), TransferDirection::Upload, meta.len); - let reader = CountingReader::new(File::open(blob)?, &guard); - self.rt.block_on(self.client.upload_file_from( - &parent, - &alt, - media_type_for(&alt), - reader, - meta.len as i64, - Vec::new(), - None, - false, - ))?; - drop(guard); - - self.db.delete_op(op.id)?; - // Dropping the pending entry hands the node back to the remote's truth: - // reads stop coming from the staged blob, and the event sync stops - // skipping it as "ahead of the server" (offline.md Phase 3a). - self.pending.lock().remove(uid); - self.cache.discard_staged(blob); - self.cache.evict(uid); - self.evict_reader(uid); - // The conflict copy is a node the tree has never seen. - let mut st = self.state.lock(); - if let Some(&ino) = st.by_uid.get(&parent) { - st.invalidate_listing(ino); - } - drop(st); - self.log_activity( - ActivityKind::Upload, - &name, - format!("{reason}; local changes uploaded as {alt}"), - false, - ); - info!(%uid, name, alt, "queued write landed as a conflict copy"); - Ok(()) - } - - /// Give up on placing a queued write anywhere the mount can see, without - /// destroying it: the op goes (it could only fail forever) but the staged - /// blob deliberately stays on disk, and the activity log says where. - /// - /// The last resort of the conflict path, and the same bargain - /// [`Core::stage_orphaned_write`] strikes at the other end: bytes we cannot - /// place are still bytes we do not get to delete. - fn abandon_to_staging( - &self, - op: &PendingOp, - blob: &Path, - uid: &NodeUid, - name: &str, - reason: &str, - ) -> Result<(), Box> { - error!(%uid, name, reason, staged = %blob.display(), - "cannot place a conflicted write; bytes kept in staging"); - self.db.delete_op(op.id)?; - self.pending.lock().remove(uid); - self.cache.evict(uid); - self.evict_reader(uid); - self.log_activity( - ActivityKind::Upload, - name, - format!("{reason}; local changes kept at {}", blob.display()), - false, - ); - Ok(()) - } - - /// Whether a folder a queued op targets has stopped being a place a node can - /// go. A failure to ask is not an answer: only a definite "trashed" or "not - /// there" counts, so a network fault leaves the op to retry normally. - fn parent_is_gone(&self, parent: &NodeUid) -> bool { - match self.fetch_node_remote(parent) { - Ok(Some(node)) => node.trashed, - Ok(None) => true, - Err(_) => false, - } - } - - /// A node's parent uid and name, as the tree currently has them. - fn node_place(&self, uid: &NodeUid) -> Option<(NodeUid, String)> { - let st = self.state.lock(); - let ino = st.by_uid.get(uid)?; - let entry = st.entries.get(ino)?; - let parent = entry.node.parent_uid.clone()?; - Some((parent, entry.node.name.clone())) - } - - /// The uid of the My Files root, which every node in the mount descends - /// from — the last resort for placing a file whose own parent is unknown. - /// - /// `None` only for a [`Core::fork_state`] sibling that has not interned its - /// root yet, which is not where the drain runs; the drain must not panic - /// over it regardless, since that would stop the queue for good. - fn root_uid(&self) -> Option { - let st = self.state.lock(); - st.entries.get(&ROOT_INO).map(|e| e.uid.clone()) - } - - /// Upload a staged revision of a file the server already knows about. - fn drain_revision(&self, op: &PendingOp) -> Result<(), Box> { - let blob = op - .blob_path - .clone() - .ok_or("pending op has no staged blob")?; - let blob = PathBuf::from(blob); - let meta: StagedWrite = serde_json::from_str(op.meta_json.as_deref().unwrap_or(""))?; - let uid = parse_node_uid(&meta.uid).ok_or("staged write has an unparseable uid")?; - - if let Some(reason) = self.revision_conflict(&uid, &meta)? { - return self.keep_as_conflict_copy(op, &blob, &meta, &uid, &reason); - } - - // An incomplete blob is authored bytes over zeros; the untouched ranges - // have to be filled from the base before it can be sent. This is the case - // the write could not resolve at release time (it was offline), and the - // reason it is safe to do now is that we are not. - if !meta.complete { - let file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(&blob)?; - let mut written = Intervals::default(); - for &(s, e) in &meta.authored { - written.add(s, e); - } - self.fill_gaps( - &uid, - &file, - meta.len, - meta.base_mtime, - meta.base_size, - &written, - ) - .map_err(|e| format!("gap-fill from base failed: {e:?}"))?; - } - - let name = { - let st = self.state.lock(); - st.by_uid - .get(&uid) - .and_then(|ino| st.entries.get(ino)) - .map(|e| e.node.name.clone()) - .unwrap_or_else(|| meta.uid.clone()) - }; - let guard = - self.transfers - .begin(&name, meta.uid.clone(), TransferDirection::Upload, meta.len); - let reader = CountingReader::new(File::open(&blob)?, &guard); - self.rt.block_on(self.client.upload_new_revision_from( - &uid, - reader, - meta.len as i64, - Vec::new(), - None, - ))?; - drop(guard); - - // Retire the op before dropping the blob: a crash between the two leaves - // an orphaned file (harmless), whereas the reverse would leave a queued - // op pointing at nothing. - self.db.delete_op(op.id)?; - self.pending.lock().remove(&uid); - - // The staged blob now matches the sealed revision, so a pinned file keeps - // it as its cached content rather than re-downloading what we just sent. - if self.cache.is_pinned(&uid) - && let Ok(bytes) = std::fs::read(&blob) - { - let _ = self.cache.store(&uid, now_secs(), meta.len, &bytes); - } - self.cache.discard_staged(&blob); - self.evict_reader(&uid); - self.refresh_after_upload(&uid); - self.log_activity(ActivityKind::Upload, &name, "uploaded", true); - info!(%uid, len = meta.len, "pending upload landed"); - Ok(()) - } - - /// Adopt the server's metadata for a node we have just uploaded a revision - /// for. - /// - /// [`State::record_pending_write`] deliberately stamps the node with the - /// moment we *accepted* the write, so `ls` reflects it before the upload - /// lands. The server stamps the sealed revision with its own time, and the - /// two differ by however long the upload took. That difference is not - /// cosmetic: [`Core::remote_baseline`] reads this node to build - /// [`StagedWrite::based_on`], and [`Core::revision_conflict`] compares that - /// baseline against the remote — so leaving our optimistic time in place - /// makes the *next* write to this file look like another device changed it - /// underneath us, and diverts it into a conflict copy over nothing. - /// - /// Best effort: a failure here costs a spurious conflict copy on the next - /// write, not this upload, which has already landed. - fn refresh_after_upload(&self, uid: &NodeUid) { - // A write queued while this one was on the wire is ahead of the remote - // again, and its optimistic size/mtime must win — the same reason - // `apply_event` refuses to overwrite a node with a queued write. - if self.pending.lock().contains_key(uid) { - return; - } - let node = match self.fetch_node_remote(uid) { - Ok(Some(node)) => node, - // Trashed or deleted under us: the tree will hear it from the event - // sync, which is better placed to unhook the inode than we are. - Ok(None) => return, - Err(e) => { - warn!(%uid, error = %e, - "refreshing metadata after an upload failed; \ - the next write to this file may conflict with itself"); - return; - } - }; - let mut st = self.state.lock(); - let Some(parent) = st - .by_uid - .get(uid) - .and_then(|ino| st.entries.get(ino)) - .map(|e| e.parent) - else { - return; - }; - st.intern(parent, node); - } - /// Download a whole file's plaintext, registering the transfer so /// `GetQueueStatus` can report its progress. `total` is the expected size /// (`0` if unknown). Streams through [`download_file_to`] so each block ticks @@ -3135,253 +2062,6 @@ impl Core { /// Metadata only: a thumbnail path is attached for photos whose thumbnail is /// already cached, and nothing is downloaded here — the front-end pulls the /// thumbnails it actually paints via [`Core::photo_thumbs`]. - fn photos_timeline( - &self, - offset: usize, - limit: usize, - kind: Option, - range: Option<(i64, i64)>, - ) -> Result>, String> { - let count = self.db.photos_count().map_err(|e| e.to_string())?; - if count == 0 { - // Nothing to serve, so this one request has to wait for the fetch — - // unless we already know the account has no photos volume and the - // answer is a fresh "no". - let known_empty = self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() == Some(0); - if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) { - return Ok(None); - } - if !self.rt.block_on(self.refresh_timeline())? { - return Ok(None); - } - } else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) { - self.spawn_timeline_refresh(); - } - - let page = self - .db - .photos_page(offset, limit, kind, range) - .map_err(|e| e.to_string())?; - Ok(Some(page.into_iter().map(|p| self.photo_item(p)).collect())) - } - - /// Project a persisted photo into the wire item the front-end paints: its - /// learned aspect ratio, its thumbnail verdict, and the on-disk path of its - /// thumbnail when one is cached (tagged with the capture time, which is the - /// only revision handle a photo carries). - fn photo_item(&self, photo: StoredPhoto) -> PhotoItem { - let thumb_path = parse_uid(&photo.uid).and_then(|uid| { - self.cache - .cached_thumbnail_path(&uid, ThumbnailType::Thumbnail.as_i32(), photo.capture_time) - .map(|p| p.display().to_string()) - }); - PhotoItem { - uid: photo.uid, - capture_time: photo.capture_time, - thumb_path, - name: photo.name, - ratio: photo.ratio, - no_thumb: photo.thumb_state == db::THUMB_NONE, - kind: photo.kind, - } - } - - /// Thumbnails for `uids`, served from the cache, fetched from the server for - /// whatever is missing, and — for the photos the server has no thumbnail for - /// at all — *generated locally* from the full file (see - /// [`Core::generate_thumbs`]). Requested on demand as tiles scroll into view, - /// so a cold timeline paints immediately and only the photos actually on - /// screen cost a round-trip. - /// - /// A photo absent from the persisted timeline is skipped: its capture time is - /// the cache's validity tag, and guessing that would poison the cache. - fn photo_thumbs(&self, uids: &[NodeUid]) -> Vec { - let ttype = ThumbnailType::Thumbnail.as_i32(); - let keys: Vec = uids.iter().map(|u| u.to_string()).collect(); - let stored = self.db.photos_by_uid(&keys).unwrap_or_default(); - let tags: HashMap = stored - .iter() - .map(|p| (p.uid.clone(), p.capture_time)) - .collect(); - - // Ask the server only for photos that are missing a cached thumbnail and - // haven't already been written off as un-thumbnailable. - let want: Vec = uids - .iter() - .filter(|uid| { - let key = uid.to_string(); - stored - .iter() - .find(|p| p.uid == key) - .is_some_and(|p| p.thumb_state != db::THUMB_NONE) - && tags.get(&key).is_some_and(|&tag| { - self.cache.cached_thumbnail_path(uid, ttype, tag).is_none() - }) - }) - .cloned() - .collect(); - - if !want.is_empty() { - let mut missing = want.clone(); - match self.rt.block_on( - self.photos() - .enumerate_thumbnails(&want, ThumbnailType::Thumbnail), - ) { - Ok(thumbs) => { - for ft in thumbs { - let Some(&tag) = tags.get(&ft.file_uid.to_string()) else { - continue; - }; - let Ok(bytes) = ft.result else { continue }; - if bytes.is_empty() { - continue; - } - if self - .cache - .store_thumbnail(&ft.file_uid, ttype, tag, &bytes) - .is_ok() - { - missing.retain(|uid| uid != &ft.file_uid); - self.record_thumb(&ft.file_uid, db::THUMB_HAVE, ratio_of(&bytes)); - } - } - } - // A failed batch is not a verdict: leave every uid in `missing` so - // the local fallback still gives those tiles an image. - Err(e) => warn!(error = %e, "batch photo thumbnails failed"), - } - - // Whatever the server had nothing for gets a thumbnail made from its - // own bytes — this is what fills in camera photos uploaded by clients - // that never generated one. Off the request path: a full-size photo - // takes far longer to fetch than the whole rest of the batch, and the - // thumbnails that *are* ready must not wait behind it. - if !missing.is_empty() { - self.spawn_generate_thumbs(missing, &tags); - } - } - - let pending = self.thumb_gen.lock(); - uids.iter() - .map(|uid| PhotoThumb { - uid: uid.to_string(), - path: tags.get(&uid.to_string()).and_then(|&tag| { - self.cache - .cached_thumbnail_path(uid, ttype, tag) - .map(|p| p.display().to_string()) - }), - pending: pending.contains(uid), - }) - .collect() - } - - /// Generate the missing thumbnails on the runtime, skipping any photo already - /// being generated. The uids are marked in-flight before the task starts, so - /// the reply this call is about to send already reports them as pending. - fn spawn_generate_thumbs(&self, uids: Vec, tags: &HashMap) { - let fresh: Vec = { - let mut inflight = self.thumb_gen.lock(); - uids.into_iter() - .filter(|uid| inflight.insert(uid.clone())) - .collect() - }; - if fresh.is_empty() { - return; - } - - let core = self.clone(); - let tags = tags.clone(); - // `generate_thumbs` blocks on the runtime itself, so it belongs on the - // blocking pool rather than on an async worker. - self.rt.spawn_blocking(move || { - core.generate_thumbs(&fresh, &tags); - let mut inflight = core.thumb_gen.lock(); - for uid in &fresh { - inflight.remove(uid); - } - }); - } - - /// Make thumbnails for photos the server has none for: download each full - /// file once, scale it to [`THUMB_EDGE`], and store the result in the thumbnail - /// cache exactly as if the server had served it. - /// - /// Bounded by [`THUMB_GEN_CONCURRENCY`] — these are full-size originals, and a - /// screenful of them at once would saturate the link. A photo whose bytes - /// can't be decoded (a codec `image` doesn't speak) is marked - /// [`db::THUMB_NONE`] and never attempted again. - fn generate_thumbs(&self, uids: &[NodeUid], tags: &HashMap) { - info!( - count = uids.len(), - "generating thumbnails the server has none for" - ); - let results: Vec<(NodeUid, ThumbAttempt)> = self.rt.block_on(async { - let mut out = Vec::with_capacity(uids.len()); - for chunk in uids.chunks(THUMB_GEN_CONCURRENCY) { - let mut set = tokio::task::JoinSet::new(); - for uid in chunk { - let client = self.client.clone(); - let uid = uid.clone(); - set.spawn(async move { - let bytes = match client.download_file(&uid).await { - Ok(bytes) => bytes, - Err(e) => { - warn!(%uid, error = %e, "photo download for thumbnail failed"); - return (uid, ThumbAttempt::Unavailable); - } - }; - // Decoding + scaling a 20 MP JPEG is CPU-bound and would - // stall the runtime's worker; hand it to the blocking pool. - let made = tokio::task::spawn_blocking(move || scale_thumbnail(&bytes)) - .await - .unwrap_or(None); - match made { - Some(thumb) => (uid, ThumbAttempt::Made(thumb)), - None => (uid, ThumbAttempt::Undecodable), - } - }); - } - while let Some(joined) = set.join_next().await { - if let Ok(result) = joined { - out.push(result); - } - } - } - out - }); - - let ttype = ThumbnailType::Thumbnail.as_i32(); - for (uid, attempt) in results { - match attempt { - ThumbAttempt::Made(thumb) => { - let Some(&tag) = tags.get(&uid.to_string()) else { - continue; - }; - match self.cache.store_thumbnail(&uid, ttype, tag, &thumb.bytes) { - Ok(()) => self.record_thumb(&uid, db::THUMB_HAVE, Some(thumb.ratio)), - Err(e) => warn!(%uid, error = %e, "storing generated thumbnail failed"), - } - } - // The photo's own bytes aren't an image we can read: no thumbnail - // will ever exist for it, so stop trying. - ThumbAttempt::Undecodable => self.record_thumb(&uid, db::THUMB_NONE, None), - // The download failed — a dropped connection, an expired link. That - // is not a verdict on the photo: leave its state alone so the next - // scroll past it tries again. - ThumbAttempt::Unavailable => {} - } - } - } - - /// Persist what a thumbnail attempt learned about a photo. - fn record_thumb(&self, uid: &NodeUid, state: i64, ratio: Option) { - if let Err(e) = self.db.photo_set_thumb(&uid.to_string(), state, ratio) { - warn!(%uid, error = %e, "recording thumbnail state failed"); - } - } - - /// Whether the listing stamped under `key` is older than `ttl` (or was never - /// fetched). fn listing_stale(&self, key: &str, ttl: Duration) -> bool { match self.db.state_i64(key).ok().flatten() { Some(ms) => now_ms().saturating_sub(ms) >= ttl.as_millis() as i64, @@ -3391,97 +2071,6 @@ impl Core { /// Re-fetch the whole photos timeline and persist it. Returns whether the /// account has a photos volume at all. - async fn refresh_timeline(&self) -> Result { - let photos = self.photos(); - if photos - .get_photos_root() - .await - .map_err(|e| format!("photos root: {e}"))? - .is_none() - { - let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 0); - let _ = self.db.set_state_i64(PHOTOS_SYNCED_MS, now_ms()); - return Ok(false); - } - let items = photos - .enumerate_timeline() - .await - .map_err(|e| format!("timeline: {e}"))?; - - // The timeline DTO carries only a uid and capture time, but the Photos - // page has to split into Photos / Videos / Raw — which needs each photo's - // name and media type. Resolve those in batches off the request path. - // Best-effort: a photo whose node we fail to resolve keeps whatever was - // learned before (or classifies from nothing, i.e. a still photo), so a - // partial resolve never blanks the timeline. - let uids: Vec = items.iter().map(|it| it.uid.clone()).collect(); - let mut meta: HashMap, Option)> = HashMap::new(); - for chunk in uids.chunks(TIMELINE_ENRICH_CHUNK) { - match photos.enumerate_nodes(chunk).await { - Ok(nodes) => { - for node in nodes { - let media_type = match &node.kind { - NodeKind::File { media_type, .. } => Some(media_type.clone()), - NodeKind::Folder => None, - }; - meta.insert(node.uid.to_string(), (Some(node.name), media_type)); - } - } - Err(e) => warn!(error = %e, "resolving photo metadata for a timeline chunk failed"), - } - } - - let rows: Vec<(String, i64, Option, Option)> = items - .iter() - .map(|it| { - let key = it.uid.to_string(); - let (name, media_type) = meta.get(&key).cloned().unwrap_or((None, None)); - (key, it.capture_time, name, media_type) - }) - .collect(); - self.db.photos_replace(&rows).map_err(|e| e.to_string())?; - let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 1); - let _ = self.db.set_state_i64(PHOTOS_SYNCED_MS, now_ms()); - Ok(true) - } - - /// Refresh the timeline off the request path, so a stale page is still served - /// at DB speed. At most one refresh runs at a time. - fn spawn_timeline_refresh(&self) { - if self.timeline_refreshing.swap(true, Ordering::SeqCst) { - return; - } - let core = self.clone(); - self.rt.spawn(async move { - if let Err(e) = core.refresh_timeline().await { - warn!(error = %e, "background timeline refresh failed"); - } - core.timeline_refreshing.store(false, Ordering::SeqCst); - }); - } - - /// Download a photo's full content into the content cache, returning its - /// on-disk path (served from cache when a fresh blob already exists). - fn open_photo(&self, uid: &NodeUid) -> Result { - let photos = self.photos(); - let node = self - .rt - .block_on(photos.get_node(uid)) - .map_err(|e| format!("photo node: {e}"))? - .ok_or("photo not found")?; - let (mtime, size) = (node.modification_time, node_size(&node)); - if let Some(p) = self.cache.cached_content_path(uid, mtime, size) { - return Ok(p); - } - let bytes = self - .download_photo_tracked(&photos, uid, &node.name, size) - .map_err(|e| format!("download photo: {e}"))?; - self.cache - .store(uid, mtime, size, &bytes) - .map_err(|e| format!("cache store: {e}"))?; - Ok(self.cache.content_path(uid)) - } - /// Download the full content of the Drive file at mountpoint-relative `rel` /// into the content cache, returning its on-disk path (served from cache /// when a fresh blob already exists). Lets a front-end open the file with @@ -4060,1065 +2649,6 @@ impl Core { Ok(()) } - // ---- devices ---------------------------------------------------------- - - /// List the account's registered devices, flagging the one *this* machine - /// syncs to so a front-end can treat it as more than another computer in the - /// list — deleting it takes this machine's synced folders down with it. - fn list_devices(&self) -> Result, String> { - let devices = self - .rt - .block_on(self.client.enumerate_devices()) - .map_err(|e| format!("list devices: {e}"))?; - // No cached device row yet means this machine syncs nothing, so none of - // the listed devices is ours. - let this_uid = self.db.device_get().ok().flatten().map(|d| d.uid); - Ok(devices - .into_iter() - .map(|d| { - let uid = d.uid.to_string(); - DeviceInfo { - this_device: this_uid.as_deref() == Some(uid.as_str()), - uid, - name: d.name.unwrap_or_else(|_| "(unnamed device)".to_string()), - device_type: device_type_str(d.device_type).to_string(), - last_sync: d.last_sync_time, - } - }) - .collect()) - } - - /// Rename a device by its uid. - fn rename_device(&self, uid: &str, name: &str) -> Result<(), String> { - if name.is_empty() { - return Err("device name must not be empty".to_string()); - } - let device_uid = DeviceUid::from(uid); - self.rt - .block_on(self.client.rename_device(&device_uid, name)) - .map_err(|e| format!("rename device: {e}"))?; - Ok(()) - } - - /// Delete (deregister) a device by its uid. - fn delete_device(&self, uid: &str) -> Result<(), String> { - let device_uid = DeviceUid::from(uid); - self.rt - .block_on(self.client.delete_device(&device_uid)) - .map_err(|e| format!("delete device: {e}"))?; - Ok(()) - } - - // ---- device folder sync (devices.md, Phase 1) ------------------------- - - /// Auto-register (or recover) this machine as a Proton Drive Device, caching - /// it so restarts reuse the same device. Recovery matches an existing remote - /// Linux device by name before creating a new one, so a lost local record - /// doesn't orphan the device's root folder. - fn ensure_device(&self) -> Result { - let name = this_hostname(); - // Enumerate the remote devices once: used both to validate any cached - // record and to recover an existing device by name. - let remote = self - .rt - .block_on(self.client.enumerate_devices()) - .map_err(|e| format!("enumerate devices: {e}"))?; - - // A cached device is only trustworthy if it still exists remotely. A - // device deleted from another client (or the web UI) leaves a stale row - // whose root folder is gone, so creating folders under it fails with - // "parent node is not a folder". Re-register in that case. - if let Some(dev) = self.db.device_get().map_err(|e| format!("db: {e:?}"))? { - if remote.iter().any(|d| d.uid.to_string() == dev.uid) { - return Ok(dev); - } - warn!(uid = %dev.uid, "cached device is gone remotely; re-registering"); - } - - // Recover: an existing remote Linux device with the same name is ours. - let existing = remote.into_iter().find(|d| { - d.device_type == DeviceType::Linux && d.name.as_deref().ok() == Some(name.as_str()) - }); - let dev = match existing { - Some(d) => StoredDevice { - uid: d.uid.to_string(), - share_id: d.share_id.to_string(), - root_uid: d.root_folder_uid.to_string(), - name, - created: d.creation_time, - }, - None => { - let d = self - .rt - .block_on(self.client.create_device(&name, DeviceType::Linux)) - .map_err(|e| format!("create device: {e}"))?; - StoredDevice { - uid: d.uid.to_string(), - share_id: d.share_id.to_string(), - root_uid: d.root_folder_uid.to_string(), - name, - created: d.creation_time, - } - } - }; - self.db.device_set(&dev).map_err(|e| format!("db: {e:?}"))?; - Ok(dev) - } - - /// An untrashed folder named `name` directly under the device root, if one - /// already exists. - fn find_device_child_folder( - &self, - root_uid: &NodeUid, - name: &str, - ) -> Result, String> { - let uids = self - .rt - .block_on(self.client.enumerate_folder_children_node_uids(root_uid)) - .map_err(|e| format!("list device root: {e}"))?; - if uids.is_empty() { - return Ok(None); - } - let nodes = self - .rt - .block_on(self.client.enumerate_nodes(&uids)) - .map_err(|e| format!("resolve device root children: {e}"))?; - Ok(nodes - .into_iter() - .find(|n| n.is_folder() && !n.trashed && n.name == name) - .map(|n| n.uid)) - } - - /// Add a local folder to this device's sync set: register the device if - /// needed, create a matching folder under the device root, upload the local - /// tree into it once, and record the mapping. Phase 1 is a one-shot upload — - /// the two-way engine (Phase 2) reconciles later changes. - fn add_sync_folder(&self, local: &Path) -> Result { - let meta = - std::fs::metadata(local).map_err(|e| format!("stat {}: {e}", local.display()))?; - if !meta.is_dir() { - return Err(format!("{} is not a directory", local.display())); - } - let local = local - .canonicalize() - .map_err(|e| format!("canonicalize {}: {e}", local.display()))?; - let local_str = local.to_string_lossy().to_string(); - - // Reject duplicates up front for a clear error (UNIQUE would also catch it). - if self - .db - .sync_folder_list() - .map_err(|e| format!("db: {e:?}"))? - .iter() - .any(|f| f.local_path == local_str) - { - return Err(format!("{} is already synced", local.display())); - } - - let name = local - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| format!("unusable folder name: {}", local.display()))? - .to_string(); - - let device = self.ensure_device()?; - let root_uid = parse_uid(&device.root_uid) - .ok_or_else(|| format!("bad device root uid: {}", device.root_uid))?; - - // The synced folder's remote root: the folder under the device root named - // after the local basename. Reuse an existing one rather than creating a - // second folder with the same name — re-adding a folder (after a removal, or - // after a failed add that had already created it) must land back on the - // original, not leave the user with two "Downloads" in their Drive. The - // reconcile treats an existing remote tree correctly: unmatched paths read as - // a conflict, not as data loss. - let remote_root = match self.find_device_child_folder(&root_uid, &name)? { - Some(uid) => { - info!(name, "reusing existing device folder"); - uid - } - None => self - .rt - .block_on( - self.client - .create_folder(&root_uid, &name, Some(now_secs())), - ) - .map_err(|e| format!("create device folder {name}: {e}"))?, - }; - - let id = self - .db - .sync_folder_add(&local_str, &remote_root.to_string(), &device.share_id) - .map_err(|e| format!("db: {e:?}"))?; - - // Hand the initial upload to the sync engine: an empty baseline against a - // full local tree reconciles as "upload everything", and the folder is - // added to the filesystem watch set in the same pass. - let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); - let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); - - info!(local = %local.display(), id, "added sync folder"); - self.db - .sync_folder_get(id) - .map_err(|e| format!("db: {e:?}"))? - .ok_or_else(|| "sync folder vanished after insert".to_string()) - } - - /// List this device's synced folders for the front-ends, each carrying the - /// live progress of its pass when one is running. - fn list_sync_folders(&self) -> Result, String> { - let progress = self.sync_progress.lock(); - Ok(self - .db - .sync_folder_list() - .map_err(|e| format!("db: {e:?}"))? - .into_iter() - .map(|f| { - let live = progress.get(&f.id).cloned(); - sync_folder_info(f, live) - }) - .collect()) - } - - /// Everything the daemon is chewing on that isn't moving bytes, for - /// `GetQueueStatus`: the registered jobs (bulk-upload scans, the local index) - /// plus a synthesized job per running sync pass, so one Activity view answers - /// "is anything still happening?" without also polling `ListSyncFolders`. - /// - /// The sync passes are folded in here rather than tracked as registry jobs - /// because the Devices page needs them per folder anyway - /// ([`SyncFolderInfo::progress`]) — this keeps one source of truth and hits - /// the db only while a pass is actually running. - fn jobs_snapshot(&self) -> Vec { - let mut jobs = self.transfers.jobs_snapshot(); - let mut passes: Vec<(i64, SyncProgress)> = self - .sync_progress - .lock() - .iter() - .map(|(id, p)| (*id, p.clone())) - .collect(); - if passes.is_empty() { - return jobs; - } - passes.sort_by_key(|(id, _)| *id); - - let names: HashMap = self - .db - .sync_folder_list() - .unwrap_or_default() - .into_iter() - .map(|f| (f.id, f.local_path)) - .collect(); - for (id, p) in passes { - // The row is titled with the folder's own name; the full local path - // is what the Devices page shows, and is far too long for this line. - let folder = names - .get(&id) - .and_then(|path| Path::new(path).file_name()) - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_else(|| "folder".to_string()); - jobs.push(match p.phase { - // The scan's total is the last pass's baseline, so a folder that - // has never synced still reports indeterminate (`total: 0`) — but - // every later pass has a real bar. A grown folder can push `done` - // past the estimate; clamp so the row never reads "600 of 500". - SyncPhase::Scanning => JobItem { - title: format!("Checking {folder}"), - detail: "Looking for changes".to_string(), - done: p.done as u64, - total: if p.total == 0 { - 0 - } else { - p.total.max(p.done) as u64 - }, - }, - SyncPhase::Applying => JobItem { - title: format!("Syncing {folder}"), - detail: p.current.clone(), - done: p.done as u64, - total: p.total.max(p.done) as u64, - }, - }); - } - jobs - } - - /// The lock guarding sync-folder `id` against concurrent reconcile/mode-switch. - pub(crate) fn sync_lock(&self, id: i64) -> Arc> { - self.sync_locks - .lock() - .entry(id) - .or_default() - .clone() - } - - /// Remove a synced folder from the sync set. `delete_remote` also deletes its - /// folder under the device root; otherwise the cloud copy is left in place. - fn remove_sync_folder(&self, id: i64, delete_remote: bool) -> Result<(), String> { - let folder = self - .db - .sync_folder_get(id) - .map_err(|e| format!("db: {e:?}"))? - .ok_or_else(|| format!("no synced folder with id {id}"))?; - // An `ondemand` folder *is* a FUSE mount over its local path, so dropping - // only the row would strand the mount: the path would keep serving a - // folder the daemon no longer tracks, and nothing would ever unmount it. - // Tear it down first — including before trashing the remote tree it - // serves, which would otherwise leave it answering for deleted nodes. - if let Some(session) = self.mounts.lock().remove(&id) { - if let Err(e) = session.umount_and_join() { - warn!(id, error = %e, "unmount on-demand folder failed"); - } else { - info!(id, path = %folder.local_path, "unmounted on-demand folder"); - } - } - if delete_remote - && let Some(uid) = parse_uid(&folder.remote_uid) - && let Err(e) = self.rt.block_on(self.client.trash_nodes(&[uid])) - { - warn!(id, error = %e, "delete remote device folder failed"); - } - if !self - .db - .sync_folder_remove(id) - .map_err(|e| format!("db: {e:?}"))? - { - return Err(format!("no synced folder with id {id}")); - } - // Stop watching the folder we just dropped. - let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); - Ok(()) - } - - /// Trigger a reconcile: one folder by id, or every folder when `id` is `None`. - fn sync_now(&self, id: Option) { - let _ = match id { - Some(id) => self.sync_tx.send(sync::SyncMsg::Reconcile(id)), - None => self.sync_tx.send(sync::SyncMsg::ReconcileAll), - }; - } - - /// A sibling Core that shares this one's client/rt/cache/db (and transfer, - /// activity, mount registries) but gets a **fresh, empty `State`** — its own - /// inode space starting at [`ROOT_INO`]. Used to root a secondary FUSE session - /// at an `ondemand` sync folder without colliding with the main mount's inodes - /// (devices.md Phase 3). - fn fork_state(&self) -> Core { - let mut fork = self.clone(); - fork.state = Arc::new(Mutex::new(State { - entries: HashMap::new(), - by_uid: HashMap::new(), - children: HashMap::new(), - next_ino: 2, - handles: HashMap::new(), - next_fh: 1, - db: self.db.clone(), - })); - fork - } - - /// Ask for a synced folder to move to `mode`, applying it now if the folder is - /// free and safe to switch, and **queueing** it otherwise. Returns the human - /// message for the reply. - /// - /// Queueing rather than rejecting is what makes the toggle usable: a folder - /// that syncs continuously (a busy Downloads folder) is almost never caught in - /// the narrow window where it is both unlocked and `idle`, so a `try_lock` - /// rejection asks the user to keep retrying until they get lucky. Instead the - /// intent is recorded and the engine applies it at the end of the pass already - /// running — which, seeing a queued `ondemand`, also stops pulling down files - /// it is about to evict ([`Core::push_pass`]). - pub(crate) fn request_sync_folder_mode(&self, id: i64, mode: &str) -> Result { - if mode != "mirror" && mode != "ondemand" { - return Err(format!("unknown mode {mode:?} (want mirror|ondemand)")); - } - let folder = self - .db - .sync_folder_get(id) - .map_err(|e| format!("db: {e:?}"))? - .ok_or_else(|| format!("no synced folder with id {id}"))?; - if folder.mode == mode { - // Already there. A queued switch the other way is the user changing - // their mind back before it landed, so withdraw it. - if folder.pending_mode.is_some() { - self.db - .sync_folder_set_pending_mode(id, None) - .map_err(|e| format!("db: {e:?}"))?; - let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); - return Ok(format!("staying {mode}")); - } - return Ok(format!("already {mode}")); - } - - match self.apply_sync_folder_mode(id, mode) { - Ok(message) => Ok(message), - // Not switchable this instant — remember the intent instead of making - // the user retry, and kick a pass to clear whatever is in the way. - Err(SwitchBlocked::NotNow) => { - self.db - .sync_folder_set_pending_mode(id, Some(mode)) - .map_err(|e| format!("db: {e:?}"))?; - let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); - Ok(match mode { - "ondemand" => format!( - "{} will go on-demand once its local changes are uploaded", - base_name(&folder.local_path) - ), - _ => format!( - "{} will start mirroring once the current sync finishes", - base_name(&folder.local_path) - ), - }) - } - Err(SwitchBlocked::Failed(e)) => Err(e), - } - } - - /// Apply a queued mode switch if the folder has one and is now able to take it. - /// Called by the sync engine after every pass, so a switch the user asked for - /// mid-sync lands as soon as the pass that blocked it is done. A folder that is - /// still not ready (its push failed, so the local copy is not safe to evict) - /// keeps its `pending_mode` and is retried after the next pass. - pub(crate) fn settle_pending_mode(&self, id: i64) { - let Ok(Some(folder)) = self.db.sync_folder_get(id) else { - return; - }; - let Some(mode) = folder.pending_mode.as_deref() else { - return; - }; - if folder.mode == mode { - let _ = self.db.sync_folder_set_pending_mode(id, None); - return; - } - match self.apply_sync_folder_mode(id, mode) { - Ok(message) => { - info!(id, mode, "applied queued mode switch"); - self.log_activity(ActivityKind::Sync, &message, "", true); - } - // Still blocked: the pass could not get everything up, so the local copy - // is not safe to evict yet. Leave the request standing — the next pass - // (poll, or the retry the engine schedules) tries again. - Err(SwitchBlocked::NotNow) => { - info!(id, mode, "queued mode switch still waiting"); - } - Err(SwitchBlocked::Failed(e)) => { - warn!(id, mode, error = %e, "queued mode switch failed; withdrawing"); - let _ = self.db.sync_folder_set_pending_mode(id, None); - self.log_activity( - ActivityKind::Sync, - format!("couldn't switch {}", base_name(&folder.local_path)), - e, - false, - ); - } - } - } - - /// Flip a synced folder between `mirror` (full local copy + two-way sync) and - /// `ondemand` (a FUSE mount over the local path; no local storage). Returns a - /// human message on success. - /// - /// - **mirror→ondemand**: require the folder in-sync (`idle`), stop watching it, - /// evict the local files to reclaim disk, then mount a secondary `ProtonFs` - /// rooted at the folder's remote node over its local path. - /// - **ondemand→mirror**: unmount, clear the stale baseline (the local tree was - /// evicted), then hand the folder back to the engine, which re-downloads it. - /// - /// [`SwitchBlocked::NotNow`] means "not yet, try after a pass" and is never an - /// error the user needs to see — callers queue on it. - fn apply_sync_folder_mode(&self, id: i64, mode: &str) -> Result { - // Hold the folder's lock for the whole switch so no reconcile pass can be - // running over the tree we are about to evict (or start while we mount over - // it). A pass in flight holds the lock for its full duration, so `try_lock` - // failing is exactly "still syncing" — and it is the only reliable signal: - // the `state` column is still `idle` in the window between `add_sync_folder` - // inserting the row and the engine picking it up. - let lock = self.sync_lock(id); - let Some(_guard) = lock.try_lock() else { - return Err(SwitchBlocked::NotNow); - }; - // Re-read under the lock: a pass that finished while we waited may have - // changed `state`. - let folder = self - .db - .sync_folder_get(id) - .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))? - .ok_or_else(|| SwitchBlocked::Failed(format!("no synced folder with id {id}")))?; - if folder.mode == mode { - return Ok(format!("already {mode}")); - } - let local = PathBuf::from(&folder.local_path); - - match mode { - "ondemand" => { - // Only flip a folder that is fully in sync — a failed reconcile means - // local edits could still be un-uploaded, and we are about to delete - // the local copy. Not an error: a pass makes this true, and the queued - // request is applied once one does. - if folder.state != "idle" { - return Err(SwitchBlocked::NotNow); - } - let root_uid = parse_uid(&folder.remote_uid).ok_or_else(|| { - SwitchBlocked::Failed(format!("bad remote uid: {}", folder.remote_uid)) - })?; - let root = self - .rt - .block_on(self.client.enumerate_nodes(std::slice::from_ref(&root_uid))) - .map_err(|e| SwitchBlocked::Failed(format!("fetch remote root: {e}")))? - .into_iter() - .next() - .ok_or_else(|| SwitchBlocked::Failed("remote folder not found".to_string()))?; - - // Reclaim the disk: empty the local dir (keep it as the mountpoint). - evict_dir_contents(&local).map_err(|e| { - SwitchBlocked::Failed(format!("evict {}: {e}", local.display())) - })?; - - let session = self - .spawn_ondemand_mount(&local, root) - .map_err(SwitchBlocked::Failed)?; - self.mounts.lock().insert(id, session); - // Persist the mode only now that the mount is actually up. Writing it - // first would strand the folder on any failure below: the engine skips - // non-mirror folders, so an `ondemand` row with no mount is a folder - // that is neither mirrored nor browsable, and nothing retries it. - // Failing before this point leaves it `mirror`, and the next pass - // re-downloads whatever eviction removed. - self.db - .sync_folder_set_mode(id, "ondemand") - .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?; - let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); - self.db.sync_folder_set_state(id, "idle", now_secs()).ok(); - info!(id, path = %local.display(), "mounted sync folder on-demand"); - Ok(format!("{} is now on-demand", local.display())) - } - _ => { - // ondemand→mirror: tear down the secondary mount first. - if let Some(session) = self.mounts.lock().remove(&id) - && let Err(e) = session.umount_and_join() - { - warn!(id, error = %e, "unmount on-demand folder failed"); - } - // The evicted local tree makes the old baseline lie ("everything - // deleted locally"); clear it so the reconcile is a pure download. - self.db - .sync_entries_clear(id) - .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?; - self.db - .sync_folder_set_mode(id, "mirror") - .map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?; - let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); - let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); - info!(id, path = %local.display(), "restored sync folder to mirror"); - Ok(format!( - "{} is mirroring again; downloading", - local.display() - )) - } - } - } - - /// Spawn a secondary FUSE session rooted at `root` over `local` on a forked - /// inode space. Clears any stale kernel mount first (a crashed run can leave - /// one, which would fail the fresh mount with EBUSY). - fn spawn_ondemand_mount(&self, local: &Path, root: Node) -> Result { - clear_stale_mount(local); - let mut config = Config::default(); - config.mount_options = vec![ - MountOption::FSName("protondrive".to_string()), - MountOption::Subtype("protondrive".to_string()), - MountOption::DefaultPermissions, - ]; - let fs = ProtonFs::new(self.fork_state(), root); - Session::new(fs, local, &config) - .and_then(|s| s.spawn()) - .map_err(|e| format!("mount {}: {e}", local.display())) - } - - /// Re-establish FUSE mounts for folders left in `ondemand` mode across a daemon - /// restart (their local dirs are empty on disk — the files live in the cloud). - /// Best-effort per folder: a missing local path or a failed remote fetch marks - /// the folder `error` and moves on rather than aborting the rest. Runs on its - /// own thread from `mount` so the network fetches never block startup - /// (devices.md Phase 4). - fn restore_ondemand_mounts(&self) { - let folders = match self.db.sync_folder_list() { - Ok(f) => f, - Err(e) => { - warn!(error = ?e, "restore on-demand: cannot list folders"); - return; - } - }; - for folder in folders { - if folder.mode != "ondemand" { - continue; - } - let local = PathBuf::from(&folder.local_path); - if !local.is_dir() { - warn!(id = folder.id, path = %local.display(), "restore on-demand: local path missing"); - let _ = self - .db - .sync_folder_set_state(folder.id, "error", now_secs()); - continue; - } - // An `ondemand` folder's local dir is empty by construction — the switch - // evicts it. Finding files there means the row is lying (a switch that - // died between persisting the mode and evicting), and mounting over them - // would hide real local data behind the remote tree. Leave the files - // alone and let the user resolve it. - match dir_is_empty(&local) { - Ok(true) => {} - Ok(false) => { - warn!( - id = folder.id, - path = %local.display(), - "restore on-demand: local dir is not empty; refusing to mount over it" - ); - let _ = self - .db - .sync_folder_set_state(folder.id, "error", now_secs()); - continue; - } - Err(e) => { - warn!(id = folder.id, path = %local.display(), error = %e, "restore on-demand: cannot read local dir"); - let _ = self - .db - .sync_folder_set_state(folder.id, "error", now_secs()); - continue; - } - } - let Some(root_uid) = parse_uid(&folder.remote_uid) else { - warn!(id = folder.id, "restore on-demand: bad remote uid"); - continue; - }; - let root = match self - .rt - .block_on(self.client.enumerate_nodes(std::slice::from_ref(&root_uid))) - { - Ok(v) => match v.into_iter().next() { - Some(n) => n, - None => { - warn!(id = folder.id, "restore on-demand: remote folder gone"); - let _ = self - .db - .sync_folder_set_state(folder.id, "error", now_secs()); - continue; - } - }, - Err(e) => { - warn!(id = folder.id, error = %e, "restore on-demand: fetch remote failed"); - let _ = self - .db - .sync_folder_set_state(folder.id, "error", now_secs()); - continue; - } - }; - match self.spawn_ondemand_mount(&local, root) { - Ok(session) => { - self.mounts.lock().insert(folder.id, session); - let _ = self.db.sync_folder_set_state(folder.id, "idle", now_secs()); - info!(id = folder.id, path = %local.display(), "remounted on-demand folder"); - } - Err(e) => { - warn!(id = folder.id, error = %e, "restore on-demand: mount failed"); - let _ = self - .db - .sync_folder_set_state(folder.id, "error", now_secs()); - } - } - } - } - - // ---- sharing a node --------------------------------------------------- - - /// Invite Proton and/or external emails to the node at `rel` at `role`. - /// Returns `(proton_invited, external_invited)`. - fn share_node( - &self, - rel: &Path, - emails: &[String], - role: &str, - message: Option<&str>, - ) -> Result<(usize, usize), String> { - let (_ino, uid) = self - .resolve_path(rel) - .map_err(|e| format!("resolve path: {e:?}"))?; - let role = role_from_str(role)?; - let invitees: Vec<(String, MemberRole)> = - emails.iter().map(|e| (e.clone(), role)).collect(); - self.rt - .block_on(self.client.invite_users(&uid, &invitees, message)) - .map_err(|e| format!("share: {e}")) - } - - /// List the members, pending invitations and public link of the node at `rel`. - fn list_share(&self, rel: &Path) -> Result<(Vec, Option), String> { - let (_ino, uid) = self - .resolve_path(rel) - .map_err(|e| format!("resolve path: {e:?}"))?; - - let mut entries = Vec::new(); - for m in self - .rt - .block_on(self.client.list_share_members(&uid)) - .map_err(|e| format!("list members: {e}"))? - { - entries.push(ShareEntry { - id: m.membership_id.to_string(), - email: m.email, - role: role_to_str(m.role).to_string(), - kind: ShareEntryKind::Member, - }); - } - for inv in self - .rt - .block_on(self.client.list_share_invitations(&uid)) - .map_err(|e| format!("list invitations: {e}"))? - { - entries.push(ShareEntry { - id: inv.invitation_id, - email: inv.invitee_email, - role: role_to_str(inv.role).to_string(), - kind: ShareEntryKind::ProtonInvite, - }); - } - for ext in self - .rt - .block_on(self.client.list_external_invitations(&uid)) - .map_err(|e| format!("list external invitations: {e}"))? - { - entries.push(ShareEntry { - id: ext.invitation_id, - email: ext.invitee_email, - role: role_to_str(ext.role).to_string(), - kind: ShareEntryKind::ExternalInvite, - }); - } - - let link = self - .rt - .block_on(self.client.get_public_link(&uid)) - .map_err(|e| format!("get public link: {e}"))? - .map(public_link_info); - - Ok((entries, link)) - } - - /// Change the role of a member or pending Proton invitation on the node at - /// `rel`. External invitations have no role-update endpoint. - fn update_share_role( - &self, - rel: &Path, - id: &str, - kind: ShareEntryKind, - role: &str, - ) -> Result<(), String> { - let (_ino, uid) = self - .resolve_path(rel) - .map_err(|e| format!("resolve path: {e:?}"))?; - let role = role_from_str(role)?; - match kind { - ShareEntryKind::Member => { - let member = self - .rt - .block_on(self.client.list_share_members(&uid)) - .map_err(|e| format!("list members: {e}"))? - .into_iter() - .find(|m| m.membership_id.to_string() == id) - .ok_or_else(|| "member not found".to_string())?; - self.rt - .block_on(self.client.update_member_role(&member, role)) - .map_err(|e| format!("update role: {e}")) - } - ShareEntryKind::ProtonInvite => { - let inv = self - .rt - .block_on(self.client.list_share_invitations(&uid)) - .map_err(|e| format!("list invitations: {e}"))? - .into_iter() - .find(|i| i.invitation_id == id) - .ok_or_else(|| "invitation not found".to_string())?; - self.rt - .block_on(self.client.update_invitation_role(&inv, role)) - .map_err(|e| format!("update role: {e}")) - } - ShareEntryKind::ExternalInvite => { - Err("an external invitation's role cannot be changed".to_string()) - } - } - } - - /// Remove a member, pending Proton invite, or external invite from the node - /// at `rel`. - fn remove_share_entry(&self, rel: &Path, id: &str, kind: ShareEntryKind) -> Result<(), String> { - let (_ino, uid) = self - .resolve_path(rel) - .map_err(|e| format!("resolve path: {e:?}"))?; - match kind { - ShareEntryKind::Member => { - let member = self - .rt - .block_on(self.client.list_share_members(&uid)) - .map_err(|e| format!("list members: {e}"))? - .into_iter() - .find(|m| m.membership_id.to_string() == id) - .ok_or_else(|| "member not found".to_string())?; - self.rt - .block_on(self.client.remove_member(&member)) - .map_err(|e| format!("remove member: {e}")) - } - ShareEntryKind::ProtonInvite => { - let inv = self - .rt - .block_on(self.client.list_share_invitations(&uid)) - .map_err(|e| format!("list invitations: {e}"))? - .into_iter() - .find(|i| i.invitation_id == id) - .ok_or_else(|| "invitation not found".to_string())?; - self.rt - .block_on(self.client.delete_invitation(&inv)) - .map_err(|e| format!("revoke invitation: {e}")) - } - ShareEntryKind::ExternalInvite => { - let ext = self - .rt - .block_on(self.client.list_external_invitations(&uid)) - .map_err(|e| format!("list external invitations: {e}"))? - .into_iter() - .find(|i| i.invitation_id == id) - .ok_or_else(|| "external invitation not found".to_string())?; - self.rt - .block_on(self.client.delete_external_invitation(&ext)) - .map_err(|e| format!("revoke external invitation: {e}")) - } - } - } - - /// Create a public link on the node at `rel`, returning its info (with URL). - fn create_public_link( - &self, - rel: &Path, - role: &str, - password: Option<&str>, - expires: Option, - ) -> Result { - let (_ino, uid) = self - .resolve_path(rel) - .map_err(|e| format!("resolve path: {e:?}"))?; - let role = role_from_str(role)?; - let link = self - .rt - .block_on( - self.client - .create_public_link(&uid, role, password, expires), - ) - .map_err(|e| format!("create public link: {e}"))?; - Ok(public_link_info(link)) - } - - /// Remove the public link `id` from the node at `rel`. - fn remove_public_link(&self, rel: &Path, id: &str) -> Result<(), String> { - let (_ino, uid) = self - .resolve_path(rel) - .map_err(|e| format!("resolve path: {e:?}"))?; - let link = self - .rt - .block_on(self.client.get_public_link(&uid)) - .map_err(|e| format!("get public link: {e}"))? - .filter(|l| l.public_link_id == id) - .ok_or_else(|| "public link not found".to_string())?; - self.rt - .block_on(self.client.remove_public_link(&link)) - .map_err(|e| format!("remove public link: {e}")) - } - - // ---- shared with me --------------------------------------------------- - - /// List nodes shared with me that I have accepted. - fn list_shared_with_me(&self) -> Result, String> { - let uids = self - .rt - .block_on(self.client.enumerate_shared_with_me_node_uids()) - .map_err(|e| format!("enumerate shared: {e}"))?; - if uids.is_empty() { - return Ok(Vec::new()); - } - let nodes = self - .rt - .block_on(self.client.enumerate_nodes(&uids)) - .map_err(|e| format!("enumerate nodes: {e}"))?; - Ok(nodes - .into_iter() - .map(|n| { - let is_dir = n.is_folder(); - let size = match &n.kind { - NodeKind::File { - claimed_size, - total_size_on_storage, - .. - } => claimed_size.unwrap_or(*total_size_on_storage).max(0) as u64, - NodeKind::Folder => 0, - }; - DirEntry { - name: n.name, - is_dir, - size, - modified: 0, - pinned: false, - cached: false, - uid: n.uid.to_string(), - path: String::new(), - } - }) - .collect()) - } - - /// Leave a shared node by its uid. - fn leave_shared(&self, uid: &str) -> Result<(), String> { - let uid = parse_uid(uid).ok_or_else(|| format!("invalid uid: {uid}"))?; - self.rt - .block_on(self.client.leave_shared_node(&uid)) - .map_err(|e| format!("leave shared: {e}")) - } - - // ---- incoming invitations --------------------------------------------- - - /// List invitations addressed to me, pending accept or reject. - fn list_invitations(&self) -> Result, String> { - let invitations = self - .rt - .block_on(self.client.list_incoming_invitations()) - .map_err(|e| format!("list invitations: {e}"))?; - Ok(invitations - .into_iter() - .map(|i| InvitationInfo { - id: i.invitation_id, - inviter_email: i.inviter_email, - name: i.node_name, - is_dir: i.is_folder, - }) - .collect()) - } - - /// Accept an invitation addressed to me by its id. - fn accept_invitation(&self, id: &str) -> Result<(), String> { - self.rt - .block_on(self.client.accept_invitation(id)) - .map_err(|e| format!("accept invitation: {e}")) - } - - /// Reject an invitation addressed to me by its id. - fn reject_invitation(&self, id: &str) -> Result<(), String> { - self.rt - .block_on(self.client.reject_invitation(id)) - .map_err(|e| format!("reject invitation: {e}")) - } - - // ---- bookmarks -------------------------------------------------------- - - /// List public links saved to my account. - fn list_bookmarks(&self) -> Result, String> { - let bookmarks = self - .rt - .block_on(self.client.list_bookmarks()) - .map_err(|e| format!("list bookmarks: {e}"))?; - Ok(bookmarks - .into_iter() - .map(|b| BookmarkInfo { - token: b.token, - url: b.url, - name: b.node_name, - is_dir: b.is_folder, - }) - .collect()) - } - - /// Save a public link (optionally password-protected) as a bookmark. - fn create_bookmark(&self, url: &str, password: Option<&str>) -> Result<(), String> { - self.rt - .block_on(self.client.create_bookmark(url, password)) - .map_err(|e| format!("create bookmark: {e}")) - } - - /// Remove a saved bookmark by its token. - fn delete_bookmark(&self, token: &str) -> Result<(), String> { - self.rt - .block_on(self.client.delete_bookmark(token)) - .map_err(|e| format!("delete bookmark: {e}")) - } - - // ---- shared by me ----------------------------------------------------- - - /// List the nodes I have shared with others, each with a summary of its share - /// state (members, pending invitations, public link). One list call enumerates - /// the shared uids; the per-node detail is then gathered best-effort — a single - /// node racing with an unshare drops from the list rather than failing the whole - /// request. - fn list_shared_by_me(&self) -> Result, String> { - let uids = self - .rt - .block_on(self.client.enumerate_shared_by_me_node_uids()) - .map_err(|e| format!("enumerate shared-by-me: {e}"))?; - if uids.is_empty() { - return Ok(Vec::new()); - } - let nodes = self - .rt - .block_on(self.client.enumerate_nodes(&uids)) - .map_err(|e| format!("enumerate nodes: {e}"))?; - let mut items = Vec::with_capacity(nodes.len()); - for n in nodes { - let uid = n.uid.clone(); - let members = self - .rt - .block_on(self.client.list_share_members(&uid)) - .map(|m| m.len()) - .unwrap_or(0); - let proton_invites = self - .rt - .block_on(self.client.list_share_invitations(&uid)) - .map(|i| i.len()) - .unwrap_or(0); - let external_invites = self - .rt - .block_on(self.client.list_external_invitations(&uid)) - .map(|i| i.len()) - .unwrap_or(0); - let link = self - .rt - .block_on(self.client.get_public_link(&uid)) - .ok() - .flatten() - .map(public_link_info); - items.push(SharedItem { - uid: uid.to_string(), - is_dir: n.is_folder(), - name: n.name, - path: self.rel_path_for_uid(&uid).unwrap_or_default(), - member_count: members, - invite_count: proton_invites + external_invites, - link, - }); - } - Ok(items) - } - - /// Best-effort mountpoint-relative path for a node already interned in the live - /// tree, by walking parent inodes to the root. `None` when the node has never - /// been seen through the mount (e.g. shared but not browsed to this session) — - /// the caller then leaves the path empty. - fn rel_path_for_uid(&self, uid: &NodeUid) -> Option { - let st = self.state.lock(); - let mut ino = *st.by_uid.get(uid)?; - let mut parts = Vec::new(); - while ino != ROOT_INO { - let entry = st.entries.get(&ino)?; - parts.push(entry.node.name.clone()); - ino = entry.parent; - } - parts.reverse(); - Some(parts.join("/")) - } // ---- activity log ----------------------------------------------------- @@ -6840,695 +4370,6 @@ fn scan_local_once( indexing.store(false, Ordering::Relaxed); } -/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path -/// must live under `mountpoint`; a relative path is taken as already relative to -/// the mount root. -fn rel_to_mount(mountpoint: &Path, path: &str) -> Result { - let p = Path::new(path); - if p.is_absolute() { - p.strip_prefix(mountpoint) - .map(Path::to_path_buf) - .map_err(|_| format!("{path} is not under the mountpoint")) - } else { - Ok(p.to_path_buf()) - } -} - -/// Handle one control-socket connection: read a single JSON request line, -/// dispatch it against `core`, and write back a JSON response line. -fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: UnixStream) { - let mut reader = BufReader::new(match stream.try_clone() { - Ok(s) => s, - Err(e) => { - warn!(error = %e, "control: clone stream failed"); - return; - } - }); - let mut line = String::new(); - if reader.read_line(&mut line).is_err() || line.trim().is_empty() { - return; - } - let response = match serde_json::from_str::(line.trim()) { - Ok(CtlRequest::Status) => { - let pins = core.cache.list_pins(); - let queued = core.db.pending_op_counts().unwrap_or_default(); - CtlResponse::Status { - username: username.to_string(), - mountpoint: mountpoint.display().to_string(), - pinned: pins.len(), - used: core.cache.usage(), - budget: core.cache.budget(), - pins, - online: core.online.load(Ordering::Relaxed), - pending_uploads: queued.uploads.max(0) as u64, - pending_changes: queued.changes.max(0) as u64, - } - } - Ok(CtlRequest::Pin { path }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.pin(&rel) { - Ok(name) => CtlResponse::Ok { - message: format!("pinned {name}"), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::Unpin { path }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.unpin(&rel) { - Ok(name) => CtlResponse::Ok { - message: format!("unpinned {name}"), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListPins) => CtlResponse::Pins { - pins: core.cache.list_pins(), - }, - Ok(CtlRequest::ListDir { path }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.list_dir(&rel) { - Ok(entries) => CtlResponse::Entries { entries }, - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::Refresh { scope }) => { - let result = match &scope { - RefreshScope::Dir { path } => { - rel_to_mount(mountpoint, path).and_then(|rel| core.refresh_dir(&rel)) - } - RefreshScope::Trash => { - core.invalidate_trash(); - Ok(()) - } - RefreshScope::Photos => { - core.invalidate_photos(); - Ok(()) - } - }; - match result { - Ok(()) => CtlResponse::Ok { - message: "refreshed".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::PhotosTimeline { - offset, - limit, - kind, - range, - }) => { - match core.photos_timeline(offset, limit, kind, range) { - Ok(Some(items)) => CtlResponse::Photos { - available: true, - items, - counts: core.db.photos_counts().ok(), - }, - Ok(None) => CtlResponse::Photos { - available: false, - items: Vec::new(), - counts: None, - }, - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::PhotoMonths { kind }) => match core.db.photos_months(kind) { - Ok(months) => CtlResponse::PhotoMonths { - months: months - .into_iter() - .map(|(year, month, count)| PhotoMonth { - year, - month, - count, - }) - .collect(), - }, - Err(e) => CtlResponse::Error { - message: e.to_string(), - }, - }, - Ok(CtlRequest::PhotoThumbs { uids }) => { - let parsed: Vec = uids.iter().filter_map(|u| parse_uid(u)).collect(); - CtlResponse::Thumbs { - items: core.photo_thumbs(&parsed), - } - } - Ok(CtlRequest::OpenPhoto { uid }) => match parse_uid(&uid) { - Some(u) => match core.open_photo(&u) { - Ok(p) => CtlResponse::FilePath { - path: p.display().to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - None => CtlResponse::Error { - message: format!("bad uid: {uid}"), - }, - }, - Ok(CtlRequest::UploadPhoto { - name, - media_type, - bytes, - capture_time, - }) => { - let photos = core.photos(); - let metadata = proton_drive_rs::PhotoUploadMetadata { - capture_time, - ..Default::default() - }; - let guard = core.transfers.begin( - name.clone(), - "", - TransferDirection::Upload, - bytes.len() as u64, - ); - let reader = CountingReader::new(std::io::Cursor::new(&bytes), &guard); - match core.rt.block_on(photos.upload_photo_from( - &name, - &media_type, - reader, - bytes.len() as i64, - Vec::new(), - metadata, - false, - )) { - Ok(uid) => { - // The photo we just uploaded belongs at the head of the - // timeline, and the GUI reloads the gallery the moment this - // reply lands — so refresh now rather than leaving it to a - // background pass that would land just after that reload. - if let Err(e) = core.rt.block_on(core.refresh_timeline()) { - warn!(error = %e, "timeline refresh after upload failed"); - } - CtlResponse::Ok { - message: format!("uploaded photo with uid {uid}"), - } - } - Err(e) => CtlResponse::Error { - message: format!("upload photo failed: {e}"), - }, - } - } - Ok(CtlRequest::OpenFile { path }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.open_file(&rel) { - Ok(p) => CtlResponse::FilePath { - path: p.display().to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::Search { query, limit }) => match core.search(&query, limit) { - Ok(hits) => CtlResponse::SearchResults { hits }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::SearchLocal { query, limit }) => match core.search_local(&query, limit) { - Ok(hits) => CtlResponse::LocalResults { - hits, - indexing: core.indexing.load(Ordering::Relaxed), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::Rename { path, new_name }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.rename(&rel, &new_name) { - Ok(name) => { - core.log_activity(ActivityKind::Rename, &name, format!("was {path}"), true); - CtlResponse::Ok { - message: format!("renamed to {name}"), - } - } - Err(e) => { - core.log_activity(ActivityKind::Rename, &path, &e, false); - CtlResponse::Error { message: e } - } - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::Move { path, new_parent }) => { - match ( - rel_to_mount(mountpoint, &path), - rel_to_mount(mountpoint, &new_parent), - ) { - (Ok(rel), Ok(parent_rel)) => match core.move_to(&rel, &parent_rel) { - Ok(name) => { - let dest = if new_parent.is_empty() { - "My files".to_string() - } else { - new_parent.clone() - }; - core.log_activity(ActivityKind::Move, &name, format!("to {dest}"), true); - CtlResponse::Ok { - message: format!("moved {name}"), - } - } - Err(e) => { - core.log_activity(ActivityKind::Move, &path, &e, false); - CtlResponse::Error { message: e } - } - }, - (Err(e), _) | (_, Err(e)) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::Delete { path }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.delete(&rel) { - Ok(name) => { - core.log_activity(ActivityKind::Trash, &name, "", true); - CtlResponse::Ok { - message: format!("trashed {name}"), - } - } - Err(e) => { - core.log_activity(ActivityKind::Trash, &path, &e, false); - CtlResponse::Error { message: e } - } - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::CreateFolder { parent, name }) => match rel_to_mount(mountpoint, &parent) { - Ok(parent_rel) => match core.create_folder(&parent_rel, &name) { - Ok(name) => { - core.log_activity(ActivityKind::CreateFolder, &name, "", true); - CtlResponse::Ok { - message: format!("created folder {name}"), - } - } - Err(e) => { - core.log_activity(ActivityKind::CreateFolder, &name, &e, false); - CtlResponse::Error { message: e } - } - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::UploadFile { - parent, - name, - bytes, - }) => match rel_to_mount(mountpoint, &parent) { - Ok(parent_rel) => match core.upload(&parent_rel, &name, &bytes) { - Ok(name) => { - core.log_activity(ActivityKind::Upload, &name, "", true); - CtlResponse::Ok { - message: format!("uploaded {name}"), - } - } - Err(e) => { - core.log_activity(ActivityKind::Upload, &name, &e, false); - CtlResponse::Error { message: e } - } - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::UploadPaths { parent, sources }) => { - match rel_to_mount(mountpoint, &parent) { - // Ack immediately and upload on a background thread: a directory tree - // far outlasts the socket read timeout. Progress and completion are - // observed through GetQueueStatus; the activity log gets the summary - // when the whole batch finishes. - Ok(parent_rel) => { - let core = core.clone(); - let paths: Vec = sources.into_iter().map(PathBuf::from).collect(); - let n = paths.len(); - std::thread::spawn(move || { - let started = Instant::now(); - match core.upload_paths(&parent_rel, &paths) { - Ok(stats) => { - // e.g. "Uploaded 700 files to Photos" — name the destination - // so the log reads like a sentence rather than a bare count. - let dest = parent_rel - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("your Drive"); - let target = format!( - "{} to {dest}", - count_noun(stats.uploaded, "file", "files") - ); - // Size · folders · elapsed, with a trailing failure count - // when some files didn't make it. - let mut parts = vec![human_bytes(stats.bytes)]; - if stats.folders > 0 { - parts.push(count_noun(stats.folders, "folder", "folders")); - } - parts.push(human_duration(started.elapsed())); - if stats.failed > 0 { - parts.push(format!("{} failed", stats.failed)); - } - core.log_activity( - ActivityKind::Upload, - target, - parts.join(" · "), - stats.failed == 0, - ); - } - Err(e) => { - warn!(error = %e, "bulk upload failed"); - core.log_activity(ActivityKind::Upload, "bulk upload", &e, false); - } - } - }); - CtlResponse::Ok { - message: format!("uploading {n} item(s)"), - } - } - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::ListTrash) => match core.list_trash() { - Ok(entries) => CtlResponse::Entries { entries }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::Restore { uids }) => match core.restore(&uids) { - Ok(n) => { - core.log_activity(ActivityKind::Restore, format!("{n} item(s)"), "", true); - CtlResponse::Ok { - message: format!("restored {n} item(s)"), - } - } - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::DeleteForever { uids }) => match core.delete_forever(&uids) { - Ok(n) => { - core.log_activity( - ActivityKind::DeleteForever, - format!("{n} item(s)"), - "", - true, - ); - CtlResponse::Ok { - message: format!("permanently deleted {n} item(s)"), - } - } - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::EmptyTrash) => match core.empty_trash() { - Ok(n) => { - core.log_activity(ActivityKind::EmptyTrash, format!("{n} item(s)"), "", true); - CtlResponse::Ok { - message: format!("emptied trash ({n} item(s))"), - } - } - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::PurgeCache) => { - let freed = core.cache.clear_unpinned(); - CtlResponse::Ok { - message: format!( - "purged {:.1} MiB of unpinned cache", - freed as f64 / 1_048_576.0 - ), - } - } - Ok(CtlRequest::GetQueueStatus) => CtlResponse::Transfers { - items: core.transfers.snapshot(), - jobs: core.jobs_snapshot(), - }, - Ok(CtlRequest::SetCacheBudget { bytes }) => { - core.cache.set_budget(bytes); - // Persist so the next mount keeps the new cap. Best-effort: a config - // write failure is reported but the live cap is already applied. - match AppDirs::new().map(|dirs| { - let mut cfg = dirs.load_config(); - cfg.cache_budget = Some(bytes); - dirs.save_config(&cfg) - }) { - Ok(Ok(())) => CtlResponse::Ok { - message: format!("cache budget set to {bytes} bytes"), - }, - Ok(Err(e)) => CtlResponse::Error { - message: format!("budget applied but config write failed: {e}"), - }, - Err(e) => CtlResponse::Error { - message: format!("budget applied but config unavailable: {e}"), - }, - } - } - Ok(CtlRequest::ListDevices) => match core.list_devices() { - Ok(items) => CtlResponse::Devices { items }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::RenameDevice { uid, name }) => match core.rename_device(&uid, &name) { - Ok(()) => CtlResponse::Ok { - message: format!("renamed device to {name}"), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::DeleteDevice { uid }) => match core.delete_device(&uid) { - Ok(()) => CtlResponse::Ok { - message: "device deleted".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::AddSyncFolder { local_path }) => { - // Registering the device and uploading a folder tree far outlasts the - // socket read timeout, so ack immediately and work on a background - // thread. The folder appears in ListSyncFolders once the row lands; - // completion (and any failures) go to the activity log. - let core = core.clone(); - let path = PathBuf::from(&local_path); - std::thread::spawn(move || { - let started = Instant::now(); - match core.add_sync_folder(&path) { - Ok(folder) => { - let name = Path::new(&folder.local_path) - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or(&folder.local_path) - .to_string(); - core.log_activity( - ActivityKind::Upload, - format!("synced {name}"), - human_duration(started.elapsed()), - folder.state != "error", - ); - } - Err(e) => { - warn!(error = %e, "add sync folder failed"); - core.log_activity(ActivityKind::Upload, "add sync folder", &e, false); - } - } - }); - CtlResponse::Ok { - message: format!("adding {local_path}"), - } - } - Ok(CtlRequest::ListSyncFolders) => match core.list_sync_folders() { - Ok(items) => CtlResponse::SyncFolders { items }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::RemoveSyncFolder { id, delete_remote }) => { - match core.remove_sync_folder(id, delete_remote) { - Ok(()) => CtlResponse::Ok { - message: "removed synced folder".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::SetSyncFolderMode { id, mode }) => { - match core.request_sync_folder_mode(id, &mode) { - Ok(message) => { - core.log_activity(ActivityKind::Upload, &message, "", true); - CtlResponse::Ok { message } - } - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::SyncNow { id }) => { - core.sync_now(id); - CtlResponse::Ok { - message: match id { - Some(id) => format!("reconciling folder {id}"), - None => "reconciling all folders".to_string(), - }, - } - } - Ok(CtlRequest::ShareNode { - path, - emails, - role, - message, - }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.share_node(&rel, &emails, &role, message.as_deref()) { - Ok((proton, external)) => { - core.log_activity( - ActivityKind::Share, - &path, - format!("{} recipient(s) as {role}", proton + external), - true, - ); - CtlResponse::Ok { - message: format!("invited {proton} Proton and {external} external user(s)"), - } - } - Err(e) => { - core.log_activity(ActivityKind::Share, &path, &e, false); - CtlResponse::Error { message: e } - } - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListShare { path }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.list_share(&rel) { - Ok((entries, link)) => CtlResponse::Share { entries, link }, - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::UpdateShareRole { - path, - id, - kind, - role, - }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.update_share_role(&rel, &id, kind, &role) { - Ok(()) => CtlResponse::Ok { - message: format!("role updated to {role}"), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::RemoveShareEntry { path, id, kind }) => { - match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.remove_share_entry(&rel, &id, kind) { - Ok(()) => { - core.log_activity(ActivityKind::Unshare, &path, "access removed", true); - CtlResponse::Ok { - message: "removed".to_string(), - } - } - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::CreatePublicLink { - path, - role, - password, - expires, - }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.create_public_link(&rel, &role, password.as_deref(), expires) { - Ok(link) => { - core.log_activity(ActivityKind::PublicLink, &path, "link created", true); - CtlResponse::PublicLink { link } - } - Err(e) => { - core.log_activity(ActivityKind::PublicLink, &path, &e, false); - CtlResponse::Error { message: e } - } - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::RemovePublicLink { path, id }) => match rel_to_mount(mountpoint, &path) { - Ok(rel) => match core.remove_public_link(&rel, &id) { - Ok(()) => { - core.log_activity(ActivityKind::Unshare, &path, "link removed", true); - CtlResponse::Ok { - message: "public link removed".to_string(), - } - } - Err(e) => CtlResponse::Error { message: e }, - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListSharedWithMe) => match core.list_shared_with_me() { - Ok(entries) => CtlResponse::Entries { entries }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::LeaveShared { uid }) => match core.leave_shared(&uid) { - Ok(()) => { - core.log_activity(ActivityKind::Unshare, "shared item", "left", true); - CtlResponse::Ok { - message: "left shared item".to_string(), - } - } - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListInvitations) => match core.list_invitations() { - Ok(items) => CtlResponse::Invitations { items }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::AcceptInvitation { id }) => match core.accept_invitation(&id) { - Ok(()) => CtlResponse::Ok { - message: "invitation accepted".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::RejectInvitation { id }) => match core.reject_invitation(&id) { - Ok(()) => CtlResponse::Ok { - message: "invitation rejected".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListBookmarks) => match core.list_bookmarks() { - Ok(items) => CtlResponse::Bookmarks { items }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::CreateBookmark { url, password }) => { - match core.create_bookmark(&url, password.as_deref()) { - Ok(()) => CtlResponse::Ok { - message: "bookmark saved".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - } - } - Ok(CtlRequest::DeleteBookmark { token }) => match core.delete_bookmark(&token) { - Ok(()) => CtlResponse::Ok { - message: "bookmark removed".to_string(), - }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListSharedByMe) => match core.list_shared_by_me() { - Ok(items) => CtlResponse::SharedByMe { items }, - Err(e) => CtlResponse::Error { message: e }, - }, - Ok(CtlRequest::ListActivity { limit }) => CtlResponse::Activity { - items: core.list_activity(limit), - }, - Err(e) => CtlResponse::Error { - message: format!("bad request: {e}"), - }, - }; - let mut out = match serde_json::to_vec(&response) { - Ok(v) => v, - Err(e) => { - warn!(error = %e, "control: serialize response failed"); - return; - } - }; - out.push(b'\n'); - let mut stream = stream; - let _ = stream.write_all(&out); -} - -/// Listen on the control socket, serving one command per connection, each on its -/// own thread. Runs on its own thread; returns only if the listener itself fails. -/// -/// Concurrent rather than serial because requests differ wildly in cost: an -/// `OpenPhoto` downloads a whole photo, and while it ran the accept loop used to -/// stall every other caller behind it — the GUI's 2s status poll, and the -/// thumbnail batches the gallery needs to paint. [`Core`] is a bundle of handles -/// (`Arc`/`Clone`), so each connection gets its own copy of it. -fn run_control_socket(core: Core, username: String, mountpoint: PathBuf, listener: UnixListener) { - info!("control socket listening"); - for conn in listener.incoming() { - match conn { - Ok(stream) => { - let core = core.clone(); - let username = username.clone(); - let mountpoint = mountpoint.clone(); - if let Err(e) = std::thread::Builder::new() - .name("pdfs-control".into()) - .spawn(move || handle_control_conn(&core, &username, &mountpoint, stream)) - { - warn!(error = %e, "control: spawn handler failed"); - } - } - Err(e) => { - warn!(error = %e, "control: accept failed"); - } - } - } -} /// Why a [`mount`] call returned. Lets the daemon decide whether to exit (clean /// shutdown) or remount (the mount went away under it). diff --git a/crates/pdfs-fuse/src/photos.rs b/crates/pdfs-fuse/src/photos.rs new file mode 100644 index 0000000..3cf4fd4 --- /dev/null +++ b/crates/pdfs-fuse/src/photos.rs @@ -0,0 +1,363 @@ +//! The photos timeline and its thumbnails. +//! +//! The timeline is a flat, newest-first projection of the photo share, persisted +//! by [`pdfs_core::db`] so the gallery opens instantly and refreshes behind the +//! user. Thumbnails the server has none for (anything a camera wrote rather than +//! a phone) are generated locally and stored as if the server had served them. + +use std::collections::HashMap; +use std::path::PathBuf; + +use pdfs_core::control::{PhotoItem, PhotoKind, PhotoThumb}; +use pdfs_core::db::{self, StoredPhoto}; +use std::sync::atomic::Ordering; + +use proton_drive_rs::{NodeKind, ThumbnailType}; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use tracing::{info, warn}; + +use super::{ + Core, PHOTOS_AVAILABLE, PHOTOS_SYNCED_MS, THUMB_GEN_CONCURRENCY, TIMELINE_ENRICH_CHUNK, + TIMELINE_TTL, ThumbAttempt, node_size, now_ms, parse_uid, ratio_of, scale_thumbnail, +}; + +impl Core { + pub(crate) fn photos_timeline( + &self, + offset: usize, + limit: usize, + kind: Option, + range: Option<(i64, i64)>, + ) -> Result>, String> { + let count = self.db.photos_count().map_err(|e| e.to_string())?; + if count == 0 { + // Nothing to serve, so this one request has to wait for the fetch — + // unless we already know the account has no photos volume and the + // answer is a fresh "no". + let known_empty = self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() == Some(0); + if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) { + return Ok(None); + } + if !self.rt.block_on(self.refresh_timeline())? { + return Ok(None); + } + } else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) { + self.spawn_timeline_refresh(); + } + + let page = self + .db + .photos_page(offset, limit, kind, range) + .map_err(|e| e.to_string())?; + Ok(Some(page.into_iter().map(|p| self.photo_item(p)).collect())) + } + + /// Project a persisted photo into the wire item the front-end paints: its + /// learned aspect ratio, its thumbnail verdict, and the on-disk path of its + /// thumbnail when one is cached (tagged with the capture time, which is the + /// only revision handle a photo carries). + pub(crate) fn photo_item(&self, photo: StoredPhoto) -> PhotoItem { + let thumb_path = parse_uid(&photo.uid).and_then(|uid| { + self.cache + .cached_thumbnail_path(&uid, ThumbnailType::Thumbnail.as_i32(), photo.capture_time) + .map(|p| p.display().to_string()) + }); + PhotoItem { + uid: photo.uid, + capture_time: photo.capture_time, + thumb_path, + name: photo.name, + ratio: photo.ratio, + no_thumb: photo.thumb_state == db::THUMB_NONE, + kind: photo.kind, + } + } + + /// Thumbnails for `uids`, served from the cache, fetched from the server for + /// whatever is missing, and — for the photos the server has no thumbnail for + /// at all — *generated locally* from the full file (see + /// [`Core::generate_thumbs`]). Requested on demand as tiles scroll into view, + /// so a cold timeline paints immediately and only the photos actually on + /// screen cost a round-trip. + /// + /// A photo absent from the persisted timeline is skipped: its capture time is + /// the cache's validity tag, and guessing that would poison the cache. + pub(crate) fn photo_thumbs(&self, uids: &[NodeUid]) -> Vec { + let ttype = ThumbnailType::Thumbnail.as_i32(); + let keys: Vec = uids.iter().map(|u| u.to_string()).collect(); + let stored = self.db.photos_by_uid(&keys).unwrap_or_default(); + let tags: HashMap = stored + .iter() + .map(|p| (p.uid.clone(), p.capture_time)) + .collect(); + + // Ask the server only for photos that are missing a cached thumbnail and + // haven't already been written off as un-thumbnailable. + let want: Vec = uids + .iter() + .filter(|uid| { + let key = uid.to_string(); + stored + .iter() + .find(|p| p.uid == key) + .is_some_and(|p| p.thumb_state != db::THUMB_NONE) + && tags.get(&key).is_some_and(|&tag| { + self.cache.cached_thumbnail_path(uid, ttype, tag).is_none() + }) + }) + .cloned() + .collect(); + + if !want.is_empty() { + let mut missing = want.clone(); + match self.rt.block_on( + self.photos() + .enumerate_thumbnails(&want, ThumbnailType::Thumbnail), + ) { + Ok(thumbs) => { + for ft in thumbs { + let Some(&tag) = tags.get(&ft.file_uid.to_string()) else { + continue; + }; + let Ok(bytes) = ft.result else { continue }; + if bytes.is_empty() { + continue; + } + if self + .cache + .store_thumbnail(&ft.file_uid, ttype, tag, &bytes) + .is_ok() + { + missing.retain(|uid| uid != &ft.file_uid); + self.record_thumb(&ft.file_uid, db::THUMB_HAVE, ratio_of(&bytes)); + } + } + } + // A failed batch is not a verdict: leave every uid in `missing` so + // the local fallback still gives those tiles an image. + Err(e) => warn!(error = %e, "batch photo thumbnails failed"), + } + + // Whatever the server had nothing for gets a thumbnail made from its + // own bytes — this is what fills in camera photos uploaded by clients + // that never generated one. Off the request path: a full-size photo + // takes far longer to fetch than the whole rest of the batch, and the + // thumbnails that *are* ready must not wait behind it. + if !missing.is_empty() { + self.spawn_generate_thumbs(missing, &tags); + } + } + + let pending = self.thumb_gen.lock(); + uids.iter() + .map(|uid| PhotoThumb { + uid: uid.to_string(), + path: tags.get(&uid.to_string()).and_then(|&tag| { + self.cache + .cached_thumbnail_path(uid, ttype, tag) + .map(|p| p.display().to_string()) + }), + pending: pending.contains(uid), + }) + .collect() + } + + /// Generate the missing thumbnails on the runtime, skipping any photo already + /// being generated. The uids are marked in-flight before the task starts, so + /// the reply this call is about to send already reports them as pending. + pub(crate) fn spawn_generate_thumbs(&self, uids: Vec, tags: &HashMap) { + let fresh: Vec = { + let mut inflight = self.thumb_gen.lock(); + uids.into_iter() + .filter(|uid| inflight.insert(uid.clone())) + .collect() + }; + if fresh.is_empty() { + return; + } + + let core = self.clone(); + let tags = tags.clone(); + // `generate_thumbs` blocks on the runtime itself, so it belongs on the + // blocking pool rather than on an async worker. + self.rt.spawn_blocking(move || { + core.generate_thumbs(&fresh, &tags); + let mut inflight = core.thumb_gen.lock(); + for uid in &fresh { + inflight.remove(uid); + } + }); + } + + /// Make thumbnails for photos the server has none for: download each full + /// file once, scale it to [`THUMB_EDGE`], and store the result in the thumbnail + /// cache exactly as if the server had served it. + /// + /// Bounded by [`THUMB_GEN_CONCURRENCY`] — these are full-size originals, and a + /// screenful of them at once would saturate the link. A photo whose bytes + /// can't be decoded (a codec `image` doesn't speak) is marked + /// [`db::THUMB_NONE`] and never attempted again. + pub(crate) fn generate_thumbs(&self, uids: &[NodeUid], tags: &HashMap) { + info!( + count = uids.len(), + "generating thumbnails the server has none for" + ); + let results: Vec<(NodeUid, ThumbAttempt)> = self.rt.block_on(async { + let mut out = Vec::with_capacity(uids.len()); + for chunk in uids.chunks(THUMB_GEN_CONCURRENCY) { + let mut set = tokio::task::JoinSet::new(); + for uid in chunk { + let client = self.client.clone(); + let uid = uid.clone(); + set.spawn(async move { + let bytes = match client.download_file(&uid).await { + Ok(bytes) => bytes, + Err(e) => { + warn!(%uid, error = %e, "photo download for thumbnail failed"); + return (uid, ThumbAttempt::Unavailable); + } + }; + // Decoding + scaling a 20 MP JPEG is CPU-bound and would + // stall the runtime's worker; hand it to the blocking pool. + let made = tokio::task::spawn_blocking(move || scale_thumbnail(&bytes)) + .await + .unwrap_or(None); + match made { + Some(thumb) => (uid, ThumbAttempt::Made(thumb)), + None => (uid, ThumbAttempt::Undecodable), + } + }); + } + while let Some(joined) = set.join_next().await { + if let Ok(result) = joined { + out.push(result); + } + } + } + out + }); + + let ttype = ThumbnailType::Thumbnail.as_i32(); + for (uid, attempt) in results { + match attempt { + ThumbAttempt::Made(thumb) => { + let Some(&tag) = tags.get(&uid.to_string()) else { + continue; + }; + match self.cache.store_thumbnail(&uid, ttype, tag, &thumb.bytes) { + Ok(()) => self.record_thumb(&uid, db::THUMB_HAVE, Some(thumb.ratio)), + Err(e) => warn!(%uid, error = %e, "storing generated thumbnail failed"), + } + } + // The photo's own bytes aren't an image we can read: no thumbnail + // will ever exist for it, so stop trying. + ThumbAttempt::Undecodable => self.record_thumb(&uid, db::THUMB_NONE, None), + // The download failed — a dropped connection, an expired link. That + // is not a verdict on the photo: leave its state alone so the next + // scroll past it tries again. + ThumbAttempt::Unavailable => {} + } + } + } + + /// Persist what a thumbnail attempt learned about a photo. + pub(crate) fn record_thumb(&self, uid: &NodeUid, state: i64, ratio: Option) { + if let Err(e) = self.db.photo_set_thumb(&uid.to_string(), state, ratio) { + warn!(%uid, error = %e, "recording thumbnail state failed"); + } + } + + /// Whether the listing stamped under `key` is older than `ttl` (or was never + /// fetched). + pub(crate) async fn refresh_timeline(&self) -> Result { + let photos = self.photos(); + if photos + .get_photos_root() + .await + .map_err(|e| format!("photos root: {e}"))? + .is_none() + { + let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 0); + let _ = self.db.set_state_i64(PHOTOS_SYNCED_MS, now_ms()); + return Ok(false); + } + let items = photos + .enumerate_timeline() + .await + .map_err(|e| format!("timeline: {e}"))?; + + // The timeline DTO carries only a uid and capture time, but the Photos + // page has to split into Photos / Videos / Raw — which needs each photo's + // name and media type. Resolve those in batches off the request path. + // Best-effort: a photo whose node we fail to resolve keeps whatever was + // learned before (or classifies from nothing, i.e. a still photo), so a + // partial resolve never blanks the timeline. + let uids: Vec = items.iter().map(|it| it.uid.clone()).collect(); + let mut meta: HashMap, Option)> = HashMap::new(); + for chunk in uids.chunks(TIMELINE_ENRICH_CHUNK) { + match photos.enumerate_nodes(chunk).await { + Ok(nodes) => { + for node in nodes { + let media_type = match &node.kind { + NodeKind::File { media_type, .. } => Some(media_type.clone()), + NodeKind::Folder => None, + }; + meta.insert(node.uid.to_string(), (Some(node.name), media_type)); + } + } + Err(e) => warn!(error = %e, "resolving photo metadata for a timeline chunk failed"), + } + } + + let rows: Vec<(String, i64, Option, Option)> = items + .iter() + .map(|it| { + let key = it.uid.to_string(); + let (name, media_type) = meta.get(&key).cloned().unwrap_or((None, None)); + (key, it.capture_time, name, media_type) + }) + .collect(); + self.db.photos_replace(&rows).map_err(|e| e.to_string())?; + let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 1); + let _ = self.db.set_state_i64(PHOTOS_SYNCED_MS, now_ms()); + Ok(true) + } + + /// Refresh the timeline off the request path, so a stale page is still served + /// at DB speed. At most one refresh runs at a time. + pub(crate) fn spawn_timeline_refresh(&self) { + if self.timeline_refreshing.swap(true, Ordering::SeqCst) { + return; + } + let core = self.clone(); + self.rt.spawn(async move { + if let Err(e) = core.refresh_timeline().await { + warn!(error = %e, "background timeline refresh failed"); + } + core.timeline_refreshing.store(false, Ordering::SeqCst); + }); + } + + /// Download a photo's full content into the content cache, returning its + /// on-disk path (served from cache when a fresh blob already exists). + pub(crate) fn open_photo(&self, uid: &NodeUid) -> Result { + let photos = self.photos(); + let node = self + .rt + .block_on(photos.get_node(uid)) + .map_err(|e| format!("photo node: {e}"))? + .ok_or("photo not found")?; + let (mtime, size) = (node.modification_time, node_size(&node)); + if let Some(p) = self.cache.cached_content_path(uid, mtime, size) { + return Ok(p); + } + let bytes = self + .download_photo_tracked(&photos, uid, &node.name, size) + .map_err(|e| format!("download photo: {e}"))?; + self.cache + .store(uid, mtime, size, &bytes) + .map_err(|e| format!("cache store: {e}"))?; + Ok(self.cache.content_path(uid)) + } + +} diff --git a/crates/pdfs-fuse/src/sharing.rs b/crates/pdfs-fuse/src/sharing.rs new file mode 100644 index 0000000..e638f1e --- /dev/null +++ b/crates/pdfs-fuse/src/sharing.rs @@ -0,0 +1,410 @@ +//! Sharing: per-node share membership, public links, the invitations this +//! account has received, saved public-link bookmarks, and the reverse view of +//! everything this account shares out. +//! +//! All of it is remote-only state — nothing here touches the inode maps or the +//! content cache, so every method is a round-trip wrapped in the crate error +//! shape the control socket expects. + +use std::path::Path; + +use pdfs_core::control::{ + BookmarkInfo, DirEntry, InvitationInfo, PublicLinkInfo, ShareEntry, ShareEntryKind, SharedItem, +}; +use proton_drive_rs::{MemberRole, NodeKind}; +use proton_drive_rs::proton_sdk::ids::NodeUid; + +use super::{Core, ROOT_INO, parse_uid, public_link_info, role_from_str, role_to_str}; + +impl Core { + // ---- sharing a node --------------------------------------------------- + + /// Invite Proton and/or external emails to the node at `rel` at `role`. + /// Returns `(proton_invited, external_invited)`. + pub(crate) fn share_node( + &self, + rel: &Path, + emails: &[String], + role: &str, + message: Option<&str>, + ) -> Result<(usize, usize), String> { + let (_ino, uid) = self + .resolve_path(rel) + .map_err(|e| format!("resolve path: {e:?}"))?; + let role = role_from_str(role)?; + let invitees: Vec<(String, MemberRole)> = + emails.iter().map(|e| (e.clone(), role)).collect(); + self.rt + .block_on(self.client.invite_users(&uid, &invitees, message)) + .map_err(|e| format!("share: {e}")) + } + + /// List the members, pending invitations and public link of the node at `rel`. + pub(crate) fn list_share(&self, rel: &Path) -> Result<(Vec, Option), String> { + let (_ino, uid) = self + .resolve_path(rel) + .map_err(|e| format!("resolve path: {e:?}"))?; + + let mut entries = Vec::new(); + for m in self + .rt + .block_on(self.client.list_share_members(&uid)) + .map_err(|e| format!("list members: {e}"))? + { + entries.push(ShareEntry { + id: m.membership_id.to_string(), + email: m.email, + role: role_to_str(m.role).to_string(), + kind: ShareEntryKind::Member, + }); + } + for inv in self + .rt + .block_on(self.client.list_share_invitations(&uid)) + .map_err(|e| format!("list invitations: {e}"))? + { + entries.push(ShareEntry { + id: inv.invitation_id, + email: inv.invitee_email, + role: role_to_str(inv.role).to_string(), + kind: ShareEntryKind::ProtonInvite, + }); + } + for ext in self + .rt + .block_on(self.client.list_external_invitations(&uid)) + .map_err(|e| format!("list external invitations: {e}"))? + { + entries.push(ShareEntry { + id: ext.invitation_id, + email: ext.invitee_email, + role: role_to_str(ext.role).to_string(), + kind: ShareEntryKind::ExternalInvite, + }); + } + + let link = self + .rt + .block_on(self.client.get_public_link(&uid)) + .map_err(|e| format!("get public link: {e}"))? + .map(public_link_info); + + Ok((entries, link)) + } + + /// Change the role of a member or pending Proton invitation on the node at + /// `rel`. External invitations have no role-update endpoint. + pub(crate) fn update_share_role( + &self, + rel: &Path, + id: &str, + kind: ShareEntryKind, + role: &str, + ) -> Result<(), String> { + let (_ino, uid) = self + .resolve_path(rel) + .map_err(|e| format!("resolve path: {e:?}"))?; + let role = role_from_str(role)?; + match kind { + ShareEntryKind::Member => { + let member = self + .rt + .block_on(self.client.list_share_members(&uid)) + .map_err(|e| format!("list members: {e}"))? + .into_iter() + .find(|m| m.membership_id.to_string() == id) + .ok_or_else(|| "member not found".to_string())?; + self.rt + .block_on(self.client.update_member_role(&member, role)) + .map_err(|e| format!("update role: {e}")) + } + ShareEntryKind::ProtonInvite => { + let inv = self + .rt + .block_on(self.client.list_share_invitations(&uid)) + .map_err(|e| format!("list invitations: {e}"))? + .into_iter() + .find(|i| i.invitation_id == id) + .ok_or_else(|| "invitation not found".to_string())?; + self.rt + .block_on(self.client.update_invitation_role(&inv, role)) + .map_err(|e| format!("update role: {e}")) + } + ShareEntryKind::ExternalInvite => { + Err("an external invitation's role cannot be changed".to_string()) + } + } + } + + /// Remove a member, pending Proton invite, or external invite from the node + /// at `rel`. + pub(crate) fn remove_share_entry(&self, rel: &Path, id: &str, kind: ShareEntryKind) -> Result<(), String> { + let (_ino, uid) = self + .resolve_path(rel) + .map_err(|e| format!("resolve path: {e:?}"))?; + match kind { + ShareEntryKind::Member => { + let member = self + .rt + .block_on(self.client.list_share_members(&uid)) + .map_err(|e| format!("list members: {e}"))? + .into_iter() + .find(|m| m.membership_id.to_string() == id) + .ok_or_else(|| "member not found".to_string())?; + self.rt + .block_on(self.client.remove_member(&member)) + .map_err(|e| format!("remove member: {e}")) + } + ShareEntryKind::ProtonInvite => { + let inv = self + .rt + .block_on(self.client.list_share_invitations(&uid)) + .map_err(|e| format!("list invitations: {e}"))? + .into_iter() + .find(|i| i.invitation_id == id) + .ok_or_else(|| "invitation not found".to_string())?; + self.rt + .block_on(self.client.delete_invitation(&inv)) + .map_err(|e| format!("revoke invitation: {e}")) + } + ShareEntryKind::ExternalInvite => { + let ext = self + .rt + .block_on(self.client.list_external_invitations(&uid)) + .map_err(|e| format!("list external invitations: {e}"))? + .into_iter() + .find(|i| i.invitation_id == id) + .ok_or_else(|| "external invitation not found".to_string())?; + self.rt + .block_on(self.client.delete_external_invitation(&ext)) + .map_err(|e| format!("revoke external invitation: {e}")) + } + } + } + + /// Create a public link on the node at `rel`, returning its info (with URL). + pub(crate) fn create_public_link( + &self, + rel: &Path, + role: &str, + password: Option<&str>, + expires: Option, + ) -> Result { + let (_ino, uid) = self + .resolve_path(rel) + .map_err(|e| format!("resolve path: {e:?}"))?; + let role = role_from_str(role)?; + let link = self + .rt + .block_on( + self.client + .create_public_link(&uid, role, password, expires), + ) + .map_err(|e| format!("create public link: {e}"))?; + Ok(public_link_info(link)) + } + + /// Remove the public link `id` from the node at `rel`. + pub(crate) fn remove_public_link(&self, rel: &Path, id: &str) -> Result<(), String> { + let (_ino, uid) = self + .resolve_path(rel) + .map_err(|e| format!("resolve path: {e:?}"))?; + let link = self + .rt + .block_on(self.client.get_public_link(&uid)) + .map_err(|e| format!("get public link: {e}"))? + .filter(|l| l.public_link_id == id) + .ok_or_else(|| "public link not found".to_string())?; + self.rt + .block_on(self.client.remove_public_link(&link)) + .map_err(|e| format!("remove public link: {e}")) + } + + // ---- shared with me --------------------------------------------------- + + /// List nodes shared with me that I have accepted. + pub(crate) fn list_shared_with_me(&self) -> Result, String> { + let uids = self + .rt + .block_on(self.client.enumerate_shared_with_me_node_uids()) + .map_err(|e| format!("enumerate shared: {e}"))?; + if uids.is_empty() { + return Ok(Vec::new()); + } + let nodes = self + .rt + .block_on(self.client.enumerate_nodes(&uids)) + .map_err(|e| format!("enumerate nodes: {e}"))?; + Ok(nodes + .into_iter() + .map(|n| { + let is_dir = n.is_folder(); + let size = match &n.kind { + NodeKind::File { + claimed_size, + total_size_on_storage, + .. + } => claimed_size.unwrap_or(*total_size_on_storage).max(0) as u64, + NodeKind::Folder => 0, + }; + DirEntry { + name: n.name, + is_dir, + size, + modified: 0, + pinned: false, + cached: false, + uid: n.uid.to_string(), + path: String::new(), + } + }) + .collect()) + } + + /// Leave a shared node by its uid. + pub(crate) fn leave_shared(&self, uid: &str) -> Result<(), String> { + let uid = parse_uid(uid).ok_or_else(|| format!("invalid uid: {uid}"))?; + self.rt + .block_on(self.client.leave_shared_node(&uid)) + .map_err(|e| format!("leave shared: {e}")) + } + + // ---- incoming invitations --------------------------------------------- + + /// List invitations addressed to me, pending accept or reject. + pub(crate) fn list_invitations(&self) -> Result, String> { + let invitations = self + .rt + .block_on(self.client.list_incoming_invitations()) + .map_err(|e| format!("list invitations: {e}"))?; + Ok(invitations + .into_iter() + .map(|i| InvitationInfo { + id: i.invitation_id, + inviter_email: i.inviter_email, + name: i.node_name, + is_dir: i.is_folder, + }) + .collect()) + } + + /// Accept an invitation addressed to me by its id. + pub(crate) fn accept_invitation(&self, id: &str) -> Result<(), String> { + self.rt + .block_on(self.client.accept_invitation(id)) + .map_err(|e| format!("accept invitation: {e}")) + } + + /// Reject an invitation addressed to me by its id. + pub(crate) fn reject_invitation(&self, id: &str) -> Result<(), String> { + self.rt + .block_on(self.client.reject_invitation(id)) + .map_err(|e| format!("reject invitation: {e}")) + } + + // ---- bookmarks -------------------------------------------------------- + + /// List public links saved to my account. + pub(crate) fn list_bookmarks(&self) -> Result, String> { + let bookmarks = self + .rt + .block_on(self.client.list_bookmarks()) + .map_err(|e| format!("list bookmarks: {e}"))?; + Ok(bookmarks + .into_iter() + .map(|b| BookmarkInfo { + token: b.token, + url: b.url, + name: b.node_name, + is_dir: b.is_folder, + }) + .collect()) + } + + /// Save a public link (optionally password-protected) as a bookmark. + pub(crate) fn create_bookmark(&self, url: &str, password: Option<&str>) -> Result<(), String> { + self.rt + .block_on(self.client.create_bookmark(url, password)) + .map_err(|e| format!("create bookmark: {e}")) + } + + /// Remove a saved bookmark by its token. + pub(crate) fn delete_bookmark(&self, token: &str) -> Result<(), String> { + self.rt + .block_on(self.client.delete_bookmark(token)) + .map_err(|e| format!("delete bookmark: {e}")) + } + + // ---- shared by me ----------------------------------------------------- + + /// List the nodes I have shared with others, each with a summary of its share + /// state (members, pending invitations, public link). One list call enumerates + /// the shared uids; the per-node detail is then gathered best-effort — a single + /// node racing with an unshare drops from the list rather than failing the whole + /// request. + pub(crate) fn list_shared_by_me(&self) -> Result, String> { + let uids = self + .rt + .block_on(self.client.enumerate_shared_by_me_node_uids()) + .map_err(|e| format!("enumerate shared-by-me: {e}"))?; + if uids.is_empty() { + return Ok(Vec::new()); + } + let nodes = self + .rt + .block_on(self.client.enumerate_nodes(&uids)) + .map_err(|e| format!("enumerate nodes: {e}"))?; + let mut items = Vec::with_capacity(nodes.len()); + for n in nodes { + let uid = n.uid.clone(); + let members = self + .rt + .block_on(self.client.list_share_members(&uid)) + .map(|m| m.len()) + .unwrap_or(0); + let proton_invites = self + .rt + .block_on(self.client.list_share_invitations(&uid)) + .map(|i| i.len()) + .unwrap_or(0); + let external_invites = self + .rt + .block_on(self.client.list_external_invitations(&uid)) + .map(|i| i.len()) + .unwrap_or(0); + let link = self + .rt + .block_on(self.client.get_public_link(&uid)) + .ok() + .flatten() + .map(public_link_info); + items.push(SharedItem { + uid: uid.to_string(), + is_dir: n.is_folder(), + name: n.name, + path: self.rel_path_for_uid(&uid).unwrap_or_default(), + member_count: members, + invite_count: proton_invites + external_invites, + link, + }); + } + Ok(items) + } + + /// Best-effort mountpoint-relative path for a node already interned in the live + /// tree, by walking parent inodes to the root. `None` when the node has never + /// been seen through the mount (e.g. shared but not browsed to this session) — + /// the caller then leaves the path empty. + pub(crate) fn rel_path_for_uid(&self, uid: &NodeUid) -> Option { + let st = self.state.lock(); + let mut ino = *st.by_uid.get(uid)?; + let mut parts = Vec::new(); + while ino != ROOT_INO { + let entry = st.entries.get(&ino)?; + parts.push(entry.node.name.clone()); + ino = entry.parent; + } + parts.reverse(); + Some(parts.join("/")) + } +} diff --git a/crates/pdfs-fuse/src/state.rs b/crates/pdfs-fuse/src/state.rs new file mode 100644 index 0000000..b074a6e --- /dev/null +++ b/crates/pdfs-fuse/src/state.rs @@ -0,0 +1,300 @@ +//! Inode bookkeeping: the maps that give a remote node a stable kernel inode, +//! the open write handles, and the interval set that tracks which bytes of a +//! staged write are authored locally. + +use std::collections::HashMap; +use std::fs::File; +use std::path::PathBuf; +use std::sync::Arc; + +use pdfs_core::cache::StagedWrite; +use pdfs_core::db::Db; +use proton_drive_rs::{Node, NodeKind}; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use tracing::warn; + + + +/// A node known to the filesystem, addressed by its kernel inode. +pub(crate) struct Entry { + pub(crate) uid: NodeUid, + pub(crate) parent: u64, + pub(crate) node: Node, +} + +/// A set of non-overlapping `[start, end)` byte ranges, kept sorted and merged. +/// Tracks which bytes of a [`WriteHandle`]'s scratch file were authored locally +/// (vs. still living only in the remote base), so reads and the commit gap-fill +/// know which regions to pull from the network. +#[derive(Clone, Default)] +pub(crate) struct Intervals(pub(crate) Vec<(u64, u64)>); + +impl Intervals { + /// Mark `[start, end)` as authored, coalescing with any touching ranges. + pub(crate) fn add(&mut self, start: u64, end: u64) { + if start >= end { + return; + } + self.0.push((start, end)); + self.0.sort_by_key(|&(s, _)| s); + let mut merged: Vec<(u64, u64)> = Vec::with_capacity(self.0.len()); + for &(s, e) in &self.0 { + match merged.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => merged.push((s, e)), + } + } + self.0 = merged; + } + + /// Drop everything at or beyond `len` (a shrink/truncate). + pub(crate) fn clip(&mut self, len: u64) { + self.0.retain(|&(s, _)| s < len); + for iv in &mut self.0 { + iv.1 = iv.1.min(len); + } + } + + /// Split `[start, end)` into contiguous `(s, e, authored)` segments, in + /// order. `authored == true` means the bytes live in the scratch file; + /// `false` means they must come from the remote base (or are a hole). + pub(crate) fn segments(&self, start: u64, end: u64) -> Vec<(u64, u64, bool)> { + let mut out = Vec::new(); + let mut pos = start; + for &(s, e) in &self.0 { + if e <= start { + continue; + } + if s >= end { + break; + } + let ws = s.max(start); + let we = e.min(end); + if pos < ws { + out.push((pos, ws, false)); + } + out.push((ws, we, true)); + pos = we; + } + if pos < end { + out.push((pos, end, false)); + } + out + } +} + +/// State for a file opened for writing. Authored bytes are staged in an on-disk +/// scratch file (positional reads/writes) rather than RAM, so a multi-GiB write +/// never balloons the daemon. On flush/release the scratch file — gap-filled +/// from the remote base where untouched — is streamed up as one new revision, +/// since the SDK seals whole revisions rather than byte ranges. +pub(crate) struct WriteHandle { + pub(crate) ino: u64, + pub(crate) uid: NodeUid, + /// Disk-backed staging buffer. Shared (`Arc`) so reads can use it without + /// holding the state lock across I/O. Accessed positionally (`read_at`/ + /// `write_at`), so a clone never disturbs another's file offset. + pub(crate) file: Arc, + /// Scratch file path, removed on release. + pub(crate) path: PathBuf, + /// Byte ranges authored into `file`. Everything else in `[0, len)` is base. + pub(crate) written: Intervals, + /// Logical file size (may exceed authored bytes after a truncate-extend). + pub(crate) len: u64, + /// Size of the remote base at open, for serving untouched ranges. + pub(crate) base_size: u64, + /// Modification time of the remote base at open, validating its block cache. + pub(crate) base_mtime: i64, + /// Whether anything diverged from the remote and needs an upload. + pub(crate) dirty: bool, +} + +/// A released write whose upload has not happened yet (offline.md Phase 3). +/// +/// The bytes live in the content cache's staging dir and the intent lives in the +/// `pending_op` table; this pairs them in memory so a read can be served without +/// a database round trip. +#[derive(Clone)] +pub(crate) struct PendingRevision { + /// Staged blob holding the written bytes. + pub(crate) path: PathBuf, + /// Which of those bytes are real, and what base the gaps refer to. + pub(crate) meta: StagedWrite, +} + +/// Mutable inode bookkeeping, guarded by a mutex because fuser drives the +/// `Filesystem` trait through `&self`. +pub(crate) struct State { + /// inode -> node metadata. + pub(crate) entries: HashMap, + /// Dedupe inodes by node uid so a node keeps a stable inode across lookups. + pub(crate) by_uid: HashMap, + /// Cached directory listings: parent inode -> child inodes. Presence of a + /// key means the directory has been enumerated. + pub(crate) children: HashMap>, + pub(crate) next_ino: u64, + /// Open write handles keyed by file handle id. Read-only opens use fh 0 and + /// have no entry here. + pub(crate) handles: HashMap, + pub(crate) next_fh: u64, + /// Unified SQLite metadata cache. Every map mutation below writes through to + /// it inside the `State` lock so the DB stays the authoritative copy across + /// restarts (see plan.md P1). + pub(crate) db: Arc, +} + +impl State { + pub(crate) fn intern(&mut self, parent: u64, node: Node) -> u64 { + if let Err(e) = self.db.upsert_node(&node) { + warn!(uid = %node.uid, error = %e, "db upsert_node failed"); + } + self.intern_mem(parent, node) + } + + /// Allocate (or reuse) a stable inode for a node, updating the hot-cache maps + /// only. Every caller owes the DB a write-through — see the callers below; + /// the split exists so a batch can pay for one transaction instead of `n`. + pub(crate) fn intern_mem(&mut self, parent: u64, node: Node) -> u64 { + if let Some(&ino) = self.by_uid.get(&node.uid) { + if let Some(e) = self.entries.get_mut(&ino) { + e.node = node; + e.parent = parent; + } + return ino; + } + let ino = self.next_ino; + self.next_ino += 1; + self.by_uid.insert(node.uid.clone(), ino); + self.entries.insert( + ino, + Entry { + uid: node.uid.clone(), + parent, + node, + }, + ); + ino + } + + /// Allocate (or reuse) a stable inode for a node that came *from* the + /// database, which is why nothing is written back. + pub(crate) fn intern_from_db(&mut self, parent: u64, node: Node) -> u64 { + self.intern_mem(parent, node) + } + + /// Allocate (or reuse) stable inodes for a whole listing, writing every node + /// through in a single transaction. + /// + /// This is what keeps `ls` on a large folder quick: one commit for the + /// listing, rather than one autocommit — and one fsync — per child. + pub(crate) fn intern_batch(&mut self, parent: u64, nodes: Vec) -> Vec { + if let Err(e) = self.db.upsert_nodes(&nodes) { + warn!(error = %e, "db upsert_nodes failed"); + } + nodes + .into_iter() + .map(|node| self.intern_mem(parent, node)) + .collect() + } + + /// Forget a node entirely: drop its inode, its uid mapping, its own cached + /// listing, its slot in its parent's listing, and its DB row. Returns + /// `(parent_ino, name)` when the node was known, so the caller can notify + /// the kernel. + pub(crate) fn forget(&mut self, uid: &NodeUid) -> Option<(u64, String)> { + let ino = self.by_uid.remove(uid)?; + if let Err(e) = self.db.delete_node(uid) { + warn!(%uid, error = %e, "db delete_node failed"); + } + let entry = self.entries.remove(&ino)?; + self.children.remove(&ino); + if let Some(kids) = self.children.get_mut(&entry.parent) { + kids.retain(|&k| k != ino); + } + Some((entry.parent, entry.node.name)) + } + + /// Move a node to a new parent and/or name within the tree, writing it + /// through like any other mutation. + /// + /// The online rename instead forgets the node and lets the destination + /// re-enumerate, which is the cheaper way to stay honest about what the + /// server did. A queued rename cannot: re-enumerating needs the network, and + /// the server has not been told yet in any case — so this *is* the tree's + /// new truth until the op drains (offline.md Phase 3b). + pub(crate) fn rename_in_place(&mut self, ino: u64, new_parent: u64, new_parent_uid: &NodeUid, name: &str) { + let Some(entry) = self.entries.get_mut(&ino) else { + return; + }; + let old_parent = entry.parent; + entry.parent = new_parent; + entry.node.name = name.to_string(); + entry.node.parent_uid = Some(new_parent_uid.clone()); + let node = entry.node.clone(); + if old_parent != new_parent { + if let Some(kids) = self.children.get_mut(&old_parent) { + kids.retain(|&k| k != ino); + } + // Only if the destination is listed: pushing into a listing that was + // never enumerated would invent a one-child folder. + if let Some(kids) = self.children.get_mut(&new_parent) + && !kids.contains(&ino) + { + kids.push(ino); + } + } + if let Err(e) = self.db.upsert_node(&node) { + warn!(uid = %node.uid, error = %e, "db upsert_node failed for a queued rename"); + } + } + + /// Drop a directory's cached child listing and mark it unlisted in the DB, + /// so the next access re-enumerates instead of trusting a stale listing. + pub(crate) fn invalidate_listing(&mut self, ino: u64) { + if self.children.remove(&ino).is_none() { + return; + } + if let Some(e) = self.entries.get(&ino) { + let uid = e.uid.clone(); + if let Err(err) = self.db.set_listed(&uid, false) { + warn!(%uid, error = %err, "db set_listed(false) failed"); + } + } + } + + /// Update a file entry's recorded plaintext size so `getattr` reflects an + /// in-progress write before the new revision is sealed. + pub(crate) fn set_size(&mut self, ino: u64, size: u64) { + if let Some(e) = self.entries.get_mut(&ino) + && let NodeKind::File { claimed_size, .. } = &mut e.node.kind + { + *claimed_size = Some(size as i64); + } + } + + /// Update a file entry's modification time (epoch seconds). + pub(crate) fn touch_mtime(&mut self, ino: u64, secs: i64) { + if let Some(e) = self.entries.get_mut(&ino) { + e.node.modification_time = secs; + } + } + + /// Record the size and mtime of a write that has been accepted but not yet + /// uploaded, persisting them like any other node mutation. + /// + /// The write-through is what makes the new size outlive the process: until + /// the op drains, the remote still holds the old revision (or, for a node + /// created offline, nothing at all), so this row is the only record that the + /// file is as long as the caller was told it is. Without it a restart serves + /// the stale size and the file reads as truncated — or empty — while its + /// bytes sit safely in staging (offline.md Phase 3). + pub(crate) fn record_pending_write(&mut self, ino: u64, size: u64, mtime: i64) { + self.set_size(ino, size); + self.touch_mtime(ino, mtime); + if let Some(e) = self.entries.get(&ino) + && let Err(err) = self.db.upsert_node(&e.node) + { + warn!(uid = %e.uid, error = %err, "db upsert_node failed for a queued write"); + } + } +} diff --git a/crates/pdfs-fuse/src/workers.rs b/crates/pdfs-fuse/src/workers.rs new file mode 100644 index 0000000..bad015f --- /dev/null +++ b/crates/pdfs-fuse/src/workers.rs @@ -0,0 +1,78 @@ +//! The thread pool that serves the FUSE handlers which touch the network. +//! +//! fuser drives `Filesystem` from a single dispatch loop, so a cold read that +//! goes to the wire would block every cheap metadata call behind it. Those +//! handlers hand their work here instead and reply from the worker. + +use std::sync::Arc; + +use parking_lot::Mutex; +use tracing::warn; + +/// How many FUSE handlers may run off the dispatch loop at once. +/// +/// `fuser`'s session loop is non-concurrent: it reads one request, calls the +/// handler, and only then reads the next. A handler that touches the network +/// therefore stalls every `getattr`/`lookup` on the mount behind it. The slow +/// handlers hand their `Reply` to this pool instead and answer from a worker, +/// which frees the loop immediately. +/// +/// Bounded on purpose: one worker can hold a 4 MiB block in flight, and an +/// unbounded pool would let read-ahead on a big file spawn threads without +/// limit. Sized just above the SDK's in-flight block cap so the semaphore +/// there, not thread count, is what bounds download memory. +pub(crate) const FUSE_WORKERS: usize = 8; + +/// Bounded thread pool behind the network-touching FUSE handlers. +/// +/// Shared by every session forked off one [`Core`] (the main mount plus each +/// on-demand sync folder), so the bound is per daemon rather than per mount. +pub(crate) struct Workers { + pub(crate) tx: std::sync::mpsc::Sender>, +} + +impl Workers { + pub(crate) fn new(n: usize) -> std::io::Result { + let (tx, rx) = std::sync::mpsc::channel::>(); + let rx = Arc::new(Mutex::new(rx)); + for i in 0..n { + let rx = rx.clone(); + std::thread::Builder::new() + .name(format!("pdfs-fuse-{i}")) + .spawn(move || { + loop { + // Held only to take the next job, never across it. + let job = rx.lock().recv(); + match job { + // A panicking handler must not cost the pool a + // worker for the rest of the run. The dropped + // `Reply` answers EIO on its own, so the caller of + // the failed op is told; the next job is unaffected. + // + // This only holds because the shared state is behind + // `parking_lot` mutexes: a `std` mutex held at the + // point of the panic would come back poisoned, and + // the worker we just rescued would die on its next + // acquisition — as would every other thread. + Ok(job) => { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job)); + } + // Every sender is gone: the daemon is shutting down. + Err(_) => break, + } + } + })?; + } + Ok(Self { tx }) + } + + /// Run `job` on a worker. Falls back to running it inline — the pre-pool + /// behaviour — if no worker is left to take it, so a shut-down pool degrades + /// to a slow mount rather than a mount that answers every read with EIO. + pub(crate) fn run(&self, job: impl FnOnce() + Send + 'static) { + if let Err(e) = self.tx.send(Box::new(job)) { + warn!("fuse worker pool is gone; serving inline"); + (e.0)(); + } + } +}