From bb9804498ce75dd75485c65f0c4cab79456d5973 Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Thu, 16 Jul 2026 18:13:48 +0200 Subject: [PATCH] added new folder icon, fixed db size (max log size), progress bars --- Cargo.lock | 84 +- crates/pdfs-cli/src/main.rs | 122 +- crates/pdfs-core/src/control.rs | 127 +- crates/pdfs-core/src/db.rs | 445 ++++- crates/pdfs-fuse/Cargo.toml | 1 + crates/pdfs-fuse/src/lib.rs | 856 +++++++++- crates/pdfs-fuse/src/sync.rs | 1457 +++++++++++++++++ crates/pdfs-fuse/src/transfers.rs | 122 +- crates/pdfs-gui/build.rs | 28 + .../actions/pdfs-folder-upload-symbolic.svg | 7 + crates/pdfs-gui/resources/pdfs.gresource.xml | 9 + crates/pdfs-gui/src/app.rs | 619 ++++++- crates/pdfs-gui/src/main.rs | 18 +- 13 files changed, 3729 insertions(+), 166 deletions(-) create mode 100644 crates/pdfs-fuse/src/sync.rs create mode 100644 crates/pdfs-gui/build.rs create mode 100644 crates/pdfs-gui/resources/icons/scalable/actions/pdfs-folder-upload-symbolic.svg create mode 100644 crates/pdfs-gui/resources/pdfs.gresource.xml diff --git a/Cargo.lock b/Cargo.lock index 5835873..1bf3bee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1671,6 +1671,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "funty" version = "2.0.0" @@ -2559,6 +2568,26 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "inotify" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "153be1941a183ec9ccd095ddbe17a8b8d435ef6c76e9e02451b933c3999af2c8" +dependencies = [ + "bitflags 2.13.0", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.4" @@ -2725,6 +2754,26 @@ dependencies = [ "zeroize", ] +[[package]] +name = "kqueue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.13.0", + "libc", +] + [[package]] name = "ksni" version = "0.2.2" @@ -2959,6 +3008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -3035,6 +3085,33 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.13.0", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -3395,6 +3472,7 @@ dependencies = [ "fuser", "image", "libc", + "notify", "pdfs-core", "proton-drive-rs", "serde_json", @@ -3664,7 +3742,7 @@ dependencies = [ [[package]] name = "proton-drive-rs" -version = "0.1.6" +version = "0.1.7" dependencies = [ "async-trait", "base64", @@ -3688,7 +3766,7 @@ dependencies = [ [[package]] name = "proton-sdk" -version = "0.1.6" +version = "0.1.7" dependencies = [ "aes-gcm 0.11.0", "async-trait", @@ -3804,7 +3882,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/crates/pdfs-cli/src/main.rs b/crates/pdfs-cli/src/main.rs index cb242e9..5ea80aa 100644 --- a/crates/pdfs-cli/src/main.rs +++ b/crates/pdfs-cli/src/main.rs @@ -10,7 +10,9 @@ use clap::{Parser, Subcommand}; use pdfs_core::auth; use pdfs_core::cache::ContentCache; use pdfs_core::config::AppDirs; -use pdfs_core::control::{Request as CtlRequest, Response as CtlResponse, ShareEntryKind}; +use pdfs_core::control::{ + Request as CtlRequest, Response as CtlResponse, ShareEntryKind, SyncPhase, +}; use pdfs_core::db::Db; #[derive(Parser)] @@ -149,6 +151,11 @@ enum Command { #[command(subcommand)] action: DeviceCmd, }, + /// Sync local folders to this machine's Proton Drive device. + Sync { + #[command(subcommand)] + action: SyncCmd, + }, /// Share a file or folder with Proton and/or external email addresses. Share { /// File/folder path, inside the mountpoint or relative to it. @@ -251,6 +258,38 @@ enum DeviceCmd { Rm { uid: String }, } +#[derive(Subcommand)] +enum SyncCmd { + /// Add a local folder to this machine's device and upload its contents. + Add { + /// Local directory to sync. + path: PathBuf, + }, + /// List this machine's synced folders. + List, + /// Remove a synced folder by id. + Rm { + /// Synced-folder id (from `sync list`). + id: i64, + /// Also delete the folder's copy in Proton Drive. + #[arg(long)] + delete_remote: bool, + }, + /// Force a sync pass now (all folders, or one by id). + Now { + /// Optional synced-folder id; omit to reconcile every folder. + id: Option, + }, + /// Switch a folder between full-copy sync and on-demand (FUSE, no local storage). + Mode { + /// Synced-folder id (from `sync list`). + id: i64, + /// `mirror` (full local copy) or `ondemand` (FUSE mount, reclaims disk). + #[arg(value_parser = ["mirror", "ondemand"])] + mode: String, + }, +} + #[derive(Subcommand)] enum PublicLinkCmd { /// Create a public link on a node. @@ -338,6 +377,7 @@ fn main() -> Result<()> { Command::DeleteForever { uids } => cmd_delete_forever(uids), Command::EmptyTrash => cmd_empty_trash(), Command::Devices { action } => cmd_devices(action), + Command::Sync { action } => cmd_sync(action), Command::Share { path, emails, @@ -389,6 +429,63 @@ fn cmd_devices(action: DeviceCmd) -> Result<()> { Ok(()) } +fn cmd_sync(action: SyncCmd) -> Result<()> { + match action { + SyncCmd::Add { path } => { + let abs = std::fs::canonicalize(&path) + .with_context(|| format!("resolve {}", path.display()))?; + let local_path = abs + .to_str() + .ok_or_else(|| anyhow!("path is not valid UTF-8"))? + .to_string(); + ok_or_bail(control_request(CtlRequest::AddSyncFolder { local_path })?)? + } + SyncCmd::List => match control_request(CtlRequest::ListSyncFolders)? { + CtlResponse::SyncFolders { items } if items.is_empty() => { + println!("No synced folders.") + } + CtlResponse::SyncFolders { items } => { + for f in items { + let sync = if f.last_sync == 0 { + "never".to_string() + } else { + f.last_sync.to_string() + }; + println!( + "[{}] {} ({}, {}, synced: {sync})", + f.id, f.local_path, f.mode, f.state + ); + // A pass in flight says what it is doing, indented under it. + if let Some(p) = &f.progress { + match p.phase { + SyncPhase::Scanning => println!(" scanning…"), + SyncPhase::Applying => println!( + " {} of {} {}", + p.done + 1, + p.total.max(p.done + 1), + p.current + ), + } + } + } + } + CtlResponse::Error { message } => bail!("{message}"), + other => bail!("unexpected response: {other:?}"), + }, + SyncCmd::Rm { id, delete_remote } => { + ok_or_bail(control_request(CtlRequest::RemoveSyncFolder { + id, + delete_remote, + })?)? + } + SyncCmd::Now { id } => ok_or_bail(control_request(CtlRequest::SyncNow { id })?)?, + SyncCmd::Mode { id, mode } => { + ok_or_bail(control_request(CtlRequest::SetSyncFolderMode { id, mode })?)? + } + } + Ok(()) +} + fn cmd_share( path: PathBuf, emails: Vec, @@ -559,6 +656,8 @@ fn activity_verb(kind: pdfs_core::control::ActivityKind) -> &'static str { use pdfs_core::control::ActivityKind::*; match kind { Upload => "upload", + Download => "download", + Sync => "sync", Rename => "rename", Move => "move", CreateFolder => "mkdir", @@ -847,8 +946,25 @@ fn cmd_unpin(path: PathBuf) -> Result<()> { fn cmd_transfers() -> Result<()> { use pdfs_core::control::TransferDirection; match control_request(CtlRequest::GetQueueStatus)? { - CtlResponse::Transfers { items } if items.is_empty() => println!("No active transfers."), - CtlResponse::Transfers { items } => { + CtlResponse::Transfers { items, jobs } if items.is_empty() && jobs.is_empty() => { + println!("Nothing in progress.") + } + CtlResponse::Transfers { items, jobs } => { + // Jobs first: they are the context for the transfers under them ("of + // 40 files", "still scanning") rather than a separate list. + for j in jobs { + let count = if j.total > 0 { + format!("{} of {}", j.done, j.total) + } else { + "—".to_string() + }; + let detail = if j.detail.is_empty() { + String::new() + } else { + format!(" — {}", j.detail) + }; + println!("… {:>9} {}{detail}", count, j.title); + } for t in items { let arrow = match t.direction { TransferDirection::Download => "↓", diff --git a/crates/pdfs-core/src/control.rs b/crates/pdfs-core/src/control.rs index 64ac4a6..651a6fa 100644 --- a/crates/pdfs-core/src/control.rs +++ b/crates/pdfs-core/src/control.rs @@ -111,9 +111,11 @@ pub enum Request { /// persist it to config so the next mount keeps it. Replies with /// [`Response::Ok`]. SetCacheBudget { bytes: u64 }, - /// Snapshot the daemon's in-flight transfers (active uploads/downloads). - /// Replies with [`Response::Transfers`]. Cheap to poll: the daemon keeps the - /// registry in memory, so a front-end can render a live progress widget. + /// Snapshot what the daemon is working on: in-flight transfers (active + /// uploads/downloads) and the longer jobs around them (scans, folder + /// skeletons, the local index, sync passes). Replies with + /// [`Response::Transfers`]. Cheap to poll: the daemon keeps the registry in + /// memory, so a front-end can render a live progress widget. GetQueueStatus, /// List what is in the account's trash. Replies with [`Response::Entries`]; /// a trashed node has no path inside the mount, so each entry carries only @@ -138,6 +140,22 @@ pub enum Request { /// Delete (deregister) a device by its uid. Replies with [`Response::Ok`]. DeleteDevice { uid: String }, + // ---- device folder sync (devices.md) ---------------------------------- + /// Add a local folder to this machine's device, uploading its tree and + /// registering the device on first use. Replies with [`Response::Ok`]. + AddSyncFolder { local_path: String }, + /// List this device's synced folders. Replies with [`Response::SyncFolders`]. + ListSyncFolders, + /// Remove a synced folder by id; `delete_remote` also trashes its cloud copy. + /// Replies with [`Response::Ok`]. + RemoveSyncFolder { id: i64, delete_remote: bool }, + /// Switch a synced folder between `mirror` and `ondemand` (Phase 3). Replies + /// with [`Response::Ok`]. + SetSyncFolderMode { id: i64, mode: String }, + /// Force a reconcile pass: one folder by id, or all when `id` is `None`. + /// Replies with [`Response::Ok`]. + SyncNow { id: Option }, + // ---- sharing a node --------------------------------------------------- /// Invite `emails` (Proton and/or external addresses, auto-detected) to the /// node at mountpoint-relative `path` at `role` ("viewer"|"editor"|"admin"), @@ -220,10 +238,11 @@ pub enum Request { // ---- activity --------------------------------------------------------- /// Fetch the daemon's recent activity log, newest first, capped at `limit` - /// entries. Replies with [`Response::Activity`]. The log lives in memory for - /// the life of the daemon: it records the mutations and transfers the daemon - /// performs (uploads, deletes, renames, shares, …), so a front-end can show a - /// running "what happened" feed without re-deriving it from anywhere. + /// entries. Replies with [`Response::Activity`]. The log is persisted, so it + /// survives a daemon restart: it records the mutations and transfers the + /// daemon performs (uploads, downloads, deletes, renames, shares, sync + /// passes, …), so a front-end can show a running "what happened" feed + /// without re-deriving it from anywhere. ListActivity { limit: usize }, } @@ -252,6 +271,54 @@ pub struct DeviceInfo { pub last_sync: Option, } +/// One synced local folder on this machine's device (in [`Response::SyncFolders`]). +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SyncFolderInfo { + /// Row id — the handle for [`Request::RemoveSyncFolder`]/[`Request::SetSyncFolderMode`]. + pub id: i64, + /// Absolute local folder path. + pub local_path: String, + /// The uid of the folder's remote root under the device root. + pub remote_uid: String, + /// `mirror` (full local copy, two-way synced) or `ondemand` (FUSE mount). + pub mode: String, + /// `idle` | `syncing` | `error` | `conflict`. + pub state: String, + /// Last successful sync, epoch seconds; `0` if never. + pub last_sync: i64, + /// What the folder's sync pass is doing right now, or `None` when no pass is + /// running. Live daemon state, not a stored column. + #[serde(default)] + pub progress: Option, +} + +/// Which stage a running sync pass is in, in a [`SyncProgress`]. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncPhase { + /// Walking the local tree, the remote tree and the stored baseline to work + /// out what changed. No per-item counts exist yet. + Scanning, + /// Applying the diff: creating folders, uploading, downloading, deleting. + Applying, +} + +/// A snapshot of a sync pass in flight (in [`SyncFolderInfo::progress`]), so a +/// front-end can say what the daemon is doing rather than just "syncing". +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SyncProgress { + pub phase: SyncPhase, + /// Items applied so far this pass. + pub done: usize, + /// Items known to need applying. This *grows* during a pass: paths are + /// classified depth by depth (a folder must exist remotely before its + /// children can be queued), so the total is what has been discovered so far, + /// not a figure fixed up front. + pub total: usize, + /// The name of an item currently being applied, or empty between items. + /// Several run at once; this is just the most recently started. + pub current: String, +} + /// One member or pending invitation on a node's share (in [`Response::Share`]). #[derive(Serialize, Deserialize, Debug, Clone)] pub struct ShareEntry { @@ -337,6 +404,9 @@ pub struct SharedItem { #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] pub enum ActivityKind { Upload, + Download, + /// A whole sync pass over one folder, summarising what it moved. + Sync, Rename, Move, CreateFolder, @@ -375,6 +445,23 @@ pub enum TransferDirection { Upload, } +/// One long-running daemon job in a [`Response::Transfers`] snapshot: work that +/// takes long enough to need reporting but doesn't move bytes over the wire — +/// walking a local tree, building a remote folder skeleton, indexing `$HOME`. +/// Byte-moving work is a [`TransferItem`] instead. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct JobItem { + /// What the job is, as a front-end would title it ("Uploading files"). + pub title: String, + /// What it is doing right now ("Scanning Photos/2024"), or empty. + pub detail: String, + /// Steps finished so far. + pub done: u64, + /// Steps known to need doing, or `0` when unknown (indeterminate progress). + /// May *grow* mid-job as more work is discovered. + pub total: u64, +} + /// One in-flight transfer in a [`Response::Transfers`] snapshot. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TransferItem { @@ -532,10 +619,19 @@ pub enum Response { /// is true while a scan of the machine is still running, so a front-end can /// say "still indexing" instead of "no matches" on a cold first launch. LocalResults { hits: Vec, indexing: bool }, - /// A snapshot of in-flight transfers (reply to [`Request::GetQueueStatus`]). - Transfers { items: Vec }, + /// A snapshot of what the daemon is working on (reply to + /// [`Request::GetQueueStatus`]): `items` are byte-moving transfers, `jobs` + /// the longer non-transfer work around them (scans, folder skeletons, the + /// local index, sync passes). Both empty means the daemon is idle. + Transfers { + items: Vec, + #[serde(default)] + jobs: Vec, + }, /// The account's devices (reply to [`Request::ListDevices`]). Devices { items: Vec }, + /// This device's synced folders (reply to [`Request::ListSyncFolders`]). + SyncFolders { items: Vec }, /// A node's share: members + pending invitations, and its public link if any /// (reply to [`Request::ListShare`]). Share { @@ -655,6 +751,19 @@ mod tests { Request::DeleteDevice { uid: "dev-1".into(), }, + Request::AddSyncFolder { + local_path: "/home/me/Docs".into(), + }, + Request::ListSyncFolders, + Request::RemoveSyncFolder { + id: 3, + delete_remote: true, + }, + Request::SetSyncFolderMode { + id: 3, + mode: "ondemand".into(), + }, + Request::SyncNow { id: Some(3) }, Request::ShareNode { path: "a/b".into(), emails: vec!["x@proton.me".into(), "y@example.com".into()], diff --git a/crates/pdfs-core/src/db.rs b/crates/pdfs-core/src/db.rs index 78b07a8..d6a5ff8 100644 --- a/crates/pdfs-core/src/db.rs +++ b/crates/pdfs-core/src/db.rs @@ -19,10 +19,20 @@ use proton_drive_rs::{Node, NodeKind}; use rusqlite::{Connection, OptionalExtension, params}; use crate::Result; +use crate::control::{ActivityEntry, ActivityKind}; use crate::localindex::LocalEntry; /// Current schema version. Bump on every forward migration added below. -const SCHEMA_VERSION: i64 = 7; +const SCHEMA_VERSION: i64 = 9; + +/// How many activity rows to keep. Older rows are pruned on insert, so the feed +/// stays a bounded "recent history" rather than growing without limit. +const ACTIVITY_KEEP: i64 = 2000; + +/// Size the WAL is truncated back to after a checkpoint. Comfortably above the +/// steady-state working set (a few MB), so the truncation only claws back the +/// outliers rather than fighting the normal write path for disk. +const WAL_SIZE_LIMIT: i64 = 64 * 1024 * 1024; /// A photo whose thumbnail state is not known yet: it has never been asked for. pub const THUMB_UNKNOWN: i64 = 0; @@ -77,6 +87,46 @@ pub struct StoredTrash { pub mtime: i64, } +/// The one Proton Drive Device this machine is registered as (devices.md). +/// Cached so restarts reuse the same device rather than creating a new one. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredDevice { + pub uid: String, + pub share_id: String, + pub root_uid: String, + pub name: String, + pub created: i64, +} + +/// One local folder the user added to this device's sync set. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredSyncFolder { + pub id: i64, + pub local_path: String, + pub remote_uid: String, + pub remote_share_id: String, + /// `mirror` (full local copy, two-way synced) or `ondemand` (FUSE mount). + pub mode: String, + /// `idle` | `syncing` | `error` | `conflict`. + pub state: String, + pub last_sync: i64, +} + +/// One per-file sync baseline row: what a path looked like on both sides at the +/// last successful sync, so the next reconcile can tell which side changed +/// (devices.md Phase 2). `remote_rev`/`remote_hash` hold the remote signature — +/// its modification time and size as strings — since no cheap content hash is +/// exposed; change detection is `(mtime, size)` on each side. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredSyncEntry { + pub rel_path: String, + pub remote_uid: Option, + pub local_mtime: i64, + pub local_size: i64, + pub remote_rev: Option, + pub remote_hash: Option, +} + /// Handle to the unified metadata database. /// /// Cheap to wrap in an `Arc`; clone the `Arc`, not this. All access goes through @@ -95,6 +145,13 @@ impl Db { conn.pragma_update(None, "journal_mode", "WAL")?; conn.pragma_update(None, "synchronous", "NORMAL")?; conn.pragma_update(None, "foreign_keys", "ON")?; + // Hand the WAL's disk back after a checkpoint. Without a limit SQLite + // reuses the file in place but never shrinks it, so a single large + // transaction (a full-index FTS rebuild in `local_finish_scan` is the + // one that reaches this size) leaves the WAL at its high-water mark + // forever — a multi-GB file next to a database two orders of magnitude + // smaller. Checkpointing is unaffected; only the file is truncated back. + conn.pragma_update(None, "journal_size_limit", WAL_SIZE_LIMIT)?; let db = Self { conn: Mutex::new(conn), @@ -161,6 +218,12 @@ impl Db { if current < 7 { tx.execute_batch(MIGRATION_V7)?; } + if current < 8 { + tx.execute_batch(MIGRATION_V8)?; + } + if current < 9 { + tx.execute_batch(MIGRATION_V9)?; + } tx.execute( "INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1) ON CONFLICT(key) DO UPDATE SET value = excluded.value", @@ -527,6 +590,266 @@ impl Db { Ok(items) } + // ---- device sync (devices.md) ----------------------------------------- + + /// The registered device for this machine, if one has been created/cached. + pub fn device_get(&self) -> Result> { + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT uid, share_id, root_uid, name, created FROM device LIMIT 1", + [], + |r| { + Ok(StoredDevice { + uid: r.get(0)?, + share_id: r.get(1)?, + root_uid: r.get(2)?, + name: r.get(3)?, + created: r.get(4)?, + }) + }, + ) + .optional() + .map_err(Into::into) + } + + /// Persist (or replace) this machine's device. The table holds a single row. + pub fn device_set(&self, dev: &StoredDevice) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute("DELETE FROM device", [])?; + conn.execute( + "INSERT INTO device (uid, share_id, root_uid, name, created) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![dev.uid, dev.share_id, dev.root_uid, dev.name, dev.created], + )?; + Ok(()) + } + + /// Add a synced folder, returning its new row id. + pub fn sync_folder_add( + &self, + local_path: &str, + remote_uid: &str, + remote_share_id: &str, + ) -> Result { + let conn = self.conn.lock().unwrap(); + conn.execute( + "INSERT INTO sync_folder (local_path, remote_uid, remote_share_id) + VALUES (?1, ?2, ?3)", + params![local_path, remote_uid, remote_share_id], + )?; + Ok(conn.last_insert_rowid()) + } + + /// Every synced folder, oldest first. + pub fn sync_folder_list(&self) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT id, local_path, remote_uid, remote_share_id, mode, state, last_sync + FROM sync_folder ORDER BY id", + )?; + let rows = stmt.query_map([], |r| { + Ok(StoredSyncFolder { + id: r.get(0)?, + local_path: r.get(1)?, + remote_uid: r.get(2)?, + remote_share_id: r.get(3)?, + mode: r.get(4)?, + state: r.get(5)?, + last_sync: r.get(6)?, + }) + })?; + let mut items = Vec::new(); + for row in rows { + items.push(row?); + } + Ok(items) + } + + /// Look up one synced folder by id. + pub fn sync_folder_get(&self, id: i64) -> Result> { + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT id, local_path, remote_uid, remote_share_id, mode, state, last_sync + FROM sync_folder WHERE id = ?1", + params![id], + |r| { + Ok(StoredSyncFolder { + id: r.get(0)?, + local_path: r.get(1)?, + remote_uid: r.get(2)?, + remote_share_id: r.get(3)?, + mode: r.get(4)?, + state: r.get(5)?, + last_sync: r.get(6)?, + }) + }, + ) + .optional() + .map_err(Into::into) + } + + /// Remove a synced folder and its per-file baseline. + pub fn sync_folder_remove(&self, id: i64) -> Result { + let mut conn = self.conn.lock().unwrap(); + let tx = conn.transaction()?; + tx.execute("DELETE FROM sync_entry WHERE folder_id = ?1", params![id])?; + let n = tx.execute("DELETE FROM sync_folder WHERE id = ?1", params![id])?; + tx.commit()?; + Ok(n > 0) + } + + /// Update a synced folder's mode (`mirror`/`ondemand`). + pub fn sync_folder_set_mode(&self, id: i64, mode: &str) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE sync_folder SET mode = ?2 WHERE id = ?1", + params![id, mode], + )?; + Ok(()) + } + + /// Update a synced folder's state and stamp `last_sync` to now. + pub fn sync_folder_set_state(&self, id: i64, state: &str, last_sync: i64) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE sync_folder SET state = ?2, last_sync = ?3 WHERE id = ?1", + params![id, state, last_sync], + )?; + Ok(()) + } + + /// The whole per-file sync baseline for a folder, keyed by relative path. + pub fn sync_entries(&self, folder_id: i64) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash + FROM sync_entry WHERE folder_id = ?1", + )?; + let rows = stmt.query_map(params![folder_id], |r| { + Ok(StoredSyncEntry { + rel_path: r.get(0)?, + remote_uid: r.get(1)?, + local_mtime: r.get(2)?, + local_size: r.get(3)?, + remote_rev: r.get(4)?, + remote_hash: r.get(5)?, + }) + })?; + let mut map = HashMap::new(); + for row in rows { + let e = row?; + map.insert(e.rel_path.clone(), e); + } + Ok(map) + } + + /// Insert or replace one baseline row. + pub fn sync_entry_upsert(&self, folder_id: i64, e: &StoredSyncEntry) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "INSERT INTO sync_entry + (folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(folder_id, rel_path) DO UPDATE SET + remote_uid = excluded.remote_uid, + local_mtime = excluded.local_mtime, + local_size = excluded.local_size, + remote_rev = excluded.remote_rev, + remote_hash = excluded.remote_hash", + params![ + folder_id, + e.rel_path, + e.remote_uid, + e.local_mtime, + e.local_size, + e.remote_rev, + e.remote_hash, + ], + )?; + Ok(()) + } + + /// Drop one baseline row (a path that left the sync set on both sides). + pub fn sync_entry_remove(&self, folder_id: i64, rel_path: &str) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "DELETE FROM sync_entry WHERE folder_id = ?1 AND rel_path = ?2", + params![folder_id, rel_path], + )?; + Ok(()) + } + + // ---- activity --------------------------------------------------------- + + /// Append one entry to the activity log, pruning back to [`ACTIVITY_KEEP`] + /// rows. `kind` round-trips through serde rather than a hand-written string + /// table, so adding an [`ActivityKind`] variant needs no change here. + pub fn activity_add(&self, entry: &ActivityEntry) -> Result<()> { + let kind = serde_json::to_string(&entry.kind)?; + let mut conn = self.conn.lock().unwrap(); + let tx = conn.transaction()?; + tx.execute( + "INSERT INTO activity (time, kind, target, detail, ok) VALUES (?1, ?2, ?3, ?4, ?5)", + params![entry.time, kind, entry.target, entry.detail, entry.ok], + )?; + tx.execute( + "DELETE FROM activity WHERE id <= ( + SELECT id FROM activity ORDER BY id DESC LIMIT 1 OFFSET ?1 + )", + params![ACTIVITY_KEEP], + )?; + tx.commit()?; + Ok(()) + } + + /// The most recent activity, newest first, capped at `limit` entries. Rows + /// whose stored `kind` no longer parses (written by an older build) are + /// skipped rather than failing the whole read. + pub fn activity_list(&self, limit: usize) -> Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT time, kind, target, detail, ok FROM activity + ORDER BY id DESC LIMIT ?1", + )?; + let rows = stmt.query_map(params![limit as i64], |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + r.get::<_, bool>(4)?, + )) + })?; + let mut out = Vec::new(); + for row in rows { + let (time, kind, target, detail, ok) = row?; + let Ok(kind) = serde_json::from_str::(&kind) else { + continue; + }; + out.push(ActivityEntry { + time, + kind, + target, + detail, + ok, + }); + } + Ok(out) + } + + /// Drop the entire baseline for a folder. Used when flipping ondemand→mirror: + /// the local tree was evicted, so the old baseline is stale and would make the + /// next reconcile mistake "locally deleted" for "must re-download". Clearing it + /// leaves an empty baseline + full remote = pure download (devices.md P3). + pub fn sync_entries_clear(&self, folder_id: i64) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "DELETE FROM sync_entry WHERE folder_id = ?1", + params![folder_id], + )?; + Ok(()) + } + /// Children of `parent`, for the offline `ensure_children` fast path when a /// listed folder's `children` map entry was trimmed mid-run. Returns `None` /// when the folder is not marked `listed` (listing unknown, must re-fetch). @@ -1106,9 +1429,86 @@ CREATE TABLE trash ( ); "; +/// Schema v8: device sync (devices.md). This machine registers as one Proton Drive +/// **Device** (a share + root folder on the main volume); `device` is a singleton +/// row cached so we reuse the same device across restarts instead of creating a +/// new one each run. `sync_folder` is one row per local folder the user added, +/// each mapped to a folder under the device root; `mode` is `mirror` (full local +/// copy, two-way synced) or `ondemand` (FUSE mount at `local_path`, no local +/// storage). `sync_entry` is the per-file baseline for three-way merge — added in +/// this migration so Phase 2 has the table, though Phase 1 leaves it empty. +const MIGRATION_V8: &str = " +CREATE TABLE device ( + uid TEXT PRIMARY KEY, + share_id TEXT NOT NULL, + root_uid TEXT NOT NULL, + name TEXT NOT NULL, + created INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE sync_folder ( + id INTEGER PRIMARY KEY, + local_path TEXT NOT NULL UNIQUE, + remote_uid TEXT NOT NULL, + remote_share_id TEXT NOT NULL, + mode TEXT NOT NULL DEFAULT 'mirror', + state TEXT NOT NULL DEFAULT 'idle', + last_sync INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE sync_entry ( + folder_id INTEGER NOT NULL, + rel_path TEXT NOT NULL, + remote_uid TEXT, + local_mtime INTEGER NOT NULL DEFAULT 0, + local_size INTEGER NOT NULL DEFAULT 0, + remote_hash TEXT, + remote_rev TEXT, + PRIMARY KEY (folder_id, rel_path) +); +"; + +const MIGRATION_V9: &str = " +CREATE TABLE activity ( + id INTEGER PRIMARY KEY, + time INTEGER NOT NULL, + kind TEXT NOT NULL, + target TEXT NOT NULL, + detail TEXT NOT NULL DEFAULT '', + ok INTEGER NOT NULL DEFAULT 1 +); + +CREATE INDEX activity_time ON activity(time DESC); +"; + #[cfg(test)] mod tests { use super::*; + + /// `Db::open` applies the pragmas that `open_in_memory` skips, so the WAL + /// settings can only be checked against a real file. + #[test] + fn open_bounds_the_wal_size() { + let path = std::env::temp_dir().join(format!("pdfs-db-wal-{}.db", std::process::id())); + let _ = std::fs::remove_file(&path); + let db = Db::open(&path).unwrap(); + let conn = db.conn.lock().unwrap(); + + let mode: String = conn + .query_row("PRAGMA journal_mode", [], |r| r.get(0)) + .unwrap(); + assert_eq!(mode, "wal"); + // Without this the WAL is reused in place but never shrinks, so one + // oversized transaction strands its high-water mark on disk forever. + let limit: i64 = conn + .query_row("PRAGMA journal_size_limit", [], |r| r.get(0)) + .unwrap(); + assert_eq!(limit, WAL_SIZE_LIMIT); + + drop(conn); + drop(db); + let _ = std::fs::remove_file(&path); + } use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId}; fn uid(link: &str) -> NodeUid { @@ -1490,6 +1890,49 @@ mod tests { assert_eq!(db.search("unique", 10).unwrap().len(), 0); } + fn activity(target: &str, kind: ActivityKind, ok: bool) -> ActivityEntry { + ActivityEntry { + time: 1700, + kind, + target: target.into(), + detail: "detail".into(), + ok, + } + } + + #[test] + fn activity_reads_back_newest_first() { + let db = Db::open_in_memory().unwrap(); + db.activity_add(&activity("a.txt", ActivityKind::Upload, true)) + .unwrap(); + db.activity_add(&activity("b.txt", ActivityKind::Download, false)) + .unwrap(); + + let items = db.activity_list(10).unwrap(); + assert_eq!(items.len(), 2); + assert_eq!(items[0].target, "b.txt"); + assert_eq!(items[0].kind, ActivityKind::Download); + assert!(!items[0].ok); + assert_eq!(items[0].detail, "detail"); + assert_eq!(items[0].time, 1700); + assert_eq!(items[1].target, "a.txt"); + + assert_eq!(db.activity_list(1).unwrap().len(), 1); + } + + #[test] + fn activity_prunes_to_the_keep_limit() { + let db = Db::open_in_memory().unwrap(); + for i in 0..(ACTIVITY_KEEP + 10) { + db.activity_add(&activity(&format!("f{i}"), ActivityKind::Upload, true)) + .unwrap(); + } + let items = db.activity_list(ACTIVITY_KEEP as usize * 2).unwrap(); + assert_eq!(items.len(), ACTIVITY_KEEP as usize); + // The newest survive; the oldest are the ones dropped. + assert_eq!(items[0].target, format!("f{}", ACTIVITY_KEEP + 9)); + } + #[test] fn opens_and_migrates() { let db = Db::open_in_memory().unwrap(); diff --git a/crates/pdfs-fuse/Cargo.toml b/crates/pdfs-fuse/Cargo.toml index e96ddd7..de34a00 100644 --- a/crates/pdfs-fuse/Cargo.toml +++ b/crates/pdfs-fuse/Cargo.toml @@ -16,3 +16,4 @@ fuser.workspace = true libc.workspace = true serde_json.workspace = true image.workspace = true +notify = "8" diff --git a/crates/pdfs-fuse/src/lib.rs b/crates/pdfs-fuse/src/lib.rs index 84f3ceb..3b9349d 100644 --- a/crates/pdfs-fuse/src/lib.rs +++ b/crates/pdfs-fuse/src/lib.rs @@ -31,7 +31,7 @@ //! set, so a multi-GiB write never buffers in RAM and only the untouched //! remainder of the file is pulled from the remote — lazily, at commit. -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{HashMap, HashSet}; use std::ffi::OsStr; use std::fs::File; use std::io::{BufRead, BufReader, Write as _}; @@ -44,30 +44,34 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use fuser::ReplyXattr; use fuser::{ - BsdFileFlags, Config, Errno, FileAttr, FileHandle, FileType, Filesystem, FopenFlags, - Generation, INodeNo, LockOwner, MountOption, Notifier, OpenAccMode, OpenFlags, RenameFlags, - ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyOpen, - ReplyWrite, Request, Session, TimeOrNow, WriteFlags, + BackgroundSession, BsdFileFlags, Config, Errno, FileAttr, FileHandle, FileType, Filesystem, + FopenFlags, Generation, INodeNo, LockOwner, MountOption, Notifier, OpenAccMode, OpenFlags, + RenameFlags, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, + ReplyOpen, ReplyWrite, Request, Session, TimeOrNow, WriteFlags, }; use pdfs_core::cache::{BLOCK_SIZE, ContentCache}; use pdfs_core::config::AppDirs; use pdfs_core::control::{ - ActivityEntry, ActivityKind, BookmarkInfo, DeviceInfo, DirEntry, InvitationInfo, LocalHit, - PhotoItem, PhotoThumb, PublicLinkInfo, Request as CtlRequest, Response as CtlResponse, - SearchHit, ShareEntry, ShareEntryKind, SharedItem, TransferDirection, + ActivityEntry, ActivityKind, BookmarkInfo, DeviceInfo, DirEntry, InvitationInfo, JobItem, + LocalHit, PhotoItem, PhotoThumb, PublicLinkInfo, Request as CtlRequest, + Response as CtlResponse, SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, + SyncPhase, SyncProgress, TransferDirection, +}; +use pdfs_core::db::{ + self, Db, StoredDevice, StoredNode, StoredPhoto, StoredSyncFolder, StoredTrash, }; -use pdfs_core::db::{self, Db, StoredNode, StoredPhoto, StoredTrash}; use pdfs_core::localindex; use proton_drive_rs::proton_sdk::error::ProtonError; use proton_drive_rs::proton_sdk::ids::{DeviceUid, DriveEventId, LinkId, NodeUid, VolumeId}; use proton_drive_rs::{ - DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, ProtonDriveClient, + DeviceType, DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, ProtonDriveClient, ProtonPhotosClient, ThumbnailType, }; +mod sync; mod transfers; use tracing::{debug, error, info, warn}; -use transfers::{CountingReader, CountingWriter, OwnedCountingReader, TransferRegistry}; +use transfers::{CountingReader, CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry}; /// Attribute/entry cache lifetime handed back to the kernel. Long because the /// Phase 2 event poller actively invalidates changed inodes; without a remote @@ -350,10 +354,14 @@ struct UploadStats { /// streamed straight from disk and ticking its own transfer-registry guard. /// Returns, per file, either `(parent_ino, new_uid)` for the caller to intern or /// `(name, error)` to log — one failure never sinks the batch. +/// +/// `job` counts files finished (either way: a failure is still one file the batch +/// no longer waits on), so a front-end can show "12 of 40" over the per-file bars. async fn run_uploads( core: Core, tasks: Vec, limit: usize, + job: Arc, ) -> Vec> { let sem = Arc::new(tokio::sync::Semaphore::new(limit)); let mut set = tokio::task::JoinSet::new(); @@ -396,6 +404,7 @@ async fn run_uploads( } let mut out = Vec::new(); while let Some(joined) = set.join_next().await { + job.step(); match joined { Ok(result) => out.push(result), Err(e) => warn!(error = %e, "upload task panicked"), @@ -436,17 +445,27 @@ struct Core { /// True while the background scanner is rebuilding the local-file index, so /// `SearchLocal` can tell a front-end "still indexing" apart from "no match". indexing: Arc, - /// Recent activity, newest at the back, capped at [`ACTIVITY_CAP`]. In memory - /// only — a running feed of the mutations and transfers the daemon performed - /// this session, served to `ListActivity`. Not persisted: it is a convenience - /// view of "what just happened", not an audit trail. - activity: Arc>>, + /// Live per-folder sync progress, keyed by sync-folder id, so `ListSyncFolders` + /// can say what a pass is doing rather than just "syncing". An entry exists + /// only while that folder's reconcile pass is running. + sync_progress: Arc>>, + /// Channel to the folder-sync engine (devices.md Phase 2): nudges it to + /// reconcile a folder, reconcile everything, or re-scan its watch set. + sync_tx: std::sync::mpsc::Sender, + /// Secondary FUSE sessions for `ondemand` sync folders, keyed by sync-folder + /// id (devices.md Phase 3). Each is a `ProtonFs` rooted at the folder's remote + /// node, mounted over its local path, sharing this Core's client/cache/db but + /// with its own inode space (`fork_state`). Held so we can unmount on toggle + /// back to `mirror` and on daemon shutdown. + mounts: Arc>>, + /// Per-sync-folder locks, held for a whole reconcile pass and for a whole + /// mode switch. A `mirror→ondemand` flip evicts the local tree and mounts + /// FUSE over it, so it must never overlap a pass that is walking and + /// uploading that same tree — the engine would upload files as they vanish + /// and then walk the FUSE mount as if it were local. + sync_locks: Arc>>>>, } -/// How many activity entries the daemon keeps in memory. Older entries fall off -/// the front as new ones arrive. -const ACTIVITY_CAP: usize = 500; - impl Core { /// Rehydrate the in-memory `State` maps from the DB on mount, so a cold /// start serves previously-seen metadata (stable inodes, instant listings) @@ -1864,28 +1883,50 @@ impl Core { // Phase 1 (sequential): build the remote folder skeleton and collect the // flat list of files to upload. Folders must exist before their children, - // so this can't be parallelised. + // so this can't be parallelised. On a deep tree this is a folder-creation + // round-trip per directory before a single byte moves — long enough that + // it needs a job of its own, or the daemon looks idle for minutes. let mut tasks = Vec::new(); let mut folders = 0usize; - for src in sources { - if let Err(e) = self.collect_uploads(pino, &parent_uid, src, &mut tasks, &mut folders) { - warn!(source = %src.display(), error = %e, "skipping source"); + { + let job = self.transfers.begin_job("Preparing upload"); + for src in sources { + if let Err(e) = + self.collect_uploads(pino, &parent_uid, src, &mut tasks, &mut folders, &job) + { + warn!(source = %src.display(), error = %e, "skipping source"); + } } } // Phase 2 (concurrent): upload the files, up to UPLOAD_CONCURRENCY at once. - let outcomes = self - .rt - .block_on(run_uploads(self.clone(), tasks, UPLOAD_CONCURRENCY)); + // Each file reports its own bytes; this job is the batch's "N of M files", + // which is the number a user actually waits on. + let job = Arc::new(self.transfers.begin_job(match tasks.len() { + 1 => "Uploading 1 file".to_string(), + n => format!("Uploading {n} files"), + })); + job.set_total(tasks.len() as u64); + let outcomes = self.rt.block_on(run_uploads( + self.clone(), + tasks, + UPLOAD_CONCURRENCY, + job.clone(), + )); + drop(job); // Phase 3 (sequential): intern each uploaded node so it shows up in the // listing without a re-enumeration. fetch_node uses `block_on`, so it must - // run here rather than inside the async batch. + // run here rather than inside the async batch — and it is a round-trip per + // file, so it too gets a job rather than a silent tail. let mut stats = UploadStats { folders, ..UploadStats::default() }; + let job = self.transfers.begin_job("Finishing upload"); + job.set_total(outcomes.len() as u64); for outcome in outcomes { + job.step(); match outcome { Ok((parent_ino, uid, size)) => { stats.uploaded += 1; @@ -1970,6 +2011,9 @@ impl Core { /// Walk one local source path, appending its files to `tasks`. A file becomes /// one task; a directory is recreated remotely and recursed into (children /// sorted for a stable order). Symlinks and other special files are skipped. + /// + /// `job` narrates the walk with the folder currently being mirrored. It stays + /// indeterminate: nothing knows the size of the tree until the walk has ended. fn collect_uploads( &self, pino: u64, @@ -1977,6 +2021,7 @@ impl Core { src: &Path, tasks: &mut Vec, folders: &mut usize, + job: &JobGuard, ) -> Result<(), String> { let meta = std::fs::symlink_metadata(src).map_err(|e| format!("stat {}: {e}", src.display()))?; @@ -1987,6 +2032,7 @@ impl Core { .to_string(); if meta.is_dir() { + job.detail(&name); let (child_ino, child_uid) = self.ensure_remote_folder(pino, parent_uid, &name)?; *folders += 1; let mut entries: Vec = std::fs::read_dir(src) @@ -1996,11 +2042,15 @@ impl Core { .collect(); entries.sort(); for entry in entries { - if let Err(e) = self.collect_uploads(child_ino, &child_uid, &entry, tasks, folders) + if let Err(e) = + self.collect_uploads(child_ino, &child_uid, &entry, tasks, folders, job) { warn!(source = %entry.display(), error = %e, "skipping entry"); } } + // Deeper folders have retitled the job by now; put this one back so the + // line tracks the walk's position rather than its deepest leaf. + job.detail(&name); } else if meta.is_file() { if name.contains('/') { return Err(format!("invalid file name: {name:?}")); @@ -2056,6 +2106,511 @@ impl Core { Ok(()) } + // ---- device folder sync (devices.md, Phase 1) ------------------------- + + /// Auto-register (or recover) this machine as a Proton Drive Device, caching + /// it so restarts reuse the same device. Recovery matches an existing remote + /// Linux device by name before creating a new one, so a lost local record + /// doesn't orphan the device's root folder. + fn ensure_device(&self) -> Result { + let name = this_hostname(); + // Enumerate the remote devices once: used both to validate any cached + // record and to recover an existing device by name. + let remote = self + .rt + .block_on(self.client.enumerate_devices()) + .map_err(|e| format!("enumerate devices: {e}"))?; + + // A cached device is only trustworthy if it still exists remotely. A + // device deleted from another client (or the web UI) leaves a stale row + // whose root folder is gone, so creating folders under it fails with + // "parent node is not a folder". Re-register in that case. + if let Some(dev) = self.db.device_get().map_err(|e| format!("db: {e:?}"))? { + if remote.iter().any(|d| d.uid.to_string() == dev.uid) { + return Ok(dev); + } + warn!(uid = %dev.uid, "cached device is gone remotely; re-registering"); + } + + // Recover: an existing remote Linux device with the same name is ours. + let existing = remote.into_iter().find(|d| { + d.device_type == DeviceType::Linux && d.name.as_deref().ok() == Some(name.as_str()) + }); + let dev = match existing { + Some(d) => StoredDevice { + uid: d.uid.to_string(), + share_id: d.share_id.to_string(), + root_uid: d.root_folder_uid.to_string(), + name, + created: d.creation_time, + }, + None => { + let d = self + .rt + .block_on(self.client.create_device(&name, DeviceType::Linux)) + .map_err(|e| format!("create device: {e}"))?; + StoredDevice { + uid: d.uid.to_string(), + share_id: d.share_id.to_string(), + root_uid: d.root_folder_uid.to_string(), + name, + created: d.creation_time, + } + } + }; + self.db.device_set(&dev).map_err(|e| format!("db: {e:?}"))?; + Ok(dev) + } + + /// An untrashed folder named `name` directly under the device root, if one + /// already exists. + fn find_device_child_folder( + &self, + root_uid: &NodeUid, + name: &str, + ) -> Result, String> { + let uids = self + .rt + .block_on(self.client.enumerate_folder_children_node_uids(root_uid)) + .map_err(|e| format!("list device root: {e}"))?; + if uids.is_empty() { + return Ok(None); + } + let nodes = self + .rt + .block_on(self.client.enumerate_nodes(&uids)) + .map_err(|e| format!("resolve device root children: {e}"))?; + Ok(nodes + .into_iter() + .find(|n| n.is_folder() && !n.trashed && n.name == name) + .map(|n| n.uid)) + } + + /// Add a local folder to this device's sync set: register the device if + /// needed, create a matching folder under the device root, upload the local + /// tree into it once, and record the mapping. Phase 1 is a one-shot upload — + /// the two-way engine (Phase 2) reconciles later changes. + fn add_sync_folder(&self, local: &Path) -> Result { + let meta = + std::fs::metadata(local).map_err(|e| format!("stat {}: {e}", local.display()))?; + if !meta.is_dir() { + return Err(format!("{} is not a directory", local.display())); + } + let local = local + .canonicalize() + .map_err(|e| format!("canonicalize {}: {e}", local.display()))?; + let local_str = local.to_string_lossy().to_string(); + + // Reject duplicates up front for a clear error (UNIQUE would also catch it). + if self + .db + .sync_folder_list() + .map_err(|e| format!("db: {e:?}"))? + .iter() + .any(|f| f.local_path == local_str) + { + return Err(format!("{} is already synced", local.display())); + } + + let name = local + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| format!("unusable folder name: {}", local.display()))? + .to_string(); + + let device = self.ensure_device()?; + let root_uid = parse_uid(&device.root_uid) + .ok_or_else(|| format!("bad device root uid: {}", device.root_uid))?; + + // The synced folder's remote root: the folder under the device root named + // after the local basename. Reuse an existing one rather than creating a + // second folder with the same name — re-adding a folder (after a removal, or + // after a failed add that had already created it) must land back on the + // original, not leave the user with two "Downloads" in their Drive. The + // reconcile treats an existing remote tree correctly: unmatched paths read as + // a conflict, not as data loss. + let remote_root = match self.find_device_child_folder(&root_uid, &name)? { + Some(uid) => { + info!(name, "reusing existing device folder"); + uid + } + None => self + .rt + .block_on( + self.client + .create_folder(&root_uid, &name, Some(now_secs())), + ) + .map_err(|e| format!("create device folder {name}: {e}"))?, + }; + + let id = self + .db + .sync_folder_add(&local_str, &remote_root.to_string(), &device.share_id) + .map_err(|e| format!("db: {e:?}"))?; + + // Hand the initial upload to the sync engine: an empty baseline against a + // full local tree reconciles as "upload everything", and the folder is + // added to the filesystem watch set in the same pass. + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); + + info!(local = %local.display(), id, "added sync folder"); + self.db + .sync_folder_get(id) + .map_err(|e| format!("db: {e:?}"))? + .ok_or_else(|| "sync folder vanished after insert".to_string()) + } + + /// List this device's synced folders for the front-ends, each carrying the + /// live progress of its pass when one is running. + fn list_sync_folders(&self) -> Result, String> { + let progress = self.sync_progress.lock().unwrap(); + Ok(self + .db + .sync_folder_list() + .map_err(|e| format!("db: {e:?}"))? + .into_iter() + .map(|f| { + let live = progress.get(&f.id).cloned(); + sync_folder_info(f, live) + }) + .collect()) + } + + /// Everything the daemon is chewing on that isn't moving bytes, for + /// `GetQueueStatus`: the registered jobs (bulk-upload scans, the local index) + /// plus a synthesized job per running sync pass, so one Activity view answers + /// "is anything still happening?" without also polling `ListSyncFolders`. + /// + /// The sync passes are folded in here rather than tracked as registry jobs + /// because the Devices page needs them per folder anyway + /// ([`SyncFolderInfo::progress`]) — this keeps one source of truth and hits + /// the db only while a pass is actually running. + fn jobs_snapshot(&self) -> Vec { + let mut jobs = self.transfers.jobs_snapshot(); + let mut passes: Vec<(i64, SyncProgress)> = self + .sync_progress + .lock() + .unwrap() + .iter() + .map(|(id, p)| (*id, p.clone())) + .collect(); + if passes.is_empty() { + return jobs; + } + passes.sort_by_key(|(id, _)| *id); + + let names: HashMap = self + .db + .sync_folder_list() + .unwrap_or_default() + .into_iter() + .map(|f| (f.id, f.local_path)) + .collect(); + for (id, p) in passes { + // The row is titled with the folder's own name; the full local path + // is what the Devices page shows, and is far too long for this line. + let folder = names + .get(&id) + .and_then(|path| Path::new(path).file_name()) + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| "folder".to_string()); + jobs.push(match p.phase { + // No counts exist until the walk has classified something, so a + // scan reports indeterminate rather than a fake 0 of 0. + SyncPhase::Scanning => JobItem { + title: format!("Checking {folder}"), + detail: "Looking for changes".to_string(), + done: 0, + total: 0, + }, + SyncPhase::Applying => JobItem { + title: format!("Syncing {folder}"), + detail: p.current.clone(), + done: p.done as u64, + total: p.total.max(p.done) as u64, + }, + }); + } + jobs + } + + /// The lock guarding sync-folder `id` against concurrent reconcile/mode-switch. + pub(crate) fn sync_lock(&self, id: i64) -> Arc> { + self.sync_locks + .lock() + .unwrap() + .entry(id) + .or_default() + .clone() + } + + /// Remove a synced folder from the sync set. `delete_remote` also deletes its + /// folder under the device root; otherwise the cloud copy is left in place. + fn remove_sync_folder(&self, id: i64, delete_remote: bool) -> Result<(), String> { + let folder = self + .db + .sync_folder_get(id) + .map_err(|e| format!("db: {e:?}"))? + .ok_or_else(|| format!("no synced folder with id {id}"))?; + if delete_remote + && let Some(uid) = parse_uid(&folder.remote_uid) + && let Err(e) = self.rt.block_on(self.client.trash_nodes(&[uid])) + { + warn!(id, error = %e, "delete remote device folder failed"); + } + if !self + .db + .sync_folder_remove(id) + .map_err(|e| format!("db: {e:?}"))? + { + return Err(format!("no synced folder with id {id}")); + } + // Stop watching the folder we just dropped. + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + Ok(()) + } + + /// Trigger a reconcile: one folder by id, or every folder when `id` is `None`. + fn sync_now(&self, id: Option) { + let _ = match id { + Some(id) => self.sync_tx.send(sync::SyncMsg::Reconcile(id)), + None => self.sync_tx.send(sync::SyncMsg::ReconcileAll), + }; + } + + /// A sibling Core that shares this one's client/rt/cache/db (and transfer, + /// activity, mount registries) but gets a **fresh, empty `State`** — its own + /// inode space starting at [`ROOT_INO`]. Used to root a secondary FUSE session + /// at an `ondemand` sync folder without colliding with the main mount's inodes + /// (devices.md Phase 3). + fn fork_state(&self) -> Core { + let mut fork = self.clone(); + fork.state = Arc::new(Mutex::new(State { + entries: HashMap::new(), + by_uid: HashMap::new(), + children: HashMap::new(), + next_ino: 2, + handles: HashMap::new(), + next_fh: 1, + db: self.db.clone(), + })); + fork + } + + /// Flip a synced folder between `mirror` (full local copy + two-way sync) and + /// `ondemand` (a FUSE mount over the local path; no local storage). Returns a + /// human message for the reply. + /// + /// - **mirror→ondemand**: require the folder in-sync (`idle`), stop watching it, + /// evict the local files to reclaim disk, then mount a secondary `ProtonFs` + /// rooted at the folder's remote node over its local path. + /// - **ondemand→mirror**: unmount, clear the stale baseline (the local tree was + /// evicted), then hand the folder back to the engine, which re-downloads it. + fn set_sync_folder_mode(&self, id: i64, mode: &str) -> Result { + if mode != "mirror" && mode != "ondemand" { + return Err(format!("unknown mode {mode:?} (want mirror|ondemand)")); + } + // Hold the folder's lock for the whole switch so no reconcile pass can be + // running over the tree we are about to evict (or start while we mount over + // it). A pass in flight holds the lock for its full duration, so `try_lock` + // failing is exactly "still syncing" — and it is the only reliable signal: + // the `state` column is still `idle` in the window between `add_sync_folder` + // inserting the row and the engine picking it up. + let lock = self.sync_lock(id); + let Ok(_guard) = lock.try_lock() else { + return Err("folder is syncing right now — wait for it to finish".to_string()); + }; + // Re-read under the lock: a pass that finished while we waited may have + // changed `state`. + let folder = self + .db + .sync_folder_get(id) + .map_err(|e| format!("db: {e:?}"))? + .ok_or_else(|| format!("no synced folder with id {id}"))?; + if folder.mode == mode { + return Ok(format!("already {mode}")); + } + let local = PathBuf::from(&folder.local_path); + + match mode { + "ondemand" => { + // Only flip a folder that is fully in sync — a failed reconcile means + // local edits could still be un-uploaded, and we are about to delete + // the local copy. + if folder.state != "idle" { + return Err(format!( + "folder is '{}', not idle — sync it before switching to on-demand", + folder.state + )); + } + let root_uid = parse_uid(&folder.remote_uid) + .ok_or_else(|| format!("bad remote uid: {}", folder.remote_uid))?; + let root = self + .rt + .block_on(self.client.enumerate_nodes(std::slice::from_ref(&root_uid))) + .map_err(|e| format!("fetch remote root: {e}"))? + .into_iter() + .next() + .ok_or_else(|| "remote folder not found".to_string())?; + + // Reclaim the disk: empty the local dir (keep it as the mountpoint). + evict_dir_contents(&local) + .map_err(|e| format!("evict {}: {e}", local.display()))?; + + let session = self.spawn_ondemand_mount(&local, root)?; + self.mounts.lock().unwrap().insert(id, session); + // Persist the mode only now that the mount is actually up. Writing it + // first would strand the folder on any failure below: the engine skips + // non-mirror folders, so an `ondemand` row with no mount is a folder + // that is neither mirrored nor browsable, and nothing retries it. + // Failing before this point leaves it `mirror`, and the next pass + // re-downloads whatever eviction removed. + self.db + .sync_folder_set_mode(id, "ondemand") + .map_err(|e| format!("db: {e:?}"))?; + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + self.db.sync_folder_set_state(id, "idle", now_secs()).ok(); + info!(id, path = %local.display(), "mounted sync folder on-demand"); + Ok(format!("{} is now on-demand", local.display())) + } + _ => { + // ondemand→mirror: tear down the secondary mount first. + if let Some(session) = self.mounts.lock().unwrap().remove(&id) + && let Err(e) = session.umount_and_join() + { + warn!(id, error = %e, "unmount on-demand folder failed"); + } + // The evicted local tree makes the old baseline lie ("everything + // deleted locally"); clear it so the reconcile is a pure download. + self.db + .sync_entries_clear(id) + .map_err(|e| format!("db: {e:?}"))?; + self.db + .sync_folder_set_mode(id, "mirror") + .map_err(|e| format!("db: {e:?}"))?; + let _ = self.sync_tx.send(sync::SyncMsg::Rewatch); + let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id)); + info!(id, path = %local.display(), "restored sync folder to mirror"); + Ok(format!( + "{} is mirroring again; downloading", + local.display() + )) + } + } + } + + /// Spawn a secondary FUSE session rooted at `root` over `local` on a forked + /// inode space. Clears any stale kernel mount first (a crashed run can leave + /// one, which would fail the fresh mount with EBUSY). + fn spawn_ondemand_mount(&self, local: &Path, root: Node) -> Result { + clear_stale_mount(local); + let mut config = Config::default(); + config.mount_options = vec![ + MountOption::FSName("protondrive".to_string()), + MountOption::Subtype("protondrive".to_string()), + MountOption::DefaultPermissions, + ]; + let fs = ProtonFs::new(self.fork_state(), root); + Session::new(fs, local, &config) + .and_then(|s| s.spawn()) + .map_err(|e| format!("mount {}: {e}", local.display())) + } + + /// Re-establish FUSE mounts for folders left in `ondemand` mode across a daemon + /// restart (their local dirs are empty on disk — the files live in the cloud). + /// Best-effort per folder: a missing local path or a failed remote fetch marks + /// the folder `error` and moves on rather than aborting the rest. Runs on its + /// own thread from `mount` so the network fetches never block startup + /// (devices.md Phase 4). + fn restore_ondemand_mounts(&self) { + let folders = match self.db.sync_folder_list() { + Ok(f) => f, + Err(e) => { + warn!(error = ?e, "restore on-demand: cannot list folders"); + return; + } + }; + for folder in folders { + if folder.mode != "ondemand" { + continue; + } + let local = PathBuf::from(&folder.local_path); + if !local.is_dir() { + warn!(id = folder.id, path = %local.display(), "restore on-demand: local path missing"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + // An `ondemand` folder's local dir is empty by construction — the switch + // evicts it. Finding files there means the row is lying (a switch that + // died between persisting the mode and evicting), and mounting over them + // would hide real local data behind the remote tree. Leave the files + // alone and let the user resolve it. + match dir_is_empty(&local) { + Ok(true) => {} + Ok(false) => { + warn!( + id = folder.id, + path = %local.display(), + "restore on-demand: local dir is not empty; refusing to mount over it" + ); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + Err(e) => { + warn!(id = folder.id, path = %local.display(), error = %e, "restore on-demand: cannot read local dir"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + } + let Some(root_uid) = parse_uid(&folder.remote_uid) else { + warn!(id = folder.id, "restore on-demand: bad remote uid"); + continue; + }; + let root = match self + .rt + .block_on(self.client.enumerate_nodes(std::slice::from_ref(&root_uid))) + { + Ok(v) => match v.into_iter().next() { + Some(n) => n, + None => { + warn!(id = folder.id, "restore on-demand: remote folder gone"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + }, + Err(e) => { + warn!(id = folder.id, error = %e, "restore on-demand: fetch remote failed"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + continue; + } + }; + match self.spawn_ondemand_mount(&local, root) { + Ok(session) => { + self.mounts.lock().unwrap().insert(folder.id, session); + let _ = self.db.sync_folder_set_state(folder.id, "idle", now_secs()); + info!(id = folder.id, path = %local.display(), "remounted on-demand folder"); + } + Err(e) => { + warn!(id = folder.id, error = %e, "restore on-demand: mount failed"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + } + } + } + } + // ---- sharing a node --------------------------------------------------- /// Invite Proton and/or external emails to the node at `rel` at `role`. @@ -2449,10 +3004,11 @@ impl Core { // ---- activity log ----------------------------------------------------- - /// Record one entry in the in-memory activity log, evicting the oldest when the - /// buffer is full. Lock-guarded and cheap; callable from any thread (the bulk - /// upload logs its summary from a background thread). - fn log_activity( + /// Append one entry to the activity log. Callable from any thread (the sync + /// engine and the bulk uploader both log from background tasks). A failed + /// write is logged and dropped: the feed is a record of work, never a reason + /// to fail the work itself. + pub(crate) fn log_activity( &self, kind: ActivityKind, target: impl Into, @@ -2466,17 +3022,70 @@ impl Core { detail: detail.into(), ok, }; - let mut log = self.activity.lock().unwrap(); - if log.len() >= ACTIVITY_CAP { - log.pop_front(); + if let Err(e) = self.db.activity_add(&entry) { + warn!(error = ?e, "could not record activity"); } - log.push_back(entry); } /// The recent activity, newest first, capped at `limit` entries. fn list_activity(&self, limit: usize) -> Vec { - let log = self.activity.lock().unwrap(); - log.iter().rev().take(limit).cloned().collect() + match self.db.activity_list(limit) { + Ok(items) => items, + Err(e) => { + warn!(error = ?e, "could not read activity"); + Vec::new() + } + } + } + + // ---- live sync progress ----------------------------------------------- + + /// Start tracking a reconcile pass over `folder_id`, in [`SyncPhase::Scanning`]. + pub(crate) fn progress_begin(&self, folder_id: i64) { + self.sync_progress.lock().unwrap().insert( + folder_id, + SyncProgress { + phase: SyncPhase::Scanning, + done: 0, + total: 0, + current: String::new(), + }, + ); + } + + /// Apply `f` to a folder's live progress, if a pass is running for it. + fn progress_update(&self, folder_id: i64, f: impl FnOnce(&mut SyncProgress)) { + if let Some(p) = self.sync_progress.lock().unwrap().get_mut(&folder_id) { + f(p); + } + } + + /// Note that `n` more items have been queued for this pass, and that it has + /// moved on from scanning to applying the diff. + pub(crate) fn progress_queued(&self, folder_id: i64, n: usize) { + self.progress_update(folder_id, |p| { + p.phase = SyncPhase::Applying; + p.total += n; + }); + } + + /// Note that work has started on `name` (shown as the pass's current item). + pub(crate) fn progress_started(&self, folder_id: i64, name: &str) { + self.progress_update(folder_id, |p| p.current = name.to_string()); + } + + /// Note that one queued item finished, whether it succeeded or not. + pub(crate) fn progress_finished(&self, folder_id: i64) { + self.progress_update(folder_id, |p| { + p.done += 1; + p.current.clear(); + }); + } + + /// Stop tracking a pass — no progress is reported for the folder until the + /// next [`progress_begin`](Self::progress_begin). + pub(crate) fn progress_end(&self, folder_id: i64) { + self.sync_progress.lock().unwrap().remove(&folder_id); } } @@ -2622,6 +3231,50 @@ fn now_secs() -> i64 { .unwrap_or(0) } +/// This machine's hostname, used to name (and later recover) its Proton Drive +/// Device. Reads the live kernel hostname, falling back to a generic label. +fn this_hostname() -> String { + std::fs::read_to_string("/proc/sys/kernel/hostname") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "Linux device".to_string()) +} + +/// Whether `dir` has no entries. +fn dir_is_empty(dir: &Path) -> std::io::Result { + Ok(std::fs::read_dir(dir)?.next().is_none()) +} + +/// Delete everything inside `dir` but keep `dir` itself (it stays as the FUSE +/// mountpoint). Used when a `mirror` folder flips to `ondemand`: the local files +/// are the disk we're reclaiming (devices.md Phase 3). +fn evict_dir_contents(dir: &Path) -> std::io::Result<()> { + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() && !path.is_symlink() { + std::fs::remove_dir_all(&path)?; + } else { + std::fs::remove_file(&path)?; + } + } + Ok(()) +} + +/// Convert a stored synced folder into its wire form for the front-ends, with +/// the live progress of its pass when one is running. +fn sync_folder_info(f: StoredSyncFolder, progress: Option) -> SyncFolderInfo { + SyncFolderInfo { + id: f.id, + local_path: f.local_path, + remote_uid: f.remote_uid, + mode: f.mode, + state: f.state, + last_sync: f.last_sync, + progress, + } +} + /// A coarse MIME type guessed from a file name's extension; Proton stores this /// on the node but an exact value is not required for correctness. fn media_type_for(name: &str) -> &'static str { @@ -3624,13 +4277,18 @@ async fn run_event_sync( /// it is deliberately kept off every hot path: it never runs on a FUSE or /// control-socket thread, and it excludes the mountpoint (walking it would fault /// every remote node in through FUSE, defeating on-demand hydration). -fn run_local_index(db: Arc, indexing: Arc, mountpoint: PathBuf) { +fn run_local_index( + db: Arc, + indexing: Arc, + transfers: Arc, + mountpoint: PathBuf, +) { loop { let age = db.local_indexed_at().ok().flatten(); let stale = age.is_none_or(|at| now_secs().saturating_sub(at) >= LOCAL_INDEX_TTL.as_secs() as i64); if stale { - scan_local_once(&db, &indexing, &mountpoint); + scan_local_once(&db, &indexing, &transfers, &mountpoint); } std::thread::sleep(LOCAL_INDEX_CHECK); } @@ -3639,7 +4297,16 @@ fn run_local_index(db: Arc, indexing: Arc, mountpoint: PathBuf) /// Walk `$HOME` once and replace the local-file index with what it finds. /// Batches stream straight into SQLite, so peak memory is one batch — not the /// whole home directory. -fn scan_local_once(db: &Db, indexing: &AtomicBool, mountpoint: &Path) { +/// +/// Reports itself as a job: the first scan after a fresh install walks the whole +/// home directory, and `indexing` alone only tells the launcher prompt to say +/// "still indexing" — nothing else showed that the daemon was busy. +fn scan_local_once( + db: &Db, + indexing: &AtomicBool, + transfers: &Arc, + mountpoint: &Path, +) { let dirs = match AppDirs::new() { Ok(d) => d, Err(e) => { @@ -3663,6 +4330,10 @@ fn scan_local_once(db: &Db, indexing: &AtomicBool, mountpoint: &Path) { indexing.store(true, Ordering::Relaxed); let started = Instant::now(); + // The walk has no idea how many files it will find, so the job counts what it + // has seen and stays indeterminate. + let job = transfers.begin_job("Indexing this computer"); + job.detail("Scanning your files"); let walked = localindex::scan(&[home], &excludes, |batch| { if let Err(e) = db.local_upsert_batch(generation, &batch) { warn!(error = %e, "local index: batch write failed"); @@ -3671,6 +4342,7 @@ fn scan_local_once(db: &Db, indexing: &AtomicBool, mountpoint: &Path) { // Prune what this scan did not see and rebuild the FTS index over the rest, // even if some batches failed — a partial index still beats none. + job.detail("Building the search index"); match db.local_finish_scan(generation, now_secs()) { Ok(indexed) => info!( walked, @@ -4035,6 +4707,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U } Ok(CtlRequest::GetQueueStatus) => CtlResponse::Transfers { items: core.transfers.snapshot(), + jobs: core.jobs_snapshot(), }, Ok(CtlRequest::SetCacheBudget { bytes }) => { core.cache.set_budget(bytes); @@ -4072,6 +4745,69 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U }, Err(e) => CtlResponse::Error { message: e }, }, + Ok(CtlRequest::AddSyncFolder { local_path }) => { + // Registering the device and uploading a folder tree far outlasts the + // socket read timeout, so ack immediately and work on a background + // thread. The folder appears in ListSyncFolders once the row lands; + // completion (and any failures) go to the activity log. + let core = core.clone(); + let path = PathBuf::from(&local_path); + std::thread::spawn(move || { + let started = Instant::now(); + match core.add_sync_folder(&path) { + Ok(folder) => { + let name = Path::new(&folder.local_path) + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(&folder.local_path) + .to_string(); + core.log_activity( + ActivityKind::Upload, + format!("synced {name}"), + human_duration(started.elapsed()), + folder.state != "error", + ); + } + Err(e) => { + warn!(error = %e, "add sync folder failed"); + core.log_activity(ActivityKind::Upload, "add sync folder", &e, false); + } + } + }); + CtlResponse::Ok { + message: format!("adding {local_path}"), + } + } + Ok(CtlRequest::ListSyncFolders) => match core.list_sync_folders() { + Ok(items) => CtlResponse::SyncFolders { items }, + Err(e) => CtlResponse::Error { message: e }, + }, + Ok(CtlRequest::RemoveSyncFolder { id, delete_remote }) => { + match core.remove_sync_folder(id, delete_remote) { + Ok(()) => CtlResponse::Ok { + message: "removed synced folder".to_string(), + }, + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::SetSyncFolderMode { id, mode }) => { + match core.set_sync_folder_mode(id, &mode) { + Ok(message) => { + core.log_activity(ActivityKind::Upload, &message, "", true); + CtlResponse::Ok { message } + } + Err(e) => CtlResponse::Error { message: e }, + } + } + Ok(CtlRequest::SyncNow { id }) => { + core.sync_now(id); + CtlResponse::Ok { + message: match id { + Some(id) => format!("reconciling folder {id}"), + None => "reconciling all folders".to_string(), + }, + } + } Ok(CtlRequest::ShareNode { path, emails, @@ -4323,6 +5059,11 @@ pub fn mount( .map_err(|e| std::io::Error::other(format!("fetch My Files root: {e}")))?; let scope = root.tree_event_scope_id(); + // The folder-sync engine (devices.md Phase 2) runs on its own thread and is + // nudged over this channel; the sender lives in Core so control-socket + // handlers can trigger reconciles. + let (sync_tx, sync_rx) = std::sync::mpsc::channel::(); + let core = Core { client: client.clone(), rt: rt.clone(), @@ -4342,18 +5083,36 @@ pub fn mount( thumb_gen: Arc::new(Mutex::new(HashSet::new())), transfers: TransferRegistry::new(), indexing: Arc::new(AtomicBool::new(false)), - activity: Arc::new(Mutex::new(VecDeque::with_capacity(ACTIVITY_CAP))), + sync_progress: Arc::new(Mutex::new(HashMap::new())), + sync_tx, + mounts: Arc::new(Mutex::new(HashMap::new())), + sync_locks: Arc::new(Mutex::new(HashMap::new())), }; + // Start the folder-sync engine. It watches every mirror folder, polls the + // remotes, and reconciles on its own thread — never in front of a FUSE call. + sync::spawn(core.clone(), sync_rx); + + // Re-establish on-demand mounts left over from a previous run (devices.md + // Phase 4). On its own thread: each remount fetches a remote node, and we + // must not block the main mount below on the network. + { + let core = core.clone(); + std::thread::Builder::new() + .name("pdfs-restore-ondemand".into()) + .spawn(move || core.restore_ondemand_mounts())?; + } + // Keep the launcher prompt's "This computer" index fresh. Its own thread: // the walk is I/O-heavy and must never sit in front of a FUSE callback. { let db = core.db.clone(); let indexing = core.indexing.clone(); + let transfers = core.transfers.clone(); let mountpoint = mountpoint.to_path_buf(); std::thread::Builder::new() .name("pdfs-localindex".into()) - .spawn(move || run_local_index(db, indexing, mountpoint))?; + .spawn(move || run_local_index(db, indexing, transfers, mountpoint))?; } let mut config = Config::default(); @@ -4454,6 +5213,15 @@ pub fn mount( } }; + // Unmount every on-demand sync folder too, or the kernel mounts linger as + // stale and the next start fails with EBUSY (devices.md Phase 3). + let secondaries: Vec<_> = core.mounts.lock().unwrap().drain().collect(); + for (id, session) in secondaries { + if let Err(e) = session.umount_and_join() { + warn!(id, error = %e, "unmount on-demand folder failed"); + } + } + let _ = std::fs::remove_file(control_socket); Ok(outcome) } diff --git a/crates/pdfs-fuse/src/sync.rs b/crates/pdfs-fuse/src/sync.rs new file mode 100644 index 0000000..0d880da --- /dev/null +++ b/crates/pdfs-fuse/src/sync.rs @@ -0,0 +1,1457 @@ +//! Two-way folder sync engine (devices.md Phase 2). +//! +//! Each `mirror`-mode synced folder is kept mirrored between a local directory +//! and a folder under this machine's Proton Drive device. A local filesystem +//! watcher ([`notify`]) and a periodic remote poll both feed a debounced +//! reconcile pass that diffs three states — the current local tree, the current +//! remote tree, and the last-synced baseline in `sync_entry` — to decide, per +//! path, which side changed and how to propagate it. +//! +//! Change detection is `(mtime, size)` on each side (no cheap content hash is +//! exposed). The baseline is refreshed from ground truth after every applied +//! action, so a file just uploaded or downloaded reads as "unchanged" on the +//! next pass and never ping-pongs. + +use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::mpsc::Receiver; +use std::time::{Duration, SystemTime}; + +use notify::{RecursiveMode, Watcher}; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use proton_drive_rs::{Node, NodeKind}; +use tracing::{info, warn}; + +use crate::transfers::{CountingWriter, OwnedCountingReader}; +use crate::{Core, now_secs, parse_uid}; +use pdfs_core::control::{ActivityKind, TransferDirection}; +use pdfs_core::db::{StoredSyncEntry, StoredSyncFolder}; + +/// Coalesce a burst of filesystem events into a single reconcile per folder. +const DEBOUNCE: Duration = Duration::from_secs(2); +/// How often to re-walk remotes so changes made on other clients are pulled in. +const POLL_INTERVAL: Duration = Duration::from_secs(120); +/// How many uploads/downloads/folder-creations a reconcile runs at once. The +/// per-item work is a network round-trip, so a folder with thousands of files is +/// unusable done one-at-a-time; this bounds concurrency like the bulk-upload +/// engine ([`crate::run_uploads`]) does. +const SYNC_CONCURRENCY: usize = 8; + +/// A message to the sync engine's control thread. +pub(crate) enum SyncMsg { + /// Reconcile one folder by id (a local change, or a targeted `SyncNow`). + Reconcile(i64), + /// Reconcile every mirror folder (periodic poll, or startup). + ReconcileAll, + /// Re-read the folder set and adjust the filesystem watches (after add/remove). + Rewatch, +} + +/// Start the sync engine: a control thread that owns the filesystem watcher and +/// serialises reconcile passes, plus a periodic poll thread. The engine receives +/// on `rx`; senders live in [`Core::sync_tx`]. +pub(crate) fn spawn(core: Core, rx: Receiver) { + if let Err(e) = std::thread::Builder::new() + .name("pdfs-sync".into()) + .spawn(move || engine_loop(core, rx)) + { + warn!(error = %e, "failed to start sync engine"); + } +} + +/// The engine control loop. Single-threaded so reconcile passes never overlap. +fn engine_loop(core: Core, rx: Receiver) { + // Paths the watcher currently covers, mapped to their folder id, so an event + // path can be resolved back to the folder it belongs to. + let watched: Mutex> = Mutex::new(Vec::new()); + + let tx_events = core.sync_tx.clone(); + let mut watcher = + match notify::recommended_watcher(move |res: notify::Result| { + let Ok(event) = res else { return }; + // Ignore pure metadata/access noise; content changes are what matter. + if matches!(event.kind, notify::EventKind::Access(_)) { + return; + } + let _ = tx_events.send(SyncMsg::ReconcileAll); + }) { + Ok(w) => w, + Err(e) => { + warn!(error = %e, "filesystem watcher unavailable; sync is poll-only"); + // Still run the loop so polling and explicit SyncNow work. + poll_only_loop(&core, rx); + return; + } + }; + + // Periodic remote poll. + { + let tx = core.sync_tx.clone(); + std::thread::Builder::new() + .name("pdfs-sync-poll".into()) + .spawn(move || { + loop { + std::thread::sleep(POLL_INTERVAL); + if tx.send(SyncMsg::ReconcileAll).is_err() { + break; + } + } + }) + .ok(); + } + + rewatch(&core, &mut watcher, &watched); + reconcile_all(&core); + + while let Ok(msg) = rx.recv() { + let mut ids: HashSet = HashSet::new(); + let mut all = false; + let mut do_rewatch = false; + classify(msg, &mut ids, &mut all, &mut do_rewatch); + // Debounce: let a burst settle, then drain everything queued behind it. + std::thread::sleep(DEBOUNCE); + while let Ok(m) = rx.try_recv() { + classify(m, &mut ids, &mut all, &mut do_rewatch); + } + + if do_rewatch { + rewatch(&core, &mut watcher, &watched); + } + if all { + reconcile_all(&core); + } else { + for id in ids { + if let Ok(Some(folder)) = core.db.sync_folder_get(id) { + core.reconcile_folder(&folder); + } + } + } + } +} + +/// Fallback loop when no filesystem watcher could be created: poll + SyncNow only. +fn poll_only_loop(core: &Core, rx: Receiver) { + { + let tx = core.sync_tx.clone(); + std::thread::spawn(move || { + loop { + std::thread::sleep(POLL_INTERVAL); + if tx.send(SyncMsg::ReconcileAll).is_err() { + break; + } + } + }); + } + reconcile_all(core); + while let Ok(msg) = rx.recv() { + let mut ids: HashSet = HashSet::new(); + let mut all = false; + let mut do_rewatch = false; + classify(msg, &mut ids, &mut all, &mut do_rewatch); + std::thread::sleep(DEBOUNCE); + while let Ok(m) = rx.try_recv() { + classify(m, &mut ids, &mut all, &mut do_rewatch); + } + if all { + reconcile_all(core); + } else { + for id in ids { + if let Ok(Some(folder)) = core.db.sync_folder_get(id) { + core.reconcile_folder(&folder); + } + } + } + } +} + +fn classify(msg: SyncMsg, ids: &mut HashSet, all: &mut bool, rewatch: &mut bool) { + match msg { + SyncMsg::Reconcile(id) => { + ids.insert(id); + } + SyncMsg::ReconcileAll => *all = true, + SyncMsg::Rewatch => *rewatch = true, + } +} + +/// Reconcile every mirror folder in turn. +fn reconcile_all(core: &Core) { + let folders = match core.db.sync_folder_list() { + Ok(f) => f, + Err(e) => { + warn!(error = ?e, "sync: cannot list folders"); + return; + } + }; + for folder in folders { + core.reconcile_folder(&folder); + } +} + +/// Bring the filesystem watches in line with the current mirror-folder set: +/// watch newly-added folders, drop removed ones. +fn rewatch( + core: &Core, + watcher: &mut notify::RecommendedWatcher, + watched: &Mutex>, +) { + let want: Vec<(PathBuf, i64)> = match core.db.sync_folder_list() { + Ok(f) => f + .into_iter() + .filter(|f| f.mode == "mirror") + .map(|f| (PathBuf::from(f.local_path), f.id)) + .collect(), + Err(e) => { + warn!(error = ?e, "sync: cannot list folders to watch"); + return; + } + }; + let mut have = watched.lock().unwrap(); + // Drop watches no longer wanted. + have.retain(|(path, _)| { + if want.iter().any(|(p, _)| p == path) { + true + } else { + let _ = watcher.unwatch(path); + false + } + }); + // Add newly wanted watches. + for (path, id) in &want { + if have.iter().any(|(p, _)| p == path) { + continue; + } + match watcher.watch(path, RecursiveMode::Recursive) { + Ok(()) => have.push((path.clone(), *id)), + Err(e) => warn!(path = %path.display(), error = %e, "sync: watch failed"), + } + } +} + +// ---- reconcile ------------------------------------------------------------ + +/// One item found while walking a local tree. +struct LocalItem { + is_dir: bool, + mtime: i64, + size: i64, +} + +/// One item found while walking a remote tree. +struct RemoteItem { + uid: NodeUid, + is_dir: bool, + mtime: i64, + size: i64, +} + +/// The result of a reconcile pass: what it moved, how many paths were kept as +/// conflict copies, and how many failed to apply (and so still need another +/// pass). The counts drive both the folder's state and its activity summary. +#[derive(Default)] +struct Outcome { + uploaded: usize, + downloaded: usize, + created: usize, + deleted: usize, + conflicts: usize, + errors: usize, +} + +impl Outcome { + /// Fold in one applied op. + fn record(&mut self, applied: &Applied) { + match applied { + Applied::Dir(..) => self.created += 1, + Applied::Uploaded => self.uploaded += 1, + Applied::Downloaded => self.downloaded += 1, + Applied::Conflict => self.conflicts += 1, + } + } + + /// Whether the pass moved nothing at all — the common case on a poll of an + /// unchanged folder, which should not add a line to the activity feed. + fn is_empty(&self) -> bool { + self.uploaded == 0 + && self.downloaded == 0 + && self.created == 0 + && self.deleted == 0 + && self.conflicts == 0 + && self.errors == 0 + } + + /// A human summary of the pass: "3 uploaded, 1 downloaded, 2 failed". + fn summary(&self) -> String { + let mut parts = Vec::new(); + for (n, label) in [ + (self.uploaded, "uploaded"), + (self.downloaded, "downloaded"), + (self.created, "folder(s) created"), + (self.deleted, "deleted"), + (self.conflicts, "conflicted"), + (self.errors, "failed"), + ] { + if n > 0 { + parts.push(format!("{n} {label}")); + } + } + parts.join(", ") + } +} + +/// A network operation queued during classification and run concurrently in a +/// per-depth batch. Parent uids are resolved up front (the parent folder is one +/// depth shallower and already created), so tasks share nothing mutable. +enum Pending { + /// Create a new remote folder under `parent`. + CreateDir { rel: String, parent: NodeUid }, + /// Upload a brand-new local file into `parent`. + UploadNew { rel: String, parent: NodeUid }, + /// Upload a changed local file as a new revision of `uid`. + UploadRevision { rel: String, uid: NodeUid }, + /// Download remote `uid` to the local path, stamping `mtime`. `size` is the + /// remote's reported size, used as the transfer's expected total. + Download { + rel: String, + uid: NodeUid, + mtime: i64, + size: i64, + }, + /// Both sides changed: set the local copy aside, then download remote `uid`. + Conflict { + rel: String, + uid: NodeUid, + mtime: i64, + size: i64, + }, +} + +impl Pending { + /// The path this op acts on, relative to the folder root. + fn rel(&self) -> &str { + match self { + Pending::CreateDir { rel, .. } + | Pending::UploadNew { rel, .. } + | Pending::UploadRevision { rel, .. } + | Pending::Download { rel, .. } + | Pending::Conflict { rel, .. } => rel, + } + } + + /// How this op reads in the activity feed. + fn kind(&self) -> ActivityKind { + match self { + Pending::CreateDir { .. } => ActivityKind::CreateFolder, + Pending::UploadNew { .. } | Pending::UploadRevision { .. } => ActivityKind::Upload, + Pending::Download { .. } | Pending::Conflict { .. } => ActivityKind::Download, + } + } + + /// The activity line's detail, which distinguishes ops that share a kind. + fn detail(&self) -> &'static str { + match self { + Pending::CreateDir { .. } => "on Drive", + Pending::UploadNew { .. } => "new file", + Pending::UploadRevision { .. } => "new version", + Pending::Download { .. } => "from Drive", + Pending::Conflict { .. } => "local changes kept as a conflict copy", + } + } +} + +/// What a [`Pending`] op did, so the reconcile can update shared state on the +/// engine thread (never inside a task). +enum Applied { + /// A folder was created; register `uid` in `remote_dirs` for its children. + Dir(String, NodeUid), + /// A local file was uploaded (new file or new revision). + Uploaded, + /// A remote file was downloaded. + Downloaded, + /// A conflict copy was kept. + Conflict, +} + +impl Core { + /// Reconcile one synced folder, updating its `state` column. `ondemand` + /// folders are live FUSE mounts, not mirrored, so they are skipped here. + pub(crate) fn reconcile_folder(&self, folder: &StoredSyncFolder) { + if folder.mode != "mirror" { + return; + } + // Hold the folder's lock for the whole pass so a mode switch can't evict the + // local tree (and mount FUSE over it) while we walk and upload it. + let lock = self.sync_lock(folder.id); + let _guard = lock.lock().unwrap(); + // `folder` was read before the lock; a switch may have landed in between, so + // re-read and re-check the mode rather than trusting the snapshot. + match self.db.sync_folder_get(folder.id) { + Ok(Some(current)) if current.mode == "mirror" => {} + Ok(Some(_)) => return, + Ok(None) => return, + Err(e) => { + warn!(id = folder.id, error = ?e, "sync: cannot re-read folder; skipping"); + return; + } + } + let Some(remote_root) = parse_uid(&folder.remote_uid) else { + warn!(id = folder.id, "sync: bad remote uid; skipping"); + return; + }; + let local_root = PathBuf::from(&folder.local_path); + if !local_root.is_dir() { + warn!(id = folder.id, path = %local_root.display(), "sync: local root missing"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + return; + } + + let _ = self + .db + .sync_folder_set_state(folder.id, "syncing", folder.last_sync); + let name = base_name(&folder.local_path); + self.progress_begin(folder.id); + let result = self.do_reconcile(folder.id, &local_root, &remote_root); + self.progress_end(folder.id); + match result { + Ok(outcome) => { + // A folder only reaches `idle` when every path applied cleanly — + // an un-uploaded file must keep it out of `idle` so it can't be + // switched to on-demand (which evicts the local copy). + let state = if outcome.errors > 0 { + "error" + } else if outcome.conflicts > 0 { + "conflict" + } else { + "idle" + }; + let _ = self.db.sync_folder_set_state(folder.id, state, now_secs()); + // Summarise a pass that actually moved something. A poll over an + // unchanged folder does nothing and says nothing — otherwise the + // feed would fill with "synced, 0 changes" every two minutes. + if !outcome.is_empty() { + self.log_activity( + ActivityKind::Sync, + name, + outcome.summary(), + outcome.errors == 0, + ); + } + // Surface a new conflict once (not on every poll while it persists). + if state == "conflict" && folder.state != "conflict" { + self.log_activity( + ActivityKind::Sync, + format!("sync conflict in {name}"), + format!("{} file(s) kept as conflict copies", outcome.conflicts), + false, + ); + } + // Surface partial failures once, too — the pass still uploaded + // everything it could, but some paths need another attempt. + if state == "error" && folder.state != "error" { + self.log_activity( + ActivityKind::Sync, + format!("sync incomplete for {name}"), + format!("{} item(s) failed; will retry", outcome.errors), + false, + ); + } + } + Err(e) => { + warn!(id = folder.id, error = %e, "sync: reconcile failed"); + let _ = self + .db + .sync_folder_set_state(folder.id, "error", now_secs()); + if folder.state != "error" { + self.log_activity( + ActivityKind::Sync, + format!("sync failed for {name}"), + e, + false, + ); + } + } + } + } + + /// The diff-and-apply core. A single path failing to apply (a transient + /// upload error, an unreadable file, a name collision) must not abort the + /// whole pass and strand every other path — otherwise one bad file in a + /// large folder leaves the sync permanently incomplete and blocks the + /// on-demand switch. So per-path failures are logged and counted in + /// [`Outcome::errors`], and the walk continues. Only a failure to establish + /// the diff itself (the walks / baseline load) aborts with `Err`. + fn do_reconcile( + &self, + folder_id: i64, + local_root: &Path, + remote_root: &NodeUid, + ) -> Result { + let mut local: HashMap = HashMap::new(); + walk_local(local_root, local_root, &mut local)?; + + // Loaded before the walk, which uses it to tell which remote files are + // unchanged and so can skip decrypting their claimed size. + let baseline = self + .db + .sync_entries(folder_id) + .map_err(|e| format!("load baseline: {e:?}"))?; + + let mut remote: HashMap = HashMap::new(); + let mut remote_dirs: HashMap = HashMap::new(); + remote_dirs.insert(String::new(), remote_root.clone()); + self.walk_remote(remote_root, "", &mut remote, &mut remote_dirs, &baseline)?; + + // Union of every path across the three states, shallow paths first so a + // parent folder is created before its children are placed inside it. + let mut paths: HashSet = HashSet::new(); + paths.extend(local.keys().cloned()); + paths.extend(remote.keys().cloned()); + paths.extend(baseline.keys().cloned()); + let mut order: Vec = paths.into_iter().collect(); + order.sort_by_key(|p| p.matches('/').count()); + + let mut outcome = Outcome::default(); + // Folders to delete, collected here and removed deepest-first at the end + // so a parent is never removed before its children. + let mut delete_local_dirs: Vec = Vec::new(); + let mut delete_remote_dirs: Vec<(String, NodeUid)> = Vec::new(); + + // Single depth-ascending pass. Network work (folder creation, file + // upload/download) is queued into `batch` and flushed concurrently at + // each depth boundary — so a parent folder is created before the deeper + // paths inside it are classified (their parent uid is then in + // `remote_dirs`), while everything within a depth runs in parallel. + // Interleaving by depth also means files start uploading as soon as + // their folder exists, instead of after every folder in the tree. + // Cheap, dependency-free work (local mkdir, baseline rows, entry + // removals, deletions) is done inline; a per-item failure is logged and + // counted, never fatal. + let mut batch: Vec = Vec::new(); + let mut batch_depth = 0usize; + + for rel in &order { + let depth = rel.matches('/').count(); + if depth > batch_depth { + self.flush_batch( + folder_id, + local_root, + &mut remote_dirs, + std::mem::take(&mut batch), + &mut outcome, + ); + batch_depth = depth; + } + + let l = local.get(rel); + let r = remote.get(rel); + let is_dir = l.map(|x| x.is_dir).or(r.map(|x| x.is_dir)).unwrap_or(false); + let in_base = baseline.contains_key(rel); + + if is_dir { + match (l.is_some(), r.is_some()) { + (true, true) => { + if let Err(e) = self.baseline_dir(folder_id, rel, &remote[rel].uid) { + warn!(rel, error = %e, "sync: folder step failed; continuing"); + outcome.errors += 1; + } + } + (true, false) if !in_base => { + // New local folder → create remotely (batched). + match remote_dirs.get(parent_rel(rel)) { + Some(parent) => batch.push(Pending::CreateDir { + rel: rel.clone(), + parent: parent.clone(), + }), + None => { + warn!(rel, "sync: no remote parent for folder; continuing"); + outcome.errors += 1; + } + } + } + (true, false) => { + // In baseline, gone remotely → the remote side deleted it. + delete_local_dirs.push(rel.clone()); + } + (false, true) if !in_base => { + // New remote folder → create locally (cheap, inline). + if let Err(e) = std::fs::create_dir_all(local_root.join(rel_to_path(rel))) + .map_err(|e| format!("mkdir {rel}: {e}")) + .and_then(|()| self.baseline_dir(folder_id, rel, &remote[rel].uid)) + { + warn!(rel, error = %e, "sync: folder step failed; continuing"); + outcome.errors += 1; + } + } + (false, true) => { + // In baseline, gone locally → the local side deleted it. + delete_remote_dirs.push((rel.clone(), remote[rel].uid.clone())); + } + (false, false) => { + let _ = self.db.sync_entry_remove(folder_id, rel); + } + } + continue; + } + + // File. + let base = baseline.get(rel); + let local_changed = l.is_some_and(|l| { + base.is_none_or(|b| b.local_mtime != l.mtime || b.local_size != l.size) + }); + let remote_changed = + r.is_some_and(|r| base.is_none_or(|b| remote_sig(b) != Some((r.mtime, r.size)))); + + match (l, r) { + (Some(_), Some(r)) => { + // With no baseline, both sides read as "changed" and fall to the + // conflict arm — the safe default for a folder re-added over + // existing remote content. + if !local_changed && !remote_changed { + // Both untouched; baseline already matches. + } else if local_changed && !remote_changed { + batch.push(Pending::UploadRevision { + rel: rel.clone(), + uid: r.uid.clone(), + }); + } else if remote_changed && !local_changed { + batch.push(Pending::Download { + rel: rel.clone(), + uid: r.uid.clone(), + mtime: r.mtime, + size: r.size, + }); + } else { + batch.push(Pending::Conflict { + rel: rel.clone(), + uid: r.uid.clone(), + mtime: r.mtime, + size: r.size, + }); + } + } + (Some(_), None) => { + if base.is_none() || local_changed { + // New local file, or remote deleted while local changed → + // (re)upload it as a new remote file. + match remote_dirs.get(parent_rel(rel)) { + Some(parent) => batch.push(Pending::UploadNew { + rel: rel.clone(), + parent: parent.clone(), + }), + None => { + warn!(rel, "sync: no remote parent for file; continuing"); + outcome.errors += 1; + } + } + } else { + // Remote deleted, local untouched → delete local. + let _ = std::fs::remove_file(local_root.join(rel_to_path(rel))); + let _ = self.db.sync_entry_remove(folder_id, rel); + self.log_activity( + ActivityKind::Trash, + base_name(rel), + "removed locally", + true, + ); + outcome.deleted += 1; + } + } + (None, Some(r)) => { + if base.is_none() || remote_changed { + // New remote file, or local deleted while remote changed → + // (re)download it. + batch.push(Pending::Download { + rel: rel.clone(), + uid: r.uid.clone(), + mtime: r.mtime, + size: r.size, + }); + } else { + // Local deleted, remote untouched → delete remote. + if let Err(e) = self + .rt + .block_on(self.client.trash_nodes(std::slice::from_ref(&r.uid))) + { + warn!(rel, error = %e, "sync: trash remote failed"); + self.log_activity( + ActivityKind::Trash, + base_name(rel), + e.to_string(), + false, + ); + outcome.errors += 1; + continue; + } + let _ = self.db.sync_entry_remove(folder_id, rel); + self.log_activity( + ActivityKind::Trash, + base_name(rel), + "removed on Drive", + true, + ); + outcome.deleted += 1; + } + } + (None, None) => { + let _ = self.db.sync_entry_remove(folder_id, rel); + } + } + } + // Flush the deepest level's batch. + self.flush_batch( + folder_id, + local_root, + &mut remote_dirs, + std::mem::take(&mut batch), + &mut outcome, + ); + + // Deferred folder deletions, deepest first. + delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count())); + for rel in delete_local_dirs { + let _ = std::fs::remove_dir_all(local_root.join(rel_to_path(&rel))); + let _ = self.db.sync_entry_remove(folder_id, &rel); + self.log_activity( + ActivityKind::Trash, + base_name(&rel), + "removed locally", + true, + ); + outcome.deleted += 1; + } + delete_remote_dirs.sort_by_key(|(p, _)| std::cmp::Reverse(p.matches('/').count())); + for (rel, uid) in delete_remote_dirs { + if let Err(e) = self.rt.block_on(self.client.trash_nodes(&[uid])) { + warn!(rel, error = %e, "sync: trash remote folder failed"); + self.log_activity(ActivityKind::Trash, base_name(&rel), e.to_string(), false); + outcome.errors += 1; + continue; + } + let _ = self.db.sync_entry_remove(folder_id, &rel); + self.log_activity( + ActivityKind::Trash, + base_name(&rel), + "removed on Drive", + true, + ); + outcome.deleted += 1; + } + + Ok(outcome) + } + + /// Recursively walk a remote folder into `out`, recording every descendant's + /// relative path, and mapping each subfolder's rel path to its uid in `dirs`. + /// + /// A file's size comes from its extended attributes, which only its own node + /// key can decrypt — and unlocking that key costs an S2K derivation (tens of + /// milliseconds) *per file*, which is the bulk of a pass over a large folder. + /// So the walk enumerates cheaply ([`enumerate_nodes_light`]) and only pays + /// for the files whose modification time has moved away from `baseline`: an + /// unchanged mtime means the recorded size still stands. + /// + /// [`enumerate_nodes_light`]: proton_drive_rs::ProtonDriveClient::enumerate_nodes_light + fn walk_remote( + &self, + folder: &NodeUid, + prefix: &str, + out: &mut HashMap, + dirs: &mut HashMap, + baseline: &HashMap, + ) -> Result<(), String> { + let uids = self + .rt + .block_on(self.client.enumerate_folder_children_node_uids(folder)) + .map_err(|e| format!("enumerate {folder}: {e}"))?; + if uids.is_empty() { + return Ok(()); + } + let nodes = self + .rt + .block_on(self.client.enumerate_nodes_light(&uids)) + .map_err(|e| format!("resolve nodes: {e}"))?; + + // Files whose recorded mtime no longer matches: their size has to be + // read for real, so these are the only node keys worth unlocking. + let stale: Vec = nodes + .iter() + .filter(|n| !n.trashed && !n.is_folder()) + .filter(|n| { + unchanged_remote_size(baseline, &join_rel(prefix, &n.name), n.modification_time) + .is_none() + }) + .map(|n| n.uid.clone()) + .collect(); + let sized: HashMap = if stale.is_empty() { + HashMap::new() + } else { + self.rt + .block_on(self.client.enumerate_nodes(&stale)) + .map_err(|e| format!("resolve nodes: {e}"))? + .iter() + .map(|n| (n.uid.to_string(), remote_file_sig(n).1)) + .collect() + }; + let stale: HashSet = stale.iter().map(|u| u.to_string()).collect(); + + for node in nodes { + if node.trashed { + continue; + } + let rel = join_rel(prefix, &node.name); + if node.is_folder() { + dirs.insert(rel.clone(), node.uid.clone()); + out.insert( + rel.clone(), + RemoteItem { + uid: node.uid.clone(), + is_dir: true, + mtime: 0, + size: 0, + }, + ); + self.walk_remote(&node.uid, &rel, out, dirs, baseline)?; + } else { + let size = if stale.contains(&node.uid.to_string()) { + match sized.get(&node.uid.to_string()) { + Some(size) => *size, + // Its size was needed but could not be read (the full + // enumeration skipped it as undecryptable). Leave it out + // rather than record a stale size, exactly as a full + // walk would have. + None => continue, + } + } else { + match unchanged_remote_size(baseline, &rel, node.modification_time) { + Some(size) => size, + None => continue, + } + }; + out.insert( + rel, + RemoteItem { + uid: node.uid, + is_dir: false, + mtime: node.modification_time, + size, + }, + ); + } + } + Ok(()) + } + + /// Record a folder's baseline row (folders carry no content signature). + fn baseline_dir(&self, folder_id: i64, rel: &str, uid: &NodeUid) -> Result<(), String> { + self.db + .sync_entry_upsert( + folder_id, + &StoredSyncEntry { + rel_path: rel.to_string(), + remote_uid: Some(uid.to_string()), + local_mtime: 0, + local_size: 0, + remote_rev: None, + remote_hash: None, + }, + ) + .map_err(|e| format!("baseline dir {rel}: {e:?}")) + } + + /// Run a batch of queued [`Pending`] ops concurrently (bounded by + /// [`SYNC_CONCURRENCY`]) and fold their results back into the reconcile's + /// shared state on the engine thread: created folders are registered in + /// `remote_dirs` (so deeper paths resolve their parent), and what each op did + /// is tallied into `outcome`. Driven by `block_on` from the (non-runtime) sync + /// engine thread, spawning the tasks onto the shared runtime. + fn flush_batch( + &self, + folder_id: i64, + local_root: &Path, + remote_dirs: &mut HashMap, + batch: Vec, + outcome: &mut Outcome, + ) { + if batch.is_empty() { + return; + } + self.progress_queued(folder_id, batch.len()); + let core = self.clone(); + let root = local_root.to_path_buf(); + let results = self.rt.block_on(async move { + let sem = Arc::new(tokio::sync::Semaphore::new(SYNC_CONCURRENCY)); + let mut set = tokio::task::JoinSet::new(); + for op in batch { + let core = core.clone(); + let sem = sem.clone(); + let root = root.clone(); + set.spawn(async move { + let _permit = sem.acquire_owned().await.expect("semaphore not closed"); + core.apply_pending(folder_id, &root, op).await + }); + } + let mut out = Vec::new(); + while let Some(joined) = set.join_next().await { + match joined { + Ok(result) => out.push(result), + Err(e) => warn!(error = %e, "sync: task panicked"), + } + } + out + }); + for result in results { + match result { + Ok(applied) => { + outcome.record(&applied); + if let Applied::Dir(rel, uid) = applied { + remote_dirs.insert(rel, uid); + } + } + Err(e) => { + warn!(error = %e, "sync: step failed; continuing"); + outcome.errors += 1; + } + } + } + } + + /// Apply one [`Pending`] op (async, so it can run concurrently in a batch), + /// reporting it to the folder's live progress and to the activity feed. Each + /// item logs its own line — the feed is meant to answer "what is it doing", + /// which a pass-level summary alone cannot. + async fn apply_pending( + &self, + folder_id: i64, + local_root: &Path, + op: Pending, + ) -> Result { + self.progress_started(folder_id, base_name(op.rel())); + let result = self.apply_one(folder_id, local_root, &op).await; + self.progress_finished(folder_id); + let name = base_name(op.rel()); + match &result { + Ok(_) => self.log_activity(op.kind(), name, op.detail(), true), + Err(e) => self.log_activity(op.kind(), name, e.clone(), false), + } + result + } + + /// The body of one [`Pending`] op, without the progress/activity bookkeeping. + async fn apply_one( + &self, + folder_id: i64, + local_root: &Path, + op: &Pending, + ) -> Result { + match op { + Pending::CreateDir { rel, parent } => { + let uid = self + .client + .create_folder(parent, base_name(rel), Some(now_secs())) + .await + .map_err(|e| format!("create remote folder {rel}: {e}"))?; + self.baseline_dir(folder_id, rel, &uid)?; + Ok(Applied::Dir(rel.clone(), uid)) + } + Pending::UploadNew { rel, parent } => { + self.upload_new(folder_id, local_root, rel, parent).await?; + Ok(Applied::Uploaded) + } + Pending::UploadRevision { rel, uid } => { + self.upload_revision(folder_id, local_root, rel, uid) + .await?; + Ok(Applied::Uploaded) + } + Pending::Download { + rel, + uid, + mtime, + size, + } => { + self.download_file(folder_id, local_root, rel, uid, *mtime, *size) + .await?; + Ok(Applied::Downloaded) + } + Pending::Conflict { + rel, + uid, + mtime, + size, + } => { + // Set the local copy aside (it re-uploads as a new file next + // pass), then take the remote version as the shared truth. + let path = local_root.join(rel_to_path(rel)); + if path.exists() { + let conflict = conflict_path(&path, now_secs()); + if let Err(e) = std::fs::rename(&path, &conflict) { + warn!(rel, error = %e, "sync: could not set aside conflict copy"); + } else { + info!(rel, "sync: kept local changes as a conflict copy"); + } + } + self.download_file(folder_id, local_root, rel, uid, *mtime, *size) + .await?; + Ok(Applied::Conflict) + } + } + } + + /// Upload a brand-new local file to its remote parent, then record baseline. + async fn upload_new( + &self, + folder_id: i64, + local_root: &Path, + rel: &str, + parent: &NodeUid, + ) -> Result<(), String> { + let path = local_root.join(rel_to_path(rel)); + let name = base_name(rel); + let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?; + let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?; + let mtime = system_mtime(&meta); + // Count the bytes as they are read, so this upload shows up in + // `GetQueueStatus` next to a manual one. The uid isn't known until the + // draft is sealed, so the transfer registers without one. + let reader = OwnedCountingReader::new( + file, + self.transfers + .begin(name, "", TransferDirection::Upload, meta.len()), + ); + let uid = self + .client + .upload_file_from( + parent, + name, + crate::media_type_for(name), + reader, + meta.len() as i64, + Vec::new(), + Some(mtime), + false, + ) + .await + .map_err(|e| format!("upload {rel}: {e}"))?; + self.record_file_baseline(folder_id, rel, &path, &uid).await + } + + /// Upload a changed local file as a new revision of an existing remote node. + async fn upload_revision( + &self, + folder_id: i64, + local_root: &Path, + rel: &str, + uid: &NodeUid, + ) -> Result<(), String> { + let path = local_root.join(rel_to_path(rel)); + let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?; + let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?; + let mtime = system_mtime(&meta); + let reader = OwnedCountingReader::new( + file, + self.transfers.begin( + base_name(rel), + uid.to_string(), + TransferDirection::Upload, + meta.len(), + ), + ); + self.client + .upload_new_revision_from(uid, reader, meta.len() as i64, Vec::new(), Some(mtime)) + .await + .map_err(|e| format!("upload revision {rel}: {e}"))?; + self.record_file_baseline(folder_id, rel, &path, uid).await + } + + /// Download a remote file to its local path (atomically via a temp file), + /// stamp the local mtime to match the remote, then record baseline. + async fn download_file( + &self, + folder_id: i64, + local_root: &Path, + rel: &str, + uid: &NodeUid, + mtime: i64, + size: i64, + ) -> Result<(), String> { + let path = local_root.join(rel_to_path(rel)); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| format!("mkdir for {rel}: {e}"))?; + } + let tmp = path.with_extension(format!( + "pdfs-tmp-{}", + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + { + let out = std::fs::File::create(&tmp).map_err(|e| format!("create tmp {rel}: {e}"))?; + // Count the bytes as they land, so the download reports progress like + // an on-demand hydration does. + let guard = self.transfers.begin( + base_name(rel), + uid.to_string(), + TransferDirection::Download, + size.max(0) as u64, + ); + let mut out = CountingWriter::new(out, &guard); + self.client + .download_file_to(uid, &mut out) + .await + .map_err(|e| { + let _ = std::fs::remove_file(&tmp); + format!("download {rel}: {e}") + })?; + out.into_inner().flush().ok(); + } + std::fs::rename(&tmp, &path).map_err(|e| format!("place {rel}: {e}"))?; + // Match local mtime to the remote's so neither side looks "changed" next pass. + if let Ok(f) = std::fs::File::options().write(true).open(&path) { + let _ = + f.set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(mtime.max(0) as u64)); + } + self.record_file_baseline(folder_id, rel, &path, uid).await + } + + /// Refresh a file's baseline from ground truth: the local stat and the + /// remote node's reported signature. Called after every upload/download so + /// the very next reconcile sees the path as unchanged. + async fn record_file_baseline( + &self, + folder_id: i64, + rel: &str, + local_path: &Path, + uid: &NodeUid, + ) -> Result<(), String> { + let (lmtime, lsize) = match std::fs::metadata(local_path) { + Ok(m) => (system_mtime(&m), m.len() as i64), + Err(e) => return Err(format!("stat {rel}: {e}")), + }; + // Re-fetch the node so the stored remote signature is exactly what a walk + // will report next time. + let (rmtime, rsize) = match self.client.enumerate_nodes(std::slice::from_ref(uid)).await { + Ok(nodes) => nodes + .first() + .map(remote_file_sig) + .unwrap_or((lmtime, lsize)), + Err(e) => { + warn!(rel, error = %e, "sync: baseline refetch failed; using local mtime"); + (lmtime, lsize) + } + }; + self.db + .sync_entry_upsert( + folder_id, + &StoredSyncEntry { + rel_path: rel.to_string(), + remote_uid: Some(uid.to_string()), + local_mtime: lmtime, + local_size: lsize, + remote_rev: Some(rmtime.to_string()), + remote_hash: Some(rsize.to_string()), + }, + ) + .map_err(|e| format!("baseline {rel}: {e:?}")) + } +} + +// ---- helpers -------------------------------------------------------------- + +/// Recursively walk a local directory into `out`, keyed by `/`-joined relative +/// path. Symlinks and other special files are skipped. +fn walk_local(root: &Path, dir: &Path, out: &mut HashMap) -> Result<(), String> { + let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?; + for entry in entries.filter_map(|e| e.ok()) { + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + let Ok(stripped) = path.strip_prefix(root) else { + continue; + }; + let Some(rel) = stripped.to_str() else { + continue; + }; + // Ignore our own in-flight download temp files. + if rel.contains(".pdfs-tmp-") { + continue; + } + if meta.is_dir() { + out.insert( + rel.to_string(), + LocalItem { + is_dir: true, + mtime: 0, + size: 0, + }, + ); + walk_local(root, &path, out)?; + } else if meta.is_file() { + out.insert( + rel.to_string(), + LocalItem { + is_dir: false, + mtime: system_mtime(&meta), + size: meta.len() as i64, + }, + ); + } + } + Ok(()) +} + +/// A remote node's `(mtime, size)` change signature. Size prefers the plaintext +/// `claimed_size`; otherwise the encrypted storage size (stable either way, and +/// only ever compared against the same node's own baseline). +fn remote_file_sig(node: &Node) -> (i64, i64) { + let size = match &node.kind { + NodeKind::File { + claimed_size, + total_size_on_storage, + .. + } => claimed_size.unwrap_or(*total_size_on_storage), + NodeKind::Folder => 0, + }; + (node.modification_time, size) +} + +/// The size the baseline recorded for `rel`, if its remote signature's mtime +/// still matches `mtime` — meaning the file has not changed on Drive, so the +/// recorded size stands and there is no need to decrypt it again. +/// +/// `None` means the size has to be read for real: either the mtime moved, or +/// there is no baseline signature to trust (a file the sync has not seen). +fn unchanged_remote_size( + baseline: &HashMap, + rel: &str, + mtime: i64, +) -> Option { + match baseline.get(rel).and_then(remote_sig) { + Some((recorded, size)) if recorded == mtime => Some(size), + _ => None, + } +} + +/// Join a child `name` onto a walk's `prefix`, giving a rel path. +fn join_rel(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}/{name}") + } +} + +/// The stored remote signature of a baseline row, if it has one. +fn remote_sig(e: &StoredSyncEntry) -> Option<(i64, i64)> { + match (&e.remote_rev, &e.remote_hash) { + (Some(m), Some(s)) => Some((m.parse().ok()?, s.parse().ok()?)), + _ => None, + } +} + +/// A file's modification time as epoch seconds (0 if unavailable). +fn system_mtime(meta: &std::fs::Metadata) -> i64 { + meta.modified() + .ok() + .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// The parent of a `/`-joined relative path (`""` for a top-level entry). +fn parent_rel(rel: &str) -> &str { + match rel.rfind('/') { + Some(i) => &rel[..i], + None => "", + } +} + +/// The final component of a `/`-joined relative path. +fn base_name(rel: &str) -> &str { + match rel.rfind('/') { + Some(i) => &rel[i + 1..], + None => rel, + } +} + +/// Turn a `/`-joined relative path into an OS path (`/` is already the separator +/// on Linux, this keeps the intent explicit). +fn rel_to_path(rel: &str) -> PathBuf { + rel.split('/').collect() +} + +/// The name for a conflict copy of `path`, e.g. `notes (sync-conflict 1700000000).txt`. +fn conflict_path(path: &Path, stamp: i64) -> PathBuf { + let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("file"); + let ext = path.extension().and_then(|s| s.to_str()); + let name = match ext { + Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"), + None => format!("{stem} (sync-conflict {stamp})"), + }; + match path.parent() { + Some(dir) => dir.join(name), + None => PathBuf::from(name), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parent_and_base_split_relative_paths() { + assert_eq!(parent_rel("a"), ""); + assert_eq!(base_name("a"), "a"); + assert_eq!(parent_rel("a/b"), "a"); + assert_eq!(base_name("a/b"), "b"); + assert_eq!(parent_rel("a/b/c.txt"), "a/b"); + assert_eq!(base_name("a/b/c.txt"), "c.txt"); + } + + #[test] + fn rel_to_path_builds_nested_os_path() { + assert_eq!(rel_to_path("a/b/c.txt"), PathBuf::from("a/b/c.txt")); + assert_eq!(rel_to_path("file"), PathBuf::from("file")); + } + + #[test] + fn join_rel_joins_onto_a_prefix_and_leaves_the_root_bare() { + assert_eq!(join_rel("", "a.txt"), "a.txt"); + assert_eq!(join_rel("d", "a.txt"), "d/a.txt"); + assert_eq!(join_rel("d/e", "a.txt"), "d/e/a.txt"); + } + + /// One baseline holding `rel` with remote signature `(mtime, size)`. + fn baseline_with(rel: &str, mtime: i64, size: i64) -> HashMap { + HashMap::from([( + rel.to_string(), + StoredSyncEntry { + rel_path: rel.to_string(), + remote_uid: Some("v~l".into()), + local_mtime: 0, + local_size: 0, + remote_rev: Some(mtime.to_string()), + remote_hash: Some(size.to_string()), + }, + )]) + } + + #[test] + fn an_unchanged_mtime_reuses_the_recorded_size() { + // The whole point: this file's node key never gets unlocked. + let base = baseline_with("a.txt", 1700, 4096); + assert_eq!(unchanged_remote_size(&base, "a.txt", 1700), Some(4096)); + } + + #[test] + fn a_moved_mtime_forces_the_size_to_be_read() { + let base = baseline_with("a.txt", 1700, 4096); + assert_eq!(unchanged_remote_size(&base, "a.txt", 1701), None); + } + + #[test] + fn a_file_missing_from_the_baseline_forces_the_size_to_be_read() { + // Nothing recorded to trust — a file the sync has never seen. + let base = baseline_with("a.txt", 1700, 4096); + assert_eq!(unchanged_remote_size(&base, "new.txt", 1700), None); + } + + #[test] + fn a_baseline_row_without_a_signature_forces_the_size_to_be_read() { + // A folder row (no remote signature) must never be mistaken for an + // unchanged file. + let base = HashMap::from([( + "d".to_string(), + StoredSyncEntry { + rel_path: "d".into(), + remote_uid: Some("v~l".into()), + local_mtime: 0, + local_size: 0, + remote_rev: None, + remote_hash: None, + }, + )]); + assert_eq!(unchanged_remote_size(&base, "d", 0), None); + } + + #[test] + fn remote_sig_round_trips_through_baseline_strings() { + let e = StoredSyncEntry { + rel_path: "x".into(), + remote_uid: Some("v~l".into()), + local_mtime: 10, + local_size: 20, + remote_rev: Some("1700".into()), + remote_hash: Some("4096".into()), + }; + assert_eq!(remote_sig(&e), Some((1700, 4096))); + } + + #[test] + fn remote_sig_is_none_without_a_stored_signature() { + let e = StoredSyncEntry { + rel_path: "d".into(), + remote_uid: Some("v~l".into()), + local_mtime: 0, + local_size: 0, + remote_rev: None, + remote_hash: None, + }; + assert_eq!(remote_sig(&e), None); + } + + #[test] + fn outcome_summarises_only_what_it_moved() { + let mut o = Outcome::default(); + assert!(o.is_empty()); + assert_eq!(o.summary(), ""); + + o.record(&Applied::Uploaded); + o.record(&Applied::Uploaded); + o.record(&Applied::Downloaded); + o.errors += 1; + assert!(!o.is_empty()); + assert_eq!(o.summary(), "2 uploaded, 1 downloaded, 1 failed"); + } + + #[test] + fn pending_ops_describe_themselves_for_the_feed() { + let op = Pending::UploadNew { + rel: "docs/report.pdf".into(), + parent: uid(), + }; + assert_eq!(op.rel(), "docs/report.pdf"); + assert_eq!(base_name(op.rel()), "report.pdf"); + assert_eq!(op.kind(), ActivityKind::Upload); + + let op = Pending::Download { + rel: "photo.jpg".into(), + uid: uid(), + mtime: 0, + size: 10, + }; + assert_eq!(op.kind(), ActivityKind::Download); + } + + fn uid() -> NodeUid { + use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId}; + NodeUid::new(VolumeId::from("vol"), LinkId::from("link")) + } + + #[test] + fn conflict_path_keeps_extension_and_directory() { + let p = conflict_path(Path::new("/home/me/docs/notes.txt"), 1700); + assert_eq!( + p, + PathBuf::from("/home/me/docs/notes (sync-conflict 1700).txt") + ); + let no_ext = conflict_path(Path::new("/home/me/README"), 42); + assert_eq!(no_ext, PathBuf::from("/home/me/README (sync-conflict 42)")); + } +} diff --git a/crates/pdfs-fuse/src/transfers.rs b/crates/pdfs-fuse/src/transfers.rs index 18dafe8..9750c4e 100644 --- a/crates/pdfs-fuse/src/transfers.rs +++ b/crates/pdfs-fuse/src/transfers.rs @@ -1,12 +1,17 @@ -//! In-memory registry of in-flight transfers, behind [`Request::GetQueueStatus`]. +//! In-memory registry of in-flight work, behind [`Request::GetQueueStatus`]. //! -//! The daemon downloads and uploads whole files through the SDK's streaming -//! [`Read`]/[`Write`] variants. Wrapping that reader/writer in a +//! Two kinds of work live here. **Transfers** move bytes: the daemon downloads +//! and uploads whole files through the SDK's streaming [`Read`]/[`Write`] +//! variants, and wrapping that reader/writer in a //! [`CountingReader`]/[`CountingWriter`] lets each block of bytes tick a -//! per-transfer counter without the SDK knowing anything about progress. A -//! [`TransferGuard`] registers a transfer on creation and deregisters it on -//! drop, so the registry always reflects exactly what is in flight — even if a -//! transfer fails partway and unwinds. +//! per-transfer counter without the SDK knowing anything about progress. +//! **Jobs** ([`JobGuard`]) cover the long stretches around them that move no +//! bytes — walking a local tree, creating the remote folder skeleton, indexing +//! `$HOME` — which otherwise look to a front-end exactly like an idle daemon. +//! +//! Both register on creation and deregister on guard drop, so the registry +//! always reflects exactly what is in flight — even if the work fails partway +//! and unwinds. //! //! [`Request::GetQueueStatus`]: pdfs_core::control::Request::GetQueueStatus @@ -16,7 +21,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; -use pdfs_core::control::{TransferDirection, TransferItem}; +use pdfs_core::control::{JobItem, TransferDirection, TransferItem}; /// One registered transfer. `done` is bumped from the I/O wrapper without the /// registry lock; the rest is immutable for the transfer's lifetime. @@ -29,11 +34,22 @@ struct Entry { started: Instant, } -/// The set of transfers currently in flight. Cloned `Arc`-style across the FUSE -/// session and the control-socket task (both share one registry via [`Core`]). +/// One registered job. `detail`/`done`/`total` all move as the job runs; `total` +/// may grow when more work is discovered mid-job. +struct JobEntry { + title: String, + detail: Mutex, + done: AtomicU64, + total: AtomicU64, +} + +/// The set of transfers and jobs currently in flight. Cloned `Arc`-style across +/// the FUSE session and the control-socket task (both share one registry via +/// [`Core`]). #[derive(Default)] pub struct TransferRegistry { inner: Mutex>>, + jobs: Mutex>>, next: AtomicU64, } @@ -68,14 +84,58 @@ impl TransferRegistry { } } - /// Snapshot every in-flight transfer for [`Response::Transfers`]. Speed is the + /// Register a long-running non-transfer job titled `title`, returning a guard + /// that carries its progress and deregisters the job when dropped. The job + /// starts indeterminate; give it a `total` once one is known. + pub fn begin_job(self: &Arc, title: impl Into) -> JobGuard { + let id = self.next.fetch_add(1, Ordering::Relaxed); + let entry = Arc::new(JobEntry { + title: title.into(), + detail: Mutex::new(String::new()), + done: AtomicU64::new(0), + total: AtomicU64::new(0), + }); + self.jobs.lock().unwrap().insert(id, entry.clone()); + JobGuard { + reg: self.clone(), + id, + entry, + } + } + + /// Snapshot every in-flight job for [`Response::Transfers`], oldest first, so + /// a front-end's rows keep their order across polls. + /// + /// [`Response::Transfers`]: pdfs_core::control::Response::Transfers + pub fn jobs_snapshot(&self) -> Vec { + let map = self.jobs.lock().unwrap(); + let mut ids: Vec<_> = map.keys().copied().collect(); + ids.sort_unstable(); + ids.iter() + .map(|id| { + let e = &map[id]; + JobItem { + title: e.title.clone(), + detail: e.detail.lock().unwrap().clone(), + done: e.done.load(Ordering::Relaxed), + total: e.total.load(Ordering::Relaxed), + } + }) + .collect() + } + + /// Snapshot every in-flight transfer for [`Response::Transfers`], oldest + /// first so a front-end's rows keep their order across polls. Speed is the /// running average since the transfer began — simple, and stable enough for a /// progress widget without per-tick sampling state. /// /// [`Response::Transfers`]: pdfs_core::control::Response::Transfers pub fn snapshot(&self) -> Vec { let map = self.inner.lock().unwrap(); - map.values() + let mut ids: Vec<_> = map.keys().copied().collect(); + ids.sort_unstable(); + ids.iter() + .map(|id| &map[id]) .map(|e| { let done = e.done.load(Ordering::Relaxed); let secs = e.started.elapsed().as_secs_f64(); @@ -120,6 +180,44 @@ impl Drop for TransferGuard { } } +/// Lifetime handle for a registered job: describe what it is doing with +/// [`detail`], size it with [`set_total`], advance it with [`step`], and the job +/// leaves the registry when this drops. Shared across the tasks of one batch as +/// an `Arc`, so a concurrent phase can report a single "N of M" line. +/// +/// [`detail`]: JobGuard::detail +/// [`set_total`]: JobGuard::set_total +/// [`step`]: JobGuard::step +pub struct JobGuard { + reg: Arc, + id: u64, + entry: Arc, +} + +impl JobGuard { + /// Say what the job is working on right now; empty clears it. + pub fn detail(&self, what: impl Into) { + *self.entry.detail.lock().unwrap() = what.into(); + } + + /// Set how many steps the job now knows it has. Safe to raise mid-job as more + /// work is discovered. + pub fn set_total(&self, total: u64) { + self.entry.total.store(total, Ordering::Relaxed); + } + + /// Mark one more step done. + pub fn step(&self) { + self.entry.done.fetch_add(1, Ordering::Relaxed); + } +} + +impl Drop for JobGuard { + fn drop(&mut self) { + self.reg.jobs.lock().unwrap().remove(&self.id); + } +} + /// A [`Write`] that tallies bytes written to a [`TransferGuard`] (download path). pub struct CountingWriter<'a, W> { inner: W, diff --git a/crates/pdfs-gui/build.rs b/crates/pdfs-gui/build.rs new file mode 100644 index 0000000..6953c08 --- /dev/null +++ b/crates/pdfs-gui/build.rs @@ -0,0 +1,28 @@ +use std::env; +use std::process::Command; + +fn main() { + let out_dir = env::var("OUT_DIR").unwrap(); + let status = Command::new("glib-compile-resources") + .args([ + "--target", + &format!("{}/pdfs.gresource", out_dir), + "--sourcedir", + "resources/icons", + "resources/pdfs.gresource.xml", + ]) + .status() + .expect("Failed to execute glib-compile-resources"); + + if !status.success() { + panic!( + "glib-compile-resources failed with exit code: {:?}", + status.code() + ); + } + + println!("cargo:rerun-if-changed=resources/pdfs.gresource.xml"); + println!( + "cargo:rerun-if-changed=resources/icons/scalable/actions/pdfs-folder-upload-symbolic.svg" + ); +} diff --git a/crates/pdfs-gui/resources/icons/scalable/actions/pdfs-folder-upload-symbolic.svg b/crates/pdfs-gui/resources/icons/scalable/actions/pdfs-folder-upload-symbolic.svg new file mode 100644 index 0000000..c06384e --- /dev/null +++ b/crates/pdfs-gui/resources/icons/scalable/actions/pdfs-folder-upload-symbolic.svg @@ -0,0 +1,7 @@ + + + + diff --git a/crates/pdfs-gui/resources/pdfs.gresource.xml b/crates/pdfs-gui/resources/pdfs.gresource.xml new file mode 100644 index 0000000..a84f6ea --- /dev/null +++ b/crates/pdfs-gui/resources/pdfs.gresource.xml @@ -0,0 +1,9 @@ + + + + + scalable/actions/pdfs-folder-upload-symbolic.svg + + diff --git a/crates/pdfs-gui/src/app.rs b/crates/pdfs-gui/src/app.rs index 318d68d..7f4db1d 100644 --- a/crates/pdfs-gui/src/app.rs +++ b/crates/pdfs-gui/src/app.rs @@ -33,9 +33,9 @@ use gtk4::glib::BoxedAnyObject; use pdfs_core::auth; use pdfs_core::config::AppDirs; use pdfs_core::control::{ - ActivityEntry, ActivityKind, BookmarkInfo, DeviceInfo, DirEntry, InvitationInfo, PhotoItem, - PublicLinkInfo, Request, Response, SearchHit, ShareEntry, ShareEntryKind, SharedItem, - TransferDirection, TransferItem, send, + ActivityEntry, ActivityKind, BookmarkInfo, DeviceInfo, DirEntry, InvitationInfo, JobItem, + PhotoItem, PublicLinkInfo, Request, Response, SearchHit, ShareEntry, ShareEntryKind, + SharedItem, SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem, send, }; use pdfs_core::service; @@ -109,15 +109,24 @@ struct PinRow { unpin: Option, } -/// One rendered transfer row in the Activity group: a name + speed line over a -/// progress bar. Retained so [`repaint_transfers`] can update the bar and label -/// in place each tick when the active set is unchanged, instead of rebuilding. +/// One rendered row in the Activity group: a description over a progress bar. +/// Retained so [`repaint_transfers`] can update the bar and label in place each +/// tick when the active set is unchanged, instead of rebuilding. struct TransferRow { row: adw::PreferencesRow, label: gtk4::Label, bar: gtk4::ProgressBar, } +/// What one Activity row should say this tick, and how far along it is — +/// `None` meaning "no total known", which the bar shows by pulsing. Jobs and +/// transfers both render to this, so the group is one list in the order the +/// daemon reports: the jobs that frame the work, then the files moving under it. +struct ActivityLine { + text: String, + fraction: Option, +} + /// All widgets the periodic refresh and the action handlers mutate, plus the /// resolved paths they act on. Wrapped in an [`Rc`] so handlers and the timeout /// closure share one instance. @@ -327,12 +336,15 @@ struct Ui { /// When the Shared page last painted good data. `None` = never / invalidated, /// forcing a fetch on next visit. See [`PAGE_TTL`]. shared_loaded_at: Cell>, - // Devices page: one section, rebuilt wholesale on each load. + // Devices page: two sections (synced folders + other devices), rebuilt + // wholesale on each load. devices_content: gtk4::Stack, devices_status: adw::StatusPage, devices_retry: gtk4::Button, devices_group: adw::PreferencesGroup, devices_rows: RefCell>, + devices_sync_group: adw::PreferencesGroup, + devices_sync_rows: RefCell>, devices_inflight: Cell, devices_loaded_at: Cell>, // Shared (by me) page: one section listing the items I've shared, each with a @@ -351,6 +363,11 @@ struct Ui { activity_group: adw::PreferencesGroup, activity_rows: RefCell>, activity_inflight: Cell, + /// Fingerprint of the feed currently on screen (see [`activity_key`]). The + /// page polls every couple of seconds and usually gets back exactly what it + /// is already showing; rebuilding all 200 rows for that would throw away the + /// user's scroll position several times a minute. + activity_key: RefCell>, } impl Ui { @@ -483,6 +500,15 @@ fn spawn_tray() { /// purple, app-wide. Named-colour overrides recolour the stock widgets (switch, /// buttons, progress fill) without per-widget styling. fn load_proton_theme() { + // Compile-in and register our custom GResources (e.g. custom icons) + let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/pdfs.gresource")); + let resource_data = glib::Bytes::from_static(bytes); + if let Ok(resource) = gio::Resource::from_data(&resource_data) { + gio::resources_register(&resource); + } else { + tracing::error!("failed to load gresource bundle"); + } + let css = format!( "@define-color accent_bg_color {PROTON_PURPLE};\n\ @define-color accent_color {PROTON_PURPLE};\n\ @@ -532,6 +558,10 @@ fn load_proton_theme() { &provider, gtk4::STYLE_PROVIDER_PRIORITY_APPLICATION, ); + + // Register custom icons directory from our GResource with the icon theme + let icon_theme = gtk4::IconTheme::for_display(&display); + icon_theme.add_resource_path("/de/nils/protondrivelinux/icons"); } } @@ -691,6 +721,8 @@ fn build_window(app: &adw::Application) { devices_retry: devices_widgets.retry.clone(), devices_group: devices_widgets.group.clone(), devices_rows: RefCell::new(Vec::new()), + devices_sync_group: devices_widgets.sync_group.clone(), + devices_sync_rows: RefCell::new(Vec::new()), devices_inflight: Cell::new(false), devices_loaded_at: Cell::new(None), shared_by_me_content: shared_by_me_widgets.content.clone(), @@ -706,6 +738,7 @@ fn build_window(app: &adw::Application) { activity_group: activity_widgets.group.clone(), activity_rows: RefCell::new(Vec::new()), activity_inflight: Cell::new(false), + activity_key: RefCell::new(None), }); ui.load_ratios(); @@ -730,7 +763,7 @@ fn build_window(app: &adw::Application) { wire_trash(&ui, &trash_widgets.list, &trash_widgets.empty); wire_shared(&ui, &shared_widgets.retry, &shared_widgets.add_bookmark); wire_shared_by_me(&ui, &shared_by_me_widgets.retry); - wire_devices(&ui, &devices_widgets.retry); + wire_devices(&ui, &devices_widgets.retry, &devices_widgets.add_folder); wire_activity(&ui, &activity_widgets.retry); wire_retry(&ui); @@ -1612,6 +1645,13 @@ fn refresh(ui: &Rc) { refresh_status(ui); refresh_transfers(ui); + // Both of these pages show work as it happens, so they follow the tick while + // they are on screen. Every other page loads on navigation only. + match ui.stack.visible_child_name().as_deref() { + Some("devices") => refresh_sync_folders(ui), + Some("activity") => refresh_activity(ui), + _ => {} + } } /// Record the mount state seen by the last status poll: gate every control that @@ -1683,21 +1723,30 @@ fn refresh_transfers(ui: &Rc) { let result = rx.recv().await; ui.transfers_inflight.set(false); match result { - Ok(Ok(Response::Transfers { items })) => repaint_transfers(&ui, &items), + Ok(Ok(Response::Transfers { items, jobs })) => repaint_transfers(&ui, &items, &jobs), // Daemon unreachable or odd reply: clear the section rather than // leave stale progress bars frozen on screen. - _ => repaint_transfers(&ui, &[]), + _ => repaint_transfers(&ui, &[], &[]), } }); } -/// Render the Activity group from `items`. Rebuilds the rows only when the set of -/// transfers changes (count differs); on the common steady tick it updates each -/// bar's fraction and the name/speed label in place, so progress animates without -/// flicker. Hides the whole group when nothing is in flight. -fn repaint_transfers(ui: &Rc, items: &[TransferItem]) { +/// Render the Activity group from a work snapshot: the daemon's jobs (scans, +/// batch counts, the local index) above the files moving under them. Rebuilds the +/// rows only when the set changes (count differs); on the common steady tick it +/// updates each bar's fraction and the label in place, so progress animates +/// without flicker. Hides the whole group when the daemon is idle. +fn repaint_transfers(ui: &Rc, items: &[TransferItem], jobs: &[JobItem]) { + let lines: Vec = jobs + .iter() + .map(job_line) + .chain(items.iter().map(transfer_line)) + .collect(); + // The wire carries in-flight transfers only, with no completion event: the // count falling to zero is what "the sync finished" looks like from here. + // Jobs are deliberately not counted — a bulk upload retires its scan job and + // starts its upload job mid-flight, which is not a thing finishing. let previous = ui.active_transfers.replace(items.len()); if items.is_empty() && previous > 0 { let files = if previous == 1 { @@ -1715,7 +1764,7 @@ fn repaint_transfers(ui: &Rc, items: &[TransferItem]) { reload_listing(ui); } - if items.is_empty() { + if lines.is_empty() { if !ui.transfer_rows.borrow().is_empty() { for tr in ui.transfer_rows.borrow_mut().drain(..) { ui.transfers_group.remove(&tr.row); @@ -1728,11 +1777,11 @@ fn repaint_transfers(ui: &Rc, items: &[TransferItem]) { ui.transfers_group.set_visible(true); // Rebuild rows only when the count changes; otherwise reuse them in place. - if ui.transfer_rows.borrow().len() != items.len() { + if ui.transfer_rows.borrow().len() != lines.len() { for tr in ui.transfer_rows.borrow_mut().drain(..) { ui.transfers_group.remove(&tr.row); } - for _ in items { + for _ in &lines { let row_box = gtk4::Box::new(gtk4::Orientation::Vertical, 4); row_box.set_margin_top(8); row_box.set_margin_bottom(8); @@ -1752,30 +1801,57 @@ fn repaint_transfers(ui: &Rc, items: &[TransferItem]) { } } - for (item, tr) in items.iter().zip(ui.transfer_rows.borrow().iter()) { - let arrow = match item.direction { - TransferDirection::Download => "↓", - TransferDirection::Upload => "↑", - }; - if item.bytes_total == 0 { - // Unknown size: pulse instead of a fraction, and omit the percentage. - tr.bar.pulse(); - tr.label.set_text(&format!( + for (line, tr) in lines.iter().zip(ui.transfer_rows.borrow().iter()) { + tr.label.set_text(&line.text); + match line.fraction { + Some(f) => tr.bar.set_fraction(f), + // No total to divide by: pulse so the bar still reads as "working". + None => tr.bar.pulse(), + } + } +} + +/// One Activity row for a daemon job: its title, plus whatever it can say about +/// where it is — a count when it has one, else what it is currently chewing on. +fn job_line(j: &JobItem) -> ActivityLine { + let text = match (j.total > 0, j.detail.is_empty()) { + (true, true) => format!("{} — {} of {}", j.title, j.done, j.total), + (true, false) => format!("{} — {} ({} of {})", j.title, j.detail, j.done, j.total), + (false, true) => format!("{}…", j.title), + (false, false) => format!("{} — {}…", j.title, j.detail), + }; + ActivityLine { + text, + fraction: (j.total > 0).then(|| (j.done as f64 / j.total as f64).min(1.0)), + } +} + +/// One Activity row for a file in flight: which way it's going, how far, how fast. +fn transfer_line(t: &TransferItem) -> ActivityLine { + let arrow = match t.direction { + TransferDirection::Download => "↓", + TransferDirection::Upload => "↑", + }; + if t.bytes_total == 0 { + ActivityLine { + text: format!( "{arrow} {} — {} ({}/s)", - item.name, - human_bytes(item.bytes_completed), - human_bytes(item.speed_bytes_sec), - )); - } else { - let fraction = (item.bytes_completed as f64 / item.bytes_total as f64).min(1.0); - tr.bar.set_fraction(fraction); - tr.label.set_text(&format!( + t.name, + human_bytes(t.bytes_completed), + human_bytes(t.speed_bytes_sec), + ), + fraction: None, + } + } else { + ActivityLine { + text: format!( "{arrow} {} — {} of {} ({}/s)", - item.name, - human_bytes(item.bytes_completed), - human_bytes(item.bytes_total), - human_bytes(item.speed_bytes_sec), - )); + t.name, + human_bytes(t.bytes_completed), + human_bytes(t.bytes_total), + human_bytes(t.speed_bytes_sec), + ), + fraction: Some((t.bytes_completed as f64 / t.bytes_total as f64).min(1.0)), } } } @@ -1999,7 +2075,7 @@ fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) { .build(); upload.add_css_class("flat"); let upload_folder = gtk4::Button::builder() - .icon_name("send-to-symbolic") + .icon_name("pdfs-folder-upload-symbolic") .tooltip_text("Upload folder") .valign(gtk4::Align::Center) .build(); @@ -3502,7 +3578,11 @@ struct SharedWidgets { struct DevicesWidgets { content: gtk4::Stack, status: adw::StatusPage, + /// "This computer" — the local folders synced to this machine's device. + sync_group: adw::PreferencesGroup, + /// "Other devices" — the account's other registered devices (read-only). group: adw::PreferencesGroup, + add_folder: gtk4::Button, retry: gtk4::Button, } @@ -3590,20 +3670,41 @@ fn build_shared_page() -> (gtk4::Widget, SharedWidgets) { ) } -/// The Devices page: one section listing every device registered to the account, -/// each row offering Rename and Remove. +/// The Computers page: a "This computer" section listing the local folders synced +/// to this machine's device (each row offering a mode toggle and Remove), plus an +/// "Other devices" section listing the account's other registered devices. fn build_devices_page() -> (gtk4::Widget, DevicesWidgets) { let title = gtk4::Label::builder() - .label("Devices") + .label("Computers") .halign(gtk4::Align::Start) .build(); title.add_css_class("title-2"); - let header = gtk4::Box::new(gtk4::Orientation::Horizontal, 8); - header.set_hexpand(true); - header.append(&title); + let titles = gtk4::Box::new(gtk4::Orientation::Vertical, 2); + titles.set_hexpand(true); + titles.append(&title); - let group = adw::PreferencesGroup::new(); - let clamp = adw::Clamp::builder().child(&group).build(); + let add_folder = gtk4::Button::builder() + .label("Add Folder") + .valign(gtk4::Align::Center) + .build(); + add_folder.add_css_class("flat"); + + let header = gtk4::Box::new(gtk4::Orientation::Horizontal, 8); + header.append(&titles); + header.append(&add_folder); + + let sync_group = adw::PreferencesGroup::builder() + .title("This computer") + .description("Local folders backed up to this device.") + .build(); + let group = adw::PreferencesGroup::builder() + .title("Other devices") + .build(); + + let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18); + groups.append(&sync_group); + groups.append(&group); + let clamp = adw::Clamp::builder().child(&groups).build(); let scroll = gtk4::ScrolledWindow::builder() .vexpand(true) .child(&clamp) @@ -3642,7 +3743,9 @@ fn build_devices_page() -> (gtk4::Widget, DevicesWidgets) { DevicesWidgets { content, status, + sync_group, group, + add_folder, retry, }, ) @@ -4391,13 +4494,15 @@ fn run_shared_mutation(ui: &Rc, req: Request, done: &'static str, failed: &' // ---- Devices page --------------------------------------------------------- -/// Install the Devices page's retry button. -fn wire_devices(ui: &Rc, retry: >k4::Button) { +/// Install the Devices page's retry button and the "Add Folder" action. +fn wire_devices(ui: &Rc, retry: >k4::Button, add_folder: >k4::Button) { let ui_retry = ui.clone(); retry.connect_clicked(move |_| { service::restart(); load_devices(&ui_retry); }); + let ui_add = ui.clone(); + add_folder.connect_clicked(move |_| prompt_add_sync_folder(&ui_add)); } /// Show a status page in place of the Devices list. @@ -4409,7 +4514,9 @@ fn devices_status(ui: &Rc, icon: &str, title: &str, description: &str, retry ui.devices_content.set_visible_child_name("status"); } -/// Fetch the device list and repaint the page. +/// Fetch this machine's synced folders and the account's other devices, then +/// repaint both sections. The two requests are chained so a single unreachable +/// daemon collapses the whole page to a status view. fn load_devices(ui: &Rc) { if ui.devices_inflight.get() { return; @@ -4423,35 +4530,64 @@ fn load_devices(ui: &Rc) { false, ); ui.busy_begin(); - let rx = spawn_request(ui.dirs.control_socket(), Request::ListDevices); + let rx = spawn_request(ui.dirs.control_socket(), Request::ListSyncFolders); + let ui = ui.clone(); + glib::spawn_future_local(async move { + let sync = rx.recv().await; + let folders = match sync { + Ok(Ok(Response::SyncFolders { items })) => items, + // The daemon answered but not as expected — treat as empty and carry + // on to the devices list rather than failing the whole page. + Ok(Ok(_)) => Vec::new(), + Ok(Err(_)) | Err(_) => { + ui.busy_end(); + ui.devices_inflight.set(false); + ui.devices_loaded_at.set(None); + devices_unreachable(&ui); + return; + } + }; + // Daemon reachable: show the list and paint the synced-folders section. + ui.devices_content.set_visible_child_name("list"); + repaint_sync_folders(&ui, &folders); + + let rx2 = spawn_request(ui.dirs.control_socket(), Request::ListDevices); + let devices = match rx2.recv().await { + Ok(Ok(Response::Devices { items })) => items, + _ => Vec::new(), + }; + ui.busy_end(); + ui.devices_inflight.set(false); + repaint_devices(&ui, &devices); + ui.devices_loaded_at.set(Some(Instant::now())); + }); +} + +/// Refresh just the synced-folders section, with no status flash, no spinner and +/// no devices round-trip. Driven by the periodic tick while the Devices page is +/// on screen: a folder's sync progress is only live if something re-reads it. +/// +/// A hiccup leaves the page as it is — the next tick tries again, and a real +/// outage is reported by [`load_devices`] on the next visit. +fn refresh_sync_folders(ui: &Rc) { + if ui.devices_inflight.get() { + return; + } + ui.devices_inflight.set(true); + let rx = spawn_request(ui.dirs.control_socket(), Request::ListSyncFolders); let ui = ui.clone(); glib::spawn_future_local(async move { let result = rx.recv().await; - ui.busy_end(); ui.devices_inflight.set(false); - match result { - Ok(Ok(Response::Devices { items })) => { - repaint_devices(&ui, &items); - ui.devices_loaded_at.set(Some(Instant::now())); - } - Ok(Ok(Response::Error { message })) => devices_status( - &ui, - "dialog-warning-symbolic", - "Couldn't read devices", - &message, - false, - ), - Ok(Ok(_)) => devices_status( - &ui, - "dialog-warning-symbolic", - "Couldn't read devices", - "Unexpected reply from the mount service.", - false, - ), - Ok(Err(_)) | Err(_) => { - ui.devices_loaded_at.set(None); - devices_unreachable(&ui); - } + let Ok(Ok(Response::SyncFolders { items })) = result else { + return; + }; + // The page may have been navigated away from, or collapsed to a status + // view, while the request was in flight. + if ui.stack.visible_child_name().as_deref() == Some("devices") + && ui.devices_content.visible_child_name().as_deref() == Some("list") + { + repaint_sync_folders(&ui, &items); } }); } @@ -4483,22 +4619,23 @@ fn devices_unreachable(ui: &Rc) { }); } -/// Rebuild the Devices section from a fresh listing. +/// Rebuild the "Other devices" section from a fresh listing. Empty is a normal +/// state (this machine may be the only device), so it shows a placeholder row +/// rather than collapsing the page. fn repaint_devices(ui: &Rc, devices: &[DeviceInfo]) { for row in ui.devices_rows.borrow_mut().drain(..) { ui.devices_group.remove(&row); } if devices.is_empty() { - devices_status( - ui, - "computer-symbolic", - "No devices", - "Desktop apps that sync a folder to this account show up here.", - false, - ); + let row = adw::ActionRow::builder() + .title("No other devices") + .subtitle("Desktop apps syncing to this account appear here.") + .build(); + row.add_prefix(>k4::Image::from_icon_name("computer-symbolic")); + ui.devices_group.add(&row); + *ui.devices_rows.borrow_mut() = vec![row.upcast()]; return; } - ui.devices_content.set_visible_child_name("list"); let mut rows: Vec = Vec::new(); for dev in devices { let row = adw::ActionRow::builder() @@ -4534,6 +4671,263 @@ fn repaint_devices(ui: &Rc, devices: &[DeviceInfo]) { *ui.devices_rows.borrow_mut() = rows; } +/// Rebuild the "This computer" section from a fresh synced-folders listing. +fn repaint_sync_folders(ui: &Rc, folders: &[SyncFolderInfo]) { + for row in ui.devices_sync_rows.borrow_mut().drain(..) { + ui.devices_sync_group.remove(&row); + } + if folders.is_empty() { + let row = adw::ActionRow::builder() + .title("No synced folders") + .subtitle("Add a folder to back it up to this device.") + .build(); + row.add_prefix(>k4::Image::from_icon_name("folder-symbolic")); + ui.devices_sync_group.add(&row); + *ui.devices_sync_rows.borrow_mut() = vec![row.upcast()]; + return; + } + let mut rows: Vec = Vec::new(); + for f in folders { + // A running pass describes itself ("Uploading photo.jpg — 12 of 40"); + // otherwise fall back to the folder's resting state. + let status = match &f.progress { + Some(p) => sync_progress_label(p), + None => sync_state_label(&f.state).to_string(), + }; + let subtitle = match f.mode.as_str() { + "ondemand" => format!("On-demand · {status}"), + _ => format!("Synced · {status}"), + }; + let row = adw::ActionRow::builder() + .title(&f.local_path) + .subtitle(&subtitle) + .build(); + row.add_prefix(>k4::Image::from_icon_name("folder-symbolic")); + let id = f.id; + + // A pass that is applying knows how far it has got, so show it: the + // subtitle alone ("syncing photo.jpg — 12 of 40") never conveys that the + // end is near. A scanning pass has no counts to draw, and this row is + // repainted on the 2s tick — too slow for a pulse to read as motion — so + // it stays text-only until the counts exist. + if let Some(p) = &f.progress + && p.phase == SyncPhase::Applying + && p.total > 0 + { + let bar = gtk4::ProgressBar::builder() + .fraction((p.done as f64 / p.total as f64).min(1.0)) + .valign(gtk4::Align::Center) + .width_request(120) + .build(); + row.add_suffix(&bar); + } + + // Sync now — only meaningful for mirror folders (on-demand has no local + // copy to reconcile). The engine also syncs on file changes and on a + // 120s poll; this is the "don't wait" button. + if f.mode != "ondemand" { + let sync_now = gtk4::Button::builder() + .icon_name("view-refresh-symbolic") + .tooltip_text("Sync this folder now") + .valign(gtk4::Align::Center) + .build(); + sync_now.add_css_class("flat"); + let ui_sync = ui.clone(); + sync_now.connect_clicked(move |_| { + let rx = spawn_request( + ui_sync.dirs.control_socket(), + Request::SyncNow { id: Some(id) }, + ); + let ui_sync = ui_sync.clone(); + glib::spawn_future_local(async move { + match rx.recv().await { + Ok(Ok(Response::Ok { .. })) => toast(&ui_sync, "Syncing folder…"), + Ok(Ok(Response::Error { message })) => { + toast_error(&ui_sync, "Couldn't sync", &message) + } + _ => toast_error( + &ui_sync, + "Couldn't sync", + "The mount service didn't respond.", + ), + } + }); + }); + row.add_suffix(&sync_now); + } + + // On-demand toggle: off = full local copy (mirror), on = FUSE mount that + // frees the disk. Set the state before wiring the handler so painting the + // current mode doesn't fire a spurious request. + let ondemand = gtk4::Switch::builder() + .tooltip_text("On-demand: keep files in the cloud, free local disk") + .valign(gtk4::Align::Center) + .active(f.mode == "ondemand") + .build(); + let ui_mode = ui.clone(); + ondemand.connect_state_set(move |_, on| { + set_sync_folder_mode(&ui_mode, id, if on { "ondemand" } else { "mirror" }); + glib::Propagation::Proceed + }); + row.add_suffix(&ondemand); + + let remove = gtk4::Button::builder() + .icon_name("user-trash-symbolic") + .tooltip_text("Stop syncing this folder") + .valign(gtk4::Align::Center) + .build(); + remove.add_css_class("flat"); + let ui_rm = ui.clone(); + let path = f.local_path.clone(); + remove.connect_clicked(move |_| prompt_remove_sync_folder(&ui_rm, id, &path)); + row.add_suffix(&remove); + ui.devices_sync_group.add(&row); + rows.push(row.upcast()); + } + *ui.devices_sync_rows.borrow_mut() = rows; +} + +/// Human label for a synced folder's `state` column. +fn sync_state_label(state: &str) -> &str { + match state { + "syncing" => "syncing…", + "error" => "sync error", + "conflict" => "needs attention", + _ => "up to date", + } +} + +/// Human label for a sync pass in flight: what it is doing, to which file, and +/// how far along it is. The total grows as deeper paths are classified, so this +/// reads "12 of 40" rather than showing a percentage that could go backwards. +fn sync_progress_label(p: &SyncProgress) -> String { + match p.phase { + SyncPhase::Scanning => "checking for changes…".to_string(), + SyncPhase::Applying => { + let count = format!("{} of {}", p.done + 1, p.total.max(p.done + 1)); + if p.current.is_empty() { + format!("syncing {count}") + } else { + format!("syncing {} — {count}", p.current) + } + } + } +} + +/// Pick a local folder and hand it to the daemon to sync to this device. +fn prompt_add_sync_folder(ui: &Rc) { + let win = ui_window(ui); + let dialog = gtk4::FileDialog::builder() + .title("Add Folder to Sync") + .build(); + let ui = ui.clone(); + dialog.select_folder(win.as_ref(), gio::Cancellable::NONE, move |res| { + let Ok(folder) = res else { return }; + let Some(local_path) = folder.path().and_then(|p| p.to_str().map(str::to_string)) else { + return; + }; + let rx = spawn_request( + ui.dirs.control_socket(), + Request::AddSyncFolder { + local_path: local_path.clone(), + }, + ); + let ui = ui.clone(); + glib::spawn_future_local(async move { + match rx.recv().await { + Ok(Ok(Response::Ok { .. })) => { + toast(&ui, "Syncing folder…"); + // The upload runs off-socket; reload shortly so the new row + // appears once the daemon has registered it. + let ui = ui.clone(); + glib::timeout_add_local_once(std::time::Duration::from_secs(2), move || { + if ui.stack.visible_child_name().as_deref() == Some("devices") { + load_devices(&ui); + } + }); + } + Ok(Ok(Response::Error { message })) => { + toast_error(&ui, "Couldn't add folder", &message) + } + _ => toast_error( + &ui, + "Couldn't add folder", + "The mount service didn't respond.", + ), + } + }); + }); +} + +/// Flip a synced folder between `mirror` and `ondemand`. Reloads after so the +/// row's subtitle and switch reflect the daemon's real state (the request may be +/// rejected, e.g. switching to on-demand while a folder is mid-sync). +fn set_sync_folder_mode(ui: &Rc, id: i64, mode: &str) { + let rx = spawn_request( + ui.dirs.control_socket(), + Request::SetSyncFolderMode { + id, + mode: mode.to_string(), + }, + ); + let ui = ui.clone(); + glib::spawn_future_local(async move { + match rx.recv().await { + Ok(Ok(Response::Ok { message })) => toast(&ui, &message), + Ok(Ok(Response::Error { message })) => { + toast_error(&ui, "Couldn't change mode", &message) + } + _ => toast_error( + &ui, + "Couldn't change mode", + "The mount service didn't respond.", + ), + } + if ui.stack.visible_child_name().as_deref() == Some("devices") { + load_devices(&ui); + } + }); +} + +/// Confirm, then stop syncing a folder. Offers to also delete the cloud copy. +fn prompt_remove_sync_folder(ui: &Rc, id: i64, path: &str) { + let win = ui_window(ui); + let dialog = adw::AlertDialog::builder() + .heading("Stop syncing folder") + .body(format!( + "Stop syncing “{path}”? Your local files stay; choose whether to also \ + delete the copy in Proton Drive." + )) + .build(); + let group = adw::PreferencesGroup::new(); + let delete_remote = adw::SwitchRow::builder() + .title("Also delete from Proton Drive") + .active(false) + .build(); + group.add(&delete_remote); + dialog.set_extra_child(Some(&group)); + dialog.add_response("cancel", "Cancel"); + dialog.add_response("remove", "Stop Syncing"); + dialog.set_response_appearance("remove", adw::ResponseAppearance::Destructive); + dialog.set_default_response(Some("cancel")); + dialog.set_close_response("cancel"); + let ui = ui.clone(); + dialog.connect_response(None, move |_, resp| { + if resp == "remove" { + run_devices_mutation( + &ui, + Request::RemoveSyncFolder { + id, + delete_remote: delete_remote.is_active(), + }, + "Stopped syncing folder", + "Couldn't stop syncing the folder", + ); + } + }); + dialog.present(win.as_ref()); +} + /// Prompt for a new device name and rename it. fn prompt_rename_device(ui: &Rc, uid: &str, current: &str) { let win = ui_window(ui); @@ -4985,9 +5379,38 @@ fn activity_status(ui: &Rc, icon: &str, title: &str, description: &str, retr ui.activity_status.set_description(Some(description)); ui.activity_retry.set_visible(retry); ui.activity_content.set_visible_child_name("status"); + // The list is no longer what is on screen, so the next repaint must not skip + // itself as a no-op and leave this status view up. + *ui.activity_key.borrow_mut() = None; +} + +/// Refresh the Activity feed in place, with no status flash and no spinner. +/// Driven by the periodic tick while the page is on screen, so a running sync +/// pass fills the feed as it works rather than only once it is done. Anything +/// other than a good answer leaves the rows alone until the next tick. +fn refresh_activity(ui: &Rc) { + if ui.activity_inflight.get() { + return; + } + ui.activity_inflight.set(true); + let rx = spawn_request( + ui.dirs.control_socket(), + Request::ListActivity { limit: 200 }, + ); + let ui = ui.clone(); + glib::spawn_future_local(async move { + let result = rx.recv().await; + ui.activity_inflight.set(false); + // The page may have been navigated away from while the request was in + // flight. + if let Ok(Ok(Response::Activity { items })) = result + && ui.stack.visible_child_name().as_deref() == Some("activity") + { + repaint_activity(&ui, &items); + } + }); } -/// Fetch the recent activity and repaint the page. fn load_activity(ui: &Rc) { if ui.activity_inflight.get() { return; @@ -5058,8 +5481,24 @@ fn activity_unreachable(ui: &Rc) { }); } -/// Rebuild the Activity section from a fresh log. +/// A cheap fingerprint of a feed: how many entries, and what the newest one is. +/// The log is append-only and newest-first, so a feed with the same length and +/// the same newest entry is the same feed. +fn activity_key(items: &[ActivityEntry]) -> String { + match items.first() { + Some(a) => format!("{}:{}:{}:{}", items.len(), a.time, a.target, a.detail), + None => String::new(), + } +} + +/// Rebuild the Activity section from a fresh log, unless it is already showing +/// exactly this feed. fn repaint_activity(ui: &Rc, items: &[ActivityEntry]) { + let key = activity_key(items); + if ui.activity_key.borrow().as_deref() == Some(key.as_str()) { + return; + } + *ui.activity_key.borrow_mut() = Some(key); for row in ui.activity_rows.borrow_mut().drain(..) { ui.activity_group.remove(&row); } @@ -5101,6 +5540,8 @@ fn repaint_activity(ui: &Rc, items: &[ActivityEntry]) { fn activity_verb(kind: ActivityKind) -> &'static str { match kind { ActivityKind::Upload => "Uploaded", + ActivityKind::Download => "Downloaded", + ActivityKind::Sync => "Synced", ActivityKind::Rename => "Renamed", ActivityKind::Move => "Moved", ActivityKind::CreateFolder => "Created folder", @@ -5118,6 +5559,8 @@ fn activity_verb(kind: ActivityKind) -> &'static str { fn activity_icon(kind: ActivityKind) -> &'static str { match kind { ActivityKind::Upload => "document-send-symbolic", + ActivityKind::Download => "document-save-symbolic", + ActivityKind::Sync => "emblem-synchronizing-symbolic", ActivityKind::Rename => "document-edit-symbolic", ActivityKind::Move => "go-jump-symbolic", ActivityKind::CreateFolder => "folder-new-symbolic", diff --git a/crates/pdfs-gui/src/main.rs b/crates/pdfs-gui/src/main.rs index 77a5e34..0c9ff22 100644 --- a/crates/pdfs-gui/src/main.rs +++ b/crates/pdfs-gui/src/main.rs @@ -18,7 +18,7 @@ use ksni::menu::StandardItem; use ksni::{MenuItem, Tray, TrayService}; use pdfs_core::auth; use pdfs_core::config::AppDirs; -use pdfs_core::control::{Request, Response, TransferDirection, TransferItem, send}; +use pdfs_core::control::{JobItem, Request, Response, TransferDirection, TransferItem, send}; use pdfs_core::service; /// How often the tray re-polls the daemon to refresh its menu. @@ -38,12 +38,18 @@ struct DriveState { sync: String, } -/// Summarise a transfer snapshot into one menu line, or empty when idle. A single +/// Summarise a work snapshot into one menu line, or empty when idle. A single /// transfer names the file and its percentage; several collapse to counts so the -/// menu stays one line regardless of queue depth. -fn sync_line(items: &[TransferItem]) -> String { +/// menu stays one line regardless of queue depth. With nothing moving bytes, a +/// running job speaks for itself — a scan or an index rebuild is still "busy", +/// and the tray saying nothing there reads as "finished". +fn sync_line(items: &[TransferItem], jobs: &[JobItem]) -> String { match items { - [] => String::new(), + [] => match jobs.first() { + Some(j) if j.total > 0 => format!("{} ({} of {})", j.title, j.done, j.total), + Some(j) => format!("{}…", j.title), + None => String::new(), + }, [t] => { let arrow = match t.direction { TransferDirection::Download => "↓", @@ -87,7 +93,7 @@ fn poll_state(socket: &Path, default_mountpoint: &Path) -> DriveState { mountpoint: PathBuf::from(mountpoint), // Same daemon is up, so a cheap follow-up poll gives the sync line. sync: match send(socket, &Request::GetQueueStatus) { - Ok(Response::Transfers { items }) => sync_line(&items), + Ok(Response::Transfers { items, jobs }) => sync_line(&items, &jobs), _ => String::new(), }, },