//! Two-way folder sync engine (devices.md Phase 2). //! //! Each `mirror`-mode synced folder is kept mirrored between a local directory //! and a folder under this machine's Proton Drive device. A local filesystem //! watcher ([`notify`]) and a periodic remote poll both feed a debounced //! reconcile pass that diffs three states — the current local tree, the current //! remote tree, and the last-synced baseline in `sync_entry` — to decide, per //! path, which side changed and how to propagate it. //! //! Change detection is `(mtime, size)` on each side (no cheap content hash is //! exposed). The baseline is refreshed from ground truth after every applied //! action, so a file just uploaded or downloaded reads as "unchanged" on the //! next pass and never ping-pongs. //! //! ## Open-for-write deferral //! //! A file still held open for writing by another process is not uploaded. //! Uploading mid-write produces a torn snapshot — a revision that was never a //! real state of the file — wasting the encrypt-and-upload cost and creating a //! revision the user never asked for. The FUSE mount path already defers until //! `close(fd)` by design; the mirror path now does the same by scanning //! `/proc/*/fd` once per reconcile pass and skipping any file whose fd flags //! include `O_WRONLY` or `O_RDWR`. Deferred files are picked up on the next //! pass, after the writer has closed them. use parking_lot::Mutex; use std::collections::{HashMap, HashSet}; use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::mpsc::{Receiver, RecvTimeoutError}; use std::time::{Duration, Instant, SystemTime}; use notify::{RecursiveMode, Watcher}; use proton_drive_rs::proton_sdk::ids::NodeUid; use proton_drive_rs::{Node, NodeKind}; use tracing::{debug, info, warn}; use crate::transfers::{CountingWriter, OwnedCountingReader}; use crate::{Core, now_secs, parse_uid}; use pdfs_core::batch; use pdfs_core::control::{ActivityKind, TransferDirection}; use pdfs_core::db::{StoredSyncEntry, StoredSyncFolder}; use pdfs_core::syncignore::IgnoreRules; mod engine; mod local_scan; mod model; mod planner; use engine::SYNC_CONCURRENCY; pub(crate) use engine::{SyncMsg, spawn}; #[cfg(test)] use engine::{classify, settle_with}; #[cfg(test)] use local_scan::is_write_mode; use local_scan::open_for_write_set; use model::{LocalItem, LocalSig, Outcome, PassAbort, Pending, RemoteItem}; pub(crate) use planner::base_name; #[cfg(test)] use planner::conflict_path; use planner::{ FilePlan, classification_order, conflict_path_with_suffix, filter_baseline, guard_local_wipe, join_rel, parent_rel, plan_file, rel_to_path, remote_sig, unchanged_remote_size, }; /// What a [`Pending`] op did, so the reconcile can update shared state on the /// engine thread (never inside a task). enum Applied { /// A folder was created; register `uid` in `remote_dirs` for its children. Dir(String, NodeUid), /// A local file was uploaded (new file or new revision). Uploaded, /// A remote file was downloaded. Downloaded, /// A conflict copy was kept. Conflict, } impl Core { /// Reconcile one synced folder, then apply any mode switch queued while it was /// busy. Both halves live here because the switch has to happen with the pass /// finished and its lock released, and the engine thread is the only place that /// is guaranteed — a switch requested over the control socket cannot wait for a /// pass that may run for minutes. pub(crate) fn reconcile_folder(&self, folder: &StoredSyncFolder) { // A paused folder skips the settle too: a queued switch to on-demand // evicts the local tree, which is only safe after a pass uploaded it. if self.sync_folder_paused(folder.id) { debug!(id = folder.id, "sync: folder paused, skipping reconcile"); return; } // `ondemand` folders are live FUSE mounts, not mirrored, so there is nothing // to reconcile — but one may still carry a queued switch back to `mirror`, // so the settle below is not skipped with it. if folder.mode == "mirror" { self.reconcile_pass(folder); } self.settle_pending_mode(folder.id); } /// One reconcile pass over a mirror folder, updating its `state` column. fn reconcile_pass(&self, folder: &StoredSyncFolder) { // Hold the folder's lock for the whole pass so a mode switch can't evict the // local tree (and mount FUSE over it) while we walk and upload it. let lock = self.sync_lock(folder.id); let _guard = lock.lock(); // `folder` was read before the lock; a switch may have landed in between, so // re-read and re-check the mode rather than trusting the snapshot. let current = match self.db.sync_folder_get(folder.id) { Ok(Some(current)) if current.mode == "mirror" => current, Ok(Some(_)) => return, Ok(None) => return, Err(e) => { warn!(id = folder.id, error = ?e, "sync: cannot re-read folder; skipping"); return; } }; // A folder waiting to go on-demand is about to have its local copy deleted, // so pulling the remote side down is work whose only result is more to evict. // All that pass owes the user is getting local changes up. let mut push_only = current.pending_mode.as_deref() == Some("ondemand"); let Some(remote_root) = parse_uid(&folder.remote_uid) else { warn!(id = folder.id, "sync: bad remote uid; skipping"); return; }; let local_root = PathBuf::from(&folder.local_path); if !local_root.is_dir() { warn!(id = folder.id, path = %local_root.display(), "sync: local root missing"); let _ = self .db .sync_folder_set_state(folder.id, "error", now_secs()); return; } let _ = self .db .sync_folder_set_state(folder.id, "syncing", folder.last_sync); let name = base_name(&folder.local_path); // Rebuilt per pass rather than cached on `Core`, so editing `.pdfsignore` // takes effect on the next reconcile instead of at the next daemon start. // The cost is one small file read against a pass that is about to walk the // whole tree. let rules = self.ignore_rules(&local_root); self.progress_begin(folder.id); let result = if push_only { self.push_pass(folder.id, &local_root, &remote_root, &rules) } else { match self.do_reconcile(folder.id, &local_root, &remote_root, &rules) { Ok(outcome) => Ok(outcome), Err(PassAbort::Failed(e)) => Err(e), // The user asked for on-demand while this pass was running. Rather // than make them wait out a walk-and-download whose results are // about to be deleted, drop it here and do the only part that still // matters — getting local changes up — so the switch lands now // instead of after the pass and another poll. Err(PassAbort::Interrupted) => { info!( id = folder.id, "sync: on-demand queued mid-pass; pushing instead" ); push_only = true; self.progress_begin(folder.id); self.push_pass(folder.id, &local_root, &remote_root, &rules) } } }; self.progress_end(folder.id); match result { Ok(outcome) => { // A folder only reaches `idle` when every path applied cleanly — // an un-uploaded file must keep it out of `idle` so it can't be // switched to on-demand (which evicts the local copy). // // A push pass is the exception on conflicts: it resolves them by // uploading the local copy under a conflict name rather than leaving // one on disk, so nothing local is left needing attention. Parking it // in `conflict` would block the very switch the pass was run for. let state = if outcome.errors > 0 { "error" } else if outcome.conflicts > 0 && !push_only { "conflict" } else { "idle" }; let _ = self.db.sync_folder_set_state(folder.id, state, now_secs()); // Summarise a pass that actually moved something. A poll over an // unchanged folder does nothing and says nothing — otherwise the // feed would fill with "synced, 0 changes" every two minutes. if !outcome.is_empty() { self.log_activity( ActivityKind::Sync, name, outcome.summary(), outcome.errors == 0, ); } // Surface a new conflict once (not on every poll while it persists). if state == "conflict" && folder.state != "conflict" { self.log_activity( ActivityKind::Sync, format!("sync conflict in {name}"), format!("{} file(s) kept as conflict copies", outcome.conflicts), false, ); } // Surface partial failures once, too — the pass still uploaded // everything it could, but some paths need another attempt. if state == "error" && folder.state != "error" { self.log_activity( ActivityKind::Sync, format!("sync incomplete for {name}"), format!("{} item(s) failed; will retry", outcome.errors), false, ); } } Err(e) => { warn!(id = folder.id, error = %e, "sync: reconcile failed"); let _ = self .db .sync_folder_set_state(folder.id, "error", now_secs()); if folder.state != "error" { self.log_activity( ActivityKind::Sync, format!("sync failed for {name}"), e, false, ); } } } } /// Compile the ignore rules for a synced folder: the global patterns from /// the config, plus any `.pdfsignore` at `local_root`. /// /// A config that cannot be read falls back to the built-in defaults rather /// than to "ignore nothing" — the defaults exclude build and VCS trees, and /// silently uploading those because a config read failed is the outcome this /// feature exists to prevent. fn ignore_rules(&self, local_root: &Path) -> IgnoreRules { let globals = match pdfs_core::config::AppDirs::new() { Ok(dirs) => dirs.load_config().resolved_ignore_patterns(), Err(e) => { warn!(error = %e, "sync: cannot resolve config dir; using default ignore patterns"); pdfs_core::syncignore::DEFAULT_IGNORE_PATTERNS .iter() .map(|s| (*s).to_string()) .collect() } }; IgnoreRules::load(local_root, &globals) } /// Whether a switch to on-demand is waiting on this folder — i.e. whether a /// pass in flight is still doing work worth doing. Read from the db rather than /// held in memory because the request arrives on the control-socket thread while /// the pass runs on the engine thread, and the db row is already the one place /// both agree on. fn ondemand_queued(&self, folder_id: i64) -> bool { matches!( self.db.sync_folder_get(folder_id), Ok(Some(f)) if f.pending_mode.as_deref() == Some("ondemand") ) } /// Get every local change up to Drive, and nothing else — the pass run for a /// folder waiting to go on-demand, whose local copy is about to be evicted. /// /// This exists because a full [`do_reconcile`](Self::do_reconcile) is both too /// slow and too wasteful to stand between the user and the switch they asked /// for. It walks the whole remote tree (the minutes-long part of a pass over a /// large folder) to answer a question the switch makes irrelevant — what the /// remote side has that we don't — and then downloads those files onto a disk /// we are about to clear. A push pass answers only "is anything here not on /// Drive yet?", which the local tree and the baseline already know, so it needs /// no remote walk at all and finishes in the time the local walk takes. /// /// The remote is consulted for exactly one thing: the files whose local copy /// changed. Those, and only those, could be a two-sided edit, so their nodes are /// fetched (cheaply, in one batch) to check whether the remote moved too. If it /// did, the local copy goes up under a conflict name rather than overwriting the /// other side. Everything else — remote edits, remote deletions, remote-only /// files — is left for the FUSE mount to show live once the switch lands. /// /// Per-path failures are counted, never fatal, exactly as in `do_reconcile`; a /// non-zero [`Outcome::errors`] leaves the folder out of `idle`, which keeps the /// queued switch waiting rather than evicting a file that never made it up. fn push_pass( &self, folder_id: i64, local_root: &Path, remote_root: &NodeUid, rules: &IgnoreRules, ) -> Result { let baseline = self .db .sync_entries(folder_id) .map_err(|e| format!("load baseline: {e:?}"))?; // Only the local side is walked here, so each baseline path is checked once. self.progress_scan_total(folder_id, baseline.len()); let mut local: HashMap = HashMap::new(); let writing = open_for_write_set(local_root); self.walk_local( folder_id, local_root, local_root, rules, &writing, &mut local, )?; // The guard compares against the paths this pass could still see: an // ignored path is absent from `local` by rule, not by loss, and counting // those as missing would trip the guard on every pass — wedging the // folder's sync permanently — the moment a rule covers the whole tree. guard_local_wipe(&filter_baseline(&baseline, rules), &local)?; // The remote folder uids come from the baseline instead of a walk: every // directory this device has synced recorded its uid there when it was // created or first matched up. let mut remote_dirs: HashMap = HashMap::new(); remote_dirs.insert(String::new(), remote_root.clone()); for (rel, item) in &local { if !item.is_dir { continue; } if let Some(uid) = baseline .get(rel) .and_then(|e| e.remote_uid.as_deref()) .and_then(parse_uid) { remote_dirs.insert(rel.clone(), uid); } } // A local directory the baseline has no uid for is either brand new or one // whose first pass never finished — and those look identical from here. // Creating it blindly would duplicate a remote folder we simply haven't // looked at, so when any exist, the remote folder tree is walked for real. // It costs one light listing per folder and no file keys, and a folder whose // baseline is complete — the case this pass is written for — skips it // entirely and touches the network only for the files it uploads. if local .iter() .any(|(rel, item)| item.is_dir && !remote_dirs.contains_key(rel)) { self.walk_remote_dirs(folder_id, remote_root, "", &mut remote_dirs)?; } // Files whose local copy has moved since the baseline: the only ones with // anything to push, and the only ones that could be a two-sided edit. let changed: Vec<&String> = local .iter() .filter(|(_, item)| !item.is_dir && !item.open_for_write) .filter(|(rel, item)| { baseline .get(*rel) .is_none_or(|b| b.local_mtime != item.mtime || b.local_size != item.size) }) .map(|(rel, _)| rel) .collect(); // One light batch tells us which of those also changed remotely. A light // node carries the modification time without unlocking the node key, and // mtime alone is what the baseline compares against. let check: Vec = changed .iter() .filter_map(|rel| baseline.get(*rel)) .filter_map(|e| e.remote_uid.as_deref()) .filter_map(parse_uid) .collect(); let remote_mtimes: HashMap = if check.is_empty() { HashMap::new() } else { self.rt .block_on(self.client.enumerate_nodes_light(&check)) .map_err(|e| format!("resolve changed nodes: {e}"))? .into_iter() .filter(|n| !n.trashed) .map(|n| (n.uid.to_string(), n.modification_time)) .collect() }; let changed: HashSet<&String> = changed.into_iter().collect(); let mut outcome = Outcome { deferred: local.values().filter(|item| item.open_for_write).count(), ..Default::default() }; let mut order: Vec<&String> = local.keys().collect(); order.sort_by_key(|p| p.matches('/').count()); // Same depth-ascending batching as `do_reconcile`: a folder is created // remotely before the paths inside it are classified, so their parent uid is // in `remote_dirs` by the time they need it. let mut batch: Vec = Vec::new(); let mut batch_depth = 0usize; for rel in order { let depth = rel.matches('/').count(); if depth > batch_depth { self.flush_batch( folder_id, local_root, &mut remote_dirs, std::mem::take(&mut batch), &mut outcome, ); batch_depth = depth; } let item = &local[rel]; if item.is_dir { // The folder is already on Drive, so only its children carry work — // but if the walk above is what found it, record the uid now so the // next pass over this folder needs no walk at all. if let Some(uid) = remote_dirs.get(rel) { if baseline .get(rel) .and_then(|e| e.remote_uid.as_deref()) .is_none() && let Err(e) = self.baseline_dir(folder_id, rel, uid) { warn!(rel, error = %e, "sync: folder step failed; continuing"); outcome.errors += 1; } continue; } match remote_dirs.get(parent_rel(rel)) { Some(parent) => batch.push(Pending::CreateDir { rel: rel.clone(), parent: parent.clone(), }), None => { warn!(rel, "sync: no remote parent for folder; continuing"); outcome.errors += 1; } } continue; } if !changed.contains(rel) { // Local copy matches the baseline, so it is already on Drive. continue; } let base = baseline.get(rel); let uid = base .and_then(|e| e.remote_uid.as_deref()) .and_then(parse_uid); match uid { // Never uploaded (or the remote node is gone): it goes up as a new file. None => match remote_dirs.get(parent_rel(rel)) { Some(parent) => batch.push(Pending::UploadNew { rel: rel.clone(), parent: parent.clone(), }), None => { warn!(rel, "sync: no remote parent for file; continuing"); outcome.errors += 1; } }, Some(uid) => { let remote_moved = match remote_mtimes.get(&uid.to_string()) { Some(mtime) => base.and_then(remote_sig).is_none_or(|(m, _)| m != *mtime), // Trashed or unreadable remotely — treat as gone and re-upload // the local copy as a new file rather than losing it. None => { match remote_dirs.get(parent_rel(rel)) { Some(parent) => batch.push(Pending::UploadNew { rel: rel.clone(), parent: parent.clone(), }), None => { warn!(rel, "sync: no remote parent for file; continuing"); outcome.errors += 1; } } continue; } }; if remote_moved { match remote_dirs.get(parent_rel(rel)) { Some(parent) => batch.push(Pending::PushConflict { rel: rel.clone(), parent: parent.clone(), }), None => { warn!(rel, "sync: no remote parent for file; continuing"); outcome.errors += 1; } } } else { batch.push(Pending::UploadRevision { rel: rel.clone(), uid, }); } } } } self.flush_batch( folder_id, local_root, &mut remote_dirs, std::mem::take(&mut batch), &mut outcome, ); // Paths the baseline knows that are no longer on disk: the user deleted them // locally, and the deletion has to reach Drive before the mount starts // showing the folder's remote contents — otherwise everything they deleted // comes back the moment the switch lands. Shallowest first, skipping anything // under a folder already trashed (its children went with it). let mut missing: Vec<&String> = baseline .keys() .filter(|rel| !local.contains_key(*rel)) .collect(); missing.sort_by_key(|p| p.matches('/').count()); let mut trashed: Vec = Vec::new(); for rel in missing { if trashed .iter() .any(|dir| rel.starts_with(&format!("{dir}/"))) { let _ = self.db.sync_entry_remove(folder_id, rel); continue; } let Some(uid) = baseline[rel].remote_uid.as_deref().and_then(parse_uid) else { let _ = self.db.sync_entry_remove(folder_id, rel); continue; }; if let Err(e) = self .rt .block_on(self.client.trash_nodes(&[uid])) .and_then(batch::into_unit) { warn!(rel, error = %e, "sync: trash remote failed"); self.log_activity(ActivityKind::Trash, base_name(rel), e.to_string(), false); outcome.errors += 1; continue; } trashed.push(rel.clone()); if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) { self.invalidate_children_of(&parent); } let _ = self.db.sync_entry_remove(folder_id, rel); self.log_activity( ActivityKind::Trash, base_name(rel), "removed on Drive", true, ); outcome.deleted += 1; } Ok(outcome) } /// The diff-and-apply core. A single path failing to apply (a transient /// upload error, an unreadable file, a name collision) must not abort the /// whole pass and strand every other path — otherwise one bad file in a /// large folder leaves the sync permanently incomplete and blocks the /// on-demand switch. So per-path failures are logged and counted in /// [`Outcome::errors`], and the walk continues. Only a failure to establish /// the diff itself (the walks / baseline load) aborts with `Err`. fn do_reconcile( &self, folder_id: i64, local_root: &Path, remote_root: &NodeUid, rules: &IgnoreRules, ) -> Result { // Loaded before the walks: the remote one uses it to tell which files are // unchanged and so can skip decrypting their claimed size, and its size is // the only estimate available of how much this pass has to check — the walks // discover the real figure only by finishing. let baseline = self .db .sync_entries(folder_id) .map_err(|e| format!("load baseline: {e:?}"))?; // Both sides are walked, so each baseline path is checked about twice. self.progress_scan_total(folder_id, baseline.len() * 2); let mut local: HashMap = HashMap::new(); let writing = open_for_write_set(local_root); self.walk_local( folder_id, local_root, local_root, rules, &writing, &mut local, )?; // See `push_pass`: the guard must not count rule-excluded paths as lost. guard_local_wipe(&filter_baseline(&baseline, rules), &local)?; let mut remote: HashMap = HashMap::new(); let mut remote_dirs: HashMap = HashMap::new(); remote_dirs.insert(String::new(), remote_root.clone()); self.walk_remote( folder_id, remote_root, "", &mut remote, &mut remote_dirs, &baseline, )?; let order = classification_order(&local, &remote, &baseline, rules); let mut outcome = Outcome::default(); // Folders to delete, collected here and removed deepest-first at the end // so a parent is never removed before its children. let mut delete_local_dirs: Vec = Vec::new(); let mut delete_remote_dirs: Vec<(String, NodeUid)> = Vec::new(); // Single depth-ascending pass. Network work (folder creation, file // upload/download) is queued into `batch` and flushed concurrently at // each depth boundary — so a parent folder is created before the deeper // paths inside it are classified (their parent uid is then in // `remote_dirs`), while everything within a depth runs in parallel. // Interleaving by depth also means files start uploading as soon as // their folder exists, instead of after every folder in the tree. // Cheap, dependency-free work (local mkdir, baseline rows, entry // removals, deletions) is done inline; a per-item failure is logged and // counted, never fatal. let mut batch: Vec = Vec::new(); let mut batch_depth = 0usize; for rel in &order { let depth = rel.matches('/').count(); if depth > batch_depth { self.flush_batch( folder_id, local_root, &mut remote_dirs, std::mem::take(&mut batch), &mut outcome, ); batch_depth = depth; // A depth boundary is the pass's natural checkpoint — everything // queued has been applied, so stopping here leaves nothing half done. if self.ondemand_queued(folder_id) { return Err(PassAbort::Interrupted); } } let l = local.get(rel); let r = remote.get(rel); let is_dir = l.map(|x| x.is_dir).or(r.map(|x| x.is_dir)).unwrap_or(false); let in_base = baseline.contains_key(rel); if is_dir { match (l.is_some(), r.is_some()) { (true, true) => { if let Err(e) = self.baseline_dir(folder_id, rel, &remote[rel].uid) { warn!(rel, error = %e, "sync: folder step failed; continuing"); outcome.errors += 1; } } (true, false) if !in_base => { // New local folder → create remotely (batched). match remote_dirs.get(parent_rel(rel)) { Some(parent) => batch.push(Pending::CreateDir { rel: rel.clone(), parent: parent.clone(), }), None => { warn!(rel, "sync: no remote parent for folder; continuing"); outcome.errors += 1; } } } (true, false) => { // In baseline, gone remotely → the remote side deleted it. delete_local_dirs.push(rel.clone()); } (false, true) if !in_base => { // New remote folder → create locally (cheap, inline). if let Err(e) = std::fs::create_dir_all(local_root.join(rel_to_path(rel))) .map_err(|e| format!("mkdir {rel}: {e}")) .and_then(|()| self.baseline_dir(folder_id, rel, &remote[rel].uid)) { warn!(rel, error = %e, "sync: folder step failed; continuing"); outcome.errors += 1; } } (false, true) => { // In baseline, gone locally → the local side deleted it. delete_remote_dirs.push((rel.clone(), remote[rel].uid.clone())); } (false, false) => { let _ = self.db.sync_entry_remove(folder_id, rel); } } continue; } match plan_file(l, r, baseline.get(rel)) { FilePlan::Unchanged => {} FilePlan::AdoptBaseline => { // First sync of an already-identical file: record the baseline // in place so future divergence is detected, without any // transfer or a spurious conflict copy. let local = l.expect("adopt baseline requires a local file"); let remote = r.expect("adopt baseline requires a remote file"); if let Err(e) = self.db.sync_entry_upsert( folder_id, &StoredSyncEntry { rel_path: rel.clone(), remote_uid: Some(remote.uid.to_string()), local_mtime: local.mtime, local_mtime_ns: local.mtime_ns, local_size: local.size, remote_rev: Some(remote.mtime.to_string()), remote_hash: Some(remote.size.to_string()), }, ) { warn!(rel, error = ?e, "sync: adopt baseline failed"); outcome.errors += 1; } } FilePlan::Deferred => { debug!( rel, "sync: file open for write by another process; deferring upload" ); outcome.deferred += 1; } FilePlan::UploadRevision => { let remote = r.expect("upload revision requires a remote file"); batch.push(Pending::UploadRevision { rel: rel.clone(), uid: remote.uid.clone(), }); } FilePlan::Download => { let remote = r.expect("download requires a remote file"); batch.push(Pending::Download { rel: rel.clone(), uid: remote.uid.clone(), mtime: remote.mtime, size: remote.size, // What the walk saw here, so the apply step can tell a // stale plan from a live one before it overwrites. local: l.map(LocalSig::from), }); } FilePlan::Conflict => { let remote = r.expect("conflict requires a remote file"); batch.push(Pending::Conflict { rel: rel.clone(), uid: remote.uid.clone(), mtime: remote.mtime, size: remote.size, }); } FilePlan::UploadNew => match remote_dirs.get(parent_rel(rel)) { Some(parent) => batch.push(Pending::UploadNew { rel: rel.clone(), parent: parent.clone(), }), None => { warn!(rel, "sync: no remote parent for file; continuing"); outcome.errors += 1; } }, FilePlan::DeleteLocal => { if let Err(e) = removed_locally(std::fs::remove_file(local_root.join(rel_to_path(rel)))) { warn!(rel, error = %e, "sync: remove local file failed"); self.log_activity( ActivityKind::Trash, base_name(rel), e.to_string(), false, ); outcome.errors += 1; continue; } let _ = self.db.sync_entry_remove(folder_id, rel); self.log_activity(ActivityKind::Trash, base_name(rel), "removed locally", true); outcome.deleted += 1; } FilePlan::DeleteRemote => { let remote = r.expect("remote deletion requires a remote file"); if let Err(e) = self .rt .block_on(self.client.trash_nodes(std::slice::from_ref(&remote.uid))) .and_then(batch::into_unit) { warn!(rel, error = %e, "sync: trash remote failed"); self.log_activity( ActivityKind::Trash, base_name(rel), e.to_string(), false, ); outcome.errors += 1; continue; } if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) { self.invalidate_children_of(&parent); } let _ = self.db.sync_entry_remove(folder_id, rel); self.log_activity( ActivityKind::Trash, base_name(rel), "removed on Drive", true, ); outcome.deleted += 1; } FilePlan::ForgetBaseline => { let _ = self.db.sync_entry_remove(folder_id, rel); } } } // Flush the deepest level's batch. self.flush_batch( folder_id, local_root, &mut remote_dirs, std::mem::take(&mut batch), &mut outcome, ); // Deferred folder deletions, deepest first. delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count())); for rel in delete_local_dirs { if let Err(e) = removed_locally(std::fs::remove_dir_all(local_root.join(rel_to_path(&rel)))) { warn!(rel, error = %e, "sync: remove local folder failed"); self.log_activity(ActivityKind::Trash, base_name(&rel), e.to_string(), false); outcome.errors += 1; continue; } let _ = self.db.sync_entry_remove(folder_id, &rel); self.log_activity( ActivityKind::Trash, base_name(&rel), "removed locally", true, ); outcome.deleted += 1; } delete_remote_dirs.sort_by_key(|(p, _)| std::cmp::Reverse(p.matches('/').count())); for (rel, uid) in delete_remote_dirs { if let Err(e) = self .rt .block_on(self.client.trash_nodes(&[uid])) .and_then(batch::into_unit) { warn!(rel, error = %e, "sync: trash remote folder failed"); self.log_activity(ActivityKind::Trash, base_name(&rel), e.to_string(), false); outcome.errors += 1; continue; } if let Some(parent) = parent_uid_of(&rel, &remote_dirs, &baseline) { self.invalidate_children_of(&parent); } let _ = self.db.sync_entry_remove(folder_id, &rel); self.log_activity( ActivityKind::Trash, base_name(&rel), "removed on Drive", true, ); outcome.deleted += 1; } Ok(outcome) } /// Recursively walk a local directory into `out`, keyed by `/`-joined relative /// path. Symlinks and other special files are skipped. Reports each entry to the /// pass's scan progress. fn walk_local( &self, folder_id: i64, root: &Path, dir: &Path, rules: &IgnoreRules, writing: &HashSet, out: &mut HashMap, ) -> Result<(), String> { walk_local_tree(root, dir, rules, writing, out, &mut |rel| { self.progress_scanned(folder_id, base_name(rel)) }) } /// Map every remote folder under `folder` to its uid by rel path, ignoring /// files. The cheap half of [`walk_remote`](Self::walk_remote): folders carry no /// content signature, so a light listing answers this without unlocking a single /// node key. Used by [`push_pass`](Self::push_pass) to tell a folder it must /// create from one it just hasn't recorded yet. fn walk_remote_dirs( &self, folder_id: i64, folder: &NodeUid, prefix: &str, dirs: &mut HashMap, ) -> Result<(), String> { let uids = self .rt .block_on(self.client.enumerate_folder_children_node_uids(folder)) .map_err(|e| format!("enumerate {folder}: {e}"))?; if uids.is_empty() { return Ok(()); } let nodes = self .rt .block_on(self.client.enumerate_nodes_light(&uids)) .map_err(|e| format!("resolve nodes: {e}"))?; for node in nodes { if node.trashed || !node.is_folder() { continue; } let rel = join_rel(prefix, &node.name); self.progress_scanned(folder_id, base_name(&rel)); dirs.insert(rel.clone(), node.uid.clone()); self.walk_remote_dirs(folder_id, &node.uid, &rel, dirs)?; } Ok(()) } /// Recursively walk a remote folder into `out`, recording every descendant's /// relative path, and mapping each subfolder's rel path to its uid in `dirs`. /// /// A file's size comes from its extended attributes, which only its own node /// key can decrypt — and unlocking that key costs an S2K derivation (tens of /// milliseconds) *per file*, which is the bulk of a pass over a large folder. /// So the walk enumerates cheaply ([`enumerate_nodes_light`]) and only pays /// for the files whose modification time has moved away from `baseline`: an /// unchanged mtime means the recorded size still stands. /// /// [`enumerate_nodes_light`]: proton_drive_rs::ProtonDriveClient::enumerate_nodes_light fn walk_remote( &self, folder_id: i64, folder: &NodeUid, prefix: &str, out: &mut HashMap, dirs: &mut HashMap, baseline: &HashMap, ) -> Result<(), PassAbort> { // Checked per folder, not per pass: this walk is the long pole on a large // tree, and a user who asks for on-demand three minutes into it should not // wait out the rest of a survey of files they want deleted. Bailing out here // is safe precisely because the walk has applied nothing — a *partial* // remote map, on the other hand, would be read as "the remote deleted // everything we haven't reached yet", so it must never reach the diff. if self.ondemand_queued(folder_id) { return Err(PassAbort::Interrupted); } let uids = self .rt .block_on(self.client.enumerate_folder_children_node_uids(folder)) .map_err(|e| format!("enumerate {folder}: {e}"))?; if uids.is_empty() { return Ok(()); } let nodes = self .rt .block_on(self.client.enumerate_nodes_light(&uids)) .map_err(|e| format!("resolve nodes: {e}"))?; // Files whose recorded mtime no longer matches: their size has to be // read for real, so these are the only node keys worth unlocking. let stale: Vec = nodes .iter() .filter(|n| !n.trashed && !n.is_folder()) .filter(|n| { unchanged_remote_size(baseline, &join_rel(prefix, &n.name), n.modification_time) .is_none() }) .map(|n| n.uid.clone()) .collect(); let sized: HashMap = if stale.is_empty() { HashMap::new() } else { self.rt .block_on(self.client.enumerate_nodes(&stale)) .map_err(|e| format!("resolve nodes: {e}"))? .iter() .map(|n| (n.uid.to_string(), remote_file_sig(n).1)) .collect() }; let stale: HashSet = stale.iter().map(|u| u.to_string()).collect(); for node in nodes { if node.trashed { continue; } let rel = join_rel(prefix, &node.name); self.progress_scanned(folder_id, base_name(&rel)); if node.is_folder() { dirs.insert(rel.clone(), node.uid.clone()); out.insert( rel.clone(), RemoteItem { uid: node.uid.clone(), is_dir: true, mtime: 0, size: 0, }, ); self.walk_remote(folder_id, &node.uid, &rel, out, dirs, baseline)?; } else { let size = if stale.contains(&node.uid.to_string()) { match sized.get(&node.uid.to_string()) { Some(size) => *size, // Its size was needed but could not be read (the full // enumeration skipped it as undecryptable). Leave it out // rather than record a stale size, exactly as a full // walk would have. None => continue, } } else { match unchanged_remote_size(baseline, &rel, node.modification_time) { Some(size) => size, None => continue, } }; out.insert( rel, RemoteItem { uid: node.uid, is_dir: false, mtime: node.modification_time, size, }, ); } } Ok(()) } /// Record a folder's baseline row (folders carry no content signature). fn baseline_dir(&self, folder_id: i64, rel: &str, uid: &NodeUid) -> Result<(), String> { self.db .sync_entry_upsert( folder_id, &StoredSyncEntry { rel_path: rel.to_string(), remote_uid: Some(uid.to_string()), local_mtime: 0, local_mtime_ns: None, local_size: 0, remote_rev: None, remote_hash: None, }, ) .map_err(|e| format!("baseline dir {rel}: {e:?}")) } /// Run a batch of queued [`Pending`] ops concurrently (bounded by /// [`SYNC_CONCURRENCY`]) and fold their results back into the reconcile's /// shared state on the engine thread: created folders are registered in /// `remote_dirs` (so deeper paths resolve their parent), and what each op did /// is tallied into `outcome`. Driven by `block_on` from the (non-runtime) sync /// engine thread, spawning the tasks onto the shared runtime. fn flush_batch( &self, folder_id: i64, local_root: &Path, remote_dirs: &mut HashMap, batch: Vec, outcome: &mut Outcome, ) { if batch.is_empty() { return; } self.progress_queued(folder_id, batch.len()); let core = self.clone(); let root = local_root.to_path_buf(); let results = self.rt.block_on(async move { let sem = Arc::new(tokio::sync::Semaphore::new(SYNC_CONCURRENCY)); let mut set = tokio::task::JoinSet::new(); for op in batch { let core = core.clone(); let sem = sem.clone(); let root = root.clone(); set.spawn(async move { let _permit = sem.acquire_owned().await.expect("semaphore not closed"); core.apply_pending(folder_id, &root, op).await }); } let mut out = Vec::new(); let mut join_errors = 0usize; while let Some(joined) = set.join_next().await { match joined { Ok(result) => out.push(result), Err(e) => { join_errors += 1; warn!(error = %e, "sync: task panicked"); } } } (out, join_errors) }); let (results, join_errors) = results; outcome.errors += join_errors; for result in results { match result { Ok(applied) => { outcome.record(&applied); if let Applied::Dir(rel, uid) = applied { remote_dirs.insert(rel, uid); } } Err(e) => { warn!(error = %e, "sync: step failed; continuing"); outcome.errors += 1; } } } } /// Apply one [`Pending`] op (async, so it can run concurrently in a batch), /// reporting it to the folder's live progress and to the activity feed. Each /// item logs its own line — the feed is meant to answer "what is it doing", /// which a pass-level summary alone cannot. async fn apply_pending( &self, folder_id: i64, local_root: &Path, op: Pending, ) -> Result { self.progress_started(folder_id, base_name(op.rel())); let result = self.apply_one(folder_id, local_root, &op).await; self.progress_finished(folder_id); let name = base_name(op.rel()); match &result { Ok(_) => self.log_activity(op.kind(), name, op.detail(), true), Err(e) => self.log_activity(op.kind(), name, e.clone(), false), } result } /// The body of one [`Pending`] op, without the progress/activity bookkeeping. async fn apply_one( &self, folder_id: i64, local_root: &Path, op: &Pending, ) -> Result { match op { Pending::CreateDir { rel, parent } => { let uid = self .client .create_folder(parent, base_name(rel), Some(now_secs())) .await .map_err(|e| format!("create remote folder {rel}: {e}"))?; // The parent has gained a child that no cached listing of it // knows about (B86). self.invalidate_children_of(parent); self.baseline_dir(folder_id, rel, &uid)?; Ok(Applied::Dir(rel.clone(), uid)) } Pending::UploadNew { rel, parent } => { self.upload_new(folder_id, local_root, rel, parent).await?; Ok(Applied::Uploaded) } Pending::UploadRevision { rel, uid } => { self.upload_revision(folder_id, local_root, rel, uid) .await?; Ok(Applied::Uploaded) } Pending::Download { rel, uid, mtime, size, local, } => { self.download_file(folder_id, local_root, rel, uid, *mtime, *size, *local) .await?; Ok(Applied::Downloaded) } Pending::Conflict { rel, uid, mtime, size, } => { // Set the local copy aside (it re-uploads as a new file next // pass), then take the remote version as the shared truth. let path = local_root.join(rel_to_path(rel)); if path.exists() { preserve_conflict_copy(&path, now_secs()) .map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?; info!(rel, "sync: kept local changes as a conflict copy"); } // The local copy has just been moved aside, so the state this // download expects to find at the path is "nothing" — anything // that appears while it runs is a *new* racing write, and gets // its own conflict copy rather than being overwritten (B39). self.download_file(folder_id, local_root, rel, uid, *mtime, *size, None) .await?; Ok(Applied::Conflict) } Pending::PushConflict { rel, parent } => { // The ordinary conflict resolution — rename the local copy aside and // let the next pass upload it — loses the file here: the folder is // going on-demand, so "next pass" comes after the local tree has been // evicted. Push the conflict copy now, while it still exists, and // leave the remote file holding the original path. let path = local_root.join(rel_to_path(rel)); let conflict = preserve_conflict_copy(&path, now_secs()) .map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?; let name = conflict .file_name() .map(|n| n.to_string_lossy().into_owned()) .ok_or_else(|| format!("conflict copy for {rel} has no name"))?; let conflict_rel = join_rel(parent_rel(rel), &name); self.upload_new(folder_id, local_root, &conflict_rel, parent) .await?; info!(rel, "sync: uploaded local changes as a conflict copy"); Ok(Applied::Conflict) } } } /// Upload a brand-new local file to its remote parent, then record baseline. async fn upload_new( &self, folder_id: i64, local_root: &Path, rel: &str, parent: &NodeUid, ) -> Result<(), String> { let path = local_root.join(rel_to_path(rel)); let name = base_name(rel); let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?; let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?; let mtime = system_mtime(&meta); // Taken from the descriptor being streamed, so it names the content that // actually goes up rather than whatever is at the path when we finish. let streamed = LocalSig::from(&meta); // Count the bytes as they are read, so this upload shows up in // `GetQueueStatus` next to a manual one. The uid isn't known until the // draft is sealed, so the transfer registers without one. let reader = OwnedCountingReader::new( file, self.transfers .begin(name, "", TransferDirection::Upload, meta.len()), ); let uid = self .client // Mirror push: the local file is authoritative. If a prior attempt was // interrupted mid-upload it left an unsealed draft of this name; recover // it even across a daemon restart (which rotates our client uid), so the // folder can reach idle instead of `AlreadyExists`-looping forever. .upload_file_replacing_draft_from( parent, name, crate::media_type_for(name), reader, meta.len() as i64, Vec::new(), Some(mtime), false, ) .await .map_err(|e| format!("upload {rel}: {e}"))?; // The parent has gained a child that no cached listing of it knows // about. Without this, a folder later switched to on-demand serves a // snapshot that predates this upload (B86). self.invalidate_children_of(parent); warn_if_torn(rel, &path, streamed); self.record_file_baseline(folder_id, rel, Some(streamed), &uid) .await } /// Upload a changed local file as a new revision of an existing remote node. async fn upload_revision( &self, folder_id: i64, local_root: &Path, rel: &str, uid: &NodeUid, ) -> Result<(), String> { let path = local_root.join(rel_to_path(rel)); let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?; let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?; let mtime = system_mtime(&meta); let streamed = LocalSig::from(&meta); let reader = OwnedCountingReader::new( file, self.transfers.begin( base_name(rel), uid.to_string(), TransferDirection::Upload, meta.len(), ), ); self.client .upload_new_revision_from(uid, reader, meta.len() as i64, Vec::new(), Some(mtime)) .await .map_err(|e| format!("upload revision {rel}: {e}"))?; warn_if_torn(rel, &path, streamed); self.record_file_baseline(folder_id, rel, Some(streamed), uid) .await } /// Download a remote file to its local path (atomically via a temp file), /// stamp the local mtime to match the remote, then record baseline. /// /// `expected_local` is the identity planning saw at the destination. The /// rename below is the destructive step, and the download in between can run /// for minutes, so the destination is re-examined right before it: a local /// file that no longer matches the plan is set aside as a conflict copy /// instead of being replaced (bugs.md B39). #[allow(clippy::too_many_arguments)] async fn download_file( &self, folder_id: i64, local_root: &Path, rel: &str, uid: &NodeUid, mtime: i64, size: i64, expected_local: Option, ) -> Result<(), String> { let path = local_root.join(rel_to_path(rel)); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).map_err(|e| format!("mkdir for {rel}: {e}"))?; } let tmp = path.with_extension(format!( "pdfs-tmp-{}", SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) )); { let out = std::fs::File::create(&tmp).map_err(|e| format!("create tmp {rel}: {e}"))?; // Count the bytes as they land, so the download reports progress like // an on-demand hydration does. let guard = self.transfers.begin( base_name(rel), uid.to_string(), TransferDirection::Download, size.max(0) as u64, ); let mut out = CountingWriter::new(out, &guard); self.client .download_file_to(uid, &mut out) .await .map_err(|e| { let _ = std::fs::remove_file(&tmp); format!("download {rel}: {e}") })?; let mut out = out.into_inner(); out.flush().map_err(|e| format!("flush tmp {rel}: {e}"))?; out.sync_all().map_err(|e| format!("sync tmp {rel}: {e}"))?; } // The last moment at which the destination can still be saved. Anything // other than the state planning classified means a writer got there // first, and its bytes are the ones nothing else has a copy of. if let Err(e) = keep_racing_local_edit(&path, expected_local) { let _ = std::fs::remove_file(&tmp); return Err(format!("set aside racing local edit for {rel}: {e}")); } std::fs::rename(&tmp, &path).map_err(|e| format!("place {rel}: {e}"))?; // Match local mtime to the remote's so neither side looks "changed" next pass. let f = std::fs::File::options() .write(true) .open(&path) .map_err(|e| format!("open placed file {rel}: {e}"))?; f.set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(mtime.max(0) as u64)) .map_err(|e| format!("set mtime for {rel}: {e}"))?; f.sync_all() .map_err(|e| format!("sync placed file {rel}: {e}"))?; if let Some(parent) = path.parent() { std::fs::File::open(parent) .and_then(|dir| dir.sync_all()) .map_err(|e| format!("sync directory for {rel}: {e}"))?; } // Here the file on disk *is* the transferred content: it was published // by a rename and stamped just above, so its stat is the identity. self.record_file_baseline(folder_id, rel, local_sig(&path), uid) .await } /// Refresh a file's baseline from ground truth: the local identity these /// bytes came from and the remote node's reported signature. Called after /// every upload/download so the very next reconcile sees the path as /// unchanged. /// /// `local` is the identity the *transferred content* corresponds to, which /// is not always the file's current stat. An upload streams the live file, /// so it passes the identity it read the first byte at: if a writer changed /// the file mid-stream, the remote now holds a torn revision, and recording /// the pre-upload identity is what makes the next pass see the path as /// locally changed and upload a whole, clean revision over it. Recording the /// post-upload stat instead declared the torn revision synchronized and left /// it as the file's content on Drive (bugs.md B40). async fn record_file_baseline( &self, folder_id: i64, rel: &str, local: Option, uid: &NodeUid, ) -> Result<(), String> { let Some(local) = local else { return Err(format!("stat {rel}: no local identity to baseline")); }; let (lmtime, lsize) = (local.mtime, local.size); // Re-fetch the node so the stored remote signature is exactly what a walk // will report next time. let (rmtime, rsize) = match self.client.enumerate_nodes(std::slice::from_ref(uid)).await { Ok(nodes) => nodes .first() .map(remote_file_sig) .unwrap_or((lmtime, lsize)), Err(e) => { warn!(rel, error = %e, "sync: baseline refetch failed; using local mtime"); (lmtime, lsize) } }; self.db .sync_entry_upsert( folder_id, &StoredSyncEntry { rel_path: rel.to_string(), remote_uid: Some(uid.to_string()), local_mtime: lmtime, local_mtime_ns: local.mtime_ns, local_size: lsize, remote_rev: Some(rmtime.to_string()), remote_hash: Some(rsize.to_string()), }, ) .map_err(|e| format!("baseline {rel}: {e:?}")) } } /// Say so when a file changed under an upload that was streaming it. /// /// Purely diagnostic — the correction is already in place, because the baseline /// records the identity the upload *read* rather than the one on disk now, so /// the next pass classifies the path as locally changed and replaces the torn /// revision. Without the log there is nothing in the journal to explain the /// extra revision (bugs.md B40). fn warn_if_torn(rel: &str, path: &Path, streamed: LocalSig) { let now = local_sig(path); if !now.is_some_and(|now| now.same_content(&streamed)) { warn!( rel, ?streamed, ?now, "sync: file changed while uploading; the remote revision may be torn \ and will be replaced on the next pass" ); } } /// The identity a reconcile pass uses to recognise a local file it has already /// looked at: `None` when nothing is at the path. fn local_sig(path: &Path) -> Option { std::fs::metadata(path).ok().map(|m| LocalSig { mtime: system_mtime(&m), mtime_ns: system_mtime_ns(&m), size: m.len() as i64, }) } /// Guard the destination of a download: if what is at `path` is no longer what /// planning classified, move it aside as a conflict copy before the caller /// renames over it. /// /// A reconcile pass decides what to do from a walk that finished before the /// transfer started, and a download of a large file can run for minutes. Without /// this, a local edit completed inside that window was replaced by the remote /// version with no copy of it kept anywhere — the one shape of loss a sync /// engine must not have (bugs.md B39). Keeping the copy makes it re-upload as a /// new file on the next pass, which is the same resolution a both-sides-changed /// conflict already gets. /// /// A path that is empty when it was expected to be is not a conflict: the /// download is about to recreate it, and there is nothing to preserve. fn keep_racing_local_edit(path: &Path, expected: Option) -> std::io::Result<()> { let actual = local_sig(path); let unchanged = match (actual, expected) { (Some(a), Some(e)) => a.same_content(&e), // Nothing there: the download is about to create it, and there is // nothing to preserve either way. (None, _) => true, (Some(_), None) => false, }; if unchanged { return Ok(()); } warn!( path = %path.display(), ?expected, ?actual, "sync: local file changed while downloading; keeping it as a conflict copy" ); preserve_conflict_copy(path, now_secs()).map(|_| ()) } /// Read a local removal as the sync engine has to read it: the *state* it was /// asking for is "this path is gone", so a path that was already gone is a /// success, not a failure. /// /// It matters because the baseline is only dropped on success. Treating /// `NotFound` as an error left the baseline in place for a file that no longer /// exists, so the same removal was re-attempted and re-reported as an error on /// every pass, forever. Every other error is still an error, and keeping the /// baseline is exactly right there: the surviving file must not be mistaken for /// a new local creation on the next pass and uploaded back (bugs.md B42). fn removed_locally(result: std::io::Result<()>) -> std::io::Result<()> { match result { Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), other => other, } } /// The body of [`Core::walk_local`], without a `Core`: recursively collect the /// local tree under `dir` into `out`, reporting each entry to `scanned`. /// /// **Every failure here is fatal to the pass, deliberately.** A `read_dir` that /// fails, an entry that cannot be stat-ed, a name that is not UTF-8 — each of /// them would, if skipped, leave `out` describing a *smaller* tree than the one /// on disk. Reconciliation reads a path present in the baseline and absent from /// the scan as "the user deleted this", and propagates that to Drive. So an /// unreadable subdirectory would trash its remote counterpart, and the deeper /// the failure, the more it takes with it (`docs/BUGS.md` B55). Refusing the /// whole pass keeps the previous baseline and leaves the remote alone; the next /// pass retries once the transient condition clears. /// /// Split out from the method so exactly that can be tested against a real tree /// with a real unreadable directory in it, which needs no `Core` at all. fn walk_local_tree( root: &Path, dir: &Path, rules: &IgnoreRules, writing: &HashSet, out: &mut HashMap, scanned: &mut dyn FnMut(&str), ) -> Result<(), String> { let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?; for entry in entries { let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?; let path = entry.path(); let meta = std::fs::symlink_metadata(&path) .map_err(|e| format!("stat {}: {e}", path.display()))?; let stripped = path.strip_prefix(root).map_err(|e| { format!( "local path {} escaped sync root {}: {e}", path.display(), root.display() ) })?; let rel = stripped.to_str().ok_or_else(|| { format!( "local path {} is not valid UTF-8; refusing a destructive sync pass", path.display() ) })?; // Ignore our own in-flight download temp files. if rel.contains(".pdfs-tmp-") { continue; } // Ignored paths are dropped here as well as from the classification // union — not for correctness (the union filter is what makes this // safe) but so an ignored `node_modules/` is never descended into or // counted against scan progress. if rules.is_ignored(rel, meta.is_dir()) { continue; } if meta.is_dir() { out.insert( rel.to_string(), LocalItem { is_dir: true, mtime: 0, mtime_ns: None, size: 0, open_for_write: false, }, ); scanned(rel); walk_local_tree(root, &path, rules, writing, out, scanned)?; } else if meta.is_file() { // Check whether any process holds this file open for writing. // The canonical path is used because /proc/*/fd links resolve // to canonical targets. let ofw = std::fs::canonicalize(&path) .map(|canon| writing.contains(&canon)) .unwrap_or(false); out.insert( rel.to_string(), LocalItem { is_dir: false, mtime: system_mtime(&meta), mtime_ns: system_mtime_ns(&meta), size: meta.len() as i64, open_for_write: ofw, }, ); scanned(rel); } } Ok(()) } /// Publish a conflict copy without overwriting an earlier conflict bearing the /// same timestamp. The original is removed only after the new copy is durable. fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result { let metadata = std::fs::metadata(path)?; for suffix in 0..=u32::MAX { let conflict = conflict_path_with_suffix(path, stamp, suffix); let mut source = std::fs::File::open(path)?; let mut target = match std::fs::OpenOptions::new() .write(true) .create_new(true) .open(&conflict) { Ok(file) => file, Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, Err(e) => return Err(e), }; if let Err(e) = std::io::copy(&mut source, &mut target).and_then(|_| target.sync_all()) { let _ = std::fs::remove_file(&conflict); return Err(e); } target.set_permissions(metadata.permissions())?; if let Ok(modified) = metadata.modified() { target.set_modified(modified)?; } target.sync_all()?; sync_parent_dir(&conflict)?; std::fs::remove_file(path)?; sync_parent_dir(&conflict)?; return Ok(conflict); } unreachable!("u32 conflict suffix space exhausted") } /// The remote uid of `rel`'s parent folder: from the pass's folder-uid map /// first, and from the baseline second for a parent the map never covered. /// /// Used to name the folder whose cached listing a remote mutation has just /// invalidated (B86). `None` is not an error — it only means this pass cannot /// say which folder to invalidate, and the caller skips it. fn parent_uid_of( rel: &str, remote_dirs: &HashMap, baseline: &HashMap, ) -> Option { let parent = parent_rel(rel); remote_dirs.get(parent).cloned().or_else(|| { baseline .get(parent) .and_then(|e| e.remote_uid.as_deref()) .and_then(parse_uid) }) } fn sync_parent_dir(path: &Path) -> std::io::Result<()> { match path.parent() { Some(parent) => std::fs::File::open(parent)?.sync_all(), None => Ok(()), } } // ---- helpers -------------------------------------------------------------- /// A remote node's `(mtime, size)` change signature. Size prefers the plaintext /// `claimed_size`; otherwise the encrypted storage size (stable either way, and /// only ever compared against the same node's own baseline). fn remote_file_sig(node: &Node) -> (i64, i64) { let size = match &node.kind { NodeKind::File { claimed_size, total_size_on_storage, .. } => claimed_size.unwrap_or(*total_size_on_storage), NodeKind::Folder => 0, }; (node.modification_time, size) } /// A file's modification time as epoch seconds (0 if unavailable). fn system_mtime(meta: &std::fs::Metadata) -> i64 { meta.modified() .ok() .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) .map(|d| d.as_secs() as i64) .unwrap_or(0) } /// The same modification time in nanoseconds, when the filesystem reports one. /// /// Whole seconds cannot distinguish an edit made in the same second as the last /// sync from no edit at all, and if the size did not change either, that file /// was never uploaded (bugs.md B25). `None` for a clock or filesystem that /// cannot answer, which falls the comparison back to seconds rather than /// asserting an mtime of zero. fn system_mtime_ns(meta: &std::fs::Metadata) -> Option { meta.modified() .ok() .and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok()) .and_then(|d| i64::try_from(d.as_nanos()).ok()) } #[cfg(test)] mod tests { use super::*; /// A removal of something already gone has produced the state the sync /// engine wanted, so its baseline is dropped rather than retried forever. /// Anything else stays an error and keeps the baseline (bugs.md B42). #[test] fn an_already_absent_path_counts_as_removed() { let root = std::env::temp_dir().join(format!("pdfs-b42-{}", std::process::id())); let missing = root.join("never-existed"); assert!(removed_locally(std::fs::remove_file(&missing)).is_ok()); assert!(removed_locally(std::fs::remove_dir_all(&missing)).is_ok()); // A directory in the way of `remove_file` is a real failure, and must // stay one: the file survives, and its baseline has to survive with it. let occupied = root.join("a-directory"); std::fs::create_dir_all(&occupied).unwrap(); assert!(removed_locally(std::fs::remove_file(&occupied)).is_err()); let _ = std::fs::remove_dir_all(&root); } /// The download guard: a destination that still matches the plan is /// replaced silently, one that has moved on since is kept (bugs.md B39). #[test] fn a_download_keeps_a_local_edit_it_did_not_plan_for() { let root = std::env::temp_dir().join(format!("pdfs-b39-{}", std::process::id())); std::fs::create_dir_all(&root).unwrap(); let path = root.join("report.txt"); // Nothing there, nothing expected: the download simply creates it. assert!(keep_racing_local_edit(&path, None).is_ok()); assert!(!path.exists()); // Unchanged since planning: no copy, the caller overwrites it. std::fs::write(&path, b"planned").unwrap(); let planned = local_sig(&path).unwrap(); keep_racing_local_edit(&path, Some(planned)).unwrap(); assert_eq!(std::fs::read(&path).unwrap(), b"planned"); // Changed since planning: the racing bytes are the only copy of // themselves, so they must survive the overwrite as a conflict copy. std::fs::write(&path, b"a much longer local edit").unwrap(); keep_racing_local_edit(&path, Some(planned)).unwrap(); let kept: Vec<_> = std::fs::read_dir(&root) .unwrap() .filter_map(|e| e.ok()) .map(|e| std::fs::read(e.path()).unwrap_or_default()) .collect(); assert!( kept.iter() .any(|bytes| bytes == b"a much longer local edit"), "the racing edit must still exist somewhere: {kept:?}" ); let _ = std::fs::remove_dir_all(&root); } /// Feed `n` `Reconcile` events `gap` apart on another thread, starting /// immediately. Returns the sender's join handle so the test can keep it /// alive (dropping the sender early would look like a disconnect). fn drip(n: usize, gap: Duration) -> (Receiver, std::thread::JoinHandle<()>) { let (tx, rx) = std::sync::mpsc::channel(); let h = std::thread::spawn(move || { for i in 0..n { if tx.send(SyncMsg::Reconcile(i as i64)).is_err() { return; } std::thread::sleep(gap); } }); (rx, h) } fn collect(rx: &Receiver, quiet: Duration, cap: Duration) -> (HashSet, Duration) { let mut ids = HashSet::new(); let (mut all, mut rewatch) = (false, false); // The caller of `settle` has always classified the opening event first. let first = rx.recv().unwrap(); classify(first, &mut ids, &mut all, &mut rewatch); let started = Instant::now(); settle_with(rx, &mut ids, &mut all, &mut rewatch, quiet, cap); (ids, started.elapsed()) } /// The bug this replaced a fixed sleep for: a save that keeps writing past /// the debounce used to be walked mid-write, uploading a torn snapshot as a /// real revision. The window has to restart on every event, so settling /// waits for the writer to actually stop. #[test] fn settling_waits_for_the_last_event_not_the_first() { // Eight events 20ms apart — a burst lasting ~160ms, far longer than the // 60ms quiet window a fixed sleep would have used. let (rx, h) = drip(8, Duration::from_millis(20)); let (ids, elapsed) = collect(&rx, Duration::from_millis(60), Duration::from_secs(30)); h.join().unwrap(); assert_eq!(ids.len(), 8, "every event in the burst is absorbed"); assert!( elapsed >= Duration::from_millis(120), "settling returned after {elapsed:?}: it stopped waiting while events \ were still arriving, which is the torn-revision bug" ); } /// A burst that ends is not waited on any longer than the quiet window. #[test] fn settling_returns_once_the_burst_stops() { let (rx, h) = drip(3, Duration::from_millis(5)); let (ids, elapsed) = collect(&rx, Duration::from_millis(50), Duration::from_secs(30)); h.join().unwrap(); assert_eq!(ids.len(), 3); assert!( elapsed < Duration::from_secs(5), "a finished burst must settle on the quiet window, not the cap" ); } /// Sustained change — copying a large tree in — must not postpone the /// reconcile for as long as the copy runs. The cap ends the wait. #[test] fn settling_is_capped_under_continuous_change() { // Events every 10ms for far longer than the 150ms cap allows. let (rx, h) = drip(200, Duration::from_millis(10)); let (_ids, elapsed) = collect(&rx, Duration::from_millis(100), Duration::from_millis(150)); assert!( elapsed < Duration::from_secs(2), "settled after {elapsed:?}; the cap should have ended the wait near 150ms" ); drop(rx); h.join().unwrap(); } fn rules_for(patterns: &[&str]) -> IgnoreRules { // No folder root is needed: these patterns are all root-relative globs, // and `load` only touches the filesystem to look for an ignore file. let globals: Vec = patterns.iter().map(|s| (*s).to_string()).collect(); IgnoreRules::load(Path::new("/nonexistent-sync-root"), &globals) } fn local_item(is_dir: bool) -> LocalItem { LocalItem { mtime_ns: None, is_dir, mtime: 1, size: 1, open_for_write: false, } } fn remote_item(is_dir: bool) -> RemoteItem { RemoteItem { uid: uid(), is_dir, mtime: 1, size: 1, } } /// The regression this whole feature has to not cause: a file that was /// synced *before* a rule started matching it is absent locally but present /// remotely and in the baseline — the exact shape `do_reconcile` answers by /// trashing the remote copy. It must never reach classification. #[test] fn a_newly_ignored_synced_file_is_never_classified_for_deletion() { let local: HashMap = HashMap::new(); let remote: HashMap = [("secrets/key.pem".to_string(), remote_item(false))] .into_iter() .collect(); let baseline = baseline_with("secrets/key.pem", 1, 1); let order = classification_order(&local, &remote, &baseline, &rules_for(&["secrets/"])); assert!( order.is_empty(), "an ignored path reached classification as {order:?}; with no local \ entry and a baseline row it would be trashed on Drive" ); } /// Filtering only the local walk is not enough, and this is why: the remote /// and baseline sides carry the path too. #[test] fn the_filter_covers_the_remote_and_baseline_sides_not_just_local() { let local: HashMap = [("target".to_string(), local_item(true))] .into_iter() .collect(); let remote: HashMap = [("target".to_string(), remote_item(true))] .into_iter() .collect(); let baseline = baseline_with("target", 1, 1); let order = classification_order(&local, &remote, &baseline, &rules_for(&["target/"])); assert!(order.is_empty(), "got {order:?}"); } #[test] fn unignored_paths_still_classify_shallowest_first() { let local: HashMap = [ ("a/b/c.txt".to_string(), local_item(false)), ("a".to_string(), local_item(true)), ("a/b".to_string(), local_item(true)), ("node_modules/x.js".to_string(), local_item(false)), ] .into_iter() .collect(); let remote: HashMap = HashMap::new(); let baseline: HashMap = HashMap::new(); let order = classification_order(&local, &remote, &baseline, &rules_for(&["node_modules/"])); assert_eq!( order, vec!["a", "a/b", "a/b/c.txt"], "a parent must be created before the paths inside it" ); } #[test] fn without_rules_every_path_is_classified() { let local: HashMap = [("node_modules".to_string(), local_item(true))] .into_iter() .collect(); let remote: HashMap = HashMap::new(); let baseline: HashMap = HashMap::new(); let order = classification_order(&local, &remote, &baseline, &IgnoreRules::empty()); assert_eq!(order, vec!["node_modules"]); } /// A rule covering the whole tree empties the local walk. Without the /// baseline being filtered to match, the wipe guard reads that as "every /// synced path vanished" and aborts the pass — every pass, forever. #[test] fn the_wipe_guard_ignores_paths_excluded_by_rule() { let mut baseline = baseline_with("build/a.o", 1, 1); baseline.extend(baseline_with("build/b.o", 1, 1)); let local: HashMap = HashMap::new(); let rules = rules_for(&["build/"]); assert!( guard_local_wipe(&baseline, &local).is_err(), "unfiltered, this is the wedge: the guard fires on a rule, not a loss" ); assert!( guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_ok(), "filtered, an all-ignored baseline leaves the guard nothing to check" ); } /// The guard must still catch a real disappearance when rules are active. #[test] fn the_wipe_guard_still_fires_on_a_real_loss_with_rules_active() { let mut baseline = baseline_with("docs/a.md", 1, 1); baseline.extend(baseline_with("docs/b.md", 1, 1)); let local: HashMap = HashMap::new(); let rules = rules_for(&["build/"]); assert!( guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_err(), "these paths are not ignored; their absence is a genuine wipe" ); } /// The guard that keeps a vanished local tree (dead mount, unplugged disk) /// from being read as "the user deleted everything" and trashing the folder /// on Drive. #[test] fn guard_local_wipe_blocks_only_a_total_disappearance() { let base: HashMap = ["a.txt", "b.txt", "d/c.txt"] .iter() .map(|r| ((*r).to_string(), ())) .collect(); let none: HashMap = HashMap::new(); let one: HashMap = [("b.txt".to_string(), ())].into_iter().collect(); assert!( guard_local_wipe(&base, &none).is_err(), "wholesale loss aborts" ); assert!( guard_local_wipe(&base, &one).is_ok(), "one survivor means the tree is readable; the rest are real deletions" ); // A one-entry folder can disappear because its mount or parent became // unavailable just as easily as a larger tree can. let single: HashMap = [("a.txt".to_string(), ())].into_iter().collect(); assert!(guard_local_wipe(&single, &none).is_err()); // Nothing synced yet: a first pass over an empty folder is not a wipe. assert!(guard_local_wipe(&none, &none).is_ok()); // Local paths that are not in the baseline are new uploads, and do not // count as survivors. let fresh: HashMap = [("new.txt".to_string(), ())].into_iter().collect(); assert!(guard_local_wipe(&base, &fresh).is_err()); } /// B54's remaining case, end to end: a folder whose local root is still /// *readable* but is no longer the tree it was — an empty directory left /// where a mount used to be, or a different device mounted over it. /// /// Nothing fails here. The scan succeeds and honestly reports an empty /// directory, so the only thing standing between the user and a trashed /// remote folder is the guard, and it has to fire for a one-file baseline /// exactly as it does for a large one. #[test] fn a_replaced_mountpoint_trips_the_guard_at_every_baseline_size() { let root = sync_test_dir("wipe-replaced-mount"); let rules = rules_for(&[]); let writing = HashSet::new(); let mut local: HashMap = HashMap::new(); let mut progress = no_progress(); walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap(); assert!(local.is_empty(), "the replacement root is genuinely empty"); for size in 1..=3 { let baseline: HashMap = (0..size).map(|i| (format!("file{i}.txt"), ())).collect(); assert!( guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_err(), "a {size}-path baseline against an empty root is not a deletion" ); } // And the guard steps aside the moment the real tree is back, so a // remounted folder resumes rather than staying wedged. std::fs::write(root.join("file0.txt"), b"back").unwrap(); let mut local: HashMap = HashMap::new(); walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap(); let baseline: HashMap = [("file0.txt".to_string(), ())].into_iter().collect(); assert!(guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_ok()); std::fs::remove_dir_all(&root).unwrap(); } /// A restored folder (features.md 5.2) starts with an empty baseline against /// an empty local directory and a full remote. That must reconcile as /// "download everything" rather than tripping the wipe guard — otherwise /// every restore would wedge its folder on the first pass. #[test] fn guard_local_wipe_is_inert_for_a_restored_folder() { let empty: HashMap = HashMap::new(); assert!(guard_local_wipe(&empty, &empty).is_ok()); } #[test] fn parent_and_base_split_relative_paths() { assert_eq!(parent_rel("a"), ""); assert_eq!(base_name("a"), "a"); assert_eq!(parent_rel("a/b"), "a"); assert_eq!(base_name("a/b"), "b"); assert_eq!(parent_rel("a/b/c.txt"), "a/b"); assert_eq!(base_name("a/b/c.txt"), "c.txt"); } #[test] fn rel_to_path_builds_nested_os_path() { assert_eq!(rel_to_path("a/b/c.txt"), PathBuf::from("a/b/c.txt")); assert_eq!(rel_to_path("file"), PathBuf::from("file")); } #[test] fn join_rel_joins_onto_a_prefix_and_leaves_the_root_bare() { assert_eq!(join_rel("", "a.txt"), "a.txt"); assert_eq!(join_rel("d", "a.txt"), "d/a.txt"); assert_eq!(join_rel("d/e", "a.txt"), "d/e/a.txt"); } /// One baseline holding `rel` with remote signature `(mtime, size)`. fn baseline_with(rel: &str, mtime: i64, size: i64) -> HashMap { HashMap::from([( rel.to_string(), StoredSyncEntry { rel_path: rel.to_string(), remote_uid: Some("v~l".into()), local_mtime: 0, local_size: 0, local_mtime_ns: None, remote_rev: Some(mtime.to_string()), remote_hash: Some(size.to_string()), }, )]) } #[test] fn an_unchanged_mtime_reuses_the_recorded_size() { // The whole point: this file's node key never gets unlocked. let base = baseline_with("a.txt", 1700, 4096); assert_eq!(unchanged_remote_size(&base, "a.txt", 1700), Some(4096)); } #[test] fn a_moved_mtime_forces_the_size_to_be_read() { let base = baseline_with("a.txt", 1700, 4096); assert_eq!(unchanged_remote_size(&base, "a.txt", 1701), None); } #[test] fn a_file_missing_from_the_baseline_forces_the_size_to_be_read() { // Nothing recorded to trust — a file the sync has never seen. let base = baseline_with("a.txt", 1700, 4096); assert_eq!(unchanged_remote_size(&base, "new.txt", 1700), None); } #[test] fn a_baseline_row_without_a_signature_forces_the_size_to_be_read() { // A folder row (no remote signature) must never be mistaken for an // unchanged file. let base = HashMap::from([( "d".to_string(), StoredSyncEntry { rel_path: "d".into(), remote_uid: Some("v~l".into()), local_mtime: 0, local_size: 0, local_mtime_ns: None, remote_rev: None, remote_hash: None, }, )]); assert_eq!(unchanged_remote_size(&base, "d", 0), None); } #[test] fn remote_sig_round_trips_through_baseline_strings() { let e = StoredSyncEntry { rel_path: "x".into(), remote_uid: Some("v~l".into()), local_mtime: 10, local_size: 20, local_mtime_ns: None, remote_rev: Some("1700".into()), remote_hash: Some("4096".into()), }; assert_eq!(remote_sig(&e), Some((1700, 4096))); } #[test] fn remote_sig_is_none_without_a_stored_signature() { let e = StoredSyncEntry { rel_path: "d".into(), remote_uid: Some("v~l".into()), local_mtime: 0, local_size: 0, local_mtime_ns: None, remote_rev: None, remote_hash: None, }; assert_eq!(remote_sig(&e), None); } #[test] fn outcome_summarises_only_what_it_moved() { let mut o = Outcome::default(); assert!(o.is_empty()); assert_eq!(o.summary(), ""); o.record(&Applied::Uploaded); o.record(&Applied::Uploaded); o.record(&Applied::Downloaded); o.errors += 1; assert!(!o.is_empty()); assert_eq!(o.summary(), "2 uploaded, 1 downloaded, 1 failed"); } #[test] fn pending_ops_describe_themselves_for_the_feed() { let op = Pending::UploadNew { rel: "docs/report.pdf".into(), parent: uid(), }; assert_eq!(op.rel(), "docs/report.pdf"); assert_eq!(base_name(op.rel()), "report.pdf"); assert_eq!(op.kind(), ActivityKind::Upload); let op = Pending::Download { rel: "photo.jpg".into(), uid: uid(), mtime: 0, size: 10, local: None, }; assert_eq!(op.kind(), ActivityKind::Download); } fn uid() -> NodeUid { use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId}; NodeUid::new(VolumeId::from("vol"), LinkId::from("link")) } #[test] fn conflict_path_keeps_extension_and_directory() { let p = conflict_path(Path::new("/home/me/docs/notes.txt"), 1700); assert_eq!( p, PathBuf::from("/home/me/docs/notes (sync-conflict 1700).txt") ); let no_ext = conflict_path(Path::new("/home/me/README"), 42); assert_eq!(no_ext, PathBuf::from("/home/me/README (sync-conflict 42)")); } #[test] fn conflict_preservation_never_replaces_an_existing_copy() { let dir = sync_test_dir("conflict-collision"); let source = dir.join("notes.txt"); let existing = conflict_path(&source, 1700); std::fs::write(&source, b"new local edit").unwrap(); std::fs::write(&existing, b"older conflict").unwrap(); let preserved = preserve_conflict_copy(&source, 1700).unwrap(); assert_eq!(std::fs::read(&existing).unwrap(), b"older conflict"); assert_eq!(std::fs::read(&preserved).unwrap(), b"new local edit"); assert_eq!( preserved.file_name().unwrap(), "notes (sync-conflict 1700-1).txt" ); assert!(!source.exists()); std::fs::remove_dir_all(dir).unwrap(); } #[test] fn conflict_preservation_failure_keeps_the_source() { let dir = sync_test_dir("conflict-failure"); // A directory cannot be copied through the regular-file preservation // path, deterministically forcing publication to fail after the source // has been discovered. let source = dir.join("not-a-file.txt"); std::fs::create_dir(&source).unwrap(); assert!(preserve_conflict_copy(&source, 1700).is_err()); assert!(source.is_dir(), "failure must not remove the source"); assert!(!conflict_path(&source, 1700).exists()); std::fs::remove_dir_all(dir).unwrap(); } /// B49's remaining cases. Preservation is the step that makes a download /// safe to publish, so every way it can fail has to leave the local file /// where it was — the download is aborted, not the file replaced. #[test] fn conflict_preservation_failures_all_keep_the_local_file() { use std::os::unix::fs::PermissionsExt as _; // An unwritable directory: the conflict copy cannot be created at all. let dir = sync_test_dir("conflict-eacces"); let source = dir.join("notes.txt"); std::fs::write(&source, b"the only copy").unwrap(); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o500)).unwrap(); let denied = preserve_conflict_copy(&source, 1700); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); assert!(denied.is_err(), "a read-only directory must fail the copy"); assert_eq!( std::fs::read(&source).unwrap(), b"the only copy", "the local file survives byte-identical" ); std::fs::remove_dir_all(&dir).unwrap(); // A source that vanished between planning and preservation. The pass // must report it rather than publish the download over the gap. let dir = sync_test_dir("conflict-enoent"); assert!(preserve_conflict_copy(&dir.join("gone.txt"), 1700).is_err()); std::fs::remove_dir_all(&dir).unwrap(); // A write error partway through the copy — induced by making the // destination name a directory, so `create_new` fails as something // other than `AlreadyExists`. let dir = sync_test_dir("conflict-eisdir"); let source = dir.join("notes.txt"); std::fs::write(&source, b"still the only copy").unwrap(); std::fs::create_dir(conflict_path(&source, 1700)).unwrap(); // Suffix 0 is taken by a directory; `create_new` on it reports // `AlreadyExists`, so preservation moves on to suffix 1 and succeeds. // What must not happen is the source being removed without a copy. let preserved = preserve_conflict_copy(&source, 1700).unwrap(); assert!(preserved.is_file()); assert_eq!(std::fs::read(&preserved).unwrap(), b"still the only copy"); assert!(!source.exists()); std::fs::remove_dir_all(&dir).unwrap(); } /// A conflict copy keeps the bytes *and* the metadata that identifies them, /// so the next pass uploads the user's version rather than a file that looks /// freshly written. #[test] fn a_conflict_copy_is_byte_and_mtime_identical() { use std::os::unix::fs::PermissionsExt as _; let dir = sync_test_dir("conflict-identity"); let source = dir.join("notes.txt"); std::fs::write(&source, b"local edit that must survive").unwrap(); let before = std::fs::metadata(&source).unwrap(); let preserved = preserve_conflict_copy(&source, 1700).unwrap(); let after = std::fs::metadata(&preserved).unwrap(); assert_eq!( std::fs::read(&preserved).unwrap(), b"local edit that must survive" ); assert_eq!(after.len(), before.len()); assert_eq!(after.modified().unwrap(), before.modified().unwrap()); assert_eq!(after.permissions().mode(), before.permissions().mode()); std::fs::remove_dir_all(&dir).unwrap(); } // ---- B55: an incomplete local scan is never a set of deletions --------- fn no_progress() -> impl FnMut(&str) { |_: &str| {} } /// A directory the pass cannot read must fail the pass. Omitting it would /// hand reconciliation a baseline-minus-subtree, which it reads as the user /// having deleted that subtree and propagates to Drive (B55). #[test] fn an_unreadable_subdirectory_fails_the_scan_instead_of_shrinking_it() { use std::os::unix::fs::PermissionsExt as _; let root = sync_test_dir("scan-eacces"); std::fs::write(root.join("keep.txt"), b"a").unwrap(); let locked = root.join("locked"); std::fs::create_dir(&locked).unwrap(); std::fs::write(locked.join("inner.txt"), b"b").unwrap(); std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap(); let rules = rules_for(&[]); let writing = HashSet::new(); let mut out = HashMap::new(); let mut progress = no_progress(); let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress); std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap(); std::fs::remove_dir_all(&root).unwrap(); let error = result.expect_err("an unreadable directory must abort the pass"); assert!( error.contains("locked"), "the error should name the subtree it could not read: {error}" ); } /// The same rule one level up: if the sync root itself cannot be read, the /// pass sees nothing, and "nothing" must never mean "delete everything". #[test] fn an_unreadable_root_fails_the_scan() { use std::os::unix::fs::PermissionsExt as _; let root = sync_test_dir("scan-root-eacces"); std::fs::write(root.join("keep.txt"), b"a").unwrap(); std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o000)).unwrap(); let rules = rules_for(&[]); let writing = HashSet::new(); let mut out = HashMap::new(); let mut progress = no_progress(); let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress); std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); std::fs::remove_dir_all(&root).unwrap(); assert!(result.is_err()); assert!(out.is_empty()); } /// A root that is gone entirely — an unmounted device folder, a removed /// directory — is the sharpest version of the same case: every baseline path /// is "missing", and the scan must refuse rather than report an empty tree. #[test] fn an_absent_root_fails_the_scan() { let root = sync_test_dir("scan-absent"); std::fs::remove_dir_all(&root).unwrap(); let rules = rules_for(&[]); let writing = HashSet::new(); let mut out = HashMap::new(); let mut progress = no_progress(); assert!(walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress).is_err()); assert!(out.is_empty()); } /// A name that is not valid UTF-8 cannot be keyed into the scan, so it /// cannot be compared against the baseline either. Refusing the pass is the /// only safe reading: skipping it would classify the file as deleted. #[test] fn a_non_utf8_name_fails_the_scan() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt as _; let root = sync_test_dir("scan-non-utf8"); std::fs::write(root.join(OsStr::from_bytes(b"bad\xffname")), b"x").unwrap(); let rules = rules_for(&[]); let writing = HashSet::new(); let mut out = HashMap::new(); let mut progress = no_progress(); let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress); std::fs::remove_dir_all(&root).unwrap(); let error = result.expect_err("a name the scan cannot key must abort the pass"); assert!(error.contains("UTF-8"), "{error}"); } /// The complement: a scan that *can* read everything reports everything, so /// the strictness above does not come at the cost of a working pass. A /// symlink is skipped by design rather than by failure, and a dangling one /// stats fine through `symlink_metadata`. #[test] fn a_readable_tree_scans_completely() { let root = sync_test_dir("scan-complete"); std::fs::write(root.join("a.txt"), b"aa").unwrap(); std::fs::create_dir(root.join("sub")).unwrap(); std::fs::write(root.join("sub/b.txt"), b"bbb").unwrap(); std::os::unix::fs::symlink("nowhere", root.join("dangling")).unwrap(); let rules = rules_for(&[]); let writing = HashSet::new(); let mut out = HashMap::new(); let mut seen: Vec = Vec::new(); let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut |rel| { seen.push(rel.to_string()) }); std::fs::remove_dir_all(&root).unwrap(); result.unwrap(); let mut keys: Vec<&str> = out.keys().map(String::as_str).collect(); keys.sort_unstable(); assert_eq!(keys, vec!["a.txt", "sub", "sub/b.txt"]); assert_eq!(out["a.txt"].size, 2); assert_eq!(out["sub/b.txt"].size, 3); assert!(out["sub"].is_dir); assert_eq!(seen.len(), 3, "every reported entry ticks scan progress"); } fn sync_test_dir(label: &str) -> PathBuf { let nonce = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .unwrap() .as_nanos(); let path = std::env::temp_dir().join(format!("pdfs-sync-{label}-{}-{nonce}", std::process::id())); std::fs::create_dir(&path).unwrap(); path } // ---- open-for-write detection tests ----------------------------------- #[test] fn is_write_mode_detects_wronly() { // O_WRONLY = 0o1 → flags octal "0100001" (O_WRONLY | O_LARGEFILE on x86_64) let fdinfo = "pos:\t0\nflags:\t0100001\nmnt_id:\t29\n"; assert!(is_write_mode(fdinfo), "O_WRONLY must be detected as write"); } #[test] fn is_write_mode_detects_rdwr() { // O_RDWR = 0o2 → flags octal "0100002" let fdinfo = "pos:\t0\nflags:\t0100002\nmnt_id:\t29\n"; assert!(is_write_mode(fdinfo), "O_RDWR must be detected as write"); } #[test] fn is_write_mode_ignores_rdonly() { // O_RDONLY = 0o0 → flags octal "0100000" (just O_LARGEFILE) let fdinfo = "pos:\t0\nflags:\t0100000\nmnt_id:\t29\n"; assert!( !is_write_mode(fdinfo), "O_RDONLY must not be detected as write" ); } #[test] fn is_write_mode_handles_bare_zero() { // A plain "0" (no O_LARGEFILE, no access flags). let fdinfo = "pos:\t0\nflags:\t0\nmnt_id:\t29\n"; assert!(!is_write_mode(fdinfo), "flags 0 is O_RDONLY"); } #[test] fn is_write_mode_returns_false_on_missing_flags_line() { let fdinfo = "pos:\t42\nmnt_id:\t29\n"; assert!(!is_write_mode(fdinfo)); } #[test] fn is_write_mode_handles_append() { // O_WRONLY | O_APPEND = 0o1 | 0o2000 = 0o2001 → with O_LARGEFILE: 0o102001 let fdinfo = "pos:\t0\nflags:\t0102001\nmnt_id:\t29\n"; assert!(is_write_mode(fdinfo), "O_WRONLY|O_APPEND is still a write"); } #[test] fn outcome_includes_deferred_in_summary() { let o = Outcome { uploaded: 3, deferred: 2, ..Default::default() }; assert!(!o.is_empty()); assert_eq!(o.summary(), "3 uploaded, 2 deferred (open for write)"); } #[test] fn outcome_deferred_alone_is_not_empty() { let mut o = Outcome::default(); assert!(o.is_empty()); o.deferred = 1; assert!(!o.is_empty()); } }