From 3ade754ea2df3050d06b6a155823a99044104ecb Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Mon, 20 Jul 2026 19:40:40 +0200 Subject: [PATCH] add .pdfsignore selective sync for mirror folders Mirror folders walked and uploaded every path under their root, so syncing a project directory pushed node_modules/, target/, and .git/ to Drive -- quota spent on files nobody wants there, and enough API traffic to invite rate limiting. Rules come from a .pdfsignore (or .protonignore) at the folder root, unioned with an ignore_patterns list in config.json that defaults to the usual build, VCS, and editor-leftover globs. Both are re-read at the start of every pass, so edits apply without restarting the daemon. The filter deliberately sits on the classification union in classification_order(), not on walk_local alone. do_reconcile classifies local | remote | baseline, so a file synced before a rule started matching it is absent from local but present in the other two -- the "local deleted, remote untouched" arm, which calls trash_nodes. Filtering only the walk would have meant that adding a line to .pdfsignore silently trashed files on Drive on the next pass. Ignored baseline rows are left in place so un-ignoring re-adopts the existing remote file instead of re-uploading it. guard_local_wipe takes a filtered baseline for the same reason: an ignored path is absent from the walk by rule rather than by loss, and counting those as missing would trip the guard on every pass once a rule covered the tree, wedging that folder's sync permanently. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01JFzNQyb4TWrCRWcnegqWxY --- README.md | 41 +++++ crates/pdfs-core/src/config.rs | 49 +++++ crates/pdfs-core/src/lib.rs | 1 + crates/pdfs-core/src/syncignore.rs | 275 +++++++++++++++++++++++++++ crates/pdfs-fuse/src/sync.rs | 287 +++++++++++++++++++++++++++-- 5 files changed, 637 insertions(+), 16 deletions(-) create mode 100644 crates/pdfs-core/src/syncignore.rs diff --git a/README.md b/README.md index ae96378..d28b5fb 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,47 @@ A fast, unofficial Proton Drive client for Linux. This client features an advanc - **Secure Credential Storage**: Integrates with the system Secret Service (GNOME Keyring, KWallet, etc.) with smart in-memory credential caching to avoid UI thread blockages. - **Proton Photos Support**: Access your Proton Photos timeline, view thumbnails, and download backed-up media natively (available in the GUI as a navigation tab and via the CLI). - **Human Verification (CAPTCHA) Recovery**: Detects sign-in gates (VPN/new IP challenges) and launches an embedded `WebKitWebView` dialog to safely complete the challenge, transparently retrying authentication with the earned token. +- **Selective Sync (`.pdfsignore`)**: Keep build trees, dependency directories, and editor leftovers out of synced folders using gitignore-style rules. + +## Selective Sync (`.pdfsignore`) + +Two-way synced folders skip paths matched by ignore rules, so syncing a project +directory does not upload `node_modules/`, `target/`, or `.git/`. + +Rules come from two places, and both apply: + +1. **Per folder** — a `.pdfsignore` file at the root of the synced folder + (`.protonignore` also works). Gitignore syntax, including negation: + + ```gitignore + # everything build-related + build/ + *.log + + # ...except this one + !important.log + ``` + +2. **Globally** — an `ignore_patterns` list in `config.json`, applied to every + synced folder. When unset, sensible defaults apply: `.git/`, `.hg/`, `.svn/`, + `node_modules/`, `target/`, `.venv/`, `__pycache__/`, `*~`, `*.swp`, `*.tmp`, + `.DS_Store`, and `Thumbs.db`. + + ```json + { + "ignore_patterns": ["node_modules/", "target/", "*.iso"] + } + ``` + + Set it to `[]` to opt out of the defaults entirely. + +Rules are re-read at the start of every sync pass, so edits take effect on the +next pass without restarting the daemon. + +**Ignoring is never destructive.** If a rule starts matching a file that was +already synced, its copy on Drive is left untouched — the file simply stops +being tracked. Removing the rule later picks the existing remote file back up +rather than re-uploading it. ## Performance & Caching diff --git a/crates/pdfs-core/src/config.rs b/crates/pdfs-core/src/config.rs index bafe216..c712970 100644 --- a/crates/pdfs-core/src/config.rs +++ b/crates/pdfs-core/src/config.rs @@ -6,6 +6,7 @@ use directories::ProjectDirs; use serde::{Deserialize, Serialize}; use crate::error::{Error, Result}; +use crate::syncignore::DEFAULT_IGNORE_PATTERNS; /// Proton requires `external-drive-{name}@{semver}-{channel}` (channel ∈ /// stable/beta/alpha); a malformed value trips the 422 anti-abuse path. @@ -34,6 +35,12 @@ pub struct AppConfig { /// path here. Defaulted for configs predating the field. #[serde(default)] pub mountpoint: Option, + /// Ignore patterns (gitignore syntax) applied to every mirror sync folder, + /// on top of each folder's own `.pdfsignore`. `None` means "use + /// [`DEFAULT_IGNORE_PATTERNS`]"; an explicit empty list disables the + /// defaults. Defaulted for configs predating the field. + #[serde(default)] + pub ignore_patterns: Option>, } impl Default for AppConfig { @@ -43,6 +50,7 @@ impl Default for AppConfig { user_agent: USER_AGENT.to_string(), cache_budget: None, mountpoint: None, + ignore_patterns: None, } } } @@ -53,6 +61,20 @@ impl AppConfig { pub fn resolved_cache_budget(&self) -> u64 { self.cache_budget.unwrap_or(DEFAULT_CACHE_BUDGET_BYTES) } + + /// The effective global sync ignore patterns: the user's explicit list, or + /// [`DEFAULT_IGNORE_PATTERNS`] when unset. An explicit empty list is + /// honoured as "no global patterns", which is why this is not a + /// `unwrap_or_default`. + pub fn resolved_ignore_patterns(&self) -> Vec { + match &self.ignore_patterns { + Some(patterns) => patterns.clone(), + None => DEFAULT_IGNORE_PATTERNS + .iter() + .map(|s| (*s).to_string()) + .collect(), + } + } } /// Resolved XDG locations for state, cache, and the default mountpoint. @@ -271,6 +293,7 @@ mod tests { user_agent: "test-agent/1.0".to_string(), cache_budget: Some(1234), mountpoint: Some("/tmp/x".to_string()), + ignore_patterns: Some(vec!["build/".to_string()]), }; let json = serde_json::to_string(&config).unwrap(); let decoded: AppConfig = serde_json::from_str(&json).unwrap(); @@ -284,4 +307,30 @@ mod tests { assert_eq!(default_config.app_version, APP_VERSION); assert_eq!(default_config.user_agent, USER_AGENT); } + + #[test] + fn a_config_predating_ignore_patterns_still_loads() { + // Every installed config on disk looks like this; the field must not be + // required or the daemon would fall back to a default config on upgrade. + let json = r#"{"app_version":"v","user_agent":"u"}"#; + let decoded: AppConfig = serde_json::from_str(json).unwrap(); + assert!(decoded.ignore_patterns.is_none()); + assert_eq!( + decoded.resolved_ignore_patterns().len(), + DEFAULT_IGNORE_PATTERNS.len(), + "an absent list means the defaults apply" + ); + } + + #[test] + fn an_explicit_empty_ignore_list_disables_the_defaults() { + let config = AppConfig { + ignore_patterns: Some(Vec::new()), + ..AppConfig::default() + }; + assert!( + config.resolved_ignore_patterns().is_empty(), + "opting out must be expressible, so this cannot be unwrap_or_default" + ); + } } diff --git a/crates/pdfs-core/src/lib.rs b/crates/pdfs-core/src/lib.rs index 27dc6f0..851378b 100644 --- a/crates/pdfs-core/src/lib.rs +++ b/crates/pdfs-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod error; pub mod localindex; pub mod service; pub mod shell; +pub mod syncignore; pub use error::{CoreError, CoreResult, Error, Result}; diff --git a/crates/pdfs-core/src/syncignore.rs b/crates/pdfs-core/src/syncignore.rs new file mode 100644 index 0000000..d2bc461 --- /dev/null +++ b/crates/pdfs-core/src/syncignore.rs @@ -0,0 +1,275 @@ +//! Selective-sync ignore rules for mirror folders (features.md 1.1). +//! +//! A mirror folder otherwise uploads every path beneath its root, which for a +//! project directory means `node_modules/`, `.git/`, and `target/` — quota +//! spent on files nobody wants on Drive, and enough API traffic to invite rate +//! limiting. +//! +//! Rules come from two places, unioned: +//! +//! * `.pdfsignore` (or `.protonignore`) at the root of the synced folder, +//! gitignore syntax. +//! * A global pattern list from the config, applied to every synced folder. +//! +//! Both are read fresh at the start of each reconcile pass, so editing the file +//! takes effect on the next pass without restarting the daemon. +//! +//! # Ignoring is not deleting +//! +//! A path that becomes ignored after it was already synced keeps its remote +//! copy and its baseline row. The sync engine drops ignored paths from +//! classification entirely rather than treating them as locally deleted — +//! see the union filter in `pdfs-fuse`'s `do_reconcile`. Editing a text file +//! must never trash data on Drive. + +use std::path::Path; + +use ignore::gitignore::{Gitignore, GitignoreBuilder}; + +/// Patterns applied to every synced folder when the config does not override +/// them. Version-control metadata, dependency and build trees, editor +/// leftovers, and OS turds — none of which belong on Drive, all of which are +/// large or rewritten constantly. +pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[ + ".git/", + ".hg/", + ".svn/", + "node_modules/", + "target/", + ".venv/", + "__pycache__/", + "*~", + "*.swp", + "*.tmp", + ".DS_Store", + "Thumbs.db", +]; + +/// Filenames accepted as a folder's ignore file, in precedence order. +const IGNORE_FILE_NAMES: &[&str] = &[".pdfsignore", ".protonignore"]; + +/// Compiled ignore rules for one synced folder. +pub struct IgnoreRules { + matcher: Option, +} + +impl IgnoreRules { + /// Build the rules for a folder rooted at `root`: `globals` first, then any + /// ignore file found at the root, so a folder-local rule can negate a + /// global one with `!pattern`. + /// + /// Malformed patterns are skipped individually rather than failing the + /// load — a typo in `.pdfsignore` must not take the folder's sync down. + pub fn load(root: &Path, globals: &[String]) -> Self { + let mut builder = GitignoreBuilder::new(root); + let mut any = false; + + for pattern in globals { + if builder.add_line(None, pattern).is_ok() { + any = true; + } + } + + for name in IGNORE_FILE_NAMES { + let path = root.join(name); + if !path.is_file() { + continue; + } + // `add` returns the first parse error but still takes the lines it + // could read, so a bad line costs that line and nothing more. + let _ = builder.add(&path); + any = true; + break; + } + + if !any { + return Self { matcher: None }; + } + Self { + matcher: builder.build().ok(), + } + } + + /// Rules matching nothing, for callers with no folder context. + pub fn empty() -> Self { + Self { matcher: None } + } + + /// Whether any rule is loaded. When false, [`is_ignored`](Self::is_ignored) + /// is always false and callers can skip filtering entirely. + pub fn is_empty(&self) -> bool { + self.matcher.is_none() + } + + /// Whether `rel` — a `/`-joined path relative to the folder root — is + /// ignored. + /// + /// A path under an ignored directory is ignored too. `Gitignore` matches + /// one path against one rule set and does not walk parents itself, so each + /// ancestor is tested as a directory. This is what makes filtering the + /// classification union sound: the engine can ask about any path in + /// isolation, in any order, without having walked its parents first. + pub fn is_ignored(&self, rel: &str, is_dir: bool) -> bool { + let Some(matcher) = &self.matcher else { + return false; + }; + if rel.is_empty() { + return false; + } + + let mut prefix = String::new(); + let mut parts = rel.split('/').peekable(); + while let Some(part) = parts.next() { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(part); + let last = parts.peek().is_none(); + // Every ancestor is a directory; only the final component's kind + // is the caller's to say. + let as_dir = if last { is_dir } else { true }; + if matcher.matched(&prefix, as_dir).is_ignore() { + return true; + } + } + false + } +} + +impl std::fmt::Debug for IgnoreRules { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IgnoreRules") + .field("active", &!self.is_empty()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A unique temp directory removed on drop; avoids a dev-dependency, as in + /// [`crate::cache`]'s tests. + struct TempDir(std::path::PathBuf); + impl TempDir { + fn new() -> Self { + use std::sync::atomic::{AtomicU64, Ordering}; + static N: AtomicU64 = AtomicU64::new(0); + let p = std::env::temp_dir().join(format!( + "pdfs-ignore-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } + fn path(&self) -> &Path { + &self.0 + } + } + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn globals() -> Vec { + DEFAULT_IGNORE_PATTERNS + .iter() + .map(|s| (*s).to_string()) + .collect() + } + + fn write(root: &Path, name: &str, body: &str) { + std::fs::write(root.join(name), body).unwrap(); + } + + #[test] + fn no_rules_at_all_ignores_nothing() { + let dir = TempDir::new(); + let rules = IgnoreRules::load(dir.path(), &[]); + assert!(rules.is_empty()); + assert!(!rules.is_ignored("anything", false)); + assert!(!rules.is_ignored("deep/nested/path.rs", false)); + } + + #[test] + fn global_defaults_match_without_an_ignore_file() { + let dir = TempDir::new(); + let rules = IgnoreRules::load(dir.path(), &globals()); + assert!(rules.is_ignored("node_modules", true)); + assert!(rules.is_ignored("target", true)); + assert!(rules.is_ignored("notes.txt~", false)); + assert!(!rules.is_ignored("src/main.rs", false)); + } + + #[test] + fn a_path_under_an_ignored_directory_is_ignored() { + let dir = TempDir::new(); + let rules = IgnoreRules::load(dir.path(), &globals()); + // The engine asks about deep paths in isolation, so each must answer + // correctly without its parents having been tested first. + assert!(rules.is_ignored("node_modules/left-pad/index.js", false)); + assert!(rules.is_ignored("node_modules/left-pad", true)); + assert!(rules.is_ignored("target/debug/build/x/out", false)); + } + + #[test] + fn a_folder_ignore_file_adds_to_the_globals() { + let dir = TempDir::new(); + write(dir.path(), ".pdfsignore", "secrets/\n*.log\n"); + let rules = IgnoreRules::load(dir.path(), &globals()); + assert!(rules.is_ignored("secrets/key.pem", false)); + assert!(rules.is_ignored("run.log", false)); + // Globals still apply alongside the file. + assert!(rules.is_ignored("node_modules", true)); + assert!(!rules.is_ignored("run.txt", false)); + } + + #[test] + fn a_folder_rule_can_negate_a_global_one() { + let dir = TempDir::new(); + write(dir.path(), ".pdfsignore", "!keep.tmp\n"); + let rules = IgnoreRules::load(dir.path(), &globals()); + assert!(rules.is_ignored("scratch.tmp", false)); + assert!(!rules.is_ignored("keep.tmp", false)); + } + + #[test] + fn protonignore_is_accepted_as_an_alias() { + let dir = TempDir::new(); + write(dir.path(), ".protonignore", "build/\n"); + let rules = IgnoreRules::load(dir.path(), &[]); + assert!(rules.is_ignored("build/out.o", false)); + } + + #[test] + fn pdfsignore_wins_when_both_files_exist() { + let dir = TempDir::new(); + write(dir.path(), ".pdfsignore", "from-pdfs/\n"); + write(dir.path(), ".protonignore", "from-proton/\n"); + let rules = IgnoreRules::load(dir.path(), &[]); + assert!(rules.is_ignored("from-pdfs/x", false)); + assert!(!rules.is_ignored("from-proton/x", false)); + } + + #[test] + fn a_malformed_pattern_does_not_take_the_rest_down() { + let dir = TempDir::new(); + // An unclosed character class is not a valid glob. + write(dir.path(), ".pdfsignore", "[unclosed\nvalid/\n"); + let rules = IgnoreRules::load(dir.path(), &globals()); + assert!(rules.is_ignored("valid/x", false)); + assert!(rules.is_ignored("node_modules", true)); + } + + #[test] + fn the_root_itself_is_never_ignored() { + let dir = TempDir::new(); + write(dir.path(), ".pdfsignore", "*\n"); + let rules = IgnoreRules::load(dir.path(), &[]); + // A blanket rule must not classify the folder root, or the pass would + // have nothing to reconcile against. + assert!(!rules.is_ignored("", true)); + } +} diff --git a/crates/pdfs-fuse/src/sync.rs b/crates/pdfs-fuse/src/sync.rs index 6ee20ce..95b7e57 100644 --- a/crates/pdfs-fuse/src/sync.rs +++ b/crates/pdfs-fuse/src/sync.rs @@ -29,6 +29,7 @@ use crate::transfers::{CountingWriter, OwnedCountingReader}; use crate::{Core, now_secs, parse_uid}; use pdfs_core::control::{ActivityKind, TransferDirection}; use pdfs_core::db::{StoredSyncEntry, StoredSyncFolder}; +use pdfs_core::syncignore::IgnoreRules; /// How long the watcher must go quiet before a burst counts as finished. /// @@ -448,6 +449,94 @@ impl From for PassAbort { /// /// One surviving path is enough to clear the guard, since the classification is /// only ever wrong about all of them at once. +/// The paths a reconcile pass will classify: the union of the local, remote, +/// and baseline states, minus anything the ignore rules exclude, shallowest +/// first so a parent folder is created before its children are placed in it. +/// +/// The ignore filter belongs *here*, on the union, rather than on the local walk +/// alone — filtering the walk alone would be actively destructive. A file synced +/// before it became ignored is absent from `local` but present in `remote` and +/// `baseline`, which is precisely the "local deleted, remote untouched" shape +/// `do_reconcile` responds to by trashing the remote copy. Adding a line to +/// `.pdfsignore` must never delete anything from Drive. +/// +/// Ignored baseline rows are deliberately left in the database rather than +/// removed: with the row intact, un-ignoring the path later re-adopts the +/// existing remote file instead of reading as a brand-new local one. +fn classification_order( + local: &HashMap, + remote: &HashMap, + baseline: &HashMap, + rules: &IgnoreRules, +) -> Vec +where + L: HasKind, + R: HasKind, +{ + let mut paths: HashSet = HashSet::new(); + paths.extend(local.keys().cloned()); + paths.extend(remote.keys().cloned()); + paths.extend(baseline.keys().cloned()); + let mut order: Vec = paths.into_iter().collect(); + if !rules.is_empty() { + order.retain(|rel| { + // A baseline-only path has no kind recorded, so it is tested as + // both; see `filter_baseline` for why erring towards "ignored" is + // the safe direction here. + let kind = local + .get(rel) + .map(HasKind::is_dir) + .or_else(|| remote.get(rel).map(HasKind::is_dir)); + match kind { + Some(is_dir) => !rules.is_ignored(rel, is_dir), + None => !rules.is_ignored(rel, false) && !rules.is_ignored(rel, true), + } + }); + } + order.sort_by_key(|p| p.matches('/').count()); + order +} + +/// Lets [`classification_order`] ask either walk's item whether it is a +/// directory without knowing which walk it came from. +trait HasKind { + fn is_dir(&self) -> bool; +} + +impl HasKind for LocalItem { + fn is_dir(&self) -> bool { + self.is_dir + } +} + +impl HasKind for RemoteItem { + fn is_dir(&self) -> bool { + self.is_dir + } +} + +/// The baseline minus its ignored paths, for [`guard_local_wipe`]. +/// +/// The guard asks "did every synced path vanish locally?", and an ignored path +/// is absent from the local walk by rule rather than by loss. Left in, a rule +/// covering the whole tree would trip the guard on every pass and wedge that +/// folder's sync for good. +/// +/// A baseline row does not record whether it was a file or a folder, so a path +/// counts as ignored if it matches as either. Erring towards "ignored" only ever +/// shrinks the set the guard checks, which weakens a safety net rather than +/// causing a deletion — the wrong direction to be wrong in is the other one. +fn filter_baseline<'a, B>( + baseline: &'a HashMap, + rules: &IgnoreRules, +) -> HashMap { + baseline + .iter() + .filter(|(rel, _)| !rules.is_ignored(rel, false) && !rules.is_ignored(rel, true)) + .map(|(rel, entry)| (rel.clone(), entry)) + .collect() +} + fn guard_local_wipe( baseline: &HashMap, local: &HashMap, @@ -529,11 +618,16 @@ impl Core { .db .sync_folder_set_state(folder.id, "syncing", folder.last_sync); let name = base_name(&folder.local_path); + // Rebuilt per pass rather than cached on `Core`, so editing `.pdfsignore` + // takes effect on the next reconcile instead of at the next daemon start. + // The cost is one small file read against a pass that is about to walk the + // whole tree. + let rules = self.ignore_rules(&local_root); self.progress_begin(folder.id); let result = if push_only { - self.push_pass(folder.id, &local_root, &remote_root) + self.push_pass(folder.id, &local_root, &remote_root, &rules) } else { - match self.do_reconcile(folder.id, &local_root, &remote_root) { + match self.do_reconcile(folder.id, &local_root, &remote_root, &rules) { Ok(outcome) => Ok(outcome), Err(PassAbort::Failed(e)) => Err(e), // The user asked for on-demand while this pass was running. Rather @@ -548,7 +642,7 @@ impl Core { ); push_only = true; self.progress_begin(folder.id); - self.push_pass(folder.id, &local_root, &remote_root) + self.push_pass(folder.id, &local_root, &remote_root, &rules) } } }; @@ -619,6 +713,27 @@ impl Core { } } + /// Compile the ignore rules for a synced folder: the global patterns from + /// the config, plus any `.pdfsignore` at `local_root`. + /// + /// A config that cannot be read falls back to the built-in defaults rather + /// than to "ignore nothing" — the defaults exclude build and VCS trees, and + /// silently uploading those because a config read failed is the outcome this + /// feature exists to prevent. + fn ignore_rules(&self, local_root: &Path) -> IgnoreRules { + let globals = match pdfs_core::config::AppDirs::new() { + Ok(dirs) => dirs.load_config().resolved_ignore_patterns(), + Err(e) => { + warn!(error = %e, "sync: cannot resolve config dir; using default ignore patterns"); + pdfs_core::syncignore::DEFAULT_IGNORE_PATTERNS + .iter() + .map(|s| (*s).to_string()) + .collect() + } + }; + IgnoreRules::load(local_root, &globals) + } + /// Whether a switch to on-demand is waiting on this folder — i.e. whether a /// pass in flight is still doing work worth doing. Read from the db rather than /// held in memory because the request arrives on the control-socket thread while @@ -658,6 +773,7 @@ impl Core { folder_id: i64, local_root: &Path, remote_root: &NodeUid, + rules: &IgnoreRules, ) -> Result { let baseline = self .db @@ -667,8 +783,12 @@ impl Core { self.progress_scan_total(folder_id, baseline.len()); let mut local: HashMap = HashMap::new(); - self.walk_local(folder_id, local_root, local_root, &mut local)?; - guard_local_wipe(&baseline, &local)?; + self.walk_local(folder_id, local_root, local_root, rules, &mut local)?; + // The guard compares against the paths this pass could still see: an + // ignored path is absent from `local` by rule, not by loss, and counting + // those as missing would trip the guard on every pass — wedging the + // folder's sync permanently — the moment a rule covers the whole tree. + guard_local_wipe(&filter_baseline(&baseline, rules), &local)?; // The remote folder uids come from the baseline instead of a walk: every // directory this device has synced recorded its uid there when it was @@ -909,6 +1029,7 @@ impl Core { folder_id: i64, local_root: &Path, remote_root: &NodeUid, + rules: &IgnoreRules, ) -> Result { // Loaded before the walks: the remote one uses it to tell which files are // unchanged and so can skip decrypting their claimed size, and its size is @@ -922,8 +1043,9 @@ impl Core { self.progress_scan_total(folder_id, baseline.len() * 2); let mut local: HashMap = HashMap::new(); - self.walk_local(folder_id, local_root, local_root, &mut local)?; - guard_local_wipe(&baseline, &local)?; + self.walk_local(folder_id, local_root, local_root, rules, &mut local)?; + // See `push_pass`: the guard must not count rule-excluded paths as lost. + guard_local_wipe(&filter_baseline(&baseline, rules), &local)?; let mut remote: HashMap = HashMap::new(); let mut remote_dirs: HashMap = HashMap::new(); @@ -937,14 +1059,7 @@ impl Core { &baseline, )?; - // Union of every path across the three states, shallow paths first so a - // parent folder is created before its children are placed inside it. - let mut paths: HashSet = HashSet::new(); - paths.extend(local.keys().cloned()); - paths.extend(remote.keys().cloned()); - paths.extend(baseline.keys().cloned()); - let mut order: Vec = paths.into_iter().collect(); - order.sort_by_key(|p| p.matches('/').count()); + let order = classification_order(&local, &remote, &baseline, rules); let mut outcome = Outcome::default(); // Folders to delete, collected here and removed deepest-first at the end @@ -1189,6 +1304,7 @@ impl Core { folder_id: i64, root: &Path, dir: &Path, + rules: &IgnoreRules, out: &mut HashMap, ) -> Result<(), String> { let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?; @@ -1208,6 +1324,13 @@ impl Core { if rel.contains(".pdfs-tmp-") { continue; } + // Ignored paths are dropped here as well as from the classification + // union — not for correctness (the union filter is what makes this + // safe) but so an ignored `node_modules/` is never descended into or + // counted against scan progress. + if rules.is_ignored(rel, meta.is_dir()) { + continue; + } if meta.is_dir() { out.insert( rel.to_string(), @@ -1218,7 +1341,7 @@ impl Core { }, ); self.progress_scanned(folder_id, base_name(rel)); - self.walk_local(folder_id, root, &path, out)?; + self.walk_local(folder_id, root, &path, rules, out)?; } else if meta.is_file() { out.insert( rel.to_string(), @@ -1895,6 +2018,138 @@ mod tests { h.join().unwrap(); } + fn rules_for(patterns: &[&str]) -> IgnoreRules { + // No folder root is needed: these patterns are all root-relative globs, + // and `load` only touches the filesystem to look for an ignore file. + let globals: Vec = patterns.iter().map(|s| (*s).to_string()).collect(); + IgnoreRules::load(Path::new("/nonexistent-sync-root"), &globals) + } + + fn local_item(is_dir: bool) -> LocalItem { + LocalItem { + is_dir, + mtime: 1, + size: 1, + } + } + + fn remote_item(is_dir: bool) -> RemoteItem { + RemoteItem { + uid: uid(), + is_dir, + mtime: 1, + size: 1, + } + } + + /// The regression this whole feature has to not cause: a file that was + /// synced *before* a rule started matching it is absent locally but present + /// remotely and in the baseline — the exact shape `do_reconcile` answers by + /// trashing the remote copy. It must never reach classification. + #[test] + fn a_newly_ignored_synced_file_is_never_classified_for_deletion() { + let local: HashMap = HashMap::new(); + let remote: HashMap = + [("secrets/key.pem".to_string(), remote_item(false))] + .into_iter() + .collect(); + let baseline = baseline_with("secrets/key.pem", 1, 1); + + let order = classification_order(&local, &remote, &baseline, &rules_for(&["secrets/"])); + + assert!( + order.is_empty(), + "an ignored path reached classification as {order:?}; with no local \ + entry and a baseline row it would be trashed on Drive" + ); + } + + /// Filtering only the local walk is not enough, and this is why: the remote + /// and baseline sides carry the path too. + #[test] + fn the_filter_covers_the_remote_and_baseline_sides_not_just_local() { + let local: HashMap = [("target".to_string(), local_item(true))] + .into_iter() + .collect(); + let remote: HashMap = [("target".to_string(), remote_item(true))] + .into_iter() + .collect(); + let baseline = baseline_with("target", 1, 1); + + let order = classification_order(&local, &remote, &baseline, &rules_for(&["target/"])); + + assert!(order.is_empty(), "got {order:?}"); + } + + #[test] + fn unignored_paths_still_classify_shallowest_first() { + let local: HashMap = [ + ("a/b/c.txt".to_string(), local_item(false)), + ("a".to_string(), local_item(true)), + ("a/b".to_string(), local_item(true)), + ("node_modules/x.js".to_string(), local_item(false)), + ] + .into_iter() + .collect(); + let remote: HashMap = HashMap::new(); + let baseline: HashMap = HashMap::new(); + + let order = classification_order(&local, &remote, &baseline, &rules_for(&["node_modules/"])); + + assert_eq!( + order, + vec!["a", "a/b", "a/b/c.txt"], + "a parent must be created before the paths inside it" + ); + } + + #[test] + fn without_rules_every_path_is_classified() { + let local: HashMap = [("node_modules".to_string(), local_item(true))] + .into_iter() + .collect(); + let remote: HashMap = HashMap::new(); + let baseline: HashMap = HashMap::new(); + + let order = classification_order(&local, &remote, &baseline, &IgnoreRules::empty()); + + assert_eq!(order, vec!["node_modules"]); + } + + /// A rule covering the whole tree empties the local walk. Without the + /// baseline being filtered to match, the wipe guard reads that as "every + /// synced path vanished" and aborts the pass — every pass, forever. + #[test] + fn the_wipe_guard_ignores_paths_excluded_by_rule() { + let mut baseline = baseline_with("build/a.o", 1, 1); + baseline.extend(baseline_with("build/b.o", 1, 1)); + let local: HashMap = HashMap::new(); + let rules = rules_for(&["build/"]); + + assert!( + guard_local_wipe(&baseline, &local).is_err(), + "unfiltered, this is the wedge: the guard fires on a rule, not a loss" + ); + assert!( + guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_ok(), + "filtered, an all-ignored baseline leaves the guard nothing to check" + ); + } + + /// The guard must still catch a real disappearance when rules are active. + #[test] + fn the_wipe_guard_still_fires_on_a_real_loss_with_rules_active() { + let mut baseline = baseline_with("docs/a.md", 1, 1); + baseline.extend(baseline_with("docs/b.md", 1, 1)); + let local: HashMap = HashMap::new(); + let rules = rules_for(&["build/"]); + + assert!( + guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_err(), + "these paths are not ignored; their absence is a genuine wipe" + ); + } + /// The guard that keeps a vanished local tree (dead mount, unplugged disk) /// from being read as "the user deleted everything" and trashing the folder /// on Drive.