//! On-demand FUSE filesystem for Proton Drive. //! //! Phase 1 is a read-only placeholder mount: directories are enumerated lazily //! from the remote on first access and file content is hydrated on `read` via //! [`ProtonDriveClient::download_range`]. Nothing is mirrored to local disk; //! every byte is fetched on demand. //! //! Phase 2 adds live remote sync: a background task polls the volume event //! cursor ([`ProtonDriveClient::enumerate_events`]) and pushes invalidations //! into the kernel via a fuser [`Notifier`], so the cache TTL can be long while //! remote changes still show up promptly. //! //! Phase 3 makes the mount writable. Each file opened for writing gets a //! `WriteHandle` whose buffer accumulates the full plaintext; on flush/release //! the buffer is sealed as a new revision via //! [`ProtonDriveClient::upload_new_revision`] (the SDK uploads whole revisions, //! not byte ranges). New files are created empty up front so they get a real //! uid; namespace ops map to `create_folder`, `trash_nodes`, `rename_node` and //! `move_node`. //! //! Phase 4 adds Files-On-Demand pinning. A control socket (see `control`) //! lets the CLI pin/unpin files; a pinned file's plaintext is downloaded once //! into the on-disk [`ContentCache`] and every later `read` is served from disk //! instead of the network. Writes and remote events evict the cache so it never //! goes stale. //! //! Reads of unpinned files no longer hit the network per call: `Core::read_range` //! fetches and caches [`BLOCK_SIZE`]-aligned blocks, so sequential/sparse reads //! reuse the on-disk block cache. Writes are disk-backed: each `WriteHandle` //! stages authored bytes in a scratch file and tracks them with an `Intervals` //! set, so a multi-GiB write never buffers in RAM and only the untouched //! remainder of the file is pulled from the remote — lazily, at commit. use parking_lot::{Condvar, Mutex}; use std::collections::{HashMap, HashSet, VecDeque}; use std::ffi::OsStr; use std::fs::File; use std::io::Write; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::FileExt; use std::os::unix::net::UnixListener; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use fuser::ReplyXattr; use fuser::{ AccessFlags, BackgroundSession, BsdFileFlags, Config, CopyFileRangeFlags, Errno, FileAttr, FileHandle, FileType, Filesystem, FopenFlags, Generation, INodeNo, InitFlags, IoctlFlags, KernelConfig, LockOwner, MountOption, Notifier, OpenAccMode, OpenFlags, RenameFlags, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyIoctl, ReplyLseek, ReplyOpen, ReplyStatfs, ReplyWrite, Request, Session, TimeOrNow, WriteFlags, }; use futures::StreamExt as _; use pdfs_core::batch; use pdfs_core::cache::{BLOCK_SIZE, Baseline, BlockGeometry, BlockSpan, ContentCache, StagedWrite}; use pdfs_core::config::{AppDirs, SweepMode}; use pdfs_core::control::{ ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo, SearchFilters, SearchHit, SearchSource, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, }; use pdfs_core::db::{ Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_UNTIL, PendingOp, PublishedSharedRoot, RenameMeta, StoredNode, StoredSyncFolder, StoredTrash, }; use pdfs_core::localindex; use pdfs_core::search::relevance_score; use pdfs_core::syncignore::is_transient_name; use pdfs_core::{Access, CoreError, CoreResult, access_for, perm_bits}; use proton_drive_rs::proton_sdk::api::ResponseCode; use proton_drive_rs::proton_sdk::error::ProtonError; use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, ShareId, VolumeId}; use proton_drive_rs::{ DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, NodeMoveItem, ProtonDriveClient, ProtonPhotosClient, SharedWithMeItem, ThumbnailType, }; mod albums; mod background; mod conflicts; mod control; use control::run_control_socket; mod devices; mod diagnostics; mod drain; mod drive; use drive::{DriveApi, RevisionRead}; mod events; mod filesystem; mod link; pub use filesystem::ProtonFs; mod mount; mod pause; mod photos; mod profile; mod queue; mod reads; mod redate; mod relocate; mod revisions; mod sharing; mod shutdown; #[cfg(test)] mod sim; mod state; mod supervisor; mod sweep; mod sync; mod systemd; mod takeout; mod transfers; mod upload; mod r#virtual; mod workers; use background::{forget_and_notify, run_event_sync, run_local_index, run_photos_event_sync}; pub(crate) use mount::is_stale_mount; pub use mount::{MountOptions, MountOutcome, mount}; use mount::{ SecondaryInsertRejection, SecondaryMount, clear_stale_mount, fuse_connection_id, spawn_session, }; use reads::{BlockFlight, BlockRing, PREFETCH_BUDGET, Prefetch, ReaderSlot, STREAM_BYPASS_MIN}; use state::{Entry, Intervals, PendingRevision, State, StateGuard, WriteHandle}; use tracing::{debug, error, info, warn}; use transfers::{CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry}; use r#virtual::{ SharedListingPlan, SharedRefreshDeadlines, disambiguate_shared_names, is_own_or_virtual_uid, is_primary_root_listing, is_virtual_uid, listing_needs_refresh, refresh_generation_is_current, shared_listing_plan, shared_with_me_uid, virtual_node, virtual_root_name, }; use workers::{FUSE_WORKERS, Lane, Workers}; /// Attribute/entry cache lifetime handed back to the kernel. Long because the /// Phase 2 event poller actively invalidates changed inodes; without a remote /// change this is how long the kernel may serve stale metadata. const TTL: Duration = Duration::from_secs(30); /// How long a `statfs(2)` answer is served from [`Core::quota`] before the /// account figures are refetched. See [`Core::account_quota_cached`]. const QUOTA_TTL: Duration = Duration::from_secs(60); /// Block size reported to the kernel and to `statfs(2)`. /// /// coreutils and most copy tools size their I/O buffers from `st_blksize`, and /// the fuser default of 512 makes them issue reads two orders of magnitude /// smaller than the [`BLOCK_SIZE`] this filesystem actually fetches in. 1 MiB is /// the largest value that stays well inside the kernel's per-request limit. const REPORTED_BLKSIZE: u32 = 1 << 20; /// Ceiling on in-flight background requests (readahead, writeback). The kernel /// default of 16 caps how much of a sequential read the kernel will run ahead of /// the application, which on a high-latency link is the whole ballgame. const MAX_BACKGROUND: u16 = 64; /// Longest single path component, as reported by `statfs(2)`. The kernel's own /// dirent limit; Proton's is not lower. const MAX_NAME_LEN: u32 = 255; /// How often the background task polls the remote event cursor. const POLL_INTERVAL: Duration = Duration::from_secs(10); /// Retry backoff for a queued upload, doubling per attempt between these. The /// floor is short because the common failure is a brief network blip; the /// ceiling keeps a persistently failing op from spinning. const DRAIN_BACKOFF_MIN: Duration = Duration::from_secs(2); const DRAIN_BACKOFF_MAX: Duration = Duration::from_secs(300); /// How long the drain worker sleeps when it has nothing due. It is woken /// directly on a new write or a reconnect, so this only bounds how late a /// backoff can fire. const DRAIN_IDLE_POLL: Duration = Duration::from_secs(30); /// How long the mount waits before asking again for a node it has just created /// when Drive does not list it yet (`Core::fetch_minted_node`). const MINTED_READ_BACK_DELAYS: [Duration; 2] = [Duration::from_millis(250), Duration::from_secs(1)]; /// Grace period before a queued revision becomes eligible for draining. /// /// Tools like aria2c preallocate a file (truncate to target size) and then write /// the real content, sometimes across separate open/close cycles. Without a /// grace period the first close drains immediately, uploading the preallocated /// (mostly-zero) content; the second close then finds its baseline stale and /// creates a conflict copy of itself. Holding the op for a short window gives /// the follow-up write time to supersede it. const DRAIN_REVISION_DEBOUNCE: Duration = Duration::from_secs(2); /// Ceiling on the *adaptive* part of that grace period. /// /// The fixed 2 seconds is enough for a preallocate-then-write tool, and wrong /// for a file that takes half a minute to upload: an editor saving every ten /// seconds queues a write, watches it start uploading, and supersedes it — each /// save uploading bytes that the next one replaces, and never catching up. /// Widening the debounce toward how long this node's last upload actually took /// makes the supersede happen in the queue, where it is free, instead of on the /// wire. Bounded so a file on a slow link still reaches Drive in a minute rather /// than deferring indefinitely, and so a one-off stall cannot park a node. const DRAIN_REVISION_DEBOUNCE_MAX: Duration = Duration::from_secs(60); /// How many nodes' measured upload times to remember for that. Bounds a map /// that would otherwise grow with every file ever written; the debounce falls /// back to [`DRAIN_REVISION_DEBOUNCE`] for anything evicted, which is the /// behaviour a node that has never been uploaded gets anyway. const UPLOAD_TIME_MEMORY: usize = 512; /// Threads draining the pending-op queue. /// /// One meant a single 10 GiB upload blocked every queued rename, trash and /// small write behind it. Ordering only has to hold per node — enforced by the /// claim query, not by the thread count — so the rest is throughput. Most ops /// spend their time waiting: a create waits for Drive to list the node it made /// before it reads it back. Three workers left a thousand small creates taking /// 0.3 s each on a LAN and 2.5 s on Wi-Fi, one after another. Uploads cannot /// crowd each other out by number: the SDK caps the blocks in flight across /// the client. const DRAIN_WORKERS: usize = 16; /// How long a staged blob that names no node is kept before it is retired. /// /// Staging is exempt from the cache budget because it holds the only copy of /// writes that have not been uploaded, so nothing there is ever evicted to /// reclaim space. That is right for a blob that still describes something. It is /// wrong for one whose sidecar is missing or names a `local~` placeholder whose /// create is gone: no future run can address it, no UI accounts for it, and it /// occupies disk for the life of the machine. Real instance: 95 such blobs, /// ~445 MiB, the oldest two months old, re-announced on every mount. /// /// A month is far longer than any transfer or any plausible "I will look at that /// later", and the retirement is announced per blob with its size, so the one /// case that matters — bytes a human still wants — has a month of warnings and a /// path to go and fetch them from. const STAGING_ORPHAN_RETAIN: Duration = Duration::from_secs(30 * 24 * 60 * 60); /// First and longest delay between probes for the network coming back after an /// offline mount (offline.md Phase 1). Doubles from min to max: a laptop shut in /// a bag is the common case, so the steady state must be cheap, while a brief /// blip should still recover in seconds. const ONLINE_PROBE_MIN: Duration = Duration::from_secs(5); const ONLINE_PROBE_MAX: Duration = Duration::from_secs(300); /// How long the persisted photos timeline stays good before a page request /// revalidates it. The SDK hands back the whole timeline at once, so it is stored /// in the DB and every page is sliced from there; a stale one is still served /// immediately and refreshed in the background. /// /// Short, because the cost of a stale timeline is a photo the user deleted on /// their phone still sitting in the gallery, and the cost of the check is one /// listing call that only runs when somebody is actually looking at the page. const TIMELINE_TTL: Duration = Duration::from_secs(60); /// How many photo nodes are resolved per [`ProtonPhotosClient::enumerate_nodes`] /// call when enriching a refreshed timeline with names and media types (for the /// Photos / Videos / Raw split). Batched so a large library is a handful of /// round-trips rather than one request per photo, and bounded so a single call /// never asks the server to decrypt the whole library at once. /// /// 150 because that is the SDK's own `MAX_BATCH_COUNT`: a larger chunk is split /// there anyway, into one full request and a ragged tail. const TIMELINE_ENRICH_CHUNK: usize = 150; /// The same, for the persisted trash listing. Shorter, because the trash is the /// one listing a user changes and then immediately looks at — though our own /// mutations also invalidate it outright, so this only covers other clients. const TRASH_TTL: Duration = Duration::from_secs(60); /// Shared roots and foreign-volume folders have no event manager. Revalidate /// them on access while online; stale persisted listings remain usable offline. const SHARED_LISTING_TTL: Duration = Duration::from_secs(60); /// How many trashed nodes are materialized per /// [`ProtonDriveClient::enumerate_nodes`] call, and persisted as a batch. The /// trash of an account that has been through a conflict storm runs to thousands /// of nodes, each costing an S2K unlock, so a single all-or-nothing call can /// take minutes and lose everything it decrypted if it fails. Same value as the /// SDK's own `MAX_BATCH_COUNT`: one chunk is exactly one request. const TRASH_MATERIALIZE_CHUNK: usize = 150; /// How long a `ListTrash` request waits for a first-ever refresh before /// answering with whatever has materialized so far. /// /// The refresh keeps running in the background either way — this bounds the /// *reply*, not the work. It has to stay well under the front-ends' 120 s /// control-socket read timeout (`pdfs_core::control`), because a request that /// outlives that timeout is a hang from the user's side: the client gives up, /// the user asks again, and the daemon accumulates another refresh. const TRASH_FIRST_WAIT: Duration = Duration::from_secs(20); /// How long one server call of a trash refresh may take before the refresh /// gives up. /// /// Refreshes are single-flight, so a call that never returns wedges the trash /// listing for the life of the daemon: every later request joins the stuck /// refresh, times out, and answers with the listing from before it. That hid /// a just-trashed folder from a permanent delete for as long as the daemon ran /// (`docs/BUGS.md` B103). One call is one request, so this is generous. const TRASH_CALL_TIMEOUT: Duration = Duration::from_secs(120); /// `sync_state` keys for the freshness stamps of the two persisted listings, and /// for whether the account has a photos volume at all (so an account without one /// doesn't re-ask the server on every page request). const PHOTOS_SYNCED_MS: &str = "photos_synced_ms"; const PHOTOS_AVAILABLE: &str = "photos_available"; /// Freshness stamp of the album listing, and of one album's contents (suffixed /// with the album uid). Separate from the timeline's: albums are enumerated by /// their own endpoints, and an album is only fetched when it is opened. const ALBUMS_SYNCED_MS: &str = "albums_synced_ms"; const ALBUM_SYNCED_PREFIX: &str = "album_synced_ms:"; const TRASH_SYNCED_MS: &str = "trash_synced_ms"; /// Set when a trash refresh finishes and cleared when one starts, so it is only /// present while the stored listing is the whole trash as of some moment. A node /// that a refresh finds and that listing lacks went to the trash since then; one /// found after an interrupted refresh may have been there all along. Unlike /// [`TRASH_SYNCED_MS`], our own mutations leave it alone. const TRASH_COMPLETE_MS: &str = "trash_complete_ms"; const SHARED_WITH_ME_NAME: &str = "shared_with_me_name"; const SHARED_WITH_ME_SYNCED_MS: &str = "shared_with_me_synced_ms"; const SHARED_FOLDER_SYNCED_PREFIX: &str = "shared_folder_synced_ms:"; /// How stale the local-file index may get before the background scanner rebuilds /// it. A rescan is a full walk of `$HOME`, so this trades index freshness against /// disk churn; the scanner also always rebuilds once per daemon start when the /// index is older than this. const LOCAL_INDEX_TTL: Duration = Duration::from_secs(15 * 60); /// How often the scanner thread wakes to check whether the index went stale. const LOCAL_INDEX_CHECK: Duration = Duration::from_secs(60); /// The FUSE root inode is always 1. const ROOT_INO: u64 = 1; /// Parent inode for a persisted node whose parent row is missing from the DB. /// No folder carries this inode, so such a node is listed by nobody and stays /// inert until a live enumeration re-parents it. const ORPHAN_INO: u64 = 0; /// Extended attribute exposing the small server-side thumbnail of a file. const XATTR_THUMBNAIL: &str = "user.proton.thumbnail"; /// Extended attribute exposing the larger server-side preview image of a file. const XATTR_PREVIEW: &str = "user.proton.preview"; /// How many "this file has no thumbnail" answers [`Core::thumbnail`] remembers /// before dropping the lot and re-learning them. Sized to cover a large browsing /// session; each entry is a uid, a type tag and an mtime. const MAX_THUMBNAIL_MISSES: usize = 8192; /// How many nodes one size-upgrade request covers. /// /// Matches the SDK's own `MAX_BATCH_COUNT`, so a chunk is exactly one request: /// chunking smaller would add round trips, larger would be split anyway and /// delay the waiters this chunking exists to release (bugs.md B14). const SIZE_UPGRADE_CHUNK: usize = 150; /// How many folders may have a size upgrade in flight at once. /// /// Each one owns a thread and an API request stream, and the single-flight only /// deduplicates *within* a folder — so without this a recursive listing scaled /// both with the number of folders it walked. Eight keeps an interactive `ls -l` /// (one folder, sometimes a couple) entirely unaffected while putting a ceiling /// on the recursive case. A folder past the cap is queued, not dropped: one of /// the running threads takes it over when its own batch ends (bugs.md B100). const MAX_SIZE_UPGRADES: usize = 8; #[derive(Clone, Debug, Eq, PartialEq)] struct RootListingSnapshot { children: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] struct RootListingChild { ino: u64, entry: Option, } #[derive(Clone, Debug, Eq, PartialEq)] struct RootListingEntry { uid: NodeUid, name: String, parent: u64, trashed: bool, unlinked: bool, } impl RootListingSnapshot { fn capture(state: &State, parent_ino: u64) -> Option { let children = state.children.get(&parent_ino)?; Some(Self { children: children .iter() .map(|ino| RootListingChild { ino: *ino, entry: state.entries.get(ino).and_then(|entry| { (!is_virtual_uid(&entry.uid)).then(|| RootListingEntry { uid: entry.uid.clone(), name: entry.node.name.clone(), parent: entry.parent, trashed: entry.node.trashed, unlinked: entry.unlinked, }) }), }) .collect(), }) } fn real_names(&self) -> HashSet { self.children .iter() .filter_map(|child| child.entry.as_ref()) .map(|entry| entry.name.clone()) .collect() } fn is_current(&self, state: &State, parent_ino: u64) -> bool { Self::capture(state, parent_ino).as_ref() == Some(self) } } struct VirtualRootPlan { node: Node, visible: bool, } /// The sync folders a search hit can be opened through, as /// (Drive-relative root path, local directory) pairs. /// /// Built once per search by [`Core::search_roots`]; resolving a hit against it /// is string arithmetic over the paths the index already carries, with no /// further queries. struct SearchRoots { roots: Vec<(String, PathBuf)>, } impl SearchRoots { /// The absolute local path for a hit at Drive-relative `path`, through the /// most specific root covering it. A node can sit below several configured /// roots, so the one leaving the shortest descendant path wins. fn resolve(&self, path: &str) -> Option { self.roots .iter() .filter_map(|(root, local)| { let relative = relative_to(root, path)?; Some((relative.split('/').count(), local.join(relative))) }) .min_by_key(|(depth, _)| *depth) .map(|(_, path)| path.to_string_lossy().into_owned()) } } /// Negative thumbnail knowledge has two different authorities. A remote miss /// only says Proton has no thumbnail to download; it must never suppress local /// generation. A local miss says this exact file revision was successfully /// inspected and could not be decoded. Both maps are bounded independently. #[derive(Default)] struct ThumbnailMissCaches { remote: HashMap<(NodeUid, i32), i64>, local: HashMap<(NodeUid, i32), i64>, } impl ThumbnailMissCaches { fn remote_contains(&self, key: &(NodeUid, i32), tag: i64) -> bool { self.remote.get(key) == Some(&tag) } fn local_contains(&self, key: &(NodeUid, i32), tag: i64) -> bool { self.local.get(key) == Some(&tag) } fn remember_remote(&mut self, key: (NodeUid, i32), tag: i64) { Self::remember(&mut self.remote, key, tag); } fn remember_local(&mut self, key: (NodeUid, i32), tag: i64) { Self::remember(&mut self.local, key, tag); } fn forget_local(&mut self, key: &(NodeUid, i32)) { self.local.remove(key); } fn remember(map: &mut HashMap<(NodeUid, i32), i64>, key: (NodeUid, i32), tag: i64) { if map.len() >= MAX_THUMBNAIL_MISSES && !map.contains_key(&key) { map.clear(); } map.insert(key, tag); } } /// `path` expressed relative to `root`, or `None` if it is not below it. The /// root itself resolves to the empty path; an empty root is the mount itself /// and covers everything. fn relative_to<'a>(root: &str, path: &'a str) -> Option<&'a str> { if root.is_empty() { return Some(path); } if path == root { return Some(""); } path.strip_prefix(root)?.strip_prefix('/') } #[derive(Clone)] struct Core { /// Drive as the sync core uses it: listings, events, creates, uploads, /// downloads, renames, moves and trashes. The real client in production, an /// in-memory Drive in the simulation tests. See [`drive`]. drive: Arc, /// The concrete client, for what [`DriveApi`] does not cover: sharing, /// Photos, devices, invitations and public links. client: ProtonDriveClient, rt: tokio::runtime::Handle, /// The daemon's My Files root. Forked on-demand sessions retain this value /// so ownership checks never mistake their own mount root for account scope. primary_root_uid: NodeUid, /// False only for a forked on-demand device mount. The synthetic shared /// directory belongs exclusively to the primary My Files mount. primary: bool, /// The one tree every mount of the daemon serves from. state: Arc>, /// The tree inode this mount's session is rooted at, set when the session /// is built ([`ProtonFs::new`]). Each fork has its own cell. root: Arc>, cache: Arc, /// Open [`RevisionReader`]s keyed by node, so the block fetches of a file /// resolve its keys and block table once instead of once per block. /// Validated by `(mtime, size)` exactly like the content cache, and bounded /// by [`MAX_OPEN_READERS`]. readers: Arc>>, /// Decrypted blocks held in memory, so the ~32 kernel reads a 4 MiB block is /// delivered in cost the block once — a download for a streaming file, a /// disk read for a cached one. See [`BlockRing`]. block_ring: Arc>, /// Block fetches in flight, so concurrent readers of one block (demand read, /// kernel read-ahead, our own prefetch) share a single download and decrypt. /// See [`BlockFlight`]. block_flight: Arc, /// One lock per file being repaired by [`Core::repair_block`], so the /// concurrent block reads that all discover the same untrustworthy block /// table share a single whole-file download instead of one each. content_repairs: Arc>>>>, /// Per-file sequential-read detection driving [`Core::prefetch`]. prefetch: Arc, /// Permits bounding speculative block fetches, so prefetch cannot queue in /// front of a read someone is waiting on. prefetch_budget: Arc, /// Threads that serve the FUSE handlers which touch the network, so the /// session's dispatch loop stays free to answer cheap metadata calls while a /// cold read is on the wire. See [`Workers`]. workers: Arc, /// Unified SQLite metadata cache: the persistence layer behind the in-memory /// `State` maps. Every mutation writes through here, and the maps rehydrate /// from it on mount (plan.md P1). db: Arc, /// Serializes only shared-list publication and access-loss invalidation. /// Network calls happen before taking it. shared_publication: Arc>, /// Invalidates in-flight shared responses when access/list events arrive. shared_generation: Arc, /// Successful refreshes suppress retries even if persisting their timestamp /// fails. Shared across primary/fork clones and cleared by access events. shared_refresh_deadlines: Arc>, /// False while the API is unreachable and we are serving the cached tree /// (offline.md Phase 1). Set by the probe thread; read by front-ends through /// `Response::Status` so the UI can say so rather than leaving the user to /// infer it from a wall of EIO. online: Arc, /// When the link went down and a condvar for its coming back; the other /// half of `online` (see [`link`]). link: Arc, /// When the user paused syncing, the Unix second it resumes by itself; /// [`pause::PAUSED_INDEFINITELY`] for "until resumed", `0` when not paused. /// Persisted, so a pause survives a restart (see [`Core::sync_paused`]). sync_paused_until: Arc, /// Writes accepted from the kernel but not yet uploaded, keyed by node /// (offline.md Phase 3). The in-memory face of the `pending_op` table, from /// which it is rebuilt on mount. /// /// Two things read it: [`Core::read_range`], because until the op drains the /// staged blob *is* the file's content and the remote still holds the old /// revision; and the drain worker, which performs the uploads. pending: Arc>>, /// Nodes removed through this daemon which a briefly stale remote listing /// may still return. Uids are not reused; an explicit restore clears one. hidden: Arc>>, /// The folder and name each create on the wire was sent with, by op id. /// Drive holds that name until the create lands and a rename or trash /// queued behind it frees it again. creating: Arc>>, /// The SHA-1 of the last revision this daemon uploaded to a file and could /// not read back, so its revision id is not known (`docs/BUGS.md` B157). sealed_unread: Arc>>, /// The node an upload has just put on Drive, from when the drain lets go /// of its blob until the tree has the node (`docs/BUGS.md` B170). landed: Arc>>, /// Creates that landed but could not be read back, by the uid Drive gave /// them, with the placeholder's. The drain adopts one before it sends /// anything else for it. unadopted: Arc>>, /// Nudges the drain workers: set true and notify to have them re-examine /// the queue instead of waiting out their backoff. drain_wake: Arc<(Mutex, Condvar)>, /// The daemon's stop signal, shared with every background loop this `Core` /// was cloned into so teardown can end and join them (bugs.md B44). shutdown: Arc, /// How long this node's last upload actually took, in milliseconds, with /// the time it was measured for eviction. Feeds /// [`Core::revision_debounce`]: a file that takes 30 seconds to send needs /// a grace period on that scale, or every save supersedes an upload that /// was already on the wire. Bounded by [`UPLOAD_TIME_MEMORY`]. upload_times: Arc>>, /// Cancellation flags for uploads currently on the wire, keyed by node. /// /// Set when a newer revision of the same file is queued: the bytes in /// flight have just been superseded, and the only thing finishing them /// achieves is spending the user's uplink on a revision the next op /// replaces. Read by the upload's own reader ([`CountingReader`]), which is /// the one thing the SDK calls often enough to notice. upload_cancel: Arc>>>, /// Uploads whose op is retired but whose bytes the content cache has not /// adopted yet, because the revision just sealed is still being read back. /// The status counts them as pending, so an empty queue means every file /// written through the mount reads without the network (`docs/BUGS.md` /// B119). landing_uploads: Arc, /// True while a background refresh of the photos timeline (resp. the trash) is /// already running, so a burst of page requests against a stale listing kicks /// off one refresh rather than one per request. timeline_refreshing: Arc, /// How far the running timeline refresh has got resolving photo metadata, /// as `(done, total)` photos; `None` when no refresh is resolving any. The /// Places view shows it while a library's locations are first read. timeline_progress: Arc>>, /// The same, for the album listing. albums_refreshing: Arc, trash_refreshing: Arc, /// True while the similar-photo pass hashes thumbnails (see /// [`Core::photo_similar`]), so asking again doesn't start a second one. similar_hashing: Arc, /// Fires whenever a trash refresh persists a batch or finishes, so a /// `ListTrash` request waiting on a first-ever refresh wakes on progress /// instead of polling or blocking for the whole run. See /// [`Core::await_trash_refresh`]. trash_progress: Arc, /// Conflict copies the sweep has already checked against the activity log /// this run, so each pass does not ask the database again. The log itself /// is what keeps a copy from being flagged again after a restart. See /// [`Core::run_conflict_sweep_loop`] and `docs/BUGS.md` B99. conflict_notified: Arc>>, /// Whether the conflict sweep may actually trash the duplicates it finds, or /// only report them. Resolved once at mount from config + environment /// ([`AppConfig::resolved_conflict_sweep`]); [`SweepMode::Off`] means the /// sweep thread is never spawned at all. See `docs/BUGS.md` B71. sweep_mode: SweepMode, /// A mkdir, create, rename or delete through the mount is recorded and /// queued, and the drain sends it; the syscall never waits for Drive. Off, /// they go to Drive inline while online, as before 3.0. local_first: bool, /// Photos whose missing thumbnail is being generated right now. A tile that is /// still on screen asks for its thumbnail again every few seconds, and each of /// those downloads is a full-size photo — so an in-flight uid is never started /// twice. thumb_gen: Arc>>, /// Global permit pool for locally generated thumbnails. The old per-request /// limit multiplied when the GUI sent several batches, allowing dozens of /// full-size images to download at once. thumb_gen_budget: Arc, /// Current ordinary-file listing generation and a wake-up for tasks waiting /// on a permit or a download when that listing is abandoned. file_thumb_generation: Arc, file_thumb_cancel: Arc, /// Progress of the explicit recursive build started from the Files toolbar. thumbnail_build: Arc>, /// Cancellation for the explicit recursive build. Separate from ordinary /// listing cancellation because the two jobs have independent lifetimes. thumbnail_build_cancelled: Arc, thumbnail_build_cancel: Arc, /// Negative thumbnail knowledge, separated by whether the remote lacked a /// thumbnail or local decoding proved this revision unsupported. /// /// Absence has to be cached or it costs a round trip every time it is asked /// for, and it is asked for constantly: an `ls -l` from an xattr-aware lister /// issues a `getxattr` per advertised name per entry, so a 65-file directory /// of videos re-probed 130 times per listing at ~186 ms each (B5). The mtime /// is the validity tag — a new revision may well have a thumbnail — matching /// how [`ContentCache::read_thumbnail`] validates the positive side. thumbnail_misses: Arc>, /// Last account quota answer and when it was learned, for `statfs(2)`. /// /// `df` and the free-space preflight of every file manager and installer ask /// for this, and the answer costs a round trip — so it is served from here /// for [`QUOTA_TTL`] and refreshed on a worker afterwards. Total account /// storage moves slowly enough that a minute-old number is a better answer /// than either a stall or the zeroes the default implementation returns. quota: Arc>>, /// Set while a `statfs` is refreshing [`Core::quota`], so the others serve /// the last figures instead of each holding a worker on the same request. quota_refreshing: Arc, /// Database maintenance that has been moved off the request thread. /// /// `VACUUM` and a deep `integrity_check` both hold the single connection for /// tens of seconds against a real install, which stops every FUSE handler /// that needs the database — so the handler acknowledges the request and a /// thread does the work, exactly as the other long control requests already /// do. See [`Maintenance`]. maintenance: Arc>, /// Folders whose listing was enumerated cheaply and is having its file sizes /// filled in right now, so a burst of `stat`s over a fresh listing starts one /// upgrade rather than one per entry. See [`Core::spawn_size_upgrade`]. /// Size upgrades currently running, per folder inode, across every mount. A `getattr` that /// needs a real size waits on the entry rather than issuing its own fetch — /// `ls -l` of a folder is one `getattr` per file, and they must collapse /// onto a single batch (bugs.md B14). size_upgrades: Arc>>>, /// Replies parked on those upgrades. Shared across every mount for the same /// reason the pool is: the bound that matters is per daemon. size_waiters: Arc, /// This mount's kernel notification channel, for telling the kernel to drop /// metadata it has cached. Set once the session exists — which is *after* /// the `Core` it is built from, hence the cell. /// /// Per mount, not per daemon: each session calls its own root inode 1. /// [`Core::fork_state`] gives each fork an empty cell of its own. Work that /// changes the tree for every mount tells every session /// ([`Core::for_each_mount`]). notifier: Arc>, /// True only after this mount's FUSE session spawned successfully. /// Registration happens earlier for the primary drain, so registration /// alone cannot answer whether the location is mounted. session_live: Arc, /// In-flight upload/download progress, served to `GetQueueStatus`. Shared /// across the FUSE session and the control-socket task. transfers: Arc, /// Open [`Request::Subscribe`](pdfs_core::control::Request::Subscribe) /// connections, which every change a front-end shows is published to. events: Arc, /// True while the background scanner is rebuilding the local-file index, so /// `SearchLocal` can tell a front-end "still indexing" apart from "no match". indexing: Arc, /// Live per-folder sync progress, keyed by sync-folder id, so `ListSyncFolders` /// can say what a pass is doing rather than just "syncing". An entry exists /// only while that folder's reconcile pass is running. sync_progress: Arc>>, /// Channel to the folder-sync engine (devices.md Phase 2): nudges it to /// reconcile a folder, reconcile everything, or re-scan its watch set. sync_tx: std::sync::mpsc::Sender, /// Secondary FUSE sessions for `ondemand` sync folders, keyed by sync-folder /// id (devices.md Phase 3). Each is a `ProtonFs` rooted at the folder's remote /// node, mounted over its local path, sharing this Core's client/cache/db and /// tree but with its own root (`fork_state`). Held so we can unmount on /// toggle back to `mirror` and on daemon shutdown. Each entry retains the /// FUSE connection id and the fork's exact liveness flag so teardown first /// makes that mount unroutable. mounts: Arc>>, /// Per-sync-folder locks, held for a whole reconcile pass and for a whole /// mode switch. A `mirror→ondemand` flip evicts the local tree and mounts /// FUSE over it, so it must never overlap a pass that is walking and /// uploading that same tree — the engine would upload files as they vanish /// and then walk the FUSE mount as if it were local. sync_locks: Arc>>>>, /// Nodes this daemon changed on the remote itself, and how many echoes of /// those changes the event feed still owes us. /// /// The feed replays our own writes back at us seconds later, and a /// `NodeUpdated` carries no revision id — nothing in it distinguishes "another /// device changed this file" from "you changed this file". Treating the echo /// as foreign is not free: it evicts the content blob we just uploaded, so the /// next read re-downloads a revision the API may not be serving yet. That is /// how a SQLite database written on the mount came back `malformed`, and how a /// mount doing bulk uploads spent its time re-fetching its own bytes. /// /// So the first event for a node we changed ourselves, within /// [`SELF_CHANGE_TTL_MS`], is consumed rather than applied: the tree already /// holds what that event describes, because [`Core::refresh_after_upload`] put /// it there. Anything beyond that applies normally, which bounds the staleness /// a genuine remote change can suffer to one event. /// /// Shared across forks like `pending`: the drain that records the change and /// the event task that consumes the echo are both per-daemon. self_changes: Arc>>, /// Every live mount: the primary and one per on-demand fork /// ([`Core::fork_state`]), with its mountpoint, root and kernel channel. /// Shared by every fork. /// /// The mounts share one tree, so a change the drain makes is seen by every /// one of them. Each forked mount used to keep its nodes in a state of its /// own, where the drain never looked: the fork's entry kept its `local~` /// placeholder uid forever (`docs/BUGS.md` B74). What is still per mount is /// the kernel's cache, which [`Core::for_each_mount`] tells every session /// about. /// /// `Weak`, so an unmounted fork is dropped rather than pinned here. states: Arc, } /// Background database maintenance: what is running, and what the last deep /// integrity check found. /// /// One at a time — both operations are whole-database and there is nothing to /// gain from overlapping them — and the result of a check outlives the request /// that asked for it, because the request is answered before the check finishes. #[derive(Default)] struct Maintenance { /// A vacuum or integrity check is running right now. running: bool, /// Findings of the most recent completed deep check. `None` means none has /// finished this run, which is *not* the same as "found nothing". integrity: Option>, } /// How long a change this daemon made stays recognisable as its own. Generous /// against a feed that can lag, but finite: past it, an event for the node is /// applied like any other. See [`Core::self_changes`]. const SELF_CHANGE_TTL_MS: i64 = 120_000; /// A remote change this daemon made, awaiting its echo from the event feed. struct SelfChange { /// When we made it, in the same epoch millis as [`now_millis`]. at_ms: i64, /// Echoes not yet seen. More than one because a single file can be created /// and then have a revision uploaded, and the feed reports both. echoes: u32, } /// Record one remote change this daemon made. Split from [`Core`] so the /// expiry and echo-counting rules can be tested against a clock the test owns. fn note_self_change(changes: &mut HashMap, uid: &NodeUid, now_ms: i64) { // Echoes that never arrived — a node trashed before its event came round, a // feed that skipped it — would otherwise accumulate for the life of the // daemon. Pruning on write keeps the map the size of the recent queue. changes.retain(|_, c| now_ms - c.at_ms < SELF_CHANGE_TTL_MS); let change = changes.entry(uid.clone()).or_insert(SelfChange { at_ms: now_ms, echoes: 0, }); change.at_ms = now_ms; change.echoes += 1; } /// Claim one echo, per [`Core::take_self_change`]. fn take_self_change( changes: &mut HashMap, uid: &NodeUid, now_ms: i64, ) -> bool { let Some(change) = changes.get_mut(uid) else { return false; }; // Too old to attribute: the feed is far enough behind that this is as likely // to be someone else's change as ours, and guessing wrong here costs the user // a stale file. if now_ms - change.at_ms >= SELF_CHANGE_TTL_MS { changes.remove(uid); return false; } change.echoes -= 1; if change.echoes == 0 { changes.remove(uid); } true } /// One live mount, as published to the shared registry. #[derive(Clone)] struct LiveMount { mountpoint: PathBuf, /// The mount's kernel channel, empty until its session is spawned. notifier: Arc>, session_live: Arc, /// The tree inode the mount is rooted at, empty until its session is built. root: Arc>, } impl LiveMount { fn is_live(&self) -> bool { self.session_live.load(Ordering::Acquire) } } struct MountedState { /// Absolute local root of this mount. mountpoint: PathBuf, /// The same cell as [`Core::notifier`]. `Weak`, so an unmounted fork is /// dropped rather than pinned here; a dead entry is reaped on the next /// registry walk. notifier: std::sync::Weak>, session_live: Arc, root: Arc>, } /// Every mount in the daemon, shared by the primary `Core` and every /// [`Core::fork_state`] clone of it. /// /// Split out from `Core` so the reap-and-walk rule can be tested on its own: a /// fork that fails to appear here, or one that lingers after unmount, is exactly /// the failure `docs/BUGS.md` B74 was. #[derive(Default)] struct StateRegistry(Mutex>); impl StateRegistry { /// Publish a mount and the channel to the session that serves it. /// /// A mount is registered exactly once by its owner. Session construction /// only flips `session_live`; it never republishes the mount. fn register( &self, mountpoint: &Path, notifier: &Arc>, session_live: Arc, root: Arc>, ) { let mut states = self.0.lock(); states.retain(|m| m.notifier.strong_count() > 0); debug_assert!( !states .iter() .any(|mounted| mounted.notifier.ptr_eq(&Arc::downgrade(notifier))), "a mount must be registered exactly once" ); states.push(MountedState { mountpoint: mountpoint.to_path_buf(), notifier: Arc::downgrade(notifier), session_live, root, }); } /// Every registered mount, reaping the entries whose session has gone. /// Returns owned handles so callers do not hold the registry lock across /// the work. fn live(&self) -> Vec { let mut states = self.0.lock(); states.retain(|m| m.notifier.strong_count() > 0); states .iter() .filter_map(|m| { Some(LiveMount { mountpoint: m.mountpoint.clone(), notifier: m.notifier.upgrade()?, session_live: m.session_live.clone(), root: m.root.clone(), }) }) .collect() } /// The live session whose local root most specifically covers `path`. /// /// On-demand roots can be nested below another location. Selecting by the /// longest component prefix ensures the nested session wins rather than the /// broader primary mount. Callers that need the relative suffix derive it /// with `path.strip_prefix(mountpoint)`. fn covering(&self, path: &Path) -> Option { self.live() .into_iter() .filter(|mount| mount.is_live() && path.starts_with(&mount.mountpoint)) .max_by_key(|mount| mount.mountpoint.components().count()) } fn is_mounted_at(&self, path: &Path) -> bool { self.covering(path) .is_some_and(|mount| mount.mountpoint == path) } /// The tree inodes of the sessions that are mounted. fn live_roots(&self) -> Vec { self.live() .into_iter() .filter(LiveMount::is_live) .filter_map(|mount| mount.root.get().copied()) .collect() } } /// Why [`Core::apply_sync_folder_mode`] did not switch a folder. The two cases are /// answered very differently: `NotNow` is the normal state of a folder that is busy /// or has local changes still to push, and the caller queues the request; `Failed` /// is a real fault the user has to hear about. enum SwitchBlocked { /// The folder is mid-pass, or not yet safe to switch. Try again after a pass. NotNow, /// A queued target was canceled or replaced before the folder lock became /// available. The newer intent owns the next transition. Superseded, /// The switch was attempted and broke. Failed(String), } impl Core { /// Start a background `VACUUM`, or refuse because maintenance is already /// running. /// /// Returns as soon as the thread is spawned; the caller acknowledges the /// request. The outcome is a log line and a job that disappears from /// `GetQueueStatus` when it is done — a vacuum has nothing to report that a /// follow-up `CacheInspect` cannot show. fn start_vacuum(&self) -> CoreResult<()> { { let mut m = self.maintenance.lock(); if m.running { return Err(CoreError::conflict( "database maintenance is already running", )); } m.running = true; } let core = self.clone(); std::thread::spawn(move || { let job = core.transfers.begin_job("Compacting the cache database"); job.detail("Rewriting the database file"); let started = Instant::now(); match core.db.vacuum() { Ok(outcome) => info!( freed_bytes = outcome.freed_bytes(), before_bytes = outcome.before_bytes, after_bytes = outcome.after_bytes, wal_frames = outcome.wal_frames_checkpointed, elapsed_ms = started.elapsed().as_millis() as u64, "cache database vacuumed" ), Err(e) => warn!(error = %e, "vacuum failed"), } core.maintenance.lock().running = false; }); Ok(()) } /// Answer a deep `CacheInspect`: hand back the last completed integrity /// check, and start one if none is running. /// /// Returns `(problems, checked, running)`. A first request gets /// `(_, false, true)` and the answer lands on a later one — the check reads /// every page of the database under the connection mutex, so the choice is /// between answering late and freezing the mount until it finishes. fn start_or_read_integrity_check(&self) -> (Vec, bool, bool) { let mut m = self.maintenance.lock(); if let Some(problems) = m.integrity.clone() { return (problems, true, m.running); } if m.running { return (Vec::new(), false, true); } m.running = true; drop(m); let core = self.clone(); std::thread::spawn(move || { let job = core.transfers.begin_job("Checking the cache database"); job.detail("Verifying every page"); let problems = match core.db.integrity_check() { Ok(problems) => problems, Err(e) => vec![format!("integrity check failed: {e:?}")], }; if problems.is_empty() { info!("cache database integrity check found no problems"); } else { warn!(problems = problems.len(), "cache database integrity check"); } let mut m = core.maintenance.lock(); m.integrity = Some(problems); m.running = false; }); (Vec::new(), false, true) } /// Register a mount so background work can reach its session. Called once /// for the primary mount and once per on-demand fork. fn register_state(&self, mountpoint: &Path) { self.states.register( mountpoint, &self.notifier, self.session_live.clone(), self.root.clone(), ); } /// The tree inode this mount is rooted at. [`ROOT_INO`] until the session /// is built, which is where a test that builds a bare tree puts its root. pub(crate) fn root_ino(&self) -> u64 { self.root.get().copied().unwrap_or(ROOT_INO) } /// Whether `uid` names an owned node reachable from the root of a mounted /// session. See [`State::owns_visible_uid`]. fn owns_visible_uid(&self, uid: &NodeUid) -> bool { let roots = self.states.live_roots(); self.state().owns_visible_uid(uid, &roots) } /// This `Core`, re-rooted onto whichever live mount most specifically covers /// the absolute path `abs`, together with `abs` relative to that mount's /// root. /// /// Everything a `Core` holds is per-daemon except the fields /// [`Core::fork_state`] replaces, so swapping exactly those turns the /// primary `Core` into the fork that owns the path — resolving from the /// fork's root and notifying its session. Without this, every /// path-addressed control request is resolved from the primary root, so /// anything under a secondary on-demand mount is rejected as "not under the /// mountpoint" and the user has no way to re-enumerate a folder they can see /// (`docs/BUGS.md` B86). /// /// `None` when no live mount covers `abs`. fn rooted_at(&self, abs: &Path) -> Option<(Core, PathBuf)> { let mount = self.states.covering(abs)?; let rel = abs.strip_prefix(&mount.mountpoint).ok()?.to_path_buf(); let mut core = self.clone(); if !Arc::ptr_eq(&mount.notifier, &self.notifier) { // Another mount, so adopt its per-mount half wholesale. `primary` // is a property of the mount, and the only mount this `Core` can // be the primary of is the one it was built for. core.primary = false; core.notifier = mount.notifier; core.session_live = mount.session_live; core.root = mount.root; } Some((core, rel)) } /// Take the tree's lock. /// /// The guard is what applies the mutation's write-throughs, after releasing /// the lock — see [`StateGuard`]. Every caller takes it this way; nothing /// locks `self.state` directly, or the writes it queues would be applied /// only when some later caller happened to take the guard. fn state(&self) -> StateGuard<'_> { StateGuard::new(self.state.lock(), &self.db) } /// Run `apply` against the tree, then tell every mounted session about the /// access changes it made. /// /// For the drain and the sync engine, which are per-daemon: a `pending_op` /// row records a uid, not the session that queued it, and any mount may be /// showing the node. fn for_each_state(&self, apply: impl FnOnce(&mut State)) { let changed = { let mut state = self.state(); apply(&mut state); state.take_access_changes() }; self.notify_access_changes(&changed); } /// Like [`Core::for_each_state`], but the closure also records what the /// kernel needs told into a [`NotifyBatch`], which goes to every mounted /// session. /// /// Recorded rather than sent: notifications are flushed after the State /// lock is released, the same rule [`Core::for_each_state`] and /// [`Core::flush_access_changes`] follow. fn for_each_mount(&self, apply: impl FnOnce(&mut State, &mut NotifyBatch)) { let mut batch = NotifyBatch::default(); apply(&mut self.state(), &mut batch); for mount in self.states.live() { batch.flush(mount.notifier.get()); } } /// Publish access changes accumulated by an intern/root refresh. The State /// lock is released before notifying the kernel because notifier callbacks /// may synchronously provoke more filesystem work. fn flush_access_changes(&self) { let changed = self.state().take_access_changes(); self.notify_access_changes(&changed); } /// Tell every mounted session that `changed` have new permissions. fn notify_access_changes(&self, changed: &[u64]) { if changed.is_empty() { return; } for mount in self.states.live() { notify_access_changes(mount.notifier.get(), changed); } } fn require_writable(&self, ino: u64) -> Result<(), Errno> { self.state().require_writable(ino) } /// Mutation admission point. Persisted and every resident authority must /// agree that the node is writable. A check completed before a downgrade is /// admitted; later queued attempts are caught by drain. No global lock is /// held across a network call. /// /// The intersection closes both halves of an access-refresh race: a DB /// downgrade cannot be hidden by stale live state, and a live downgrade /// cannot be hidden by a restored DB row. Missing UIDs fail closed. fn require_uid_writable(&self, uid: &NodeUid) -> Result<(), Errno> { match self.uid_write_authority(uid) { WriteAuthority::Writable => Ok(()), // A uid nobody can speak for is refused here just as a denied one // is: a syscall is a live request against a live tree, and a stale // handle must not be admitted because the tree forgot the node. // The drain, which replays intent recorded long ago, is the one // caller that has to tell the two apart — see // [`Core::uid_write_authority`]. WriteAuthority::Denied | WriteAuthority::Unknown => Err(Errno::EACCES), } } /// The full answer behind [`Core::require_uid_writable`]: whether every /// authority agrees this node is writable, refuses it, or has never heard /// of it. /// /// The third case is the one worth naming. A node absent from the local /// tree yields no access row, and collapsing that into "denied" told the /// drain to wait for a permission change that was never coming, because /// nothing about the node's permissions was ever the problem (B83). pub(crate) fn uid_write_authority(&self, uid: &NodeUid) -> WriteAuthority { let live_access: Vec = self.state().access_by_uid(uid).into_iter().collect(); uid_write_authority(&self.db, uid, &live_access) } } /// Whether a node may be written, as agreed by every authority that has an /// opinion — or the fact that none of them does. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum WriteAuthority { /// The persisted tree and every resident mount agree: writable. Writable, /// At least one authority refuses it, or could not be consulted. Fails /// closed, so a lookup error lands here. Denied, /// The node is not in the local tree, so there is no authority to consult. /// Not the same as a refusal, and callers that can ask the remote should. Unknown, } fn uid_write_authority(db: &Db, uid: &NodeUid, live_access: &[Access]) -> WriteAuthority { let persisted = match db.effective_node_access(uid) { Ok(Some(access)) => access, Ok(None) => return WriteAuthority::Unknown, Err(error) => { error!(%uid, %error, "db effective access lookup failed"); return WriteAuthority::Denied; } }; if persisted.writable() && live_access.iter().all(|access| access.writable()) { WriteAuthority::Writable } else { WriteAuthority::Denied } } /// Move `uid` to `name` in `new_parent` within one mount, for a rename or move /// Drive has made, and record what the kernel has to forget. Returns the name /// the mount knew the node by, and whether the node was moved rather than /// forgotten, which writes its row through. `None` when the mount does not /// hold the node. /// /// The inode is kept wherever the mount shows the new parent, as the mount's /// own rename keeps it (`State::relocate`). A mount that does not show the new /// parent only forgets the node; to it, the node is gone. The row stays either /// way (`docs/BUGS.md` B117). fn relocate_in_mount( st: &mut State, notify: &mut NotifyBatch, uid: &NodeUid, new_parent: &NodeUid, name: &str, queued: bool, ) -> Option<(String, bool)> { let ino = *st.by_uid.get(uid)?; let entry = st.entries.get(&ino)?; let (from_parent, from_name) = (entry.parent, entry.node.name.clone()); match st.by_uid.get(new_parent).copied() { Some(to_parent) => { if queued { // Listing again would ask Drive, which still has the old name; // the kernel's pages of both folders go instead. st.rename_in_place(ino, to_parent, new_parent, name); notify.inval_inode(from_parent); if to_parent != from_parent { notify.inval_inode(to_parent); } } else { st.relocate(ino, from_parent, to_parent, new_parent, name); } notify.inval_entry(from_parent, from_name.clone()); notify.inval_entry(to_parent, name.to_string()); Some((from_name, true)) } None => { st.forget_mem(uid); notify.delete(from_parent, ino, from_name.clone()); Some((from_name, false)) } } } /// One kernel notification, recorded while the State lock is held so it can be /// sent once it is not. /// /// `inval_inode` calls into `invalidate_inode_pages2`, which can block waiting /// on pages that this daemon's own workers have to fill — and those workers take /// the State lock. Sending from inside the lock is that deadlock's shape; a /// batch is the smallest way to keep the inode/mount pairing while moving the /// send outside it. enum KernelNotice { InvalInode(u64), InvalEntry { parent: u64, name: String, }, Delete { parent: u64, child: u64, name: String, }, } /// Notifications accumulated under the State lock. Flushed by /// [`Core::for_each_mount`] to every session once the lock is released. #[derive(Default)] struct NotifyBatch(Vec); impl NotifyBatch { fn inval_inode(&mut self, ino: u64) { self.0.push(KernelNotice::InvalInode(ino)); } fn inval_entry(&mut self, parent: u64, name: String) { self.0.push(KernelNotice::InvalEntry { parent, name }); } fn delete(&mut self, parent: u64, child: u64, name: String) { self.0.push(KernelNotice::Delete { parent, child, name, }); } fn extend_inodes(&mut self, inodes: &[u64]) { self.0 .extend(inodes.iter().map(|&ino| KernelNotice::InvalInode(ino))); } /// Send everything recorded to one session. A mount whose session has not /// been spawned has nothing to tell the kernel. A session that never looked /// a node up answers `ENOENT`, which is ignored like any other refusal. fn flush(&self, notifier: Option<&KernelChannel>) { let Some(notifier) = notifier else { return }; for notice in &self.0 { let _ = match notice { KernelNotice::InvalInode(ino) => notifier.inval_inode(*ino), KernelNotice::InvalEntry { parent, name } => { notifier.inval_entry(*parent, OsStr::new(name)) } KernelNotice::Delete { parent, child, name, } => notifier.delete(*parent, *child, OsStr::new(name)), }; } } } /// One FUSE session's notification channel. The tree has one inode space for /// every mount, and each session knows its own root as [`ROOT_INO`], so this /// names the root that way and every other inode as it is. pub(crate) struct KernelChannel { notifier: Notifier, /// The tree inode of the session's root. root: u64, } impl KernelChannel { fn new(notifier: Notifier, root: u64) -> Self { Self { notifier, root } } pub(crate) fn inval_inode(&self, ino: u64) -> std::io::Result<()> { let ino = filesystem::kernel_ino(self.root, ino); off_the_runtime(|| self.notifier.inval_inode(ino, 0, 0)) } /// Drop the cached attributes of `ino` but not its page cache. fn inval_attrs(&self, ino: u64) -> std::io::Result<()> { let ino = filesystem::kernel_ino(self.root, ino); off_the_runtime(|| self.notifier.inval_inode(ino, -1, 0)) } pub(crate) fn inval_entry(&self, parent: u64, name: &OsStr) -> std::io::Result<()> { let parent = filesystem::kernel_ino(self.root, parent); off_the_runtime(|| self.notifier.inval_entry(parent, name)) } fn delete(&self, parent: u64, child: u64, name: &OsStr) -> std::io::Result<()> { let parent = filesystem::kernel_ino(self.root, parent); let child = filesystem::kernel_ino(self.root, child); off_the_runtime(|| self.notifier.delete(parent, child, name)) } } /// Send a notice to the kernel without holding up the runtime. /// /// The kernel answers an invalidation only once it has the pages' locks, and /// a read holds those until the daemon answers it. Sent from a runtime worker, /// the notice blocked that worker in the kernel; when it was the one driving /// the timers and I/O, nothing drove them any more, so the read's fetch never /// finished and the daemon hung (`docs/BUGS.md` B130). On a worker the notice /// hands the worker's tasks to another thread first; anywhere else it is sent /// as it is. fn off_the_runtime(notice: impl FnOnce() -> T) -> T { match tokio::runtime::Handle::try_current() { Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => { tokio::task::block_in_place(notice) } _ => notice(), } } fn notify_access_changes(notifier: Option<&KernelChannel>, changed: &[u64]) { let Some(notifier) = notifier else { return }; for &ino in changed { let _ = notifier.inval_inode(ino); } } fn require_rename_access( require: impl Fn(&NodeUid) -> Result<(), Errno>, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_uid: &NodeUid, ) -> Result<(), Errno> { for authority in [uid, old_parent_uid, new_parent_uid] { require(authority)?; } Ok(()) } fn require_node_parent_access( require: impl Fn(&NodeUid) -> Result<(), Errno>, uid: &NodeUid, parent_uid: &NodeUid, ) -> Result<(), Errno> { require(uid)?; require(parent_uid) } fn preserve_on_access_denied( access: Result<(), Errno>, accepted_bytes: bool, preserve: impl FnOnce(), ) -> Result<(), Errno> { if let Err(error) = access { if error.code() == libc::EACCES && accepted_bytes { preserve(); } return Err(error); } Ok(()) } /// Fold the staged blob of an undrained write into the blob at `src` that is /// about to supersede it, so the newer write inherits the older one's bytes /// instead of losing them. /// /// Only the ranges the older write authored and the newer one did not are /// copied, so a small edit over a small edit moves almost nothing. What neither /// wrote is untouched remote content, which is why `base_size`, `base_mtime` and /// the baseline all come from the write being superseded: that is the revision /// those ranges were last observed against, and the one the drain must gap-fill /// and conflict-check them against. fn merge_over_pending( meta: &mut StagedWrite, src: &Path, previous: &PendingRevision, ) -> std::io::Result<()> { let mut written = Intervals::default(); for &(s, e) in &meta.authored { written.add(s, e); } let mut earlier = Intervals::default(); for &(s, e) in &previous.meta.authored { earlier.add(s, e); } let blob = File::open(&previous.path)?; let dst = std::fs::OpenOptions::new().write(true).open(src)?; for (s, e, authored) in written.clone().segments(0, meta.len) { if authored { continue; } for (ps, pe, have) in earlier.segments(s, e.min(previous.meta.len)) { if !have { continue; } let mut buf = vec![0u8; (pe - ps) as usize]; blob.read_exact_at(&mut buf, ps)?; dst.write_all_at(&buf, ps)?; written.add(ps, pe); } } dst.sync_all()?; meta.base_size = previous.meta.base_size; meta.base_mtime = previous.meta.base_mtime; meta.based_on = previous.meta.based_on.clone(); meta.authored = written .segments(0, meta.len) .into_iter() .filter(|&(_, _, authored)| authored) .map(|(s, e, _)| (s, e)) .collect(); meta.complete = StagedWrite::covers(&meta.authored, meta.len); Ok(()) } /// Retire the last resident reference to an unlinked inode without touching its /// persisted node row. The namespace path already chose whether that row is /// disposable or retained as queued-trash authority. fn release_unlinked_entry(state: &mut State, ino: u64) -> Option { let entry = state.entries.get_mut(&ino)?; entry.open_count = entry.open_count.saturating_sub(1); if entry.open_count != 0 || !entry.unlinked { return None; } let uid = entry.uid.clone(); state.forget_mem(&uid); Some(uid) } fn release_must_retain_queued_trash(db: &Db, uid: &NodeUid) -> pdfs_core::Result { db.has_pending_op(&uid.to_string(), OP_TRASH) } /// The size of a staged blob that has outlived [`STAGING_ORPHAN_RETAIN`], or /// `None` while it is younger than that — or while its age cannot be read at /// all, which is never grounds for deleting bytes. fn orphan_past_retention(blob: &Path) -> Option { let meta = std::fs::metadata(blob).ok()?; let age = meta.modified().ok()?.elapsed().ok()?; (age >= STAGING_ORPHAN_RETAIN).then_some(meta.len()) } fn release_can_discard_unlinked(db: &Db, uid: &NodeUid) -> bool { match release_must_retain_queued_trash(db, uid) { Ok(retain) => !retain, Err(error) => { warn!(%uid, %error, "queued-trash lookup failed; retaining unlinked state"); false } } } struct AcceptedShares { uids: Vec, /// `None` means duplicate listing rows disagreed on provenance. share_ids: HashMap>, } fn accepted_share_provenance(items: Vec) -> AcceptedShares { let mut uids = Vec::new(); let mut share_ids = HashMap::new(); for item in items { match share_ids.entry(item.uid.clone()) { std::collections::hash_map::Entry::Vacant(entry) => { uids.push(item.uid); entry.insert(Some(item.share_id)); } std::collections::hash_map::Entry::Occupied(mut entry) => { if entry.get().as_ref() != Some(&item.share_id) { entry.insert(None); } } } } AcceptedShares { uids, share_ids } } fn shared_root_access(nodes: &[&Node], expected_share: Option<&ShareId>) -> Access { let Some(expected_share) = expected_share else { return Access::Viewer; }; let mut observed = None; for node in nodes { let Some(membership) = node.membership.as_ref() else { return Access::Viewer; }; if &membership.share_id != expected_share { return Access::Viewer; } let Some(role) = membership.role_exact() else { return Access::Viewer; }; let access = access_for(Some(role), Access::Viewer, true); if observed.is_some_and(|prior| prior != access) { return Access::Viewer; } observed = Some(access); } observed.unwrap_or(Access::Viewer) } fn prepare_shared_roots( accepted: &AcceptedShares, mut materialized: Vec, parent: &NodeUid, ) -> Vec { materialized.retain(|node| accepted.share_ids.contains_key(&node.uid)); materialized.sort_by(|a, b| { a.uid .to_string() .cmp(&b.uid.to_string()) .then_with(|| a.name.cmp(&b.name)) }); let mut published = Vec::new(); let mut start = 0usize; while start < materialized.len() { let uid = materialized[start].uid.clone(); let mut end = start + 1; while end < materialized.len() && materialized[end].uid == uid { end += 1; } let access = shared_root_access( &materialized[start..end].iter().collect::>(), accepted.share_ids.get(&uid).and_then(Option::as_ref), ); let mut node = materialized[start].clone(); node.parent_uid = Some(parent.clone()); node.trashed = false; published.push(PublishedSharedRoot { node, access }); start = end; } let mut named: Vec = published.iter().map(|root| root.node.clone()).collect(); disambiguate_shared_names(&mut named); let names: HashMap = named .into_iter() .map(|node| (node.uid, node.name)) .collect(); for root in &mut published { if let Some(name) = names.get(&root.node.uid) { root.node.name.clone_from(name); } } published } impl Core { /// Rename `uid` on Drive, riding out an "out of date" answer from a stale /// name hash. See [`retry_out_of_date`]. pub(crate) fn rename_remote(&self, uid: &NodeUid, name: &str) -> Result<(), ProtonError> { self.change_remote(uid, || { self.block_on_bounded(self.drive.rename_node(uid, name, None)) }) } /// Move `uid` into the folder `parent` on Drive, like [`Core::rename_remote`]. pub(crate) fn move_remote(&self, uid: &NodeUid, parent: &NodeUid) -> Result<(), ProtonError> { self.change_remote(uid, || { self.block_on_bounded(self.drive.move_node(uid, parent)) }) } /// Move `uid` into the folder `parent` on Drive, renaming it to /// `target_name` on the way when one is given, in one request. /// /// Done as a rename and then a move, a sibling in the source folder that /// held the new name made Drive refuse the first half, although the /// destination was free (docs/BUGS.md B122). One request also leaves no /// half-applied state between the two. pub(crate) fn move_rename_remote( &self, uid: &NodeUid, parent: &NodeUid, target_name: Option<&str>, ) -> Result<(), ProtonError> { self.change_remote(uid, || { let item = NodeMoveItem { uid: uid.clone(), target_name: target_name.map(str::to_string), }; self.block_on_bounded(async { let mut outcomes = std::pin::pin!(self.drive.move_nodes_streaming(vec![item], parent.clone())); match outcomes.next().await { Some(Ok((_, outcome))) => outcome, Some(Err(e)) => Err(e), None => Err(ProtonError::invalid_operation(format!( "move of {uid} reported no outcome" ))), } }) }) } /// Run a rename or move of `uid`, then drop the SDK's now stale copy of it. fn change_remote( &self, uid: &NodeUid, call: impl FnMut() -> Result<(), ProtonError>, ) -> Result<(), ProtonError> { retry_out_of_date(call, || { self.forget_sdk_node(uid); std::thread::sleep(OUT_OF_DATE_RETRY_DELAY); }) .inspect(|()| self.changed_remote(uid)) } /// Bookkeeping after a rename or move of `uid` landed: the SDK's copy of it /// is stale, and the feed will echo the change back. Without the second, /// that echo reads as someone else's edit and evicts the cached bytes a /// partial write is about to gap-fill from (docs/BUGS.md B123). pub(crate) fn changed_remote(&self, uid: &NodeUid) { self.forget_sdk_node(uid); self.note_self_change(uid); } /// Drop the SDK's cached copy of `uid` after this daemon renamed or moved it. /// /// The SDK sends a node's name hash from its entity cache, and nothing but /// the event feed's echo of the change refreshes that entry. Until the next /// poll, up to [`POLL_INTERVAL`] later, each rename or move of the node would /// send the old hash and Drive would refuse it as out of date (docs/BUGS.md /// B121). Feeding the SDK the echo now makes the next call read the link. pub(crate) fn forget_sdk_node(&self, uid: &NodeUid) { let changed = DriveEvent::NodeUpdated { id: DriveEventId::from("pdfs-local-change"), node_uid: uid.clone(), parent_node_uid: None, is_trashed: false, is_shared: false, }; if let Err(error) = self.block_on_bounded(self.drive.invalidate_caches_for_event(&changed)) { debug!(%uid, %error, "could not drop the SDK's cached node"); } } /// Record that this daemon just changed `uid` on the remote, so the event /// feed's echo of that change is recognised instead of re-applied. Called /// from the drain once the remote has actually accepted the change and the /// tree has been brought level with it. See [`Core::self_changes`]. pub(crate) fn note_self_change(&self, uid: &NodeUid) { note_self_change(&mut self.self_changes.lock(), uid, now_millis()); } /// Claim one outstanding echo for `uid`, reporting whether this event is one /// this daemon caused. Consuming rather than merely testing is what bounds /// the suppression: the next event for the node applies normally. fn take_self_change(&self, uid: &NodeUid) -> bool { take_self_change(&mut self.self_changes.lock(), uid, now_millis()) } /// Rehydrate the in-memory `State` maps from the DB on mount, so a cold /// start serves previously-seen metadata (stable inodes, instant listings) /// without re-hitting the API. The root inode is already installed by /// [`ProtonFs::new`]; this fills in every other persisted node and rebuilds /// the child listings of folders the DB records as fully enumerated. fn hydrate(&self) { let stored = match self.db.load_all() { Ok(s) => s, Err(e) => { warn!(error = %e, "db load_all failed; mounting cold"); return; } }; if stored.is_empty() { return; } // A write queued by a previous run carries the optimistic size the file // grew to, which the sealed remote revision does not yet reflect. The DB // node still holds the pre-write `claimed_size` (often 0 for a fresh // file), so without this a queued 500 MB file `stat`s as 0 bytes until // the drain lands — reads still serve from the staged blob, but `ls` // lying about the size reads as data loss. `hydrate_pending` ran before // us, so the pending map is populated; snapshot it here rather than hold // both locks at once. let pending_sizes: HashMap = { let pending = self.pending.lock(); pending .iter() .map(|(uid, pr)| (uid.clone(), pr.meta.len)) .collect() }; // `hydrate_pending` runs first and restores queued-trash tombstones. // Their node rows remain persisted as drain-time access authority, but // must not reappear in the mounted namespace after a restart. let hidden = self.hidden.lock().clone(); let mut st = self.state(); // Pass 1: give every uid its inode from its local id (the root is // already mapped). for sn in &stored { if hidden.contains(&sn.node.uid) || st.by_uid.contains_key(&sn.node.uid) { continue; } st.by_uid .insert(sn.node.uid.clone(), state::lid_ino(sn.lid)); } // Pass 2: materialize entries, resolving each parent's inode by uid. // Track folders flagged complete so their listings rebuild in pass 3. let mut listed_dirs: Vec = Vec::new(); for sn in stored { let StoredNode { mut node, listed, .. } = sn; if hidden.contains(&node.uid) { continue; } let Some(&ino) = st.by_uid.get(&node.uid) else { continue; }; // Re-apply a queued write's optimistic size so `stat` matches what // reads (served from the staged blob) already return. if let Some(&len) = pending_sizes.get(&node.uid) && let NodeKind::File { claimed_size, .. } = &mut node.kind { *claimed_size = Some(len as i64); } if listed && node.is_folder() { listed_dirs.push(ino); } // A root entry is owned by `ProtonFs::new`; don't overwrite it. if st.roots.contains(&ino) { continue; } // A node whose parent row never made it to disk must not be adopted // by the root: it would surface as a phantom top-level entry. let parent = node .parent_uid .as_ref() .and_then(|p| st.by_uid.get(p).copied()) .unwrap_or(ORPHAN_INO); let uid = node.uid.clone(); st.entries.insert( ino, Entry { uid, parent, node, access: pdfs_core::Access::Unknown, lookup_count: 1, open_count: 0, unlinked: false, }, ); } st.hydrate_access(); // Pass 3: rebuild child listings for fully-enumerated folders. let listings = listed_children(&st.entries, listed_dirs); st.children.extend(listings); info!(nodes = st.entries.len(), "hydrated metadata cache from db"); } /// Rebuild the in-memory pending map from the `pending_op` table on mount /// (offline.md Phase 3). /// /// A queued write survives a restart — that is the point of persisting it — /// so until the drain worker gets to it, reads of that file must still come /// from its staged blob rather than the remote's older revision. /// /// A row whose blob has gone missing is dropped: there is nothing left to /// upload, and keeping it would fail forever. fn hydrate_pending(&self) { let ops = match self.db.pending_ops() { Ok(ops) => ops, Err(e) => { error!(error = %e, "loading pending uploads failed"); return; } }; let mut map = self.pending.lock(); let mut restored = 0usize; for op in ops { let Some(uid) = parse_node_uid(&op.uid) else { error!(uid = %op.uid, id = op.id, "pending op has an unparseable uid; dropping"); let _ = self.db.delete_op(op.id); continue; }; if op.kind == OP_TRASH { self.hidden.lock().insert(uid.clone()); } restored += 1; // Only a revision must have a blob. A create carries none until // something is written to it (`touch` offline is a legitimate op // with nothing to serve), and a rename never has one. A trash keeps // the blob of a create it withdrew only to find the node by, not // to serve. All still have to be replayed, so only the blob — if // any — is checked. if op.kind == OP_TRASH || (op.blob_path.is_none() && op.kind != OP_REVISION) { continue; } let parsed = op .meta_json .as_deref() .and_then(|j| serde_json::from_str::(j).ok()) .zip(op.blob_path.as_deref().map(PathBuf::from)); let Some((meta, path)) = parsed else { error!(uid = %op.uid, id = op.id, "pending op is unreadable; dropping"); self.drop_unrecoverable_op(&op, &uid); restored -= 1; continue; }; if !path.exists() { error!(%uid, path = %path.display(), "staged blob is gone; dropping pending op"); self.drop_unrecoverable_op(&op, &uid); restored -= 1; continue; } map.insert(uid, PendingRevision { path, meta }); } if restored > 0 { info!(count = restored, "restored pending ops"); } } /// Re-attach staged blobs that no queued op refers to. /// /// `staging/` holds the only copy of writes the kernel has already been /// told succeeded. Three release-path failures put bytes there and return /// `EIO` without leaving a `pending_op` row behind — an access change /// mid-release, a write overlapping an undrained incomplete edit, a create /// that drained out from under its own handle. Until now nothing ever /// walked the directory again: `hydrate_pending` reads the op table and /// [`recover_fsynced_writes`](Self::recover_fsynced_writes) reads /// `recovery/`, so an orphaned staged blob was invisible to the daemon and /// to the user alike, and stayed that way until someone found the file. /// /// Runs once at mount, after `hydrate_pending`, so `pending` already /// describes what the queue owns. An orphan whose sidecar names a real node /// is queued as an ordinary revision — the condition that made it fail is /// gone by definition, since the op it collided with has since drained. /// Anything that cannot be addressed (no sidecar, an unparseable uid, a /// `local~` placeholder whose create is gone) is left in place and named in /// the log: unexplained bytes are never evidence that user data is /// disposable. fn reconcile_staging(&self) { let staged = self.cache.staged_writes(); if staged.is_empty() { return; } let claimed = match self.db.op_blob_paths() { Ok(paths) => paths, Err(e) => { error!(error = %e, "cannot read queued blob paths; skipping staging reconcile"); return; } }; let mut queued = 0usize; let mut stranded = 0usize; let mut retired = 0usize; let mut retired_bytes = 0u64; let mut example: Option = None; for (blob, meta) in staged { if claimed.contains(&blob.to_string_lossy().into_owned()) { continue; } let addressable = meta.as_ref().and_then(|m| parse_node_uid(&m.uid)); let Some((meta, uid)) = meta.zip(addressable).filter(|(_, uid)| !is_local_uid(uid)) else { if let Some(bytes) = orphan_past_retention(&blob) { retired += 1; retired_bytes += bytes; warn!(blob = %blob.display(), bytes, age_days = STAGING_ORPHAN_RETAIN.as_secs() / 86_400, "retiring staged bytes that name no node and have outlived retention"); self.cache.discard_staged(&blob); continue; } stranded += 1; example.get_or_insert_with(|| blob.clone()); continue; }; if self.pending.lock().contains_key(&uid) { // A newer write for the same node is queued and carries its own // blob. These bytes are older; keeping them is right, uploading // them over the newer ones is not. stranded += 1; warn!(%uid, blob = %blob.display(), "staged bytes are superseded by a queued write; kept for recovery"); continue; } let op = PendingOp { id: 0, kind: OP_REVISION.to_string(), uid: uid.to_string(), parent_uid: None, name: None, blob_path: Some(blob.to_string_lossy().into_owned()), meta_json: Some(serde_json::to_string(&meta).unwrap_or_default()), created_at: now_millis(), attempts: 0, last_error: None, next_attempt_at: now_millis(), }; match self.db.enqueue_op(&op) { Ok((_, superseded)) => { if let Some(old) = superseded { self.cache.discard_staged(Path::new(&old)); } self.pending.lock().insert( uid.clone(), PendingRevision { path: blob.clone(), meta, }, ); queued += 1; } Err(e) => { stranded += 1; error!(%uid, blob = %blob.display(), error = %e, "cannot queue an orphaned staged write; bytes kept"); } } } if stranded > 0 { // One line, not one per blob. A tree of them says exactly as much as // a single count and an example does, and said it on every mount: // 86 identical warnings for the same 86 files, boot after boot. warn!( count = stranded, example = %example.map(|b| b.display().to_string()).unwrap_or_default(), "staged bytes belong to no queued upload and name no node; kept for recovery" ); } if queued > 0 || stranded > 0 || retired > 0 { info!( queued, stranded, retired, retired_bytes, "reconciled orphaned staged writes" ); } } /// Queue the writes that an unclean shutdown caught between `fsync(2)` and /// `close(2)`. /// /// `fsync` promises the bytes survive a crash, but the queueing that makes a /// write outlive the daemon happens at `release`. A crash in between used to /// lose the data outright, because the scratch directory is cleared at open. /// Now `fsync` leaves a sidecar, `ContentCache::open` moves those blobs to /// `recovery/`, and this walks them into the same staging + queued-op path a /// normal release takes. /// /// Runs after [`hydrate_pending`](Self::hydrate_pending), which is what makes /// the incomplete-blob check in [`enqueue_staged_write`](Self::enqueue_staged_write) /// meaningful: a recovered partial write whose earlier write is still queued /// must not gap-fill from a remote revision that no longer describes the /// file. With `pending` already loaded, that case is detected and the bytes /// are parked rather than mis-filled. /// /// Failure is per-write and never fatal: a node that no longer exists, or an /// op that cannot be queued, leaves its blob in `recovery/` for the next run /// (and for a human) instead of taking the mount down. fn recover_fsynced_writes(&self) { let recovered = self.cache.recovered_writes(); if recovered.is_empty() { return; } let mut queued = 0usize; for (blob, meta) in recovered { let Some(uid) = parse_node_uid(&meta.uid) else { error!(uid = %meta.uid, "recovered write has an unparseable uid; keeping bytes"); continue; }; // If the write is incomplete and an earlier edit is still queued, merging it // now would use the wrong remote base. enqueue_staged_write would abandon it // to staging; keep it in recovery instead so the drain loop can queue it // once the queue clears. if !meta.complete && self.pending.lock().contains_key(&uid) { continue; } // The inode is only used to stamp the in-memory tree with the new // size, and nothing is interned this early — `hydrate` reads the // size back off `pending` when the node is first looked up, so 0 // (no such inode) is correct rather than merely tolerable. match self.enqueue_staged_write(&uid, 0, &blob, meta) { Ok(()) => { self.cache.discard_recovered(&blob); queued += 1; } Err(e) => { error!(%uid, blob = %blob.display(), error = ?e, "cannot queue a recovered write; bytes kept for the next run"); // Some failures (a partial write parked by // `stage_orphaned_write`) still consume the blob. Its sidecar // would then describe nothing, so retire it — the bytes are // in staging, which is where a human looks for them. if !blob.exists() { self.cache.discard_recovered(&blob); } } } } if queued > 0 { info!( count = queued, "recovered fsynced writes from an unclean shutdown" ); } } /// Discard an op that can never be performed, because the bytes it was to /// upload are gone from staging (something outside the daemon deleted them). /// /// For a node that only ever existed locally, the placeholder goes too. Its /// content is unrecoverable and nothing will ever mint it a real uid, so /// leaving the row would strand a file in the tree that can be listed but /// never read and never uploaded. fn drop_unrecoverable_op(&self, op: &PendingOp, uid: &NodeUid) { let _ = self.db.delete_op(op.id); if is_local_uid(uid) { error!(%uid, name = op.name.as_deref().unwrap_or("?"), "discarding a node whose only copy was lost"); if let Err(e) = self.db.delete_node(uid) { warn!(%uid, error = %e, "db delete_node failed for lost local node"); } } } /// Whether `ino`'s listing is already in memory, i.e. whether /// [`Core::ensure_children`] would return without touching the network. /// Lets a handler decide between answering inline and handing off to a /// worker, at the cost of one uncontended map lookup. fn children_cached(&self, ino: u64) -> bool { self.state().children.contains_key(&ino) } /// Re-apply the optimistic size of any queued write to `nodes`. /// /// A node that arrives from the remote (or from its DB row) carries the size /// of the revision the *server* holds, which for a file with a write still /// queued is the pre-write size — often 0 for a file created moments ago. /// Interning it as-is silently reverts the optimistic size that /// `record_pending_write` stamped, and a file that stats as 0 bytes is a file /// the kernel will not issue a single `read` for: `cat` prints nothing and /// the staged blob that `read_range` would have served is never asked for. /// That reads as data loss even though nothing is lost (B11). /// /// [`Core::hydrate`] does the same thing for the restart case. This covers /// every *live* re-enumeration — which is what a rename or move triggers, /// since both invalidate the listings they touch. /// /// Snapshots the pending map and returns before any caller takes the state /// lock: no site in the daemon holds `pending` and `state` at once, and this /// is not the place to become the first. fn stamp_pending_sizes(&self, nodes: &mut [Node]) { let sizes: HashMap = { let pending = self.pending.lock(); if pending.is_empty() { return; } pending .iter() .map(|(uid, pr)| (uid.clone(), pr.meta.len)) .collect() }; apply_pending_sizes(nodes, &sizes); } /// Read the inputs for virtual-root publication without changing SQLite. /// /// The caller validates the listing snapshot after these reads. Keeping all /// writes out of this phase prevents an invalidated/repopulated root listing /// from changing the synthetic node's persisted visibility or FTS ancestry. fn prepare_virtual_root(&self, real_names: &HashSet) -> Result { let pinned = self.db.state_str(SHARED_WITH_ME_NAME).map_err(|error| { error!(%error, "reading shared-root display name failed"); Errno::EIO })?; let (name, visible) = virtual_root_name(real_names, pinned.as_deref()); let uid = shared_with_me_uid(); let previous_node = self.db.node_by_uid(&uid.to_string()).map_err(|error| { error!(%error, "reading synthetic shared root failed"); Errno::EIO })?; let mut node = previous_node.unwrap_or_else(|| { virtual_node(self.primary_root_uid.clone(), name.clone(), now_secs()) }); node.parent_uid = Some(self.primary_root_uid.clone()); node.name = name; node.kind = NodeKind::Folder; // A later real-name collision suppresses both the dentry and its search // hit. The row and pinned name remain, and clearing the collision // restores both by writing `trashed = false` again. node.trashed = !visible; node.membership = None; Ok(VirtualRootPlan { node, visible }) } /// Reconcile the synthetic dentry against an already-known primary-root /// listing. The node remains persisted and pinned even while a real folder /// of the same name temporarily suppresses its dentry. fn reconcile_virtual_root_dentry( &self, parent_ino: u64, snapshot: RootListingSnapshot, ) -> Result<(), Errno> { let _publication = self.shared_publication.lock(); let plan = self.prepare_virtual_root(&snapshot.real_names())?; let mut st = self.state(); // State methods already establish the state -> DB lock order. No path // holds the DB connection while acquiring state, including shared event // publication, so keeping state locked across this one transaction // closes the crash window without introducing an inverse order. publish_virtual_root_in_listing(&self.db, &mut st, parent_ino, &snapshot, plan)?; drop(st); self.flush_access_changes(); Ok(()) } fn resident_root_listing_snapshot(&self, ino: u64) -> Result { RootListingSnapshot::capture(&self.state(), ino).ok_or(Errno::EAGAIN) } fn shared_folder_freshness_key(uid: &NodeUid) -> String { format!("{SHARED_FOLDER_SYNCED_PREFIX}{uid}") } fn shared_listing_stale(&self, key: &str) -> bool { if self .shared_refresh_deadlines .lock() .is_fresh(key, Instant::now()) { return false; } self.listing_stale(key, SHARED_LISTING_TTL) } fn mark_shared_refresh_success(&self, key: &str) { self.shared_refresh_deadlines .lock() .mark(key, Instant::now(), SHARED_LISTING_TTL); if let Err(error) = self.db.set_state_i64(key, now_millis()) { warn!(key, %error, "persisting shared-list freshness failed"); } } fn invalidate_shared_refreshes(&self) { self.shared_generation.fetch_add(1, Ordering::SeqCst); self.shared_refresh_deadlines.lock().clear(); } fn is_own_or_virtual(&self, uid: &NodeUid) -> bool { is_own_or_virtual_uid(uid, &self.primary_root_uid.volume_id) } fn ensure_shared_children(&self, ino: u64) -> Result<(), Errno> { let online = self.online.load(Ordering::Relaxed); let resident = self.state().children.contains_key(&ino); match shared_listing_plan( resident, online, self.shared_listing_stale(SHARED_WITH_ME_SYNCED_MS), ) { SharedListingPlan::Resident => return Ok(()), SharedListingPlan::Persisted => { // A completed snapshot remains useful offline even after an // event expired its freshness stamp. let nodes = self .db .visible_children(&shared_with_me_uid()) .map_err(|error| { error!(%error, "loading persisted shared roots failed"); Errno::EIO })?; let mut st = self.state(); if st.children.contains_key(&ino) { return Ok(()); } let children = nodes .into_iter() .map(|node| { let access = st .share_access .get(&node.uid) .copied() .unwrap_or(Access::Viewer); st.intern_published_share_root(ino, node, access) }) .collect(); st.children.insert(ino, children); drop(st); self.flush_access_changes(); return Ok(()); } SharedListingPlan::Refresh => {} } let generation = self.shared_generation.load(Ordering::SeqCst); let accepted = accepted_share_provenance( self.rt .block_on(self.client.enumerate_shared_with_me()) .map_err(|error| { error!(%error, "enumerating shared roots failed"); Errno::EIO })?, ); let nodes = if accepted.uids.is_empty() { Vec::new() } else { self.rt .block_on(self.drive.enumerate_nodes_light(&accepted.uids)) .map_err(|error| { error!(%error, "materializing shared roots failed"); Errno::EIO })? }; let virtual_uid = shared_with_me_uid(); let published = prepare_shared_roots(&accepted, nodes, &virtual_uid); let _publication = self.shared_publication.lock(); if !refresh_generation_is_current(generation, self.shared_generation.load(Ordering::SeqCst)) { return Err(Errno::EAGAIN); } let removed = match self .db .publish_shared_roots(&virtual_uid, &accepted.uids, &published) { Ok(removed) => removed, Err(error) => { error!(%error, "publishing shared-root listing failed"); let mut st = self.state(); for uid in &accepted.uids { st.downgrade_shared_subtree(uid); } drop(st); self.flush_access_changes(); return Err(Errno::EIO); } }; let snapshot = self.db.visible_children(&virtual_uid).map_err(|error| { error!(%error, "loading published shared-root listing failed"); Errno::EIO })?; let published_access: HashMap = published .iter() .map(|root| (root.node.uid.clone(), root.access)) .collect(); let mut st = self.state(); for uid in &removed { st.hide_shared_root(uid); } let mut children = Vec::with_capacity(snapshot.len()); for node in snapshot { let uid = node.uid.clone(); let access = published_access .get(&uid) .copied() .unwrap_or(Access::Viewer); let child = st.intern_published_share_root(ino, node, access); children.push(child); } st.children.insert(ino, children); drop(st); self.flush_access_changes(); self.mark_shared_refresh_success(SHARED_WITH_ME_SYNCED_MS); if let Some(notifier) = self.notifier.get() { let _ = notifier.inval_inode(ino); } Ok(()) } /// The nodes of `before`, queued in `folder_uid` when its listing was asked /// of Drive, that are no longer queued in `after`, as the DB has them now. /// /// Their create landed while the listing was on the wire, after Drive took /// it, and so is in neither. One unlinked or moved away meanwhile is not /// under the folder by that name any more and stays out. fn landed_meanwhile( &self, folder_uid: &NodeUid, before: Vec, after: &[Node], ) -> Vec { let still: HashSet<&str> = after.iter().map(|node| node.name.as_str()).collect(); let landed: HashSet = before .into_iter() .map(|node| node.name) .filter(|name| !still.contains(name.as_str())) .collect(); if landed.is_empty() { return Vec::new(); } match self.db.known_children(folder_uid) { Ok(known) => known .into_iter() .filter(|node| landed.contains(&node.name)) .collect(), Err(e) => { warn!(%folder_uid, error = %e, "db known_children failed"); Vec::new() } } } /// Serve a listing without the network: the one already in memory if any, /// else whatever children the DB still knows. The fallback only answers /// until the link is back, when [`Core::relist_offline_listings`] drops it: /// a listing dropped as stale may lack a child another device made, and a /// create over that name lands as a conflict copy rather than an `EIO` now. fn adopt_stale_listing( &self, ino: u64, folder_uid: &NodeUid, cached: bool, primary_root: bool, ) -> Result<(), Errno> { if cached { return Ok(()); } let nodes = self.db.known_children(folder_uid).map_err(|e| { warn!(%folder_uid, error = %e, "db known_children failed"); Errno::EIO })?; debug!(%folder_uid, count = nodes.len(), "offline: listing a folder from what the DB knows"); self.link.stale_listings.lock().insert(ino); self.adopt_db_listing(ino, folder_uid, nodes, primary_root) } /// Relist every folder [`Core::adopt_stale_listing`] served while offline. pub(crate) fn relist_offline_listings(&self) { let stale = std::mem::take(&mut *self.link.stale_listings.lock()); if stale.is_empty() { return; } let mut st = self.state(); for &ino in &stale { st.invalidate_listing(ino); } drop(st); for mount in self.states.live() { if let Some(notifier) = mount.notifier.get() { for &ino in &stale { let _ = notifier.inval_inode(ino); } } } } /// Install `nodes`, read from the DB, as `ino`'s listing. fn adopt_db_listing( &self, ino: u64, folder_uid: &NodeUid, mut nodes: Vec, primary_root: bool, ) -> Result<(), Errno> { let folder_uid = folder_uid.clone(); if primary_root { nodes.retain(|node| !is_virtual_uid(&node.uid)); } // Before the lock: a DB row carries the size the server last // sealed, which a queued write is ahead of (B11). self.stamp_pending_sizes(&mut nodes); let hidden = self.hidden.lock().clone(); let mut st = self.state(); if st.children.contains_key(&ino) { return Ok(()); } let mut child_inos = Vec::with_capacity(nodes.len()); let mut needs_size = Vec::new(); for node in nodes { if !node_visible(&node, &folder_uid, &hidden) { continue; } if matches!( &node.kind, NodeKind::File { claimed_size: None, .. } ) { needs_size.push(node.uid.clone()); } child_inos.push(st.intern_from_db(ino, node)); } st.children.insert(ino, child_inos); let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap()); drop(st); self.flush_access_changes(); if let Some(snapshot) = root_snapshot { self.reconcile_virtual_root_dentry(ino, snapshot)?; } // Rows persisted from a cheap enumeration whose upgrade never // ran (a restart in between, say) still owe their real sizes. self.spawn_size_upgrade(ino, needs_size); Ok(()) } /// Enumerate `ino`'s children from the remote and cache them. No-op if the /// directory has already been listed. Network I/O happens without the lock /// held so concurrent metadata reads aren't blocked behind a fetch. /// /// A listing that changed while it was being published answers EAGAIN: the /// root listed again after a restart, an event, a newer refresh of a shared /// folder. The listing is still owed, so it is taken again rather than /// handed to the lookup or open that asked (B159). fn ensure_children(&self, ino: u64) -> Result<(), Errno> { for _ in 1..LISTING_ATTEMPTS { match self.list_children(ino) { Err(e) if e.code() == Errno::EAGAIN.code() => { debug!(ino, "listing changed while it was published; listing again"); } done => return done, } } self.list_children(ino) } fn list_children(&self, ino: u64) -> Result<(), Errno> { let (folder_uid, cached) = { let st = self.state(); match st.entries.get(&ino) { Some(e) => (e.uid.clone(), st.children.contains_key(&ino)), None => return Err(Errno::ENOENT), } }; if is_virtual_uid(&folder_uid) { return self.ensure_shared_children(ino); } let primary_root = is_primary_root_listing(self.primary, &folder_uid, &self.primary_root_uid); let foreign = !self.is_own_or_virtual(&folder_uid) && !is_local_uid(&folder_uid); let foreign_key = foreign.then(|| Self::shared_folder_freshness_key(&folder_uid)); let refresh_foreign = foreign_key.as_deref().is_some_and(|key| { listing_needs_refresh( self.online.load(Ordering::Relaxed), self.shared_listing_stale(key), ) }); if cached && !refresh_foreign { if primary_root { self.reconcile_virtual_root_dentry(ino, self.resident_root_listing_snapshot(ino)?)?; } return Ok(()); } // Offline fast path: a folder the DB still records as fully enumerated // can be rebuilt from disk without hitting the API, even if its listing // was trimmed from the hot cache mid-run. let cached_nodes = if refresh_foreign { Ok(None) } else { self.db.children_if_listed(&folder_uid) }; match cached_nodes { Ok(Some(nodes)) => return self.adopt_db_listing(ino, &folder_uid, nodes, primary_root), Ok(None) => {} Err(e) => warn!(%folder_uid, error = %e, "db children_if_listed failed"), } // Drive knows nothing by a stand-in. Until its mkdir lands the folder // is only here, and once it has, the DB holds what Drive does. A // listing dropped meanwhile was asked of Drive, and every lookup in // the folder failed with EIO (`docs/BUGS.md` B125). if is_local_uid(&folder_uid) { let nodes = self.db.known_children(&folder_uid).map_err(|e| { warn!(%folder_uid, error = %e, "db known_children failed"); Errno::EIO })?; return self.adopt_db_listing(ino, &folder_uid, nodes, primary_root); } if !self.is_online() { return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root); } let refresh_generation = refresh_foreign.then(|| self.shared_generation.load(Ordering::SeqCst)); // What is queued before Drive is asked: a create that lands while the // listing is on the wire is in neither Drive's answer nor the queue read // after it (`docs/BUGS.md` B166). let queued_before = (!foreign) .then(|| self.db.queued_children(&folder_uid).ok()) .flatten(); let uids = match self .rt .block_on(self.drive.enumerate_folder_children_node_uids(&folder_uid)) { Ok(uids) => uids, Err(e) if self.lost_link(&e, "list folder") => { return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root); } Err(e) => { error!(%folder_uid, error = %e, "enumerate folder children failed"); return Err(Errno::EIO); } }; // Cheap enumeration: `Light` skips unlocking each *file's* node key, // which is an S2K derivation per file and was ~74% of the cost of a cold // listing (B12 — measured with `perf`, 64% of cycles in SHA-256 alone). // Folders are unlocked either way; their keys are what the children are // decrypted with, so the walk cannot proceed without them. // // The price is that files come back without a `claimed_size`, so // `node_size` falls back to the *ciphertext* size until // `spawn_size_upgrade` below fills the real one in. let mut nodes = match self.rt.block_on(self.drive.enumerate_nodes_light(&uids)) { Ok(nodes) => nodes, Err(e) if self.lost_link(&e, "list folder") => { return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root); } Err(e) => { error!(%folder_uid, error = %e, "enumerate nodes failed"); return Err(Errno::EIO); } }; // Drive shows none of what is still queued for the folder (B132). // A shared folder's listing is published as Drive has it. if !foreign { match self.db.queued_children(&folder_uid) { Ok((mut queued, gone)) => { if let Some((before, _)) = queued_before { queued.extend(self.landed_meanwhile(&folder_uid, before, &queued)); } overlay_queued(&mut nodes, queued, &gone) } Err(e) => warn!(%folder_uid, error = %e, "db queued_children failed"), } } // Same as the DB path above: the remote's size for a file with a write // still queued is the pre-write one (B11). self.stamp_pending_sizes(&mut nodes); let hidden = self.hidden.lock().clone(); let mut filtered_nodes: Vec = nodes .into_iter() .filter(|node| node_visible(node, &folder_uid, &hidden)) .collect(); if foreign { for node in &mut filtered_nodes { node.parent_uid = Some(folder_uid.clone()); } let _publication = self.shared_publication.lock(); if !refresh_generation_is_current( refresh_generation.expect("foreign refresh has a generation"), self.shared_generation.load(Ordering::SeqCst), ) { return Err(Errno::EAGAIN); } let removed = self .db .publish_foreign_children(&folder_uid, &uids, &filtered_nodes) .map_err(|error| { error!(%folder_uid, %error, "publishing foreign-folder listing failed"); Errno::EIO })?; let mut snapshot = self.db.visible_children(&folder_uid).map_err(|error| { error!(%folder_uid, %error, "loading published foreign-folder listing failed"); Errno::EIO })?; self.stamp_pending_sizes(&mut snapshot); snapshot.retain(|node| node_visible(node, &folder_uid, &hidden)); let needs_size: Vec = snapshot .iter() .filter(|node| { matches!( &node.kind, NodeKind::File { claimed_size: None, .. } ) }) .map(|node| node.uid.clone()) .collect(); let mut st = self.state(); for uid in &removed { st.hide_foreign_subtree(uid); } let children = snapshot .into_iter() .map(|node| st.intern_from_db(ino, node)) .collect(); st.children.insert(ino, children); drop(st); self.flush_access_changes(); self.mark_shared_refresh_success( foreign_key .as_deref() .expect("foreign refresh has a freshness key"), ); self.spawn_size_upgrade(ino, needs_size); return Ok(()); } let mut st = self.state(); // Lost the race? Another thread already populated it. if st.children.contains_key(&ino) { return Ok(()); } let mut child_inos = Vec::with_capacity(filtered_nodes.len()); // Files whose real size the cheap enumeration could not read. Collected // before interning so the upgrade below has the uids without re-walking. let needs_size: Vec = filtered_nodes .iter() .filter(|n| { matches!( &n.kind, NodeKind::File { claimed_size: None, .. } ) }) .map(|n| n.uid.clone()) .collect(); let inos = st.intern_batch(ino, filtered_nodes); child_inos.extend(inos); st.children.insert(ino, child_inos); let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap()); // Record the listing as complete so a later restart (or a trimmed hot // cache) can rebuild it from the DB without the API. if let Err(e) = self.db.set_listed(&folder_uid, true) { warn!(%folder_uid, error = %e, "db set_listed(true) failed"); } drop(st); self.flush_access_changes(); if let Some(snapshot) = root_snapshot { self.reconcile_virtual_root_dentry(ino, snapshot)?; } self.spawn_size_upgrade(ino, needs_size); Ok(()) } /// Resolve the real size of every file in `parent`'s listing still carrying /// a provisional one, returning when they are in `state`. /// /// Called when a `stat` lands on such a file. That covers the paths /// [`Core::ensure_children`] cannot: a listing rebuilt from the DB, or one /// restored by [`Core::hydrate`] on mount, whose rows were persisted before /// an earlier upgrade had a chance to run. Gathers the whole folder rather /// than the one file asked for, because a `stat` of one entry in a listing /// almost always means a `stat` of all of them. fn upgrade_sizes_for_parent( &self, ino: u64, uid: &NodeUid, parent: u64, ) -> Option> { let provisional = |e: &Entry| { matches!( &e.node.kind, NodeKind::File { claimed_size: None, .. } ) }; let (key, mut missing): (u64, Vec) = { let st = self.state(); match st.children.get(&parent) { // The listing is resident: batch the whole folder under its // inode, so the rest of an `ls -l` rides along on this fetch. Some(kids) => ( parent, kids.iter() .filter_map(|k| st.entries.get(k)) .filter(|e| provisional(e)) .map(|e| e.uid.clone()) .collect(), ), // It is not, and returning here is what let a provisional size // reach the caller anyway: a rename invalidates its parents' // listings, so a freshly renamed file always landed in this // branch. Resolve the single node instead, keyed by its own // inode — folder and file inodes share one space, so the two // single-flight keys cannot collide. (Same shape as B4: an // early return that assumed the hot cache was authoritative.) None => (ino, Vec::new()), } }; // The node that was actually asked about is never optional, whichever // branch produced the batch. if !missing.iter().any(|u| u == uid) { missing.push(uid.clone()); } // Returned, not awaited here: the caller is a `getattr` that must not // answer with a provisional size (bugs.md B14), and it parks its reply // on this batch rather than holding a thread until it lands. self.upgrade_sizes(key, missing) } /// Fill in the true sizes of files a `Light` enumeration returned without /// one, on a worker, after the listing has already been served. /// /// This is the other half of the split in [`Core::ensure_children`]. The /// listing itself needs only names and parentage, so it is served from the /// cheap enumeration immediately; the S2K-per-file work that produces /// `claimed_size` happens here, off the path the user is waiting on. /// /// **Sizes are provisional until this lands.** `node_size` falls back to /// `total_size_on_storage`, the *ciphertext* size, which is slightly larger /// than the real one. A read must not plan on it: since B84 a size the /// revision disagrees with fails the read, so `read` settles the size first /// ([`Core::resolve_provisional_size`], bugs.md B100). In `stat` it is a /// discrepancy that closes within a round trip, not a repeat of B11 (which /// reported **0** and so suppressed reads entirely). /// /// Single-flight per folder: a `stat` of every entry in a fresh listing is /// the normal case, and each one must not start its own upgrade. fn spawn_size_upgrade(&self, folder_ino: u64, uids: Vec) { self.upgrade_sizes(folder_ino, uids); } /// Start (or join) the size upgrade for `key`, returning the batch a caller /// can wait on — `None` when there was nothing to upgrade. /// /// Single-flight per `key`. The fetch runs on its own thread rather than on /// the [`Workers`] pool: callers wait on `Lane::Meta`, so a batch queued onto /// that lane could have a wide enough `ls -l` fill it with threads waiting /// for a job that can never be scheduled. `Lane::Transfer` would swap that /// for starvation behind bulk reads. One short-lived thread per folder, /// bounded by the single-flight, avoids both. fn upgrade_sizes(&self, key: u64, uids: Vec) -> Option> { if uids.is_empty() { return None; } let claim = claim_size_upgrade(&mut self.size_upgrades.lock(), key, uids); match claim { SizeUpgradeClaim::Joined(slot) | SizeUpgradeClaim::Queued(slot) => Some(slot), SizeUpgradeClaim::Start(slot, uids) => { let core = self.clone(); let worker = slot.clone(); std::thread::spawn(move || core.run_size_upgrade(key, uids, worker)); Some(slot) } } } /// Hold `respond` back until `ino` has a real size, `slot`'s batch ends, or /// [`SizeUpgrade::WAIT`] elapses — then run it, on the queue's thread. /// /// The caller returns immediately. That is the whole point: the thread it /// was running on is a FUSE dispatch thread or a [`Workers`] one, and both /// are needed by everything else on the mount (audit F4). fn await_size( &self, ino: u64, slot: Arc, respond: impl FnOnce() + Send + 'static, ) { let core = self.clone(); let waiter = SizeWaiter { slot, resolved: Box::new(move || core.size_is_real(ino)), deadline: Instant::now() + SizeUpgrade::WAIT, respond: Box::new(respond), }; if !self.size_waiters.park(waiter) { return; } let queue = self.size_waiters.clone(); if let Err(error) = std::thread::Builder::new() .name("pdfs-size-waiters".into()) .spawn(move || queue.serve()) { // Without a thread nothing would ever answer these. A provisional // size is wrong; a `stat` that never returns wedges the caller. warn!(%error, "no thread for parked size waiters; answering provisionally"); self.size_waiters.answer_all(); } } /// Fetch `uids` in chunks, applying and announcing each one as it lands. /// /// Chunked so a waiter is released as soon as *its* file is resolved. A /// single 793-node batch took ~80 s, which outran [`SizeUpgrade::WAIT`] and /// put provisional sizes back in front of callers — the bug this was /// supposed to fix (bugs.md B14). /// /// When the batch ends, the thread takes over the oldest batch the cap held /// back, so at most [`MAX_SIZE_UPGRADES`] threads run however many folders /// a recursive walk touches (bugs.md B100). fn run_size_upgrade(&self, key: u64, uids: Vec, slot: Arc) { let (mut key, mut uids, mut slot) = (key, uids, slot); loop { for chunk in uids.chunks(SIZE_UPGRADE_CHUNK) { let result = self.rt.block_on(self.drive.enumerate_nodes(chunk)); self.apply_size_upgrade(key, result); slot.chunk_done(); } // Both of these must happen however the loop ended: a folder whose // upgrade failed has to be retryable, and its waiters released. let next = next_size_upgrade(&mut self.size_upgrades.lock(), key); slot.finish(); let Some(next) = next else { return; }; (key, uids, slot) = next; } } /// Settle the size of a file whose listing only carries the provisional /// one, before a read plans its blocks on it or a write handle starts from /// it. Returns the real size, or `None` when the fetch did not produce one. /// /// A provisional size is the ciphertext size. The read path checks the /// size it is given against the revision's own block table and fails the /// read on a mismatch (B84), so reading on the provisional size is an `EIO` /// every time. `getattr` usually settles the size first, but not when its /// wait times out, so the read cannot rely on it (bugs.md B100). fn resolve_provisional_size(&self, ino: u64, uid: &NodeUid) -> Option { let parent = self.state().entries.get(&ino)?.parent; // Without a network the answer is `None` either way; the bound keeps a // dead link from holding the open or read for the SDK's two minutes. if !self.is_online() { return None; } let result = self.block_on_bounded(self.drive.enumerate_nodes(std::slice::from_ref(uid))); if let Err(e) = &result { self.lost_link(e, "settle a provisional size"); } self.apply_size_upgrade(parent, result); let st = self.state(); match &st.entries.get(&ino)?.node.kind { NodeKind::File { claimed_size: Some(size), .. } => Some((*size).max(0) as u64), _ => None, } } /// Wait, up to [`SizeUpgrade::WAIT`] as a `stat` would, for the real size /// of every file in `ino`'s listing that still carries a provisional one. /// /// For callers that read a whole listing's sizes straight from `state` and /// do not run on a FUSE thread: `pdfs ls` and the app's file browser. A /// listing just fetched has the ciphertext size of every file in it until /// its upgrade lands, and they showed those (`docs/BUGS.md` B115). fn settle_listing_sizes(&self, ino: u64) { // Offline the fetch cannot succeed; the wait would only delay the // answer it ends up giving anyway. if !self.is_online() { return; } let missing: Vec = { let st = self.state(); st.children .get(&ino) .map(|kids| { kids.iter() .filter_map(|k| st.entries.get(k)) .filter(|e| { !is_local_uid(&e.uid) && matches!( &e.node.kind, NodeKind::File { claimed_size: None, .. } ) }) .map(|e| e.uid.clone()) .collect() }) .unwrap_or_default() }; if let Some(slot) = self.upgrade_sizes(ino, missing) { slot.wait_finished(Instant::now() + SizeUpgrade::WAIT); } } /// Whether `ino` has a real size — the condition a waiter is waiting on. /// A node that vanished counts as resolved; there is nothing left to wait /// for, and its caller will find the `ENOENT` for itself. fn size_is_real(&self, ino: u64) -> bool { let st = self.state(); st.entries.get(&ino).is_none_or(|e| { !matches!( &e.node.kind, NodeKind::File { claimed_size: None, .. } ) }) } /// Adopt the sizes a completed upgrade fetched. Split from /// [`Core::upgrade_sizes`] so the single-flight bookkeeping there has one /// exit path rather than one per early return. fn apply_size_upgrade(&self, folder_ino: u64, result: Result, ProtonError>) { let core = self; { let mut nodes = match result { Ok(nodes) => nodes, Err(e) => { // Not fatal: the listing stands, sizes just stay provisional // until something invalidates and re-enumerates it. warn!(folder_ino, error = %e, "size upgrade failed; sizes stay provisional"); return; } }; // A queued write is ahead of anything the server can report, so its // optimistic size must survive this just as it survives a re-listing // (B11). core.stamp_pending_sizes(&mut nodes); let mut changed: Vec = Vec::new(); let mut st = core.state(); for node in nodes { // Only adopt the size. Re-interning wholesale would also adopt a // name or parent that a rename/move may have changed locally // while this was in flight, undoing it. let Some(&ino) = st.by_uid.get(&node.uid) else { continue; }; changed.push(ino); // An open write handle is ahead of the server too (B109). if st.active_writes.contains_key(&ino) { continue; } let NodeKind::File { claimed_size, .. } = &node.kind else { continue; }; let (Some(size), Some(entry)) = (*claimed_size, st.entries.get_mut(&ino)) else { continue; }; if let NodeKind::File { claimed_size, .. } = &mut entry.node.kind { *claimed_size = Some(size); } } let updated: Vec = st .children .get(&folder_ino) .map(|kids| { kids.iter() .filter_map(|k| st.entries.get(k).map(|e| e.node.clone())) .collect() }) .unwrap_or_default(); drop(st); if let Err(e) = core.db.upsert_nodes(&updated) { warn!(folder_ino, error = %e, "persisting upgraded sizes failed"); } // Without this the corrected size is invisible for the length of the // attr TTL: the kernel answers `stat` from the provisional attrs it // cached while the listing was being served, so `ls -l` reports the // ciphertext size for up to 30 s even though the daemon has had the // real one all along. Notify *after* the DB write, so a re-`getattr` // provoked by the invalidation cannot race the persistence. // // Attributes only (offset -1): the content did not change, and a // page-cache invalidation deadlocks when this runs inside a `read` // (B100 resolves sizes there). The kernel holds the page lock of // that read until we reply, and the invalidation waits for the // same lock, so the read never answered (docs/BUGS.md B110). for mount in core.states.live() { if let Some(notifier) = mount.notifier.get() { for &ino in &changed { let _ = notifier.inval_attrs(ino); } } } debug!(folder_ino, files = updated.len(), "filled in listing sizes"); } } /// Resolve a child `name` within `parent` to its `(inode, uid)`, ensuring /// the parent's listing is cached first. fn lookup_child(&self, parent: u64, name: &str) -> Result<(u64, NodeUid), Errno> { self.ensure_children(parent)?; let st = self.state(); st.children .get(&parent) .and_then(|kids| { kids.iter().copied().find_map(|ino| { st.entries .get(&ino) .filter(|e| e.node.name == name) .map(|e| (ino, e.uid.clone())) }) }) .ok_or(Errno::ENOENT) } /// Walk a mountpoint-relative path to its `(inode, uid)`, enumerating each /// directory on the way as needed. Leading `/` and `.` components are /// ignored; `..` is rejected. fn resolve_path(&self, rel: &Path) -> Result<(u64, NodeUid), Errno> { let mut ino = self.root_ino(); let mut uid = { let st = self.state(); st.entries .get(&ino) .map(|e| e.uid.clone()) .ok_or(Errno::ENOENT)? }; for comp in rel.components() { match comp { Component::RootDir | Component::CurDir => continue, Component::Normal(name) => { let (child_ino, child_uid) = self.lookup_child(ino, &name.to_string_lossy())?; ino = child_ino; uid = child_uid; } _ => return Err(Errno::EINVAL), } } Ok((ino, uid)) } /// [`resolve_path`](Self::resolve_path) for the request-serving side. /// /// The kernel-facing resolver answers in `Errno` because that is what the /// FUSE reply needs. A front-end needs the opposite: `{:?}` of a libc /// constant is not something to show a user, and "does not exist" and "the /// API is down" want different copy and different buttons. This is the one /// place that knows enough to tell them apart, so it is the place that does. fn resolve(&self, rel: &Path) -> CoreResult<(u64, NodeUid)> { self.resolve_path(rel) .map_err(|e| self.errno_error(e, &format!("could not resolve {}", rel.display()))) } /// Resolve a wire-format uid only when one of this daemon's locations /// proves that it owns the node. /// /// The primary and on-demand mounts prove residency through the tree and /// the roots in [`StateRegistry`]. Mirror locations have no tree, so their root /// and last-synced descendants are resolved through `sync_folder` and /// `sync_entry` instead. pub(crate) fn resolve_anywhere(&self, uid: &str) -> CoreResult { resolve_anywhere_with( uid, |uid| self.owns_visible_uid(uid), |uid| self.db.mirror_contains_uid(&uid.to_string()), ) } fn source_parent_uid(&self, ino: u64, rel: &Path) -> CoreResult { let state = self.state(); let entry = state .entries .get(&ino) .ok_or_else(|| CoreError::not_found(format!("could not resolve {}", rel.display())))?; entry .node .parent_uid .clone() .or_else(|| { state .entries .get(&entry.parent) .map(|parent| parent.uid.clone()) }) .ok_or_else(|| CoreError::invalid("the mount root cannot be mutated")) } /// Classify a failure that arrived as an `Errno`. /// /// The internal paths speak `Errno` because they also serve FUSE, where a /// number is the whole vocabulary. Everything crossing the control socket /// has to be turned back into something a person can read, and this is the /// one place that knows how — a call site holding only an `Errno` has /// already lost the context needed to say what went wrong. /// /// `Errno` is neither `PartialEq` nor structural-match, so this compares /// raw codes rather than the `libc` constants. fn errno_error(&self, e: Errno, what: &str) -> CoreError { match e.code() { libc::ENOENT => CoreError::not_found(format!("{what}: no such file or folder")), libc::EACCES | libc::EPERM => CoreError::denied(format!("{what}: not allowed")), // These paths walk the tree lazily, so a cold node needs the API. // Offline that surfaces as EIO, which on its own would read to the // user as a broken file rather than a missing network. libc::EIO if !self.online.load(Ordering::Relaxed) => CoreError::offline(), libc::EINVAL => CoreError::invalid(format!("{what}: not a usable path")), libc::ENOSPC => CoreError::new(ErrorKind::Quota, format!("{what}: out of space")), libc::EEXIST => CoreError::conflict(format!("{what}: already exists")), libc::ENOTEMPTY => CoreError::conflict(format!("{what}: folder is not empty")), _ => CoreError::internal(format!("{what}: {e:?}")), } } /// Fetch a single node's current metadata from the remote. fn fetch_node(&self, uid: &NodeUid) -> Result { match self.fetch_node_remote(uid) { Ok(Some(node)) => Ok(node), Ok(None) => Err(Errno::ENOENT), Err(e) => { if !self.lost_link(&e, "fetch node") { error!(%uid, error = %e, "enumerate node failed"); } Err(Errno::EIO) } } } /// Read back a node just created on the remote. /// /// Drive can answer a create before a lookup lists the new node, so an /// empty answer is asked again for a moment. `None` means the node exists /// but could not be read back. The mount then queues the create, and the /// drain finds the node by name and adopts it: the application gets its /// file or folder either way, never an `ENOENT` for something that exists. fn fetch_minted_node(&self, uid: &NodeUid, name: &str) -> Option { // The feed echoes the create. Unclaimed, it would take the echo the // upload of the file's bytes records, and that one would then evict // the bytes just uploaded (docs/BUGS.md B123). self.note_self_change(uid); let node = self.read_back(uid).ok(); if node.is_none() { warn!(%uid, name, "Drive made the node but did not hand it back"); } node } /// [`Core::fetch_node`] for a node Drive has just made, asked again for a /// moment while Drive does not list it yet. /// /// A file Drive lists without an active revision is not listed yet either: /// the create commits one before it answers, so this answer is from before /// the commit, with no size. Taken, it emptied the new file in the tree, /// and a read of it right after found nothing (`docs/BUGS.md` B163). pub(crate) fn read_back(&self, uid: &NodeUid) -> Result { let mut delays = MINTED_READ_BACK_DELAYS.iter(); loop { let fetched = match self.fetch_node(uid) { Ok(node) if is_unrevised(&node) => Err(Errno::ENOENT), fetched => fetched, }; match fetched { Err(e) if e == Errno::ENOENT && let Some(delay) = delays.next() => { std::thread::sleep(*delay); } fetched => return fetched, } } } /// [`Core::fetch_node`] without the collapse to an `Errno`, for the drain: /// resolving a conflict turns on *why* a call failed, and "the node is not /// there" (`Ok(None)`) is a different outcome from "we could not ask". fn fetch_node_remote(&self, uid: &NodeUid) -> Result, ProtonError> { match self.block_on_bounded(self.drive.enumerate_nodes(std::slice::from_ref(uid))) { Ok(nodes) => Ok(nodes.into_iter().next()), // An unknown uid is reported either as an empty result or as an // outright refusal, depending on the endpoint. Err(e) if is_gone(&e) => Ok(None), Err(e) => Err(e), } } /// Point a handle closed while its file's create landed at the uid Drive /// gave the node. /// /// `Core::retire_create` moves the row to that uid before it repoints the /// open handles. A handle closed in between still named the placeholder, /// which no row has any more, and its write was refused as if access had /// changed. The inode is the row's local id, which finds it either way. pub(crate) fn follow_landed_create(&self, h: &mut WriteHandle) { if !is_local_uid(&h.uid) { return; } if let Some(real) = self.landed_row_uid(h.ino) { debug!(local = %h.uid, %real, "write handle closed as its create landed"); h.uid = real; } } /// The uid Drive gave the node at `ino`, once its create has landed, or /// `None` while it is still queued. Read from the row, which has it before /// the tree does. pub(crate) fn landed_row_uid(&self, ino: u64) -> Option { state::ino_lid(ino).and_then(|lid| self.landed_lid_uid(lid)) } /// [`Core::landed_row_uid`] for a node known by its stand-in `uid`. pub(crate) fn landed_stand_in_uid(&self, uid: &NodeUid) -> Option { pdfs_core::db::local_lid(&uid.to_string()).and_then(|lid| self.landed_lid_uid(lid)) } fn landed_lid_uid(&self, lid: i64) -> Option { self.db .uid_of_lid(lid) .ok() .flatten() .and_then(|uid| parse_node_uid(&uid)) .filter(|uid| !is_local_uid(uid)) } /// Accept a released write handle's bytes and queue their upload /// (offline.md Phase 3). /// /// This is what makes a copy into the mount run at disk speed: the caller's /// `close` returns once the bytes are staged on local disk and the intent is /// in `pending_op`, instead of waiting out a full upload inside the FUSE /// handler. It is also what makes an offline write succeed rather than EIO — /// the queued op simply waits for the network. /// /// The scratch file is *moved* into staging, never copied: it is the only /// copy of what the user wrote. fn queue_revision(&self, h: &mut WriteHandle) -> Result<(), Errno> { // Closing an untouched writable handle is not a mutation. It remains // valid after a downgrade and only owes cleanup of its empty scratch. if !h.dirty { self.cache.clear_scratch_durable(&h.path); let _ = std::fs::remove_file(&h.path); return Ok(()); } preserve_on_access_denied(self.require_uid_writable(&h.uid), h.dirty, || { let meta = self.recovery_meta(h); self.stage_orphaned_write( &h.uid, h.ino, &h.path, &meta, "access changed before the write could be queued", ); })?; // The handle is being retired either way, so any durability sidecar an // `fsync` left has done its job: from here the bytes are tracked as a // staged write and a queued op, and a sidecar outliving them would offer // recovery a second, stale copy of the same write. self.cache.clear_scratch_durable(&h.path); if is_local_uid(&h.uid) && !self.db.has_create_op(&h.uid.to_string()).unwrap_or(true) { // A create that landed since the handle last looked left no op // either. It moved the row to its uid in the same transaction, so // the row says which it was (`docs/BUGS.md` B176). self.follow_landed_create(h); if is_local_uid(&h.uid) { debug!(uid = %h.uid, "local node was unlinked before creation; dropping revision"); let _ = std::fs::remove_file(&h.path); return Ok(()); } } // Materialize as much of the full content as is free to materialize. A // complete blob is uploadable without the network and lets a later write // to the same file supersede this one without any reconstruction at all. // // What is deliberately *not* done here is fetching the untouched ranges // from the remote: that is a download of everything the edit did not // touch, and this runs on the FUSE dispatch loop, so closing a large // file after a small edit stopped every other request on the mount for // the length of it (audit F1). An incomplete blob is first-class — // `read_pending` resolves its gaps against the base, and `drain_revision` // fills them on the drain thread — so the network half of the fill // belongs there, and only there. // // A node that exists only locally has no remote base to fill from: its // untouched ranges live in the blob queued by an earlier write, which // `merge_over_pending` folds in below. if let Err(e) = h.file.set_len(h.len) { error!(uid = %h.uid, error = %e, "resize scratch file failed"); return Err(Errno::EIO); } let mut written = h.written.clone(); if !(is_local_uid(&h.uid) && h.base_size > 0) { // A local node has no remote base, so nothing of it can be fetched // or cached: passing zero says so, and leaves the fill doing only // the part that still applies — claiming the zeroed tail. let base_size = if is_local_uid(&h.uid) { 0 } else { h.base_size }; self.fill_gaps_cached( &h.uid, &h.file, h.len, h.base_mtime, base_size, &mut written, ); } let authored: Vec<(u64, u64)> = written .segments(0, h.len) .into_iter() .filter(|&(_, _, authored)| authored) .map(|(s, e, _)| (s, e)) .collect(); let meta = StagedWrite { uid: h.uid.to_string(), len: h.len, base_size: h.base_size, base_mtime: h.base_mtime, complete: StagedWrite::covers(&authored, h.len), authored, based_on: self.remote_baseline( &h.uid, h.base_mtime, h.base_size, h.base_revision_id.clone(), ), }; let complete = meta.complete; self.enqueue_staged_write(&h.uid, h.ino, &h.path, meta)?; debug!(uid = %h.uid, len = h.len, complete, "queued revision upload"); Ok(()) } /// Describe the locally authored ranges without gap-filling them. /// /// Used when a write was accepted by the kernel but a queue-time access /// check now denies it. Building this metadata is side-effect free, so the /// permission guard still precedes cache, database, and tree mutation. fn recovery_meta(&self, h: &WriteHandle) -> StagedWrite { let authored: Vec<(u64, u64)> = h .written .segments(0, h.len) .into_iter() .filter(|&(_, _, authored)| authored) .map(|(start, end, _)| (start, end)) .collect(); StagedWrite { uid: h.uid.to_string(), len: h.len, base_size: h.base_size, base_mtime: h.base_mtime, complete: StagedWrite::covers(&authored, h.len), authored, based_on: self.remote_baseline( &h.uid, h.base_mtime, h.base_size, h.base_revision_id.clone(), ), } } /// The remote revision a change to `uid` is being made against, for /// [`StagedWrite::based_on`]. /// /// Normally that is simply the base the handle opened over. The exception is /// a write that supersedes a still-queued one: its "base" is the previous /// *staged blob*, whose size and mtime are ours, not the server's — so the /// baseline is inherited from the op being superseded, which is the last one /// that actually observed the remote. Without that, chaining two writes /// before the queue drains would leave the drain comparing the remote /// against a revision it never had, and cutting a conflict copy over /// nothing. /// /// `None` for a node that has never existed remotely: there is no revision /// to conflict with until its create drains. fn remote_baseline( &self, uid: &NodeUid, base_mtime: i64, base_size: u64, base_revision_id: Option, ) -> Option { if is_local_uid(uid) { return None; } match self.pending.lock().get(uid) { Some(p) => p.meta.based_on.clone(), None => Some(Baseline { mtime: base_mtime, size: base_size, hash: None, revision_id: base_revision_id, }), } } /// Move a file holding a node's intended new content into staging and queue /// the upload that will make it the remote's content. Shared by the release /// of a write handle and by a path-based truncate. /// /// `src` is consumed either way: on success it is *moved* into staging, and /// on the refusal below it is moved there too, just without an op to upload /// it. It is the only copy of what the user wrote, so nothing here may /// simply delete it. fn enqueue_staged_write( &self, uid: &NodeUid, ino: u64, src: &Path, mut meta: StagedWrite, ) -> Result<(), Errno> { preserve_on_access_denied(self.require_uid_writable(uid), true, || { self.stage_orphaned_write( uid, ino, src, &meta, "access changed before the staged write could be queued", ); })?; // An incomplete blob's gaps refer to the *remote* base. If an earlier // write to this file is still queued, the remote no longer holds that // base — the previous staged blob does — so taking those gaps at face // value would fill them from the wrong revision. Fold the blob being // superseded into this one instead: its authored bytes are a local copy // of exactly the ranges this write did not touch, and what neither wrote // is still the remote content that the earlier write was based on, which // is why the baseline is inherited along with them. let previous = self.pending.lock().get(uid).cloned(); if !meta.complete && let Some(previous) = previous && let Err(error) = merge_over_pending(&mut meta, src, &previous) { // Without the earlier bytes the gaps cannot be resolved against // anything, so this is where the write stops being uploadable. // Keep it recoverable rather than corrupting the file. warn!(%uid, %error, "folding in the undrained edit failed"); self.stage_orphaned_write( uid, ino, src, &meta, "write overlaps an undrained incomplete edit", ); return Err(Errno::EIO); } let path = self.cache.stage_write(&meta, src).map_err(|e| { error!(%uid, error = %e, "staging write failed"); // The scratch file is still the only copy of these bytes, and the // caller is releasing the handle it belongs to — without a // durability sidecar `discard_unmarked_scratch` deletes it at the // next mount, the one place the "never delete on a failure path" // invariant would break. Mark it first, whatever went wrong. if let Err(mark) = self.cache.mark_scratch_durable(src, &meta) { error!(%uid, error = %mark, "could not mark an unstageable write durable; bytes at risk"); } if e.is_disk_full() { // Free cached blobs so the retry has somewhere to land, and tell // the caller what is actually wrong. self.cache.emergency_evict(); Errno::ENOSPC } else { Errno::EIO } })?; // Whatever this write supersedes, it supersedes now — including an // upload already on the wire, whose bytes describe a revision the row // written just below replaces. Signalled before the enqueue so the // in-flight reader stops at its next block rather than after however // much of the file is left. self.cancel_upload(uid); let mut uid = uid.clone(); let len = meta.len; let superseded = 'queued: { if is_local_uid(&uid) { // The node has no server-side identity to hang a revision on, so // the bytes ride on the create that will mint it. Attached and // recorded under one hold of `pending`, which is what the drain // takes to retire a landed create: it either sees this blob and // carries it over, or has already finished and this write // queues against the real uid below (`Core::retire_create`). let meta_json = serde_json::to_string(&meta).unwrap_or_default(); let mut pending = self.pending.lock(); let attached = self .db .attach_blob_to_create(&uid.to_string(), &path.to_string_lossy(), &meta_json) .map_err(|e| { error!(%uid, error = %e, "attaching write to queued create failed"); Errno::EIO })?; if let Some(attached) = attached { pending.insert(uid.clone(), PendingRevision { path, meta }); break 'queued attached.superseded; } drop(pending); // The create landed while this handle was open, and its row // already has the uid Drive gave it. let Some(real) = self.landed_row_uid(ino) else { error!(%uid, staged = %path.display(), "queued create vanished under a write; bytes kept in staging"); return Err(Errno::EIO); }; debug!(local = %uid, %real, "create landed under an open write; queueing a revision"); uid = real; meta.uid = uid.to_string(); let previous = self.pending.lock().get(&uid).cloned(); meta.based_on = previous.as_ref().and_then(|p| p.meta.based_on.clone()); if !meta.complete && let Some(previous) = previous && let Err(error) = merge_over_pending(&mut meta, &path, &previous) { error!(%uid, %error, staged = %path.display(), "folding in the write the create carried failed; bytes kept in staging"); return Err(Errno::EIO); } } let op = PendingOp { id: 0, kind: OP_REVISION.to_string(), uid: uid.to_string(), parent_uid: None, name: None, blob_path: Some(path.to_string_lossy().into_owned()), meta_json: Some(serde_json::to_string(&meta).unwrap_or_default()), created_at: now_millis(), attempts: 0, last_error: None, next_attempt_at: now_millis() + self.revision_debounce(&uid).as_millis() as i64, }; let (_id, superseded) = self.db.enqueue_op(&op).map_err(|e| { error!(%uid, error = %e, "queueing upload failed"); Errno::EIO })?; self.pending .lock() .insert(uid.clone(), PendingRevision { path, meta }); superseded }; if let Some(old) = superseded { self.cache.discard_staged(Path::new(&old)); } let uid = &uid; // Reflect the write in the tree straight away: `ls` must show the new // size and mtime even though the remote still holds the old revision. let now = now_secs(); let updated = { let mut st = self.state(); st.record_pending_write(ino, len, now) }; // The only record that the file is as long as the caller was told. If it // cannot be written the write did not fully take, and saying so beats // acknowledging a `close(2)` and then serving the old size — the bytes // themselves are already staged and queued either way. Written out here // rather than inside `State` so the commit does not hold the inode lock. let recorded = match updated { Some(node) => self.db.upsert_node(&node), None => Ok(()), }; // Cached blobs and open readers describe the superseded revision. Reads // come from the staged blob until the op drains, so just drop them. // Done even when the row above failed: the staged blob is the file's // content now either way, and leaving a stale one cached would serve it. self.cache.evict(uid); self.evict_reader(uid); self.wake_drain(); if let Err(e) = recorded { error!(%uid, error = %e, "recording a queued write's size failed"); return Err(if e.is_disk_full() { Errno::ENOSPC } else { Errno::EIO }); } Ok(()) } /// Queue the new content of a path-based truncate — `> file`, or any /// `setattr(size=…)` arriving without a write handle. /// /// No bytes have been authored at truncate time, which is why this path was /// never staged and instead resized the remote content inline. That is also /// why a shell redirect failed offline *before* the write that follows it: /// the truncate itself needed the network. Staging a blob describing the /// result puts it on the same queue as every other write. /// /// The blob is a hole of the new length; what is real about it is `authored`: /// /// - `> file` (size 0) is the whole point and needs nothing at all — an /// empty file is complete content, so it queues and drains offline. /// - Extending past the end authors the new tail (zeros, by definition) and /// leaves the head to be gap-filled from the base. /// - Shrinking authors nothing: every remaining byte still comes from the /// base, so it is the drain that has to fetch it. fn queue_truncate(&self, ino: u64, size: u64) -> Result<(), Errno> { let (uid, base_mtime, base_size, base_revision_id) = { let st = self.state(); match st.entries.get(&ino) { Some(e) if e.node.is_file() => ( e.uid.clone(), e.node.modification_time, node_size(&e.node), node_revision_id(&e.node), ), Some(_) => return Err(Errno::EISDIR), None => return Err(Errno::ENOENT), } }; self.require_uid_writable(&uid)?; if size == base_size { return Ok(()); } // A close immediately followed by truncate is common (`write`, close, // `truncate(2)`) and usually beats the write-back debounce. In that // case the remote is *not* our base: the complete staged revision is. // Compose from those local bytes so the truncate can supersede the // queued upload without fetching stale remote content or returning EIO. let pending_base = self.pending.lock().get(&uid).cloned(); if pending_base .as_ref() .is_some_and(|pending| !pending.meta.complete) { // An incomplete pending blob still has holes referring to the // remote revision. Stacking another incomplete transform on it is // not representable safely; preserve the existing refusal. return Err(Errno::EIO); } let (authored, complete) = if size == 0 { // An empty file has no content to be missing. (Vec::new(), true) } else if base_size == 0 { // Nothing to gap-fill from: every byte is a zero this truncate // defines. (vec![(0, size)], true) } else if size > base_size { (vec![(base_size, size)], false) } else { (Vec::new(), false) }; let (file, path) = self.cache.create_scratch().map_err(|e| { error!(%uid, error = %e, "create scratch file for truncate failed"); Errno::EIO })?; if let Some(pending) = &pending_base && let Err(e) = copy_pending_for_truncate(pending, &file) { error!(%uid, source = %pending.path.display(), error = %e, "copy pending revision for truncate failed"); let _ = std::fs::remove_file(&path); return Err(Errno::EIO); } file.set_len(size).map_err(|e| { error!(%uid, error = %e, "resize scratch file for truncate failed"); let _ = std::fs::remove_file(&path); Errno::EIO })?; let meta = match pending_base { Some(pending) => StagedWrite { uid: uid.to_string(), len: size, base_size: pending.meta.len, base_mtime: pending.meta.base_mtime, authored: if size == 0 { Vec::new() } else { vec![(0, size)] }, complete: true, // Both local revisions descend from the same last-observed // remote revision. Never replace this with the optimistic // mtime/size currently published by State. based_on: pending.meta.based_on, }, None => StagedWrite { uid: uid.to_string(), len: size, base_size, base_mtime, authored, complete, based_on: self.remote_baseline(&uid, base_mtime, base_size, base_revision_id), }, }; self.enqueue_staged_write(&uid, ino, &path, meta)?; debug!(%uid, size, complete, "queued truncate"); Ok(()) } /// Invent a node under `parent_uid` and queue the op that will make it real /// (offline.md Phase 3b). Returns the node to intern, exactly as the online /// path returns the one the server minted. /// /// The parent may itself be a placeholder — `mkdir -p` offline, or `cp -r` of /// a tree. That is fine: the op records the parent it was made under by local /// id, and is not sent until the parent's row has the uid it landed as. fn queue_local_node( &self, parent_uid: &NodeUid, name: &str, is_dir: bool, hold: bool, ) -> Result { self.require_uid_writable(parent_uid)?; // The row the tree would give it anyway, now: the node goes by its // local id until it lands. let lid = self .db .add_local_row(&parent_uid.to_string(), name, is_dir, now_secs()) .map_err(|e| { error!(%parent_uid, name, error = %e, "giving a local node its row failed"); Errno::EIO })?; let uid = local_node_uid(lid); let node = local_node(uid.clone(), parent_uid.clone(), name.to_string(), is_dir); let op = PendingOp { id: 0, kind: if is_dir { OP_MKDIR } else { OP_CREATE }.to_string(), uid: uid.to_string(), parent_uid: Some(parent_uid.to_string()), name: Some(name.to_string()), blob_path: None, meta_json: None, created_at: now_millis(), attempts: 0, last_error: None, // Parked when the name is transient: its bytes ride on this create // (via `attach_blob_to_create`) and must not upload until a rename to // the finished name un-parks it (docs/BUGS.md B70). next_attempt_at: if hold { PARK_UNTIL } else { 0 }, }; self.db.enqueue_op(&op).map_err(|e| { error!(%parent_uid, name, error = %e, "queueing local node failed"); Errno::EIO })?; debug!(%uid, %parent_uid, name, is_dir, "created node offline; queued"); // A folder is due now; without this the workers slept through it to // the idle poll (B160). A file is not: its bytes ride on the create, and // the release that attaches them wakes the drain. if is_dir { self.wake_drain(); } Ok(node) } /// Queue giving a node a new parent and/or name, and apply it to the tree /// now (offline.md Phase 3b). /// /// The op records the desired end state rather than the step, so it both /// supersedes any earlier queued rename and lets the drain skip whichever /// half the remote already agrees with. /// /// `new_parent_uid` may be a placeholder — moving a file into a folder /// created offline — which is why this cannot simply be the online call with /// a retry around it: the API would 404 on a `local~` parent. The op waits /// for that folder's drain to rewrite it, exactly as a queued create does. fn queue_rename( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_ino: u64, new_parent_uid: &NodeUid, new_name: &str, ) -> Result<(), Errno> { self.require_uid_writable(uid)?; self.require_uid_writable(old_parent_uid)?; self.require_uid_writable(new_parent_uid)?; self.queue_rename_authorized( ino, uid, old_parent_uid, new_parent_ino, new_parent_uid, new_name, ) } /// Enqueue a rename whose access was linearized immediately before a /// replacement victim was removed. Rechecking here could reject after that /// destructive half had already completed. fn queue_rename_authorized( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_ino: u64, new_parent_uid: &NodeUid, new_name: &str, ) -> Result<(), Errno> { self.enqueue_rename(ino, uid, old_parent_uid, new_parent_uid, new_name)?; self.state .lock() .rename_in_place(ino, new_parent_ino, new_parent_uid, new_name); self.wake_drain(); debug!(%uid, %new_parent_uid, new_name, "renamed offline; queued"); Ok(()) } /// Record the queued rename op, and nothing else: the caller moves the node /// in the tree, through the kernel or around it. fn enqueue_rename( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_uid: &NodeUid, new_name: &str, ) -> Result<(), Errno> { // A rename that replaces a queued one keeps where the node started. let meta = match self.db.pending_op_meta(&uid.to_string(), OP_RENAME) { Ok(Some(json)) => serde_json::from_str::(&json).map_err(|error| { error!(%uid, %error, "queued rename authority metadata is invalid"); Errno::EIO })?, Ok(None) => RenameMeta { original_parent_uid: old_parent_uid.to_string(), original_name: self .state .lock() .entries .get(&ino) .map(|entry| entry.node.name.clone()), }, Err(error) => { error!(%uid, %error, "reading queued rename authority failed"); return Err(Errno::EIO); } }; let meta_json = serde_json::to_string(&meta).map_err(|error| { error!(%uid, %error, "serializing queued rename authority failed"); Errno::EIO })?; let op = PendingOp { id: 0, kind: OP_RENAME.to_string(), uid: uid.to_string(), parent_uid: Some(new_parent_uid.to_string()), name: Some(new_name.to_string()), blob_path: None, meta_json: Some(meta_json), created_at: now_millis(), attempts: 0, last_error: None, next_attempt_at: 0, }; self.db.enqueue_op(&op).map_err(|e| { error!(%uid, new_name, error = %e, "queueing rename failed"); Errno::EIO })?; Ok(()) } /// Queue trashing a node the server knows about, and drop it from the tree /// now (offline.md Phase 3b). /// /// Anything else queued for this node is discarded first: the user has said /// the file should not exist, so uploading bytes to it or renaming it are /// both work towards an outcome nobody wants any more. That does throw away /// staged bytes that never landed — which is precisely what deleting an /// un-uploaded file means, and the alternative (upload it, then trash it) is /// worse in every way. fn queue_trash(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> { self.require_uid_writable(uid)?; // The bytes below are about to be discarded, and a drain worker may be // reading them onto the wire right now. Stop it first: with several // workers the trash op this queues can be claimed while that upload is // still running, and the two would race to decide whether the file // exists. self.cancel_upload(uid); let (_, blobs) = self .db .replace_ops_with_trash(&uid.to_string(), name, now_millis()) .map_err(|e| { error!(%uid, error = %e, "queueing trash failed"); Errno::EIO })?; self.release_dropped_ops(uid, blobs); self.hidden.lock().insert(uid.clone()); // Withdrawn from the tree, which every mount serves from, so no other // mount goes on serving a file the user just trashed (`docs/BUGS.md` // B74). self.for_each_state(|st| { st.unlink_mem(uid); }); self.cache.evict(uid); self.evict_reader(uid); self.wake_drain(); debug!(%uid, name, "trashed offline; queued"); Ok(()) } /// Drop every op queued against a node, and the staged bytes they own. fn discard_queued_ops(&self, uid: &NodeUid) -> Result<(), Errno> { // Same reason as `queue_trash`: an upload may be reading the very blobs // this is about to unlink. self.cancel_upload(uid); let blobs = self.db.delete_ops_for_uid(&uid.to_string()).map_err(|e| { error!(%uid, error = %e, "dropping queued ops failed"); Errno::EIO })?; self.release_dropped_ops(uid, blobs); Ok(()) } /// Drop every op queued against a node that just left the namespace while /// a handle still has it open, but keep the bytes that handle reads. /// /// The ops cannot wait for the final close: the node is already trashed, and /// a revision the drain sends in the meantime lands on it after the trash — /// an editor's save loop then found the name it was renaming onto still /// taken (`docs/BUGS.md` B106). The pending entry and its staged blob are /// what the open handle reads, so they stay until the final close drops /// them. fn withdraw_queued_ops(&self, uid: &NodeUid) -> Result<(), Errno> { self.cancel_upload(uid); let blobs = self.db.delete_ops_for_uid(&uid.to_string()).map_err(|e| { error!(%uid, error = %e, "withdrawing queued ops failed"); Errno::EIO })?; let read_from = self.pending_blob(uid); let blobs = blobs .into_iter() .filter(|blob| read_from.as_deref() != Some(Path::new(blob))) .collect(); self.release_dropped_below(blobs); Ok(()) } /// Let go of what the ops just dropped for `uid` held: their pending /// entries, any upload still reading their bytes, and the bytes. /// /// Removing a folder also drops the queued revisions of the files below it /// (`docs/BUGS.md` B102), so the entries to forget are found by blob, not /// only by `uid`. fn release_dropped_ops(&self, uid: &NodeUid, blobs: Vec) { self.pending.lock().remove(uid); self.release_dropped_below(blobs); } /// [`Core::release_dropped_ops`] without touching the node's own pending /// entry. fn release_dropped_below(&self, blobs: Vec) { let below: Vec = { let mut pending = self.pending.lock(); let below: Vec = pending .iter() .filter(|(_, p)| blobs.iter().any(|b| p.path == Path::new(b))) .map(|(k, _)| k.clone()) .collect(); for k in &below { pending.remove(k); } below }; for k in &below { self.cancel_upload(k); } for blob in blobs { self.cache.discard_staged(Path::new(&blob)); } } /// Which staged blob is currently queued for `uid`, if any. /// /// The path identifies the revision: publication stages a fresh file and /// swaps the map entry to point at it, so an unchanged path means the base a /// caller sampled earlier is still the base now (bugs.md B32). fn pending_blob(&self, uid: &NodeUid) -> Option { self.pending.lock().get(uid).map(|p| p.path.clone()) } /// Nudge the drain worker to re-examine the queue now. fn wake_drain(&self) { let (lock, cv) = &*self.drain_wake; *lock.lock() = true; cv.notify_all(); } /// Keep a write we cannot safely queue, so the bytes are recoverable even /// though the caller is getting an error. See [`Core::queue_revision`]. fn stage_orphaned_write( &self, uid: &NodeUid, ino: u64, src: &Path, meta: &StagedWrite, reason: &str, ) { match self.cache.preserve_write(meta, src) { Ok(staged) => { self.cache.clear_scratch_durable(src); error!( %uid, staged = %staged.display(), %reason, "cannot queue write; bytes kept in staging" ); let name = { let st = self.state(); st.entries .get(&ino) .map(|e| e.node.name.clone()) .unwrap_or_default() }; self.log_activity( ActivityKind::Upload, &name, format!( "write not queued ({reason}); changes kept at {}", staged.display() ), false, ); } Err(e) => { let surviving = e.surviving_path.clone(); let marker = e.marker_path.clone(); error!( %uid, error = %e, surviving = %surviving.display(), marker = ?marker, %reason, "write rescue was only partially published; bytes retained" ); let name = { let st = self.state(); st.entries .get(&ino) .map(|entry| entry.node.name.clone()) .unwrap_or_default() }; self.log_activity( ActivityKind::Upload, &name, format!( "write not queued ({reason}); changes retained at {}", surviving.display() ), false, ); } } } /// Download a whole file's plaintext, registering the transfer so /// `GetQueueStatus` can report its progress. `total` is the expected size /// (`0` if unknown). Streams through [`download_file_to`] so each block ticks /// the progress counter. /// /// [`download_file_to`]: ProtonDriveClient::download_file_to fn download_file_tracked( &self, uid: &NodeUid, name: &str, total: u64, ) -> std::result::Result, ProtonError> { let guard = self .transfers .begin(name, uid.to_string(), TransferDirection::Download, total); let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard); self.rt .block_on(self.drive.download_file_to(uid, &mut out))?; Ok(out.into_inner()) } /// Download a whole file directly into `out`, without retaining its /// plaintext in memory. This is the disk-backed twin of /// [`Core::download_file_tracked`], and preserves the same transfer /// accounting while allowing callers that only need an on-disk file to /// keep memory use independent of file size. fn download_file_tracked_to( &self, uid: &NodeUid, name: &str, total: u64, out: W, ) -> std::result::Result { let guard = self .transfers .begin(name, uid.to_string(), TransferDirection::Download, total); let mut out = CountingWriter::new(out, &guard); self.rt .block_on(self.drive.download_file_to(uid, &mut out))?; Ok(out.into_inner()) } /// Like [`download_file_tracked`] for a photo, streaming through the photos /// client's [`download_photo_to`]. /// /// [`download_file_tracked`]: Core::download_file_tracked /// [`download_photo_to`]: ProtonPhotosClient::download_photo_to fn download_photo_tracked( &self, photos: &ProtonPhotosClient, uid: &NodeUid, name: &str, total: u64, ) -> std::result::Result, ProtonError> { let guard = self .transfers .begin(name, uid.to_string(), TransferDirection::Download, total); let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard); self.rt.block_on(photos.download_photo_to(uid, &mut out))?; Ok(out.into_inner()) } /// Pin the node at mountpoint-relative `rel`. A file downloads its full /// plaintext into the content cache; a folder records a recursive pin and /// downloads every descendant file (selective sync). Returns a human message. fn pin(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (name, is_folder, mtime, size) = { let st = self.state(); let e = st .entries .get(&ino) .ok_or_else(|| CoreError::not_found("node vanished"))?; ( e.node.name.clone(), e.node.is_folder(), e.node.modification_time, node_size(&e.node), ) }; if is_folder { // Record the recursive pin first so every descendant is eviction- // exempt before we start filling the cache with the subtree. self.cache .add_pin(&uid, rel, true) .map_err(|e| CoreError::from_api(&e, "pin"))?; let n = self.pin_subtree(ino)?; return Ok(format!("{name} ({n} files)")); } let bytes = self .download_file_tracked(&uid, &name, size) .map_err(|e| CoreError::from_api(&e, "download"))?; self.cache .store(&uid, mtime, size, &bytes) .map_err(|e| CoreError::internal(format!("cache store: {e}")))?; self.cache .add_pin(&uid, rel, false) .map_err(|e| CoreError::from_api(&e, "pin"))?; Ok(name) } /// Download and cache every file in the subtree rooted at folder `ino`, /// returning the count cached (already-fresh blobs counted, not re-fetched). /// Walks the tree depth-first, enumerating each folder so a cold subtree is /// fully discovered; the lock is dropped before each network download. fn pin_subtree(&self, ino: u64) -> CoreResult { let mut files: Vec<(NodeUid, String, i64, u64)> = Vec::new(); let mut stack = vec![ino]; while let Some(dir) = stack.pop() { self.ensure_children(dir) .map_err(|e| self.errno_error(e, "enumerate"))?; let st = self.state(); if let Some(kids) = st.children.get(&dir) { for &k in kids { if let Some(e) = st.entries.get(&k) { if e.node.is_folder() { stack.push(k); } else { files.push(( e.uid.clone(), e.node.name.clone(), e.node.modification_time, node_size(&e.node), )); } } } } } let mut count = 0; for (uid, name, mtime, size) in files { if self.cache.is_cached(&uid, mtime, size) { count += 1; continue; } match self.download_file_tracked(&uid, &name, size) { Ok(bytes) => { if self.cache.store(&uid, mtime, size, &bytes).is_ok() { count += 1; } } Err(e) => warn!(%uid, error = %e, "pin subtree: download failed"), } } Ok(count) } /// Fetch a thumbnail of `ttype` for the file at `ino`, served from the cache /// when fresh and otherwise downloaded from the remote and cached. Returns /// `Ok(None)` when the node is not a file or has no thumbnail of that type. fn thumbnail(&self, ino: u64, ttype: ThumbnailType) -> Result>, Errno> { let (uid, mtime) = { let st = self.state(); match st.entries.get(&ino) { Some(e) if e.node.is_file() => (e.uid.clone(), e.node.modification_time), Some(_) => return Ok(None), None => return Err(Errno::ENOENT), } }; if let Some(bytes) = self.cache.read_thumbnail(&uid, ttype.as_i32(), mtime) { return Ok(Some(bytes)); } // "This file has no thumbnail" is an answer worth remembering: without // it every listing pays a round trip per file to be told nothing (B5). let key = (uid.clone(), ttype.as_i32()); if self.thumbnail_misses.lock().remote_contains(&key, mtime) { return Ok(None); } let bytes = self .rt .block_on(self.client.download_thumbnail(&uid, ttype)) .map_err(|e| { warn!(%uid, error = %e, "download thumbnail failed"); Errno::EIO })?; match &bytes { Some(bytes) => { let _ = self .cache .store_thumbnail(&uid, ttype.as_i32(), mtime, bytes); } None => { self.thumbnail_misses.lock().remember_remote(key, mtime); } } Ok(bytes) } /// Unpin the node at `rel`, evicting its cached content. For a folder, also /// evicts every descendant's cached blob (the subtree is no longer kept). fn unpin(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (name, is_folder) = { let st = self.state(); st.entries .get(&ino) .map(|e| (e.node.name.clone(), e.node.is_folder())) .unwrap_or_default() }; self.cache .remove_pin(&uid) .map_err(|e| CoreError::from_api(&e, "unpin"))?; // A recursively-pinned folder's descendants were eviction-exempt; now // that the pin is gone, reclaim their blobs eagerly instead of waiting // for budget pressure. Descendants come from the DB node tree. if is_folder && let Ok(uids) = self.db.descendants(&uid.to_string()) { for s in uids { if let Some(u) = parse_uid(&s) { self.cache.evict(&u); self.evict_reader(&u); } } } Ok(name) } /// A Photos API handle sharing this Core's Drive client and session, so it /// reuses the daemon's single authenticated session rather than logging in /// again (Proton refresh tokens are single-use). Cheap — the Drive client /// is `Clone` over `Arc`-backed state. fn photos(&self) -> ProtonPhotosClient { ProtonPhotosClient::from_drive_client(self.client.clone()) } /// List the directory at mountpoint-relative `rel` for the in-app browser: /// the same lazy remote enumeration `readdir` uses, projected into /// serializable [`DirEntry`]s (with per-file pin state). fn list_dir(&self, rel: &Path) -> CoreResult> { let (ino, _uid) = self.resolve(rel)?; self.ensure_children(ino) .map_err(|e| self.errno_error(e, "enumerate"))?; self.settle_listing_sizes(ino); // Snapshot the listing, then drop the lock before touching the on-disk // pin registry so a slow disk read doesn't block FUSE metadata ops. let rows: Vec<(String, bool, u64, i64, NodeUid, u64)> = { let st = self.state(); st.children .get(&ino) .map(|kids| { kids.iter() .filter_map(|&k| st.entries.get(&k).map(|e| (k, e))) .map(|(k, e)| { ( e.node.name.clone(), e.node.is_folder(), node_size(&e.node), e.node.modification_time, e.uid.clone(), k, ) }) .collect() }) .unwrap_or_default() }; let issues = self.db.node_issues().unwrap_or_default(); Ok(rows .into_iter() .map(|(name, is_dir, size, modified, uid, ino)| DirEntry { name, is_dir, size, modified, pinned: self.cache.is_pinned(&uid), cached: !is_dir && self.cache.is_cached(&uid, modified, size), uid: uid.to_string(), // Listing entries live in the requested dir; the caller derives // the path from its name. Left empty. path: String::new(), // My own tree: a role only means something under a share, and // this listing never crosses into one. role: String::new(), shared_by: String::new(), shared_at: 0, shared_by_unverified: false, trashed_at: 0, trashed_from: None, issue: state::ino_lid(ino) .and_then(|lid| issues.get(&lid)) .map(|issue| SyncIssue::parse(issue)), }) .collect()) } /// Every uid that is pinned right now, as one set. /// /// `cache.is_pinned` is a direct lookup plus an ancestor walk; asking it per /// candidate was a thousand walks per keystroke. The recursive expansion is /// the same query, run once. fn pinned_set(&self) -> HashSet { self.db .pinned_uids() .unwrap_or_default() .into_iter() .collect() } /// The local locations a search result can be opened through, resolved once /// per search rather than once per hit. /// /// An on-demand folder only answers while its mount is live — without the /// mount its `local_path` is an ordinary empty directory. A mirror folder /// needs no mount: its files are real local copies the sync loop pushes /// back, so it is a valid open target whenever it is configured. /// /// This used to be a per-hit call that re-read the sync-folder list and ran /// an ancestor walk per folder, against a candidate pool of up to a /// thousand: the whole cost of a keystroke was in here. fn search_roots(&self) -> SearchRoots { let live_mounts = self.mounts.lock(); let folders = self.db.sync_folder_list().unwrap_or_default(); SearchRoots { roots: folders .into_iter() .filter(|folder| match folder.mode.as_str() { "ondemand" => live_mounts.contains_key(&folder.id), "mirror" => true, _ => false, }) .filter_map(|folder| { let root = self.db.node_path(&folder.remote_uid).ok().flatten()?; Some((root, PathBuf::from(folder.local_path))) }) .collect(), } } /// Full-text search node names against the local SQLite index, mapping each /// DB hit to the wire [`SearchHit`] (resolving live pin state from the cache, /// which the DB doesn't track). Pure local lookup — never hits the network. /// Every pin, with `cached` saying whether a pinned file's content is on /// disk, so a front-end listing pins can badge them without a round-trip /// per file. fn list_pins(&self) -> Vec { let mut pins = self.cache.list_pins(); for pin in pins.iter_mut().filter(|pin| pin.is_dir == Some(false)) { if let Ok(Some(node)) = self.db.node_by_uid(&pin.uid) { pin.cached = self.cache .is_cached(&node.uid, node.modification_time, node_size(&node)); } } pins } fn search(&self, query: &str, limit: usize, scope: Option<&str>) -> CoreResult> { let hits = self .db .search_in(query, limit, scope) .map_err(|e| CoreError::from_api(&e, "search"))?; let roots = self.search_roots(); let pinned = self.pinned_set(); Ok(hits .into_iter() .map(|h| SearchHit { name: h.node.name.clone(), is_dir: h.node.is_folder(), size: node_size(&h.node), modified: h.node.modification_time, pinned: pinned.contains(&h.node.uid.to_string()), cached: !h.node.is_folder() && self.cache.is_cached( &h.node.uid, h.node.modification_time, node_size(&h.node), ), uid: h.node.uid.to_string(), mounted_path: roots.resolve(&h.path), path: h.path, score: 0, }) .collect()) } /// Search the index of files on this machine (outside Drive), built by the /// background scanner in [`run_local_index`]. Pure local lookup, never hits /// the network — and never touches the FUSE mount, which the scanner excludes. fn search_local(&self, query: &str, limit: usize) -> CoreResult> { let hits = self .db .search_local(query, limit) .map_err(|e| CoreError::from_api(&e, "local search"))?; Ok(hits .into_iter() .map(|h| LocalHit { name: h.name, path: h.path, is_dir: h.is_dir, size: h.size.max(0) as u64, modified: h.mtime, score: 0, }) .collect()) } /// Search both prompt sources with identical query and per-source limits. /// Keeping the composition in the daemon core gives every control surface /// the same semantics while preserving the legacy source-specific methods. fn search_v2( &self, query: &str, limit: usize, filters: &SearchFilters, ) -> CoreResult<(Vec, Vec)> { if query.trim().is_empty() || limit == 0 { return Ok((Vec::new(), Vec::new())); } // Candidate generation is intentionally broader than the final result // cap: filtering and fuzzy rejection happen below, so limiting at the // SQLite boundary to `limit` would recreate the old empty-filter bug. let candidate_limit = limit.saturating_mul(10).clamp(100, 1_000); let mut drive_hits = if filters.sources.contains(&SearchSource::Drive) { let roots = self.search_roots(); let pinned = self.pinned_set(); self.db .search_candidates(query, candidate_limit) .map_err(|e| CoreError::from_api(&e, "search candidates"))? .into_iter() .filter(|hit| filters.kind.accepts(&hit.node.name, hit.node.is_folder())) .filter_map(|hit| { let parent = Path::new(&hit.path) .parent() .map_or_else(String::new, |path| path.display().to_string()); let uid = hit.node.uid.to_string(); let is_pinned = pinned.contains(&uid); let score = relevance_score(query, &hit.node.name, &parent)? + if is_pinned { 250 } else { 0 }; Some(SearchHit { name: hit.node.name.clone(), is_dir: hit.node.is_folder(), size: node_size(&hit.node), modified: hit.node.modification_time, pinned: is_pinned, cached: !hit.node.is_folder() && self.cache.is_cached( &hit.node.uid, hit.node.modification_time, node_size(&hit.node), ), uid, mounted_path: roots.resolve(&hit.path), path: hit.path, score, }) }) .collect() } else { Vec::new() }; let mut local_hits = if filters.sources.contains(&SearchSource::Local) { self.db .search_local_candidates(query, candidate_limit) .map_err(|e| CoreError::from_api(&e, "local search candidates"))? .into_iter() .filter(|hit| filters.kind.accepts(&hit.name, hit.is_dir)) .filter_map(|hit| { let parent = Path::new(&hit.path) .parent() .map_or_else(String::new, |path| path.display().to_string()); Some(LocalHit { score: relevance_score(query, &hit.name, &parent)?, name: hit.name, path: hit.path, is_dir: hit.is_dir, size: hit.size.max(0) as u64, modified: hit.mtime, }) }) .collect() } else { Vec::new() }; drive_hits.sort_by(|a, b| { b.score .cmp(&a.score) .then_with(|| a.name.cmp(&b.name)) .then_with(|| a.path.cmp(&b.path)) }); local_hits.sort_by(|a, b| { b.score .cmp(&a.score) .then_with(|| a.name.cmp(&b.name)) .then_with(|| a.path.cmp(&b.path)) }); drive_hits.truncate(limit); local_hits.truncate(limit); Ok((drive_hits, local_hits)) } /// A page of the photos timeline (newest first), sliced out of the DB. /// `Ok(None)` when the account has no photos volume. /// /// Stale-while-revalidate: a persisted timeline is served *immediately*, and /// refreshed on the runtime if it is older than [`TIMELINE_TTL`] — so opening /// the app paints from disk rather than waiting on a full `enumerate_timeline` /// (which returns the whole timeline, not a page). Only an empty DB blocks on /// the network, i.e. the very first run. /// /// Metadata only: a thumbnail path is attached for photos whose thumbnail is /// already cached, and nothing is downloaded here — the front-end pulls the /// thumbnails it actually paints via [`Core::photo_thumbs`]. fn listing_stale(&self, key: &str, ttl: Duration) -> bool { match self.db.state_i64(key).ok().flatten() { Some(ms) => now_ms().saturating_sub(ms) >= ttl.as_millis() as i64, None => true, } } /// Re-fetch the whole photos timeline and persist it. Returns whether the /// account has a photos volume at all. /// Download the full content of the Drive file at mountpoint-relative `rel` /// into the content cache, returning its on-disk path (served from cache /// when a fresh blob already exists). Lets a front-end open the file with /// the user's default application without pinning it. fn open_file(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (name, mtime, size) = { let st = self.state(); let e = st .entries .get(&ino) .ok_or_else(|| CoreError::not_found("node vanished"))?; if !e.node.is_file() { return Err(CoreError::invalid("not a regular file")); } ( e.node.name.clone(), e.node.modification_time, node_size(&e.node), ) }; self.fetch_content(&uid, &name, mtime, size) } /// [`open_file`](Self::open_file) addressed by uid instead of by path. /// /// A search hit is a metadata-index row, and the index covers every /// location this daemon knows — including on-demand sync folders and /// mirrors, whose nodes have no path inside the primary mount's tree. /// Walking such a hit's path from the mount root answers ENOENT even though /// the node is perfectly reachable, so front-ends address hits by uid. /// /// Metadata comes from the live tree, then the index, then the API — the /// last covers a node the index knows but no location holds. Unlike the /// mutating by-uid handlers this does not demand a /// [`resolve_anywhere`](Self::resolve_anywhere) residency proof: reading /// one's own file is what the API authorizes anyway, and requiring /// residency is exactly what turned an openable search hit into "no such /// file or folder". [`open_shared_file`](Self::open_shared_file) already /// works this way. Reserved (local/virtual) uids name no remote node and /// are still rejected. fn open_file_uid(&self, raw_uid: &str) -> CoreResult { let uid = parse_uid(raw_uid) .filter(|uid| !is_local_uid(uid) && !is_virtual_uid(uid)) .ok_or_else(|| CoreError::invalid(format!("invalid uid: {raw_uid}")))?; let live = self .state .lock() .entries .values() .find(|e| e.uid == uid) .map(|e| e.node.clone()); let node = match live { Some(node) => node, None => match self .db .node_by_uid(&uid.to_string()) .map_err(|e| CoreError::from_api(&e, "load node"))? { Some(node) => node, None => self .rt .block_on(self.drive.get_node(&uid)) .map_err(|e| CoreError::from_api(&e, "get node"))? .ok_or_else(|| CoreError::not_found("node vanished"))?, }, }; if !node.is_file() { return Err(CoreError::invalid("not a regular file")); } self.fetch_content(&uid, &node.name, node.modification_time, node_size(&node)) } /// Materialise a file's full content into the content cache, returning its /// on-disk path. Serves a fresh cached blob without touching the network. fn fetch_content( &self, uid: &NodeUid, name: &str, mtime: i64, size: u64, ) -> CoreResult { if let Some(p) = self.cache.cached_content_path(uid, mtime, size) { return Ok(p); } // Keep the temporary file beside the final cache blob. `store_file` // can then adopt it with a hard link and atomic rename, while the SDK // streams decrypted blocks to disk instead of building a file-sized // `Vec`. A unique name also keeps concurrent opens of the same node // from sharing a partially written staging file. static OPEN_TEMP_SEQ: AtomicU64 = AtomicU64::new(0); let seq = OPEN_TEMP_SEQ.fetch_add(1, Ordering::Relaxed); let tmp = self.cache.content_path(uid).with_extension(format!( "open-{}-{}-{seq}.tmp", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) )); let file = File::options() .write(true) .create_new(true) .open(&tmp) .map_err(|e| CoreError::internal(format!("create download temp: {e}")))?; let file = match self.download_file_tracked_to(uid, name, size, file) { Ok(file) => file, Err(e) => { let _ = std::fs::remove_file(&tmp); return Err(CoreError::from_api(&e, "download")); } }; if let Err(e) = file.sync_all() { let _ = std::fs::remove_file(&tmp); return Err(CoreError::internal(format!("sync download temp: {e}"))); } if let Err(e) = self.cache.store_file(uid, mtime, size, &tmp) { let _ = std::fs::remove_file(&tmp); return Err(CoreError::internal(format!("cache store: {e}"))); } let _ = std::fs::remove_file(&tmp); Ok(self.cache.content_path(uid)) } /// Drop the cached child listing of `rel`'s parent directory so the next /// `ListDir` re-enumerates it from the server. No-op when the parent can't be /// resolved (e.g. `rel` is the root). Resolves the parent without holding the /// state lock, then invalidates under it. fn invalidate_parent_listing(&self, rel: &Path) { let parent = rel.parent().unwrap_or_else(|| Path::new("")); if let Ok((pino, _)) = self.resolve_path(parent) { self.state().invalidate_listing(pino); } } /// Drop the cached child listing of the folder `parent`, everywhere it is /// held — naming the folder by uid rather than by inode. /// /// [`Core::invalidate_parent_listing`] is the path-based sibling, and it /// only reaches *this* mount's paths. A remote mutation made by the /// sync engine needs more than that, for two reasons. The folder it changed /// is usually not resident in the tree at the time — a mirror folder /// has no FUSE session at all — so there is no inode to name. And the stale /// listing that outlives the pass is not a hot-cache map but the `listed` /// flag on the folder's DB row, which survives a daemon restart and is what /// a later `ensure_children` trusts. /// /// So the flag is cleared unconditionally, and the resident listing is /// invalidated on top of it. That is what makes a mirror folder switched to /// on-demand re-enumerate instead of serving a snapshot that predates the /// engine's own uploads — `docs/BUGS.md` B86. pub(crate) fn invalidate_children_of(&self, parent: &NodeUid) { if let Err(e) = self.db.set_listed(parent, false) { warn!(uid = %parent, error = ?e, "could not clear cached listing"); } self.for_each_state(|st| { if let Some(&ino) = st.by_uid.get(parent) { st.invalidate_listing(ino); } }); } /// Rename a file or folder to `new_name`. `rel` is mountpoint-relative. /// Mirrors the FUSE `rename` write path: queued whenever the mount queues /// ([`Core::queue_relocation`]), otherwise renamed on the remote, then /// moved to its new name in every mount with the parent listing dropped /// so the next `ListDir` re-enumerates. fn rename(&self, rel: &Path, new_name: &str) -> CoreResult { if new_name.is_empty() || new_name.contains('/') { return Err(CoreError::invalid(format!("invalid name: {new_name:?}"))); } let (ino, uid) = self.resolve(rel)?; let old_parent_uid = self.source_parent_uid(ino, rel)?; require_rename_access( |authority| self.require_uid_writable(authority), &uid, &old_parent_uid, &old_parent_uid, ) .map_err(|error| self.errno_error(error, "rename access"))?; if self.queues_relocation(&uid, &old_parent_uid) { self.queue_relocation( ino, &uid, &old_parent_uid, &old_parent_uid, &rel.with_file_name(new_name), new_name, )?; return Ok(new_name.to_string()); } self.rename_remote(&uid, new_name) .map_err(|e| CoreError::from_api(&e, "rename"))?; self.relocate_everywhere(&uid, &old_parent_uid, new_name); self.invalidate_parent_listing(rel); Ok(new_name.to_string()) } /// Whether a rename or move asked for over the control socket goes /// through the queue. It does whenever the same change through the mount /// would, and whenever the node has anything queued: sent straight to /// Drive, it would land first, and the queued change landing after it /// would undo it (`docs/BUGS.md` B149). fn queues_relocation(&self, uid: &NodeUid, new_parent_uid: &NodeUid) -> bool { !self.sends_inline() || is_local_uid(uid) || is_local_uid(new_parent_uid) || self.db.has_any_op(&uid.to_string()).unwrap_or(true) } /// Queue renaming or moving the node at `ino` to `dest`, whose last /// component is `name`, and apply it to every mount now, as the mount's /// own queued rename does. /// /// The listings stay: Drive has not been told yet, so listing either /// folder again would bring back the old name. fn queue_relocation( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_uid: &NodeUid, dest: &Path, name: &str, ) -> CoreResult<()> { // Drive refuses a taken name, but only once the queue sends the change. match self.resolve_path(dest) { Ok((found, _)) if found != ino => { return Err(CoreError::conflict(format!( "{} already exists", dest.display() ))); } Ok(_) => {} Err(e) if e.code() == libc::ENOENT => {} Err(e) => { return Err(self.errno_error(e, &format!("could not resolve {}", dest.display()))); } } let old_name = self .state() .entries .get(&ino) .map(|entry| entry.node.name.clone()) .unwrap_or_default(); let mut uid = uid.clone(); // A node whose create is still queued is that op: rewriting its target // is the whole change, as through the mount. if is_local_uid(&uid) { match self .db .rewrite_op_target(&uid.to_string(), &new_parent_uid.to_string(), name) { Ok(true) => { self.relocate_queued_everywhere(&uid, new_parent_uid, name); // A parked transient file given a finished name is meant // to reach Drive now (`docs/BUGS.md` B70). if is_transient_name(&old_name) && !is_transient_name(name) && let Err(e) = self.db.set_create_hold(&uid.to_string(), false) { warn!(%uid, error = %e, "un-parking a finalized transient create failed"); } self.wake_drain(); return Ok(()); } // The create landed meanwhile: rename what Drive made. Ok(false) => { uid = self.landed_row_uid(ino).ok_or_else(|| { CoreError::conflict(format!( "{old_name} changed while it was being renamed; try again" )) })?; } Err(e) => return Err(CoreError::internal(format!("db: {e}"))), } } self.enqueue_rename(ino, &uid, old_parent_uid, new_parent_uid, name) .map_err(|e| self.errno_error(e, "queue the change"))?; self.relocate_queued_everywhere(&uid, new_parent_uid, name); self.wake_drain(); debug!(%uid, %new_parent_uid, name, "renamed over the control socket; queued"); Ok(()) } /// Bring every mount level with a rename or move Drive has made of `uid`, /// to `name` in `new_parent`, and have each kernel session drop the name it /// cached for it. Returns the name a mount knew the node by. /// /// For changes that do not come through the mount. The kernel made a /// rename through the mount itself, but one made over the control socket /// leaves the old name cached for [`TTL`] (`docs/BUGS.md` B114). /// /// The node is moved, not forgotten, and so is its row. Forgetting it, as /// this used to, deleted the row, and until a listing brought it back /// nothing local knew the node: `pdfs sync queue` showed its queued upload /// by raw id, and the drain re-read the node and put the upload off /// (`docs/BUGS.md` B117). fn relocate_everywhere( &self, uid: &NodeUid, new_parent: &NodeUid, name: &str, ) -> Option { self.relocate_with(uid, new_parent, name, false) } /// [`Core::relocate_everywhere`] for a change the queue has not sent yet. /// Each mount keeps its listings, which Drive cannot bring up to date /// until the change lands. fn relocate_queued_everywhere(&self, uid: &NodeUid, new_parent: &NodeUid, name: &str) { self.relocate_with(uid, new_parent, name, true); } fn relocate_with( &self, uid: &NodeUid, new_parent: &NodeUid, name: &str, queued: bool, ) -> Option { let mut old_name = None; let mut moved_row = false; self.for_each_mount(|st, notify| { if let Some((from_name, relocated)) = relocate_in_mount(st, notify, uid, new_parent, name, queued) { old_name = Some(from_name); moved_row |= relocated; } }); if !moved_row { self.move_row(uid, new_parent, name); } old_name } /// Point the persisted row of `uid` at `name` in `new_parent`, for a node /// no mount holds where it now is. fn move_row(&self, uid: &NodeUid, new_parent: &NodeUid, name: &str) { let moved = self.db.node_by_uid(&uid.to_string()).and_then(|node| { let Some(mut node) = node else { return Ok(()); }; node.name = name.to_string(); node.parent_uid = Some(new_parent.clone()); self.db.upsert_node(&node) }); if let Err(error) = moved { warn!(%uid, ?error, "could not move the node's row along with it"); } } /// Forget `uid` in every mount, and have each kernel session drop the name /// it cached for it. Returns the name a mount knew the node by. /// /// For a removal that does not come through the mount. The kernel made an /// unlink through the mount itself, but one made over the control socket /// leaves the old name cached for [`TTL`], pointing at an inode the mount no /// longer has (`docs/BUGS.md` B114). Every mount, not only this one, so a /// fork showing the same node does not keep serving it (`docs/BUGS.md` /// B74). fn forget_everywhere(&self, uid: &NodeUid) -> Option { let mut name = None; self.for_each_mount(|st, notify| { if let Some(entry) = st.by_uid.get(uid).and_then(|ino| st.entries.get(ino)) { name = Some(entry.node.name.clone()); } forget_and_notify(st, notify, uid); }); name } /// Move a file or folder into the folder at `new_parent_rel`. Both paths are /// mountpoint-relative. Queued whenever the mount queues /// ([`Core::queue_relocation`]); otherwise moved on the remote, then in /// every mount, with both listings invalidated so each re-enumerates on /// next access. fn move_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let old_parent_uid = self.source_parent_uid(ino, rel)?; let (pino, new_parent_uid) = self .resolve_path(new_parent_rel) .map_err(|e| self.errno_error(e, "resolve new parent"))?; require_rename_access( |authority| self.require_uid_writable(authority), &uid, &old_parent_uid, &new_parent_uid, ) .map_err(|error| self.errno_error(error, "move access"))?; let name = self .state() .entries .get(&ino) .map(|entry| entry.node.name.clone()) .unwrap_or_default(); if self.queues_relocation(&uid, &new_parent_uid) { if self.state().is_ancestor_of(ino, pino) { return Err(CoreError::invalid("cannot move a folder into itself")); } self.queue_relocation( ino, &uid, &old_parent_uid, &new_parent_uid, &new_parent_rel.join(&name), &name, )?; return Ok(name); } self.move_remote(&uid, &new_parent_uid) .map_err(|e| CoreError::from_api(&e, "move"))?; self.relocate_everywhere(&uid, &new_parent_uid, &name); self.invalidate_parent_listing(rel); self.state().invalidate_listing(pino); Ok(name) } /// Trash a file or folder. `rel` is mountpoint-relative. Forgets the node, /// evicts any cached content, and invalidates the parent listing. fn delete(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let parent_uid = self.source_parent_uid(ino, rel)?; require_node_parent_access( |authority| self.require_uid_writable(authority), &uid, &parent_uid, ) .map_err(|error| self.errno_error(error, "trash access"))?; self.rt .block_on(self.drive.trash_nodes(std::slice::from_ref(&uid))) .and_then(batch::into_unit) .map_err(|e| CoreError::from_api(&e, "trash"))?; // Uploads queued for the node or anything below it would now only fail // against the trash, forever, holding their bytes (`docs/BUGS.md` B102). if let Err(error) = self.discard_queued_ops(&uid) { warn!(%uid, ?error, "remote trash landed but queued-op cleanup failed"); } let name = self.forget_everywhere(&uid).unwrap_or_default(); self.cache.evict(&uid); self.evict_reader(&uid); self.invalidate_parent_listing(rel); self.invalidate_trash(); Ok(name) } /// Remove the node a `rename` is about to replace, so the new name is free /// for the API call that follows. /// /// `rename(2)` promises to replace an existing destination atomically. Proton /// offers no such primitive — `rename_node` refuses a name that is already /// taken — so this is the first half of an emulation that is *not* atomic: /// see [`Core::restore_replaced`] for the other half (bugs.md B13). /// /// A node whose own creation is still queued has never reached the server, so /// dropping its queued ops is the whole removal; nothing goes to the wire and /// it works offline. A create that landed before its ops were dropped left /// none: Drive has the node, so it is trashed there like any other /// (`docs/BUGS.md` B175). /// /// A handle still open on the replaced node keeps reading its bytes, as /// `rename(2)` promises: the inode stays as an unlinked entry and its /// staged bytes, cache and reader go at the final close, exactly as for an /// `unlink` under an open handle. Its queued ops go now, or the drain would /// upload to the node after it was trashed (`docs/BUGS.md` B106). fn remove_replaced(&self, ino: u64, uid: &NodeUid, name: &str) -> Result<(), Errno> { let landed; let uid = if is_local_uid(uid) { if self.is_open_anywhere(uid) { self.withdraw_queued_ops(uid)?; } else { self.discard_queued_ops(uid)?; } self.for_each_state(|st| { st.forget_or_unlink(uid); }); match self.landed_row_uid(ino) { Some(real) => { debug!(local = %uid, %real, name, "create landed under a replace"); landed = real; &landed } None => { debug!(%uid, name, "replaced a node whose create was still queued"); return Ok(()); } } } else { uid }; // Local-first or offline, or the network goes away under the call: // queue the trash. The rename that follows queues as well, behind it. if !self.sends_inline() { return self.queue_trash(uid, name); } self.require_uid_writable(uid)?; if let Err(e) = self .block_on_bounded(self.drive.trash_nodes(std::slice::from_ref(uid))) .and_then(batch::into_unit) { if self.lost_link(&e, "trash a replaced node") { return self.queue_trash(uid, name); } error!(%uid, name, error = %e, "trashing the node a rename replaces failed"); self.log_activity(ActivityKind::Trash, name, e.to_string(), false); return Err(Errno::EIO); } // The node was trashed on the server; withdraw it from every inode // space that had it, not only the mount the rename came through. let open_now = self.is_open_anywhere(uid); self.for_each_state(|st| { st.forget_or_unlink(uid); }); self.hidden.lock().insert(uid.clone()); if open_now { if let Err(error) = self.withdraw_queued_ops(uid) { error!(%uid, "remote replacement landed but queued-op cleanup failed"); return Err(error); } } else { if let Err(error) = self.discard_queued_ops(uid) { error!(%uid, "remote replacement landed but queued-op cleanup failed"); return Err(error); } self.cache.evict(uid); self.evict_reader(uid); } self.invalidate_trash(); // The node is recoverable from the trash, but only if the user knows it // went there — a rename is not an operation anyone expects to trash // something, so this is the only record that it happened. self.log_activity( ActivityKind::Trash, name, "replaced by a rename from the mount", true, ); Ok(()) } /// Whether any mount holds an open handle on `uid`. fn is_open_anywhere(&self, uid: &NodeUid) -> bool { let mut open = false; self.for_each_state(|st| { open |= st .by_uid .get(uid) .and_then(|ino| st.entries.get(ino)) .is_some_and(|entry| entry.open_count > 0); }); open } /// Put back the node [`Core::remove_replaced`] trashed, after the rename it /// was clearing the way for failed anyway. /// /// Best-effort by construction: if the restore also fails there is nothing /// further to try, and the node is still in the trash where `pdfs restore` /// can reach it. Says so loudly in that case, because the alternative is a /// file the user believes was only renamed quietly sitting in the trash. /// /// A restored node comes back to the mount as the ordinary restore brings /// it: out of `hidden`, and through a fresh listing of `parent` in every /// mount that shows it (`docs/BUGS.md` B124). fn restore_replaced(&self, victim: Option<&(u64, NodeUid)>, parent: &NodeUid, name: &str) { let Some((_, uid)) = victim else { return }; if is_local_uid(uid) { // Its queued create was discarded and cannot be reconstructed from // here; the caller's error is what the user acts on. warn!(%uid, name, "a rename failed after discarding a queued node it replaced"); return; } match self .rt .block_on(self.drive.restore_nodes(std::slice::from_ref(uid))) .and_then(batch::into_unit) { Ok(()) => { self.hidden.lock().remove(uid); self.for_each_mount(|st, notify| { if let Some(&ino) = st.by_uid.get(parent) { st.invalidate_listing(ino); notify.inval_entry(ino, name.to_string()); } }); self.invalidate_trash(); debug!(%uid, name, "restored the node a failed rename had replaced"); } Err(e) => { error!(%uid, name, error = %e, "restoring a replaced node failed; it stays in the trash"); self.log_activity(ActivityKind::Restore, name, e.to_string(), false); } } } /// List the account's trash, from the DB. Trashed nodes are outside the /// mounted tree — the FUSE side forgot them when they were trashed — so the /// listing is persisted in its own table rather than derived from `State`, and /// each entry is identified by its uid (its only remaining handle) with an /// empty path. /// /// Stale-while-revalidate, like the photos timeline: a persisted listing comes /// back at DB speed and is refreshed in the background past [`TRASH_TTL`]. /// Our own trash mutations invalidate it outright (see /// [`Core::invalidate_trash`]), so the TTL only covers changes made elsewhere. /// /// The refresh always runs *off* the request: a never-fetched trash waits for /// it, but only for [`TRASH_FIRST_WAIT`], and then answers with the batches /// that have landed so far. Blocking the request on the whole refresh is what /// made a large trash unlistable — the refresh outran the front-end's read /// timeout, the user asked again, and each attempt left another full refresh /// grinding in the daemon (`docs/BUGS.md` B72). fn list_trash(&self) -> CoreResult> { let never_fetched = self.db.state_i64(TRASH_SYNCED_MS).ok().flatten().is_none(); if never_fetched || self.listing_stale(TRASH_SYNCED_MS, TRASH_TTL) { self.spawn_trash_refresh(); } if never_fetched { self.await_trash_refresh(TRASH_FIRST_WAIT); } // Most of a trash comes from a handful of folders. let mut origins: HashMap> = HashMap::new(); Ok(self .db .trash_list() .map_err(CoreError::from)? .into_iter() .map(|item| DirEntry { trashed_from: item.parent_uid.as_ref().and_then(|parent| { origins .entry(parent.clone()) .or_insert_with(|| self.db.node_path(parent).ok().flatten()) .clone() }), trashed_at: item.trashed_at.unwrap_or(0), name: item.name, is_dir: item.is_dir, size: item.size.max(0) as u64, modified: item.mtime, // A trashed node can't be pinned or served from the mount, so its // content is never current cache: report neither. pinned: false, cached: false, uid: item.uid, path: String::new(), role: String::new(), shared_by: String::new(), shared_at: 0, shared_by_unverified: false, issue: None, }) .collect()) } /// Re-fetch the trash listing from the server and persist it. /// /// Materialized in chunks of [`TRASH_MATERIALIZE_CHUNK`], each persisted as /// it lands: decrypting a trashed node costs an S2K unlock, so a trash of a /// few thousand nodes takes minutes, and an all-or-nothing write would throw /// that away on the first failure and show the user nothing until the very /// end. Every batch is a usable listing. async fn refresh_trash(&self) -> CoreResult<()> { let started = Instant::now(); let uids = within_trash_call("enumerate trash", self.drive.enumerate_trash_node_uids()) .await? .map_err(|e| CoreError::from_api(&e, "enumerate trash"))?; info!( count = uids.len(), elapsed_ms = started.elapsed().as_millis() as u64, "trash refresh: enumerated" ); // What the listing held before, and whether it was the whole trash: the // chunks below overwrite it, so both are read up front. let seen = self.db.trash_seen().unwrap_or_default(); let complete = self .db .state_i64(TRASH_COMPLETE_MS) .ok() .flatten() .is_some(); let _ = self.db.clear_state(TRASH_COMPLETE_MS); let now = now_secs(); if uids.is_empty() { self.db.trash_replace(&[]).map_err(CoreError::from)?; } let mut items: Vec = Vec::with_capacity(uids.len()); for chunk in uids.chunks(TRASH_MATERIALIZE_CHUNK) { let chunk_started = Instant::now(); let nodes = within_trash_call("enumerate nodes", self.drive.enumerate_nodes(chunk)) .await? .map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?; items.extend(nodes.into_iter().map(|node| StoredTrash { uid: node.uid.to_string(), name: node.name.clone(), is_dir: node.is_folder(), size: node_size(&node) as i64, mtime: node.modification_time, // The folder the node will go back to, which is also what tells // a restore whether that folder is itself in the trash. parent_uid: node.parent_uid.as_ref().map(|uid| uid.to_string()), trashed_at: first_seen_in_trash(&seen, &node.uid.to_string(), complete, now), })); // Cumulative, so the table is always a prefix of the real trash // rather than a mix of this refresh and the last one. self.db.trash_replace(&items).map_err(CoreError::from)?; self.trash_progress.notify_waiters(); debug!( materialized = items.len(), total = uids.len(), chunk_ms = chunk_started.elapsed().as_millis() as u64, "trash refresh: batch" ); } let _ = self.db.set_state_i64(TRASH_SYNCED_MS, now_ms()); let _ = self.db.set_state_i64(TRASH_COMPLETE_MS, now_ms()); info!( count = items.len(), elapsed_ms = started.elapsed().as_millis() as u64, "trash refresh: done" ); Ok(()) } /// Refresh the trash off the request path. At most one refresh at a time — /// including the one a first-ever [`Core::list_trash`] waits on, so a burst /// of requests against an unfetched trash joins one refresh instead of /// starting one apiece. fn spawn_trash_refresh(&self) { if self.trash_refreshing.swap(true, Ordering::SeqCst) { return; } let core = self.clone(); self.rt.spawn(async move { let refreshed = core.refresh_trash().await; core.trash_refreshing.store(false, Ordering::SeqCst); match refreshed { Ok(_) => core.events.publish(&[Topic::Trash]), Err(e) => warn!(error = %e, "background trash refresh failed"), } // After the flag, so a waiter that wakes on this sees it cleared. core.trash_progress.notify_waiters(); }); } /// Wait for the in-flight trash refresh to make progress, for at most /// `budget`. Returns early when it finishes; never cancels it — the refresh /// is a detached task, so giving up here costs the work nothing. fn await_trash_refresh(&self, budget: Duration) { let deadline = Instant::now() + budget; self.rt.block_on(async { while self.trash_refreshing.load(Ordering::SeqCst) { let Some(left) = deadline.checked_duration_since(Instant::now()) else { break; }; let notified = self.trash_progress.notified(); tokio::pin!(notified); // Register before re-checking the flag: a refresh that finishes // in between must not leave us waiting for a notify that has // already fired. notified.as_mut().enable(); if !self.trash_refreshing.load(Ordering::SeqCst) { break; } if tokio::time::timeout(left, notified).await.is_err() { warn!( budget_s = budget.as_secs(), "trash refresh still running; answering with what has materialized" ); break; } } }); } /// Drop the persisted trash listing's freshness stamp after we changed the /// trash ourselves. The next Trash page then *waits* for a fresh listing /// rather than painting one from before the mutation — the user just made the /// change and is about to look straight at it. fn invalidate_trash(&self) { let _ = self.db.clear_state(TRASH_SYNCED_MS); } /// Drop the persisted photos timeline's freshness stamp, so the next timeline /// read fetches rather than serving what it already has. fn invalidate_photos(&self) { let _ = self.db.clear_state(PHOTOS_SYNCED_MS); } /// Drop one folder's cached child listing (`rel` is mountpoint-relative), so /// the next `ListDir`/`readdir` re-enumerates it from the server. Backs /// [`CtlRequest::Refresh`] with a [`RefreshScope::Dir`] scope. fn refresh_dir(&self, rel: &Path) -> CoreResult<()> { let (ino, _uid) = self.resolve(rel)?; self.state().invalidate_listing(ino); Ok(()) } /// Parse wire uids (`volume~link`) into [`NodeUid`]s, rejecting the whole /// batch if any is malformed — a partial trash mutation is worse than none. fn parse_uids(uids: &[String]) -> CoreResult> { if uids.is_empty() { return Err(CoreError::invalid("no nodes given")); } uids.iter() .map(|u| parse_uid(u).ok_or_else(|| CoreError::invalid(format!("invalid uid: {u}")))) .collect() } /// Restore trashed nodes to the folders they were trashed from, together /// with the rest of the trashed tree they belong to. /// /// The trash is a flat list, but a user deletes a shape: a folder, and often /// items inside it that were trashed separately beforehand. Restoring one of /// those in isolation is not what anyone means by it — a folder would come /// back without the contents that were deleted with it, and a file would come /// back inside a folder that is still in the trash, which is to say nowhere /// the user can see. So a restore takes the whole connected piece: /// every trashed descendant of what was asked for, and every trashed ancestor /// above it. See [`expand_restore`] for the exact rule. /// /// The waves are restored shallowest first, because a node cannot be put back /// under a parent that is still trashed. /// /// The destination parents are read *before* the restore — a restored node /// reappears in a listing the daemon may already have cached, so each one is /// invalidated and re-enumerated on next access. fn restore(&self, uids: &[String]) -> CoreResult { let requested = Self::parse_uids(uids)?; // A listing we could not read is not a reason to refuse the restore: it // only costs the expansion, and the requested nodes still go back. let relations = self.db.trash_parents().unwrap_or_default(); let waves = expand_restore(uids, &relations); let waves: Vec> = if waves.is_empty() { vec![requested.clone()] } else { waves .iter() .map(|wave| { wave.iter() .filter_map(|uid| parse_uid(uid)) .collect::>() }) .filter(|wave: &Vec| !wave.is_empty()) .collect() }; let all: Vec = waves.iter().flatten().cloned().collect(); let parents: Vec = self .rt .block_on(self.drive.enumerate_nodes(&all)) .map_err(|e| CoreError::from_api(&e, "enumerate nodes"))? .into_iter() .filter_map(|n| n.parent_uid) .collect(); // Streamed per-node outcomes: a uid the server refuses (already restored, // gone) leaves the rest of the batch restored, and each node is unhidden // as *its* batch lands rather than after the last one — a daemon killed // half way through a large restore leaves local state describing what the // server actually did. Only a batch that restored nothing is an error the // caller can act on. let mut restored: Vec = Vec::with_capacity(all.len()); let mut first_error: Option = None; for wave in &waves { self.rt .block_on(async { let mut outcomes = std::pin::pin!(self.drive.restore_nodes_streaming(wave)); while let Some(item) = outcomes.next().await { let (uid, outcome) = item?; match outcome { Ok(()) => { self.hidden.lock().remove(&uid); restored.push(uid); } Err(e) => { warn!(%uid, error = %e, "restore failed for a node"); first_error.get_or_insert(e); } } } Ok::<(), ProtonError>(()) }) .map_err(|e| CoreError::from_api(&e, "restore"))?; } if restored.is_empty() && let Some(error) = first_error { return Err(CoreError::from_api(&error, "restore")); } // Keyed by uid, so it applies to every mount: a restored node reappears // in whichever mounts show its parent, not only the primary one. self.for_each_state(|st| { for parent in &parents { if let Some(&ino) = st.by_uid.get(parent) { st.invalidate_listing(ino); } } }); self.invalidate_trash(); Ok(restored.len()) } /// Permanently delete trashed nodes. Irreversible on the server; locally it /// drops any metadata and cached content the node still owns. fn delete_forever(&self, uids: &[String]) -> CoreResult { let parsed = Self::parse_uids(uids)?; // Streamed, and each node's local state is dropped as its batch lands. // A permanent delete is irreversible, so the useful property is that a // daemon interrupted mid-batch has already forgotten exactly the nodes // the server destroyed — never more, never fewer. let mut deleted = 0usize; let mut first_error: Option = None; self.rt .block_on(async { let mut outcomes = std::pin::pin!(self.drive.delete_nodes_streaming(&parsed)); while let Some(item) = outcomes.next().await { let (uid, outcome) = item?; match outcome { Ok(()) => { self.drop_local(std::slice::from_ref(&uid)); deleted += 1; } Err(e) => { warn!(%uid, error = %e, "permanent delete failed for a node"); first_error.get_or_insert(e); } } } Ok::<(), ProtonError>(()) }) .map_err(|e| CoreError::from_api(&e, "delete"))?; if deleted == 0 && let Some(error) = first_error { return Err(CoreError::from_api(&error, "delete")); } self.invalidate_trash(); Ok(deleted) } /// Permanently delete everything in the trash. The uids are listed first so /// the blobs of items trashed by *another* client — which this daemon may /// still hold in its cache — are reclaimed too, not just the ones it trashed. fn empty_trash(&self) -> CoreResult { let uids = self .rt .block_on(self.drive.enumerate_trash_node_uids()) .map_err(|e| CoreError::from_api(&e, "enumerate trash"))?; self.rt .block_on(self.client.empty_trash()) .map_err(|e| CoreError::from_api(&e, "empty trash"))?; self.drop_local(&uids); self.invalidate_trash(); Ok(uids.len()) } /// Forget every trace of nodes that no longer exist anywhere: their inode and /// DB row, and their cached content. fn drop_local(&self, uids: &[NodeUid]) { // These nodes are gone from the server, and a sync-folder fork showing // the same uid would otherwise keep serving them (`docs/BUGS.md` B74). self.for_each_state(|st| { for uid in uids { st.forget(uid); } }); for uid in uids { self.cache.evict(uid); self.evict_reader(uid); } } /// Create a folder named `name` under the mountpoint-relative `parent_rel`. /// Interns the new node directly so it shows up without a re-enumeration. fn create_folder(&self, parent_rel: &Path, name: &str) -> CoreResult { if name.is_empty() || name.contains('/') { return Err(CoreError::invalid(format!("invalid name: {name:?}"))); } let (pino, parent_uid) = self.resolve(parent_rel)?; self.ensure_children(pino) .map_err(|e| self.errno_error(e, "enumerate"))?; self.require_uid_writable(&parent_uid) .map_err(|error| self.errno_error(error, "create folder access"))?; let new_uid = self .rt .block_on( self.drive .create_folder(&parent_uid, name, Some(now_secs())), ) .map_err(|e| CoreError::from_api(&e, "create folder"))?; let node = self .fetch_minted_node(&new_uid, name) .ok_or_else(|| minted_but_unlisted(name))?; let mut st = self.state(); let ino = st.intern(pino, node); if let Some(kids) = st.children.get_mut(&pino) && !kids.contains(&ino) { kids.push(ino); } drop(st); self.flush_access_changes(); Ok(name.to_string()) } // ---- activity log ----------------------------------------------------- /// Append one entry to the activity log. Callable from any thread (the sync /// engine and the bulk uploader both log from background tasks). A failed /// write is logged and dropped: the feed is a record of work, never a reason /// to fail the work itself. pub(crate) fn log_activity( &self, kind: ActivityKind, target: impl Into, // `Display` rather than `Into` so a classified `CoreError` can be // logged as-is, without the caller flattening it first. detail: impl std::fmt::Display, ok: bool, ) { let entry = ActivityEntry { time: now_secs(), kind, target: target.into(), detail: detail.to_string(), ok, }; if let Err(e) = self.db.activity_add(&entry) { warn!(error = ?e, "could not record activity"); return; } // A sync pass is how a conflict copy reaches the listing, and the sweep // reports the ones it finds as conflicts. if matches!(entry.kind, ActivityKind::Conflict | ActivityKind::Sync) { self.events.publish(&[Topic::Activity, Topic::Conflicts]); } else { self.events.publish(&[Topic::Activity]); } } /// The recent activity, newest first, capped at `limit` entries. fn list_activity(&self, limit: usize) -> Vec { match self.db.activity_list(limit) { Ok(items) => items, Err(e) => { warn!(error = ?e, "could not read activity"); Vec::new() } } } // ---- live sync progress ----------------------------------------------- /// Start tracking a reconcile pass over `folder_id`, in [`SyncPhase::Scanning`]. pub(crate) fn progress_begin(&self, folder_id: i64) { self.sync_progress.lock().insert( folder_id, SyncProgress { phase: SyncPhase::Scanning, done: 0, total: 0, current: String::new(), }, ); } /// Apply `f` to a folder's live progress, if a pass is running for it. fn progress_update(&self, folder_id: i64, f: impl FnOnce(&mut SyncProgress)) { if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) { f(p); } } /// Set the number of items the scan expects to check, from the size of the /// last pass's baseline. Only an estimate — the folder may have grown since — /// but it turns the scan from an indeterminate pulse into a bar that moves, /// which is the difference between "it's stuck" and "it's working" on a folder /// whose walk takes minutes. pub(crate) fn progress_scan_total(&self, folder_id: i64, n: usize) { self.progress_update(folder_id, |p| p.total = n); } /// Note that the scan has checked one more item, named `name`. pub(crate) fn progress_scanned(&self, folder_id: i64, name: &str) { self.progress_update(folder_id, |p| { p.done += 1; p.current = name.to_string(); }); } /// Note that `n` more items have been queued for this pass, and that it has /// moved on from scanning to applying the diff. The scan's counts are dropped: /// they measured a different quantity (items checked, not items to apply), so /// carrying them over would start the applying bar at a meaningless fraction. pub(crate) fn progress_queued(&self, folder_id: i64, n: usize) { self.progress_update(folder_id, |p| { if p.phase == SyncPhase::Scanning { p.phase = SyncPhase::Applying; p.done = 0; p.total = 0; p.current.clear(); } p.total += n; }); } /// Note that work has started on `name` (shown as the pass's current item). pub(crate) fn progress_started(&self, folder_id: i64, name: &str) { self.progress_update(folder_id, |p| p.current = name.to_string()); } /// Note that one queued item finished, whether it succeeded or not. pub(crate) fn progress_finished(&self, folder_id: i64) { self.progress_update(folder_id, |p| { p.done += 1; p.current.clear(); }); } /// Stop tracking a pass — no progress is reported for the folder until the /// next [`progress_begin`](Self::progress_begin). pub(crate) fn progress_end(&self, folder_id: i64) { self.sync_progress.lock().remove(&folder_id); } } fn publish_virtual_root_in_listing( db: &Db, state: &mut State, parent_ino: u64, snapshot: &RootListingSnapshot, plan: VirtualRootPlan, ) -> Result<(), Errno> { if !snapshot.is_current(state, parent_ino) { return Err(Errno::EAGAIN); } db.publish_virtual_root(SHARED_WITH_ME_NAME, &plan.node) .map_err(|error| { error!(%error, "atomically publishing synthetic shared root failed"); Errno::EIO })?; let uid = plan.node.uid.clone(); state.share_access.insert(uid, Access::Viewer); let virtual_ino = state.intern_mem(parent_ino, plan.node); let published = reconcile_virtual_root_in_listing(state, parent_ino, snapshot, virtual_ino, plan.visible); debug_assert!( published, "snapshot cannot change while the state lock is held" ); Ok(()) } /// Update the synthetic dentry only while the parent listing is still known. /// /// Event invalidation can remove the listing after its real names were captured. /// Recreating it here would publish a synthetic-only partial snapshot as complete. fn reconcile_virtual_root_in_listing( state: &mut State, parent_ino: u64, snapshot: &RootListingSnapshot, virtual_ino: u64, visible: bool, ) -> bool { if !snapshot.is_current(state, parent_ino) { return false; } let Some(children) = state.children.get_mut(&parent_ino) else { return false; }; children.retain(|ino| *ino != virtual_ino); if visible { children.push(virtual_ino); } true } fn resolve_anywhere_with( raw_uid: &str, resident: impl FnOnce(&NodeUid) -> bool, mirrored: impl FnOnce(&NodeUid) -> pdfs_core::Result, ) -> CoreResult { let uid = parse_uid(raw_uid).ok_or_else(|| CoreError::invalid(format!("invalid uid: {raw_uid}")))?; if is_local_uid(&uid) || is_virtual_uid(&uid) { return Err(CoreError::invalid(format!("reserved uid: {raw_uid}"))); } if resident(&uid) { return Ok(uid); } if mirrored(&uid)? { return Ok(uid); } Err(CoreError::not_found(format!( "node is not present in any location: {raw_uid}" ))) } /// Map a [`MemberRole`] to its wire string. fn role_to_str(role: MemberRole) -> &'static str { match role { MemberRole::Viewer => "viewer", MemberRole::Editor => "editor", MemberRole::Admin => "admin", MemberRole::Inherited => "inherited", } } /// Parse a wire role string into a [`MemberRole`]. "inherited" is read-only and /// rejected here, since it cannot be sent when inviting or updating. fn role_from_str(role: &str) -> CoreResult { match role.to_lowercase().as_str() { "viewer" => Ok(MemberRole::Viewer), "editor" => Ok(MemberRole::Editor), "admin" => Ok(MemberRole::Admin), other => Err(CoreError::invalid(format!("invalid role: {other}"))), } } /// Map a device type to a display string. fn device_type_str(t: proton_drive_rs::DeviceType) -> &'static str { match t { proton_drive_rs::DeviceType::Windows => "Windows", proton_drive_rs::DeviceType::MacOs => "MacOs", proton_drive_rs::DeviceType::Linux => "Linux", } } /// Convert an SDK [`PublicLink`](proton_drive_rs::PublicLink) into the wire form. fn public_link_info(link: proton_drive_rs::PublicLink) -> PublicLinkInfo { PublicLinkInfo { id: link.public_link_id, url: link.url, role: role_to_str(link.role).to_string(), expires: link.expiration_time, has_password: link.has_custom_password, created: link.creation_time, } } /// Parse a `volume~link` uid display string back into a [`NodeUid`]. Front-ends /// receive uids as strings over the control socket and pass them back verbatim. pub(crate) fn parse_uid(s: &str) -> Option { let (vol, link) = s.split_once('~')?; if vol.is_empty() || link.is_empty() || link.contains('~') { return None; } Some(NodeUid::new(VolumeId::from(vol), LinkId::from(link))) } /// Current wall-clock time as epoch seconds (0 if the clock is before the epoch). fn now_secs() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs() as i64) .unwrap_or(0) } /// Current wall-clock time as epoch milliseconds, the resolution `pending_op` /// timestamps and backoff deadlines are kept in. fn now_millis() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as i64) .unwrap_or(0) } /// Parse a [`NodeUid`] back from its `Display` form (`volume~link`), which is /// how one is persisted in `pending_op.uid` and a [`StagedWrite`] sidecar. The /// SDK has no `FromStr`, and neither id contains a `~`. fn parse_node_uid(s: &str) -> Option { parse_uid(s) } /// The uid a node made on this machine goes by until it lands, so it can be /// interned, listed and written to before the server has ever heard of it /// (offline.md Phase 3b): its row's local id, as [`pdfs_core::db::local_uid`] /// has it. fn local_node_uid(lid: i64) -> NodeUid { NodeUid::new(VolumeId::from(LOCAL_VOLUME), LinkId::from(lid.to_string())) } /// Whether this node exists only on this machine, so far. Such a uid is /// meaningless to the API and must never be sent to it. fn is_local_uid(uid: &NodeUid) -> bool { uid.volume_id.as_str() == LOCAL_VOLUME } /// [`is_local_uid`] for a uid in its persisted `Display` form. fn is_local_uid_str(s: &str) -> bool { s.split_once('~') .is_some_and(|(vol, _)| vol == LOCAL_VOLUME) } /// The API's response code for a failed call, when it failed *at* the API. /// /// Takes `&dyn Error` so it reads a [`ProtonError`] equally well through the /// boxes the drain deals in, where the concrete type survives but the static one /// does not. `None` covers both "not an API error at all" (a transport failure, /// which is what being offline looks like) and "not a `ProtonError`". fn api_code(e: &(dyn std::error::Error + 'static)) -> Option { match e.downcast_ref::() { Some(ProtonError::Api(api)) => Some(api.code), _ => None, } } /// How often [`Core::ensure_children`] takes a listing that changed while it /// was being published. const LISTING_ATTEMPTS: usize = 4; /// How often a rename or move that Drive calls out of date is retried, and how /// long apart. const OUT_OF_DATE_RETRIES: u32 = 20; const OUT_OF_DATE_RETRY_DELAY: Duration = Duration::from_millis(500); /// Run a remote rename or move, retrying while Drive answers /// `InvalidRequirements`. /// /// Both send the node's current name hash. The SDK takes it from its entity /// cache, which still holds the old hash when the node changed since it was /// cached, and Drive then rejects the request as "out of date" (docs/BUGS.md /// B111, B121). `wait` drops that entry before the next attempt. Any other /// error returns at once. fn retry_out_of_date( mut call: impl FnMut() -> Result<(), ProtonError>, mut wait: impl FnMut(), ) -> Result<(), ProtonError> { let mut attempts = 0u32; loop { match call() { Err(e) if api_code(&e) == Some(ResponseCode::InvalidRequirements) && attempts < OUT_OF_DATE_RETRIES => { attempts += 1; debug!(attempts, error = %e, "Drive called the request out of date; retrying"); wait(); } result => return result, } } } /// Whether a call failed because the name it asked for is already in use. /// /// The queue makes this reachable in a way the synchronous path never was: a /// mutation queued while offline is applied against a server that may have /// gained a file of that name in the meantime. fn is_already_exists(e: &(dyn std::error::Error + 'static)) -> bool { api_code(e) == Some(ResponseCode::AlreadyExists) } /// Whether a call failed because the node it addressed is not there. /// The error for a folder Drive created but did not hand back, where there is /// no queue to adopt it later. Saying it was created keeps the caller from /// taking a retry's "already exists" for someone else's folder. fn minted_but_unlisted(name: &str) -> CoreError { CoreError::remote(format!( "created folder {name}, but Drive did not list it yet; it shows on the next refresh" )) } fn is_gone(e: &(dyn std::error::Error + 'static)) -> bool { api_code(e) == Some(ResponseCode::DoesNotExist) } /// A variant of `name` to fall back on when the remote already has that name. /// /// Deliberately the same shape the sync engine uses for its conflict copies /// (`sync.rs`, `conflict_path`), so the two halves of the product name the same /// situation the same way and a user only has to learn it once. fn conflict_name(name: &str, stamp: i64) -> String { let path = Path::new(name); let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(name); match path.extension().and_then(|s| s.to_str()) { Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"), None => format!("{stem} (sync-conflict {stamp})"), } } /// Fabricate the node the server would have returned, for a `create`/`mkdir` /// that could not reach it. Everything the kernel asks about a fresh node — /// name, kind, size, times — is knowable locally; the uid is the only invention, /// and the drain replaces it with the real one. fn local_node(uid: NodeUid, parent_uid: NodeUid, name: String, is_dir: bool) -> Node { let now = now_secs(); Node { uid, parent_uid: Some(parent_uid), kind: if is_dir { NodeKind::Folder } else { NodeKind::File { media_type: media_type_for(&name).to_string(), total_size_on_storage: 0, // No revision has been sealed: nothing has been uploaded yet. active_revision_state: None, active_revision_id: None, content_sha1: None, claimed_size: Some(0), claimed_modification_time: None, } }, name, creation_time: now, modification_time: now, trashed: false, is_shared: false, is_shared_publicly: false, signature_email: None, membership: None, // A locally fabricated node is on the main volume, never a photo/album. photo: None, album: None, // Nothing signed it: it has never been near the crypto layer. verification: Default::default(), direct_role: None, share_id: None, } } /// This machine's hostname, used to name (and later recover) its Proton Drive /// Device. Reads the live kernel hostname, falling back to a generic label. fn this_hostname() -> String { std::fs::read_to_string("/proc/sys/kernel/hostname") .ok() .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .unwrap_or_else(|| "Linux device".to_string()) } /// Whether `dir` has no entries. fn dir_is_empty(dir: &Path) -> std::io::Result { Ok(std::fs::read_dir(dir)?.next().is_none()) } /// Delete everything inside `dir` but keep `dir` itself (it stays as the FUSE /// mountpoint). Used when a `mirror` folder flips to `ondemand`: the local files /// are the disk we're reclaiming (devices.md Phase 3). fn evict_dir_contents(dir: &Path) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let path = entry?.path(); if path.is_dir() && !path.is_symlink() { std::fs::remove_dir_all(&path)?; } else { std::fs::remove_file(&path)?; } } Ok(()) } /// Convert a stored synced folder into its wire form for the front-ends, with /// the live progress of its pass when one is running. fn sync_folder_info( f: StoredSyncFolder, progress: Option, paused: bool, ) -> SyncFolderInfo { SyncFolderInfo { id: f.id, local_path: f.local_path, remote_uid: f.remote_uid, mode: f.mode, pending_mode: f.pending_mode, state: f.state, last_sync: f.last_sync, progress, paused, } } /// A coarse MIME type guessed from a file name's extension; Proton stores this /// on the node but an exact value is not required for correctness. fn media_type_for(name: &str) -> &'static str { let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase()); match ext.as_deref() { Some("txt" | "md" | "log") => "text/plain", Some("png") => "image/png", Some("jpg" | "jpeg") => "image/jpeg", Some("gif") => "image/gif", Some("pdf") => "application/pdf", Some("json") => "application/json", Some("html" | "htm") => "text/html", _ => "application/octet-stream", } } /// The plaintext size, in bytes, that a node reports. /// Overwrite each file node's `claimed_size` with the optimistic size of its /// queued write, where it has one. Folders and nodes with nothing queued are /// left alone. See [`Core::stamp_pending_sizes`] for why this exists. fn apply_pending_sizes(nodes: &mut [Node], sizes: &HashMap) { for node in nodes { if let Some(&len) = sizes.get(&node.uid) && let NodeKind::File { claimed_size, .. } = &mut node.kind { *claimed_size = Some(len as i64); } } } /// A size upgrade in flight for one folder, so the `getattr`s that need its /// result wait for the one batch instead of each fetching its own node. /// /// A plain `Condvar` rather than a channel: the waiters do not want a value, /// only the edge, and there may be hundreds of them for one folder. #[derive(Default)] struct Progress { /// The whole batch has been applied (or failed). Nothing more is coming. done: bool, /// Bumped every time a chunk lands, so a waiter can tell "no progress since /// I last looked" from "progress happened while I was looking". generation: u64, } /// A size upgrade in flight for one folder, so the `getattr`s and `lookup`s that /// need its result wait for the one batch instead of each fetching its own node. /// /// Waiters are released **per chunk**, not once at the end: a waiter only cares /// about its own file, and waiting for the other 792 is what let a large folder /// outrun the timeout (bugs.md B14). #[derive(Default)] struct SizeUpgrade { inner: Mutex, ready: Condvar, /// The batch, while it waits for a thread because [`MAX_SIZE_UPGRADES`] /// were already running, with when it was queued so the oldest goes first. /// `None` once a thread has it. queued: Mutex)>>, } /// What [`Core::upgrade_sizes`] does with one folder's batch. enum SizeUpgradeClaim { /// A batch for this folder is already running or queued; wait on it. Joined(Arc), /// The cap is reached. The batch waits for a running thread to take it. Queued(Arc), /// Start a thread for the batch. Start(Arc, Vec), } /// Register the batch for `key` in `in_flight` and decide who runs it. /// /// Past the cap this used to return nothing, and the caller answered with the /// provisional size at once. Nothing retried the folder until the next `stat`, /// so a recursive walk over more than [`MAX_SIZE_UPGRADES`] fresh folders showed /// ciphertext sizes for most of them (bugs.md B100). Queued, the batch is still /// fetched, and a waiter gets the real size unless [`SizeUpgrade::WAIT`] runs out. fn claim_size_upgrade( in_flight: &mut HashMap>, key: u64, uids: Vec, ) -> SizeUpgradeClaim { if let Some(existing) = in_flight.get(&key) { return SizeUpgradeClaim::Joined(existing.clone()); } let running = in_flight .values() .filter(|slot| slot.queued.lock().is_none()) .count(); let slot = Arc::new(SizeUpgrade::default()); in_flight.insert(key, slot.clone()); if running >= MAX_SIZE_UPGRADES { *slot.queued.lock() = Some((Instant::now(), uids)); return SizeUpgradeClaim::Queued(slot); } SizeUpgradeClaim::Start(slot, uids) } /// Retire the finished batch for `done` and hand its thread the oldest queued /// batch, if there is one. fn next_size_upgrade( in_flight: &mut HashMap>, done: u64, ) -> Option<(u64, Vec, Arc)> { in_flight.remove(&done); let (key, slot) = in_flight .iter() .filter_map(|(key, slot)| slot.queued.lock().as_ref().map(|(at, _)| (*at, *key, slot))) .min_by_key(|(at, _, _)| *at) .map(|(_, key, slot)| (key, slot.clone()))?; let (_, uids) = slot.queued.lock().take()?; Some((key, uids, slot)) } impl SizeUpgrade { /// How long a caller will wait for a real size before answering with the /// provisional one. /// /// A `stat` that never returns is far worse than one that is briefly wrong: /// on timeout the caller falls back to the pre-fix behaviour rather than /// wedging whatever is listing the directory. const WAIT: Duration = Duration::from_secs(10); /// Whether the batch has ended, however it ended. A waiter that sees this /// has nothing left to wait for. fn is_finished(&self) -> bool { self.inner.lock().done } /// Announce that a chunk has been applied. Every waiter re-checks its own /// node; the ones it resolved return, the rest go back to sleep. fn chunk_done(&self) { self.inner.lock().generation += 1; self.ready.notify_all(); } /// Block until the batch has ended or `deadline` has passed. Only for a /// caller that is not on a FUSE or [`Workers`] thread; those park their /// reply with [`Core::await_size`] instead. fn wait_finished(&self, deadline: Instant) { let mut progress = self.inner.lock(); while !progress.done { if self.ready.wait_until(&mut progress, deadline).timed_out() { return; } } } /// Release every waiter for good. The worker must reach this on all paths. fn finish(&self) { let mut progress = self.inner.lock(); progress.done = true; progress.generation += 1; drop(progress); self.ready.notify_all(); } } /// How often [`SizeWaitQueue::serve`] re-tests its parked waiters. /// /// The predicates read `state`, which the applying thread writes under its own /// lock, so this polls rather than waiting on each slot's condvar: one thread /// cannot wait on a hundred condvars, and the whole point of the queue is that /// there is only one thread. Twenty milliseconds is far below the round trip a /// waiter is waiting on and only ticks while something is actually parked. const SIZE_WAIT_POLL: Duration = Duration::from_millis(20); /// A reply held back until a size upgrade resolves the node it describes. struct SizeWaiter { /// The batch this reply is waiting on. When it ends, so does the wait. slot: Arc, /// Whether *this* waiter's node now has a real size. /// /// Called with no queue lock held: it reaches into `state`, and the thread /// applying a chunk holds `state` before it announces the chunk — taking /// them in the other order would close the cycle. resolved: Box bool + Send>, /// When to answer with the provisional size anyway. A `stat` that never /// returns is far worse than one that is briefly wrong. deadline: Instant, /// Sends the FUSE reply. Runs on the serving thread. respond: Box, } #[derive(Default)] struct Parked { waiters: Vec, /// Whether a thread is currently serving `waiters`. The thread clears this /// as it retires, so the next park starts a new one. running: bool, } /// Every reply parked on a size upgrade, served by at most one thread. /// /// This exists because waiting used to happen on the [`Workers`] pool. Sizes /// are single-flighted per folder, so an `ls -l` across N cold folders parks a /// waiter per entry — and each of those held a pool thread asleep on a condvar /// for up to [`SizeUpgrade::WAIT`]. With enough folders that is every thread in /// both lanes, including the ones reserved for metadata precisely so that this /// could not happen (audit F4). Parking the `Reply` instead costs one thread /// for the whole mount, and only while something is parked. #[derive(Default)] struct SizeWaitQueue { parked: Mutex, /// Woken when a waiter is added, so a serving thread re-computes how long /// it may sleep. added: Condvar, } impl SizeWaitQueue { /// Park `waiter`. Returns whether the caller must start a serving thread. fn park(&self, waiter: SizeWaiter) -> bool { let mut parked = self.parked.lock(); parked.waiters.push(waiter); if parked.running { drop(parked); self.added.notify_all(); return false; } parked.running = true; true } /// Answer every waiter whose node has resolved, whose batch has ended, or /// whose deadline has passed; sleep until the next one might. Returns when /// nothing is parked any more. fn serve(&self) { loop { // Taken out of the lock wholesale: the predicates and the replies // both take locks of their own, and none of them may be called // while this one is held. let taken = std::mem::take(&mut self.parked.lock().waiters); let now = Instant::now(); let mut keep = Vec::with_capacity(taken.len()); for waiter in taken { if now >= waiter.deadline || waiter.slot.is_finished() || (waiter.resolved)() { (waiter.respond)(); } else { keep.push(waiter); } } let mut parked = self.parked.lock(); parked.waiters.append(&mut keep); if parked.waiters.is_empty() { parked.running = false; return; } let until = parked .waiters .iter() .map(|w| w.deadline) .min() .unwrap_or(now) .min(Instant::now() + SIZE_WAIT_POLL); self.added.wait_until(&mut parked, until); } } /// Answer everything parked, immediately, and retire the serving thread. /// Used when there is no thread to serve with: a provisional size now beats /// a reply that never comes. fn answer_all(&self) { let taken = { let mut parked = self.parked.lock(); parked.running = false; std::mem::take(&mut parked.waiters) }; for waiter in taken { (waiter.respond)(); } } } /// Whether a `rename` may replace an existing destination, per POSIX. /// /// Split out from the handler because it is the part that is pure and the part /// that is dangerous: every `Err` here is a refusal that happens *before* /// anything is trashed, and getting one wrong turns a refusal into the /// destruction of the destination (bugs.md B13). /// /// `dst_empty` is only meaningful when `dst_dir`; pass `true` otherwise. fn check_replaceable(src_dir: bool, dst_dir: bool, dst_empty: bool) -> Result<(), Errno> { match (src_dir, dst_dir) { // A non-directory may not replace a directory, or vice versa. (false, true) => Err(Errno::EISDIR), (true, false) => Err(Errno::ENOTDIR), // Proton trashes a folder with its whole subtree, so replacing a // non-empty one would silently take its contents with it. POSIX says // ENOTEMPTY, which is also the safe answer. (true, true) if !dst_empty => Err(Errno::ENOTEMPTY), _ => Ok(()), } } /// Whether `node` is a file Drive lists without an active revision, as it does /// one whose create has not been committed. fn is_unrevised(node: &Node) -> bool { matches!( node.kind, NodeKind::File { active_revision_id: None, .. } ) } fn node_size(node: &Node) -> u64 { match &node.kind { NodeKind::Folder => 0, NodeKind::File { claimed_size, total_size_on_storage, .. } => claimed_size.unwrap_or(*total_size_on_storage).max(0) as u64, } } /// `call`, failed with a remote error once it has run for /// [`TRASH_CALL_TIMEOUT`]. async fn within_trash_call(what: &str, call: impl Future) -> CoreResult { tokio::time::timeout(TRASH_CALL_TIMEOUT, call) .await .map_err(|_| { CoreError::new( ErrorKind::Remote, format!("{what}: no answer within {}s", TRASH_CALL_TIMEOUT.as_secs()), ) }) } /// When a node a trash refresh found went to the trash, as far as the daemon /// can tell. A node the stored listing already had keeps the time it had. A new /// one went there since the last refresh, so now is close, but only when that /// refresh saw the whole trash (`complete`): after a first or interrupted one, /// a node missing from the listing may be years old, and no date beats a wrong /// one. fn first_seen_in_trash( seen: &HashMap>, uid: &str, complete: bool, now: i64, ) -> Option { match seen.get(uid) { Some(at) => *at, None if complete => Some(now), None => None, } } /// Expand a restore request across the trashed tree, and order it so a parent is /// always restored before its children. /// /// `relations` is the persisted trash listing as `(uid, parent_uid)` pairs; a /// parent that is not itself in the listing is a live folder, which ends the /// walk upward. The rule is: /// /// - every trashed descendant of a requested node joins the restore, so a folder /// comes back with the items that were trashed inside it; /// - every trashed ancestor of a requested node joins it too, so a file never /// lands in a folder that is still in the trash; /// - a requested uid the listing does not know about is restored on its own, /// because a stale or half-materialised listing must not silently drop it. /// /// The result is one wave per depth, shallowest first. Waves are disjoint and /// together contain each uid exactly once. fn expand_restore( requested: &[String], relations: &[(String, Option)], ) -> Vec> { let parents: HashMap<&str, Option<&str>> = relations .iter() .map(|(uid, parent)| (uid.as_str(), parent.as_deref())) .collect(); let mut children: HashMap<&str, Vec<&str>> = HashMap::new(); for (uid, parent) in relations { // Only a trashed parent: a live one keeps its child as a root here. if let Some(parent) = parent.as_deref() && parents.contains_key(parent) { children.entry(parent).or_default().push(uid.as_str()); } } let mut selected: HashSet<&str> = HashSet::new(); for uid in requested { let uid = uid.as_str(); if !parents.contains_key(uid) { selected.insert(uid); continue; } // Upward: every trashed ancestor, so the destination folder exists again. let mut cursor = Some(uid); while let Some(current) = cursor { // A parent outside the listing is live: the walk stops under it. if !parents.contains_key(current) || !selected.insert(current) { break; } cursor = parents.get(current).copied().flatten(); } // Downward: the whole trashed subtree, so the folder comes back full. let mut stack = vec![uid]; while let Some(current) = stack.pop() { for child in children.get(current).into_iter().flatten() { if selected.insert(child) { stack.push(child); } } } } // Depth within the *trash*, not within the drive: a node whose parent is live // (or unknown) is at depth 0 and can be restored straight away. let mut waves: Vec> = Vec::new(); for uid in &selected { let mut depth = 0usize; let mut cursor = parents.get(uid).copied().flatten(); // Only trashed ancestors count, and the walk is bounded by the listing's // size: server data that somehow describes a cycle must not spin here. while let Some(current) = cursor && parents.contains_key(current) && depth < parents.len() { depth += 1; cursor = parents.get(current).copied().flatten(); } if waves.len() <= depth { waves.resize(depth + 1, Vec::new()); } waves[depth].push((*uid).to_string()); } // Stable output, so a restore of the same trash is the same sequence of calls // every time — the batches are what shows up in the log and in the tests. for wave in &mut waves { wave.sort(); } waves.retain(|wave| !wave.is_empty()); waves } /// The server revision id of a node's active revision, if it is a file that has /// one. The stable identity the drain conflict-checks against (see [`Baseline`]). fn node_revision_id(node: &Node) -> Option { match &node.kind { NodeKind::Folder => None, NodeKind::File { active_revision_id, .. } => active_revision_id.clone(), } } /// The plaintext content SHA-1 of a file node, if its active revision carried one. /// A download-free content fingerprint the conflict sweep uses to prove two files /// hold identical bytes before it removes one. fn node_content_sha1(node: &Node) -> Option { match &node.kind { NodeKind::Folder => None, NodeKind::File { content_sha1, .. } => content_sha1.clone(), } } /// `"1 file"` / `"3 files"` — a count with a correctly pluralised noun, for /// human-readable activity-log lines. fn count_noun(n: usize, one: &str, many: &str) -> String { format!("{n} {}", if n == 1 { one } else { many }) } /// Seed a truncate scratch file from a complete queued revision. /// /// `File` writes use an explicit offset so this is independent of either /// descriptor's cursor and remains correct when the destination came from the /// cache's scratch allocator. fn copy_pending_for_truncate(pending: &PendingRevision, destination: &File) -> std::io::Result<()> { let source = File::open(&pending.path)?; destination.set_len(0)?; let mut offset = 0u64; let mut buffer = vec![0u8; 1024 * 1024]; loop { let count = source.read_at(&mut buffer, offset)?; if count == 0 { break; } destination.write_all_at(&buffer[..count], offset)?; offset += count as u64; } destination.set_len(offset) } /// Whether a rename must be represented as a durable desired end state. /// Local-first, offline, or into a not-yet-uploaded directory, the API is not /// called in the syscall. Only remote-to-remote renames sent inline /// ([`Core::sends_inline`]) stay synchronous, so a successful kernel reply means /// the remote namespace has already reached the same end state. fn rename_needs_queue( inline: bool, destination_is_local: bool, _parent_changed: bool, _name_changed: bool, ) -> bool { !inline || destination_is_local } /// Convert one kernel pathname component into the UTF-8 name accepted by Drive. /// Linux filesystems conventionally cap a component at `NAME_MAX` (255 bytes), /// but the FUSE kernel path can pass a longer component through to userspace. fn fuse_name(name: &OsStr) -> Result { let bytes = name.as_bytes(); if bytes.len() > 255 { return Err(Errno::ENAMETOOLONG); } if bytes.is_empty() || bytes == b"." || bytes == b".." { return Err(Errno::EINVAL); } name.to_str().map(str::to_owned).ok_or(Errno::EILSEQ) } fn node_visible(node: &Node, folder_uid: &NodeUid, hidden: &HashSet) -> bool { !node.trashed && node.uid != *folder_uid && !hidden.contains(&node.uid) } /// The children `entries` hold for each folder in `listed`, gathered in one /// pass over them. /// /// The root is its own parent (that is what `..` resolves to), so it would /// count as its own child; a directory listed inside itself makes the kernel /// fail the lookup with EIO, taking the whole listing down with it. /// /// A pass per listed folder took 1.7 s on a tree of 37,000 nodes with 4,600 of /// them listed, at every start before the mount came up (`docs/BUGS.md` B167). fn listed_children(entries: &HashMap, listed: Vec) -> HashMap> { let mut children: HashMap> = listed.into_iter().map(|ino| (ino, Vec::new())).collect(); for (&ino, entry) in entries { if ino != entry.parent && !entry.node.trashed && let Some(kids) = children.get_mut(&entry.parent) { kids.push(ino); } } children } /// Lay what is queued for a folder over Drive's listing of it, from /// `Db::queued_children`: a node a queued rename moved away is dropped, and a /// node a queued create, mkdir or rename put there is listed as the DB has it. /// /// That node holds its name here: a create Drive already has but whose answer /// is still on its way, or which the drain is still reading back, is listed by /// Drive under its new uid and by the queue under its stand-in. Both were /// listed, and a 128-file folder listed 129 (`docs/BUGS.md` B164). fn overlay_queued(nodes: &mut Vec, queued: Vec, gone: &HashSet) { let here: HashSet = queued.iter().map(|node| node.uid.to_string()).collect(); let names: HashSet<&str> = queued.iter().map(|node| node.name.as_str()).collect(); nodes.retain(|node| { let uid = node.uid.to_string(); !gone.contains(&uid) && !here.contains(&uid) && !names.contains(node.name.as_str()) }); nodes.extend(queued); } /// Bytes rendered with a binary unit and one decimal place (e.g. `"1.2 GB"`), /// for the activity log. Uses 1024-based steps but the shorter SI labels. fn human_bytes(bytes: u64) -> String { const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; let mut val = bytes as f64; let mut unit = 0; while val >= 1024.0 && unit < UNITS.len() - 1 { val /= 1024.0; unit += 1; } if unit == 0 { format!("{bytes} B") } else { format!("{val:.1} {}", UNITS[unit]) } } /// A compact elapsed-time label for the activity log: `"820ms"`, `"43s"`, or /// `"2m 5s"`. fn human_duration(d: Duration) -> String { let secs = d.as_secs(); if secs == 0 { format!("{}ms", d.as_millis()) } else if secs < 60 { format!("{secs}s") } else { format!("{}m {}s", secs / 60, secs % 60) } } /// Wall clock in milliseconds since the epoch — the unit the persisted listings /// stamp their freshness in. fn now_ms() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as i64) .unwrap_or(0) } fn unix_secs(secs: i64) -> SystemTime { if secs >= 0 { UNIX_EPOCH + Duration::from_secs(secs as u64) } else { UNIX_EPOCH - Duration::from_secs(secs.unsigned_abs()) } } #[cfg(test)] mod merge_over_pending_tests { use super::{PendingRevision, merge_over_pending}; use pdfs_core::cache::{Baseline, StagedWrite}; use std::io::Write as _; use std::path::PathBuf; /// A unique temp directory removed on drop; avoids a dev-dependency, as in /// `state.rs`. struct TempDir(PathBuf); impl TempDir { fn new() -> Self { use std::sync::atomic::{AtomicU64, Ordering}; static N: AtomicU64 = AtomicU64::new(0); let p = std::env::temp_dir().join(format!( "pdfs-merge-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed) )); std::fs::create_dir_all(&p).unwrap(); TempDir(p) } } impl Drop for TempDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } fn staged(len: u64, base_size: u64, authored: &[(u64, u64)]) -> StagedWrite { StagedWrite { uid: "vol~link".into(), len, base_size, base_mtime: 100, complete: StagedWrite::covers(authored, len), authored: authored.to_vec(), based_on: Some(Baseline { mtime: 100, size: base_size, hash: None, revision_id: Some("r1".into()), }), } } /// Write a file of `len` bytes with `fill` at every offset in `ranges`. fn blob( dir: &std::path::Path, name: &str, len: u64, ranges: &[(u64, u64)], fill: u8, ) -> std::path::PathBuf { let path = dir.join(name); let mut bytes = vec![0u8; len as usize]; for &(s, e) in ranges { bytes[s as usize..e as usize].fill(fill); } let mut f = std::fs::File::create(&path).unwrap(); f.write_all(&bytes).unwrap(); path } /// The case the merge exists for: two small edits to different parts of a /// file, the second closing before the first has drained. The second blob /// must come out holding both, or the first edit is silently lost. #[test] fn the_superseding_write_inherits_the_earlier_ones_bytes() { let dir = TempDir::new(); let previous = PendingRevision { path: blob(&dir.0, "old", 64, &[(0, 16)], 0xAA), meta: staged(64, 64, &[(0, 16)]), }; let src = blob(&dir.0, "new", 64, &[(32, 48)], 0xBB); let mut meta = staged(64, 64, &[(32, 48)]); merge_over_pending(&mut meta, &src, &previous).unwrap(); assert_eq!(meta.authored, vec![(0, 16), (32, 48)]); assert!( !meta.complete, "the untouched middle still comes from the base" ); let merged = std::fs::read(&src).unwrap(); assert!( merged[0..16].iter().all(|&b| b == 0xAA), "earlier edit kept" ); assert!(merged[32..48].iter().all(|&b| b == 0xBB), "newer edit kept"); } /// Where they overlap, the newer write wins — it is the later state of the /// file, and the older bytes were never visible to anyone after it landed. #[test] fn the_newer_write_wins_the_ranges_it_authored() { let dir = TempDir::new(); let previous = PendingRevision { path: blob(&dir.0, "old", 32, &[(0, 32)], 0xAA), meta: staged(32, 32, &[(0, 32)]), }; let src = blob(&dir.0, "new", 32, &[(8, 16)], 0xBB); let mut meta = staged(32, 32, &[(8, 16)]); merge_over_pending(&mut meta, &src, &previous).unwrap(); assert_eq!(meta.authored, vec![(0, 32)]); assert!(meta.complete, "between them they cover the file"); let merged = std::fs::read(&src).unwrap(); assert!( merged[8..16].iter().all(|&b| b == 0xBB), "newer edit survives" ); assert!(merged[16..32].iter().all(|&b| b == 0xAA)); } /// The baseline has to come from the write being superseded. This one's own /// "base" is the earlier *staged blob*, whose size and mtime are ours, not /// the server's — filling or conflict-checking against that would compare /// the remote to a revision it never had. #[test] fn the_baseline_is_inherited_from_the_write_being_superseded() { let dir = TempDir::new(); let mut earlier = staged(64, 64, &[(0, 16)]); earlier.base_mtime = 42; earlier.base_size = 50; earlier.based_on = Some(Baseline { mtime: 42, size: 50, hash: None, revision_id: Some("original".into()), }); let previous = PendingRevision { path: blob(&dir.0, "old", 64, &[(0, 16)], 0xAA), meta: earlier, }; let src = blob(&dir.0, "new", 64, &[(32, 48)], 0xBB); // Opened over the staged blob, so this write thinks its base is 64/100. let mut meta = staged(64, 64, &[(32, 48)]); merge_over_pending(&mut meta, &src, &previous).unwrap(); assert_eq!(meta.base_mtime, 42); assert_eq!(meta.base_size, 50); assert_eq!( meta.based_on.unwrap().revision_id.as_deref(), Some("original") ); } /// A shrink must not drag the earlier write's tail back in. #[test] fn a_truncating_write_does_not_inherit_past_its_own_length() { let dir = TempDir::new(); let previous = PendingRevision { path: blob(&dir.0, "old", 64, &[(0, 64)], 0xAA), meta: staged(64, 64, &[(0, 64)]), }; let src = blob(&dir.0, "new", 16, &[(8, 16)], 0xBB); let mut meta = staged(16, 64, &[(8, 16)]); merge_over_pending(&mut meta, &src, &previous).unwrap(); assert_eq!(meta.authored, vec![(0, 16)]); assert!(meta.complete); assert_eq!(std::fs::metadata(&src).unwrap().len(), 16); } } #[cfg(test)] mod size_upgrade_tests { use super::{ LinkId, MAX_SIZE_UPGRADES, NodeUid, SizeUpgrade, SizeUpgradeClaim, SizeWaitQueue, SizeWaiter, VolumeId, claim_size_upgrade, next_size_upgrade, }; use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use std::time::{Duration, Instant}; /// Park a waiter the way [`Core::await_size`] does, and hand back the /// channel its reply is sent down, so a test can time the release. fn park( queue: &Arc, slot: Arc, resolved: impl Fn() -> bool + Send + 'static, deadline: Instant, ) -> mpsc::Receiver<()> { let (tx, rx) = mpsc::channel(); let waiter = SizeWaiter { slot, resolved: Box::new(resolved), deadline, respond: Box::new(move || { let _ = tx.send(()); }), }; if queue.park(waiter) { let queue = queue.clone(); std::thread::spawn(move || queue.serve()); } rx } /// Long enough that a test never trips the deadline by accident, short /// enough that one which is *meant* to does not take ten seconds. fn far_off() -> Instant { Instant::now() + Duration::from_secs(30) } /// The point of the per-chunk design: a waiter is answered as soon as *its /// own* file is resolved, without waiting for the rest of the folder. A /// single batch over 793 nodes took ~80 s, which outran the timeout and /// handed back the provisional size this is all meant to prevent. #[test] fn a_waiter_returns_on_the_chunk_that_resolves_it() { let queue = Arc::new(SizeWaitQueue::default()); let slot = Arc::new(SizeUpgrade::default()); let mine = Arc::new(AtomicBool::new(false)); let worker = slot.clone(); let flag = mine.clone(); let t = std::thread::spawn(move || { // Our node lands in the first chunk; two more follow. std::thread::sleep(Duration::from_millis(30)); flag.store(true, Ordering::SeqCst); worker.chunk_done(); std::thread::sleep(Duration::from_millis(300)); worker.chunk_done(); worker.finish(); }); let started = Instant::now(); let rx = park(&queue, slot, move || mine.load(Ordering::SeqCst), far_off()); rx.recv().expect("the waiter is answered"); // Released by the first chunk, not the last. assert!(started.elapsed() < Duration::from_millis(200)); t.join().unwrap(); } /// `pdfs ls` waits for the whole batch, so it lists no ciphertext size /// (`docs/BUGS.md` B115), but no longer than a `stat` would. #[test] fn a_listing_waits_for_the_whole_size_batch_but_not_forever() { let slot = Arc::new(SizeUpgrade::default()); let worker = slot.clone(); let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(30)); worker.chunk_done(); std::thread::sleep(Duration::from_millis(60)); worker.finish(); }); let started = Instant::now(); slot.wait_finished(started + Duration::from_secs(5)); // Past the first chunk, up to the end of the batch. assert!(started.elapsed() >= Duration::from_millis(90)); assert!(slot.is_finished()); t.join().unwrap(); let stuck = SizeUpgrade::default(); let started = Instant::now(); stuck.wait_finished(started + Duration::from_millis(50)); assert!(started.elapsed() < Duration::from_secs(1)); assert!(!stuck.is_finished()); } /// A chunk that does not resolve this waiter must not answer it. #[test] fn an_unrelated_chunk_does_not_release_a_waiter() { let queue = Arc::new(SizeWaitQueue::default()); let slot = Arc::new(SizeUpgrade::default()); let worker = slot.clone(); let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(30)); worker.chunk_done(); std::thread::sleep(Duration::from_millis(120)); worker.finish(); }); let started = Instant::now(); // Never resolved: only `finish` can end this wait. let rx = park(&queue, slot, || false, far_off()); rx.recv().expect("the waiter is answered"); assert!(started.elapsed() >= Duration::from_millis(140)); t.join().unwrap(); } /// A batch that ends without resolving the node — a failed fetch — still /// answers its waiters, who fall back to the provisional size. #[test] fn finish_releases_a_waiter_that_was_never_resolved() { let queue = Arc::new(SizeWaitQueue::default()); let slot = Arc::new(SizeUpgrade::default()); let worker = slot.clone(); let t = std::thread::spawn(move || { std::thread::sleep(Duration::from_millis(30)); worker.finish(); }); let started = Instant::now(); let rx = park(&queue, slot, || false, far_off()); rx.recv().expect("the waiter is answered"); assert!(started.elapsed() < SizeUpgrade::WAIT); t.join().unwrap(); } /// Already resolved before parking: answered on the first pass. This is the /// follower that arrives after the chunk it needed has landed. #[test] fn an_already_resolved_waiter_does_not_block() { let queue = Arc::new(SizeWaitQueue::default()); let started = Instant::now(); let rx = park(&queue, Arc::new(SizeUpgrade::default()), || true, far_off()); rx.recv().expect("the waiter is answered"); assert!(started.elapsed() < Duration::from_millis(100)); } /// Finishing before anyone parks must not strand the late arrival — the /// flag is what is checked, not the notification, which it would miss. #[test] fn waiting_after_finish_returns_at_once() { let queue = Arc::new(SizeWaitQueue::default()); let slot = Arc::new(SizeUpgrade::default()); slot.finish(); let started = Instant::now(); let rx = park(&queue, slot, || false, far_off()); rx.recv().expect("the waiter is answered"); assert!(started.elapsed() < Duration::from_millis(100)); } /// Every waiter is answered, not just the first: one `ls -l` puts one /// waiter per file on the same folder. #[test] fn all_waiters_are_released() { let queue = Arc::new(SizeWaitQueue::default()); let slot = Arc::new(SizeUpgrade::default()); let waiters: Vec<_> = (0..8) .map(|_| park(&queue, slot.clone(), || false, far_off())) .collect(); std::thread::sleep(Duration::from_millis(20)); slot.finish(); for rx in waiters { rx.recv().expect("every waiter is answered"); } } /// The deadline is the backstop: a batch that never finishes must not wedge /// the caller's `stat` forever. #[test] fn a_waiter_gives_up_at_the_deadline() { let queue = Arc::new(SizeWaitQueue::default()); let started = Instant::now(); // Nothing will ever resolve or finish this. let rx = park( &queue, Arc::new(SizeUpgrade::default()), || false, Instant::now() + Duration::from_millis(120), ); rx.recv().expect("the waiter is answered"); assert!(started.elapsed() >= Duration::from_millis(120)); } /// The serving thread retires when nothing is parked, and a later park /// starts a new one — otherwise the second `ls -l` of the session would /// hang forever. #[test] fn the_queue_serves_again_after_it_has_drained() { let queue = Arc::new(SizeWaitQueue::default()); for _ in 0..3 { let rx = park(&queue, Arc::new(SizeUpgrade::default()), || true, far_off()); rx.recv().expect("the waiter is answered"); } } /// With no thread to serve them, waiters are answered on the spot rather /// than never: a provisional size beats a `stat` that does not return. #[test] fn answer_all_releases_everything_parked() { let queue = SizeWaitQueue::default(); let (tx, rx) = mpsc::channel(); for _ in 0..4 { let tx = tx.clone(); queue.park(SizeWaiter { slot: Arc::new(SizeUpgrade::default()), resolved: Box::new(|| false), deadline: far_off(), respond: Box::new(move || { let _ = tx.send(()); }), }); } drop(tx); queue.answer_all(); assert_eq!(rx.into_iter().count(), 4); } fn uids(links: &[&str]) -> Vec { links .iter() .map(|link| NodeUid::new(VolumeId::from("vol"), LinkId::from(*link))) .collect() } /// Past the cap a folder used to get no upgrade at all, so a recursive /// walk reported ciphertext sizes for most of what it listed (bugs.md B100). #[test] fn a_folder_past_the_cap_is_queued_not_dropped() { let mut in_flight = HashMap::new(); for key in 0..MAX_SIZE_UPGRADES as u64 { let claim = claim_size_upgrade(&mut in_flight, key, uids(&["a"])); assert!(matches!(claim, SizeUpgradeClaim::Start(..))); } let key = MAX_SIZE_UPGRADES as u64; let claim = claim_size_upgrade(&mut in_flight, key, uids(&["late"])); let SizeUpgradeClaim::Queued(slot) = claim else { panic!("expected the batch to be queued"); }; assert!(Arc::ptr_eq(&slot, &in_flight[&key])); // A second caller for the queued folder waits on the same batch. let again = claim_size_upgrade(&mut in_flight, key, uids(&["late"])); assert!(matches!(again, SizeUpgradeClaim::Joined(s) if Arc::ptr_eq(&s, &slot))); } #[test] fn a_finished_batch_hands_its_thread_the_oldest_queued_one() { let mut in_flight = HashMap::new(); for key in 0..MAX_SIZE_UPGRADES as u64 { claim_size_upgrade(&mut in_flight, key, uids(&["a"])); } let first = MAX_SIZE_UPGRADES as u64; claim_size_upgrade(&mut in_flight, first, uids(&["first"])); std::thread::sleep(Duration::from_millis(2)); claim_size_upgrade(&mut in_flight, first + 1, uids(&["second"])); let (key, taken, slot) = next_size_upgrade(&mut in_flight, 0).expect("a queued batch"); assert_eq!((key, taken), (first, uids(&["first"]))); assert!(slot.queued.lock().is_none()); assert!(!in_flight.contains_key(&0)); // Still at the cap: the handed-over batch counts as running. let running = in_flight .values() .filter(|s| s.queued.lock().is_none()) .count(); assert_eq!(running, MAX_SIZE_UPGRADES); let (key, _, _) = next_size_upgrade(&mut in_flight, 1).expect("the other one"); assert_eq!(key, first + 1); assert!(next_size_upgrade(&mut in_flight, 2).is_none()); } } #[cfg(test)] mod replace_tests { use super::check_replaceable; /// The case that motivated all of this: rsync renaming its temp file over /// the real one. Two plain files, and it has to be allowed — refusing is /// what made every rsync transfer fail at the last step (bugs.md B13). #[test] fn a_file_may_replace_a_file() { assert!(check_replaceable(false, false, true).is_ok()); } #[test] fn an_empty_directory_may_be_replaced_by_a_directory() { assert!(check_replaceable(true, true, true).is_ok()); } /// Proton trashes a folder with its whole subtree, so allowing this would /// discard every file under the destination without ever naming them. #[test] fn a_non_empty_directory_is_never_replaced() { let e = check_replaceable(true, true, false).expect_err("must refuse"); assert_eq!(e.code(), libc::ENOTEMPTY); } #[test] fn the_two_ends_must_agree_on_being_a_directory() { assert_eq!( check_replaceable(false, true, true) .expect_err("a file may not replace a directory") .code(), libc::EISDIR ); assert_eq!( check_replaceable(true, false, true) .expect_err("a directory may not replace a file") .code(), libc::ENOTDIR ); } /// `dst_empty` describes a directory; it must not leak into the file case, /// where callers pass `true` by convention. #[test] fn emptiness_is_ignored_when_the_destination_is_a_file() { assert!(check_replaceable(false, false, false).is_ok()); } } #[cfg(test)] mod local_uid_tests { use super::*; #[test] fn a_local_node_uid_is_recognisable_and_round_trips() { let uid = local_node_uid(42); assert!(is_local_uid(&uid)); assert!(is_local_uid_str(&uid.to_string())); assert_eq!(uid.to_string(), pdfs_core::db::local_uid(42)); // It has to survive the trip through `pending_op.uid` as text, like any // other uid does, and give back the row it stands for. let parsed = parse_node_uid(&uid.to_string()).expect("parses back"); assert_eq!(parsed, uid); assert_eq!(pdfs_core::db::local_lid(&uid.to_string()), Some(42)); } #[test] fn local_node_uids_are_distinct_per_row() { // Two files created in the same millisecond must not collide — the whole // queue is keyed by uid. assert_ne!(local_node_uid(1), local_node_uid(2)); } #[test] fn a_real_uid_is_never_mistaken_for_a_placeholder() { let real = NodeUid::new(VolumeId::from("vol1"), LinkId::from("link1")); assert!(!is_local_uid(&real)); assert!(!is_local_uid_str("vol1~link1")); // Not a uid at all. assert!(!is_local_uid_str("local")); // The sentinel is the *volume*; a link that merely says "local" is real. assert!(!is_local_uid_str("vol1~local")); } #[test] fn a_placeholder_file_reports_itself_as_empty_and_unsealed() { let parent = NodeUid::new(VolumeId::from("vol1"), LinkId::from("dir")); let node = local_node(local_node_uid(7), parent.clone(), "notes.txt".into(), false); assert_eq!(node.name, "notes.txt"); assert_eq!(node.parent_uid, Some(parent)); assert!(!node.trashed); match node.kind { NodeKind::File { claimed_size, active_revision_state, ref media_type, .. } => { assert_eq!(claimed_size, Some(0)); // Nothing has been uploaded, so there is no sealed revision. assert!(active_revision_state.is_none()); assert_eq!(media_type, "text/plain"); } NodeKind::Folder => panic!("expected a file"), } } #[test] fn a_placeholder_folder_is_a_folder() { let parent = NodeUid::new(VolumeId::from("vol1"), LinkId::from("root")); let node = local_node(local_node_uid(7), parent, "photos".into(), true); assert!(node.is_folder()); } } #[cfg(test)] mod pending_size_tests { use super::{Node, NodeKind, NodeUid, apply_pending_sizes, node_size}; use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId}; use std::collections::HashMap; fn uid(link: &str) -> NodeUid { NodeUid::new(VolumeId::from("vol"), LinkId::from(link)) } fn file(link: &str, claimed: i64) -> Node { Node { uid: uid(link), parent_uid: Some(uid("parent")), kind: NodeKind::File { media_type: "text/plain".into(), total_size_on_storage: 0, active_revision_state: None, active_revision_id: None, content_sha1: None, claimed_size: Some(claimed), claimed_modification_time: None, }, name: link.into(), creation_time: 100, modification_time: 100, trashed: false, is_shared: false, is_shared_publicly: false, signature_email: None, membership: None, photo: None, album: None, verification: Default::default(), direct_role: None, share_id: None, } } fn folder(link: &str) -> Node { Node { kind: NodeKind::Folder, ..file(link, 0) } } /// B11: a re-enumeration mid-write must not revert the size to the /// server's. A file that stats as 0 gets no `read` from the kernel at all, /// so the staged blob is never served and the file reads as empty. #[test] fn a_queued_write_keeps_its_optimistic_size_through_a_re_enumeration() { let mut nodes = vec![file("queued", 0), file("settled", 4096)]; let sizes = HashMap::from([(uid("queued"), 3)]); apply_pending_sizes(&mut nodes, &sizes); assert_eq!( node_size(&nodes[0]), 3, "the remote's pre-write size must not win over the queued write's" ); assert_eq!( node_size(&nodes[1]), 4096, "a file with nothing queued keeps the size the server reported" ); } /// The map is keyed by uid and says nothing about kind; a folder that /// somehow collides must not grow a `claimed_size`. #[test] fn folders_are_left_alone() { let mut nodes = vec![folder("dir")]; apply_pending_sizes(&mut nodes, &HashMap::from([(uid("dir"), 999)])); assert!(matches!(nodes[0].kind, NodeKind::Folder)); assert_eq!(node_size(&nodes[0]), 0); } /// The common case: nothing queued, nothing touched. #[test] fn an_empty_pending_map_changes_nothing() { let mut nodes = vec![file("a", 10), file("b", 20)]; apply_pending_sizes(&mut nodes, &HashMap::new()); assert_eq!(node_size(&nodes[0]), 10); assert_eq!(node_size(&nodes[1]), 20); } } #[cfg(test)] mod tests { use super::{ Access, AccessFlags, Errno, HashMap, Intervals, PendingRevision, RootListingSnapshot, SELF_CHANGE_TTL_MS, ShareId, SharedWithMeItem, StateRegistry, VirtualRootPlan, accepted_share_provenance, conflict_name, copy_pending_for_truncate, expand_restore, first_seen_in_trash, fuse_name, is_stale_mount, node_visible, note_self_change, off_the_runtime, orphan_past_retention, parse_node_uid, prepare_shared_roots, preserve_on_access_denied, publish_virtual_root_in_listing, reconcile_virtual_root_in_listing, release_can_discard_unlinked, release_must_retain_queued_trash, release_unlinked_entry, rename_needs_queue, require_node_parent_access, require_rename_access, resolve_anywhere_with, shared_with_me_uid, take_self_change, uid_write_authority, virtual_node, }; use super::{Db, WriteAuthority}; use std::time::Duration; /// Staged bytes that name no node are kept for a month, because the only /// copy of a write must outlive the bug that stranded it. Past that, one /// boot retires them instead of re-announcing them every boot forever. #[test] fn a_stranded_blob_is_retired_only_once_it_outlives_retention() { let dir = std::env::temp_dir().join(format!("pdfs-orphan-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let blob = dir.join("blob"); std::fs::write(&blob, b"twelve bytes").unwrap(); // Written just now: still recoverable, still kept. assert_eq!(orphan_past_retention(&blob), None); set_mtime_ago( &blob, super::STAGING_ORPHAN_RETAIN + Duration::from_secs(60), ); assert_eq!(orphan_past_retention(&blob), Some(12)); // A blob that is not there is not a blob to retire. std::fs::remove_file(&blob).unwrap(); assert_eq!(orphan_past_retention(&blob), None); let _ = std::fs::remove_dir_all(&dir); } /// Backdate a file's mtime, which is what the retention check reads. fn set_mtime_ago(path: &std::path::Path, ago: Duration) { let when = std::time::SystemTime::now() - ago; let secs = when .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs() as libc::time_t; let times = [ libc::timeval { tv_sec: secs, tv_usec: 0, }, libc::timeval { tv_sec: secs, tv_usec: 0, }, ]; let c_path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); // SAFETY: both pointers are valid for the call and nothing keeps them. let rc = unsafe { libc::utimes(c_path.as_ptr(), times.as_ptr()) }; assert_eq!(rc, 0, "utimes: {}", std::io::Error::last_os_error()); } /// `(uid, parent)` pairs in the shape [`Db::trash_parents`] returns. fn relations(pairs: &[(&str, Option<&str>)]) -> Vec<(String, Option)> { pairs .iter() .map(|(uid, parent)| (uid.to_string(), parent.map(str::to_string))) .collect() } fn uids(values: &[&str]) -> Vec { values.iter().map(|v| v.to_string()).collect() } /// Restoring a folder brings back what was trashed inside it, and the folder /// goes first: the server has nowhere to put a child whose parent is trashed. #[test] fn restoring_a_folder_restores_its_trashed_contents_parents_first() { let trash = relations(&[ ("dir", Some("live")), ("file", Some("dir")), ("deep", Some("sub")), ("sub", Some("dir")), ]); assert_eq!( expand_restore(&uids(&["dir"]), &trash), vec![ vec!["dir".to_string()], vec!["file".to_string(), "sub".to_string()], vec!["deep".to_string()], ] ); } /// Restoring a file inside a trashed folder restores the folder too — /// otherwise the file comes back somewhere the user cannot reach. #[test] fn restoring_content_restores_the_trashed_parents_above_it() { let trash = relations(&[ ("dir", Some("live")), ("sub", Some("dir")), ("file", Some("sub")), ("other", Some("dir")), ]); // `other` is a sibling of `sub`, not an ancestor of `file`: it stays put. assert_eq!( expand_restore(&uids(&["file"]), &trash), vec![ vec!["dir".to_string()], vec!["sub".to_string()], vec!["file".to_string()], ] ); } /// A node whose parent is not in the trash is already at the top: one wave, /// no ancestors invented. #[test] fn a_node_under_a_live_parent_restores_on_its_own() { let trash = relations(&[("file", Some("live")), ("elsewhere", None)]); assert_eq!( expand_restore(&uids(&["file"]), &trash), vec![vec!["file".to_string()]] ); } /// The listing is materialised in chunks and can be stale, so a uid it does /// not know about must still be restored rather than dropped. #[test] fn an_unlisted_uid_is_still_restored() { let trash = relations(&[("dir", None)]); assert_eq!( expand_restore(&uids(&["missing"]), &trash), vec![vec!["missing".to_string()]] ); } /// Asking for both a folder and something inside it restores each node once. #[test] fn overlapping_requests_do_not_duplicate_nodes() { let trash = relations(&[("dir", None), ("file", Some("dir"))]); assert_eq!( expand_restore(&uids(&["dir", "file"]), &trash), vec![vec!["dir".to_string()], vec!["file".to_string()]] ); } /// A node keeps the time it was first seen in the trash; a new one is dated /// now only when the previous refresh saw the whole trash. #[test] fn trash_dates_are_kept_and_only_guessed_after_a_complete_refresh() { let seen = HashMap::from([("old".to_string(), Some(5)), ("unknown".to_string(), None)]); assert_eq!(first_seen_in_trash(&seen, "old", true, 9), Some(5)); assert_eq!(first_seen_in_trash(&seen, "old", false, 9), Some(5)); assert_eq!(first_seen_in_trash(&seen, "unknown", true, 9), None); assert_eq!(first_seen_in_trash(&seen, "new", true, 9), Some(9)); assert_eq!(first_seen_in_trash(&seen, "new", false, 9), None); } /// A parent chain that loops back on itself must terminate. #[test] fn a_cyclic_listing_terminates() { let trash = relations(&[("a", Some("b")), ("b", Some("a"))]); let waves = expand_restore(&uids(&["a"]), &trash); let total: usize = waves.iter().map(Vec::len).sum(); assert_eq!(total, 2); } /// The collapse [`Core::require_uid_writable`] performs, without a `Core`: /// every non-writable authority is one `EACCES` on the syscall path. fn require_uid_access(db: &Db, uid: &NodeUid, live: &[Access]) -> Result<(), Errno> { match uid_write_authority(db, uid, live) { WriteAuthority::Writable => Ok(()), WriteAuthority::Denied | WriteAuthority::Unknown => Err(Errno::EACCES), } } use crate::filesystem::access_allowed; use pdfs_core::cache::{Baseline, StagedWrite}; use pdfs_core::db::{OP_REVISION, OP_TRASH, PendingOp}; fn session_flag(live: bool) -> std::sync::Arc { std::sync::Arc::new(std::sync::atomic::AtomicBool::new(live)) } /// A mount as the registry sees it, with the cells its `Core` would own. struct TestMount { notifier: std::sync::Arc>, live: std::sync::Arc, root: std::sync::Arc>, } impl TestMount { fn register(registry: &StateRegistry, mountpoint: &str, root: Option) -> Self { let mount = Self { notifier: std::sync::Arc::new(std::sync::OnceLock::new()), live: session_flag(false), root: std::sync::Arc::new(std::sync::OnceLock::new()), }; if let Some(root) = root { mount.root.set(root).unwrap(); } registry.register( std::path::Path::new(mountpoint), &mount.notifier, mount.live.clone(), mount.root.clone(), ); mount } fn spawn(&self) { self.live.store(true, std::sync::atomic::Ordering::Release); } } /// Whether `uid` is visible from the root a single-mount test installs. fn visible(state: &crate::state::State, uid: &NodeUid) -> bool { state.owns_visible_uid(uid, &[super::ROOT_INO]) } /// The registry is how background work reaches every mounted session's /// kernel cache, so the walk has to see *every* registered mount — and /// only the ones still mounted (`docs/BUGS.md` B74). #[test] fn state_registry_walks_every_live_mount_and_reaps_the_rest() { let registry = StateRegistry::default(); assert!(registry.live().is_empty(), "a fresh registry has no mounts"); // The primary mount plus two on-demand forks. let primary = TestMount::register(®istry, "/mnt/primary", None); let fork_a = TestMount::register(®istry, "/mnt/fork-a", None); let fork_b = TestMount::register(®istry, "/mnt/fork-b", None); assert_eq!(registry.live().len(), 3, "every registered mount is walked"); // `live()` must not itself pin a mount alive, or an unmounted fork could // never be reaped. assert_eq!( std::sync::Arc::strong_count(&fork_a.notifier), 1, "the registry holds forks weakly" ); // Unmounting a fork drops its `Core`; the dead entry goes on the next walk. drop(fork_a); assert_eq!(registry.live().len(), 2, "an unmounted fork is dropped"); assert_eq!( registry.0.lock().len(), 2, "and its slot is reaped, not merely skipped" ); drop(primary); drop(fork_b); assert!( registry.live().is_empty(), "the last mount going away empties the registry" ); } #[test] fn state_registry_covering_uses_longest_prefix_and_ignores_dead_forks() { let registry = StateRegistry::default(); let primary_path = std::path::Path::new("/home/me/ProtonDrive"); let nested_path = std::path::Path::new("/home/me/ProtonDrive/Device"); let sibling_path = std::path::Path::new("/home/me/Archive"); let primary = TestMount::register(®istry, "/home/me/ProtonDrive", Some(2)); let nested = TestMount::register(®istry, "/home/me/ProtonDrive/Device", Some(3)); let sibling = TestMount::register(®istry, "/home/me/Archive", Some(4)); sibling.spawn(); assert_eq!(registry.live().len(), 3); assert!(!registry.is_mounted_at(primary_path)); assert!(!registry.is_mounted_at(nested_path)); assert_eq!( registry.live_roots(), vec![4], "an unspawned session's root must not authorize resident uids" ); assert!( registry .covering(&nested_path.join("folder/file.txt")) .is_none(), "a registered but unspawned session must not win path covering" ); primary.spawn(); nested.spawn(); let mut roots = registry.live_roots(); roots.sort(); assert_eq!(roots, vec![2, 3, 4]); let covering = registry .covering(&nested_path.join("folder/file.txt")) .unwrap(); assert_eq!(covering.mountpoint, nested_path); assert!(std::sync::Arc::ptr_eq(&covering.notifier, &nested.notifier)); drop(covering); let covering = registry.covering(nested_path).unwrap().mountpoint; assert_eq!(covering, nested_path, "an exact mountpoint covers itself"); let covering = registry .covering(std::path::Path::new("/home/me/ProtonDrive/Device/folder/")) .unwrap() .mountpoint; assert_eq!( covering, nested_path, "a trailing separator does not change component-prefix selection" ); let covering = registry .covering(std::path::Path::new( "/home/me/ProtonDrive/DeviceBackup/file", )) .unwrap() .mountpoint; assert_eq!( covering, primary_path, "/Device must not string-prefix-match /DeviceBackup" ); let covering = registry.covering(&sibling_path.join("file")).unwrap(); assert_eq!(covering.mountpoint, sibling_path); assert!(std::sync::Arc::ptr_eq( &covering.notifier, &sibling.notifier )); drop(covering); assert!( registry .covering(std::path::Path::new("/home/me/ProtonDriveBackup/file")) .is_none(), "/ProtonDrive must not string-prefix-match /ProtonDriveBackup" ); assert!( registry .covering(std::path::Path::new("/tmp/outside")) .is_none(), "an outside path has no covering mount" ); assert!(registry.is_mounted_at(primary_path)); assert!(registry.is_mounted_at(nested_path)); assert!( !registry.is_mounted_at(&nested_path.join("folder")), "a path covered by a session is not itself a mountpoint" ); // This is also the failed-fork case: the caller registered the fork, // session construction failed, and its last `Core` was dropped. It // must not make ListLocations report mounted. drop(nested); assert!(!registry.is_mounted_at(nested_path)); assert_eq!(registry.live().len(), 2); } /// The routing half of `docs/BUGS.md` B86: a path under a secondary mount /// has to resolve to *that* mount's root and a suffix relative to it, or /// `pdfs ls` and `pdfs refresh` can only ever name the primary mount and a /// stale on-demand folder has no escape hatch. #[test] fn covering_routes_a_path_to_the_mount_that_owns_it() { let registry = StateRegistry::default(); let primary_path = std::path::Path::new("/home/me/ProtonDrive"); let nested_path = std::path::Path::new("/home/me/ProtonDrive/Device"); let primary = TestMount::register(®istry, "/home/me/ProtonDrive", Some(2)); let nested = TestMount::register(®istry, "/home/me/ProtonDrive/Device", Some(3)); primary.spawn(); nested.spawn(); let deep = nested_path.join("sub/file.txt"); let mount = registry.covering(&deep).expect("nested mount covers"); assert_eq!(mount.mountpoint, nested_path, "the nested mount wins"); assert_eq!( mount.root.get(), Some(&3), "the request must be resolved from the nested root" ); assert!( std::sync::Arc::ptr_eq(&mount.notifier, &nested.notifier), "and tell that mount's session, not the primary's" ); assert_eq!( deep.strip_prefix(&mount.mountpoint).unwrap(), std::path::Path::new("sub/file.txt"), "the suffix is relative to the mount that owns it, not the primary" ); let shallow = primary_path.join("Documents"); let mount = registry.covering(&shallow).expect("primary mount covers"); assert_eq!( mount.root.get(), Some(&2), "a path outside the nested root still routes to the primary" ); assert!( registry .covering(std::path::Path::new("/tmp/outside")) .is_none(), "a path under no mount routes nowhere, and the caller reports that" ); } /// Every mount shares one tree, so a node under a fork's root and one under /// the primary root are both found there — but only while the mount whose /// root they sit under is mounted. #[test] fn a_uid_is_visible_under_the_root_of_any_mounted_session() { let (mut state, _dir) = rooted_state("primary-volume", "primary"); let primary_uid = state.entries[&super::ROOT_INO].uid.clone(); let device_root = node_helper_in_volume("device-volume", "device", None, "device", true); let fork_root = state.install_root(device_root); let fork_node = node_helper_in_volume("device-volume", "fork", Some("device"), "fork", true); let fork_uid = fork_node.uid.clone(); let fork_ino = state.intern(fork_root, fork_node); state.children.insert(fork_root, vec![fork_ino]); let both = [super::ROOT_INO, fork_root]; assert!(state.owns_visible_uid(&primary_uid, &both)); assert!(state.owns_visible_uid(&fork_uid, &both)); assert!( !state.owns_visible_uid(&super::parse_uid("device-volume~missing").unwrap(), &both) ); assert!( !state.owns_visible_uid(&fork_uid, &[super::ROOT_INO]), "an unmounted on-demand root must stop authorizing its uids" ); } /// Each session calls its own root inode 1, and nothing else may come out /// as 1 or come in from it. #[test] fn each_session_names_its_own_root_as_the_kernel_root() { let fork_root = 41; assert_eq!( super::filesystem::kernel_ino(fork_root, fork_root).0, super::ROOT_INO ); assert_eq!(super::filesystem::kernel_ino(fork_root, 7).0, 7); assert_eq!(super::filesystem::kernel_ino(super::ROOT_INO, 7).0, 7); } #[test] fn a_missing_listing_is_unknown_not_absent() { let (mut state, _dir) = rooted_state("device-volume", "device"); let child_node = node_helper_in_volume("device-volume", "child", Some("device"), "child.txt", false); let child_uid = child_node.uid.clone(); let child_ino = state.intern(super::ROOT_INO, child_node); state.children.insert(super::ROOT_INO, vec![child_ino]); assert!(visible(&state, &child_uid)); state.invalidate_listing(super::ROOT_INO); assert!( visible(&state, &child_uid), "an invalidated listing is unknown and must not hide a valid resident child" ); state.children.insert(super::ROOT_INO, Vec::new()); assert!( !visible(&state, &child_uid), "a present listing that omits the child proves it is no longer visible" ); } #[test] fn virtual_root_reconcile_does_not_recreate_an_invalidated_listing() { let (mut state, _dir) = rooted_state("primary-volume", "root"); let root_uid = state.entries[&super::ROOT_INO].uid.clone(); let child_node = node_helper_in_volume("primary-volume", "child", Some("root"), "child.txt", false); let child_uid = child_node.uid.clone(); let child_ino = state.intern(super::ROOT_INO, child_node); state.children.insert(super::ROOT_INO, vec![child_ino]); // Simulate ensure_children observing a complete primary-root listing. let snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap(); let detected_names = snapshot.real_names(); assert_eq!( detected_names, std::collections::HashSet::from(["child.txt".into()]) ); // An event wins the race before virtual-root dentry publication. state.invalidate_listing(super::ROOT_INO); let virtual_ino = state.intern_mem( super::ROOT_INO, virtual_node(root_uid, "Shared with me".into(), 0), ); assert!( !reconcile_virtual_root_in_listing( &mut state, super::ROOT_INO, &snapshot, virtual_ino, true, ), "the invalidated snapshot must not publish" ); assert!( !state.children.contains_key(&super::ROOT_INO), "reconciliation must leave an invalidated listing absent" ); assert!( visible(&state, &child_uid), "the absent listing remains unknown until refresh, so a valid resident uid is allowed" ); } #[test] fn virtual_root_reconcile_rejects_repopulated_changed_snapshot() { let (mut state, _dir) = rooted_state("primary-volume", "root"); let root_uid = state.entries[&super::ROOT_INO].uid.clone(); let child = node_helper_in_volume("primary-volume", "child", Some("root"), "child.txt", false); let child_ino = state.intern(super::ROOT_INO, child); state.children.insert(super::ROOT_INO, vec![child_ino]); let stale_snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap(); let mut persisted_virtual = virtual_node(root_uid.clone(), "Shared with me".into(), 0); persisted_virtual.trashed = true; state .flushed_db() .publish_virtual_root(super::SHARED_WITH_ME_NAME, &persisted_virtual) .unwrap(); let mut indexed_descendant = node_helper_in_volume( "foreign-volume", "inside", None, "SnapshotRaceFindable", false, ); indexed_descendant.parent_uid = Some(shared_with_me_uid()); state.flushed_db().upsert_node(&indexed_descendant).unwrap(); assert!( state .db .search("SnapshotRaceFindable", 10) .unwrap() .is_empty(), "the hidden synthetic ancestor must hide descendant search hits" ); state.invalidate_listing(super::ROOT_INO); state.entries.get_mut(&child_ino).unwrap().node.name = "Shared with me".into(); state.children.insert(super::ROOT_INO, vec![child_ino]); let replacement_snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap(); assert_ne!(stale_snapshot, replacement_snapshot); assert_eq!( replacement_snapshot.real_names(), std::collections::HashSet::from(["Shared with me".into()]) ); let mut stale_node = persisted_virtual.clone(); stale_node.trashed = false; let stale_plan = VirtualRootPlan { node: stale_node.clone(), visible: true, }; let db = state.flushed_db(); let error = publish_virtual_root_in_listing( &db, &mut state, super::ROOT_INO, &stale_snapshot, stale_plan, ) .unwrap_err(); assert_eq!( error.code(), Errno::EAGAIN.code(), "stale work must fail before persistence or dentry publication" ); assert_eq!( state.children[&super::ROOT_INO], vec![child_ino], "the replacement listing must remain untouched" ); assert!( state .db .node_by_uid(&shared_with_me_uid().to_string()) .unwrap() .unwrap() .trashed, "stale work must not change persisted visibility" ); assert!( state .db .search("SnapshotRaceFindable", 10) .unwrap() .is_empty(), "stale work must not change FTS ancestry" ); let virtual_ino = state.intern_mem(super::ROOT_INO, stale_node); state .children .get_mut(&super::ROOT_INO) .unwrap() .push(virtual_ino); let current_snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap(); state.entries.get_mut(&virtual_ino).unwrap().node.trashed = false; assert!( reconcile_virtual_root_in_listing( &mut state, super::ROOT_INO, ¤t_snapshot, virtual_ino, false, ), "the synthetic node's own visibility update must not stale the real-child snapshot" ); assert_eq!(state.children[&super::ROOT_INO], vec![child_ino]); } #[test] fn virtual_root_publication_failure_is_atomic_and_leaves_state_unchanged() { let (mut state, _dir) = rooted_state("primary-volume", "root"); let root_uid = state.entries[&super::ROOT_INO].uid.clone(); let child = node_helper_in_volume("primary-volume", "child", Some("root"), "child.txt", false); let child_ino = state.intern(super::ROOT_INO, child); state.children.insert(super::ROOT_INO, vec![child_ino]); let snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap(); let mut descendant = node_helper_in_volume( "foreign-volume", "inside", None, "AtomicRollbackFindable", false, ); descendant.parent_uid = Some(shared_with_me_uid()); state.flushed_db().upsert_node(&descendant).unwrap(); // A node whose parent row is not cached — the synthetic root is not one // until this publication commits — is indexed under the path it can // actually be resolved to, which is its bare name. Publication reindexes // the subtree beneath the synthetic ancestor, so that path is what the // rollback has to restore. let path_before = { let hits = state .flushed_db() .search("AtomicRollbackFindable", 10) .unwrap(); assert_eq!(hits.len(), 1, "the descendant starts out searchable"); hits[0].path.clone() }; assert_eq!(path_before, "AtomicRollbackFindable"); // The pin is the transaction's final statement. Failing it proves the // earlier access, node, and descendant FTS work rolls back with it. state .flushed_db() .with_conn(|conn| { conn.execute_batch( "CREATE TRIGGER reject_virtual_root_pin BEFORE INSERT ON sync_state WHEN NEW.key = 'shared_with_me_name' BEGIN SELECT RAISE(ABORT, 'injected virtual-root pin failure'); END;", )?; Ok(()) }) .unwrap(); let entries_before = state.entries.len(); let by_uid_before = state.by_uid.len(); let next_ino_before = state.next_ino; let access_before = state.share_access.clone(); let db = state.flushed_db(); let result = publish_virtual_root_in_listing( &db, &mut state, super::ROOT_INO, &snapshot, VirtualRootPlan { node: virtual_node(root_uid, "Shared with me".into(), 0), visible: true, }, ); assert_eq!(result.unwrap_err().code(), Errno::EIO.code()); assert_eq!( RootListingSnapshot::capture(&state, super::ROOT_INO), Some(snapshot), "the resident listing must not change on DB failure" ); assert_eq!(state.entries.len(), entries_before); assert_eq!(state.by_uid.len(), by_uid_before); assert_eq!(state.next_ino, next_ino_before); assert_eq!(state.share_access, access_before); assert!( !state.by_uid.contains_key(&shared_with_me_uid()), "the synthetic inode must not be interned before commit" ); assert_eq!( state .flushed_db() .state_str(super::SHARED_WITH_ME_NAME) .unwrap(), None ); assert!( state .db .node_by_uid(&shared_with_me_uid().to_string()) .unwrap() .is_none() ); assert_eq!( state .flushed_db() .share_access(&shared_with_me_uid()) .unwrap(), None ); let hits = state .flushed_db() .search("AtomicRollbackFindable", 10) .unwrap(); assert_eq!(hits.len(), 1); assert_eq!( hits[0].path, path_before, "the descendant's indexed path must roll back with the failed pin, \ not keep the synthetic ancestor the publication tried to add" ); } #[test] fn open_unlinked_and_revoked_residents_are_not_visible() { let (mut state, _dir) = rooted_state("device-volume", "device"); let open_node = node_helper_in_volume("device-volume", "open", Some("device"), "open.txt", false); let open_uid = open_node.uid.clone(); let open_ino = state.intern(super::ROOT_INO, open_node); state.children.insert(super::ROOT_INO, vec![open_ino]); state.entries.get_mut(&open_ino).unwrap().open_count = 1; let revoked_node = node_helper_in_volume("device-volume", "revoked", Some("device"), "revoked", true); let revoked_uid = revoked_node.uid.clone(); let revoked_ino = state.intern(super::ROOT_INO, revoked_node); let child_node = node_helper_in_volume( "device-volume", "revoked-child", Some("revoked"), "child.txt", false, ); let child_uid = child_node.uid.clone(); let child_ino = state.intern(revoked_ino, child_node); state .children .get_mut(&super::ROOT_INO) .unwrap() .push(revoked_ino); state.children.insert(revoked_ino, vec![child_ino]); assert!(visible(&state, &open_uid)); assert!(visible(&state, &child_uid)); state.unlink_mem(&open_uid); assert!( !visible(&state, &open_uid), "an open inode retained after unlink is not addressable" ); state.hide_shared_root(&revoked_uid); assert!(!visible(&state, &revoked_uid)); assert!( !visible(&state, &child_uid), "a retained descendant beneath a revoked root is not reachable" ); } #[test] fn foreign_broken_and_cyclic_residents_are_not_visible() { let (mut state, _dir) = rooted_state("own-volume", "root"); let foreign_node = node_helper_in_volume("foreign-volume", "shared", Some("root"), "shared", true); let foreign_uid = foreign_node.uid.clone(); let foreign_ino = state.intern(super::ROOT_INO, foreign_node); let broken_node = node_helper_in_volume("own-volume", "broken", Some("root"), "broken", true); let broken_uid = broken_node.uid.clone(); let _broken_ino = state.intern(super::ROOT_INO, broken_node); let cycle_a_node = node_helper_in_volume("own-volume", "cycle-a", Some("root"), "a", true); let cycle_a_uid = cycle_a_node.uid.clone(); let cycle_a = state.intern(super::ROOT_INO, cycle_a_node); let cycle_b_node = node_helper_in_volume("own-volume", "cycle-b", Some("cycle-a"), "b", true); let cycle_b = state.intern(cycle_a, cycle_b_node); let cycle_b_uid = state.entries[&cycle_b].uid.clone(); let cycle_a_entry = state.entries.get_mut(&cycle_a).unwrap(); cycle_a_entry.parent = cycle_b; cycle_a_entry.node.parent_uid = Some(cycle_b_uid); state.children.insert(super::ROOT_INO, vec![foreign_ino]); state.children.insert(cycle_a, vec![cycle_b]); state.children.insert(cycle_b, vec![cycle_a]); assert!( !visible(&state, &foreign_uid), "incoming-share residents are outside the mounted root's volume" ); assert!( !visible(&state, &broken_uid), "an interned node without a visible parent dentry is not reachable" ); assert!( !visible(&state, &cycle_a_uid), "a cyclic resident parent chain is not reachable" ); } #[test] fn resolve_anywhere_prefers_resident_state_then_falls_back_to_mirror() { let resident = resolve_anywhere_with( "vol~resident", |_| true, |_| panic!("resident uid must not query mirror bookkeeping"), ) .unwrap(); assert_eq!(resident.to_string(), "vol~resident"); let mirror = resolve_anywhere_with("vol~mirror", |_| false, |_| Ok(true)).unwrap(); assert_eq!(mirror.to_string(), "vol~mirror"); } #[test] fn resolve_anywhere_classifies_invalid_missing_and_db_errors() { for raw_uid in ["not-a-uid", "~link", "vol~", "vol~link~extra"] { let invalid = resolve_anywhere_with( raw_uid, |_| panic!("invalid uid must not query live state"), |_| panic!("invalid uid must not query mirror bookkeeping"), ) .unwrap_err(); assert_eq!( invalid.kind, pdfs_core::control::ErrorKind::Invalid, "{raw_uid}" ); } for raw_uid in ["local~pending", "virtual~sharedwithme"] { let reserved = resolve_anywhere_with( raw_uid, |_| panic!("reserved uid must not query live state"), |_| panic!("reserved uid must not query mirror bookkeeping"), ) .unwrap_err(); assert_eq!( reserved.kind, pdfs_core::control::ErrorKind::Invalid, "{raw_uid}" ); } let missing = resolve_anywhere_with("vol~missing", |_| false, |_| Ok(false)).unwrap_err(); assert_eq!(missing.kind, pdfs_core::control::ErrorKind::NotFound); let db_error = resolve_anywhere_with( "vol~db", |_| false, |_| Err(pdfs_core::Error::Other("lookup failed".into())), ) .unwrap_err(); assert_eq!(db_error.kind, pdfs_core::control::ErrorKind::Internal); assert!(db_error.message.contains("lookup failed")); } /// An event feed carries no revision id, so the only thing that tells a /// remote change apart from the echo of one this daemon just made is this /// bookkeeping. Getting it wrong is expensive in both directions: too eager /// and we evict the bytes we uploaded and re-download them from an API that /// may still be serving the previous revision (a SQLite file on the mount /// read back `malformed`); too generous and a real change from another device /// is ignored. #[test] fn a_self_change_is_claimed_once_and_expires() { let mut changes = HashMap::new(); let uid = parse_node_uid("vol~link").unwrap(); let other = parse_node_uid("vol~other").unwrap(); let t0 = 1_000_000; assert!( !take_self_change(&mut changes, &uid, t0), "a node we never touched is never ours" ); note_self_change(&mut changes, &uid, t0); assert!( take_self_change(&mut changes, &uid, t0 + 5_000), "the echo of our own change is claimed" ); assert!( !take_self_change(&mut changes, &uid, t0 + 6_000), "and only once, so a later foreign change still applies" ); // A create followed by a revision upload is two changes to one node, and // the feed reports both. note_self_change(&mut changes, &uid, t0); note_self_change(&mut changes, &uid, t0); assert!(take_self_change(&mut changes, &uid, t0 + 1)); assert!(take_self_change(&mut changes, &uid, t0 + 2)); assert!(!take_self_change(&mut changes, &uid, t0 + 3)); // Past the window the attribution is a guess, and a wrong guess leaves // the user looking at a stale file. note_self_change(&mut changes, &uid, t0); assert!(!take_self_change( &mut changes, &uid, t0 + SELF_CHANGE_TTL_MS )); assert!( !changes.contains_key(&uid), "an expired change is dropped, not left to be re-tested" ); // An echo that never arrives must not accumulate for the life of the // daemon; the next recorded change prunes it. note_self_change(&mut changes, &uid, t0); note_self_change(&mut changes, &other, t0 + SELF_CHANGE_TTL_MS); assert_eq!( changes.len(), 1, "recording a change prunes the ones whose echo never came" ); } /// The predicate must answer *only* for a dead FUSE connection. A healthy /// directory and an absent path are both "not stale" — widening it to any /// `metadata` error would make the on-demand restore lazily unmount paths /// that are simply missing. #[test] fn is_stale_mount_is_narrow() { let dir = std::env::temp_dir().join(format!("pdfs-stale-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); assert!(!is_stale_mount(&dir)); assert!(!is_stale_mount(&dir.join("no-such-entry"))); let _ = std::fs::remove_dir_all(&dir); } /// Flatten `segments` into a readable form for assertions. fn segs(iv: &Intervals, start: u64, end: u64) -> Vec<(u64, u64, bool)> { iv.segments(start, end) } /// The conflict copy has to stay openable, so the extension survives — and /// the shape matches the sync engine's `conflict_path` so the two features /// name the same situation the same way. #[test] fn conflict_name_keeps_the_extension() { assert_eq!( conflict_name("notes.txt", 1700), "notes (sync-conflict 1700).txt" ); assert_eq!(conflict_name("README", 42), "README (sync-conflict 42)"); assert_eq!( conflict_name("archive.tar.gz", 7), "archive.tar (sync-conflict 7).gz", "only the last extension is one, as everywhere else" ); } #[test] fn append_merges_into_one_run() { // Sequential writes (an append) coalesce into a single authored range. let mut iv = Intervals::default(); iv.add(0, 10); iv.add(10, 20); iv.add(20, 25); assert_eq!(segs(&iv, 0, 25), vec![(0, 25, true)]); } #[test] fn partial_overwrite_leaves_base_gap() { // Author [0,4) and [8,12); [4,8) stays base. A read of [0,12) must stitch // authored / base / authored in order. let mut iv = Intervals::default(); iv.add(0, 4); iv.add(8, 12); assert_eq!( segs(&iv, 0, 12), vec![(0, 4, true), (4, 8, false), (8, 12, true)] ); } #[test] fn overlapping_writes_coalesce() { let mut iv = Intervals::default(); iv.add(0, 10); iv.add(5, 15); iv.add(14, 20); assert_eq!(segs(&iv, 0, 20), vec![(0, 20, true)]); } #[test] fn segments_clamp_to_request_window() { let mut iv = Intervals::default(); iv.add(0, 100); // A sub-window of one big authored range is a single authored segment. assert_eq!(segs(&iv, 20, 50), vec![(20, 50, true)]); // A window entirely outside any authored range is all base. let empty = Intervals::default(); assert_eq!(segs(&empty, 0, 8), vec![(0, 8, false)]); } #[test] fn truncate_shrink_drops_tail() { // Grow-then-shrink: clip removes/truncates authored ranges past the end. let mut iv = Intervals::default(); iv.add(0, 100); iv.clip(40); assert_eq!(segs(&iv, 0, 40), vec![(0, 40, true)]); // Authored ranges wholly past the new end disappear. let mut iv2 = Intervals::default(); iv2.add(0, 10); iv2.add(50, 60); iv2.clip(40); assert_eq!(segs(&iv2, 0, 40), vec![(0, 10, true), (10, 40, false)]); } #[test] fn truncate_extend_authors_zero_tail() { // setattr grow claims the new tail as authored (defined zeros), so commit // never pulls it from the base. let mut iv = Intervals::default(); iv.add(0, 10); // base content authored over let old_len = 10u64; let new_len = 30u64; iv.add(old_len, new_len); assert_eq!(segs(&iv, 0, 30), vec![(0, 30, true)]); } fn pending_revision(dir: &TestDir, bytes: &[u8], complete: bool) -> PendingRevision { let path = dir.0.join("pending"); std::fs::write(&path, bytes).unwrap(); PendingRevision { path, meta: StagedWrite { uid: "vol~file".into(), len: bytes.len() as u64, base_size: 3, base_mtime: 42, authored: if complete { vec![(0, bytes.len() as u64)] } else { vec![(3, bytes.len() as u64)] }, complete, based_on: Some(Baseline { mtime: 42, size: 3, hash: None, revision_id: None, }), }, } } #[test] fn complete_pending_revision_can_seed_immediate_shrink() { let (_, dir) = state_test_helper(); let pending = pending_revision(&dir, b"0123456789", true); let destination_path = dir.0.join("shrink"); let destination = std::fs::File::create(&destination_path).unwrap(); copy_pending_for_truncate(&pending, &destination).unwrap(); destination.set_len(4).unwrap(); assert_eq!(std::fs::read(destination_path).unwrap(), b"0123"); } #[test] fn complete_pending_revision_can_seed_immediate_growth_and_zero() { let (_, dir) = state_test_helper(); let pending = pending_revision(&dir, b"0123456789", true); let destination_path = dir.0.join("grow"); let destination = std::fs::File::create(&destination_path).unwrap(); copy_pending_for_truncate(&pending, &destination).unwrap(); destination.set_len(14).unwrap(); assert_eq!( std::fs::read(&destination_path).unwrap(), b"0123456789\0\0\0\0" ); destination.set_len(0).unwrap(); assert!(std::fs::read(destination_path).unwrap().is_empty()); } #[test] fn incomplete_pending_revision_is_not_a_safe_truncate_base() { let (_, dir) = state_test_helper(); let pending = pending_revision(&dir, b"\0\0\0authored", false); assert!(!pending.meta.complete); assert_ne!(pending.meta.authored, vec![(0, pending.meta.len)]); } #[test] fn online_combined_cross_directory_rename_stays_synchronous() { assert!(!rename_needs_queue(true, false, true, true)); assert!(rename_needs_queue(false, false, false, true)); assert!(rename_needs_queue(true, true, true, false)); } #[test] fn one_step_online_namespace_changes_stay_synchronous() { assert!(!rename_needs_queue(true, false, false, true)); assert!(!rename_needs_queue(true, false, true, false)); assert!(!rename_needs_queue(true, false, false, false)); } #[test] fn fuse_names_enforce_linux_component_limits() { use std::ffi::OsStr; assert_eq!( fuse_name(OsStr::new("x".repeat(255).as_str())) .unwrap() .len(), 255 ); assert_eq!( fuse_name(OsStr::new("x".repeat(256).as_str())) .expect_err("256-byte component must be refused") .code(), libc::ENAMETOOLONG ); assert_eq!( fuse_name(OsStr::new(".")) .expect_err("dot is not an ordinary component") .code(), libc::EINVAL ); assert_eq!( fuse_name(OsStr::new("unicodé-文件")).unwrap(), "unicodé-文件" ); } #[test] fn local_trash_tombstones_hide_stale_remote_entries() { let folder_uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("folder")); let node = node_helper("child", "folder", "child", false); assert!(node_visible( &node, &folder_uid, &std::collections::HashSet::new() )); assert!(!node_visible( &node, &folder_uid, &std::collections::HashSet::from([node.uid.clone()]) )); } #[test] fn denied_queue_guard_preserves_accepted_bytes_and_pending_count() { use std::cell::Cell; let (mut state, _dir) = state_test_helper(); let root = state.intern(0, node_helper("root", "none", "root", true)); let shared = state.intern(root, node_helper("shared", "root", "shared", true)); state.entries.get_mut(&shared).unwrap().access = Access::Viewer; let uid = state.entries[&shared].uid.clone(); let db = state.flushed_db(); let before = db.pending_ops().unwrap().len(); let preserved = Cell::new(false); let result = preserve_on_access_denied( require_uid_access(&db, &uid, &[Access::Viewer]), true, || preserved.set(true), ); assert_eq!(result.unwrap_err().code(), libc::EACCES); assert!(preserved.get(), "accepted bytes must enter recovery"); assert_eq!(db.pending_ops().unwrap().len(), before); let untouched = Cell::new(false); assert!(preserve_on_access_denied(Ok(()), true, || untouched.set(true)).is_ok()); assert!( !untouched.get(), "a permitted write must continue to the normal queue path" ); // The same fail-closed rule applies before hydration and to a stale UID. db.set_share_access(&uid, Access::Viewer).unwrap(); assert_eq!( require_uid_access(&db, &uid, &[]).unwrap_err().code(), libc::EACCES ); let stale = NodeUid::new(VolumeId::from("vol"), LinkId::from("stale")); assert_eq!( require_uid_access(&db, &stale, &[]).unwrap_err().code(), libc::EACCES ); // Same refusal on the syscall path, but a distinguishable one // underneath: the drain asks the remote about `Unknown` rather than // waiting on a permission change (B83). assert_eq!( uid_write_authority(&db, &stale, &[]), WriteAuthority::Unknown, "a uid the tree has no row for is unknown, not denied" ); assert_eq!( uid_write_authority(&db, &uid, &[]), WriteAuthority::Denied, "a uid the tree knows and refuses is denied" ); assert_eq!(db.pending_ops().unwrap().len(), before); } #[test] fn persisted_and_live_queue_authorities_must_both_be_writable() { let (mut state, _dir) = state_test_helper(); let root = state.intern(0, node_helper("root", "none", "root", true)); let node = state.intern(root, node_helper("shared", "root", "shared", true)); let uid = state.entries[&node].uid.clone(); let db = state.flushed_db(); db.set_share_access(&uid, Access::Viewer).unwrap(); assert_eq!( require_uid_access(&db, &uid, &[Access::Editor]) .unwrap_err() .code(), libc::EACCES, "a persisted downgrade must beat stale writable live state" ); db.set_share_access(&uid, Access::Editor).unwrap(); assert_eq!( require_uid_access(&db, &uid, &[Access::Viewer]) .unwrap_err() .code(), libc::EACCES, "a live downgrade must beat restored writable persisted state" ); assert!(require_uid_access(&db, &uid, &[Access::Editor]).is_ok()); } #[test] fn shared_listing_provenance_must_agree_before_editor_is_granted() { use proton_drive_rs::ShareMembership; use proton_drive_rs::proton_sdk::ids::ShareMembershipId; let uid = NodeUid::new(VolumeId::from("foreign"), LinkId::from("root")); let share_a = ShareId::from("share-a"); let share_b = ShareId::from("share-b"); let parent = shared_with_me_uid(); let item = |share_id: ShareId| SharedWithMeItem { uid: uid.clone(), share_id, }; let node = |share_id: Option, permissions: i32| { let mut node = node_helper("root", "none", "Shared", true); node.uid = uid.clone(); node.membership = share_id.map(|share_id| ShareMembership { share_id, membership_id: ShareMembershipId::from("membership"), permissions, invite_time: None, inviter_email: None, inviter_verification: None, }); node }; let accepted = accepted_share_provenance(vec![item(share_a.clone()), item(share_a.clone())]); assert_eq!( prepare_shared_roots(&accepted, vec![node(Some(share_a.clone()), 6)], &parent)[0] .access, Access::Editor ); for materialized in [ node(None, 6), node(Some(share_b.clone()), 6), node(Some(share_a.clone()), 12345), ] { assert_eq!( prepare_shared_roots(&accepted, vec![materialized], &parent)[0].access, Access::Viewer ); } let conflicting = accepted_share_provenance(vec![item(share_a), item(share_b)]); assert_eq!( prepare_shared_roots( &conflicting, vec![node(Some(ShareId::from("share-a")), 6)], &parent )[0] .access, Access::Viewer, "duplicate UID provenance disagreement must fail closed" ); } #[test] fn virtual_bulk_upload_destination_is_denied_by_runtime_authority() { let (mut state, _dir) = state_test_helper(); let root = state.intern(0, node_helper("root", "none", "My Files", true)); let mut virtual_node = node_helper("sharedwithme", "root", "Shared with me", true); virtual_node.uid = shared_with_me_uid(); state .flushed_db() .set_share_access(&virtual_node.uid, Access::Viewer) .unwrap(); state .share_access .insert(virtual_node.uid.clone(), Access::Viewer); let virtual_ino = state.intern(root, virtual_node); let uid = state.entries[&virtual_ino].uid.clone(); assert_eq!( require_uid_access(&state.db, &uid, &[Access::Viewer]) .unwrap_err() .code(), libc::EACCES ); } fn function_source<'a>(source: &'a str, signature: &str) -> &'a str { let start = source.find(signature).expect("function signature exists"); let body = &source[start..]; let open = body.find('{').expect("function body starts"); let mut depth = 0usize; for (offset, byte) in body.as_bytes()[open..].iter().enumerate() { match byte { b'{' => depth += 1, b'}' => { depth -= 1; if depth == 0 { return &body[..open + offset + 1]; } } _ => {} } } panic!("function body closes") } fn assert_before(source: &str, first: &str, second: &str) { let first = source.find(first).expect("first operation exists"); let second = source.find(second).expect("second operation exists"); assert!(first < second, "{first} must precede {second}"); } #[test] fn production_queue_guards_precede_mutating_work() { let source = include_str!("lib.rs"); let revision = function_source(source, "fn queue_revision("); assert_before(revision, "if !h.dirty", "preserve_on_access_denied"); assert_before(revision, "preserve_on_access_denied", "fill_gaps"); let staged = function_source(source, "fn enqueue_staged_write("); assert_before(staged, "preserve_on_access_denied", "self.pending.lock()"); assert_before( staged, "preserve_on_access_denied", "self.cache.stage_write", ); let truncate = function_source(source, "fn queue_truncate("); assert_before(truncate, "require_uid_writable", "self.pending.lock()"); assert_before(truncate, "require_uid_writable", "create_scratch"); for (signature, first_side_effect) in [ ("fn queue_local_node(", "add_local_row"), ("fn queue_rename(", "queue_rename_authorized"), ("fn queue_trash(", "replace_ops_with_trash"), ] { let function = function_source(source, signature); assert_before(function, "require_uid_writable", first_side_effect); } let trash = function_source(source, "fn queue_trash("); assert_before(trash, "replace_ops_with_trash", "release_dropped_ops"); // §5, parallel drain: every path that unlinks a staged blob or hands // the same node's work to another worker has to stop the upload that // may be reading those bytes *first*. Late is a superseded revision on // the wire, or two workers deciding whether the file exists. for (signature, mutation) in [ ("fn enqueue_staged_write(", "attach_blob_to_create"), ("fn queue_trash(", "replace_ops_with_trash"), ("fn discard_queued_ops(", "delete_ops_for_uid"), ("fn withdraw_queued_ops(", "delete_ops_for_uid"), ] { let function = function_source(source, signature); assert_before(function, "cancel_upload", mutation); } let filesystem = include_str!("filesystem.rs"); for signature in ["fn serve_unlink(", "fn serve_rmdir("] { let function = function_source(filesystem, signature); assert_before(function, "require_writable", "lookup_child"); } let rename = function_source(filesystem, "fn serve_rename("); assert_before(rename, "require_rename_access", "remove_replaced"); assert_before(rename, "remove_replaced", "queue_rename_authorized"); for (signature, gate, remote_call) in [ ("fn serve_create(", "require_uid_writable", "upload_file"), ("fn serve_mkdir(", "require_uid_writable", "create_folder"), ("fn trash_child(", "require_uid_writable", "trash_nodes"), ("fn serve_rename(", "require_rename_access", "rename_remote"), ] { let function = function_source(filesystem, signature); assert_before(function, gate, remote_call); } let control_mkdir = function_source(source, "fn create_folder("); assert_before( control_mkdir, "require_uid_writable", ".create_folder(&parent_uid", ); for (signature, gate, remote_call, settle) in [ ( "fn rename(&self, rel: &Path", "require_rename_access", ".rename_remote", "self.relocate_everywhere(&uid", ), ( "fn move_to(&self, rel: &Path", "require_rename_access", ".move_remote", "self.relocate_everywhere(&uid", ), ( "fn delete(&self, rel: &Path", "require_node_parent_access", ".trash_nodes", "self.forget_everywhere(&uid)", ), ] { let function = function_source(source, signature); assert_before(function, "source_parent_uid", gate); assert_before(function, gate, remote_call); // The kernel did not see this change, so it has to be told the // old name is gone (`docs/BUGS.md` B114). assert_before(function, remote_call, settle); } let upload = include_str!("upload.rs"); let run_uploads = function_source(upload, "async fn run_uploads("); assert_before(run_uploads, "require_uid_writable", ".upload_file_from"); let bulk = function_source(upload, "pub(super) fn upload_paths("); assert_before(bulk, "require_uid_writable", "collect_uploads"); let folder = function_source(upload, "fn ensure_remote_folder("); assert_before(folder, "require_uid_writable(parent_uid)", ".create_folder"); } #[test] fn a_listing_for_the_cli_settles_sizes_before_reading_them() { // A fresh listing holds ciphertext sizes until its upgrade lands // (`docs/BUGS.md` B115). let list = function_source(include_str!("lib.rs"), "fn list_dir(&self, rel: &Path)"); assert_before( list, "self.ensure_children(ino)", "self.settle_listing_sizes(ino)", ); assert_before(list, "self.settle_listing_sizes(ino)", "node_size(&e.node)"); } #[test] fn a_replaced_node_survives_its_open_handles() { // A rename over a file someone is reading must leave that reader on the // replaced bytes; forgetting the inode outright made its next read EIO. let source = include_str!("lib.rs"); let replaced = function_source(source, "fn remove_replaced("); assert!(replaced.contains("forget_or_unlink(uid)")); assert!(!replaced.contains("st.forget(uid)")); assert_before( replaced, "if self.is_open_anywhere(uid)", "self.discard_queued_ops(uid)", ); assert_before(replaced, "if open_now", "self.cache.evict(uid)"); // ...but its queued ops go at once, open or not: a revision drained // after the trash landed on the trashed node and kept its name taken. assert_eq!(replaced.matches("self.withdraw_queued_ops(uid)").count(), 2); } #[test] fn a_write_closed_as_its_create_lands_is_queued() { // The release followed a landing create, found none, and the create // landed before it asked for the create op. Finding none, it took the // file for one unlinked before it was created and dropped the write // (B176). The landing removes the op and moves the row at once, so a // missing op is checked against the row. let revision = function_source(include_str!("lib.rs"), "fn queue_revision("); assert_before( revision, "self.db.has_create_op(", "self.follow_landed_create(h)", ); assert_before( revision, "self.follow_landed_create(h)", "dropping revision", ); } #[test] fn a_replaced_node_put_back_shows_in_the_mount_again() { // The restore after a failed rename left the node hidden and out of // the listing, so a create by its name failed with ENOENT (B124). let restore = function_source(include_str!("lib.rs"), "fn restore_replaced("); assert_before( restore, "self.drive.restore_nodes(", "self.hidden.lock().remove(uid)", ); assert!(restore.contains("st.invalidate_listing(ino)")); assert!(restore.contains("notify.inval_entry(ino, name.to_string())")); } #[test] fn a_size_upgrade_leaves_the_page_cache_alone() { // Reads resolve provisional sizes inline, while the kernel holds the // page lock of that read; dropping pages here waited on it forever. let upgrade = function_source(include_str!("lib.rs"), "fn apply_size_upgrade("); assert!(upgrade.contains("notifier.inval_attrs(ino)")); assert!(!upgrade.contains("notifier.inval_inode(")); let read = function_source(include_str!("filesystem.rs"), " fn read("); assert!(read.contains("core.resolve_provisional_size(ino.0, &uid)")); } fn api_error(code: super::ResponseCode) -> super::ProtonError { super::ProtonError::Api(proton_drive_rs::proton_sdk::error::ProtonApiError { code, http_status: 422, message: "This file or folder was out of date, rename failed.".into(), details: None, }) } #[test] fn a_rename_called_out_of_date_is_retried_until_it_lands() { let mut calls = 0; let mut waits = 0; let result = super::retry_out_of_date( || { calls += 1; if calls < 3 { Err(api_error(super::ResponseCode::InvalidRequirements)) } else { Ok(()) } }, || waits += 1, ); assert!(result.is_ok()); assert_eq!((calls, waits), (3, 2)); } #[test] fn a_rename_gives_up_on_other_errors_and_after_its_retries() { let mut calls = 0; let taken = super::retry_out_of_date( || { calls += 1; Err(api_error(super::ResponseCode::AlreadyExists)) }, || {}, ); assert!(taken.is_err()); assert_eq!(calls, 1, "a taken name is not a stale read"); let mut calls = 0; let stale = super::retry_out_of_date( || { calls += 1; Err(api_error(super::ResponseCode::InvalidRequirements)) }, || {}, ); assert!(stale.is_err()); assert_eq!(calls, super::OUT_OF_DATE_RETRIES + 1); } #[test] fn every_remote_rename_and_move_rides_out_a_stale_hash() { for source in [ include_str!("lib.rs"), include_str!("filesystem.rs"), include_str!("drain.rs"), include_str!("relocate.rs"), ] { assert_eq!( source.matches(concat!("drive.rename_node", "(")).count(), usize::from(source.contains("fn rename_remote(")), "call Core::rename_remote instead of the SDK rename directly" ); assert_eq!( source.matches(concat!("drive.move_node", "(")).count(), usize::from(source.contains("fn move_remote(")), "call Core::move_remote instead of the SDK move directly" ); assert_eq!( source .matches(concat!("drive.move_nodes_streaming", "(")) .count(), usize::from(source.contains("fn move_rename_remote(")), "call Core::move_rename_remote instead of the SDK move directly" ); } } #[test] fn a_rename_or_move_drops_the_sdk_copy_and_expects_its_echo() { // The SDK would send the old name hash with the node's next rename or // move until the event poll caught up (B121), and the echo of the // change, taken for someone else's, evicted the cached bytes (B123). let change = function_source(include_str!("lib.rs"), "fn change_remote("); assert!( change.contains("self.forget_sdk_node(uid)"), "forget the node before each retry" ); assert!(change.contains("self.changed_remote(uid)")); let moved = function_source(include_str!("lib.rs"), "fn move_rename_remote("); assert!(moved.contains("self.change_remote(uid")); let changed = function_source(include_str!("lib.rs"), "fn changed_remote("); assert!(changed.contains("self.forget_sdk_node(uid)")); assert!(changed.contains("self.note_self_change(uid)")); } #[test] fn a_listing_changed_while_published_is_taken_again() { // The root listed again after a restart while a lookup was publishing // the "Shared with me" entry, and the lookup answered EAGAIN (B159). let ensure = function_source(include_str!("lib.rs"), "fn ensure_children("); assert_before( ensure, "for _ in 1..LISTING_ATTEMPTS", "Errno::EAGAIN.code()", ); assert_eq!(ensure.matches("self.list_children(ino)").count(), 2); assert!( !ensure.contains("?"), "an attempt's error must not skip the retry" ); } #[test] fn a_folder_lists_from_the_db_while_the_link_is_down() { // A listing dropped as stale mid-run, then a drop: every create in that // folder looks the name up first, and answered EIO for want of it. let ensure = function_source(include_str!("lib.rs"), "fn list_children("); assert_before( ensure, "if !self.is_online()", "enumerate_folder_children_node_uids", ); assert_eq!(ensure.matches("adopt_stale_listing(").count(), 3); let online = function_source(include_str!("link.rs"), "fn mark_online("); assert!(online.contains("self.relist_offline_listings();")); } #[test] fn a_remote_read_gives_up_at_its_deadline() { // The SDK bounds requests, not reads; a stalled link held one `read` // for 34 minutes. let remote = function_source(include_str!("reads.rs"), "fn read_range_remote("); assert_before( remote, "let fetch = async", "tokio::time::timeout(budget, fetch)", ); assert!(remote.contains("let deadline = Instant::now() + READ_FETCH_TIMEOUT;")); // Built inside the runtime: a FUSE worker has no reactor, and a timer // built there panicked every remote read. assert!(remote.contains("async { tokio::time::timeout(budget, fetch).await }")); } #[test] fn a_kernel_notice_sent_from_the_runtime_leaves_its_timers_running() { // A notice waits in the kernel for a read whose fetch waits on a // timer of the same runtime (B130). Here the runtime's one worker sends // it, and the timer has to fire for the notice to return. let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() .build() .unwrap(); let (fetched, fetch) = std::sync::mpsc::channel(); rt.spawn(async move { tokio::time::sleep(Duration::from_millis(10)).await; let _ = fetched.send(()); }); let notice = rt.spawn(async move { off_the_runtime(|| fetch.recv_timeout(Duration::from_secs(5))) }); assert!(rt.block_on(notice).unwrap().is_ok()); // Every notice to a session goes through it. let lib = include_str!("lib.rs"); for method in [ "fn inval_inode(&self", "fn inval_attrs(&self", "fn inval_entry(&self", "fn delete(&self, parent: u64", ] { let source = function_source(lib, method); assert_eq!(source.matches("self.notifier.").count(), 1, "{method}"); assert!( source.contains("off_the_runtime(|| self.notifier."), "{method}" ); } } #[test] fn a_deadline_built_under_block_on_needs_no_ambient_runtime() { let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() .build() .unwrap(); let answered = std::thread::spawn(move || { rt.block_on(async { tokio::time::timeout(Duration::from_secs(5), async { 7 }).await }) }) .join() .expect("a worker thread outside the runtime can wait on the deadline"); assert_eq!(answered.unwrap(), 7); } #[test] fn a_trashed_open_node_sends_nothing_more() { // Trashing a file under an open handle withdraws its queued upload at // once; the bytes the handle reads go with the last close. let withdraw = function_source(include_str!("lib.rs"), "fn withdraw_queued_ops("); assert_before(withdraw, "self.pending_blob(uid)", "release_dropped_below"); assert!(!withdraw.contains("self.pending.lock().remove(uid)")); let filesystem = include_str!("filesystem.rs"); let trash = function_source(filesystem, "fn trash_child("); assert_eq!( trash.matches("self.core.withdraw_queued_ops(&uid)").count(), 2 ); let release = function_source(filesystem, "fn finish_release("); assert_before( release, "let withdrawn = self.core.pending_blob(&uid)", "discard_queued_ops", ); assert_before(release, "discard_queued_ops", "discard_staged(&blob)"); // A drain worker that claimed the op before it was withdrawn checks // again before any byte leaves. let drain = function_source(include_str!("drain.rs"), "pub(crate) fn drain_revision("); let upload = &drain[drain.find("begin_cancellable_upload").unwrap()..]; assert_before(upload, "self.db.op_exists(op.id)", "File::open(&blob)"); } #[test] fn an_online_rename_writes_the_move_through_before_it_answers() { // Through the raw lock, the move stayed in the state's outbox, and the // next lookup rebuilt the destination from a database without it // (B135). let rename: String = function_source(include_str!("filesystem.rs"), "fn serve_rename(") .split_whitespace() .collect(); assert!(rename.contains("self.core.state().relocate(")); assert!(!rename.contains(".lock().relocate(")); } #[test] fn a_paused_drain_claims_nothing() { // A claimed create counts as sent, so trashing its folder kept it as // a trash, although the paused worker only handed it back (B178). let worker: String = function_source(include_str!("drain.rs"), "pub(crate) fn run_pending_drain(") .split_whitespace() .collect(); assert!(worker.contains("(online&&!paused).then(||self.db.claim_next_due_op(now))")); } #[test] fn a_landed_upload_stays_pending_until_its_bytes_are_cached() { let drain = function_source(include_str!("drain.rs"), "pub(crate) fn drain_revision("); assert_before( drain, "let _landing = LandingUpload::begin(&self.landing_uploads)", "self.db.delete_op(op.id)", ); assert_before(drain, "self.db.delete_op(op.id)", ".store_file(&uid,"); let status = function_source(include_str!("control.rs"), "fn status_response("); assert_before( status, "core.db.pending_op_counts()", "core.landing_uploads.load(", ); assert!(status.contains("queued.uploads.max(0) as u64 + landing")); } #[test] fn replacement_rename_authorizes_source_and_both_parents() { use std::cell::RefCell; let source = NodeUid::new(VolumeId::from("vol"), LinkId::from("source")); let old_parent = NodeUid::new(VolumeId::from("vol"), LinkId::from("old")); let new_parent = NodeUid::new(VolumeId::from("vol"), LinkId::from("new")); let checked = RefCell::new(Vec::new()); require_rename_access( |uid| { checked.borrow_mut().push(uid.to_string()); Ok(()) }, &source, &old_parent, &new_parent, ) .unwrap(); assert_eq!( checked.into_inner(), vec!["vol~source", "vol~old", "vol~new"] ); } #[test] fn writable_nested_node_cannot_mutate_a_viewer_parent_namespace() { let source = NodeUid::new(VolumeId::from("vol"), LinkId::from("nested-editor")); let viewer_parent = NodeUid::new(VolumeId::from("vol"), LinkId::from("viewer-parent")); let writable_destination = NodeUid::new(VolumeId::from("vol"), LinkId::from("destination")); let access = |uid: &NodeUid| { if uid == &viewer_parent { Err(Errno::EACCES) } else { Ok(()) } }; assert_eq!( require_node_parent_access(access, &source, &viewer_parent) .unwrap_err() .code(), libc::EACCES, "delete must authorize the source namespace" ); assert_eq!( require_rename_access(access, &source, &viewer_parent, &writable_destination,) .unwrap_err() .code(), libc::EACCES, "rename and move must authorize the original parent" ); } #[test] fn access_handler_logic_matches_advertised_permissions() { assert!(access_allowed(false, Access::Viewer, AccessFlags::F_OK)); assert!(access_allowed(false, Access::Viewer, AccessFlags::R_OK)); assert!(!access_allowed(false, Access::Viewer, AccessFlags::W_OK)); assert!(!access_allowed(false, Access::Owner, AccessFlags::X_OK)); assert!(access_allowed(true, Access::Viewer, AccessFlags::X_OK)); assert!(access_allowed( true, Access::Editor, AccessFlags::R_OK | AccessFlags::W_OK | AccessFlags::X_OK )); } use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId}; use proton_drive_rs::{Node, NodeKind}; #[test] fn test_posix_unlink_and_rmdir_checks() { let (mut st, _dir) = state_test_helper(); let parent = st.intern(0, node_helper("parent_uid", "none", "parent", true)); let folder_child = st.intern( parent, node_helper("folder_child", "parent_uid", "subfolder", true), ); let file_child = st.intern( parent, node_helper("file_child", "parent_uid", "file.txt", false), ); st.children.insert(parent, vec![folder_child, file_child]); // unlink on folder -> EISDIR check let folder_entry = st.entries.get(&folder_child).unwrap(); assert!(folder_entry.node.is_folder(), "folder_child is a folder"); // rmdir on file -> ENOTDIR check let file_entry = st.entries.get(&file_child).unwrap(); assert!(!file_entry.node.is_folder(), "file_child is a file"); // rmdir on non-empty parent -> ENOTEMPTY check assert!(st.has_children(parent), "parent is not empty"); } struct TestDir(std::path::PathBuf); impl Drop for TestDir { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } fn state_test_helper() -> (crate::state::State, TestDir) { use std::sync::atomic::{AtomicU64, Ordering}; static N: AtomicU64 = AtomicU64::new(0); let dir_path = std::env::temp_dir().join(format!( "pdfs-lib-test-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed) )); std::fs::create_dir_all(&dir_path).unwrap(); let db = pdfs_core::db::Db::open(&dir_path.join("cache.db")).unwrap(); let share_access = db.all_share_access().unwrap(); let st = crate::state::State::new(std::sync::Arc::new(db), share_access); (st, TestDir(dir_path)) } fn rooted_state(volume: &str, root_link: &str) -> (crate::state::State, TestDir) { let (mut state, dir) = state_test_helper(); let root = node_helper_in_volume(volume, root_link, None, "root", true); // At the root inode, as `ProtonFs::new` puts it. state.by_uid.insert(root.uid.clone(), super::ROOT_INO); state.entries.insert( super::ROOT_INO, crate::state::Entry { uid: root.uid.clone(), parent: super::ROOT_INO, node: root, access: Access::Owner, lookup_count: 1, open_count: 0, unlinked: false, }, ); state.roots.insert(super::ROOT_INO); (state, dir) } fn node_helper_in_volume( volume: &str, id: &str, parent: Option<&str>, name: &str, is_dir: bool, ) -> Node { let mut node = node_helper(id, "none", name, is_dir); node.uid = NodeUid::new(VolumeId::from(volume), LinkId::from(id)); node.parent_uid = parent.map(|parent| NodeUid::new(VolumeId::from(volume), LinkId::from(parent))); node } /// A rename or move made over the control socket keeps the node and its /// row, so the queue and the drain still know it (`docs/BUGS.md` B117), /// and tells the kernel the old name is gone (`docs/BUGS.md` B114). #[test] fn a_control_move_keeps_the_node_and_its_row_and_drops_the_old_name() { use super::{KernelNotice, NotifyBatch, relocate_in_mount}; let (mut state, _dir) = state_test_helper(); let src = state.intern(0, node_helper("src", "none", "src", true)); let dst = state.intern(0, node_helper("dst", "none", "dst", true)); let moved = node_helper("moved", "src", "a.txt", false); let uid = moved.uid.clone(); let ino = state.intern(src, moved); state.children.insert(src, vec![ino]); state.children.insert(dst, vec![]); let dst_uid = state.entries[&dst].uid.clone(); let mut notify = NotifyBatch::default(); let outcome = relocate_in_mount(&mut state, &mut notify, &uid, &dst_uid, "b.txt", false); assert_eq!(outcome, Some(("a.txt".to_string(), true))); assert_eq!(state.by_uid.get(&uid), Some(&ino), "the inode survives"); assert_eq!(state.entries[&ino].parent, dst); let row = state .flushed_db() .node_by_uid(&uid.to_string()) .unwrap() .expect("the row survives the move"); assert_eq!(row.name, "b.txt"); assert_eq!(row.parent_uid.as_ref(), Some(&dst_uid)); assert!(notify.0.iter().any(|notice| matches!( notice, KernelNotice::InvalEntry { parent, name } if *parent == src && name == "a.txt" ))); // A mount that does not show the new parent forgets the node, and the // kernel the name, but the row stays for the caller to move. let elsewhere = node_helper("elsewhere", "none", "elsewhere", true).uid; let mut notify = NotifyBatch::default(); let outcome = relocate_in_mount(&mut state, &mut notify, &uid, &elsewhere, "c.txt", false); assert_eq!(outcome, Some(("b.txt".to_string(), false))); assert!(!state.by_uid.contains_key(&uid)); assert!(notify.0.iter().any(|notice| matches!( notice, KernelNotice::Delete { parent, child, name } if *parent == dst && *child == ino && name == "b.txt" ))); assert!( state .flushed_db() .node_by_uid(&uid.to_string()) .unwrap() .is_some() ); } /// A control-socket move that is queued keeps both listings: Drive still /// has the old name, so listing again would bring it back (B149). #[test] fn a_queued_control_move_keeps_both_listings() { use super::{KernelNotice, NotifyBatch, relocate_in_mount}; let (mut state, _dir) = state_test_helper(); let src = state.intern(0, node_helper("src", "none", "src", true)); let dst = state.intern(0, node_helper("dst", "none", "dst", true)); let moved = node_helper("moved", "src", "a.txt", false); let uid = moved.uid.clone(); let ino = state.intern(src, moved); state.children.insert(src, vec![ino]); state.children.insert(dst, vec![]); let dst_uid = state.entries[&dst].uid.clone(); let mut notify = NotifyBatch::default(); let outcome = relocate_in_mount(&mut state, &mut notify, &uid, &dst_uid, "b.txt", true); assert_eq!(outcome, Some(("a.txt".to_string(), true))); assert_eq!(state.children.get(&src), Some(&vec![])); assert_eq!(state.children.get(&dst), Some(&vec![ino])); assert_eq!(state.entries[&ino].node.name, "b.txt"); for folder in [src, dst] { assert!( notify.0.iter().any( |notice| matches!(notice, KernelNotice::InvalInode(ino) if *ino == folder) ) ); } assert!(notify.0.iter().any(|notice| matches!( notice, KernelNotice::InvalEntry { parent, name } if *parent == src && name == "a.txt" ))); } #[test] fn queued_trash_of_open_file_survives_last_release_until_atomic_completion() { let (mut state, _dir) = state_test_helper(); let parent = state.intern(0, node_helper("parent", "none", "parent", true)); let file_node = node_helper("open-trash", "parent", "dirty.txt", false); let uid = file_node.uid.clone(); let ino = state.intern(parent, file_node); state.children.insert(parent, vec![ino]); state.entries.get_mut(&ino).unwrap().open_count = 1; // The typed DB operation atomically replaces the old revision with the // one intent that must survive the open handle's final release. state .flushed_db() .enqueue_op(&PendingOp { id: 0, kind: OP_REVISION.to_string(), uid: uid.to_string(), parent_uid: None, name: None, blob_path: None, meta_json: None, created_at: 1, attempts: 0, last_error: None, next_attempt_at: 0, }) .unwrap(); let (_, blobs) = state .flushed_db() .replace_ops_with_trash(&uid.to_string(), "dirty.txt", 2) .unwrap(); assert!(blobs.is_empty()); let queued_counts = state.flushed_db().pending_op_counts().unwrap(); state.unlink_mem(&uid); assert!(state.entries[&ino].unlinked); assert!(!state.children[&parent].contains(&ino)); assert!( state .flushed_db() .node_by_uid(&uid.to_string()) .unwrap() .is_some() ); let released = release_unlinked_entry(&mut state, ino).unwrap(); assert_eq!(released, uid); assert!(!state.by_uid.contains_key(&uid)); assert!(release_must_retain_queued_trash(&state.db, &uid).unwrap()); assert!( state .flushed_db() .node_by_uid(&uid.to_string()) .unwrap() .is_some(), "last release keeps persisted drain authority" ); let pending = state.flushed_db().pending_ops().unwrap(); assert_eq!(pending.len(), 1, "release must not add a revision"); assert_eq!(pending[0].kind, OP_TRASH); let after_release = state.flushed_db().pending_op_counts().unwrap(); assert_eq!(after_release.uploads, 0); assert_eq!(after_release.changes, 1); assert_eq!(after_release.uploads, queued_counts.uploads); assert_eq!(after_release.changes, queued_counts.changes); state .flushed_db() .complete_trash_op(pending[0].id, &uid) .unwrap(); assert!(state.flushed_db().pending_ops().unwrap().is_empty()); assert!( state .flushed_db() .node_by_uid(&uid.to_string()) .unwrap() .is_none() ); } #[test] fn queued_trash_lookup_failure_conservatively_retains_unlinked_state() { let (mut state, _dir) = state_test_helper(); let uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("uncertain-trash")); state .flushed_db() .with_conn(|conn| { conn.execute_batch("DROP TABLE pending_op")?; Ok(()) }) .unwrap(); assert!(release_must_retain_queued_trash(&state.db, &uid).is_err()); assert!( !release_can_discard_unlinked(&state.db, &uid), "database uncertainty must retain the tombstone and authority row" ); } #[test] fn hydrated_listings_hold_each_folders_live_children() { let entry = |id: &str, parent: u64, is_dir: bool, trashed: bool| { let mut node = node_helper(id, "none", id, is_dir); node.trashed = trashed; crate::state::Entry { uid: node.uid.clone(), parent, node, access: Access::Owner, lookup_count: 1, open_count: 0, unlinked: false, } }; let entries: HashMap = [ (1, entry("root", 1, true, false)), (2, entry("a", 1, true, false)), (3, entry("b", 1, true, false)), (4, entry("a1", 2, false, false)), (5, entry("a2", 2, false, true)), (6, entry("b1", 3, false, false)), ] .into_iter() .collect(); let mut listings = super::listed_children(&entries, vec![1, 2]); listings.values_mut().for_each(|kids| kids.sort()); assert_eq!(listings, HashMap::from([(1, vec![2, 3]), (2, vec![4])])); } fn node_helper(id: &str, parent: &str, name: &str, is_dir: bool) -> Node { let uid = NodeUid::new(VolumeId::from("vol"), LinkId::from(id)); let parent_uid = if parent == "none" { None } else { Some(NodeUid::new(VolumeId::from("vol"), LinkId::from(parent))) }; Node { uid, parent_uid, name: name.to_string(), kind: if is_dir { NodeKind::Folder } else { NodeKind::File { media_type: "text/plain".into(), total_size_on_storage: 0, active_revision_state: None, active_revision_id: None, content_sha1: None, claimed_size: Some(0), claimed_modification_time: None, } }, creation_time: 100, modification_time: 100, trashed: false, is_shared: false, is_shared_publicly: false, signature_email: None, membership: None, photo: None, album: None, verification: Default::default(), direct_role: None, share_id: None, } } } #[cfg(test)] mod thumbnail_miss_tests { use super::*; #[test] fn remote_misses_never_suppress_local_generation() { let uid = parse_uid("volume~node").expect("uid"); let key = (uid, ThumbnailType::Thumbnail.as_i32()); let mut misses = ThumbnailMissCaches::default(); misses.remember_remote(key.clone(), 42); assert!(misses.remote_contains(&key, 42)); assert!(!misses.local_contains(&key, 42)); misses.remember_local(key.clone(), 42); assert!(misses.local_contains(&key, 42)); misses.forget_local(&key); assert!(!misses.local_contains(&key, 42)); assert!(misses.remote_contains(&key, 42)); } #[test] fn local_miss_cache_is_bounded_on_gui_only_paths() { let mut misses = ThumbnailMissCaches::default(); for index in 0..=MAX_THUMBNAIL_MISSES { let uid = parse_uid(&format!("volume~node-{index}")).expect("uid"); misses.remember_local((uid, ThumbnailType::Thumbnail.as_i32()), 42); } assert_eq!(misses.local.len(), 1); } } #[cfg(test)] mod search_root_tests { use super::*; fn roots(pairs: &[(&str, &str)]) -> SearchRoots { SearchRoots { roots: pairs .iter() .map(|(root, local)| ((*root).to_string(), PathBuf::from(local))) .collect(), } } #[test] fn a_hit_below_a_root_resolves_through_it() { let roots = roots(&[("Videos/Anime", "/home/u/anime")]); assert_eq!( roots.resolve("Videos/Anime/s01/e01.mkv").as_deref(), Some("/home/u/anime/s01/e01.mkv") ); } #[test] fn a_hit_outside_every_root_resolves_nowhere() { let roots = roots(&[("Videos/Anime", "/home/u/anime")]); assert_eq!(roots.resolve("Documents/tax.pdf"), None); } #[test] fn a_sibling_sharing_a_name_prefix_is_not_below_the_root() { // `Videos/Anime2` starts with `Videos/Anime` as a string and is a // different folder; only a `/` boundary makes it a descendant. let roots = roots(&[("Videos/Anime", "/home/u/anime")]); assert_eq!(roots.resolve("Videos/Anime2/e01.mkv"), None); } #[test] fn the_most_specific_root_wins() { let roots = roots(&[ ("Videos", "/home/u/videos"), ("Videos/Anime", "/home/u/anime"), ]); assert_eq!( roots.resolve("Videos/Anime/e01.mkv").as_deref(), Some("/home/u/anime/e01.mkv"), "the root leaving the shortest descendant path is the one the user \ configured for this content" ); } #[test] fn a_root_covering_the_whole_drive_resolves_everything() { let roots = roots(&[("", "/home/u/drive")]); assert_eq!( roots.resolve("Documents/tax.pdf").as_deref(), Some("/home/u/drive/Documents/tax.pdf") ); } #[test] fn the_root_folder_itself_resolves_to_its_local_directory() { let roots = roots(&[("Videos/Anime", "/home/u/anime")]); assert_eq!( roots.resolve("Videos/Anime").as_deref(), Some("/home/u/anime/") ); } }