//! On-demand FUSE filesystem for Proton Drive. //! //! Phase 1 is a read-only placeholder mount: directories are enumerated lazily //! from the remote on first access and file content is hydrated on `read` via //! [`ProtonDriveClient::download_range`]. Nothing is mirrored to local disk; //! every byte is fetched on demand. //! //! Phase 2 adds live remote sync: a background task polls the volume event //! cursor ([`ProtonDriveClient::enumerate_events`]) and pushes invalidations //! into the kernel via a fuser [`Notifier`], so the cache TTL can be long while //! remote changes still show up promptly. //! //! Phase 3 makes the mount writable. Each file opened for writing gets a //! `WriteHandle` whose buffer accumulates the full plaintext; on flush/release //! the buffer is sealed as a new revision via //! [`ProtonDriveClient::upload_new_revision`] (the SDK uploads whole revisions, //! not byte ranges). New files are created empty up front so they get a real //! uid; namespace ops map to `create_folder`, `trash_nodes`, `rename_node` and //! `move_node`. //! //! Phase 4 adds Files-On-Demand pinning. A control socket (see `control`) //! lets the CLI pin/unpin files; a pinned file's plaintext is downloaded once //! into the on-disk [`ContentCache`] and every later `read` is served from disk //! instead of the network. Writes and remote events evict the cache so it never //! goes stale. //! //! Reads of unpinned files no longer hit the network per call: `Core::read_range` //! fetches and caches [`BLOCK_SIZE`]-aligned blocks, so sequential/sparse reads //! reuse the on-disk block cache. Writes are disk-backed: each `WriteHandle` //! stages authored bytes in a scratch file and tracks them with an `Intervals` //! set, so a multi-GiB write never buffers in RAM and only the untouched //! remainder of the file is pulled from the remote — lazily, at commit. use parking_lot::{Condvar, Mutex}; use std::collections::{HashMap, HashSet, VecDeque}; use std::ffi::OsStr; use std::fs::File; use std::io::Write; use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::FileExt; use std::os::unix::net::UnixListener; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use fuser::ReplyXattr; use fuser::{ AccessFlags, BackgroundSession, BsdFileFlags, Config, CopyFileRangeFlags, Errno, FileAttr, FileHandle, FileType, Filesystem, FopenFlags, Generation, INodeNo, InitFlags, IoctlFlags, KernelConfig, LockOwner, MountOption, Notifier, OpenAccMode, OpenFlags, RenameFlags, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyIoctl, ReplyLseek, ReplyOpen, ReplyStatfs, ReplyWrite, Request, Session, TimeOrNow, WriteFlags, }; use futures::StreamExt as _; use pdfs_core::batch; use pdfs_core::cache::{BLOCK_SIZE, Baseline, BlockGeometry, BlockSpan, ContentCache, StagedWrite}; use pdfs_core::config::{AppDirs, SweepMode}; use pdfs_core::control::{ ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo, SearchFilters, SearchHit, SearchSource, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, }; use pdfs_core::db::{ Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_UNTIL, PendingOp, PublishedSharedRoot, RenameMeta, StoredNode, StoredSyncFolder, StoredTrash, }; use pdfs_core::localindex; use pdfs_core::search::relevance_score; use pdfs_core::syncignore::is_transient_name; use pdfs_core::{Access, CoreError, CoreResult, access_for, perm_bits}; use proton_drive_rs::proton_sdk::api::ResponseCode; use proton_drive_rs::proton_sdk::error::ProtonError; use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, ShareId, VolumeId}; use proton_drive_rs::{ AvailableNameTarget, DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, NodeMoveItem, ProtonDriveClient, ProtonPhotosClient, SharedWithMeItem, ThumbnailType, }; mod albums; mod attach; mod background; mod conflicts; mod control; use control::run_control_socket; mod devices; mod diagnostics; mod drain; mod drive; use drive::{DriveApi, RevisionRead}; mod events; mod filesystem; mod keeper; mod link; pub use filesystem::ProtonFs; mod mount; mod pause; mod photos; mod profile; mod queue; mod reads; mod redate; mod relocate; mod revisions; mod sharing; mod shutdown; #[cfg(test)] mod sim; mod state; mod supervisor; mod sweep; mod sync; mod systemd; mod takeout; mod transfers; mod upload; mod r#virtual; mod workers; use background::{forget_and_notify, run_event_sync, run_local_index, run_photos_event_sync}; use link::BATCH_CALL_TIMEOUT; pub(crate) use mount::is_stale_mount; pub use mount::prepare_mountpoint; pub use mount::{MountOptions, MountOutcome, mount}; use mount::{ SecondaryInsertRejection, SecondaryMount, clear_stale_mount, fuse_connection_id, spawn_session, }; use reads::{BlockFlight, BlockRing, PREFETCH_BUDGET, Prefetch, ReaderSlot, STREAM_BYPASS_MIN}; use state::{Entry, Intervals, PendingRevision, State, StateGuard, WriteHandle}; pub use systemd::{idle_sleep, status as notify_status, stretch_watchdog_for_startup}; use tracing::{debug, error, info, warn}; use transfers::{CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry}; use r#virtual::{ SharedListingPlan, SharedRefreshDeadlines, disambiguate_shared_names, is_own_or_virtual_uid, is_primary_root_listing, is_virtual_uid, listing_needs_refresh, refresh_generation_is_current, shared_listing_plan, shared_with_me_uid, virtual_node, virtual_root_name, }; use workers::{FUSE_WORKERS, Lane, Workers}; /// Attribute/entry cache lifetime handed back to the kernel. Long because the /// Phase 2 event poller actively invalidates changed inodes; without a remote /// change this is how long the kernel may serve stale metadata. const TTL: Duration = Duration::from_secs(30); /// How long a `statfs(2)` answer is served from [`Core::quota`] before the /// account figures are refetched. See [`Core::account_quota_cached`]. const QUOTA_TTL: Duration = Duration::from_secs(60); /// Block size reported to the kernel and to `statfs(2)`. /// /// coreutils and most copy tools size their I/O buffers from `st_blksize`, and /// the fuser default of 512 makes them issue reads two orders of magnitude /// smaller than the [`BLOCK_SIZE`] this filesystem actually fetches in. 1 MiB is /// the largest value that stays well inside the kernel's per-request limit. const REPORTED_BLKSIZE: u32 = 1 << 20; /// Ceiling on in-flight background requests (readahead, writeback). The kernel /// default of 16 caps how much of a sequential read the kernel will run ahead of /// the application, which on a high-latency link is the whole ballgame. const MAX_BACKGROUND: u16 = 64; /// Longest single path component, as reported by `statfs(2)`. The kernel's own /// dirent limit; Proton's is not lower. const MAX_NAME_LEN: u32 = 255; /// How often the background task polls the remote event cursor. const POLL_INTERVAL: Duration = Duration::from_secs(10); /// Retry backoff for a queued upload, doubling per attempt between these. The /// floor is short because the common failure is a brief network blip; the /// ceiling keeps a persistently failing op from spinning. const DRAIN_BACKOFF_MIN: Duration = Duration::from_secs(2); const DRAIN_BACKOFF_MAX: Duration = Duration::from_secs(300); /// How long the drain worker sleeps when it has nothing due. It is woken /// directly on a new write or a reconnect, so this only bounds how late a /// backoff can fire. const DRAIN_IDLE_POLL: Duration = Duration::from_secs(30); /// How long the mount waits before asking again for a node it has just created /// when Drive does not list it yet (`Core::fetch_minted_node`). const MINTED_READ_BACK_DELAYS: [Duration; 2] = [Duration::from_millis(250), Duration::from_secs(1)]; /// Grace period before a queued revision becomes eligible for draining. /// /// Tools like aria2c preallocate a file (truncate to target size) and then write /// the real content, sometimes across separate open/close cycles. Without a /// grace period the first close drains immediately, uploading the preallocated /// (mostly-zero) content; the second close then finds its baseline stale and /// creates a conflict copy of itself. Holding the op for a short window gives /// the follow-up write time to supersede it. const DRAIN_REVISION_DEBOUNCE: Duration = Duration::from_secs(2); /// Ceiling on the *adaptive* part of that grace period. /// /// The fixed 2 seconds is enough for a preallocate-then-write tool, and wrong /// for a file that takes half a minute to upload: an editor saving every ten /// seconds queues a write, watches it start uploading, and supersedes it — each /// save uploading bytes that the next one replaces, and never catching up. /// Widening the debounce toward how long this node's last upload actually took /// makes the supersede happen in the queue, where it is free, instead of on the /// wire. Bounded so a file on a slow link still reaches Drive in a minute rather /// than deferring indefinitely, and so a one-off stall cannot park a node. const DRAIN_REVISION_DEBOUNCE_MAX: Duration = Duration::from_secs(60); /// How many nodes' measured upload times to remember for that. Bounds a map /// that would otherwise grow with every file ever written; the debounce falls /// back to [`DRAIN_REVISION_DEBOUNCE`] for anything evicted, which is the /// behaviour a node that has never been uploaded gets anyway. const UPLOAD_TIME_MEMORY: usize = 512; /// Threads draining the pending-op queue. /// /// One meant a single 10 GiB upload blocked every queued rename, trash and /// small write behind it. Ordering only has to hold per node — enforced by the /// claim query, not by the thread count — so the rest is throughput. Most ops /// spend their time waiting: a create waits for Drive to list the node it made /// before it reads it back. Three workers left a thousand small creates taking /// 0.3 s each on a LAN and 2.5 s on Wi-Fi, one after another. Uploads cannot /// crowd each other out by number: the SDK caps the blocks in flight across /// the client. const DRAIN_WORKERS: usize = 16; /// How long a staged blob that names no node is kept before it is retired. /// /// Staging is exempt from the cache budget because it holds the only copy of /// writes that have not been uploaded, so nothing there is ever evicted to /// reclaim space. That is right for a blob that still describes something. It is /// wrong for one whose sidecar is missing or names a `local~` placeholder whose /// create is gone: no future run can address it, no UI accounts for it, and it /// occupies disk for the life of the machine. Real instance: 95 such blobs, /// ~445 MiB, the oldest two months old, re-announced on every mount. /// /// A month is far longer than any transfer or any plausible "I will look at that /// later", and the retirement is announced per blob with its size, so the one /// case that matters — bytes a human still wants — has a month of warnings and a /// path to go and fetch them from. const STAGING_ORPHAN_RETAIN: Duration = Duration::from_secs(30 * 24 * 60 * 60); /// First and longest delay between probes for the network coming back after an /// offline mount. Doubles from min to max: a laptop shut in /// a bag is the common case, so the steady state must be cheap, while a brief /// blip should still recover in seconds. const ONLINE_PROBE_MIN: Duration = Duration::from_secs(5); const ONLINE_PROBE_MAX: Duration = Duration::from_secs(300); /// How long the persisted photos timeline stays good before a page request /// revalidates it. The SDK hands back the whole timeline at once, so it is stored /// in the DB and every page is sliced from there; a stale one is still served /// immediately and refreshed in the background. /// /// Short, because the cost of a stale timeline is a photo the user deleted on /// their phone still sitting in the gallery, and the cost of the check is one /// listing call that only runs when somebody is actually looking at the page. const TIMELINE_TTL: Duration = Duration::from_secs(60); /// How many photo nodes are resolved per [`ProtonPhotosClient::enumerate_nodes`] /// call when enriching a refreshed timeline with names and media types (for the /// Photos / Videos / Raw split). Batched so a large library is a handful of /// round-trips rather than one request per photo, and bounded so a single call /// never asks the server to decrypt the whole library at once. /// /// 150 because that is the SDK's own `MAX_BATCH_COUNT`: a larger chunk is split /// there anyway, into one full request and a ragged tail. const TIMELINE_ENRICH_CHUNK: usize = 150; /// The same, for the persisted trash listing. Shorter, because the trash is the /// one listing a user changes and then immediately looks at — though our own /// mutations also invalidate it outright, so this only covers other clients. const TRASH_TTL: Duration = Duration::from_secs(60); /// Shared roots and foreign-volume folders have no event manager. Revalidate /// them on access while online; stale persisted listings remain usable offline. const SHARED_LISTING_TTL: Duration = Duration::from_secs(60); /// How many trashed nodes are materialized per /// [`ProtonDriveClient::enumerate_nodes`] call, and persisted as a batch. The /// trash of an account that has been through a conflict storm runs to thousands /// of nodes, each costing an S2K unlock, so a single all-or-nothing call can /// take minutes and lose everything it decrypted if it fails. Same value as the /// SDK's own `MAX_BATCH_COUNT`: one chunk is exactly one request. const TRASH_MATERIALIZE_CHUNK: usize = 150; /// How long a `ListTrash` request waits for a first-ever refresh before /// answering with whatever has materialized so far. /// /// The refresh keeps running in the background either way — this bounds the /// *reply*, not the work. It has to stay well under the front-ends' 120 s /// control-socket read timeout (`pdfs_core::control`), because a request that /// outlives that timeout is a hang from the user's side: the client gives up, /// the user asks again, and the daemon accumulates another refresh. const TRASH_FIRST_WAIT: Duration = Duration::from_secs(20); /// How long one server call of a trash refresh may take before the refresh /// gives up. /// /// Refreshes are single-flight, so a call that never returns wedges the trash /// listing for the life of the daemon: every later request joins the stuck /// refresh, times out, and answers with the listing from before it. That hid /// a just-trashed folder from a permanent delete for as long as the daemon ran /// (`docs/BUGS.md` B103). One call is one request, so this is generous. const TRASH_CALL_TIMEOUT: Duration = Duration::from_secs(120); /// `sync_state` keys for the freshness stamps of the two persisted listings, and /// for whether the account has a photos volume at all (so an account without one /// doesn't re-ask the server on every page request). const PHOTOS_SYNCED_MS: &str = "photos_synced_ms"; const PHOTOS_AVAILABLE: &str = "photos_available"; /// Freshness stamp of the album listing, and of one album's contents (suffixed /// with the album uid). Separate from the timeline's: albums are enumerated by /// their own endpoints, and an album is only fetched when it is opened. const ALBUMS_SYNCED_MS: &str = "albums_synced_ms"; const ALBUM_SYNCED_PREFIX: &str = "album_synced_ms:"; const TRASH_SYNCED_MS: &str = "trash_synced_ms"; /// Set when a trash refresh finishes and cleared when one starts, so it is only /// present while the stored listing is the whole trash as of some moment. A node /// that a refresh finds and that listing lacks went to the trash since then; one /// found after an interrupted refresh may have been there all along. Unlike /// [`TRASH_SYNCED_MS`], our own mutations leave it alone. const TRASH_COMPLETE_MS: &str = "trash_complete_ms"; const SHARED_WITH_ME_NAME: &str = "shared_with_me_name"; const SHARED_WITH_ME_SYNCED_MS: &str = "shared_with_me_synced_ms"; const SHARED_FOLDER_SYNCED_PREFIX: &str = "shared_folder_synced_ms:"; /// How stale the local-file index may get before the background scanner rebuilds /// it. A rescan is a full walk of `$HOME`, so this trades index freshness against /// disk churn; the scanner also always rebuilds once per daemon start when the /// index is older than this. const LOCAL_INDEX_TTL: Duration = Duration::from_secs(15 * 60); /// How often the scanner thread wakes to check whether the index went stale. const LOCAL_INDEX_CHECK: Duration = Duration::from_secs(60); /// The FUSE root inode is always 1. const ROOT_INO: u64 = 1; /// Parent inode for a persisted node whose parent row is missing from the DB. /// No folder carries this inode, so such a node is listed by nobody and stays /// inert until a live enumeration re-parents it. const ORPHAN_INO: u64 = 0; /// Extended attribute exposing the small server-side thumbnail of a file. const XATTR_THUMBNAIL: &str = "user.proton.thumbnail"; /// Extended attribute exposing the larger server-side preview image of a file. const XATTR_PREVIEW: &str = "user.proton.preview"; /// How many "this file has no thumbnail" answers [`Core::thumbnail`] remembers /// before dropping the lot and re-learning them. Sized to cover a large browsing /// session; each entry is a uid, a type tag and an mtime. const MAX_THUMBNAIL_MISSES: usize = 8192; /// How many nodes one size-upgrade request covers. /// /// Matches the SDK's own `MAX_BATCH_COUNT`, so a chunk is exactly one request: /// chunking smaller would add round trips, larger would be split anyway and /// delay the waiters this chunking exists to release (`docs/BUGS.md` B14). const SIZE_UPGRADE_CHUNK: usize = 150; /// How many folders may have a size upgrade in flight at once. /// /// Each one owns a thread and an API request stream, and the single-flight only /// deduplicates *within* a folder — so without this a recursive listing scaled /// both with the number of folders it walked. Eight keeps an interactive `ls -l` /// (one folder, sometimes a couple) entirely unaffected while putting a ceiling /// on the recursive case. A folder past the cap is queued, not dropped: one of /// the running threads takes it over when its own batch ends (`docs/BUGS.md` B100). const MAX_SIZE_UPGRADES: usize = 8; #[derive(Clone, Debug, Eq, PartialEq)] struct RootListingSnapshot { children: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] struct RootListingChild { ino: u64, entry: Option, } #[derive(Clone, Debug, Eq, PartialEq)] struct RootListingEntry { uid: NodeUid, name: String, parent: u64, trashed: bool, unlinked: bool, } impl RootListingSnapshot { fn capture(state: &State, parent_ino: u64) -> Option { let children = state.children.get(&parent_ino)?; Some(Self { children: children .iter() .map(|ino| RootListingChild { ino: *ino, entry: state.entries.get(ino).and_then(|entry| { (!is_virtual_uid(&entry.uid)).then(|| RootListingEntry { uid: entry.uid.clone(), name: entry.node.name.clone(), parent: entry.parent, trashed: entry.node.trashed, unlinked: entry.unlinked, }) }), }) .collect(), }) } fn real_names(&self) -> HashSet { self.children .iter() .filter_map(|child| child.entry.as_ref()) .map(|entry| entry.name.clone()) .collect() } fn is_current(&self, state: &State, parent_ino: u64) -> bool { Self::capture(state, parent_ino).as_ref() == Some(self) } } struct VirtualRootPlan { node: Node, visible: bool, } /// The sync folders a search hit can be opened through, as /// (Drive-relative root path, local directory) pairs. /// /// Built once per search by [`Core::search_roots`]; resolving a hit against it /// is string arithmetic over the paths the index already carries, with no /// further queries. struct SearchRoots { roots: Vec<(String, PathBuf)>, } impl SearchRoots { /// The absolute local path for a hit at Drive-relative `path`, through the /// most specific root covering it. A node can sit below several configured /// roots, so the one leaving the shortest descendant path wins. fn resolve(&self, path: &str) -> Option { self.roots .iter() .filter_map(|(root, local)| { let relative = relative_to(root, path)?; Some((relative.split('/').count(), local.join(relative))) }) .min_by_key(|(depth, _)| *depth) .map(|(_, path)| path.to_string_lossy().into_owned()) } } /// Negative thumbnail knowledge has two different authorities. A remote miss /// only says Proton has no thumbnail to download; it must never suppress local /// generation. A local miss says this exact file revision was successfully /// inspected and could not be decoded. Both maps are bounded independently. #[derive(Default)] struct ThumbnailMissCaches { remote: HashMap<(NodeUid, i32), i64>, local: HashMap<(NodeUid, i32), i64>, } impl ThumbnailMissCaches { fn remote_contains(&self, key: &(NodeUid, i32), tag: i64) -> bool { self.remote.get(key) == Some(&tag) } fn local_contains(&self, key: &(NodeUid, i32), tag: i64) -> bool { self.local.get(key) == Some(&tag) } fn remember_remote(&mut self, key: (NodeUid, i32), tag: i64) { Self::remember(&mut self.remote, key, tag); } fn remember_local(&mut self, key: (NodeUid, i32), tag: i64) { Self::remember(&mut self.local, key, tag); } fn forget_local(&mut self, key: &(NodeUid, i32)) { self.local.remove(key); } fn remember(map: &mut HashMap<(NodeUid, i32), i64>, key: (NodeUid, i32), tag: i64) { if map.len() >= MAX_THUMBNAIL_MISSES && !map.contains_key(&key) { map.clear(); } map.insert(key, tag); } } /// `path` expressed relative to `root`, or `None` if it is not below it. The /// root itself resolves to the empty path; an empty root is the mount itself /// and covers everything. fn relative_to<'a>(root: &str, path: &'a str) -> Option<&'a str> { if root.is_empty() { return Some(path); } if path == root { return Some(""); } path.strip_prefix(root)?.strip_prefix('/') } #[derive(Clone)] struct Core { /// Drive as the sync core uses it: listings, events, creates, uploads, /// downloads, renames, moves and trashes. The real client in production, an /// in-memory Drive in the simulation tests. See [`drive`]. drive: Arc, /// The concrete client, for what [`DriveApi`] does not cover: sharing, /// Photos, devices, invitations and public links. client: ProtonDriveClient, rt: tokio::runtime::Handle, /// The daemon's My Files root. Forked on-demand sessions retain this value /// so ownership checks never mistake their own mount root for account scope. primary_root_uid: NodeUid, /// False only for a forked on-demand device mount. The synthetic shared /// directory belongs exclusively to the primary My Files mount. primary: bool, /// The one tree every mount of the daemon serves from. state: Arc>, /// The tree inode this mount's session is rooted at, set when the session /// is built ([`ProtonFs::new`]). Each fork has its own cell. root: Arc>, cache: Arc, /// Open [`RevisionReader`]s keyed by node, so the block fetches of a file /// resolve its keys and block table once instead of once per block. /// Validated by `(mtime, size)` exactly like the content cache, and bounded /// by [`MAX_OPEN_READERS`]. readers: Arc>>, /// Decrypted blocks held in memory, so the ~32 kernel reads a 4 MiB block is /// delivered in cost the block once — a download for a streaming file, a /// disk read for a cached one. See [`BlockRing`]. block_ring: Arc>, /// Block fetches in flight, so concurrent readers of one block (demand read, /// kernel read-ahead, our own prefetch) share a single download and decrypt. /// See [`BlockFlight`]. block_flight: Arc, /// One lock per file being repaired by [`Core::repair_block`], so the /// concurrent block reads that all discover the same untrustworthy block /// table share a single whole-file download instead of one each. content_repairs: Arc>>>>, /// Per-file sequential-read detection driving [`Core::prefetch`]. prefetch: Arc, /// Permits bounding speculative block fetches, so prefetch cannot queue in /// front of a read someone is waiting on. prefetch_budget: Arc, /// Threads that serve the FUSE handlers which touch the network, so the /// session's dispatch loop stays free to answer cheap metadata calls while a /// cold read is on the wire. See [`Workers`]. workers: Arc, /// Unified SQLite metadata cache: the persistence layer behind the in-memory /// `State` maps. Every mutation writes through here, and the maps rehydrate /// from it on mount. db: Arc, /// Serializes only shared-list publication and access-loss invalidation. /// Network calls happen before taking it. shared_publication: Arc>, /// Invalidates in-flight shared responses when access/list events arrive. shared_generation: Arc, /// Successful refreshes suppress retries even if persisting their timestamp /// fails. Shared across primary/fork clones and cleared by access events. shared_refresh_deadlines: Arc>, /// False while the API is unreachable and we are serving the cached tree. /// Set by the probe thread; read by front-ends through /// `Response::Status` so the UI can say so rather than leaving the user to /// infer it from a wall of EIO. online: Arc, /// When the link went down and a condvar for its coming back; the other /// half of `online` (see [`link`]). link: Arc, /// When the user paused syncing, the Unix second it resumes by itself; /// [`pause::PAUSED_INDEFINITELY`] for "until resumed", `0` when not paused. /// Persisted, so a pause survives a restart (see [`Core::sync_paused`]). sync_paused_until: Arc, /// Writes accepted from the kernel but not yet uploaded, keyed by node. /// The in-memory face of the `pending_op` table, from /// which it is rebuilt on mount. /// /// Two things read it: [`Core::read_range`], because until the op drains the /// staged blob *is* the file's content and the remote still holds the old /// revision; and the drain worker, which performs the uploads. pending: Arc>>, /// Nodes removed through this daemon which a briefly stale remote listing /// may still return. Uids are not reused; an explicit restore clears one. hidden: Arc>>, /// The folder and name each create on the wire was sent with, by op id. /// Drive holds that name until the create lands and a rename or trash /// queued behind it frees it again. creating: Arc>>, /// The SHA-1 of the last revision this daemon uploaded to a file and could /// not read back, so its revision id is not known (`docs/BUGS.md` B157). sealed_unread: Arc>>, /// The node an upload has just put on Drive, from when the drain lets go /// of its blob until the tree has the node (`docs/BUGS.md` B170). landed: Arc>>, /// Creates that landed but could not be read back, by the uid Drive gave /// them, with the placeholder's. The drain adopts one before it sends /// anything else for it. unadopted: Arc>>, /// Nudges the drain workers: set true and notify to have them re-examine /// the queue instead of waiting out their backoff. drain_wake: Arc<(Mutex, Condvar)>, /// The pins the keeper owes a check (`keeper`). pin_work: Arc>, /// The daemon's stop signal, shared with every background loop this `Core` /// was cloned into so teardown can end and join them (`docs/BUGS.md` B44). shutdown: Arc, /// How long this node's last upload actually took, in milliseconds, with /// the time it was measured for eviction. Feeds /// [`Core::revision_debounce`]: a file that takes 30 seconds to send needs /// a grace period on that scale, or every save supersedes an upload that /// was already on the wire. Bounded by [`UPLOAD_TIME_MEMORY`]. upload_times: Arc>>, /// Cancellation flags for uploads currently on the wire, keyed by node. /// /// Set when a newer revision of the same file is queued: the bytes in /// flight have just been superseded, and the only thing finishing them /// achieves is spending the user's uplink on a revision the next op /// replaces. Read by the upload's own reader ([`CountingReader`]), which is /// the one thing the SDK calls often enough to notice. upload_cancel: Arc>>>, /// Uploads whose op is retired but whose bytes the content cache has not /// adopted yet, because the revision just sealed is still being read back. /// The status counts them as pending, so an empty queue means every file /// written through the mount reads without the network (`docs/BUGS.md` /// B119). landing_uploads: Arc, /// True while a background refresh of the photos timeline (resp. the trash) is /// already running, so a burst of page requests against a stale listing kicks /// off one refresh rather than one per request. timeline_refreshing: Arc, /// How far the running timeline refresh has got resolving photo metadata, /// as `(done, total)` photos; `None` when no refresh is resolving any. The /// Places view shows it while a library's locations are first read. timeline_progress: Arc>>, /// The same, for the album listing. albums_refreshing: Arc, trash_refreshing: Arc, /// True while the similar-photo pass hashes thumbnails (see /// [`Core::photo_similar`]), so asking again doesn't start a second one. similar_hashing: Arc, /// Fires whenever a trash refresh persists a batch or finishes, so a /// `ListTrash` request waiting on a first-ever refresh wakes on progress /// instead of polling or blocking for the whole run. See /// [`Core::await_trash_refresh`]. trash_progress: Arc, /// Conflict copies the sweep has already checked against the activity log /// this run, so each pass does not ask the database again. The log itself /// is what keeps a copy from being flagged again after a restart. See /// [`Core::run_conflict_sweep_loop`] and `docs/BUGS.md` B99. conflict_notified: Arc>>, /// Whether the conflict sweep may actually trash the duplicates it finds, or /// only report them. Resolved once at mount from config + environment /// ([`AppConfig::resolved_conflict_sweep`]); [`SweepMode::Off`] means the /// sweep thread is never spawned at all. See `docs/BUGS.md` B71. sweep_mode: SweepMode, /// A mkdir, create, rename or delete through the mount is recorded and /// queued, and the drain sends it; the syscall never waits for Drive. Off, /// they go to Drive inline while online, as before 3.0. local_first: bool, /// Photos whose missing thumbnail is being generated right now. A tile that is /// still on screen asks for its thumbnail again every few seconds, and each of /// those downloads is a full-size photo — so an in-flight uid is never started /// twice. thumb_gen: Arc>>, /// Global permit pool for locally generated thumbnails. The old per-request /// limit multiplied when the GUI sent several batches, allowing dozens of /// full-size images to download at once. thumb_gen_budget: Arc, /// Current ordinary-file listing generation and a wake-up for tasks waiting /// on a permit or a download when that listing is abandoned. file_thumb_generation: Arc, file_thumb_cancel: Arc, /// Progress of the explicit recursive build started from the Files toolbar. thumbnail_build: Arc>, /// Cancellation for the explicit recursive build. Separate from ordinary /// listing cancellation because the two jobs have independent lifetimes. thumbnail_build_cancelled: Arc, thumbnail_build_cancel: Arc, /// Negative thumbnail knowledge, separated by whether the remote lacked a /// thumbnail or local decoding proved this revision unsupported. /// /// Absence has to be cached or it costs a round trip every time it is asked /// for, and it is asked for constantly: an `ls -l` from an xattr-aware lister /// issues a `getxattr` per advertised name per entry, so a 65-file directory /// of videos re-probed 130 times per listing at ~186 ms each (B5). The mtime /// is the validity tag — a new revision may well have a thumbnail — matching /// how [`ContentCache::read_thumbnail`] validates the positive side. thumbnail_misses: Arc>, /// Last account quota answer and when it was learned, for `statfs(2)`. /// /// `df` and the free-space preflight of every file manager and installer ask /// for this, and the answer costs a round trip — so it is served from here /// for [`QUOTA_TTL`] and refreshed on a worker afterwards. Total account /// storage moves slowly enough that a minute-old number is a better answer /// than either a stall or the zeroes the default implementation returns. quota: Arc>>, /// Set while a `statfs` is refreshing [`Core::quota`], so the others serve /// the last figures instead of each holding a worker on the same request. quota_refreshing: Arc, /// Database maintenance that has been moved off the request thread. /// /// `VACUUM` and a deep `integrity_check` both hold the single connection for /// tens of seconds against a real install, which stops every FUSE handler /// that needs the database — so the handler acknowledges the request and a /// thread does the work, exactly as the other long control requests already /// do. See [`Maintenance`]. maintenance: Arc>, /// Folders whose listing was enumerated cheaply and is having its file sizes /// filled in right now, so a burst of `stat`s over a fresh listing starts one /// upgrade rather than one per entry. See [`Core::spawn_size_upgrade`]. /// Size upgrades currently running, per folder inode, across every mount. A `getattr` that /// needs a real size waits on the entry rather than issuing its own fetch — /// `ls -l` of a folder is one `getattr` per file, and they must collapse /// onto a single batch (`docs/BUGS.md` B14). size_upgrades: Arc>>>, /// Replies parked on those upgrades. Shared across every mount for the same /// reason the pool is: the bound that matters is per daemon. size_waiters: Arc, /// This mount's kernel notification channel, for telling the kernel to drop /// metadata it has cached. Set once the session exists — which is *after* /// the `Core` it is built from, hence the cell. /// /// Per mount, not per daemon: each session calls its own root inode 1. /// [`Core::fork_state`] gives each fork an empty cell of its own. Work that /// changes the tree for every mount tells every session /// ([`Core::for_each_mount`]). notifier: Arc>, /// True only after this mount's FUSE session spawned successfully. /// Registration happens earlier for the primary drain, so registration /// alone cannot answer whether the location is mounted. session_live: Arc, /// In-flight upload/download progress, served to `GetQueueStatus`. Shared /// across the FUSE session and the control-socket task. transfers: Arc, /// Open [`Request::Subscribe`](pdfs_core::control::Request::Subscribe) /// connections, which every change a front-end shows is published to. events: Arc, /// True while the background scanner is rebuilding the local-file index, so /// `SearchLocal` can tell a front-end "still indexing" apart from "no match". indexing: Arc, /// Live per-folder sync progress, keyed by sync-folder id, so `ListSyncFolders` /// can say what a pass is doing rather than just "syncing". An entry exists /// only while that folder's reconcile pass is running. sync_progress: Arc>>, /// Whether the folder-sync engine is alive, and which pass it is on. sync_engine: Arc, /// Channel to the folder-sync engine: nudges it to /// reconcile a folder, reconcile everything, or re-scan its watch set. sync_tx: std::sync::mpsc::Sender, /// Secondary FUSE sessions for `ondemand` sync folders, keyed by sync-folder /// id. Each is a `ProtonFs` rooted at the folder's remote /// node, mounted over its local path, sharing this Core's client/cache/db and /// tree but with its own root (`fork_state`). Held so we can unmount on /// toggle back to `mirror` and on daemon shutdown. Each entry retains the /// FUSE connection id and the fork's exact liveness flag so teardown first /// makes that mount unroutable. mounts: Arc>>, /// Per-sync-folder locks, held for a whole reconcile pass and for a whole /// mode switch. A `mirror→ondemand` flip evicts the local tree and mounts /// FUSE over it, so it must never overlap a pass that is walking and /// uploading that same tree — the engine would upload files as they vanish /// and then walk the FUSE mount as if it were local. sync_locks: Arc>>>>, /// Per-sync-folder paths the last pass could not read, so the activity feed /// says so when the set changes rather than on every poll (B196). sync_unscanned: Arc>>>, /// Nodes this daemon changed on the remote itself, and how many echoes of /// those changes the event feed still owes us. /// /// The feed replays our own writes back at us seconds later, and a /// `NodeUpdated` carries no revision id — nothing in it distinguishes "another /// device changed this file" from "you changed this file". Treating the echo /// as foreign is not free: it evicts the content blob we just uploaded, so the /// next read re-downloads a revision the API may not be serving yet. That is /// how a SQLite database written on the mount came back `malformed`, and how a /// mount doing bulk uploads spent its time re-fetching its own bytes. /// /// So the first event for a node we changed ourselves, within /// [`SELF_CHANGE_TTL_MS`], is consumed rather than applied: the tree already /// holds what that event describes, because [`Core::refresh_after_upload`] put /// it there. Anything beyond that applies normally, which bounds the staleness /// a genuine remote change can suffer to one event. /// /// Shared across forks like `pending`: the drain that records the change and /// the event task that consumes the echo are both per-daemon. self_changes: Arc>>, /// Every live mount: the primary and one per on-demand fork /// ([`Core::fork_state`]), with its mountpoint, root and kernel channel. /// Shared by every fork. /// /// The mounts share one tree, so a change the drain makes is seen by every /// one of them. Each forked mount used to keep its nodes in a state of its /// own, where the drain never looked: the fork's entry kept its `local~` /// placeholder uid forever (`docs/BUGS.md` B74). What is still per mount is /// the kernel's cache, which [`Core::for_each_mount`] tells every session /// about. /// /// `Weak`, so an unmounted fork is dropped rather than pinned here. states: Arc, } /// Background database maintenance: what is running, and what the last deep /// integrity check found. /// /// One at a time — both operations are whole-database and there is nothing to /// gain from overlapping them — and the result of a check outlives the request /// that asked for it, because the request is answered before the check finishes. #[derive(Default)] struct Maintenance { /// A vacuum or integrity check is running right now. running: bool, /// Findings of the most recent completed deep check. `None` means none has /// finished this run, which is *not* the same as "found nothing". integrity: Option>, } /// How long a change this daemon made stays recognisable as its own. Generous /// against a feed that can lag, but finite: past it, an event for the node is /// applied like any other. See [`Core::self_changes`]. const SELF_CHANGE_TTL_MS: i64 = 120_000; /// A remote change this daemon made, awaiting its echo from the event feed. struct SelfChange { /// When we made it, in the same epoch millis as [`now_millis`]. at_ms: i64, /// Echoes not yet seen. More than one because a single file can be created /// and then have a revision uploaded, and the feed reports both. echoes: u32, } /// Record one remote change this daemon made. Split from [`Core`] so the /// expiry and echo-counting rules can be tested against a clock the test owns. fn note_self_change(changes: &mut HashMap, uid: &NodeUid, now_ms: i64) { // Echoes that never arrived — a node trashed before its event came round, a // feed that skipped it — would otherwise accumulate for the life of the // daemon. Pruning on write keeps the map the size of the recent queue. changes.retain(|_, c| now_ms - c.at_ms < SELF_CHANGE_TTL_MS); let change = changes.entry(uid.clone()).or_insert(SelfChange { at_ms: now_ms, echoes: 0, }); change.at_ms = now_ms; change.echoes += 1; } /// Claim one echo, per [`Core::take_self_change`]. fn take_self_change( changes: &mut HashMap, uid: &NodeUid, now_ms: i64, ) -> bool { let Some(change) = changes.get_mut(uid) else { return false; }; // Too old to attribute: the feed is far enough behind that this is as likely // to be someone else's change as ours, and guessing wrong here costs the user // a stale file. if now_ms - change.at_ms >= SELF_CHANGE_TTL_MS { changes.remove(uid); return false; } change.echoes -= 1; if change.echoes == 0 { changes.remove(uid); } true } /// One live mount, as published to the shared registry. #[derive(Clone)] struct LiveMount { mountpoint: PathBuf, /// The mount's kernel channel, empty until its session is spawned. notifier: Arc>, session_live: Arc, /// The tree inode the mount is rooted at, empty until its session is built. root: Arc>, } impl LiveMount { fn is_live(&self) -> bool { self.session_live.load(Ordering::Acquire) } } struct MountedState { /// Absolute local root of this mount. mountpoint: PathBuf, /// The same cell as [`Core::notifier`]. `Weak`, so an unmounted fork is /// dropped rather than pinned here; a dead entry is reaped on the next /// registry walk. notifier: std::sync::Weak>, session_live: Arc, root: Arc>, } /// Every mount in the daemon, shared by the primary `Core` and every /// [`Core::fork_state`] clone of it. /// /// Split out from `Core` so the reap-and-walk rule can be tested on its own: a /// fork that fails to appear here, or one that lingers after unmount, is exactly /// the failure `docs/BUGS.md` B74 was. #[derive(Default)] struct StateRegistry(Mutex>); impl StateRegistry { /// Publish a mount and the channel to the session that serves it. /// /// A mount is registered exactly once by its owner. Session construction /// only flips `session_live`; it never republishes the mount. fn register( &self, mountpoint: &Path, notifier: &Arc>, session_live: Arc, root: Arc>, ) { let mut states = self.0.lock(); states.retain(|m| m.notifier.strong_count() > 0); debug_assert!( !states .iter() .any(|mounted| mounted.notifier.ptr_eq(&Arc::downgrade(notifier))), "a mount must be registered exactly once" ); states.push(MountedState { mountpoint: mountpoint.to_path_buf(), notifier: Arc::downgrade(notifier), session_live, root, }); } /// Every registered mount, reaping the entries whose session has gone. /// Returns owned handles so callers do not hold the registry lock across /// the work. fn live(&self) -> Vec { let mut states = self.0.lock(); states.retain(|m| m.notifier.strong_count() > 0); states .iter() .filter_map(|m| { Some(LiveMount { mountpoint: m.mountpoint.clone(), notifier: m.notifier.upgrade()?, session_live: m.session_live.clone(), root: m.root.clone(), }) }) .collect() } /// The live session whose local root most specifically covers `path`. /// /// On-demand roots can be nested below another location. Selecting by the /// longest component prefix ensures the nested session wins rather than the /// broader primary mount. Callers that need the relative suffix derive it /// with `path.strip_prefix(mountpoint)`. fn covering(&self, path: &Path) -> Option { self.live() .into_iter() .filter(|mount| mount.is_live() && path.starts_with(&mount.mountpoint)) .max_by_key(|mount| mount.mountpoint.components().count()) } fn is_mounted_at(&self, path: &Path) -> bool { self.covering(path) .is_some_and(|mount| mount.mountpoint == path) } /// The tree inodes of the sessions that are mounted. fn live_roots(&self) -> Vec { self.live() .into_iter() .filter(LiveMount::is_live) .filter_map(|mount| mount.root.get().copied()) .collect() } } /// Why [`Core::apply_sync_folder_mode`] did not switch a folder. The two cases are /// answered very differently: `NotNow` is the normal state of a folder that is busy /// or has local changes still to push, and the caller queues the request; `Failed` /// is a real fault the user has to hear about. enum SwitchBlocked { /// The folder is mid-pass, or not yet safe to switch. Try again after a pass. NotNow, /// A queued target was canceled or replaced before the folder lock became /// available. The newer intent owns the next transition. Superseded, /// The switch was attempted and broke. Failed(String), } impl Core { /// Start a background `VACUUM`, or refuse because maintenance is already /// running. /// /// Returns as soon as the thread is spawned; the caller acknowledges the /// request. The outcome is a log line and a job that disappears from /// `GetQueueStatus` when it is done — a vacuum has nothing to report that a /// follow-up `CacheInspect` cannot show. fn start_vacuum(&self) -> CoreResult<()> { { let mut m = self.maintenance.lock(); if m.running { return Err(CoreError::conflict( "database maintenance is already running", )); } m.running = true; } let core = self.clone(); std::thread::spawn(move || { let job = core.transfers.begin_job("Compacting the cache database"); job.detail("Rewriting the database file"); let started = Instant::now(); match core.db.vacuum() { Ok(outcome) => info!( freed_bytes = outcome.freed_bytes(), before_bytes = outcome.before_bytes, after_bytes = outcome.after_bytes, wal_frames = outcome.wal_frames_checkpointed, elapsed_ms = started.elapsed().as_millis() as u64, "cache database vacuumed" ), Err(e) => warn!(error = %e, "vacuum failed"), } core.maintenance.lock().running = false; }); Ok(()) } /// Answer a deep `CacheInspect`: hand back the last completed integrity /// check, and start one if none is running. /// /// Returns `(problems, checked, running)`. A first request gets /// `(_, false, true)` and the answer lands on a later one — the check reads /// every page of the database under the connection mutex, so the choice is /// between answering late and freezing the mount until it finishes. fn start_or_read_integrity_check(&self) -> (Vec, bool, bool) { let mut m = self.maintenance.lock(); if let Some(problems) = m.integrity.clone() { return (problems, true, m.running); } if m.running { return (Vec::new(), false, true); } m.running = true; drop(m); let core = self.clone(); std::thread::spawn(move || { let job = core .transfers .begin_background_job("Checking the cache database"); job.detail("Verifying every page"); let problems = match core.db.integrity_check() { Ok(problems) => problems, Err(e) => vec![format!("integrity check failed: {e:?}")], }; if problems.is_empty() { info!("cache database integrity check found no problems"); } else { warn!(problems = problems.len(), "cache database integrity check"); } let mut m = core.maintenance.lock(); m.integrity = Some(problems); m.running = false; }); (Vec::new(), false, true) } /// Register a mount so background work can reach its session. Called once /// for the primary mount and once per on-demand fork. fn register_state(&self, mountpoint: &Path) { self.states.register( mountpoint, &self.notifier, self.session_live.clone(), self.root.clone(), ); } /// The tree inode this mount is rooted at. [`ROOT_INO`] until the session /// is built, which is where a test that builds a bare tree puts its root. pub(crate) fn root_ino(&self) -> u64 { self.root.get().copied().unwrap_or(ROOT_INO) } /// Whether `uid` names an owned node reachable from the root of a mounted /// session. See [`State::owns_visible_uid`]. fn owns_visible_uid(&self, uid: &NodeUid) -> bool { let roots = self.states.live_roots(); self.state().owns_visible_uid(uid, &roots) } /// This `Core`, re-rooted onto whichever live mount most specifically covers /// the absolute path `abs`, together with `abs` relative to that mount's /// root. /// /// Everything a `Core` holds is per-daemon except the fields /// [`Core::fork_state`] replaces, so swapping exactly those turns the /// primary `Core` into the fork that owns the path — resolving from the /// fork's root and notifying its session. Without this, every /// path-addressed control request is resolved from the primary root, so /// anything under a secondary on-demand mount is rejected as "not under the /// mountpoint" and the user has no way to re-enumerate a folder they can see /// (`docs/BUGS.md` B86). /// /// `None` when no live mount covers `abs`. fn rooted_at(&self, abs: &Path) -> Option<(Core, PathBuf)> { let mount = self.states.covering(abs)?; let rel = abs.strip_prefix(&mount.mountpoint).ok()?.to_path_buf(); let mut core = self.clone(); if !Arc::ptr_eq(&mount.notifier, &self.notifier) { // Another mount, so adopt its per-mount half wholesale. `primary` // is a property of the mount, and the only mount this `Core` can // be the primary of is the one it was built for. core.primary = false; core.notifier = mount.notifier; core.session_live = mount.session_live; core.root = mount.root; } Some((core, rel)) } /// Take the tree's lock. /// /// The guard is what applies the mutation's write-throughs, after releasing /// the lock — see [`StateGuard`]. Every caller takes it this way; nothing /// locks `self.state` directly, or the writes it queues would be applied /// only when some later caller happened to take the guard. fn state(&self) -> StateGuard<'_> { StateGuard::new(self.state.lock(), &self.db) } /// Run `apply` against the tree, then tell every mounted session about the /// access changes it made. /// /// For the drain and the sync engine, which are per-daemon: a `pending_op` /// row records a uid, not the session that queued it, and any mount may be /// showing the node. fn for_each_state(&self, apply: impl FnOnce(&mut State)) { let changed = { let mut state = self.state(); apply(&mut state); state.take_access_changes() }; self.notify_access_changes(&changed); } /// Like [`Core::for_each_state`], but the closure also records what the /// kernel needs told into a [`NotifyBatch`], which goes to every mounted /// session. /// /// Recorded rather than sent: notifications are flushed after the State /// lock is released, the same rule [`Core::for_each_state`] and /// [`Core::flush_access_changes`] follow. fn for_each_mount(&self, apply: impl FnOnce(&mut State, &mut NotifyBatch)) { let mut batch = NotifyBatch::default(); apply(&mut self.state(), &mut batch); for mount in self.states.live() { batch.flush(mount.notifier.get()); } } /// Publish access changes accumulated by an intern/root refresh. The State /// lock is released before notifying the kernel because notifier callbacks /// may synchronously provoke more filesystem work. fn flush_access_changes(&self) { let changed = self.state().take_access_changes(); self.notify_access_changes(&changed); } /// Tell every mounted session that `changed` have new permissions. fn notify_access_changes(&self, changed: &[u64]) { if changed.is_empty() { return; } for mount in self.states.live() { notify_access_changes(mount.notifier.get(), changed); } } fn require_writable(&self, ino: u64) -> Result<(), Errno> { self.state().require_writable(ino) } /// Mutation admission point. Persisted and every resident authority must /// agree that the node is writable. A check completed before a downgrade is /// admitted; later queued attempts are caught by drain. No global lock is /// held across a network call. /// /// The intersection closes both halves of an access-refresh race: a DB /// downgrade cannot be hidden by stale live state, and a live downgrade /// cannot be hidden by a restored DB row. Missing UIDs fail closed. fn require_uid_writable(&self, uid: &NodeUid) -> Result<(), Errno> { match self.uid_write_authority(uid) { WriteAuthority::Writable => Ok(()), // A uid nobody can speak for is refused here just as a denied one // is: a syscall is a live request against a live tree, and a stale // handle must not be admitted because the tree forgot the node. // The drain, which replays intent recorded long ago, is the one // caller that has to tell the two apart — see // [`Core::uid_write_authority`]. WriteAuthority::Denied | WriteAuthority::Unknown => Err(Errno::EACCES), } } /// The full answer behind [`Core::require_uid_writable`]: whether every /// authority agrees this node is writable, refuses it, or has never heard /// of it. /// /// The third case is the one worth naming. A node absent from the local /// tree yields no access row, and collapsing that into "denied" told the /// drain to wait for a permission change that was never coming, because /// nothing about the node's permissions was ever the problem (B83). pub(crate) fn uid_write_authority(&self, uid: &NodeUid) -> WriteAuthority { let live_access: Vec = self.state().access_by_uid(uid).into_iter().collect(); uid_write_authority(&self.db, uid, &live_access) } } /// Whether a node may be written, as agreed by every authority that has an /// opinion — or the fact that none of them does. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum WriteAuthority { /// The persisted tree and every resident mount agree: writable. Writable, /// At least one authority refuses it, or could not be consulted. Fails /// closed, so a lookup error lands here. Denied, /// The node is not in the local tree, so there is no authority to consult. /// Not the same as a refusal, and callers that can ask the remote should. Unknown, } fn uid_write_authority(db: &Db, uid: &NodeUid, live_access: &[Access]) -> WriteAuthority { let persisted = match db.effective_node_access(uid) { Ok(Some(access)) => access, Ok(None) => return WriteAuthority::Unknown, Err(error) => { error!(%uid, %error, "db effective access lookup failed"); return WriteAuthority::Denied; } }; if persisted.writable() && live_access.iter().all(|access| access.writable()) { WriteAuthority::Writable } else { WriteAuthority::Denied } } /// Move `uid` to `name` in `new_parent` within one mount, for a rename or move /// Drive has made, and record what the kernel has to forget. Returns the name /// the mount knew the node by, and whether the node was moved rather than /// forgotten, which writes its row through. `None` when the mount does not /// hold the node. /// /// The inode is kept wherever the mount shows the new parent, as the mount's /// own rename keeps it (`State::relocate`). A mount that does not show the new /// parent only forgets the node; to it, the node is gone. The row stays either /// way (`docs/BUGS.md` B117). fn relocate_in_mount( st: &mut State, notify: &mut NotifyBatch, uid: &NodeUid, new_parent: &NodeUid, name: &str, queued: bool, ) -> Option<(String, bool)> { let ino = *st.by_uid.get(uid)?; let entry = st.entries.get(&ino)?; let (from_parent, from_name) = (entry.parent, entry.node.name.clone()); match st.by_uid.get(new_parent).copied() { Some(to_parent) => { if queued { // Listing again would ask Drive, which still has the old name; // the kernel's pages of both folders go instead. st.rename_in_place(ino, to_parent, new_parent, name); notify.inval_inode(from_parent); if to_parent != from_parent { notify.inval_inode(to_parent); } } else { st.relocate(ino, from_parent, to_parent, new_parent, name); } notify.inval_entry(from_parent, from_name.clone()); notify.inval_entry(to_parent, name.to_string()); Some((from_name, true)) } None => { st.forget_mem(uid); notify.delete(from_parent, ino, from_name.clone()); Some((from_name, false)) } } } /// One kernel notification, recorded while the State lock is held so it can be /// sent once it is not. /// /// `inval_inode` calls into `invalidate_inode_pages2`, which can block waiting /// on pages that this daemon's own workers have to fill — and those workers take /// the State lock. Sending from inside the lock is that deadlock's shape; a /// batch is the smallest way to keep the inode/mount pairing while moving the /// send outside it. enum KernelNotice { InvalInode(u64), InvalEntry { parent: u64, name: String, }, Delete { parent: u64, child: u64, name: String, }, } /// Notifications accumulated under the State lock. Flushed by /// [`Core::for_each_mount`] to every session once the lock is released. #[derive(Default)] struct NotifyBatch(Vec); impl NotifyBatch { fn inval_inode(&mut self, ino: u64) { self.0.push(KernelNotice::InvalInode(ino)); } fn inval_entry(&mut self, parent: u64, name: String) { self.0.push(KernelNotice::InvalEntry { parent, name }); } fn delete(&mut self, parent: u64, child: u64, name: String) { self.0.push(KernelNotice::Delete { parent, child, name, }); } fn extend_inodes(&mut self, inodes: &[u64]) { self.0 .extend(inodes.iter().map(|&ino| KernelNotice::InvalInode(ino))); } /// Send everything recorded to one session. A mount whose session has not /// been spawned has nothing to tell the kernel. A session that never looked /// a node up answers `ENOENT`, which is ignored like any other refusal. fn flush(&self, notifier: Option<&KernelChannel>) { let Some(notifier) = notifier else { return }; for notice in &self.0 { let _ = match notice { KernelNotice::InvalInode(ino) => notifier.inval_inode(*ino), KernelNotice::InvalEntry { parent, name } => { notifier.inval_entry(*parent, OsStr::new(name)) } KernelNotice::Delete { parent, child, name, } => notifier.delete(*parent, *child, OsStr::new(name)), }; } } } /// One FUSE session's notification channel. The tree has one inode space for /// every mount, and each session knows its own root as [`ROOT_INO`], so this /// names the root that way and every other inode as it is. pub(crate) struct KernelChannel { notifier: Notifier, /// The tree inode of the session's root. root: u64, } impl KernelChannel { fn new(notifier: Notifier, root: u64) -> Self { Self { notifier, root } } pub(crate) fn inval_inode(&self, ino: u64) -> std::io::Result<()> { let ino = filesystem::kernel_ino(self.root, ino); off_the_runtime(|| self.notifier.inval_inode(ino, 0, 0)) } /// Drop the cached attributes of `ino` but not its page cache. fn inval_attrs(&self, ino: u64) -> std::io::Result<()> { let ino = filesystem::kernel_ino(self.root, ino); off_the_runtime(|| self.notifier.inval_inode(ino, -1, 0)) } pub(crate) fn inval_entry(&self, parent: u64, name: &OsStr) -> std::io::Result<()> { let parent = filesystem::kernel_ino(self.root, parent); off_the_runtime(|| self.notifier.inval_entry(parent, name)) } fn delete(&self, parent: u64, child: u64, name: &OsStr) -> std::io::Result<()> { let parent = filesystem::kernel_ino(self.root, parent); let child = filesystem::kernel_ino(self.root, child); off_the_runtime(|| self.notifier.delete(parent, child, name)) } } /// Send a notice to the kernel without holding up the runtime. /// /// The kernel answers an invalidation only once it has the pages' locks, and /// a read holds those until the daemon answers it. Sent from a runtime worker, /// the notice blocked that worker in the kernel; when it was the one driving /// the timers and I/O, nothing drove them any more, so the read's fetch never /// finished and the daemon hung (`docs/BUGS.md` B130). On a worker the notice /// hands the worker's tasks to another thread first; anywhere else it is sent /// as it is. fn off_the_runtime(notice: impl FnOnce() -> T) -> T { match tokio::runtime::Handle::try_current() { Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => { tokio::task::block_in_place(notice) } _ => notice(), } } fn notify_access_changes(notifier: Option<&KernelChannel>, changed: &[u64]) { let Some(notifier) = notifier else { return }; for &ino in changed { let _ = notifier.inval_inode(ino); } } fn require_rename_access( require: impl Fn(&NodeUid) -> Result<(), Errno>, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_uid: &NodeUid, ) -> Result<(), Errno> { for authority in [uid, old_parent_uid, new_parent_uid] { require(authority)?; } Ok(()) } fn require_node_parent_access( require: impl Fn(&NodeUid) -> Result<(), Errno>, uid: &NodeUid, parent_uid: &NodeUid, ) -> Result<(), Errno> { require(uid)?; require(parent_uid) } fn preserve_on_access_denied( access: Result<(), Errno>, accepted_bytes: bool, preserve: impl FnOnce(), ) -> Result<(), Errno> { if let Err(error) = access { if error.code() == libc::EACCES && accepted_bytes { preserve(); } return Err(error); } Ok(()) } /// Fold the staged blob of an undrained write into the blob at `src` that is /// about to supersede it, so the newer write inherits the older one's bytes /// instead of losing them. /// /// Only the ranges the older write authored and the newer one did not are /// copied, so a small edit over a small edit moves almost nothing. What neither /// wrote is untouched remote content, which is why `base_size`, `base_mtime` and /// the baseline all come from the write being superseded: that is the revision /// those ranges were last observed against, and the one the drain must gap-fill /// and conflict-check them against. fn merge_over_pending( meta: &mut StagedWrite, src: &Path, previous: &PendingRevision, ) -> std::io::Result<()> { let mut written = Intervals::default(); for &(s, e) in &meta.authored { written.add(s, e); } let mut earlier = Intervals::default(); for &(s, e) in &previous.meta.authored { earlier.add(s, e); } let blob = File::open(&previous.path)?; let dst = std::fs::OpenOptions::new().write(true).open(src)?; for (s, e, authored) in written.clone().segments(0, meta.len) { if authored { continue; } for (ps, pe, have) in earlier.segments(s, e.min(previous.meta.len)) { if !have { continue; } let mut buf = vec![0u8; (pe - ps) as usize]; blob.read_exact_at(&mut buf, ps)?; dst.write_all_at(&buf, ps)?; written.add(ps, pe); } } dst.sync_all()?; meta.base_size = previous.meta.base_size; meta.base_mtime = previous.meta.base_mtime; meta.based_on = previous.meta.based_on.clone(); meta.authored = written .segments(0, meta.len) .into_iter() .filter(|&(_, _, authored)| authored) .map(|(s, e, _)| (s, e)) .collect(); meta.complete = StagedWrite::covers(&meta.authored, meta.len); Ok(()) } /// Retire the last resident reference to an unlinked inode without touching its /// persisted node row. The namespace path already chose whether that row is /// disposable or retained as queued-trash authority. fn release_unlinked_entry(state: &mut State, ino: u64) -> Option { let entry = state.entries.get_mut(&ino)?; entry.open_count = entry.open_count.saturating_sub(1); if entry.open_count != 0 || !entry.unlinked { return None; } let uid = entry.uid.clone(); state.forget_mem(&uid); Some(uid) } fn release_must_retain_queued_trash(db: &Db, uid: &NodeUid) -> pdfs_core::Result { db.has_pending_op(&uid.to_string(), OP_TRASH) } /// The size of a staged blob that has outlived [`STAGING_ORPHAN_RETAIN`], or /// `None` while it is younger than that — or while its age cannot be read at /// all, which is never grounds for deleting bytes. fn orphan_past_retention(blob: &Path) -> Option { let meta = std::fs::metadata(blob).ok()?; let age = meta.modified().ok()?.elapsed().ok()?; (age >= STAGING_ORPHAN_RETAIN).then_some(meta.len()) } fn release_can_discard_unlinked(db: &Db, uid: &NodeUid) -> bool { match release_must_retain_queued_trash(db, uid) { Ok(retain) => !retain, Err(error) => { warn!(%uid, %error, "queued-trash lookup failed; retaining unlinked state"); false } } } struct AcceptedShares { uids: Vec, /// `None` means duplicate listing rows disagreed on provenance. share_ids: HashMap>, } fn accepted_share_provenance(items: Vec) -> AcceptedShares { let mut uids = Vec::new(); let mut share_ids = HashMap::new(); for item in items { match share_ids.entry(item.uid.clone()) { std::collections::hash_map::Entry::Vacant(entry) => { uids.push(item.uid); entry.insert(Some(item.share_id)); } std::collections::hash_map::Entry::Occupied(mut entry) => { if entry.get().as_ref() != Some(&item.share_id) { entry.insert(None); } } } } AcceptedShares { uids, share_ids } } fn shared_root_access(nodes: &[&Node], expected_share: Option<&ShareId>) -> Access { let Some(expected_share) = expected_share else { return Access::Viewer; }; let mut observed = None; for node in nodes { let Some(membership) = node.membership.as_ref() else { return Access::Viewer; }; if &membership.share_id != expected_share { return Access::Viewer; } let Some(role) = membership.role_exact() else { return Access::Viewer; }; let access = access_for(Some(role), Access::Viewer, true); if observed.is_some_and(|prior| prior != access) { return Access::Viewer; } observed = Some(access); } observed.unwrap_or(Access::Viewer) } fn prepare_shared_roots( accepted: &AcceptedShares, mut materialized: Vec, parent: &NodeUid, ) -> Vec { materialized.retain(|node| accepted.share_ids.contains_key(&node.uid)); materialized.sort_by(|a, b| { a.uid .to_string() .cmp(&b.uid.to_string()) .then_with(|| a.name.cmp(&b.name)) }); let mut published = Vec::new(); let mut start = 0usize; while start < materialized.len() { let uid = materialized[start].uid.clone(); let mut end = start + 1; while end < materialized.len() && materialized[end].uid == uid { end += 1; } let access = shared_root_access( &materialized[start..end].iter().collect::>(), accepted.share_ids.get(&uid).and_then(Option::as_ref), ); let mut node = materialized[start].clone(); node.parent_uid = Some(parent.clone()); node.trashed = false; published.push(PublishedSharedRoot { node, access }); start = end; } let mut named: Vec = published.iter().map(|root| root.node.clone()).collect(); disambiguate_shared_names(&mut named); let names: HashMap = named .into_iter() .map(|node| (node.uid, node.name)) .collect(); for root in &mut published { if let Some(name) = names.get(&root.node.uid) { root.node.name.clone_from(name); } } published } impl Core { /// Rename `uid` on Drive, riding out an "out of date" answer from a stale /// name hash. See [`retry_out_of_date`]. pub(crate) fn rename_remote(&self, uid: &NodeUid, name: &str) -> Result<(), ProtonError> { self.change_remote(uid, || { self.block_on_bounded(self.drive.rename_node(uid, name, None)) }) } /// Move `uid` into the folder `parent` on Drive, like [`Core::rename_remote`]. pub(crate) fn move_remote(&self, uid: &NodeUid, parent: &NodeUid) -> Result<(), ProtonError> { self.change_remote(uid, || { self.block_on_bounded(self.drive.move_node(uid, parent)) }) } /// Move `uid` into the folder `parent` on Drive, renaming it to /// `target_name` on the way when one is given, in one request. /// /// Done as a rename and then a move, a sibling in the source folder that /// held the new name made Drive refuse the first half, although the /// destination was free (docs/BUGS.md B122). One request also leaves no /// half-applied state between the two. pub(crate) fn move_rename_remote( &self, uid: &NodeUid, parent: &NodeUid, target_name: Option<&str>, ) -> Result<(), ProtonError> { self.change_remote(uid, || { let item = NodeMoveItem { uid: uid.clone(), target_name: target_name.map(str::to_string), }; self.block_on_bounded(async { let mut outcomes = std::pin::pin!(self.drive.move_nodes_streaming(vec![item], parent.clone())); match outcomes.next().await { Some(Ok((_, outcome))) => outcome, Some(Err(e)) => Err(e), None => Err(ProtonError::invalid_operation(format!( "move of {uid} reported no outcome" ))), } }) }) } /// Run a rename or move of `uid`, then drop the SDK's now stale copy of it. fn change_remote( &self, uid: &NodeUid, call: impl FnMut() -> Result<(), ProtonError>, ) -> Result<(), ProtonError> { retry_out_of_date(call, || { self.forget_sdk_node(uid); std::thread::sleep(OUT_OF_DATE_RETRY_DELAY); }) .inspect(|()| self.changed_remote(uid)) } /// Bookkeeping after a rename or move of `uid` landed: the SDK's copy of it /// is stale, and the feed will echo the change back. Without the second, /// that echo reads as someone else's edit and evicts the cached bytes a /// partial write is about to gap-fill from (docs/BUGS.md B123). pub(crate) fn changed_remote(&self, uid: &NodeUid) { self.forget_sdk_node(uid); self.note_self_change(uid); } /// Drop the SDK's cached copy of `uid` after this daemon renamed or moved it. /// /// The SDK sends a node's name hash from its entity cache, and nothing but /// the event feed's echo of the change refreshes that entry. Until the next /// poll, up to [`POLL_INTERVAL`] later, each rename or move of the node would /// send the old hash and Drive would refuse it as out of date (docs/BUGS.md /// B121). Feeding the SDK the echo now makes the next call read the link. pub(crate) fn forget_sdk_node(&self, uid: &NodeUid) { let changed = DriveEvent::NodeUpdated { id: DriveEventId::from("pdfs-local-change"), node_uid: uid.clone(), parent_node_uid: None, is_trashed: false, is_shared: false, }; if let Err(error) = self.block_on_bounded(self.drive.invalidate_caches_for_event(&changed)) { debug!(%uid, %error, "could not drop the SDK's cached node"); } } /// Record that this daemon just changed `uid` on the remote, so the event /// feed's echo of that change is recognised instead of re-applied. Called /// from the drain once the remote has actually accepted the change and the /// tree has been brought level with it. See [`Core::self_changes`]. pub(crate) fn note_self_change(&self, uid: &NodeUid) { note_self_change(&mut self.self_changes.lock(), uid, now_millis()); } /// Claim one outstanding echo for `uid`, reporting whether this event is one /// this daemon caused. Consuming rather than merely testing is what bounds /// the suppression: the next event for the node applies normally. fn take_self_change(&self, uid: &NodeUid) -> bool { take_self_change(&mut self.self_changes.lock(), uid, now_millis()) } /// Rehydrate the in-memory `State` maps from the DB on mount, so a cold /// start serves previously-seen metadata (stable inodes, instant listings) /// without re-hitting the API. The root inode is already installed by /// [`ProtonFs::new`]; this fills in every other persisted node and rebuilds /// the child listings of folders the DB records as fully enumerated. fn hydrate(&self) { let stored = match self.db.load_all() { Ok(s) => s, Err(e) => { warn!(error = %e, "db load_all failed; mounting cold"); return; } }; if stored.is_empty() { return; } // A write queued by a previous run carries the optimistic size the file // grew to, which the sealed remote revision does not yet reflect. The DB // node still holds the pre-write `claimed_size` (often 0 for a fresh // file), so without this a queued 500 MB file `stat`s as 0 bytes until // the drain lands — reads still serve from the staged blob, but `ls` // lying about the size reads as data loss. `hydrate_pending` ran before // us, so the pending map is populated; snapshot it here rather than hold // both locks at once. let pending_sizes: HashMap = { let pending = self.pending.lock(); pending .iter() .map(|(uid, pr)| (uid.clone(), pr.meta.len)) .collect() }; // `hydrate_pending` runs first and restores queued-trash tombstones. // Their node rows remain persisted as drain-time access authority, but // must not reappear in the mounted namespace after a restart. let hidden = self.hidden.lock().clone(); let mut st = self.state(); // Pass 1: give every uid its inode from its local id (the root is // already mapped). for sn in &stored { if hidden.contains(&sn.node.uid) || st.by_uid.contains_key(&sn.node.uid) { continue; } st.by_uid .insert(sn.node.uid.clone(), state::lid_ino(sn.lid)); } // Pass 2: materialize entries, resolving each parent's inode by uid. // Track folders flagged complete so their listings rebuild in pass 3. let mut listed_dirs: Vec = Vec::new(); for sn in stored { let StoredNode { mut node, listed, .. } = sn; if hidden.contains(&node.uid) { continue; } let Some(&ino) = st.by_uid.get(&node.uid) else { continue; }; // Re-apply a queued write's optimistic size so `stat` matches what // reads (served from the staged blob) already return. if let Some(&len) = pending_sizes.get(&node.uid) && let NodeKind::File { claimed_size, .. } = &mut node.kind { *claimed_size = Some(len as i64); } if listed && node.is_folder() { listed_dirs.push(ino); } // A root entry is owned by `ProtonFs::new`; don't overwrite it. if st.roots.contains(&ino) { continue; } // A node whose parent row never made it to disk must not be adopted // by the root: it would surface as a phantom top-level entry. let parent = node .parent_uid .as_ref() .and_then(|p| st.by_uid.get(p).copied()) .unwrap_or(ORPHAN_INO); let uid = node.uid.clone(); st.entries.insert( ino, Entry { uid, parent, node, access: pdfs_core::Access::Unknown, lookup_count: 1, open_count: 0, unlinked: false, }, ); } st.hydrate_access(); // Pass 3: rebuild child listings for fully-enumerated folders. let listings = listed_children(&st.entries, listed_dirs); st.children.extend(listings); info!(nodes = st.entries.len(), "hydrated metadata cache from db"); } /// Rebuild the in-memory pending map from the `pending_op` table on mount. /// /// A queued write survives a restart — that is the point of persisting it — /// so until the drain worker gets to it, reads of that file must still come /// from its staged blob rather than the remote's older revision. /// /// A row whose blob has gone missing is dropped: there is nothing left to /// upload, and keeping it would fail forever. fn hydrate_pending(&self) { let ops = match self.db.pending_ops() { Ok(ops) => ops, Err(e) => { error!(error = %e, "loading pending uploads failed"); return; } }; let mut map = self.pending.lock(); let mut restored = 0usize; for op in ops { let Some(uid) = parse_node_uid(&op.uid) else { error!(uid = %op.uid, id = op.id, "pending op has an unparseable uid; dropping"); let _ = self.db.delete_op(op.id); continue; }; if op.kind == OP_TRASH { self.hidden.lock().insert(uid.clone()); } restored += 1; // Only a revision must have a blob. A create carries none until // something is written to it (`touch` offline is a legitimate op // with nothing to serve), and a rename never has one. A trash keeps // the blob of a create it withdrew only to find the node by, not // to serve. All still have to be replayed, so only the blob — if // any — is checked. if op.kind == OP_TRASH || (op.blob_path.is_none() && op.kind != OP_REVISION) { continue; } let parsed = op .meta_json .as_deref() .and_then(|j| serde_json::from_str::(j).ok()) .zip(op.blob_path.as_deref().map(PathBuf::from)); let Some((meta, path)) = parsed else { error!(uid = %op.uid, id = op.id, "pending op is unreadable; dropping"); self.drop_unrecoverable_op(&op, &uid); restored -= 1; continue; }; if !path.exists() { error!(%uid, path = %path.display(), "staged blob is gone; dropping pending op"); self.drop_unrecoverable_op(&op, &uid); restored -= 1; continue; } map.insert(uid, PendingRevision { path, meta }); } if restored > 0 { info!(count = restored, "restored pending ops"); } } /// Re-attach staged blobs that no queued op refers to. /// /// `staging/` holds the only copy of writes the kernel has already been /// told succeeded. Three release-path failures put bytes there and return /// `EIO` without leaving a `pending_op` row behind — an access change /// mid-release, a write overlapping an undrained incomplete edit, a create /// that drained out from under its own handle. Until now nothing ever /// walked the directory again: `hydrate_pending` reads the op table and /// [`recover_fsynced_writes`](Self::recover_fsynced_writes) reads /// `recovery/`, so an orphaned staged blob was invisible to the daemon and /// to the user alike, and stayed that way until someone found the file. /// /// Runs once at mount, after `hydrate_pending`, so `pending` already /// describes what the queue owns. An orphan whose sidecar names a real node /// is queued as an ordinary revision — the condition that made it fail is /// gone by definition, since the op it collided with has since drained. /// Anything that cannot be addressed (no sidecar, an unparseable uid, a /// `local~` placeholder whose create is gone) is left in place and named in /// the log: unexplained bytes are never evidence that user data is /// disposable. fn reconcile_staging(&self) { let staged = self.cache.staged_writes(); if staged.is_empty() { return; } let claimed = match self.db.op_blob_paths() { Ok(paths) => paths, Err(e) => { error!(error = %e, "cannot read queued blob paths; skipping staging reconcile"); return; } }; let mut queued = 0usize; let mut stranded = 0usize; let mut retired = 0usize; let mut retired_bytes = 0u64; let mut example: Option = None; for (blob, meta) in staged { if claimed.contains(&blob.to_string_lossy().into_owned()) { continue; } let addressable = meta.as_ref().and_then(|m| parse_node_uid(&m.uid)); let Some((meta, uid)) = meta.zip(addressable).filter(|(_, uid)| !is_local_uid(uid)) else { if let Some(bytes) = orphan_past_retention(&blob) { retired += 1; retired_bytes += bytes; warn!(blob = %blob.display(), bytes, age_days = STAGING_ORPHAN_RETAIN.as_secs() / 86_400, "retiring staged bytes that name no node and have outlived retention"); self.cache.discard_staged(&blob); continue; } stranded += 1; example.get_or_insert_with(|| blob.clone()); continue; }; if self.pending.lock().contains_key(&uid) { // A newer write for the same node is queued and carries its own // blob. These bytes are older; keeping them is right, uploading // them over the newer ones is not. stranded += 1; warn!(%uid, blob = %blob.display(), "staged bytes are superseded by a queued write; kept for recovery"); continue; } if self .db .has_pending_op(&uid.to_string(), OP_TRASH) .unwrap_or(false) { // The node was deleted while a handle still read these bytes, // and the daemon stopped before its trash went out. Queued again, // they lost their blob when the trash landed and retried on it // for good (`docs/BUGS.md` B188). info!(%uid, blob = %blob.display(), "staged bytes belong to a node whose trash is queued; discarded"); self.cache.discard_staged(&blob); continue; } let op = PendingOp { id: 0, kind: OP_REVISION.to_string(), uid: uid.to_string(), parent_uid: None, name: None, blob_path: Some(blob.to_string_lossy().into_owned()), meta_json: Some(serde_json::to_string(&meta).unwrap_or_default()), created_at: now_millis(), attempts: 0, last_error: None, next_attempt_at: now_millis(), }; match self.db.enqueue_op(&op) { Ok((_, superseded)) => { if let Some(old) = superseded { self.cache.discard_staged(Path::new(&old)); } self.pending.lock().insert( uid.clone(), PendingRevision { path: blob.clone(), meta, }, ); queued += 1; } Err(e) => { stranded += 1; error!(%uid, blob = %blob.display(), error = %e, "cannot queue an orphaned staged write; bytes kept"); } } } if stranded > 0 { // One line, not one per blob. A tree of them says exactly as much as // a single count and an example does, and said it on every mount: // 86 identical warnings for the same 86 files, boot after boot. warn!( count = stranded, example = %example.map(|b| b.display().to_string()).unwrap_or_default(), "staged bytes belong to no queued upload and name no node; kept for recovery" ); } if queued > 0 || stranded > 0 || retired > 0 { info!( queued, stranded, retired, retired_bytes, "reconciled orphaned staged writes" ); } } /// Queue the writes that an unclean shutdown caught between `fsync(2)` and /// `close(2)`. /// /// `fsync` promises the bytes survive a crash, but the queueing that makes a /// write outlive the daemon happens at `release`. A crash in between used to /// lose the data outright, because the scratch directory is cleared at open. /// Now `fsync` leaves a sidecar, `ContentCache::open` moves those blobs to /// `recovery/`, and this walks them into the same staging + queued-op path a /// normal release takes. /// /// Runs after [`hydrate_pending`](Self::hydrate_pending), which is what makes /// the incomplete-blob check in [`enqueue_staged_write`](Self::enqueue_staged_write) /// meaningful: a recovered partial write whose earlier write is still queued /// must not gap-fill from a remote revision that no longer describes the /// file. With `pending` already loaded, that case is detected and the bytes /// are parked rather than mis-filled. /// /// Failure is per-write and never fatal: a node that no longer exists, or an /// op that cannot be queued, leaves its blob in `recovery/` for the next run /// (and for a human) instead of taking the mount down. fn recover_fsynced_writes(&self) { let recovered = self.cache.recovered_writes(); if recovered.is_empty() { return; } let mut queued = 0usize; for (blob, meta) in recovered { let Some(uid) = parse_node_uid(&meta.uid) else { error!(uid = %meta.uid, "recovered write has an unparseable uid; keeping bytes"); continue; }; // If the write is incomplete and an earlier edit is still queued, merging it // now would use the wrong remote base. enqueue_staged_write would abandon it // to staging; keep it in recovery instead so the drain loop can queue it // once the queue clears. if !meta.complete && self.pending.lock().contains_key(&uid) { continue; } // The inode is only used to stamp the in-memory tree with the new // size, and nothing is interned this early — `hydrate` reads the // size back off `pending` when the node is first looked up, so 0 // (no such inode) is correct rather than merely tolerable. match self.enqueue_staged_write(&uid, 0, &blob, meta) { Ok(()) => { self.cache.discard_recovered(&blob); queued += 1; } Err(e) => { error!(%uid, blob = %blob.display(), error = ?e, "cannot queue a recovered write; bytes kept for the next run"); // Some failures (a partial write parked by // `stage_orphaned_write`) still consume the blob. Its sidecar // would then describe nothing, so retire it — the bytes are // in staging, which is where a human looks for them. if !blob.exists() { self.cache.discard_recovered(&blob); } } } } if queued > 0 { info!( count = queued, "recovered fsynced writes from an unclean shutdown" ); } } /// Discard an op that can never be performed, because the bytes it was to /// upload are gone from staging (something outside the daemon deleted them). /// /// For a node that only ever existed locally, the placeholder goes too. Its /// content is unrecoverable and nothing will ever mint it a real uid, so /// leaving the row would strand a file in the tree that can be listed but /// never read and never uploaded. fn drop_unrecoverable_op(&self, op: &PendingOp, uid: &NodeUid) { let _ = self.db.delete_op(op.id); if is_local_uid(uid) { error!(%uid, name = op.name.as_deref().unwrap_or("?"), "discarding a node whose only copy was lost"); if let Err(e) = self.db.delete_node(uid) { warn!(%uid, error = %e, "db delete_node failed for lost local node"); } } } /// Whether [`Core::ensure_children`] would answer for `ino` without /// touching the network. Lets a handler decide between answering inline /// and handing off to a worker. /// /// A resident listing is not enough on its own. A shared folder's, or the /// Shared with me root's, is fetched again once it is a minute old, and /// that fetch ran on the dispatch loop, so every request on the mount /// waited for Drive to answer (`docs/BUGS.md` B219). fn children_cached(&self, ino: u64) -> bool { let (uid, resident) = { let st = self.state(); match st.entries.get(&ino) { Some(e) => (e.uid.clone(), st.children.contains_key(&ino)), // `list_children` answers ENOENT without the network. None => return true, } }; if !resident { return false; } let freshness = if is_virtual_uid(&uid) { Some(SHARED_WITH_ME_SYNCED_MS.to_string()) } else if !self.is_own_or_virtual(&uid) && !is_local_uid(&uid) { Some(Self::shared_folder_freshness_key(&uid)) } else { None }; freshness.is_none_or(|key| { !listing_needs_refresh( self.online.load(Ordering::Relaxed), self.shared_listing_stale(&key), ) }) } /// Re-apply the optimistic size of any queued write to `nodes`. /// /// A node that arrives from the remote (or from its DB row) carries the size /// of the revision the *server* holds, which for a file with a write still /// queued is the pre-write size — often 0 for a file created moments ago. /// Interning it as-is silently reverts the optimistic size that /// `record_pending_write` stamped, and a file that stats as 0 bytes is a file /// the kernel will not issue a single `read` for: `cat` prints nothing and /// the staged blob that `read_range` would have served is never asked for. /// That reads as data loss even though nothing is lost (B11). /// /// [`Core::hydrate`] does the same thing for the restart case. This covers /// every *live* re-enumeration — which is what a rename or move triggers, /// since both invalidate the listings they touch. /// /// Snapshots the pending map and returns before any caller takes the state /// lock: no site in the daemon holds `pending` and `state` at once, and this /// is not the place to become the first. fn stamp_pending_sizes(&self, nodes: &mut [Node]) { let sizes: HashMap = { let pending = self.pending.lock(); if pending.is_empty() { return; } pending .iter() .map(|(uid, pr)| (uid.clone(), pr.meta.len)) .collect() }; apply_pending_sizes(nodes, &sizes); } /// Read the inputs for virtual-root publication without changing SQLite. /// /// The caller validates the listing snapshot after these reads. Keeping all /// writes out of this phase prevents an invalidated/repopulated root listing /// from changing the synthetic node's persisted visibility or FTS ancestry. fn prepare_virtual_root(&self, real_names: &HashSet) -> Result { let pinned = self.db.state_str(SHARED_WITH_ME_NAME).map_err(|error| { error!(%error, "reading shared-root display name failed"); Errno::EIO })?; let (name, visible) = virtual_root_name(real_names, pinned.as_deref()); let uid = shared_with_me_uid(); let previous_node = self.db.node_by_uid(&uid.to_string()).map_err(|error| { error!(%error, "reading synthetic shared root failed"); Errno::EIO })?; let mut node = previous_node.unwrap_or_else(|| { virtual_node(self.primary_root_uid.clone(), name.clone(), now_secs()) }); node.parent_uid = Some(self.primary_root_uid.clone()); node.name = name; node.kind = NodeKind::Folder; // A later real-name collision suppresses both the dentry and its search // hit. The row and pinned name remain, and clearing the collision // restores both by writing `trashed = false` again. node.trashed = !visible; node.membership = None; Ok(VirtualRootPlan { node, visible }) } /// Reconcile the synthetic dentry against an already-known primary-root /// listing. The node remains persisted and pinned even while a real folder /// of the same name temporarily suppresses its dentry. fn reconcile_virtual_root_dentry( &self, parent_ino: u64, snapshot: RootListingSnapshot, ) -> Result<(), Errno> { let _publication = self.shared_publication.lock(); let plan = self.prepare_virtual_root(&snapshot.real_names())?; let mut st = self.state(); // State methods already establish the state -> DB lock order. No path // holds the DB connection while acquiring state, including shared event // publication, so keeping state locked across this one transaction // closes the crash window without introducing an inverse order. publish_virtual_root_in_listing(&self.db, &mut st, parent_ino, &snapshot, plan)?; drop(st); self.flush_access_changes(); Ok(()) } fn resident_root_listing_snapshot(&self, ino: u64) -> Result { RootListingSnapshot::capture(&self.state(), ino).ok_or(Errno::EAGAIN) } fn shared_folder_freshness_key(uid: &NodeUid) -> String { format!("{SHARED_FOLDER_SYNCED_PREFIX}{uid}") } fn shared_listing_stale(&self, key: &str) -> bool { if self .shared_refresh_deadlines .lock() .is_fresh(key, Instant::now()) { return false; } self.listing_stale(key, SHARED_LISTING_TTL) } fn mark_shared_refresh_success(&self, key: &str) { self.shared_refresh_deadlines .lock() .mark(key, Instant::now(), SHARED_LISTING_TTL); if let Err(error) = self.db.set_state_i64(key, now_millis()) { warn!(key, %error, "persisting shared-list freshness failed"); } } fn invalidate_shared_refreshes(&self) { self.shared_generation.fetch_add(1, Ordering::SeqCst); self.shared_refresh_deadlines.lock().clear(); } fn is_own_or_virtual(&self, uid: &NodeUid) -> bool { is_own_or_virtual_uid(uid, &self.primary_root_uid.volume_id) } fn ensure_shared_children(&self, ino: u64) -> Result<(), Errno> { let online = self.online.load(Ordering::Relaxed); let resident = self.state().children.contains_key(&ino); match shared_listing_plan( resident, online, self.shared_listing_stale(SHARED_WITH_ME_SYNCED_MS), ) { SharedListingPlan::Resident => return Ok(()), SharedListingPlan::Persisted => { // A completed snapshot remains useful offline even after an // event expired its freshness stamp. let nodes = self .db .visible_children(&shared_with_me_uid()) .map_err(|error| { error!(%error, "loading persisted shared roots failed"); Errno::EIO })?; let mut st = self.state(); if st.children.contains_key(&ino) { return Ok(()); } let children = nodes .into_iter() .map(|node| { let access = st .share_access .get(&node.uid) .copied() .unwrap_or(Access::Viewer); st.intern_published_share_root(ino, node, access) }) .collect(); st.children.insert(ino, children); drop(st); self.flush_access_changes(); return Ok(()); } SharedListingPlan::Refresh => {} } let generation = self.shared_generation.load(Ordering::SeqCst); let accepted = accepted_share_provenance( self.block_on_within(BATCH_CALL_TIMEOUT, self.client.enumerate_shared_with_me()) .map_err(|error| { error!(%error, "enumerating shared roots failed"); Errno::EIO })?, ); let nodes = if accepted.uids.is_empty() { Vec::new() } else { self.block_on_within( BATCH_CALL_TIMEOUT, self.drive.enumerate_nodes_light(&accepted.uids), ) .map_err(|error| { error!(%error, "materializing shared roots failed"); Errno::EIO })? }; let virtual_uid = shared_with_me_uid(); let published = prepare_shared_roots(&accepted, nodes, &virtual_uid); let _publication = self.shared_publication.lock(); if !refresh_generation_is_current(generation, self.shared_generation.load(Ordering::SeqCst)) { return Err(Errno::EAGAIN); } let removed = match self .db .publish_shared_roots(&virtual_uid, &accepted.uids, &published) { Ok(removed) => removed, Err(error) => { error!(%error, "publishing shared-root listing failed"); let mut st = self.state(); for uid in &accepted.uids { st.downgrade_shared_subtree(uid); } drop(st); self.flush_access_changes(); return Err(Errno::EIO); } }; let snapshot = self.db.visible_children(&virtual_uid).map_err(|error| { error!(%error, "loading published shared-root listing failed"); Errno::EIO })?; let published_access: HashMap = published .iter() .map(|root| (root.node.uid.clone(), root.access)) .collect(); let mut st = self.state(); for uid in &removed { st.hide_shared_root(uid); } let mut children = Vec::with_capacity(snapshot.len()); for node in snapshot { let uid = node.uid.clone(); let access = published_access .get(&uid) .copied() .unwrap_or(Access::Viewer); let child = st.intern_published_share_root(ino, node, access); children.push(child); } st.children.insert(ino, children); drop(st); self.flush_access_changes(); self.mark_shared_refresh_success(SHARED_WITH_ME_SYNCED_MS); if let Some(notifier) = self.notifier.get() { let _ = notifier.inval_inode(ino); } Ok(()) } /// The nodes of `before`, queued in `folder_uid` when its listing was asked /// of Drive, that are no longer queued in `after`, as the DB has them now. /// /// Their create landed while the listing was on the wire, after Drive took /// it, and so is in neither. One unlinked or moved away meanwhile is not /// under the folder by that name any more and stays out. fn landed_meanwhile( &self, folder_uid: &NodeUid, before: Vec, after: &[Node], ) -> Vec { let still: HashSet<&str> = after.iter().map(|node| node.name.as_str()).collect(); let landed: HashSet = before .into_iter() .map(|node| node.name) .filter(|name| !still.contains(name.as_str())) .collect(); if landed.is_empty() { return Vec::new(); } match self.db.known_children(folder_uid) { Ok(known) => known .into_iter() .filter(|node| landed.contains(&node.name)) .collect(), Err(e) => { warn!(%folder_uid, error = %e, "db known_children failed"); Vec::new() } } } /// Serve a listing without the network: the one already in memory if any, /// else whatever children the DB still knows. The fallback only answers /// until the link is back, when [`Core::relist_offline_listings`] drops it: /// a listing dropped as stale may lack a child another device made, and a /// create over that name lands as a conflict copy rather than an `EIO` now. fn adopt_stale_listing( &self, ino: u64, folder_uid: &NodeUid, cached: bool, primary_root: bool, ) -> Result<(), Errno> { if cached { return Ok(()); } let nodes = self.db.known_children(folder_uid).map_err(|e| { warn!(%folder_uid, error = %e, "db known_children failed"); Errno::EIO })?; debug!(%folder_uid, count = nodes.len(), "offline: listing a folder from what the DB knows"); self.link.stale_listings.lock().insert(ino); self.adopt_db_listing(ino, folder_uid, nodes, primary_root) } /// Relist every folder [`Core::adopt_stale_listing`] served while offline. pub(crate) fn relist_offline_listings(&self) { let stale = std::mem::take(&mut *self.link.stale_listings.lock()); if stale.is_empty() { return; } let mut st = self.state(); for &ino in &stale { st.invalidate_listing(ino); } drop(st); for mount in self.states.live() { if let Some(notifier) = mount.notifier.get() { for &ino in &stale { let _ = notifier.inval_inode(ino); } } } } /// Install `nodes`, read from the DB, as `ino`'s listing. fn adopt_db_listing( &self, ino: u64, folder_uid: &NodeUid, mut nodes: Vec, primary_root: bool, ) -> Result<(), Errno> { let folder_uid = folder_uid.clone(); if primary_root { nodes.retain(|node| !is_virtual_uid(&node.uid)); } // Before the lock: a DB row carries the size the server last // sealed, which a queued write is ahead of (B11). self.stamp_pending_sizes(&mut nodes); let hidden = self.hidden.lock().clone(); let mut st = self.state(); if st.children.contains_key(&ino) { return Ok(()); } let mut child_inos = Vec::with_capacity(nodes.len()); let mut needs_size = Vec::new(); for node in nodes { if !node_visible(&node, &folder_uid, &hidden) { continue; } if matches!( &node.kind, NodeKind::File { claimed_size: None, .. } ) { needs_size.push(node.uid.clone()); } child_inos.push(st.intern_from_db(ino, node)); } st.children.insert(ino, child_inos); let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap()); drop(st); self.flush_access_changes(); if let Some(snapshot) = root_snapshot { self.reconcile_virtual_root_dentry(ino, snapshot)?; } // Rows persisted from a cheap enumeration whose upgrade never // ran (a restart in between, say) still owe their real sizes. self.spawn_size_upgrade(ino, needs_size); Ok(()) } /// Enumerate `ino`'s children from the remote and cache them. No-op if the /// directory has already been listed. Network I/O happens without the lock /// held so concurrent metadata reads aren't blocked behind a fetch. /// /// A listing that changed while it was being published answers EAGAIN: the /// root listed again after a restart, an event, a newer refresh of a shared /// folder. The listing is still owed, so it is taken again rather than /// handed to the lookup or open that asked (B159). fn ensure_children(&self, ino: u64) -> Result<(), Errno> { for _ in 1..LISTING_ATTEMPTS { match self.list_children(ino) { Err(e) if e.code() == Errno::EAGAIN.code() => { debug!(ino, "listing changed while it was published; listing again"); } done => return done, } } self.list_children(ino) } fn list_children(&self, ino: u64) -> Result<(), Errno> { let (folder_uid, cached) = { let st = self.state(); match st.entries.get(&ino) { Some(e) => (e.uid.clone(), st.children.contains_key(&ino)), None => return Err(Errno::ENOENT), } }; if is_virtual_uid(&folder_uid) { return self.ensure_shared_children(ino); } let primary_root = is_primary_root_listing(self.primary, &folder_uid, &self.primary_root_uid); let foreign = !self.is_own_or_virtual(&folder_uid) && !is_local_uid(&folder_uid); let foreign_key = foreign.then(|| Self::shared_folder_freshness_key(&folder_uid)); let refresh_foreign = foreign_key.as_deref().is_some_and(|key| { listing_needs_refresh( self.online.load(Ordering::Relaxed), self.shared_listing_stale(key), ) }); if cached && !refresh_foreign { if primary_root { self.reconcile_virtual_root_dentry(ino, self.resident_root_listing_snapshot(ino)?)?; } return Ok(()); } // Offline fast path: a folder the DB still records as fully enumerated // can be rebuilt from disk without hitting the API, even if its listing // was trimmed from the hot cache mid-run. let cached_nodes = if refresh_foreign { Ok(None) } else { self.db.children_if_listed(&folder_uid) }; match cached_nodes { Ok(Some(nodes)) => return self.adopt_db_listing(ino, &folder_uid, nodes, primary_root), Ok(None) => {} Err(e) => warn!(%folder_uid, error = %e, "db children_if_listed failed"), } // Drive knows nothing by a stand-in. Until its mkdir lands the folder // is only here, and once it has, the DB holds what Drive does. A // listing dropped meanwhile was asked of Drive, and every lookup in // the folder failed with EIO (`docs/BUGS.md` B125). if is_local_uid(&folder_uid) { let nodes = self.db.known_children(&folder_uid).map_err(|e| { warn!(%folder_uid, error = %e, "db known_children failed"); Errno::EIO })?; return self.adopt_db_listing(ino, &folder_uid, nodes, primary_root); } if !self.is_online() { return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root); } let refresh_generation = refresh_foreign.then(|| self.shared_generation.load(Ordering::SeqCst)); // What is queued before Drive is asked: a create that lands while the // listing is on the wire is in neither Drive's answer nor the queue read // after it (`docs/BUGS.md` B166). let queued_before = (!foreign) .then(|| self.db.queued_children(&folder_uid).ok()) .flatten(); let uids = match self .rt .block_on(self.drive.enumerate_folder_children_node_uids(&folder_uid)) { Ok(uids) => uids, Err(e) if self.lost_link(&e, "list folder") => { return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root); } Err(e) => { error!(%folder_uid, error = %e, "enumerate folder children failed"); return Err(Errno::EIO); } }; // Cheap enumeration: `Light` skips unlocking each *file's* node key, // which is an S2K derivation per file and was ~74% of the cost of a cold // listing (B12 — measured with `perf`, 64% of cycles in SHA-256 alone). // Folders are unlocked either way; their keys are what the children are // decrypted with, so the walk cannot proceed without them. // // The price is that files come back without a `claimed_size`, so // `node_size` falls back to the *ciphertext* size until // `spawn_size_upgrade` below fills the real one in. let mut nodes = match self.rt.block_on(self.drive.enumerate_nodes_light(&uids)) { Ok(nodes) => nodes, Err(e) if self.lost_link(&e, "list folder") => { return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root); } Err(e) => { error!(%folder_uid, error = %e, "enumerate nodes failed"); return Err(Errno::EIO); } }; // Drive shows none of what is still queued for the folder (B132). // A shared folder's listing is published as Drive has it. if !foreign { match self.db.queued_children(&folder_uid) { Ok((mut queued, gone)) => { if let Some((before, _)) = queued_before { queued.extend(self.landed_meanwhile(&folder_uid, before, &queued)); } overlay_queued(&mut nodes, queued, &gone) } Err(e) => warn!(%folder_uid, error = %e, "db queued_children failed"), } } // Same as the DB path above: the remote's size for a file with a write // still queued is the pre-write one (B11). self.stamp_pending_sizes(&mut nodes); let hidden = self.hidden.lock().clone(); let mut filtered_nodes: Vec = nodes .into_iter() .filter(|node| node_visible(node, &folder_uid, &hidden)) .collect(); if foreign { for node in &mut filtered_nodes { node.parent_uid = Some(folder_uid.clone()); } let _publication = self.shared_publication.lock(); if !refresh_generation_is_current( refresh_generation.expect("foreign refresh has a generation"), self.shared_generation.load(Ordering::SeqCst), ) { return Err(Errno::EAGAIN); } let removed = self .db .publish_foreign_children(&folder_uid, &uids, &filtered_nodes) .map_err(|error| { error!(%folder_uid, %error, "publishing foreign-folder listing failed"); Errno::EIO })?; let mut snapshot = self.db.visible_children(&folder_uid).map_err(|error| { error!(%folder_uid, %error, "loading published foreign-folder listing failed"); Errno::EIO })?; self.stamp_pending_sizes(&mut snapshot); snapshot.retain(|node| node_visible(node, &folder_uid, &hidden)); let needs_size: Vec = snapshot .iter() .filter(|node| { matches!( &node.kind, NodeKind::File { claimed_size: None, .. } ) }) .map(|node| node.uid.clone()) .collect(); let mut st = self.state(); for uid in &removed { st.hide_foreign_subtree(uid); } let children = snapshot .into_iter() .map(|node| st.intern_from_db(ino, node)) .collect(); st.children.insert(ino, children); drop(st); self.flush_access_changes(); self.mark_shared_refresh_success( foreign_key .as_deref() .expect("foreign refresh has a freshness key"), ); self.spawn_size_upgrade(ino, needs_size); return Ok(()); } let preassigned = state::inos_for(&self.db, &filtered_nodes); let mut st = self.state(); // Lost the race? Another thread already populated it. if st.children.contains_key(&ino) { return Ok(()); } let mut child_inos = Vec::with_capacity(filtered_nodes.len()); // Files whose real size the cheap enumeration could not read. Collected // before interning so the upgrade below has the uids without re-walking. let needs_size: Vec = filtered_nodes .iter() .filter(|n| { matches!( &n.kind, NodeKind::File { claimed_size: None, .. } ) }) .map(|n| n.uid.clone()) .collect(); let inos = st.intern_batch_with_inos(ino, filtered_nodes, preassigned); child_inos.extend(inos); st.children.insert(ino, child_inos); let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap()); // Record the listing as complete so a later restart (or a trimmed hot // cache) can rebuild it from the DB without the API. Written with the // children's rows when the lock goes, not under it. st.record_listed(folder_uid.clone()); drop(st); self.flush_access_changes(); if let Some(snapshot) = root_snapshot { self.reconcile_virtual_root_dentry(ino, snapshot)?; } self.spawn_size_upgrade(ino, needs_size); Ok(()) } /// Resolve the real size of every file in `parent`'s listing still carrying /// a provisional one, returning when they are in `state`. /// /// Called when a `stat` lands on such a file. That covers the paths /// [`Core::ensure_children`] cannot: a listing rebuilt from the DB, or one /// restored by [`Core::hydrate`] on mount, whose rows were persisted before /// an earlier upgrade had a chance to run. Gathers the whole folder rather /// than the one file asked for, because a `stat` of one entry in a listing /// almost always means a `stat` of all of them. fn upgrade_sizes_for_parent( &self, ino: u64, uid: &NodeUid, parent: u64, ) -> Option> { let provisional = |e: &Entry| { matches!( &e.node.kind, NodeKind::File { claimed_size: None, .. } ) }; let (key, mut missing): (u64, Vec) = { let st = self.state(); match st.children.get(&parent) { // The listing is resident: batch the whole folder under its // inode, so the rest of an `ls -l` rides along on this fetch. Some(kids) => ( parent, kids.iter() .filter_map(|k| st.entries.get(k)) .filter(|e| provisional(e)) .map(|e| e.uid.clone()) .collect(), ), // It is not, and returning here is what let a provisional size // reach the caller anyway: a rename invalidates its parents' // listings, so a freshly renamed file always landed in this // branch. Resolve the single node instead, keyed by its own // inode — folder and file inodes share one space, so the two // single-flight keys cannot collide. (Same shape as B4: an // early return that assumed the hot cache was authoritative.) None => (ino, Vec::new()), } }; // The node that was actually asked about is never optional, whichever // branch produced the batch. if !missing.iter().any(|u| u == uid) { missing.push(uid.clone()); } // Returned, not awaited here: the caller is a `getattr` that must not // answer with a provisional size (`docs/BUGS.md` B14), and it parks its reply // on this batch rather than holding a thread until it lands. self.upgrade_sizes(key, missing) } /// Fill in the true sizes of files a `Light` enumeration returned without /// one, on a worker, after the listing has already been served. /// /// This is the other half of the split in [`Core::ensure_children`]. The /// listing itself needs only names and parentage, so it is served from the /// cheap enumeration immediately; the S2K-per-file work that produces /// `claimed_size` happens here, off the path the user is waiting on. /// /// **Sizes are provisional until this lands.** `node_size` falls back to /// `total_size_on_storage`, the *ciphertext* size, which is slightly larger /// than the real one. A read must not plan on it: since B84 a size the /// revision disagrees with fails the read, so `read` settles the size first /// ([`Core::resolve_provisional_size`], `docs/BUGS.md` B100). In `stat` it is a /// discrepancy that closes within a round trip, not a repeat of B11 (which /// reported **0** and so suppressed reads entirely). /// /// Single-flight per folder: a `stat` of every entry in a fresh listing is /// the normal case, and each one must not start its own upgrade. fn spawn_size_upgrade(&self, folder_ino: u64, uids: Vec) { self.upgrade_sizes(folder_ino, uids); } /// Start (or join) the size upgrade for `key`, returning the batch a caller /// can wait on — `None` when there was nothing to upgrade. /// /// Single-flight per `key`. The fetch runs on its own thread rather than on /// the [`Workers`] pool: callers wait on `Lane::Meta`, so a batch queued onto /// that lane could have a wide enough `ls -l` fill it with threads waiting /// for a job that can never be scheduled. `Lane::Transfer` would swap that /// for starvation behind bulk reads. One short-lived thread per folder, /// bounded by the single-flight, avoids both. fn upgrade_sizes(&self, key: u64, uids: Vec) -> Option> { if uids.is_empty() { return None; } let claim = claim_size_upgrade(&mut self.size_upgrades.lock(), key, uids); match claim { SizeUpgradeClaim::Joined(slot) | SizeUpgradeClaim::Queued(slot) => Some(slot), SizeUpgradeClaim::Start(slot, uids) => { let core = self.clone(); let worker = slot.clone(); std::thread::spawn(move || core.run_size_upgrade(key, uids, worker)); Some(slot) } } } /// Hold `respond` back until `ino` has a real size, `slot`'s batch ends, or /// [`SizeUpgrade::WAIT`] elapses — then run it, on the queue's thread. /// /// The caller returns immediately. That is the whole point: the thread it /// was running on is a FUSE dispatch thread or a [`Workers`] one, and both /// are needed by everything else on the mount (audit F4). fn await_size( &self, ino: u64, slot: Arc, respond: impl FnOnce() + Send + 'static, ) { let core = self.clone(); let waiter = SizeWaiter { slot, resolved: Box::new(move || core.size_is_real(ino)), deadline: Instant::now() + SizeUpgrade::WAIT, respond: Box::new(respond), }; if !self.size_waiters.park(waiter) { return; } let queue = self.size_waiters.clone(); if let Err(error) = std::thread::Builder::new() .name("pdfs-size-waiters".into()) .spawn(move || queue.serve()) { // Without a thread nothing would ever answer these. A provisional // size is wrong; a `stat` that never returns wedges the caller. warn!(%error, "no thread for parked size waiters; answering provisionally"); self.size_waiters.answer_all(); } } /// Fetch `uids` in chunks, applying and announcing each one as it lands. /// /// Chunked so a waiter is released as soon as *its* file is resolved. A /// single 793-node batch took ~80 s, which outran [`SizeUpgrade::WAIT`] and /// put provisional sizes back in front of callers — the bug this was /// supposed to fix (`docs/BUGS.md` B14). /// /// When the batch ends, the thread takes over the oldest batch the cap held /// back, so at most [`MAX_SIZE_UPGRADES`] threads run however many folders /// a recursive walk touches (`docs/BUGS.md` B100). fn run_size_upgrade(&self, key: u64, uids: Vec, slot: Arc) { let (mut key, mut uids, mut slot) = (key, uids, slot); loop { for chunk in uids.chunks(SIZE_UPGRADE_CHUNK) { let result = self.block_on_within(BATCH_CALL_TIMEOUT, self.drive.enumerate_nodes(chunk)); self.apply_size_upgrade(key, result); slot.chunk_done(); } // Both of these must happen however the loop ended: a folder whose // upgrade failed has to be retryable, and its waiters released. let next = next_size_upgrade(&mut self.size_upgrades.lock(), key); slot.finish(); let Some(next) = next else { return; }; (key, uids, slot) = next; } } /// Settle the size of a file whose listing only carries the provisional /// one, before a read plans its blocks on it or a write handle starts from /// it. Returns the real size, or `None` when the fetch did not produce one. /// /// A provisional size is the ciphertext size. The read path checks the /// size it is given against the revision's own block table and fails the /// read on a mismatch (B84), so reading on the provisional size is an `EIO` /// every time. `getattr` usually settles the size first, but not when its /// wait times out, so the read cannot rely on it (`docs/BUGS.md` B100). fn resolve_provisional_size(&self, ino: u64, uid: &NodeUid) -> Option { let parent = self.state().entries.get(&ino)?.parent; // Without a network the answer is `None` either way; the bound keeps a // dead link from holding the open or read for the SDK's two minutes. if !self.is_online() { return None; } let result = self.block_on_bounded(self.drive.enumerate_nodes(std::slice::from_ref(uid))); if let Err(e) = &result { self.lost_link(e, "settle a provisional size"); } self.apply_size_upgrade(parent, result); let st = self.state(); match &st.entries.get(&ino)?.node.kind { NodeKind::File { claimed_size: Some(size), .. } => Some((*size).max(0) as u64), _ => None, } } /// Wait, up to [`SizeUpgrade::WAIT`] as a `stat` would, for the real size /// of every file in `ino`'s listing that still carries a provisional one. /// /// For callers that read a whole listing's sizes straight from `state` and /// do not run on a FUSE thread: `pdfs ls` and the app's file browser. A /// listing just fetched has the ciphertext size of every file in it until /// its upgrade lands, and they showed those (`docs/BUGS.md` B115). fn settle_listing_sizes(&self, ino: u64) { // Offline the fetch cannot succeed; the wait would only delay the // answer it ends up giving anyway. if !self.is_online() { return; } let missing: Vec = { let st = self.state(); st.children .get(&ino) .map(|kids| { kids.iter() .filter_map(|k| st.entries.get(k)) .filter(|e| { !is_local_uid(&e.uid) && matches!( &e.node.kind, NodeKind::File { claimed_size: None, .. } ) }) .map(|e| e.uid.clone()) .collect() }) .unwrap_or_default() }; if let Some(slot) = self.upgrade_sizes(ino, missing) { slot.wait_finished(Instant::now() + SizeUpgrade::WAIT); } } /// Whether `ino` has a real size — the condition a waiter is waiting on. /// A node that vanished counts as resolved; there is nothing left to wait /// for, and its caller will find the `ENOENT` for itself. fn size_is_real(&self, ino: u64) -> bool { let st = self.state(); st.entries.get(&ino).is_none_or(|e| { !matches!( &e.node.kind, NodeKind::File { claimed_size: None, .. } ) }) } /// Adopt the sizes a completed upgrade fetched. Split from /// [`Core::upgrade_sizes`] so the single-flight bookkeeping there has one /// exit path rather than one per early return. fn apply_size_upgrade(&self, folder_ino: u64, result: Result, ProtonError>) { let core = self; { let mut nodes = match result { Ok(nodes) => nodes, Err(e) => { // Not fatal: the listing stands, sizes just stay provisional // until something invalidates and re-enumerates it. warn!(folder_ino, error = %e, "size upgrade failed; sizes stay provisional"); return; } }; // A queued write is ahead of anything the server can report, so its // optimistic size must survive this just as it survives a re-listing // (B11). core.stamp_pending_sizes(&mut nodes); let mut changed: Vec = Vec::new(); let mut st = core.state(); for node in nodes { // Only adopt the size. Re-interning wholesale would also adopt a // name or parent that a rename/move may have changed locally // while this was in flight, undoing it. let Some(&ino) = st.by_uid.get(&node.uid) else { continue; }; changed.push(ino); // An open write handle is ahead of the server too (B109). if st.active_writes.contains_key(&ino) { continue; } let NodeKind::File { claimed_size, .. } = &node.kind else { continue; }; let (Some(size), Some(entry)) = (*claimed_size, st.entries.get_mut(&ino)) else { continue; }; if let NodeKind::File { claimed_size, .. } = &mut entry.node.kind { *claimed_size = Some(size); } } let updated: Vec = st .children .get(&folder_ino) .map(|kids| { kids.iter() .filter_map(|k| st.entries.get(k).map(|e| e.node.clone())) .collect() }) .unwrap_or_default(); drop(st); if let Err(e) = core.db.upsert_nodes(&updated) { warn!(folder_ino, error = %e, "persisting upgraded sizes failed"); } // Without this the corrected size is invisible for the length of the // attr TTL: the kernel answers `stat` from the provisional attrs it // cached while the listing was being served, so `ls -l` reports the // ciphertext size for up to 30 s even though the daemon has had the // real one all along. Notify *after* the DB write, so a re-`getattr` // provoked by the invalidation cannot race the persistence. // // Attributes only (offset -1): the content did not change, and a // page-cache invalidation deadlocks when this runs inside a `read` // (B100 resolves sizes there). The kernel holds the page lock of // that read until we reply, and the invalidation waits for the // same lock, so the read never answered (docs/BUGS.md B110). for mount in core.states.live() { if let Some(notifier) = mount.notifier.get() { for &ino in &changed { let _ = notifier.inval_attrs(ino); } } } debug!(folder_ino, files = updated.len(), "filled in listing sizes"); } } /// Resolve a child `name` within `parent` to its `(inode, uid)`, ensuring /// the parent's listing is cached first. fn lookup_child(&self, parent: u64, name: &str) -> Result<(u64, NodeUid), Errno> { self.ensure_children(parent)?; let st = self.state(); st.children .get(&parent) .and_then(|kids| { kids.iter().copied().find_map(|ino| { st.entries .get(&ino) .filter(|e| e.node.name == name) .map(|e| (ino, e.uid.clone())) }) }) .ok_or(Errno::ENOENT) } /// Walk a mountpoint-relative path to its `(inode, uid)`, enumerating each /// directory on the way as needed. Leading `/` and `.` components are /// ignored; `..` is rejected. fn resolve_path(&self, rel: &Path) -> Result<(u64, NodeUid), Errno> { let mut ino = self.root_ino(); let mut uid = { let st = self.state(); st.entries .get(&ino) .map(|e| e.uid.clone()) .ok_or(Errno::ENOENT)? }; for comp in rel.components() { match comp { Component::RootDir | Component::CurDir => continue, Component::Normal(name) => { let (child_ino, child_uid) = self.lookup_child(ino, &name.to_string_lossy())?; ino = child_ino; uid = child_uid; } _ => return Err(Errno::EINVAL), } } Ok((ino, uid)) } /// [`resolve_path`](Self::resolve_path) for the request-serving side. /// /// The kernel-facing resolver answers in `Errno` because that is what the /// FUSE reply needs. A front-end needs the opposite: `{:?}` of a libc /// constant is not something to show a user, and "does not exist" and "the /// API is down" want different copy and different buttons. This is the one /// place that knows enough to tell them apart, so it is the place that does. fn resolve(&self, rel: &Path) -> CoreResult<(u64, NodeUid)> { self.resolve_path(rel) .map_err(|e| self.errno_error(e, &format!("could not resolve {}", rel.display()))) } /// Resolve a wire-format uid only when one of this daemon's locations /// proves that it owns the node. /// /// The primary and on-demand mounts prove residency through the tree and /// the roots in [`StateRegistry`]. Mirror locations have no tree, so their root /// and last-synced descendants are resolved through `sync_folder` and /// `sync_entry` instead. pub(crate) fn resolve_anywhere(&self, uid: &str) -> CoreResult { resolve_anywhere_with( uid, |uid| self.owns_visible_uid(uid), |uid| self.db.mirror_contains_uid(&uid.to_string()), ) } /// [`Core::resolve_anywhere`], widened to the Photos timeline for sharing. /// /// A photo lives on the photos volume, which no location shows, so the /// residency proof above never finds it. The timeline in the `photos` table /// is this daemon's proof instead: it lists only photos on the user's own /// volume, which are theirs to share. A photo in an album someone else /// shared is not in it and stays refused. pub(crate) fn resolve_shareable(&self, uid: &str) -> CoreResult { resolve_anywhere_with( uid, |uid| self.owns_visible_uid(uid), |uid| { let uid = uid.to_string(); Ok(self.db.mirror_contains_uid(&uid)? || !self .db .photos_by_uid(std::slice::from_ref(&uid))? .is_empty()) }, ) } fn source_parent_uid(&self, ino: u64, rel: &Path) -> CoreResult { let state = self.state(); let entry = state .entries .get(&ino) .ok_or_else(|| CoreError::not_found(format!("could not resolve {}", rel.display())))?; entry .node .parent_uid .clone() .or_else(|| { state .entries .get(&entry.parent) .map(|parent| parent.uid.clone()) }) .ok_or_else(|| CoreError::invalid("the mount root cannot be mutated")) } /// Classify a failure that arrived as an `Errno`. /// /// The internal paths speak `Errno` because they also serve FUSE, where a /// number is the whole vocabulary. Everything crossing the control socket /// has to be turned back into something a person can read, and this is the /// one place that knows how — a call site holding only an `Errno` has /// already lost the context needed to say what went wrong. /// /// `Errno` is neither `PartialEq` nor structural-match, so this compares /// raw codes rather than the `libc` constants. fn errno_error(&self, e: Errno, what: &str) -> CoreError { match e.code() { libc::ENOENT => CoreError::not_found(format!("{what}: no such file or folder")), libc::EACCES | libc::EPERM => CoreError::denied(format!("{what}: not allowed")), // These paths walk the tree lazily, so a cold node needs the API. // Offline that surfaces as EIO, which on its own would read to the // user as a broken file rather than a missing network. libc::EIO if !self.online.load(Ordering::Relaxed) => CoreError::offline(), libc::EINVAL => CoreError::invalid(format!("{what}: not a usable path")), libc::ENOSPC => CoreError::new(ErrorKind::Quota, format!("{what}: out of space")), libc::EEXIST => CoreError::conflict(format!("{what}: already exists")), libc::ENOTEMPTY => CoreError::conflict(format!("{what}: folder is not empty")), _ => CoreError::internal(format!("{what}: {e:?}")), } } /// Fetch a single node's current metadata from the remote. fn fetch_node(&self, uid: &NodeUid) -> Result { match self.fetch_node_remote(uid) { Ok(Some(node)) => Ok(node), Ok(None) => Err(Errno::ENOENT), Err(e) => { if !self.lost_link(&e, "fetch node") { error!(%uid, error = %e, "enumerate node failed"); } Err(Errno::EIO) } } } /// Read back a node just created on the remote. /// /// Drive can answer a create before a lookup lists the new node, so an /// empty answer is asked again for a moment. `None` means the node exists /// but could not be read back. The mount then queues the create, and the /// drain finds the node by name and adopts it: the application gets its /// file or folder either way, never an `ENOENT` for something that exists. fn fetch_minted_node(&self, uid: &NodeUid, name: &str) -> Option { // The feed echoes the create. Unclaimed, it would take the echo the // upload of the file's bytes records, and that one would then evict // the bytes just uploaded (docs/BUGS.md B123). self.note_self_change(uid); let node = self.read_back(uid).ok(); if node.is_none() { warn!(%uid, name, "Drive made the node but did not hand it back"); } node } /// [`Core::fetch_node`] for a node Drive has just made, asked again for a /// moment while Drive does not list it yet. /// /// A file Drive lists without an active revision is not listed yet either: /// the create commits one before it answers, so this answer is from before /// the commit, with no size. Taken, it emptied the new file in the tree, /// and a read of it right after found nothing (`docs/BUGS.md` B163). pub(crate) fn read_back(&self, uid: &NodeUid) -> Result { let mut delays = MINTED_READ_BACK_DELAYS.iter(); loop { let fetched = match self.fetch_node(uid) { Ok(node) if is_unrevised(&node) => Err(Errno::ENOENT), fetched => fetched, }; match fetched { Err(e) if e == Errno::ENOENT && let Some(delay) = delays.next() => { std::thread::sleep(*delay); } fetched => return fetched, } } } /// [`Core::fetch_node`] without the collapse to an `Errno`, for the drain: /// resolving a conflict turns on *why* a call failed, and "the node is not /// there" (`Ok(None)`) is a different outcome from "we could not ask". fn fetch_node_remote(&self, uid: &NodeUid) -> Result, ProtonError> { match self.block_on_bounded(self.drive.enumerate_nodes(std::slice::from_ref(uid))) { Ok(nodes) => Ok(nodes.into_iter().next()), // An unknown uid is reported either as an empty result or as an // outright refusal, depending on the endpoint. Err(e) if is_gone(&e) => Ok(None), Err(e) => Err(e), } } /// Point a handle closed while its file's create landed at the uid Drive /// gave the node. /// /// `Core::retire_create` moves the row to that uid before it repoints the /// open handles. A handle closed in between still named the placeholder, /// which no row has any more, and its write was refused as if access had /// changed. The inode is the row's local id, which finds it either way. pub(crate) fn follow_landed_create(&self, h: &mut WriteHandle) { if !is_local_uid(&h.uid) { return; } if let Some(real) = self.landed_row_uid(h.ino) { debug!(local = %h.uid, %real, "write handle closed as its create landed"); h.uid = real; } } /// The uid Drive gave the node at `ino`, once its create has landed, or /// `None` while it is still queued. Read from the row, which has it before /// the tree does. pub(crate) fn landed_row_uid(&self, ino: u64) -> Option { state::ino_lid(ino).and_then(|lid| self.landed_lid_uid(lid)) } /// [`Core::landed_row_uid`] for a node known by its stand-in `uid`. pub(crate) fn landed_stand_in_uid(&self, uid: &NodeUid) -> Option { pdfs_core::db::local_lid(&uid.to_string()).and_then(|lid| self.landed_lid_uid(lid)) } fn landed_lid_uid(&self, lid: i64) -> Option { self.db .uid_of_lid(lid) .ok() .flatten() .and_then(|uid| parse_node_uid(&uid)) .filter(|uid| !is_local_uid(uid)) } /// Queue what the write handles the kernel never released hold. /// /// `close(2)` returns before the kernel sends `release`, and the stop /// aborts the FUSE connection with the releases still queued: a file a /// program had just saved was never queued, and Drive kept it as it was /// (`docs/BUGS.md` B190). Runs once every session has ended, so no release /// can race it. A handle still open in a program goes the same way, with /// what it wrote so far; one on an unlinked file only leaves its scratch. pub(crate) fn queue_unreleased_writes(&self) { let handles: Vec<(WriteHandle, bool)> = { let mut st = self.state(); st.handles.clear(); let inos: Vec = st.active_writes.keys().copied().collect(); inos.into_iter() .filter_map(|ino| { let unlinked = st.entries.get(&ino).is_some_and(|e| e.unlinked); st.active_writes.remove(&ino).map(|h| (h, unlinked)) }) .collect() }; for (mut h, unlinked) in handles { if unlinked { self.cache.clear_scratch_durable(&h.path); let _ = std::fs::remove_file(&h.path); continue; } self.follow_landed_create(&mut h); match self.queue_revision(&mut h) { Ok(()) => info!(uid = %h.uid, "queued a write the kernel never released"), Err(error) => { warn!(uid = %h.uid, ?error, "cannot queue a write the kernel never released") } } } } /// Accept a released write handle's bytes and queue their upload. /// /// This is what makes a copy into the mount run at disk speed: the caller's /// `close` returns once the bytes are staged on local disk and the intent is /// in `pending_op`, instead of waiting out a full upload inside the FUSE /// handler. It is also what makes an offline write succeed rather than EIO — /// the queued op simply waits for the network. /// /// The scratch file is *moved* into staging, never copied: it is the only /// copy of what the user wrote. fn queue_revision(&self, h: &mut WriteHandle) -> Result<(), Errno> { // Closing an untouched writable handle is not a mutation. It remains // valid after a downgrade and only owes cleanup of its empty scratch. if !h.dirty { self.cache.clear_scratch_durable(&h.path); let _ = std::fs::remove_file(&h.path); // A new file's create waits for this release to wake the drain // (`Core::queue_local_node`). Closed empty, it attached nothing // that would, and sat until the idle poll (`docs/BUGS.md` B192). if is_local_uid(&h.uid) { self.wake_drain(); } return Ok(()); } preserve_on_access_denied(self.require_uid_writable(&h.uid), h.dirty, || { let meta = self.recovery_meta(h); self.stage_orphaned_write( &h.uid, h.ino, &h.path, &meta, "access changed before the write could be queued", ); })?; // The handle is being retired either way, so any durability sidecar an // `fsync` left has done its job: from here the bytes are tracked as a // staged write and a queued op, and a sidecar outliving them would offer // recovery a second, stale copy of the same write. self.cache.clear_scratch_durable(&h.path); if is_local_uid(&h.uid) && !self.db.has_create_op(&h.uid.to_string()).unwrap_or(true) { // A create that landed since the handle last looked left no op // either. It moved the row to its uid in the same transaction, so // the row says which it was (`docs/BUGS.md` B176). self.follow_landed_create(h); if is_local_uid(&h.uid) { debug!(uid = %h.uid, "local node was unlinked before creation; dropping revision"); let _ = std::fs::remove_file(&h.path); return Ok(()); } } // Materialize as much of the full content as is free to materialize. A // complete blob is uploadable without the network and lets a later write // to the same file supersede this one without any reconstruction at all. // // What is deliberately *not* done here is fetching the untouched ranges // from the remote: that is a download of everything the edit did not // touch, and this runs on the FUSE dispatch loop, so closing a large // file after a small edit stopped every other request on the mount for // the length of it (audit F1). An incomplete blob is first-class — // `read_pending` resolves its gaps against the base, and `drain_revision` // fills them on the drain thread — so the network half of the fill // belongs there, and only there. // // A node that exists only locally has no remote base to fill from: its // untouched ranges live in the blob queued by an earlier write, which // `merge_over_pending` folds in below. if let Err(e) = h.file.set_len(h.len) { error!(uid = %h.uid, error = %e, "resize scratch file failed"); return Err(Errno::EIO); } let mut written = h.written.clone(); if !(is_local_uid(&h.uid) && h.base_size > 0) { // A local node has no remote base, so nothing of it can be fetched // or cached: passing zero says so, and leaves the fill doing only // the part that still applies — claiming the zeroed tail. let base_size = if is_local_uid(&h.uid) { 0 } else { h.base_size }; self.fill_gaps_cached( &h.uid, &h.file, h.len, h.base_mtime, base_size, &mut written, ); } let authored: Vec<(u64, u64)> = written .segments(0, h.len) .into_iter() .filter(|&(_, _, authored)| authored) .map(|(s, e, _)| (s, e)) .collect(); let meta = StagedWrite { uid: h.uid.to_string(), len: h.len, base_size: h.base_size, base_mtime: h.base_mtime, complete: StagedWrite::covers(&authored, h.len), authored, based_on: self.remote_baseline( &h.uid, h.base_mtime, h.base_size, h.base_revision_id.clone(), ), }; let complete = meta.complete; self.enqueue_staged_write(&h.uid, h.ino, &h.path, meta)?; debug!(uid = %h.uid, len = h.len, complete, "queued revision upload"); Ok(()) } /// Describe the locally authored ranges without gap-filling them. /// /// Used when a write was accepted by the kernel but a queue-time access /// check now denies it. Building this metadata is side-effect free, so the /// permission guard still precedes cache, database, and tree mutation. fn recovery_meta(&self, h: &WriteHandle) -> StagedWrite { let authored: Vec<(u64, u64)> = h .written .segments(0, h.len) .into_iter() .filter(|&(_, _, authored)| authored) .map(|(start, end, _)| (start, end)) .collect(); StagedWrite { uid: h.uid.to_string(), len: h.len, base_size: h.base_size, base_mtime: h.base_mtime, complete: StagedWrite::covers(&authored, h.len), authored, based_on: self.remote_baseline( &h.uid, h.base_mtime, h.base_size, h.base_revision_id.clone(), ), } } /// The remote revision a change to `uid` is being made against, for /// [`StagedWrite::based_on`]. /// /// Normally that is simply the base the handle opened over. The exception is /// a write that supersedes a still-queued one: its "base" is the previous /// *staged blob*, whose size and mtime are ours, not the server's — so the /// baseline is inherited from the op being superseded, which is the last one /// that actually observed the remote. Without that, chaining two writes /// before the queue drains would leave the drain comparing the remote /// against a revision it never had, and cutting a conflict copy over /// nothing. /// /// `None` for a node that has never existed remotely: there is no revision /// to conflict with until its create drains. fn remote_baseline( &self, uid: &NodeUid, base_mtime: i64, base_size: u64, base_revision_id: Option, ) -> Option { if is_local_uid(uid) { return None; } match self.pending.lock().get(uid) { Some(p) => p.meta.based_on.clone(), None => Some(Baseline { mtime: base_mtime, size: base_size, hash: None, revision_id: base_revision_id, }), } } /// Move a file holding a node's intended new content into staging and queue /// the upload that will make it the remote's content. Shared by the release /// of a write handle and by a path-based truncate. /// /// `src` is consumed either way: on success it is *moved* into staging, and /// on the refusal below it is moved there too, just without an op to upload /// it. It is the only copy of what the user wrote, so nothing here may /// simply delete it. fn enqueue_staged_write( &self, uid: &NodeUid, ino: u64, src: &Path, mut meta: StagedWrite, ) -> Result<(), Errno> { preserve_on_access_denied(self.require_uid_writable(uid), true, || { self.stage_orphaned_write( uid, ino, src, &meta, "access changed before the staged write could be queued", ); })?; // An incomplete blob's gaps refer to the *remote* base. If an earlier // write to this file is still queued, the remote no longer holds that // base — the previous staged blob does — so taking those gaps at face // value would fill them from the wrong revision. Fold the blob being // superseded into this one instead: its authored bytes are a local copy // of exactly the ranges this write did not touch, and what neither wrote // is still the remote content that the earlier write was based on, which // is why the baseline is inherited along with them. let previous = self.pending.lock().get(uid).cloned(); if !meta.complete && let Some(previous) = previous && let Err(error) = merge_over_pending(&mut meta, src, &previous) { // Without the earlier bytes the gaps cannot be resolved against // anything, so this is where the write stops being uploadable. // Keep it recoverable rather than corrupting the file. warn!(%uid, %error, "folding in the undrained edit failed"); self.stage_orphaned_write( uid, ino, src, &meta, "write overlaps an undrained incomplete edit", ); return Err(Errno::EIO); } let path = self.cache.stage_write(&meta, src).map_err(|e| { error!(%uid, error = %e, "staging write failed"); // The scratch file is still the only copy of these bytes, and the // caller is releasing the handle it belongs to — without a // durability sidecar `discard_unmarked_scratch` deletes it at the // next mount, the one place the "never delete on a failure path" // invariant would break. Mark it first, whatever went wrong. if let Err(mark) = self.cache.mark_scratch_durable(src, &meta) { error!(%uid, error = %mark, "could not mark an unstageable write durable; bytes at risk"); } if e.is_disk_full() { // Free cached blobs so the retry has somewhere to land, and tell // the caller what is actually wrong. self.cache.emergency_evict(); Errno::ENOSPC } else { Errno::EIO } })?; // Whatever this write supersedes, it supersedes now — including an // upload already on the wire, whose bytes describe a revision the row // written just below replaces. Signalled before the enqueue so the // in-flight reader stops at its next block rather than after however // much of the file is left. self.cancel_upload(uid); let mut uid = uid.clone(); let len = meta.len; let superseded = 'queued: { if is_local_uid(&uid) { // The node has no server-side identity to hang a revision on, so // the bytes ride on the create that will mint it. Attached and // recorded under one hold of `pending`, which is what the drain // takes to retire a landed create: it either sees this blob and // carries it over, or has already finished and this write // queues against the real uid below (`Core::retire_create`). let meta_json = serde_json::to_string(&meta).unwrap_or_default(); let mut pending = self.pending.lock(); let attached = self .db .attach_blob_to_create(&uid.to_string(), &path.to_string_lossy(), &meta_json) .map_err(|e| { error!(%uid, error = %e, "attaching write to queued create failed"); Errno::EIO })?; if let Some(attached) = attached { pending.insert(uid.clone(), PendingRevision { path, meta }); break 'queued attached.superseded; } drop(pending); // The create landed while this handle was open, and its row // already has the uid Drive gave it. let Some(real) = self.landed_row_uid(ino) else { error!(%uid, staged = %path.display(), "queued create vanished under a write; bytes kept in staging"); return Err(Errno::EIO); }; debug!(local = %uid, %real, "create landed under an open write; queueing a revision"); uid = real; meta.uid = uid.to_string(); let previous = self.pending.lock().get(&uid).cloned(); meta.based_on = previous.as_ref().and_then(|p| p.meta.based_on.clone()); if !meta.complete && let Some(previous) = previous && let Err(error) = merge_over_pending(&mut meta, &path, &previous) { error!(%uid, %error, staged = %path.display(), "folding in the write the create carried failed; bytes kept in staging"); return Err(Errno::EIO); } } let op = PendingOp { id: 0, kind: OP_REVISION.to_string(), uid: uid.to_string(), parent_uid: None, name: None, blob_path: Some(path.to_string_lossy().into_owned()), meta_json: Some(serde_json::to_string(&meta).unwrap_or_default()), created_at: now_millis(), attempts: 0, last_error: None, next_attempt_at: now_millis() + self.revision_debounce(&uid).as_millis() as i64, }; let (_id, superseded) = self.db.enqueue_op(&op).map_err(|e| { error!(%uid, error = %e, "queueing upload failed"); Errno::EIO })?; self.pending .lock() .insert(uid.clone(), PendingRevision { path, meta }); superseded }; if let Some(old) = superseded { self.cache.discard_staged(Path::new(&old)); } let uid = &uid; // Reflect the write in the tree straight away: `ls` must show the new // size and mtime even though the remote still holds the old revision. let now = now_secs(); let updated = { let mut st = self.state(); st.record_pending_write(ino, len, now) }; // The only record that the file is as long as the caller was told. If it // cannot be written the write did not fully take, and saying so beats // acknowledging a `close(2)` and then serving the old size — the bytes // themselves are already staged and queued either way. Written out here // rather than inside `State` so the commit does not hold the inode lock. let recorded = match updated { Some(node) => self.db.upsert_node(&node), None => Ok(()), }; // Cached blobs and open readers describe the superseded revision. Reads // come from the staged blob until the op drains, so just drop them. // Done even when the row above failed: the staged blob is the file's // content now either way, and leaving a stale one cached would serve it. self.cache.evict(uid); self.evict_reader(uid); self.wake_drain(); if let Err(e) = recorded { error!(%uid, error = %e, "recording a queued write's size failed"); return Err(if e.is_disk_full() { Errno::ENOSPC } else { Errno::EIO }); } Ok(()) } /// Queue the new content of a path-based truncate — `> file`, or any /// `setattr(size=…)` arriving without a write handle. /// /// No bytes have been authored at truncate time, which is why this path was /// never staged and instead resized the remote content inline. That is also /// why a shell redirect failed offline *before* the write that follows it: /// the truncate itself needed the network. Staging a blob describing the /// result puts it on the same queue as every other write. /// /// The blob is a hole of the new length; what is real about it is `authored`: /// /// - `> file` (size 0) is the whole point and needs nothing at all — an /// empty file is complete content, so it queues and drains offline. /// - Extending past the end authors the new tail (zeros, by definition) and /// leaves the head to be gap-filled from the base. /// - Shrinking authors nothing: every remaining byte still comes from the /// base, so it is the drain that has to fetch it. fn queue_truncate(&self, ino: u64, size: u64) -> Result<(), Errno> { let (uid, base_mtime, base_size, base_revision_id) = { let st = self.state(); match st.entries.get(&ino) { Some(e) if e.node.is_file() => ( e.uid.clone(), e.node.modification_time, node_size(&e.node), node_revision_id(&e.node), ), Some(_) => return Err(Errno::EISDIR), None => return Err(Errno::ENOENT), } }; self.require_uid_writable(&uid)?; if size == base_size { return Ok(()); } // A close immediately followed by truncate is common (`write`, close, // `truncate(2)`) and usually beats the write-back debounce. In that // case the remote is *not* our base: the complete staged revision is. // Compose from those local bytes so the truncate can supersede the // queued upload without fetching stale remote content or returning EIO. let pending_base = self.pending.lock().get(&uid).cloned(); if pending_base .as_ref() .is_some_and(|pending| !pending.meta.complete) { // An incomplete pending blob still has holes referring to the // remote revision. Stacking another incomplete transform on it is // not representable safely; preserve the existing refusal. return Err(Errno::EIO); } let (authored, complete) = if size == 0 { // An empty file has no content to be missing. (Vec::new(), true) } else if base_size == 0 { // Nothing to gap-fill from: every byte is a zero this truncate // defines. (vec![(0, size)], true) } else if size > base_size { (vec![(base_size, size)], false) } else { (Vec::new(), false) }; let (file, path) = self.cache.create_scratch().map_err(|e| { error!(%uid, error = %e, "create scratch file for truncate failed"); Errno::EIO })?; if let Some(pending) = &pending_base && let Err(e) = copy_pending_for_truncate(pending, &file) { error!(%uid, source = %pending.path.display(), error = %e, "copy pending revision for truncate failed"); let _ = std::fs::remove_file(&path); return Err(Errno::EIO); } file.set_len(size).map_err(|e| { error!(%uid, error = %e, "resize scratch file for truncate failed"); let _ = std::fs::remove_file(&path); Errno::EIO })?; let meta = match pending_base { Some(pending) => StagedWrite { uid: uid.to_string(), len: size, base_size: pending.meta.len, base_mtime: pending.meta.base_mtime, authored: if size == 0 { Vec::new() } else { vec![(0, size)] }, complete: true, // Both local revisions descend from the same last-observed // remote revision. Never replace this with the optimistic // mtime/size currently published by State. based_on: pending.meta.based_on, }, None => StagedWrite { uid: uid.to_string(), len: size, base_size, base_mtime, authored, complete, based_on: self.remote_baseline(&uid, base_mtime, base_size, base_revision_id), }, }; self.enqueue_staged_write(&uid, ino, &path, meta)?; debug!(%uid, size, complete, "queued truncate"); Ok(()) } /// Invent a node under `parent_uid` and queue the op that will make it real. /// Returns the node to intern, exactly as the online /// path returns the one the server minted. /// /// The parent may itself be a placeholder — `mkdir -p` offline, or `cp -r` of /// a tree. That is fine: the op records the parent it was made under by local /// id, and is not sent until the parent's row has the uid it landed as. fn queue_local_node( &self, parent_uid: &NodeUid, name: &str, is_dir: bool, hold: bool, ) -> Result { self.require_uid_writable(parent_uid)?; // The row the tree would give it anyway, now: the node goes by its // local id until it lands. let lid = self .db .add_local_row(&parent_uid.to_string(), name, is_dir, now_secs()) .map_err(|e| { error!(%parent_uid, name, error = %e, "giving a local node its row failed"); Errno::EIO })?; let uid = local_node_uid(lid); let node = local_node(uid.clone(), parent_uid.clone(), name.to_string(), is_dir); let op = PendingOp { id: 0, kind: if is_dir { OP_MKDIR } else { OP_CREATE }.to_string(), uid: uid.to_string(), parent_uid: Some(parent_uid.to_string()), name: Some(name.to_string()), blob_path: None, meta_json: None, created_at: now_millis(), attempts: 0, last_error: None, // Parked when the name is transient: its bytes ride on this create // (via `attach_blob_to_create`) and must not upload until a rename to // the finished name un-parks it (docs/BUGS.md B70). next_attempt_at: if hold { PARK_UNTIL } else { 0 }, }; self.db.enqueue_op(&op).map_err(|e| { error!(%parent_uid, name, error = %e, "queueing local node failed"); Errno::EIO })?; debug!(%uid, %parent_uid, name, is_dir, "created node offline; queued"); // A folder is due now; without this the workers slept through it to // the idle poll (B160). A file is not: its bytes ride on the create, and // its release wakes the drain, with bytes or without (B192). if is_dir { self.wake_drain(); } Ok(node) } /// Queue giving a node a new parent and/or name, and apply it to the tree /// now. /// /// The op records the desired end state rather than the step, so it both /// supersedes any earlier queued rename and lets the drain skip whichever /// half the remote already agrees with. /// /// `new_parent_uid` may be a placeholder — moving a file into a folder /// created offline — which is why this cannot simply be the online call with /// a retry around it: the API would 404 on a `local~` parent. The op waits /// for that folder's drain to rewrite it, exactly as a queued create does. fn queue_rename( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_ino: u64, new_parent_uid: &NodeUid, new_name: &str, ) -> Result<(), Errno> { self.require_uid_writable(uid)?; self.require_uid_writable(old_parent_uid)?; self.require_uid_writable(new_parent_uid)?; self.queue_rename_authorized( ino, uid, old_parent_uid, new_parent_ino, new_parent_uid, new_name, ) } /// Enqueue a rename whose access was linearized immediately before a /// replacement victim was removed. Rechecking here could reject after that /// destructive half had already completed. fn queue_rename_authorized( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_ino: u64, new_parent_uid: &NodeUid, new_name: &str, ) -> Result<(), Errno> { self.enqueue_rename(ino, uid, old_parent_uid, new_parent_uid, new_name)?; self.state .lock() .rename_in_place(ino, new_parent_ino, new_parent_uid, new_name); self.wake_drain(); debug!(%uid, %new_parent_uid, new_name, "renamed offline; queued"); Ok(()) } /// Record the queued rename op, and nothing else: the caller moves the node /// in the tree, through the kernel or around it. fn enqueue_rename( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_uid: &NodeUid, new_name: &str, ) -> Result<(), Errno> { // A rename that replaces a queued one keeps where the node started. let meta = match self.db.pending_op_meta(&uid.to_string(), OP_RENAME) { Ok(Some(json)) => serde_json::from_str::(&json).map_err(|error| { error!(%uid, %error, "queued rename authority metadata is invalid"); Errno::EIO })?, Ok(None) => RenameMeta { original_parent_uid: old_parent_uid.to_string(), original_name: self .state .lock() .entries .get(&ino) .map(|entry| entry.node.name.clone()), }, Err(error) => { error!(%uid, %error, "reading queued rename authority failed"); return Err(Errno::EIO); } }; let meta_json = serde_json::to_string(&meta).map_err(|error| { error!(%uid, %error, "serializing queued rename authority failed"); Errno::EIO })?; let op = PendingOp { id: 0, kind: OP_RENAME.to_string(), uid: uid.to_string(), parent_uid: Some(new_parent_uid.to_string()), name: Some(new_name.to_string()), blob_path: None, meta_json: Some(meta_json), created_at: now_millis(), attempts: 0, last_error: None, next_attempt_at: 0, }; self.db.enqueue_op(&op).map_err(|e| { error!(%uid, new_name, error = %e, "queueing rename failed"); Errno::EIO })?; Ok(()) } /// Queue trashing a node the server knows about, and drop it from the tree /// now. /// /// Anything else queued for this node is discarded first: the user has said /// the file should not exist, so uploading bytes to it or renaming it are /// both work towards an outcome nobody wants any more. That does throw away /// staged bytes that never landed — which is precisely what deleting an /// un-uploaded file means, and the alternative (upload it, then trash it) is /// worse in every way. /// /// A file still open keeps the bytes its handle reads, the queued ones and /// the cached ones, as when the trash is sent at once: a partial write /// fills its gaps from them, offline too (`docs/BUGS.md` B186). The final /// close or the trash landing, whichever comes last, lets go of them. /// /// `open` is whether the caller found the file open. A create that landed /// under a delete is trashed by its real uid before the tree has it, so /// asking the tree alone took an open file for closed (`docs/BUGS.md` /// B257). fn queue_trash(&self, uid: &NodeUid, name: &str, open: bool) -> Result<(), Errno> { self.require_uid_writable(uid)?; // The bytes below are about to be discarded, and a drain worker may be // reading them onto the wire right now. Stop it first: with several // workers the trash op this queues can be claimed while that upload is // still running, and the two would race to decide whether the file // exists. self.cancel_upload(uid); let (_, blobs) = self .db .replace_ops_with_trash(&uid.to_string(), name, now_millis()) .map_err(|e| { error!(%uid, error = %e, "queueing trash failed"); Errno::EIO })?; let open_now = open || self.is_open_anywhere(uid); if open_now { let read_from = self.pending_blob(uid); self.release_dropped_below( blobs .into_iter() .filter(|blob| read_from.as_deref() != Some(Path::new(blob))) .collect(), ); } else { self.release_dropped_ops(uid, blobs); } self.hidden.lock().insert(uid.clone()); // Withdrawn from the tree, which every mount serves from, so no other // mount goes on serving a file the user just trashed (`docs/BUGS.md` // B74). self.for_each_state(|st| { st.unlink_mem(uid); }); if !open_now { self.cache.evict(uid); self.evict_reader(uid); } self.wake_drain(); debug!(%uid, name, "trashed offline; queued"); Ok(()) } /// Drop every op queued against a node, and the staged bytes they own. fn discard_queued_ops(&self, uid: &NodeUid) -> Result<(), Errno> { // Same reason as `queue_trash`: an upload may be reading the very blobs // this is about to unlink. self.cancel_upload(uid); let blobs = self.db.delete_ops_for_uid(&uid.to_string()).map_err(|e| { error!(%uid, error = %e, "dropping queued ops failed"); Errno::EIO })?; self.release_dropped_ops(uid, blobs); Ok(()) } /// Drop every op queued against a node that just left the namespace while /// a handle still has it open, but keep the bytes that handle reads. /// /// The ops cannot wait for the final close: the node is already trashed, and /// a revision the drain sends in the meantime lands on it after the trash — /// an editor's save loop then found the name it was renaming onto still /// taken (`docs/BUGS.md` B106). The pending entry and its staged blob are /// what the open handle reads, so they stay until the final close drops /// them. fn withdraw_queued_ops(&self, uid: &NodeUid) -> Result<(), Errno> { self.cancel_upload(uid); let blobs = self.db.delete_ops_for_uid(&uid.to_string()).map_err(|e| { error!(%uid, error = %e, "withdrawing queued ops failed"); Errno::EIO })?; let read_from = self.pending_blob(uid); let blobs = blobs .into_iter() .filter(|blob| read_from.as_deref() != Some(Path::new(blob))) .collect(); self.release_dropped_below(blobs); Ok(()) } /// Let go of what the ops just dropped for `uid` held: their pending /// entries, any upload still reading their bytes, and the bytes. /// /// Removing a folder also drops the queued revisions of the files below it /// (`docs/BUGS.md` B102), so the entries to forget are found by blob, not /// only by `uid`. fn release_dropped_ops(&self, uid: &NodeUid, blobs: Vec) { self.pending.lock().remove(uid); self.release_dropped_below(blobs); } /// [`Core::release_dropped_ops`] without touching the node's own pending /// entry. fn release_dropped_below(&self, blobs: Vec) { let below: Vec = { let mut pending = self.pending.lock(); let below: Vec = pending .iter() .filter(|(_, p)| blobs.iter().any(|b| p.path == Path::new(b))) .map(|(k, _)| k.clone()) .collect(); for k in &below { pending.remove(k); } below }; for k in &below { self.cancel_upload(k); } for blob in blobs { self.cache.discard_staged(Path::new(&blob)); } } /// Which staged blob is currently queued for `uid`, if any. /// /// The path identifies the revision: publication stages a fresh file and /// swaps the map entry to point at it, so an unchanged path means the base a /// caller sampled earlier is still the base now (`docs/BUGS.md` B32). fn pending_blob(&self, uid: &NodeUid) -> Option { self.pending.lock().get(uid).map(|p| p.path.clone()) } /// Nudge the drain worker to re-examine the queue now. fn wake_drain(&self) { let (lock, cv) = &*self.drain_wake; *lock.lock() = true; cv.notify_all(); } /// Keep a write we cannot safely queue, so the bytes are recoverable even /// though the caller is getting an error. See [`Core::queue_revision`]. fn stage_orphaned_write( &self, uid: &NodeUid, ino: u64, src: &Path, meta: &StagedWrite, reason: &str, ) { match self.cache.preserve_write(meta, src) { Ok(staged) => { self.cache.clear_scratch_durable(src); error!( %uid, staged = %staged.display(), %reason, "cannot queue write; bytes kept in staging" ); let name = { let st = self.state(); st.entries .get(&ino) .map(|e| e.node.name.clone()) .unwrap_or_default() }; self.log_activity( ActivityKind::Upload, &name, format!( "write not queued ({reason}); changes kept at {}", staged.display() ), false, ); } Err(e) => { let surviving = e.surviving_path.clone(); let marker = e.marker_path.clone(); error!( %uid, error = %e, surviving = %surviving.display(), marker = ?marker, %reason, "write rescue was only partially published; bytes retained" ); let name = { let st = self.state(); st.entries .get(&ino) .map(|entry| entry.node.name.clone()) .unwrap_or_default() }; self.log_activity( ActivityKind::Upload, &name, format!( "write not queued ({reason}); changes retained at {}", surviving.display() ), false, ); } } } /// Download a whole file's plaintext, registering the transfer so /// `GetQueueStatus` can report its progress. `total` is the expected size /// (`0` if unknown). Streams through [`download_file_to`] so each block ticks /// the progress counter. /// /// [`download_file_to`]: ProtonDriveClient::download_file_to fn download_file_tracked( &self, uid: &NodeUid, name: &str, total: u64, ) -> std::result::Result, ProtonError> { let guard = self .transfers .begin(name, uid.to_string(), TransferDirection::Download, total); let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard); self.rt .block_on(self.drive.download_file_to(uid, &mut out))?; Ok(out.into_inner()) } /// Download a whole file directly into `out`, without retaining its /// plaintext in memory. This is the disk-backed twin of /// [`Core::download_file_tracked`], and preserves the same transfer /// accounting while allowing callers that only need an on-disk file to /// keep memory use independent of file size. fn download_file_tracked_to( &self, uid: &NodeUid, name: &str, total: u64, out: W, ) -> std::result::Result { let guard = self .transfers .begin(name, uid.to_string(), TransferDirection::Download, total); let mut out = CountingWriter::new(out, &guard); self.rt .block_on(self.drive.download_file_to(uid, &mut out))?; Ok(out.into_inner()) } /// Like [`download_file_tracked`] for a photo, streaming through the photos /// client's [`download_photo_to`]. /// /// [`download_file_tracked`]: Core::download_file_tracked /// [`download_photo_to`]: ProtonPhotosClient::download_photo_to fn download_photo_tracked( &self, photos: &ProtonPhotosClient, uid: &NodeUid, name: &str, total: u64, ) -> std::result::Result, ProtonError> { let guard = self .transfers .begin(name, uid.to_string(), TransferDirection::Download, total); let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard); self.rt.block_on(photos.download_photo_to(uid, &mut out))?; Ok(out.into_inner()) } /// Pin the node at mountpoint-relative `rel`. A file downloads its full /// plaintext into the content cache before this returns. A folder records a /// recursive pin and downloads every descendant file on a background thread, /// reported as a job: a big folder far outlasts the socket read timeout. /// Returns a human message. /// /// `recorded` is the path the pin registry keeps for the front ends to /// open the pin by: `rel` itself on the primary mount, which they join to /// the mountpoint, and the absolute local path on an on-demand folder's /// mount, whose node is not reachable under the primary mountpoint at all. fn pin(&self, rel: &Path, recorded: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (name, is_folder, mtime, size) = { let st = self.state(); let e = st .entries .get(&ino) .ok_or_else(|| CoreError::not_found("node vanished"))?; ( e.node.name.clone(), e.node.is_folder(), e.node.modification_time, node_size(&e.node), ) }; if is_folder { // Offline, every download would fail and the folder would only look // kept. Refuse instead, as a file pin does. if !self.is_online() { return Err(CoreError::offline()); } // Record the recursive pin first so every descendant is eviction- // exempt before we start filling the cache with the subtree. self.cache .add_pin(&uid, recorded, true) .map_err(|e| CoreError::from_api(&e, "pin"))?; let core = self.clone(); let folder = name.clone(); std::thread::Builder::new() .name("pin-folder".into()) .spawn(move || core.pin_subtree(ino, &uid, &folder)) .map_err(|e| CoreError::internal(format!("spawn pin: {e}")))?; return Ok(format!("{name}: downloading its files in the background")); } // Stored under a provisional size, the copy would never match the // plaintext it holds (`docs/BUGS.md` B282). let size = match self.kept_file(ino) { Some(file) if file.uid == uid => file.size, _ => size, }; self.fetch_content(&uid, &name, mtime, size)?; self.cache .add_pin(&uid, recorded, false) .map_err(|e| CoreError::from_api(&e, "pin"))?; Ok(name) } /// Download and cache every file in the subtree rooted at the pinned folder /// `ino` (`folder`, named `name`), as a job the front ends show. Walks the /// tree depth-first, enumerating each folder so a cold subtree is fully /// discovered; the lock is dropped before each network download. Stops when /// the folder is unpinned or the daemon stops. fn pin_subtree(&self, ino: u64, folder: &NodeUid, name: &str) { let job = self .transfers .begin_job(format!("Making {name} available offline")); match self.pin_subtree_files(ino) { Ok(files) => { job.set_total(files.len() as u64); let mut failed = 0usize; for ino in files { if self.shutdown.is_stopping() || !self.cache.is_pinned(folder) { return; } // A file with a queued write, or no readable size, is // left to the drain and the keeper (`keeper::kept_file`). if let Some(file) = self.kept_file(ino) { job.detail(file.name.clone()); if let Err(e) = self.fetch_pinned(&file.uid, &file.name, file.mtime, file.size) { warn!(uid = %file.uid, error = %e, "pin folder: download failed"); failed += 1; } } job.step(); } info!(folder = %name, failed, "pin folder: done"); } Err(e) => warn!(folder = %name, error = %e, "pin folder: enumerate failed"), } } /// Cache the file `uid` for a folder pin. Its `mtime` and `size` were read /// before the download, and a create landing or a remote edit can change /// them while it runs. The blob is then stored under the old ones and reads /// as stale, so the file is listed as kept but not available offline. Fetch /// it again under the new ones, a few times at most. fn fetch_pinned(&self, uid: &NodeUid, name: &str, mtime: i64, size: u64) -> CoreResult<()> { let mut fetched = (mtime, size); for _ in 0..3 { self.fetch_content(uid, name, fetched.0, fetched.1)?; let current = { let st = self.state(); st.by_uid .get(uid) .and_then(|ino| st.entries.get(ino)) .map(|e| (e.node.modification_time, node_size(&e.node))) }; match current { Some(now) if now != fetched => fetched = now, _ => break, } } Ok(()) } /// The inode of every file in the subtree rooted at folder `ino`. fn pin_subtree_files(&self, ino: u64) -> CoreResult> { let mut files: Vec = Vec::new(); let mut stack = vec![ino]; while let Some(dir) = stack.pop() { if self.shutdown.is_stopping() { break; } self.ensure_children(dir) .map_err(|e| self.errno_error(e, "enumerate"))?; let st = self.state(); if let Some(kids) = st.children.get(&dir) { for &k in kids { if let Some(e) = st.entries.get(&k) { if e.node.is_folder() { stack.push(k); } else { files.push(k); } } } } } Ok(files) } /// Fetch a thumbnail of `ttype` for the file at `ino`, served from the cache /// when fresh and otherwise downloaded from the remote and cached. Returns /// `Ok(None)` when the node is not a file or has no thumbnail of that type. fn thumbnail(&self, ino: u64, ttype: ThumbnailType) -> Result>, Errno> { let (uid, mtime) = { let st = self.state(); match st.entries.get(&ino) { Some(e) if e.node.is_file() => (e.uid.clone(), e.node.modification_time), Some(_) => return Ok(None), None => return Err(Errno::ENOENT), } }; if let Some(bytes) = self.cache.read_thumbnail(&uid, ttype.as_i32(), mtime) { return Ok(Some(bytes)); } // "This file has no thumbnail" is an answer worth remembering: without // it every listing pays a round trip per file to be told nothing (B5). let key = (uid.clone(), ttype.as_i32()); if self.thumbnail_misses.lock().remote_contains(&key, mtime) { return Ok(None); } // A file manager asks for one per file in a folder, on the metadata // workers. Offline, or on a stalled link, each one held a worker for // the SDK's whole retry budget, and every cold lookup, create and // rename on the mount queued behind them (`docs/BUGS.md` B220). if !self.is_online() { return Ok(None); } let bytes = self .block_on_bounded(self.client.download_thumbnail(&uid, ttype)) .map_err(|e| { warn!(%uid, error = %e, "download thumbnail failed"); Errno::EIO })?; match &bytes { Some(bytes) => { let _ = self .cache .store_thumbnail(&uid, ttype.as_i32(), mtime, bytes); } None => { self.thumbnail_misses.lock().remember_remote(key, mtime); } } Ok(bytes) } /// Unpin the node at `rel`, evicting its cached content. For a folder, also /// evicts every descendant's cached blob (the subtree is no longer kept). /// A node kept only because a folder above it is pinned is refused: there /// is no pin of its own to remove, and it would stay kept all the same. fn unpin(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (name, is_folder) = { let st = self.state(); st.entries .get(&ino) .map(|e| (e.node.name.clone(), e.node.is_folder())) .unwrap_or_default() }; if self.pinned_by_folder(ino) { return Err(CoreError::conflict(format!( "{name} is kept offline because a folder above it is; unpin that folder instead" ))); } self.cache .remove_pin(&uid) .map_err(|e| CoreError::from_api(&e, "unpin"))?; // A recursively-pinned folder's descendants were eviction-exempt; now // that the pin is gone, reclaim their blobs eagerly instead of waiting // for budget pressure. Descendants come from the DB node tree. One // pinned on its own, or under a pinned subfolder, is still kept. if is_folder && let Ok(uids) = self.db.descendants(&uid.to_string()) { let still_pinned = self.pinned_set(); for s in uids { if still_pinned.contains(&s) { continue; } if let Some(u) = parse_uid(&s) { self.cache.evict(&u); self.evict_reader(&u); } } } Ok(name) } /// Whether the node at `ino` is kept offline because a folder above it is /// pinned, whether or not it is also pinned on its own. fn pinned_by_folder(&self, ino: u64) -> bool { let parent = { let st = self.state(); st.entries .get(&ino) .filter(|e| e.parent != ino) .and_then(|e| st.entries.get(&e.parent)) .map(|p| p.uid.clone()) }; parent.is_some_and(|uid| self.cache.is_pinned(&uid)) } /// A Photos API handle sharing this Core's Drive client and session, so it /// reuses the daemon's single authenticated session rather than logging in /// again (Proton refresh tokens are single-use). Cheap — the Drive client /// is `Clone` over `Arc`-backed state. fn photos(&self) -> ProtonPhotosClient { ProtonPhotosClient::from_drive_client(self.client.clone()) } /// List the directory at mountpoint-relative `rel` for the in-app browser: /// the same lazy remote enumeration `readdir` uses, projected into /// serializable [`DirEntry`]s (with per-file pin state). fn list_dir(&self, rel: &Path) -> CoreResult> { let (ino, uid) = self.resolve(rel)?; self.ensure_children(ino) .map_err(|e| self.errno_error(e, "enumerate"))?; self.settle_listing_sizes(ino); // Every entry shares this folder as its parent, so one lookup says // whether they are all kept through it. let pinned_by_folder = self.cache.is_pinned(&uid); // Snapshot the listing, then drop the lock before touching the on-disk // pin registry so a slow disk read doesn't block FUSE metadata ops. let rows: Vec<(String, bool, u64, i64, NodeUid, u64)> = { let st = self.state(); st.children .get(&ino) .map(|kids| { kids.iter() .filter_map(|&k| st.entries.get(&k).map(|e| (k, e))) .map(|(k, e)| { ( e.node.name.clone(), e.node.is_folder(), node_size(&e.node), e.node.modification_time, e.uid.clone(), k, ) }) .collect() }) .unwrap_or_default() }; let issues = self.db.node_issues().unwrap_or_default(); Ok(rows .into_iter() .map(|(name, is_dir, size, modified, uid, ino)| DirEntry { name, is_dir, size, modified, pinned: pinned_by_folder || self.cache.is_pinned(&uid), pinned_by_folder, cached: !is_dir && self.cache.is_cached(&uid, modified, size), uid: uid.to_string(), // Listing entries live in the requested dir; the caller derives // the path from its name. Left empty. path: String::new(), // My own tree: a role only means something under a share, and // this listing never crosses into one. role: String::new(), shared_by: String::new(), shared_at: 0, shared_by_unverified: false, trashed_at: 0, trashed_from: None, issue: state::ino_lid(ino) .and_then(|lid| issues.get(&lid)) .map(|issue| SyncIssue::parse(issue)), }) .collect()) } /// Every uid that is pinned right now, as one set. /// /// `cache.is_pinned` is a direct lookup plus an ancestor walk; asking it per /// candidate was a thousand walks per keystroke. The recursive expansion is /// the same query, run once. fn pinned_set(&self) -> HashSet { self.db .pinned_uids() .unwrap_or_default() .into_iter() .collect() } /// The local locations a search result can be opened through, resolved once /// per search rather than once per hit. /// /// An on-demand folder only answers while its mount is live — without the /// mount its `local_path` is an ordinary empty directory. A mirror folder /// needs no mount: its files are real local copies the sync loop pushes /// back, so it is a valid open target whenever it is configured. /// /// This used to be a per-hit call that re-read the sync-folder list and ran /// an ancestor walk per folder, against a candidate pool of up to a /// thousand: the whole cost of a keystroke was in here. fn search_roots(&self) -> SearchRoots { let live_mounts = self.mounts.lock(); let folders = self.db.sync_folder_list().unwrap_or_default(); SearchRoots { roots: folders .into_iter() .filter(|folder| match folder.mode.as_str() { "ondemand" => live_mounts.contains_key(&folder.id), "mirror" => true, _ => false, }) .filter_map(|folder| { let root = self.db.node_path(&folder.remote_uid).ok().flatten()?; Some((root, PathBuf::from(folder.local_path))) }) .collect(), } } /// Full-text search node names against the local SQLite index, mapping each /// DB hit to the wire [`SearchHit`] (resolving live pin state from the cache, /// which the DB doesn't track). Pure local lookup — never hits the network. /// Every pin, with `cached` saying whether a pinned file's content is on /// disk, so a front-end listing pins can badge them without a round-trip /// per file. fn list_pins(&self) -> Vec { let mut pins = self.cache.list_pins(); for pin in pins.iter_mut().filter(|pin| pin.is_dir == Some(false)) { if let Ok(Some(node)) = self.db.node_by_uid(&pin.uid) { pin.cached = self.cache .is_cached(&node.uid, node.modification_time, node_size(&node)); } } pins } fn search(&self, query: &str, limit: usize, scope: Option<&str>) -> CoreResult> { let hits = self .db .search_in(query, limit, scope) .map_err(|e| CoreError::from_api(&e, "search"))?; let roots = self.search_roots(); let pinned = self.pinned_set(); Ok(hits .into_iter() .map(|h| SearchHit { name: h.node.name.clone(), is_dir: h.node.is_folder(), size: node_size(&h.node), modified: h.node.modification_time, pinned: pinned.contains(&h.node.uid.to_string()), cached: !h.node.is_folder() && self.cache.is_cached( &h.node.uid, h.node.modification_time, node_size(&h.node), ), uid: h.node.uid.to_string(), mounted_path: roots.resolve(&h.path), path: h.path, score: 0, }) .collect()) } /// Search the index of files on this machine (outside Drive), built by the /// background scanner in [`run_local_index`]. Pure local lookup, never hits /// the network — and never touches the FUSE mount, which the scanner excludes. fn search_local(&self, query: &str, limit: usize) -> CoreResult> { let hits = self .db .search_local(query, limit) .map_err(|e| CoreError::from_api(&e, "local search"))?; Ok(hits .into_iter() .map(|h| LocalHit { name: h.name, path: h.path, is_dir: h.is_dir, size: h.size.max(0) as u64, modified: h.mtime, score: 0, }) .collect()) } /// Search both prompt sources with identical query and per-source limits. /// Keeping the composition in the daemon core gives every control surface /// the same semantics while preserving the legacy source-specific methods. fn search_v2( &self, query: &str, limit: usize, filters: &SearchFilters, ) -> CoreResult<(Vec, Vec)> { if query.trim().is_empty() || limit == 0 { return Ok((Vec::new(), Vec::new())); } // Candidate generation is intentionally broader than the final result // cap: filtering and fuzzy rejection happen below, so limiting at the // SQLite boundary to `limit` would recreate the old empty-filter bug. let candidate_limit = limit.saturating_mul(10).clamp(100, 1_000); let mut drive_hits = if filters.sources.contains(&SearchSource::Drive) { let roots = self.search_roots(); let pinned = self.pinned_set(); self.db .search_candidates(query, candidate_limit) .map_err(|e| CoreError::from_api(&e, "search candidates"))? .into_iter() .filter(|hit| filters.kind.accepts(&hit.node.name, hit.node.is_folder())) .filter_map(|hit| { let parent = Path::new(&hit.path) .parent() .map_or_else(String::new, |path| path.display().to_string()); let uid = hit.node.uid.to_string(); let is_pinned = pinned.contains(&uid); let score = relevance_score(query, &hit.node.name, &parent)? + if is_pinned { 250 } else { 0 }; Some(SearchHit { name: hit.node.name.clone(), is_dir: hit.node.is_folder(), size: node_size(&hit.node), modified: hit.node.modification_time, pinned: is_pinned, cached: !hit.node.is_folder() && self.cache.is_cached( &hit.node.uid, hit.node.modification_time, node_size(&hit.node), ), uid, mounted_path: roots.resolve(&hit.path), path: hit.path, score, }) }) .collect() } else { Vec::new() }; let mut local_hits = if filters.sources.contains(&SearchSource::Local) { self.db .search_local_candidates(query, candidate_limit) .map_err(|e| CoreError::from_api(&e, "local search candidates"))? .into_iter() .filter(|hit| filters.kind.accepts(&hit.name, hit.is_dir)) .filter_map(|hit| { let parent = Path::new(&hit.path) .parent() .map_or_else(String::new, |path| path.display().to_string()); Some(LocalHit { score: relevance_score(query, &hit.name, &parent)?, name: hit.name, path: hit.path, is_dir: hit.is_dir, size: hit.size.max(0) as u64, modified: hit.mtime, }) }) .collect() } else { Vec::new() }; drive_hits.sort_by(|a, b| { b.score .cmp(&a.score) .then_with(|| a.name.cmp(&b.name)) .then_with(|| a.path.cmp(&b.path)) }); local_hits.sort_by(|a, b| { b.score .cmp(&a.score) .then_with(|| a.name.cmp(&b.name)) .then_with(|| a.path.cmp(&b.path)) }); drive_hits.truncate(limit); local_hits.truncate(limit); Ok((drive_hits, local_hits)) } /// A page of the photos timeline (newest first), sliced out of the DB. /// `Ok(None)` when the account has no photos volume. /// /// Stale-while-revalidate: a persisted timeline is served *immediately*, and /// refreshed on the runtime if it is older than [`TIMELINE_TTL`] — so opening /// the app paints from disk rather than waiting on a full `enumerate_timeline` /// (which returns the whole timeline, not a page). Only an empty DB blocks on /// the network, i.e. the very first run. /// /// Metadata only: a thumbnail path is attached for photos whose thumbnail is /// already cached, and nothing is downloaded here — the front-end pulls the /// thumbnails it actually paints via [`Core::photo_thumbs`]. fn listing_stale(&self, key: &str, ttl: Duration) -> bool { match self.db.state_i64(key).ok().flatten() { Some(ms) => now_ms().saturating_sub(ms) >= ttl.as_millis() as i64, None => true, } } /// Re-fetch the whole photos timeline and persist it. Returns whether the /// account has a photos volume at all. /// Download the full content of the Drive file at mountpoint-relative `rel` /// into the content cache, returning its on-disk path (served from cache /// when a fresh blob already exists). Lets a front-end open the file with /// the user's default application without pinning it. fn open_file(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (name, mtime, size) = { let st = self.state(); let e = st .entries .get(&ino) .ok_or_else(|| CoreError::not_found("node vanished"))?; if !e.node.is_file() { return Err(CoreError::invalid("not a regular file")); } ( e.node.name.clone(), e.node.modification_time, node_size(&e.node), ) }; self.fetch_content(&uid, &name, mtime, size) } /// [`open_file`](Self::open_file) addressed by uid instead of by path. /// /// A search hit is a metadata-index row, and the index covers every /// location this daemon knows — including on-demand sync folders and /// mirrors, whose nodes have no path inside the primary mount's tree. /// Walking such a hit's path from the mount root answers ENOENT even though /// the node is perfectly reachable, so front-ends address hits by uid. /// /// Metadata comes from the live tree, then the index, then the API — the /// last covers a node the index knows but no location holds. Unlike the /// mutating by-uid handlers this does not demand a /// [`resolve_anywhere`](Self::resolve_anywhere) residency proof: reading /// one's own file is what the API authorizes anyway, and requiring /// residency is exactly what turned an openable search hit into "no such /// file or folder". [`open_shared_file`](Self::open_shared_file) already /// works this way. Reserved (local/virtual) uids name no remote node and /// are still rejected. fn open_file_uid(&self, raw_uid: &str) -> CoreResult { let uid = parse_uid(raw_uid) .filter(|uid| !is_local_uid(uid) && !is_virtual_uid(uid)) .ok_or_else(|| CoreError::invalid(format!("invalid uid: {raw_uid}")))?; let live = self .state .lock() .entries .values() .find(|e| e.uid == uid) .map(|e| e.node.clone()); let node = match live { Some(node) => node, None => match self .db .node_by_uid(&uid.to_string()) .map_err(|e| CoreError::from_api(&e, "load node"))? { Some(node) => node, None => self .rt .block_on(self.drive.get_node(&uid)) .map_err(|e| CoreError::from_api(&e, "get node"))? .ok_or_else(|| CoreError::not_found("node vanished"))?, }, }; if !node.is_file() { return Err(CoreError::invalid("not a regular file")); } self.fetch_content(&uid, &node.name, node.modification_time, node_size(&node)) } /// Materialise a file's full content into the content cache, returning its /// on-disk path. Serves a fresh cached blob without touching the network. fn fetch_content( &self, uid: &NodeUid, name: &str, mtime: i64, size: u64, ) -> CoreResult { if let Some(p) = self.cache.cached_content_path(uid, mtime, size) { return Ok(p); } // Keep the temporary file beside the final cache blob. `store_file` // can then adopt it with a hard link and atomic rename, while the SDK // streams decrypted blocks to disk instead of building a file-sized // `Vec`. A unique name also keeps concurrent opens of the same node // from sharing a partially written staging file. static OPEN_TEMP_SEQ: AtomicU64 = AtomicU64::new(0); let seq = OPEN_TEMP_SEQ.fetch_add(1, Ordering::Relaxed); let tmp = self.cache.content_path(uid).with_extension(format!( "open-{}-{}-{seq}.tmp", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) )); let file = File::options() .write(true) .create_new(true) .open(&tmp) .map_err(|e| CoreError::internal(format!("create download temp: {e}")))?; let file = match self.download_file_tracked_to(uid, name, size, file) { Ok(file) => file, Err(e) => { let _ = std::fs::remove_file(&tmp); return Err(CoreError::from_api(&e, "download")); } }; if let Err(e) = file.sync_all() { let _ = std::fs::remove_file(&tmp); return Err(CoreError::internal(format!("sync download temp: {e}"))); } if let Err(e) = self.cache.store_file(uid, mtime, size, &tmp) { let _ = std::fs::remove_file(&tmp); return Err(CoreError::internal(format!("cache store: {e}"))); } let _ = std::fs::remove_file(&tmp); Ok(self.cache.content_path(uid)) } /// Drop the cached child listing of `rel`'s parent directory so the next /// `ListDir` re-enumerates it from the server. No-op when the parent can't be /// resolved (e.g. `rel` is the root). Resolves the parent without holding the /// state lock, then invalidates under it. fn invalidate_parent_listing(&self, rel: &Path) { let parent = rel.parent().unwrap_or_else(|| Path::new("")); if let Ok((pino, _)) = self.resolve_path(parent) { self.state().invalidate_listing(pino); } } /// Drop the cached child listing of the folder `parent`, everywhere it is /// held — naming the folder by uid rather than by inode. /// /// [`Core::invalidate_parent_listing`] is the path-based sibling, and it /// only reaches *this* mount's paths. A remote mutation made by the /// sync engine needs more than that, for two reasons. The folder it changed /// is usually not resident in the tree at the time — a mirror folder /// has no FUSE session at all — so there is no inode to name. And the stale /// listing that outlives the pass is not a hot-cache map but the `listed` /// flag on the folder's DB row, which survives a daemon restart and is what /// a later `ensure_children` trusts. /// /// So the flag is cleared unconditionally, and the resident listing is /// invalidated on top of it. That is what makes a mirror folder switched to /// on-demand re-enumerate instead of serving a snapshot that predates the /// engine's own uploads — `docs/BUGS.md` B86. pub(crate) fn invalidate_children_of(&self, parent: &NodeUid) { if let Err(e) = self.db.set_listed(parent, false) { warn!(uid = %parent, error = ?e, "could not clear cached listing"); } self.for_each_state(|st| { if let Some(&ino) = st.by_uid.get(parent) { st.invalidate_listing(ino); } }); } /// Rename a file or folder to `new_name`. `rel` is mountpoint-relative. /// Mirrors the FUSE `rename` write path: queued whenever the mount queues /// ([`Core::queue_relocation`]), otherwise renamed on the remote, then /// moved to its new name in every mount with the parent listing dropped /// so the next `ListDir` re-enumerates. fn rename(&self, rel: &Path, new_name: &str) -> CoreResult { if new_name.is_empty() || new_name.contains('/') { return Err(CoreError::invalid(format!("invalid name: {new_name:?}"))); } let (ino, uid) = self.resolve(rel)?; let old_parent_uid = self.source_parent_uid(ino, rel)?; require_rename_access( |authority| self.require_uid_writable(authority), &uid, &old_parent_uid, &old_parent_uid, ) .map_err(|error| self.errno_error(error, "rename access"))?; if self.queues_relocation(&uid, &old_parent_uid) { self.queue_relocation( ino, &uid, &old_parent_uid, &old_parent_uid, &rel.with_file_name(new_name), new_name, )?; return Ok(new_name.to_string()); } self.rename_remote(&uid, new_name) .map_err(|e| CoreError::from_api(&e, "rename"))?; self.relocate_everywhere(&uid, &old_parent_uid, new_name); self.invalidate_parent_listing(rel); Ok(new_name.to_string()) } /// Whether a rename or move asked for over the control socket goes /// through the queue. It does whenever the same change through the mount /// would, and whenever the node has anything queued: sent straight to /// Drive, it would land first, and the queued change landing after it /// would undo it (`docs/BUGS.md` B149). fn queues_relocation(&self, uid: &NodeUid, new_parent_uid: &NodeUid) -> bool { !self.sends_inline() || is_local_uid(uid) || is_local_uid(new_parent_uid) || self.db.has_any_op(&uid.to_string()).unwrap_or(true) } /// Queue renaming or moving the node at `ino` to `dest`, whose last /// component is `name`, and apply it to every mount now, as the mount's /// own queued rename does. /// /// The listings stay: Drive has not been told yet, so listing either /// folder again would bring back the old name. fn queue_relocation( &self, ino: u64, uid: &NodeUid, old_parent_uid: &NodeUid, new_parent_uid: &NodeUid, dest: &Path, name: &str, ) -> CoreResult<()> { // Drive refuses a taken name, but only once the queue sends the change. match self.resolve_path(dest) { Ok((found, _)) if found != ino => { return Err(CoreError::conflict(format!( "{} already exists", dest.display() ))); } Ok(_) => {} Err(e) if e.code() == libc::ENOENT => {} Err(e) => { return Err(self.errno_error(e, &format!("could not resolve {}", dest.display()))); } } let old_name = self .state() .entries .get(&ino) .map(|entry| entry.node.name.clone()) .unwrap_or_default(); let mut uid = uid.clone(); // A node whose create is still queued is that op: rewriting its target // is the whole change, as through the mount. if is_local_uid(&uid) { match self .db .rewrite_op_target(&uid.to_string(), &new_parent_uid.to_string(), name) { Ok(true) => { self.relocate_queued_everywhere(&uid, new_parent_uid, name); // A parked transient file given a finished name is meant // to reach Drive now (`docs/BUGS.md` B70). if is_transient_name(&old_name) && !is_transient_name(name) && let Err(e) = self.db.set_create_hold(&uid.to_string(), false) { warn!(%uid, error = %e, "un-parking a finalized transient create failed"); } self.wake_drain(); return Ok(()); } // The create landed meanwhile: rename what Drive made. Ok(false) => { uid = self.landed_row_uid(ino).ok_or_else(|| { CoreError::conflict(format!( "{old_name} changed while it was being renamed; try again" )) })?; } Err(e) => return Err(CoreError::internal(format!("db: {e}"))), } } self.enqueue_rename(ino, &uid, old_parent_uid, new_parent_uid, name) .map_err(|e| self.errno_error(e, "queue the change"))?; self.relocate_queued_everywhere(&uid, new_parent_uid, name); self.wake_drain(); debug!(%uid, %new_parent_uid, name, "renamed over the control socket; queued"); Ok(()) } /// Bring every mount level with a rename or move Drive has made of `uid`, /// to `name` in `new_parent`, and have each kernel session drop the name it /// cached for it. Returns the name a mount knew the node by. /// /// For changes that do not come through the mount. The kernel made a /// rename through the mount itself, but one made over the control socket /// leaves the old name cached for [`TTL`] (`docs/BUGS.md` B114). /// /// The node is moved, not forgotten, and so is its row. Forgetting it, as /// this used to, deleted the row, and until a listing brought it back /// nothing local knew the node: `pdfs sync queue` showed its queued upload /// by raw id, and the drain re-read the node and put the upload off /// (`docs/BUGS.md` B117). fn relocate_everywhere( &self, uid: &NodeUid, new_parent: &NodeUid, name: &str, ) -> Option { self.relocate_with(uid, new_parent, name, false) } /// [`Core::relocate_everywhere`] for a change the queue has not sent yet. /// Each mount keeps its listings, which Drive cannot bring up to date /// until the change lands. fn relocate_queued_everywhere(&self, uid: &NodeUid, new_parent: &NodeUid, name: &str) { self.relocate_with(uid, new_parent, name, true); } fn relocate_with( &self, uid: &NodeUid, new_parent: &NodeUid, name: &str, queued: bool, ) -> Option { let mut old_name = None; let mut moved_row = false; self.for_each_mount(|st, notify| { if let Some((from_name, relocated)) = relocate_in_mount(st, notify, uid, new_parent, name, queued) { old_name = Some(from_name); moved_row |= relocated; } }); if !moved_row { self.move_row(uid, new_parent, name); } old_name } /// Point the persisted row of `uid` at `name` in `new_parent`, for a node /// no mount holds where it now is. fn move_row(&self, uid: &NodeUid, new_parent: &NodeUid, name: &str) { let moved = self.db.node_by_uid(&uid.to_string()).and_then(|node| { let Some(mut node) = node else { return Ok(()); }; node.name = name.to_string(); node.parent_uid = Some(new_parent.clone()); self.db.upsert_node(&node) }); if let Err(error) = moved { warn!(%uid, ?error, "could not move the node's row along with it"); } } /// Forget `uid` in every mount, and have each kernel session drop the name /// it cached for it. Returns the name a mount knew the node by. /// /// For a removal that does not come through the mount. The kernel made an /// unlink through the mount itself, but one made over the control socket /// leaves the old name cached for [`TTL`], pointing at an inode the mount no /// longer has (`docs/BUGS.md` B114). Every mount, not only this one, so a /// fork showing the same node does not keep serving it (`docs/BUGS.md` /// B74). fn forget_everywhere(&self, uid: &NodeUid) -> Option { let mut name = None; self.for_each_mount(|st, notify| { if let Some(entry) = st.by_uid.get(uid).and_then(|ino| st.entries.get(ino)) { name = Some(entry.node.name.clone()); } forget_and_notify(st, notify, uid); }); name } /// Move a file or folder into the folder at `new_parent_rel`. Both paths are /// mountpoint-relative. Queued whenever the mount queues /// ([`Core::queue_relocation`]); otherwise moved on the remote, then in /// every mount, with both listings invalidated so each re-enumerates on /// next access. fn move_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let old_parent_uid = self.source_parent_uid(ino, rel)?; let (pino, new_parent_uid) = self .resolve_path(new_parent_rel) .map_err(|e| self.errno_error(e, "resolve new parent"))?; require_rename_access( |authority| self.require_uid_writable(authority), &uid, &old_parent_uid, &new_parent_uid, ) .map_err(|error| self.errno_error(error, "move access"))?; let name = self .state() .entries .get(&ino) .map(|entry| entry.node.name.clone()) .unwrap_or_default(); if self.queues_relocation(&uid, &new_parent_uid) { if self.state().is_ancestor_of(ino, pino) { return Err(CoreError::invalid("cannot move a folder into itself")); } self.queue_relocation( ino, &uid, &old_parent_uid, &new_parent_uid, &new_parent_rel.join(&name), &name, )?; return Ok(name); } self.move_remote(&uid, &new_parent_uid) .map_err(|e| CoreError::from_api(&e, "move"))?; self.relocate_everywhere(&uid, &new_parent_uid, &name); self.invalidate_parent_listing(rel); self.state().invalidate_listing(pino); Ok(name) } /// Copy a file or folder into `new_parent_rel` on Proton Drive, which copies /// the content there: nothing is downloaded or uploaded. Both paths are /// mountpoint-relative. A name already taken in the folder gets the next /// free ` (n)` suffix. Returns the copy's name. /// /// Drive copies what it holds, so the copy is refused offline and while the /// mount still owes Drive part of it: a node or folder only queued locally, /// a file with a queued upload or unsaved writes, or — for a folder, whose /// queued changes could sit anywhere below it — any queued op at all. fn copy_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let (pino, new_parent_uid) = self .resolve_path(new_parent_rel) .map_err(|e| self.errno_error(e, "resolve new parent"))?; self.ensure_children(pino) .map_err(|e| self.errno_error(e, "enumerate"))?; self.require_uid_writable(&new_parent_uid) .map_err(|error| self.errno_error(error, "copy access"))?; let (name, is_folder, unsaved) = { let st = self.state(); let entry = st .entries .get(&ino) .ok_or_else(|| CoreError::not_found(format!("{} is gone", rel.display())))?; // Unsaved writes, or a release still queueing them, to the file or, // for a folder, to any file below it. let unsaved = st .active_writes .iter() .filter(|(_, w)| w.dirty) .map(|(&open, _)| open) .chain(st.releasing.iter().copied()) .any(|open| relocate::ino_is_under(&st, open, ino)); (entry.node.name.clone(), entry.node.is_folder(), unsaved) }; // Unlike a move, a copy cannot wait in the queue: it needs the node as // Drive has it, so it goes now or not at all, local-first or not. if !self.is_online() { return Err(CoreError::offline()); } let owed = unsaved || is_local_uid(&uid) || is_local_uid(&new_parent_uid) || self.db.has_any_op(&uid.to_string()).unwrap_or(true) || (is_folder && self .db .pending_op_counts() .map(|counts| counts.uploads + counts.changes > 0) .unwrap_or(true)); if owed { return Err(CoreError::conflict(format!( "{name} has changes that have not reached Proton Drive yet; \ copy it once they have synced" ))); } let target = if is_folder { AvailableNameTarget::Folder } else { AvailableNameTarget::File }; let copy_name = self .rt .block_on( self.drive .get_available_name(&new_parent_uid, &name, target), ) .map_err(|e| CoreError::from_api(&e, "copy"))?; let new_uid = self .rt .block_on( self.drive .copy_node(&uid, &new_parent_uid, Some(©_name)), ) .map_err(|e| CoreError::from_api(&e, "copy"))?; let node = self .fetch_minted_node(&new_uid, ©_name) .ok_or_else(|| minted_but_unlisted(©_name))?; let mut st = self.state(); let copy_ino = st.intern(pino, node); if let Some(kids) = st.children.get_mut(&pino) && !kids.contains(©_ino) { kids.push(copy_ino); } drop(st); self.flush_access_changes(); Ok(copy_name) } /// What the folder at `rel` holds on Proton Drive: bytes on storage and the /// number of items below it. Drive adds it up, so nothing is listed here; /// trashed items count, and the bytes are encrypted ones. pub(crate) fn folder_size(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let is_folder = self .state() .entries .get(&ino) .is_some_and(|entry| entry.node.is_folder()); if !is_folder { return Err(CoreError::invalid(format!( "{} is not a folder", rel.display() ))); } self.folder_size_for_uid(&uid) } /// [`Core::folder_size`] for a folder addressed through any daemon location. pub(crate) fn folder_size_by_uid(&self, uid: &str) -> CoreResult { let uid = self.resolve_anywhere(uid)?; self.folder_size_for_uid(&uid) } fn folder_size_for_uid(&self, uid: &NodeUid) -> CoreResult { if is_local_uid(uid) { return Err(CoreError::conflict("the folder is not on Proton Drive yet")); } if !self.is_online() { return Err(CoreError::offline()); } self.rt .block_on(self.client.get_folder_size(uid)) .map_err(|e| CoreError::from_api(&e, "folder size")) } /// Trash a file or folder. `rel` is mountpoint-relative. Forgets the node, /// evicts any cached content, and invalidates the parent listing. fn delete(&self, rel: &Path) -> CoreResult { let (ino, uid) = self.resolve(rel)?; let parent_uid = self.source_parent_uid(ino, rel)?; require_node_parent_access( |authority| self.require_uid_writable(authority), &uid, &parent_uid, ) .map_err(|error| self.errno_error(error, "trash access"))?; self.rt .block_on(self.drive.trash_nodes(std::slice::from_ref(&uid))) .and_then(batch::into_unit) .map_err(|e| CoreError::from_api(&e, "trash"))?; // Uploads queued for the node or anything below it would now only fail // against the trash, forever, holding their bytes (`docs/BUGS.md` B102). if let Err(error) = self.discard_queued_ops(&uid) { warn!(%uid, ?error, "remote trash landed but queued-op cleanup failed"); } let name = self.forget_everywhere(&uid).unwrap_or_default(); self.cache.evict(&uid); self.evict_reader(&uid); self.invalidate_parent_listing(rel); self.invalidate_trash(); Ok(name) } /// Remove the node a `rename` is about to replace, so the new name is free /// for the API call that follows. /// /// `rename(2)` promises to replace an existing destination atomically. Proton /// offers no such primitive — `rename_node` refuses a name that is already /// taken — so this is the first half of an emulation that is *not* atomic: /// see [`Core::restore_replaced`] for the other half (`docs/BUGS.md` B13). /// /// A node whose own creation is still queued has never reached the server, so /// dropping its queued ops is the whole removal; nothing goes to the wire and /// it works offline. A create that landed before its ops were dropped left /// none: Drive has the node, so it is trashed there like any other /// (`docs/BUGS.md` B175). /// /// A handle still open on the replaced node keeps reading its bytes, as /// `rename(2)` promises: the inode stays as an unlinked entry and its /// staged bytes, cache and reader go at the final close, exactly as for an /// `unlink` under an open handle. Its queued ops go now, or the drain would /// upload to the node after it was trashed (`docs/BUGS.md` B106). fn remove_replaced(&self, ino: u64, uid: &NodeUid, name: &str) -> Result<(), Errno> { // Asked under the uid the tree has now: a landed create's real uid is // not in the tree yet (`docs/BUGS.md` B257). let open = self.is_open_anywhere(uid); let landed; let uid = if is_local_uid(uid) { if open { self.withdraw_queued_ops(uid)?; } else { self.discard_queued_ops(uid)?; } self.for_each_state(|st| { st.forget_or_unlink(uid); }); match self.landed_row_uid(ino) { Some(real) => { debug!(local = %uid, %real, name, "create landed under a replace"); landed = real; &landed } None => { debug!(%uid, name, "replaced a node whose create was still queued"); return Ok(()); } } } else { uid }; // Local-first or offline, or the network goes away under the call: // queue the trash. The rename that follows queues as well, behind it. if !self.sends_inline() { return self.queue_trash(uid, name, open); } self.require_uid_writable(uid)?; if let Err(e) = self .block_on_bounded(self.drive.trash_nodes(std::slice::from_ref(uid))) .and_then(batch::into_unit) { if self.lost_link(&e, "trash a replaced node") { return self.queue_trash(uid, name, open); } error!(%uid, name, error = %e, "trashing the node a rename replaces failed"); self.log_activity(ActivityKind::Trash, name, e.to_string(), false); return Err(Errno::EIO); } // The node was trashed on the server; withdraw it from every inode // space that had it, not only the mount the rename came through. let open_now = self.is_open_anywhere(uid); self.for_each_state(|st| { st.forget_or_unlink(uid); }); self.hidden.lock().insert(uid.clone()); if open_now { if let Err(error) = self.withdraw_queued_ops(uid) { error!(%uid, "remote replacement landed but queued-op cleanup failed"); return Err(error); } } else { if let Err(error) = self.discard_queued_ops(uid) { error!(%uid, "remote replacement landed but queued-op cleanup failed"); return Err(error); } self.cache.evict(uid); self.evict_reader(uid); } self.invalidate_trash(); // The node is recoverable from the trash, but only if the user knows it // went there — a rename is not an operation anyone expects to trash // something, so this is the only record that it happened. self.log_activity( ActivityKind::Trash, name, "replaced by a rename from the mount", true, ); Ok(()) } /// Whether any mount holds an open handle on `uid`. fn is_open_anywhere(&self, uid: &NodeUid) -> bool { let mut open = false; self.for_each_state(|st| { open |= st .by_uid .get(uid) .and_then(|ino| st.entries.get(ino)) .is_some_and(|entry| entry.open_count > 0); }); open } /// Put back the node [`Core::remove_replaced`] trashed, after the rename it /// was clearing the way for failed anyway. /// /// Best-effort by construction: if the restore also fails there is nothing /// further to try, and the node is still in the trash where `pdfs restore` /// can reach it. Says so loudly in that case, because the alternative is a /// file the user believes was only renamed quietly sitting in the trash. /// /// A restored node comes back to the mount as the ordinary restore brings /// it: out of `hidden`, and through a fresh listing of `parent` in every /// mount that shows it (`docs/BUGS.md` B124). fn restore_replaced(&self, victim: Option<&(u64, NodeUid)>, parent: &NodeUid, name: &str) { let Some((_, uid)) = victim else { return }; if is_local_uid(uid) { // Its queued create was discarded and cannot be reconstructed from // here; the caller's error is what the user acts on. warn!(%uid, name, "a rename failed after discarding a queued node it replaced"); return; } match self .rt .block_on(self.drive.restore_nodes(std::slice::from_ref(uid))) .and_then(batch::into_unit) { Ok(()) => { self.hidden.lock().remove(uid); self.for_each_mount(|st, notify| { if let Some(&ino) = st.by_uid.get(parent) { st.invalidate_listing(ino); notify.inval_entry(ino, name.to_string()); } }); self.invalidate_trash(); debug!(%uid, name, "restored the node a failed rename had replaced"); } Err(e) => { error!(%uid, name, error = %e, "restoring a replaced node failed; it stays in the trash"); self.log_activity(ActivityKind::Restore, name, e.to_string(), false); } } } /// List the account's trash, from the DB. Trashed nodes are outside the /// mounted tree — the FUSE side forgot them when they were trashed — so the /// listing is persisted in its own table rather than derived from `State`, and /// each entry is identified by its uid (its only remaining handle) with an /// empty path. /// /// Stale-while-revalidate, like the photos timeline: a persisted listing comes /// back at DB speed and is refreshed in the background past [`TRASH_TTL`]. /// Our own trash mutations invalidate it outright (see /// [`Core::invalidate_trash`]), so the TTL only covers changes made elsewhere. /// /// The refresh always runs *off* the request: a never-fetched trash waits for /// it, but only for [`TRASH_FIRST_WAIT`], and then answers with the batches /// that have landed so far. Blocking the request on the whole refresh is what /// made a large trash unlistable — the refresh outran the front-end's read /// timeout, the user asked again, and each attempt left another full refresh /// grinding in the daemon (`docs/BUGS.md` B72). fn list_trash(&self) -> CoreResult> { let never_fetched = self.db.state_i64(TRASH_SYNCED_MS).ok().flatten().is_none(); if never_fetched || self.listing_stale(TRASH_SYNCED_MS, TRASH_TTL) { self.spawn_trash_refresh(); } if never_fetched { self.await_trash_refresh(TRASH_FIRST_WAIT); } // Most of a trash comes from a handful of folders. let mut origins: HashMap> = HashMap::new(); Ok(self .db .trash_list() .map_err(CoreError::from)? .into_iter() .map(|item| DirEntry { trashed_from: item.parent_uid.as_ref().and_then(|parent| { origins .entry(parent.clone()) .or_insert_with(|| self.db.node_path(parent).ok().flatten()) .clone() }), trashed_at: item.trashed_at.unwrap_or(0), name: item.name, is_dir: item.is_dir, size: item.size.max(0) as u64, modified: item.mtime, // A trashed node can't be pinned or served from the mount, so its // content is never current cache: report neither. pinned: false, pinned_by_folder: false, cached: false, uid: item.uid, path: String::new(), role: String::new(), shared_by: String::new(), shared_at: 0, shared_by_unverified: false, issue: None, }) .collect()) } /// Re-fetch the trash listing from the server and persist it. /// /// Materialized in chunks of [`TRASH_MATERIALIZE_CHUNK`], each persisted as /// it lands: decrypting a trashed node costs an S2K unlock, so a trash of a /// few thousand nodes takes minutes, and an all-or-nothing write would throw /// that away on the first failure and show the user nothing until the very /// end. Every batch is a usable listing. async fn refresh_trash(&self) -> CoreResult<()> { let started = Instant::now(); let uids = within_trash_call("enumerate trash", self.drive.enumerate_trash_node_uids()) .await? .map_err(|e| CoreError::from_api(&e, "enumerate trash"))?; info!( count = uids.len(), elapsed_ms = started.elapsed().as_millis() as u64, "trash refresh: enumerated" ); // What the listing held before, and whether it was the whole trash: the // chunks below overwrite it, so both are read up front. let seen = self.db.trash_seen().unwrap_or_default(); let complete = self .db .state_i64(TRASH_COMPLETE_MS) .ok() .flatten() .is_some(); let _ = self.db.clear_state(TRASH_COMPLETE_MS); let now = now_secs(); if uids.is_empty() { self.db.trash_replace(&[]).map_err(CoreError::from)?; } let mut items: Vec = Vec::with_capacity(uids.len()); for chunk in uids.chunks(TRASH_MATERIALIZE_CHUNK) { let chunk_started = Instant::now(); let nodes = within_trash_call("enumerate nodes", self.drive.enumerate_nodes(chunk)) .await? .map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?; items.extend(nodes.into_iter().map(|node| StoredTrash { uid: node.uid.to_string(), name: node.name.clone(), is_dir: node.is_folder(), size: node_size(&node) as i64, mtime: node.modification_time, // The folder the node will go back to, which is also what tells // a restore whether that folder is itself in the trash. parent_uid: node.parent_uid.as_ref().map(|uid| uid.to_string()), trashed_at: first_seen_in_trash(&seen, &node.uid.to_string(), complete, now), })); // Cumulative, so the table is always a prefix of the real trash // rather than a mix of this refresh and the last one. self.db.trash_replace(&items).map_err(CoreError::from)?; self.trash_progress.notify_waiters(); debug!( materialized = items.len(), total = uids.len(), chunk_ms = chunk_started.elapsed().as_millis() as u64, "trash refresh: batch" ); } let _ = self.db.set_state_i64(TRASH_SYNCED_MS, now_ms()); let _ = self.db.set_state_i64(TRASH_COMPLETE_MS, now_ms()); info!( count = items.len(), elapsed_ms = started.elapsed().as_millis() as u64, "trash refresh: done" ); Ok(()) } /// Refresh the trash off the request path. At most one refresh at a time — /// including the one a first-ever [`Core::list_trash`] waits on, so a burst /// of requests against an unfetched trash joins one refresh instead of /// starting one apiece. fn spawn_trash_refresh(&self) { if self.trash_refreshing.swap(true, Ordering::SeqCst) { return; } let core = self.clone(); self.rt.spawn(async move { let refreshed = core.refresh_trash().await; core.trash_refreshing.store(false, Ordering::SeqCst); match refreshed { Ok(_) => core.events.publish(&[Topic::Trash]), Err(e) => warn!(error = %e, "background trash refresh failed"), } // After the flag, so a waiter that wakes on this sees it cleared. core.trash_progress.notify_waiters(); }); } /// Wait for the in-flight trash refresh to make progress, for at most /// `budget`. Returns early when it finishes; never cancels it — the refresh /// is a detached task, so giving up here costs the work nothing. fn await_trash_refresh(&self, budget: Duration) { let deadline = Instant::now() + budget; self.rt.block_on(async { while self.trash_refreshing.load(Ordering::SeqCst) { let Some(left) = deadline.checked_duration_since(Instant::now()) else { break; }; let notified = self.trash_progress.notified(); tokio::pin!(notified); // Register before re-checking the flag: a refresh that finishes // in between must not leave us waiting for a notify that has // already fired. notified.as_mut().enable(); if !self.trash_refreshing.load(Ordering::SeqCst) { break; } if tokio::time::timeout(left, notified).await.is_err() { warn!( budget_s = budget.as_secs(), "trash refresh still running; answering with what has materialized" ); break; } } }); } /// Drop the persisted trash listing's freshness stamp after we changed the /// trash ourselves. The next Trash page then *waits* for a fresh listing /// rather than painting one from before the mutation — the user just made the /// change and is about to look straight at it. fn invalidate_trash(&self) { let _ = self.db.clear_state(TRASH_SYNCED_MS); } /// Drop the persisted photos timeline's freshness stamp, so the next timeline /// read fetches rather than serving what it already has. fn invalidate_photos(&self) { let _ = self.db.clear_state(PHOTOS_SYNCED_MS); } /// Drop one folder's cached child listing (`rel` is mountpoint-relative), so /// the next `ListDir`/`readdir` re-enumerates it from the server. Backs /// [`CtlRequest::Refresh`] with a [`RefreshScope::Dir`] scope. fn refresh_dir(&self, rel: &Path) -> CoreResult<()> { let (ino, _uid) = self.resolve(rel)?; self.state().invalidate_listing(ino); Ok(()) } /// Parse wire uids (`volume~link`) into [`NodeUid`]s, rejecting the whole /// batch if any is malformed — a partial trash mutation is worse than none. fn parse_uids(uids: &[String]) -> CoreResult> { if uids.is_empty() { return Err(CoreError::invalid("no nodes given")); } uids.iter() .map(|u| parse_uid(u).ok_or_else(|| CoreError::invalid(format!("invalid uid: {u}")))) .collect() } /// Restore trashed nodes to the folders they were trashed from, together /// with the rest of the trashed tree they belong to. /// /// The trash is a flat list, but a user deletes a shape: a folder, and often /// items inside it that were trashed separately beforehand. Restoring one of /// those in isolation is not what anyone means by it — a folder would come /// back without the contents that were deleted with it, and a file would come /// back inside a folder that is still in the trash, which is to say nowhere /// the user can see. So a restore takes the whole connected piece: /// every trashed descendant of what was asked for, and every trashed ancestor /// above it. See [`expand_restore`] for the exact rule. /// /// The waves are restored shallowest first, because a node cannot be put back /// under a parent that is still trashed. /// /// The destination parents are read *before* the restore — a restored node /// reappears in a listing the daemon may already have cached, so each one is /// invalidated and re-enumerated on next access. fn restore(&self, uids: &[String]) -> CoreResult { let requested = Self::parse_uids(uids)?; // A listing we could not read is not a reason to refuse the restore: it // only costs the expansion, and the requested nodes still go back. let relations = self.db.trash_parents().unwrap_or_default(); let waves = expand_restore(uids, &relations); let waves: Vec> = if waves.is_empty() { vec![requested.clone()] } else { waves .iter() .map(|wave| { wave.iter() .filter_map(|uid| parse_uid(uid)) .collect::>() }) .filter(|wave: &Vec| !wave.is_empty()) .collect() }; let all: Vec = waves.iter().flatten().cloned().collect(); let parents: Vec = self .rt .block_on(self.drive.enumerate_nodes(&all)) .map_err(|e| CoreError::from_api(&e, "enumerate nodes"))? .into_iter() .filter_map(|n| n.parent_uid) .collect(); // Streamed per-node outcomes: a uid the server refuses (already restored, // gone) leaves the rest of the batch restored, and each node is unhidden // as *its* batch lands rather than after the last one — a daemon killed // half way through a large restore leaves local state describing what the // server actually did. Only a batch that restored nothing is an error the // caller can act on. let mut restored: Vec = Vec::with_capacity(all.len()); let mut first_error: Option = None; for wave in &waves { self.rt .block_on(async { let mut outcomes = std::pin::pin!(self.drive.restore_nodes_streaming(wave)); while let Some(item) = outcomes.next().await { let (uid, outcome) = item?; match outcome { Ok(()) => { self.hidden.lock().remove(&uid); restored.push(uid); } Err(e) => { warn!(%uid, error = %e, "restore failed for a node"); first_error.get_or_insert(e); } } } Ok::<(), ProtonError>(()) }) .map_err(|e| CoreError::from_api(&e, "restore"))?; } if restored.is_empty() && let Some(error) = first_error { return Err(CoreError::from_api(&error, "restore")); } // Keyed by uid, so it applies to every mount: a restored node reappears // in whichever mounts show its parent, not only the primary one. self.for_each_state(|st| { for parent in &parents { if let Some(&ino) = st.by_uid.get(parent) { st.invalidate_listing(ino); } } }); self.invalidate_trash(); Ok(restored.len()) } /// Permanently delete trashed nodes. Irreversible on the server; locally it /// drops any metadata and cached content the node still owns. fn delete_forever(&self, uids: &[String]) -> CoreResult { let parsed = Self::parse_uids(uids)?; // Streamed, and each node's local state is dropped as its batch lands. // A permanent delete is irreversible, so the useful property is that a // daemon interrupted mid-batch has already forgotten exactly the nodes // the server destroyed — never more, never fewer. let mut deleted = 0usize; let mut first_error: Option = None; self.rt .block_on(async { let mut outcomes = std::pin::pin!(self.drive.delete_nodes_streaming(&parsed)); while let Some(item) = outcomes.next().await { let (uid, outcome) = item?; match outcome { Ok(()) => { self.drop_local(std::slice::from_ref(&uid)); deleted += 1; } Err(e) => { warn!(%uid, error = %e, "permanent delete failed for a node"); first_error.get_or_insert(e); } } } Ok::<(), ProtonError>(()) }) .map_err(|e| CoreError::from_api(&e, "delete"))?; if deleted == 0 && let Some(error) = first_error { return Err(CoreError::from_api(&error, "delete")); } self.invalidate_trash(); Ok(deleted) } /// Permanently delete everything in the trash. The uids are listed first so /// the blobs of items trashed by *another* client — which this daemon may /// still hold in its cache — are reclaimed too, not just the ones it trashed. fn empty_trash(&self) -> CoreResult { let uids = self .rt .block_on(self.drive.enumerate_trash_node_uids()) .map_err(|e| CoreError::from_api(&e, "enumerate trash"))?; self.rt .block_on(self.client.empty_trash()) .map_err(|e| CoreError::from_api(&e, "empty trash"))?; self.drop_local(&uids); self.invalidate_trash(); Ok(uids.len()) } /// Forget every trace of nodes that no longer exist anywhere: their inode and /// DB row, and their cached content. fn drop_local(&self, uids: &[NodeUid]) { // These nodes are gone from the server, and a sync-folder fork showing // the same uid would otherwise keep serving them (`docs/BUGS.md` B74). self.for_each_state(|st| { for uid in uids { st.forget(uid); } }); for uid in uids { self.cache.evict(uid); self.evict_reader(uid); } } /// Create a folder named `name` under the mountpoint-relative `parent_rel`. /// Interns the new node directly so it shows up without a re-enumeration. fn create_folder(&self, parent_rel: &Path, name: &str) -> CoreResult { if name.is_empty() || name.contains('/') { return Err(CoreError::invalid(format!("invalid name: {name:?}"))); } let (pino, parent_uid) = self.resolve(parent_rel)?; self.ensure_children(pino) .map_err(|e| self.errno_error(e, "enumerate"))?; self.require_uid_writable(&parent_uid) .map_err(|error| self.errno_error(error, "create folder access"))?; let new_uid = self .rt .block_on( self.drive .create_folder(&parent_uid, name, Some(now_secs())), ) .map_err(|e| CoreError::from_api(&e, "create folder"))?; let node = self .fetch_minted_node(&new_uid, name) .ok_or_else(|| minted_but_unlisted(name))?; let mut st = self.state(); let ino = st.intern(pino, node); if let Some(kids) = st.children.get_mut(&pino) && !kids.contains(&ino) { kids.push(ino); } drop(st); self.flush_access_changes(); Ok(name.to_string()) } // ---- activity log ----------------------------------------------------- /// Append one entry to the activity log. Callable from any thread (the sync /// engine and the bulk uploader both log from background tasks). A failed /// write is logged and dropped: the feed is a record of work, never a reason /// to fail the work itself. pub(crate) fn log_activity( &self, kind: ActivityKind, target: impl Into, // `Display` rather than `Into` so a classified `CoreError` can be // logged as-is, without the caller flattening it first. detail: impl std::fmt::Display, ok: bool, ) { let entry = ActivityEntry { time: now_secs(), kind, target: target.into(), detail: detail.to_string(), ok, }; if let Err(e) = self.db.activity_add(&entry) { warn!(error = ?e, "could not record activity"); return; } // A sync pass is how a conflict copy reaches the listing, and the sweep // reports the ones it finds as conflicts. if matches!(entry.kind, ActivityKind::Conflict | ActivityKind::Sync) { self.events.publish(&[Topic::Activity, Topic::Conflicts]); } else { self.events.publish(&[Topic::Activity]); } } /// The recent activity, newest first, capped at `limit` entries. fn list_activity(&self, limit: usize) -> Vec { match self.db.activity_list(limit) { Ok(items) => items, Err(e) => { warn!(error = ?e, "could not read activity"); Vec::new() } } } // ---- live sync progress ----------------------------------------------- /// Start tracking a reconcile pass over `folder_id`, in [`SyncPhase::Scanning`]. pub(crate) fn progress_begin(&self, folder_id: i64) { self.sync_progress.lock().insert( folder_id, SyncProgress { phase: SyncPhase::Scanning, done: 0, total: 0, current: String::new(), }, ); } /// Apply `f` to a folder's live progress, if a pass is running for it. fn progress_update(&self, folder_id: i64, f: impl FnOnce(&mut SyncProgress)) { if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) { f(p); } self.sync_engine.moved(); } /// Set the number of items the scan expects to check, from the size of the /// last pass's baseline. Only an estimate — the folder may have grown since — /// but it turns the scan from an indeterminate pulse into a bar that moves, /// which is the difference between "it's stuck" and "it's working" on a folder /// whose walk takes minutes. pub(crate) fn progress_scan_total(&self, folder_id: i64, n: usize) { self.progress_update(folder_id, |p| p.total = n); } /// Note that the scan has checked one more item, named `name`. pub(crate) fn progress_scanned(&self, folder_id: i64, name: &str) { self.progress_update(folder_id, |p| { p.done += 1; p.current = name.to_string(); }); } /// Note that `n` more items have been queued for this pass, and that it has /// moved on from scanning to applying the diff. The scan's counts are dropped: /// they measured a different quantity (items checked, not items to apply), so /// carrying them over would start the applying bar at a meaningless fraction. pub(crate) fn progress_queued(&self, folder_id: i64, n: usize) { self.progress_update(folder_id, |p| { if p.phase == SyncPhase::Scanning { p.phase = SyncPhase::Applying; p.done = 0; p.total = 0; p.current.clear(); } p.total += n; }); } /// Note that work has started on `name` (shown as the pass's current item). pub(crate) fn progress_started(&self, folder_id: i64, name: &str) { self.progress_update(folder_id, |p| p.current = name.to_string()); } /// Note that one queued item finished, whether it succeeded or not. pub(crate) fn progress_finished(&self, folder_id: i64) { self.progress_update(folder_id, |p| { p.done += 1; p.current.clear(); }); } /// Stop tracking a pass — no progress is reported for the folder until the /// next [`progress_begin`](Self::progress_begin). pub(crate) fn progress_end(&self, folder_id: i64) { self.sync_progress.lock().remove(&folder_id); } } fn publish_virtual_root_in_listing( db: &Db, state: &mut State, parent_ino: u64, snapshot: &RootListingSnapshot, plan: VirtualRootPlan, ) -> Result<(), Errno> { if !snapshot.is_current(state, parent_ino) { return Err(Errno::EAGAIN); } db.publish_virtual_root(SHARED_WITH_ME_NAME, &plan.node) .map_err(|error| { error!(%error, "atomically publishing synthetic shared root failed"); Errno::EIO })?; let uid = plan.node.uid.clone(); state.share_access.insert(uid, Access::Viewer); let virtual_ino = state.intern_mem(parent_ino, plan.node); let published = reconcile_virtual_root_in_listing(state, parent_ino, snapshot, virtual_ino, plan.visible); debug_assert!( published, "snapshot cannot change while the state lock is held" ); Ok(()) } /// Update the synthetic dentry only while the parent listing is still known. /// /// Event invalidation can remove the listing after its real names were captured. /// Recreating it here would publish a synthetic-only partial snapshot as complete. fn reconcile_virtual_root_in_listing( state: &mut State, parent_ino: u64, snapshot: &RootListingSnapshot, virtual_ino: u64, visible: bool, ) -> bool { if !snapshot.is_current(state, parent_ino) { return false; } let Some(children) = state.children.get_mut(&parent_ino) else { return false; }; children.retain(|ino| *ino != virtual_ino); if visible { children.push(virtual_ino); } true } fn resolve_anywhere_with( raw_uid: &str, resident: impl FnOnce(&NodeUid) -> bool, mirrored: impl FnOnce(&NodeUid) -> pdfs_core::Result, ) -> CoreResult { let uid = parse_uid(raw_uid).ok_or_else(|| CoreError::invalid(format!("invalid uid: {raw_uid}")))?; if is_local_uid(&uid) || is_virtual_uid(&uid) { return Err(CoreError::invalid(format!("reserved uid: {raw_uid}"))); } if resident(&uid) { return Ok(uid); } if mirrored(&uid)? { return Ok(uid); } Err(CoreError::not_found(format!( "node is not present in any location: {raw_uid}" ))) } /// Map a [`MemberRole`] to its wire string. fn role_to_str(role: MemberRole) -> &'static str { match role { MemberRole::Viewer => "viewer", MemberRole::Editor => "editor", MemberRole::Admin => "admin", MemberRole::Inherited => "inherited", } } /// Parse a wire role string into a [`MemberRole`]. "inherited" is read-only and /// rejected here, since it cannot be sent when inviting or updating. fn role_from_str(role: &str) -> CoreResult { match role.to_lowercase().as_str() { "viewer" => Ok(MemberRole::Viewer), "editor" => Ok(MemberRole::Editor), "admin" => Ok(MemberRole::Admin), other => Err(CoreError::invalid(format!("invalid role: {other}"))), } } /// Map a device type to a display string. fn device_type_str(t: proton_drive_rs::DeviceType) -> &'static str { match t { proton_drive_rs::DeviceType::Windows => "Windows", proton_drive_rs::DeviceType::MacOs => "MacOs", proton_drive_rs::DeviceType::Linux => "Linux", } } /// Convert an SDK [`PublicLink`](proton_drive_rs::PublicLink) into the wire form. fn public_link_info(link: proton_drive_rs::PublicLink) -> PublicLinkInfo { PublicLinkInfo { id: link.public_link_id, url: link.url, role: role_to_str(link.role).to_string(), expires: link.expiration_time, has_password: link.has_custom_password, created: link.creation_time, } } /// Parse a `volume~link` uid display string back into a [`NodeUid`]. Front-ends /// receive uids as strings over the control socket and pass them back verbatim. pub(crate) fn parse_uid(s: &str) -> Option { let (vol, link) = s.split_once('~')?; if vol.is_empty() || link.is_empty() || link.contains('~') { return None; } Some(NodeUid::new(VolumeId::from(vol), LinkId::from(link))) } /// Current wall-clock time as epoch seconds (0 if the clock is before the epoch). fn now_secs() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs() as i64) .unwrap_or(0) } /// Current wall-clock time as epoch milliseconds, the resolution `pending_op` /// timestamps and backoff deadlines are kept in. fn now_millis() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as i64) .unwrap_or(0) } /// Parse a [`NodeUid`] back from its `Display` form (`volume~link`), which is /// how one is persisted in `pending_op.uid` and a [`StagedWrite`] sidecar. The /// SDK has no `FromStr`, and neither id contains a `~`. fn parse_node_uid(s: &str) -> Option { parse_uid(s) } /// The uid a node made on this machine goes by until it lands, so it can be /// interned, listed and written to before the server has ever heard of it: /// its row's local id, as [`pdfs_core::db::local_uid`] /// has it. fn local_node_uid(lid: i64) -> NodeUid { NodeUid::new(VolumeId::from(LOCAL_VOLUME), LinkId::from(lid.to_string())) } /// Whether this node exists only on this machine, so far. Such a uid is /// meaningless to the API and must never be sent to it. fn is_local_uid(uid: &NodeUid) -> bool { uid.volume_id.as_str() == LOCAL_VOLUME } /// [`is_local_uid`] for a uid in its persisted `Display` form. fn is_local_uid_str(s: &str) -> bool { s.split_once('~') .is_some_and(|(vol, _)| vol == LOCAL_VOLUME) } /// The API's response code for a failed call, when it failed *at* the API. /// /// Takes `&dyn Error` so it reads a [`ProtonError`] equally well through the /// boxes the drain deals in, where the concrete type survives but the static one /// does not. `None` covers both "not an API error at all" (a transport failure, /// which is what being offline looks like) and "not a `ProtonError`". fn api_code(e: &(dyn std::error::Error + 'static)) -> Option { match e.downcast_ref::() { Some(ProtonError::Api(api)) => Some(api.code), _ => None, } } /// How often [`Core::ensure_children`] takes a listing that changed while it /// was being published. const LISTING_ATTEMPTS: usize = 4; /// How often a rename or move that Drive calls out of date is retried, and how /// long apart. const OUT_OF_DATE_RETRIES: u32 = 20; const OUT_OF_DATE_RETRY_DELAY: Duration = Duration::from_millis(500); /// Run a remote rename or move, retrying while Drive answers /// `InvalidRequirements`. /// /// Both send the node's current name hash. The SDK takes it from its entity /// cache, which still holds the old hash when the node changed since it was /// cached, and Drive then rejects the request as "out of date" (docs/BUGS.md /// B111, B121). `wait` drops that entry before the next attempt. Any other /// error returns at once. fn retry_out_of_date( mut call: impl FnMut() -> Result<(), ProtonError>, mut wait: impl FnMut(), ) -> Result<(), ProtonError> { let mut attempts = 0u32; loop { match call() { Err(e) if api_code(&e) == Some(ResponseCode::InvalidRequirements) && attempts < OUT_OF_DATE_RETRIES => { attempts += 1; debug!(attempts, error = %e, "Drive called the request out of date; retrying"); wait(); } result => return result, } } } /// Whether a call failed because the name it asked for is already in use. /// /// The queue makes this reachable in a way the synchronous path never was: a /// mutation queued while offline is applied against a server that may have /// gained a file of that name in the meantime. fn is_already_exists(e: &(dyn std::error::Error + 'static)) -> bool { api_code(e) == Some(ResponseCode::AlreadyExists) } /// Whether a call failed because the node it addressed is not there. /// The error for a folder Drive created but did not hand back, where there is /// no queue to adopt it later. Saying it was created keeps the caller from /// taking a retry's "already exists" for someone else's folder. fn minted_but_unlisted(name: &str) -> CoreError { CoreError::remote(format!( "created folder {name}, but Drive did not list it yet; it shows on the next refresh" )) } fn is_gone(e: &(dyn std::error::Error + 'static)) -> bool { api_code(e) == Some(ResponseCode::DoesNotExist) } /// A variant of `name` to fall back on when the remote already has that name. /// /// Deliberately the same shape the sync engine uses for its conflict copies /// (`sync.rs`, `conflict_path`), so the two halves of the product name the same /// situation the same way and a user only has to learn it once. fn conflict_name(name: &str, stamp: i64) -> String { conflict_name_as(name, conflict_label(), stamp) } /// [`conflict_name`] with the machine's label given. fn conflict_name_as(name: &str, label: &str, stamp: i64) -> String { let path = Path::new(name); let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(name); let tag = conflict_tag(label, stamp, 0); match path.extension().and_then(|s| s.to_str()) { Some(ext) => format!("{stem} (sync-conflict {tag}).{ext}"), None => format!("{stem} (sync-conflict {tag})"), } } /// This machine's name as its conflict copies carry it (`docs/BUGS.md` B251). /// /// Two computers syncing one folder that each keep a copy of the same file in /// the same second would otherwise make the same name on Drive, and nobody /// could tell which computer a copy came from. Read once, from the hostname. fn conflict_label() -> &'static str { static LABEL: OnceLock = OnceLock::new(); LABEL.get_or_init(|| conflict_label_of(&this_hostname())) } /// `host` as one token: anything but ASCII letters, digits, `-`, `_` and `.` /// becomes `-`, so the label holds no space for `conflict_base_name` to trip /// on and no `/` or `)` to break the name. fn conflict_label_of(host: &str) -> String { let label: String = host .chars() .take(32) .map(|c| { if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') { c } else { '-' } }) .collect(); label.trim_matches('-').to_string() } /// The inside of a conflict copy's marker: `