Files
proton-drive-linux/crates/pdfs-fuse/src/lib.rs
T
nvrl b093010b64 Merge branch 'fix/never-synced-label' into fix/mount-failure-surfaced
# Conflicts:
#	docs/BUGS.md
#	po/de.po
#	po/en_GB.po
#	po/es.po
#	po/fr.po
#	po/it.po
#	po/nl.po
#	po/pdfs.pot
#	po/pl.po
#	po/pt_BR.po
2026-10-31 19:26:34 +01:00

10123 lines
413 KiB
Rust

//! On-demand FUSE filesystem for Proton Drive.
//!
//! Phase 1 is a read-only placeholder mount: directories are enumerated lazily
//! from the remote on first access and file content is hydrated on `read` via
//! [`ProtonDriveClient::download_range`]. Nothing is mirrored to local disk;
//! every byte is fetched on demand.
//!
//! Phase 2 adds live remote sync: a background task polls the volume event
//! cursor ([`ProtonDriveClient::enumerate_events`]) and pushes invalidations
//! into the kernel via a fuser [`Notifier`], so the cache TTL can be long while
//! remote changes still show up promptly.
//!
//! Phase 3 makes the mount writable. Each file opened for writing gets a
//! `WriteHandle` whose buffer accumulates the full plaintext; on flush/release
//! the buffer is sealed as a new revision via
//! [`ProtonDriveClient::upload_new_revision`] (the SDK uploads whole revisions,
//! not byte ranges). New files are created empty up front so they get a real
//! uid; namespace ops map to `create_folder`, `trash_nodes`, `rename_node` and
//! `move_node`.
//!
//! Phase 4 adds Files-On-Demand pinning. A control socket (see `control`)
//! lets the CLI pin/unpin files; a pinned file's plaintext is downloaded once
//! into the on-disk [`ContentCache`] and every later `read` is served from disk
//! instead of the network. Writes and remote events evict the cache so it never
//! goes stale.
//!
//! Reads of unpinned files no longer hit the network per call: `Core::read_range`
//! fetches and caches [`BLOCK_SIZE`]-aligned blocks, so sequential/sparse reads
//! reuse the on-disk block cache. Writes are disk-backed: each `WriteHandle`
//! stages authored bytes in a scratch file and tracks them with an `Intervals`
//! set, so a multi-GiB write never buffers in RAM and only the untouched
//! remainder of the file is pulled from the remote — lazily, at commit.
use parking_lot::{Condvar, Mutex};
use std::collections::{HashMap, HashSet, VecDeque};
use std::ffi::OsStr;
use std::fs::File;
use std::io::Write;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::FileExt;
use std::os::unix::net::UnixListener;
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use fuser::ReplyXattr;
use fuser::{
AccessFlags, BackgroundSession, BsdFileFlags, Config, CopyFileRangeFlags, Errno, FileAttr,
FileHandle, FileType, Filesystem, FopenFlags, Generation, INodeNo, InitFlags, IoctlFlags,
KernelConfig, LockOwner, MountOption, Notifier, OpenAccMode, OpenFlags, RenameFlags, ReplyAttr,
ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyIoctl, ReplyLseek,
ReplyOpen, ReplyStatfs, ReplyWrite, Request, Session, TimeOrNow, WriteFlags,
};
use futures::StreamExt as _;
use pdfs_core::batch;
use pdfs_core::cache::{BLOCK_SIZE, Baseline, BlockGeometry, BlockSpan, ContentCache, StagedWrite};
use pdfs_core::config::{AppDirs, SweepMode};
use pdfs_core::control::{
ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo,
SearchFilters, SearchHit, SearchSource, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
ThumbnailBuildStatus, Topic, TransferDirection,
};
use pdfs_core::db::{
Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_UNTIL, PendingOp,
PublishedSharedRoot, RenameMeta, StoredNode, StoredSyncFolder, StoredTrash,
};
use pdfs_core::localindex;
use pdfs_core::search::relevance_score;
use pdfs_core::syncignore::is_transient_name;
use pdfs_core::{Access, CoreError, CoreResult, access_for, perm_bits};
use proton_drive_rs::proton_sdk::api::ResponseCode;
use proton_drive_rs::proton_sdk::error::ProtonError;
use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, ShareId, VolumeId};
use proton_drive_rs::{
AvailableNameTarget, DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, NodeMoveItem,
ProtonDriveClient, ProtonPhotosClient, SharedWithMeItem, ThumbnailType,
};
mod albums;
mod background;
mod conflicts;
mod control;
use control::run_control_socket;
mod devices;
mod diagnostics;
mod drain;
mod drive;
use drive::{DriveApi, RevisionRead};
mod events;
mod filesystem;
mod link;
pub use filesystem::ProtonFs;
mod mount;
mod pause;
mod photos;
mod profile;
mod queue;
mod reads;
mod redate;
mod relocate;
mod revisions;
mod sharing;
mod shutdown;
#[cfg(test)]
mod sim;
mod state;
mod supervisor;
mod sweep;
mod sync;
mod systemd;
mod takeout;
mod transfers;
mod upload;
mod r#virtual;
mod workers;
use background::{forget_and_notify, run_event_sync, run_local_index, run_photos_event_sync};
use link::BATCH_CALL_TIMEOUT;
pub(crate) use mount::is_stale_mount;
pub use mount::{MountOptions, MountOutcome, mount};
use mount::{
SecondaryInsertRejection, SecondaryMount, clear_stale_mount, fuse_connection_id, spawn_session,
};
use reads::{BlockFlight, BlockRing, PREFETCH_BUDGET, Prefetch, ReaderSlot, STREAM_BYPASS_MIN};
use state::{Entry, Intervals, PendingRevision, State, StateGuard, WriteHandle};
pub use systemd::{idle_sleep, status as notify_status};
use tracing::{debug, error, info, warn};
use transfers::{CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry};
use r#virtual::{
SharedListingPlan, SharedRefreshDeadlines, disambiguate_shared_names, is_own_or_virtual_uid,
is_primary_root_listing, is_virtual_uid, listing_needs_refresh, refresh_generation_is_current,
shared_listing_plan, shared_with_me_uid, virtual_node, virtual_root_name,
};
use workers::{FUSE_WORKERS, Lane, Workers};
/// Attribute/entry cache lifetime handed back to the kernel. Long because the
/// Phase 2 event poller actively invalidates changed inodes; without a remote
/// change this is how long the kernel may serve stale metadata.
const TTL: Duration = Duration::from_secs(30);
/// How long a `statfs(2)` answer is served from [`Core::quota`] before the
/// account figures are refetched. See [`Core::account_quota_cached`].
const QUOTA_TTL: Duration = Duration::from_secs(60);
/// Block size reported to the kernel and to `statfs(2)`.
///
/// coreutils and most copy tools size their I/O buffers from `st_blksize`, and
/// the fuser default of 512 makes them issue reads two orders of magnitude
/// smaller than the [`BLOCK_SIZE`] this filesystem actually fetches in. 1 MiB is
/// the largest value that stays well inside the kernel's per-request limit.
const REPORTED_BLKSIZE: u32 = 1 << 20;
/// Ceiling on in-flight background requests (readahead, writeback). The kernel
/// default of 16 caps how much of a sequential read the kernel will run ahead of
/// the application, which on a high-latency link is the whole ballgame.
const MAX_BACKGROUND: u16 = 64;
/// Longest single path component, as reported by `statfs(2)`. The kernel's own
/// dirent limit; Proton's is not lower.
const MAX_NAME_LEN: u32 = 255;
/// How often the background task polls the remote event cursor.
const POLL_INTERVAL: Duration = Duration::from_secs(10);
/// Retry backoff for a queued upload, doubling per attempt between these. The
/// floor is short because the common failure is a brief network blip; the
/// ceiling keeps a persistently failing op from spinning.
const DRAIN_BACKOFF_MIN: Duration = Duration::from_secs(2);
const DRAIN_BACKOFF_MAX: Duration = Duration::from_secs(300);
/// How long the drain worker sleeps when it has nothing due. It is woken
/// directly on a new write or a reconnect, so this only bounds how late a
/// backoff can fire.
const DRAIN_IDLE_POLL: Duration = Duration::from_secs(30);
/// How long the mount waits before asking again for a node it has just created
/// when Drive does not list it yet (`Core::fetch_minted_node`).
const MINTED_READ_BACK_DELAYS: [Duration; 2] = [Duration::from_millis(250), Duration::from_secs(1)];
/// Grace period before a queued revision becomes eligible for draining.
///
/// Tools like aria2c preallocate a file (truncate to target size) and then write
/// the real content, sometimes across separate open/close cycles. Without a
/// grace period the first close drains immediately, uploading the preallocated
/// (mostly-zero) content; the second close then finds its baseline stale and
/// creates a conflict copy of itself. Holding the op for a short window gives
/// the follow-up write time to supersede it.
const DRAIN_REVISION_DEBOUNCE: Duration = Duration::from_secs(2);
/// Ceiling on the *adaptive* part of that grace period.
///
/// The fixed 2 seconds is enough for a preallocate-then-write tool, and wrong
/// for a file that takes half a minute to upload: an editor saving every ten
/// seconds queues a write, watches it start uploading, and supersedes it — each
/// save uploading bytes that the next one replaces, and never catching up.
/// Widening the debounce toward how long this node's last upload actually took
/// makes the supersede happen in the queue, where it is free, instead of on the
/// wire. Bounded so a file on a slow link still reaches Drive in a minute rather
/// than deferring indefinitely, and so a one-off stall cannot park a node.
const DRAIN_REVISION_DEBOUNCE_MAX: Duration = Duration::from_secs(60);
/// How many nodes' measured upload times to remember for that. Bounds a map
/// that would otherwise grow with every file ever written; the debounce falls
/// back to [`DRAIN_REVISION_DEBOUNCE`] for anything evicted, which is the
/// behaviour a node that has never been uploaded gets anyway.
const UPLOAD_TIME_MEMORY: usize = 512;
/// Threads draining the pending-op queue.
///
/// One meant a single 10 GiB upload blocked every queued rename, trash and
/// small write behind it. Ordering only has to hold per node — enforced by the
/// claim query, not by the thread count — so the rest is throughput. Most ops
/// spend their time waiting: a create waits for Drive to list the node it made
/// before it reads it back. Three workers left a thousand small creates taking
/// 0.3 s each on a LAN and 2.5 s on Wi-Fi, one after another. Uploads cannot
/// crowd each other out by number: the SDK caps the blocks in flight across
/// the client.
const DRAIN_WORKERS: usize = 16;
/// How long a staged blob that names no node is kept before it is retired.
///
/// Staging is exempt from the cache budget because it holds the only copy of
/// writes that have not been uploaded, so nothing there is ever evicted to
/// reclaim space. That is right for a blob that still describes something. It is
/// wrong for one whose sidecar is missing or names a `local~` placeholder whose
/// create is gone: no future run can address it, no UI accounts for it, and it
/// occupies disk for the life of the machine. Real instance: 95 such blobs,
/// ~445 MiB, the oldest two months old, re-announced on every mount.
///
/// A month is far longer than any transfer or any plausible "I will look at that
/// later", and the retirement is announced per blob with its size, so the one
/// case that matters — bytes a human still wants — has a month of warnings and a
/// path to go and fetch them from.
const STAGING_ORPHAN_RETAIN: Duration = Duration::from_secs(30 * 24 * 60 * 60);
/// First and longest delay between probes for the network coming back after an
/// offline mount. Doubles from min to max: a laptop shut in
/// a bag is the common case, so the steady state must be cheap, while a brief
/// blip should still recover in seconds.
const ONLINE_PROBE_MIN: Duration = Duration::from_secs(5);
const ONLINE_PROBE_MAX: Duration = Duration::from_secs(300);
/// How long the persisted photos timeline stays good before a page request
/// revalidates it. The SDK hands back the whole timeline at once, so it is stored
/// in the DB and every page is sliced from there; a stale one is still served
/// immediately and refreshed in the background.
///
/// Short, because the cost of a stale timeline is a photo the user deleted on
/// their phone still sitting in the gallery, and the cost of the check is one
/// listing call that only runs when somebody is actually looking at the page.
const TIMELINE_TTL: Duration = Duration::from_secs(60);
/// How many photo nodes are resolved per [`ProtonPhotosClient::enumerate_nodes`]
/// call when enriching a refreshed timeline with names and media types (for the
/// Photos / Videos / Raw split). Batched so a large library is a handful of
/// round-trips rather than one request per photo, and bounded so a single call
/// never asks the server to decrypt the whole library at once.
///
/// 150 because that is the SDK's own `MAX_BATCH_COUNT`: a larger chunk is split
/// there anyway, into one full request and a ragged tail.
const TIMELINE_ENRICH_CHUNK: usize = 150;
/// The same, for the persisted trash listing. Shorter, because the trash is the
/// one listing a user changes and then immediately looks at — though our own
/// mutations also invalidate it outright, so this only covers other clients.
const TRASH_TTL: Duration = Duration::from_secs(60);
/// Shared roots and foreign-volume folders have no event manager. Revalidate
/// them on access while online; stale persisted listings remain usable offline.
const SHARED_LISTING_TTL: Duration = Duration::from_secs(60);
/// How many trashed nodes are materialized per
/// [`ProtonDriveClient::enumerate_nodes`] call, and persisted as a batch. The
/// trash of an account that has been through a conflict storm runs to thousands
/// of nodes, each costing an S2K unlock, so a single all-or-nothing call can
/// take minutes and lose everything it decrypted if it fails. Same value as the
/// SDK's own `MAX_BATCH_COUNT`: one chunk is exactly one request.
const TRASH_MATERIALIZE_CHUNK: usize = 150;
/// How long a `ListTrash` request waits for a first-ever refresh before
/// answering with whatever has materialized so far.
///
/// The refresh keeps running in the background either way — this bounds the
/// *reply*, not the work. It has to stay well under the front-ends' 120 s
/// control-socket read timeout (`pdfs_core::control`), because a request that
/// outlives that timeout is a hang from the user's side: the client gives up,
/// the user asks again, and the daemon accumulates another refresh.
const TRASH_FIRST_WAIT: Duration = Duration::from_secs(20);
/// How long one server call of a trash refresh may take before the refresh
/// gives up.
///
/// Refreshes are single-flight, so a call that never returns wedges the trash
/// listing for the life of the daemon: every later request joins the stuck
/// refresh, times out, and answers with the listing from before it. That hid
/// a just-trashed folder from a permanent delete for as long as the daemon ran
/// (`docs/BUGS.md` B103). One call is one request, so this is generous.
const TRASH_CALL_TIMEOUT: Duration = Duration::from_secs(120);
/// `sync_state` keys for the freshness stamps of the two persisted listings, and
/// for whether the account has a photos volume at all (so an account without one
/// doesn't re-ask the server on every page request).
const PHOTOS_SYNCED_MS: &str = "photos_synced_ms";
const PHOTOS_AVAILABLE: &str = "photos_available";
/// Freshness stamp of the album listing, and of one album's contents (suffixed
/// with the album uid). Separate from the timeline's: albums are enumerated by
/// their own endpoints, and an album is only fetched when it is opened.
const ALBUMS_SYNCED_MS: &str = "albums_synced_ms";
const ALBUM_SYNCED_PREFIX: &str = "album_synced_ms:";
const TRASH_SYNCED_MS: &str = "trash_synced_ms";
/// Set when a trash refresh finishes and cleared when one starts, so it is only
/// present while the stored listing is the whole trash as of some moment. A node
/// that a refresh finds and that listing lacks went to the trash since then; one
/// found after an interrupted refresh may have been there all along. Unlike
/// [`TRASH_SYNCED_MS`], our own mutations leave it alone.
const TRASH_COMPLETE_MS: &str = "trash_complete_ms";
const SHARED_WITH_ME_NAME: &str = "shared_with_me_name";
const SHARED_WITH_ME_SYNCED_MS: &str = "shared_with_me_synced_ms";
const SHARED_FOLDER_SYNCED_PREFIX: &str = "shared_folder_synced_ms:";
/// How stale the local-file index may get before the background scanner rebuilds
/// it. A rescan is a full walk of `$HOME`, so this trades index freshness against
/// disk churn; the scanner also always rebuilds once per daemon start when the
/// index is older than this.
const LOCAL_INDEX_TTL: Duration = Duration::from_secs(15 * 60);
/// How often the scanner thread wakes to check whether the index went stale.
const LOCAL_INDEX_CHECK: Duration = Duration::from_secs(60);
/// The FUSE root inode is always 1.
const ROOT_INO: u64 = 1;
/// Parent inode for a persisted node whose parent row is missing from the DB.
/// No folder carries this inode, so such a node is listed by nobody and stays
/// inert until a live enumeration re-parents it.
const ORPHAN_INO: u64 = 0;
/// Extended attribute exposing the small server-side thumbnail of a file.
const XATTR_THUMBNAIL: &str = "user.proton.thumbnail";
/// Extended attribute exposing the larger server-side preview image of a file.
const XATTR_PREVIEW: &str = "user.proton.preview";
/// How many "this file has no thumbnail" answers [`Core::thumbnail`] remembers
/// before dropping the lot and re-learning them. Sized to cover a large browsing
/// session; each entry is a uid, a type tag and an mtime.
const MAX_THUMBNAIL_MISSES: usize = 8192;
/// How many nodes one size-upgrade request covers.
///
/// Matches the SDK's own `MAX_BATCH_COUNT`, so a chunk is exactly one request:
/// chunking smaller would add round trips, larger would be split anyway and
/// delay the waiters this chunking exists to release (`docs/BUGS.md` B14).
const SIZE_UPGRADE_CHUNK: usize = 150;
/// How many folders may have a size upgrade in flight at once.
///
/// Each one owns a thread and an API request stream, and the single-flight only
/// deduplicates *within* a folder — so without this a recursive listing scaled
/// both with the number of folders it walked. Eight keeps an interactive `ls -l`
/// (one folder, sometimes a couple) entirely unaffected while putting a ceiling
/// on the recursive case. A folder past the cap is queued, not dropped: one of
/// the running threads takes it over when its own batch ends (`docs/BUGS.md` B100).
const MAX_SIZE_UPGRADES: usize = 8;
#[derive(Clone, Debug, Eq, PartialEq)]
struct RootListingSnapshot {
children: Vec<RootListingChild>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct RootListingChild {
ino: u64,
entry: Option<RootListingEntry>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct RootListingEntry {
uid: NodeUid,
name: String,
parent: u64,
trashed: bool,
unlinked: bool,
}
impl RootListingSnapshot {
fn capture(state: &State, parent_ino: u64) -> Option<Self> {
let children = state.children.get(&parent_ino)?;
Some(Self {
children: children
.iter()
.map(|ino| RootListingChild {
ino: *ino,
entry: state.entries.get(ino).and_then(|entry| {
(!is_virtual_uid(&entry.uid)).then(|| RootListingEntry {
uid: entry.uid.clone(),
name: entry.node.name.clone(),
parent: entry.parent,
trashed: entry.node.trashed,
unlinked: entry.unlinked,
})
}),
})
.collect(),
})
}
fn real_names(&self) -> HashSet<String> {
self.children
.iter()
.filter_map(|child| child.entry.as_ref())
.map(|entry| entry.name.clone())
.collect()
}
fn is_current(&self, state: &State, parent_ino: u64) -> bool {
Self::capture(state, parent_ino).as_ref() == Some(self)
}
}
struct VirtualRootPlan {
node: Node,
visible: bool,
}
/// The sync folders a search hit can be opened through, as
/// (Drive-relative root path, local directory) pairs.
///
/// Built once per search by [`Core::search_roots`]; resolving a hit against it
/// is string arithmetic over the paths the index already carries, with no
/// further queries.
struct SearchRoots {
roots: Vec<(String, PathBuf)>,
}
impl SearchRoots {
/// The absolute local path for a hit at Drive-relative `path`, through the
/// most specific root covering it. A node can sit below several configured
/// roots, so the one leaving the shortest descendant path wins.
fn resolve(&self, path: &str) -> Option<String> {
self.roots
.iter()
.filter_map(|(root, local)| {
let relative = relative_to(root, path)?;
Some((relative.split('/').count(), local.join(relative)))
})
.min_by_key(|(depth, _)| *depth)
.map(|(_, path)| path.to_string_lossy().into_owned())
}
}
/// Negative thumbnail knowledge has two different authorities. A remote miss
/// only says Proton has no thumbnail to download; it must never suppress local
/// generation. A local miss says this exact file revision was successfully
/// inspected and could not be decoded. Both maps are bounded independently.
#[derive(Default)]
struct ThumbnailMissCaches {
remote: HashMap<(NodeUid, i32), i64>,
local: HashMap<(NodeUid, i32), i64>,
}
impl ThumbnailMissCaches {
fn remote_contains(&self, key: &(NodeUid, i32), tag: i64) -> bool {
self.remote.get(key) == Some(&tag)
}
fn local_contains(&self, key: &(NodeUid, i32), tag: i64) -> bool {
self.local.get(key) == Some(&tag)
}
fn remember_remote(&mut self, key: (NodeUid, i32), tag: i64) {
Self::remember(&mut self.remote, key, tag);
}
fn remember_local(&mut self, key: (NodeUid, i32), tag: i64) {
Self::remember(&mut self.local, key, tag);
}
fn forget_local(&mut self, key: &(NodeUid, i32)) {
self.local.remove(key);
}
fn remember(map: &mut HashMap<(NodeUid, i32), i64>, key: (NodeUid, i32), tag: i64) {
if map.len() >= MAX_THUMBNAIL_MISSES && !map.contains_key(&key) {
map.clear();
}
map.insert(key, tag);
}
}
/// `path` expressed relative to `root`, or `None` if it is not below it. The
/// root itself resolves to the empty path; an empty root is the mount itself
/// and covers everything.
fn relative_to<'a>(root: &str, path: &'a str) -> Option<&'a str> {
if root.is_empty() {
return Some(path);
}
if path == root {
return Some("");
}
path.strip_prefix(root)?.strip_prefix('/')
}
#[derive(Clone)]
struct Core {
/// Drive as the sync core uses it: listings, events, creates, uploads,
/// downloads, renames, moves and trashes. The real client in production, an
/// in-memory Drive in the simulation tests. See [`drive`].
drive: Arc<dyn DriveApi>,
/// The concrete client, for what [`DriveApi`] does not cover: sharing,
/// Photos, devices, invitations and public links.
client: ProtonDriveClient,
rt: tokio::runtime::Handle,
/// The daemon's My Files root. Forked on-demand sessions retain this value
/// so ownership checks never mistake their own mount root for account scope.
primary_root_uid: NodeUid,
/// False only for a forked on-demand device mount. The synthetic shared
/// directory belongs exclusively to the primary My Files mount.
primary: bool,
/// The one tree every mount of the daemon serves from.
state: Arc<Mutex<State>>,
/// The tree inode this mount's session is rooted at, set when the session
/// is built ([`ProtonFs::new`]). Each fork has its own cell.
root: Arc<OnceLock<u64>>,
cache: Arc<ContentCache>,
/// Open [`RevisionReader`]s keyed by node, so the block fetches of a file
/// resolve its keys and block table once instead of once per block.
/// Validated by `(mtime, size)` exactly like the content cache, and bounded
/// by [`MAX_OPEN_READERS`].
readers: Arc<Mutex<HashMap<NodeUid, ReaderSlot>>>,
/// Decrypted blocks held in memory, so the ~32 kernel reads a 4 MiB block is
/// delivered in cost the block once — a download for a streaming file, a
/// disk read for a cached one. See [`BlockRing`].
block_ring: Arc<Mutex<BlockRing>>,
/// Block fetches in flight, so concurrent readers of one block (demand read,
/// kernel read-ahead, our own prefetch) share a single download and decrypt.
/// See [`BlockFlight`].
block_flight: Arc<BlockFlight>,
/// One lock per file being repaired by [`Core::repair_block`], so the
/// concurrent block reads that all discover the same untrustworthy block
/// table share a single whole-file download instead of one each.
content_repairs: Arc<tokio::sync::Mutex<HashMap<NodeUid, Arc<tokio::sync::Mutex<()>>>>>,
/// Per-file sequential-read detection driving [`Core::prefetch`].
prefetch: Arc<Prefetch>,
/// Permits bounding speculative block fetches, so prefetch cannot queue in
/// front of a read someone is waiting on.
prefetch_budget: Arc<tokio::sync::Semaphore>,
/// Threads that serve the FUSE handlers which touch the network, so the
/// session's dispatch loop stays free to answer cheap metadata calls while a
/// cold read is on the wire. See [`Workers`].
workers: Arc<Workers>,
/// Unified SQLite metadata cache: the persistence layer behind the in-memory
/// `State` maps. Every mutation writes through here, and the maps rehydrate
/// from it on mount.
db: Arc<Db>,
/// Serializes only shared-list publication and access-loss invalidation.
/// Network calls happen before taking it.
shared_publication: Arc<Mutex<()>>,
/// Invalidates in-flight shared responses when access/list events arrive.
shared_generation: Arc<AtomicU64>,
/// Successful refreshes suppress retries even if persisting their timestamp
/// fails. Shared across primary/fork clones and cleared by access events.
shared_refresh_deadlines: Arc<Mutex<SharedRefreshDeadlines>>,
/// False while the API is unreachable and we are serving the cached tree.
/// Set by the probe thread; read by front-ends through
/// `Response::Status` so the UI can say so rather than leaving the user to
/// infer it from a wall of EIO.
online: Arc<AtomicBool>,
/// When the link went down and a condvar for its coming back; the other
/// half of `online` (see [`link`]).
link: Arc<link::Link>,
/// When the user paused syncing, the Unix second it resumes by itself;
/// [`pause::PAUSED_INDEFINITELY`] for "until resumed", `0` when not paused.
/// Persisted, so a pause survives a restart (see [`Core::sync_paused`]).
sync_paused_until: Arc<AtomicI64>,
/// Writes accepted from the kernel but not yet uploaded, keyed by node.
/// The in-memory face of the `pending_op` table, from
/// which it is rebuilt on mount.
///
/// Two things read it: [`Core::read_range`], because until the op drains the
/// staged blob *is* the file's content and the remote still holds the old
/// revision; and the drain worker, which performs the uploads.
pending: Arc<Mutex<HashMap<NodeUid, PendingRevision>>>,
/// Nodes removed through this daemon which a briefly stale remote listing
/// may still return. Uids are not reused; an explicit restore clears one.
hidden: Arc<Mutex<HashSet<NodeUid>>>,
/// The folder and name each create on the wire was sent with, by op id.
/// Drive holds that name until the create lands and a rename or trash
/// queued behind it frees it again.
creating: Arc<Mutex<HashMap<i64, (NodeUid, String)>>>,
/// The SHA-1 of the last revision this daemon uploaded to a file and could
/// not read back, so its revision id is not known (`docs/BUGS.md` B157).
sealed_unread: Arc<Mutex<HashMap<NodeUid, String>>>,
/// The node an upload has just put on Drive, from when the drain lets go
/// of its blob until the tree has the node (`docs/BUGS.md` B170).
landed: Arc<Mutex<HashMap<NodeUid, Node>>>,
/// Creates that landed but could not be read back, by the uid Drive gave
/// them, with the placeholder's. The drain adopts one before it sends
/// anything else for it.
unadopted: Arc<Mutex<HashMap<NodeUid, NodeUid>>>,
/// Nudges the drain workers: set true and notify to have them re-examine
/// the queue instead of waiting out their backoff.
drain_wake: Arc<(Mutex<bool>, Condvar)>,
/// The daemon's stop signal, shared with every background loop this `Core`
/// was cloned into so teardown can end and join them (`docs/BUGS.md` B44).
shutdown: Arc<shutdown::Shutdown>,
/// How long this node's last upload actually took, in milliseconds, with
/// the time it was measured for eviction. Feeds
/// [`Core::revision_debounce`]: a file that takes 30 seconds to send needs
/// a grace period on that scale, or every save supersedes an upload that
/// was already on the wire. Bounded by [`UPLOAD_TIME_MEMORY`].
upload_times: Arc<Mutex<HashMap<String, (u64, i64)>>>,
/// Cancellation flags for uploads currently on the wire, keyed by node.
///
/// Set when a newer revision of the same file is queued: the bytes in
/// flight have just been superseded, and the only thing finishing them
/// achieves is spending the user's uplink on a revision the next op
/// replaces. Read by the upload's own reader ([`CountingReader`]), which is
/// the one thing the SDK calls often enough to notice.
upload_cancel: Arc<Mutex<HashMap<NodeUid, Arc<AtomicBool>>>>,
/// Uploads whose op is retired but whose bytes the content cache has not
/// adopted yet, because the revision just sealed is still being read back.
/// The status counts them as pending, so an empty queue means every file
/// written through the mount reads without the network (`docs/BUGS.md`
/// B119).
landing_uploads: Arc<AtomicU64>,
/// True while a background refresh of the photos timeline (resp. the trash) is
/// already running, so a burst of page requests against a stale listing kicks
/// off one refresh rather than one per request.
timeline_refreshing: Arc<AtomicBool>,
/// How far the running timeline refresh has got resolving photo metadata,
/// as `(done, total)` photos; `None` when no refresh is resolving any. The
/// Places view shows it while a library's locations are first read.
timeline_progress: Arc<Mutex<Option<(usize, usize)>>>,
/// The same, for the album listing.
albums_refreshing: Arc<AtomicBool>,
trash_refreshing: Arc<AtomicBool>,
/// True while the similar-photo pass hashes thumbnails (see
/// [`Core::photo_similar`]), so asking again doesn't start a second one.
similar_hashing: Arc<AtomicBool>,
/// Fires whenever a trash refresh persists a batch or finishes, so a
/// `ListTrash` request waiting on a first-ever refresh wakes on progress
/// instead of polling or blocking for the whole run. See
/// [`Core::await_trash_refresh`].
trash_progress: Arc<tokio::sync::Notify>,
/// Conflict copies the sweep has already checked against the activity log
/// this run, so each pass does not ask the database again. The log itself
/// is what keeps a copy from being flagged again after a restart. See
/// [`Core::run_conflict_sweep_loop`] and `docs/BUGS.md` B99.
conflict_notified: Arc<Mutex<HashSet<NodeUid>>>,
/// Whether the conflict sweep may actually trash the duplicates it finds, or
/// only report them. Resolved once at mount from config + environment
/// ([`AppConfig::resolved_conflict_sweep`]); [`SweepMode::Off`] means the
/// sweep thread is never spawned at all. See `docs/BUGS.md` B71.
sweep_mode: SweepMode,
/// A mkdir, create, rename or delete through the mount is recorded and
/// queued, and the drain sends it; the syscall never waits for Drive. Off,
/// they go to Drive inline while online, as before 3.0.
local_first: bool,
/// Photos whose missing thumbnail is being generated right now. A tile that is
/// still on screen asks for its thumbnail again every few seconds, and each of
/// those downloads is a full-size photo — so an in-flight uid is never started
/// twice.
thumb_gen: Arc<Mutex<HashSet<NodeUid>>>,
/// Global permit pool for locally generated thumbnails. The old per-request
/// limit multiplied when the GUI sent several batches, allowing dozens of
/// full-size images to download at once.
thumb_gen_budget: Arc<tokio::sync::Semaphore>,
/// Current ordinary-file listing generation and a wake-up for tasks waiting
/// on a permit or a download when that listing is abandoned.
file_thumb_generation: Arc<AtomicU64>,
file_thumb_cancel: Arc<tokio::sync::Notify>,
/// Progress of the explicit recursive build started from the Files toolbar.
thumbnail_build: Arc<Mutex<ThumbnailBuildStatus>>,
/// Cancellation for the explicit recursive build. Separate from ordinary
/// listing cancellation because the two jobs have independent lifetimes.
thumbnail_build_cancelled: Arc<AtomicBool>,
thumbnail_build_cancel: Arc<tokio::sync::Notify>,
/// Negative thumbnail knowledge, separated by whether the remote lacked a
/// thumbnail or local decoding proved this revision unsupported.
///
/// Absence has to be cached or it costs a round trip every time it is asked
/// for, and it is asked for constantly: an `ls -l` from an xattr-aware lister
/// issues a `getxattr` per advertised name per entry, so a 65-file directory
/// of videos re-probed 130 times per listing at ~186 ms each (B5). The mtime
/// is the validity tag — a new revision may well have a thumbnail — matching
/// how [`ContentCache::read_thumbnail`] validates the positive side.
thumbnail_misses: Arc<Mutex<ThumbnailMissCaches>>,
/// Last account quota answer and when it was learned, for `statfs(2)`.
///
/// `df` and the free-space preflight of every file manager and installer ask
/// for this, and the answer costs a round trip — so it is served from here
/// for [`QUOTA_TTL`] and refreshed on a worker afterwards. Total account
/// storage moves slowly enough that a minute-old number is a better answer
/// than either a stall or the zeroes the default implementation returns.
quota: Arc<Mutex<Option<(std::time::Instant, i64, i64)>>>,
/// Set while a `statfs` is refreshing [`Core::quota`], so the others serve
/// the last figures instead of each holding a worker on the same request.
quota_refreshing: Arc<AtomicBool>,
/// Database maintenance that has been moved off the request thread.
///
/// `VACUUM` and a deep `integrity_check` both hold the single connection for
/// tens of seconds against a real install, which stops every FUSE handler
/// that needs the database — so the handler acknowledges the request and a
/// thread does the work, exactly as the other long control requests already
/// do. See [`Maintenance`].
maintenance: Arc<Mutex<Maintenance>>,
/// Folders whose listing was enumerated cheaply and is having its file sizes
/// filled in right now, so a burst of `stat`s over a fresh listing starts one
/// upgrade rather than one per entry. See [`Core::spawn_size_upgrade`].
/// Size upgrades currently running, per folder inode, across every mount. A `getattr` that
/// needs a real size waits on the entry rather than issuing its own fetch —
/// `ls -l` of a folder is one `getattr` per file, and they must collapse
/// onto a single batch (`docs/BUGS.md` B14).
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
/// Replies parked on those upgrades. Shared across every mount for the same
/// reason the pool is: the bound that matters is per daemon.
size_waiters: Arc<SizeWaitQueue>,
/// This mount's kernel notification channel, for telling the kernel to drop
/// metadata it has cached. Set once the session exists — which is *after*
/// the `Core` it is built from, hence the cell.
///
/// Per mount, not per daemon: each session calls its own root inode 1.
/// [`Core::fork_state`] gives each fork an empty cell of its own. Work that
/// changes the tree for every mount tells every session
/// ([`Core::for_each_mount`]).
notifier: Arc<OnceLock<KernelChannel>>,
/// True only after this mount's FUSE session spawned successfully.
/// Registration happens earlier for the primary drain, so registration
/// alone cannot answer whether the location is mounted.
session_live: Arc<AtomicBool>,
/// In-flight upload/download progress, served to `GetQueueStatus`. Shared
/// across the FUSE session and the control-socket task.
transfers: Arc<TransferRegistry>,
/// Open [`Request::Subscribe`](pdfs_core::control::Request::Subscribe)
/// connections, which every change a front-end shows is published to.
events: Arc<events::EventHub>,
/// True while the background scanner is rebuilding the local-file index, so
/// `SearchLocal` can tell a front-end "still indexing" apart from "no match".
indexing: Arc<AtomicBool>,
/// Live per-folder sync progress, keyed by sync-folder id, so `ListSyncFolders`
/// can say what a pass is doing rather than just "syncing". An entry exists
/// only while that folder's reconcile pass is running.
sync_progress: Arc<Mutex<HashMap<i64, SyncProgress>>>,
/// Whether the folder-sync engine is alive, and which pass it is on.
sync_engine: Arc<sync::EngineHealth>,
/// Channel to the folder-sync engine: nudges it to
/// reconcile a folder, reconcile everything, or re-scan its watch set.
sync_tx: std::sync::mpsc::Sender<sync::SyncMsg>,
/// Secondary FUSE sessions for `ondemand` sync folders, keyed by sync-folder
/// id. Each is a `ProtonFs` rooted at the folder's remote
/// node, mounted over its local path, sharing this Core's client/cache/db and
/// tree but with its own root (`fork_state`). Held so we can unmount on
/// toggle back to `mirror` and on daemon shutdown. Each entry retains the
/// FUSE connection id and the fork's exact liveness flag so teardown first
/// makes that mount unroutable.
mounts: Arc<Mutex<mount::SecondaryMountRegistry<SecondaryMount>>>,
/// Per-sync-folder locks, held for a whole reconcile pass and for a whole
/// mode switch. A `mirror→ondemand` flip evicts the local tree and mounts
/// FUSE over it, so it must never overlap a pass that is walking and
/// uploading that same tree — the engine would upload files as they vanish
/// and then walk the FUSE mount as if it were local.
sync_locks: Arc<Mutex<HashMap<i64, Arc<Mutex<()>>>>>,
/// Per-sync-folder paths the last pass could not read, so the activity feed
/// says so when the set changes rather than on every poll (B196).
sync_unscanned: Arc<Mutex<HashMap<i64, Vec<String>>>>,
/// Nodes this daemon changed on the remote itself, and how many echoes of
/// those changes the event feed still owes us.
///
/// The feed replays our own writes back at us seconds later, and a
/// `NodeUpdated` carries no revision id — nothing in it distinguishes "another
/// device changed this file" from "you changed this file". Treating the echo
/// as foreign is not free: it evicts the content blob we just uploaded, so the
/// next read re-downloads a revision the API may not be serving yet. That is
/// how a SQLite database written on the mount came back `malformed`, and how a
/// mount doing bulk uploads spent its time re-fetching its own bytes.
///
/// So the first event for a node we changed ourselves, within
/// [`SELF_CHANGE_TTL_MS`], is consumed rather than applied: the tree already
/// holds what that event describes, because [`Core::refresh_after_upload`] put
/// it there. Anything beyond that applies normally, which bounds the staleness
/// a genuine remote change can suffer to one event.
///
/// Shared across forks like `pending`: the drain that records the change and
/// the event task that consumes the echo are both per-daemon.
self_changes: Arc<Mutex<HashMap<NodeUid, SelfChange>>>,
/// Every live mount: the primary and one per on-demand fork
/// ([`Core::fork_state`]), with its mountpoint, root and kernel channel.
/// Shared by every fork.
///
/// The mounts share one tree, so a change the drain makes is seen by every
/// one of them. Each forked mount used to keep its nodes in a state of its
/// own, where the drain never looked: the fork's entry kept its `local~`
/// placeholder uid forever (`docs/BUGS.md` B74). What is still per mount is
/// the kernel's cache, which [`Core::for_each_mount`] tells every session
/// about.
///
/// `Weak`, so an unmounted fork is dropped rather than pinned here.
states: Arc<StateRegistry>,
}
/// Background database maintenance: what is running, and what the last deep
/// integrity check found.
///
/// One at a time — both operations are whole-database and there is nothing to
/// gain from overlapping them — and the result of a check outlives the request
/// that asked for it, because the request is answered before the check finishes.
#[derive(Default)]
struct Maintenance {
/// A vacuum or integrity check is running right now.
running: bool,
/// Findings of the most recent completed deep check. `None` means none has
/// finished this run, which is *not* the same as "found nothing".
integrity: Option<Vec<String>>,
}
/// How long a change this daemon made stays recognisable as its own. Generous
/// against a feed that can lag, but finite: past it, an event for the node is
/// applied like any other. See [`Core::self_changes`].
const SELF_CHANGE_TTL_MS: i64 = 120_000;
/// A remote change this daemon made, awaiting its echo from the event feed.
struct SelfChange {
/// When we made it, in the same epoch millis as [`now_millis`].
at_ms: i64,
/// Echoes not yet seen. More than one because a single file can be created
/// and then have a revision uploaded, and the feed reports both.
echoes: u32,
}
/// Record one remote change this daemon made. Split from [`Core`] so the
/// expiry and echo-counting rules can be tested against a clock the test owns.
fn note_self_change(changes: &mut HashMap<NodeUid, SelfChange>, uid: &NodeUid, now_ms: i64) {
// Echoes that never arrived — a node trashed before its event came round, a
// feed that skipped it — would otherwise accumulate for the life of the
// daemon. Pruning on write keeps the map the size of the recent queue.
changes.retain(|_, c| now_ms - c.at_ms < SELF_CHANGE_TTL_MS);
let change = changes.entry(uid.clone()).or_insert(SelfChange {
at_ms: now_ms,
echoes: 0,
});
change.at_ms = now_ms;
change.echoes += 1;
}
/// Claim one echo, per [`Core::take_self_change`].
fn take_self_change(
changes: &mut HashMap<NodeUid, SelfChange>,
uid: &NodeUid,
now_ms: i64,
) -> bool {
let Some(change) = changes.get_mut(uid) else {
return false;
};
// Too old to attribute: the feed is far enough behind that this is as likely
// to be someone else's change as ours, and guessing wrong here costs the user
// a stale file.
if now_ms - change.at_ms >= SELF_CHANGE_TTL_MS {
changes.remove(uid);
return false;
}
change.echoes -= 1;
if change.echoes == 0 {
changes.remove(uid);
}
true
}
/// One live mount, as published to the shared registry.
#[derive(Clone)]
struct LiveMount {
mountpoint: PathBuf,
/// The mount's kernel channel, empty until its session is spawned.
notifier: Arc<OnceLock<KernelChannel>>,
session_live: Arc<AtomicBool>,
/// The tree inode the mount is rooted at, empty until its session is built.
root: Arc<OnceLock<u64>>,
}
impl LiveMount {
fn is_live(&self) -> bool {
self.session_live.load(Ordering::Acquire)
}
}
struct MountedState {
/// Absolute local root of this mount.
mountpoint: PathBuf,
/// The same cell as [`Core::notifier`]. `Weak`, so an unmounted fork is
/// dropped rather than pinned here; a dead entry is reaped on the next
/// registry walk.
notifier: std::sync::Weak<OnceLock<KernelChannel>>,
session_live: Arc<AtomicBool>,
root: Arc<OnceLock<u64>>,
}
/// Every mount in the daemon, shared by the primary `Core` and every
/// [`Core::fork_state`] clone of it.
///
/// Split out from `Core` so the reap-and-walk rule can be tested on its own: a
/// fork that fails to appear here, or one that lingers after unmount, is exactly
/// the failure `docs/BUGS.md` B74 was.
#[derive(Default)]
struct StateRegistry(Mutex<Vec<MountedState>>);
impl StateRegistry {
/// Publish a mount and the channel to the session that serves it.
///
/// A mount is registered exactly once by its owner. Session construction
/// only flips `session_live`; it never republishes the mount.
fn register(
&self,
mountpoint: &Path,
notifier: &Arc<OnceLock<KernelChannel>>,
session_live: Arc<AtomicBool>,
root: Arc<OnceLock<u64>>,
) {
let mut states = self.0.lock();
states.retain(|m| m.notifier.strong_count() > 0);
debug_assert!(
!states
.iter()
.any(|mounted| mounted.notifier.ptr_eq(&Arc::downgrade(notifier))),
"a mount must be registered exactly once"
);
states.push(MountedState {
mountpoint: mountpoint.to_path_buf(),
notifier: Arc::downgrade(notifier),
session_live,
root,
});
}
/// Every registered mount, reaping the entries whose session has gone.
/// Returns owned handles so callers do not hold the registry lock across
/// the work.
fn live(&self) -> Vec<LiveMount> {
let mut states = self.0.lock();
states.retain(|m| m.notifier.strong_count() > 0);
states
.iter()
.filter_map(|m| {
Some(LiveMount {
mountpoint: m.mountpoint.clone(),
notifier: m.notifier.upgrade()?,
session_live: m.session_live.clone(),
root: m.root.clone(),
})
})
.collect()
}
/// The live session whose local root most specifically covers `path`.
///
/// On-demand roots can be nested below another location. Selecting by the
/// longest component prefix ensures the nested session wins rather than the
/// broader primary mount. Callers that need the relative suffix derive it
/// with `path.strip_prefix(mountpoint)`.
fn covering(&self, path: &Path) -> Option<LiveMount> {
self.live()
.into_iter()
.filter(|mount| mount.is_live() && path.starts_with(&mount.mountpoint))
.max_by_key(|mount| mount.mountpoint.components().count())
}
fn is_mounted_at(&self, path: &Path) -> bool {
self.covering(path)
.is_some_and(|mount| mount.mountpoint == path)
}
/// The tree inodes of the sessions that are mounted.
fn live_roots(&self) -> Vec<u64> {
self.live()
.into_iter()
.filter(LiveMount::is_live)
.filter_map(|mount| mount.root.get().copied())
.collect()
}
}
/// Why [`Core::apply_sync_folder_mode`] did not switch a folder. The two cases are
/// answered very differently: `NotNow` is the normal state of a folder that is busy
/// or has local changes still to push, and the caller queues the request; `Failed`
/// is a real fault the user has to hear about.
enum SwitchBlocked {
/// The folder is mid-pass, or not yet safe to switch. Try again after a pass.
NotNow,
/// A queued target was canceled or replaced before the folder lock became
/// available. The newer intent owns the next transition.
Superseded,
/// The switch was attempted and broke.
Failed(String),
}
impl Core {
/// Start a background `VACUUM`, or refuse because maintenance is already
/// running.
///
/// Returns as soon as the thread is spawned; the caller acknowledges the
/// request. The outcome is a log line and a job that disappears from
/// `GetQueueStatus` when it is done — a vacuum has nothing to report that a
/// follow-up `CacheInspect` cannot show.
fn start_vacuum(&self) -> CoreResult<()> {
{
let mut m = self.maintenance.lock();
if m.running {
return Err(CoreError::conflict(
"database maintenance is already running",
));
}
m.running = true;
}
let core = self.clone();
std::thread::spawn(move || {
let job = core.transfers.begin_job("Compacting the cache database");
job.detail("Rewriting the database file");
let started = Instant::now();
match core.db.vacuum() {
Ok(outcome) => info!(
freed_bytes = outcome.freed_bytes(),
before_bytes = outcome.before_bytes,
after_bytes = outcome.after_bytes,
wal_frames = outcome.wal_frames_checkpointed,
elapsed_ms = started.elapsed().as_millis() as u64,
"cache database vacuumed"
),
Err(e) => warn!(error = %e, "vacuum failed"),
}
core.maintenance.lock().running = false;
});
Ok(())
}
/// Answer a deep `CacheInspect`: hand back the last completed integrity
/// check, and start one if none is running.
///
/// Returns `(problems, checked, running)`. A first request gets
/// `(_, false, true)` and the answer lands on a later one — the check reads
/// every page of the database under the connection mutex, so the choice is
/// between answering late and freezing the mount until it finishes.
fn start_or_read_integrity_check(&self) -> (Vec<String>, bool, bool) {
let mut m = self.maintenance.lock();
if let Some(problems) = m.integrity.clone() {
return (problems, true, m.running);
}
if m.running {
return (Vec::new(), false, true);
}
m.running = true;
drop(m);
let core = self.clone();
std::thread::spawn(move || {
let job = core
.transfers
.begin_background_job("Checking the cache database");
job.detail("Verifying every page");
let problems = match core.db.integrity_check() {
Ok(problems) => problems,
Err(e) => vec![format!("integrity check failed: {e:?}")],
};
if problems.is_empty() {
info!("cache database integrity check found no problems");
} else {
warn!(problems = problems.len(), "cache database integrity check");
}
let mut m = core.maintenance.lock();
m.integrity = Some(problems);
m.running = false;
});
(Vec::new(), false, true)
}
/// Register a mount so background work can reach its session. Called once
/// for the primary mount and once per on-demand fork.
fn register_state(&self, mountpoint: &Path) {
self.states.register(
mountpoint,
&self.notifier,
self.session_live.clone(),
self.root.clone(),
);
}
/// The tree inode this mount is rooted at. [`ROOT_INO`] until the session
/// is built, which is where a test that builds a bare tree puts its root.
pub(crate) fn root_ino(&self) -> u64 {
self.root.get().copied().unwrap_or(ROOT_INO)
}
/// Whether `uid` names an owned node reachable from the root of a mounted
/// session. See [`State::owns_visible_uid`].
fn owns_visible_uid(&self, uid: &NodeUid) -> bool {
let roots = self.states.live_roots();
self.state().owns_visible_uid(uid, &roots)
}
/// This `Core`, re-rooted onto whichever live mount most specifically covers
/// the absolute path `abs`, together with `abs` relative to that mount's
/// root.
///
/// Everything a `Core` holds is per-daemon except the fields
/// [`Core::fork_state`] replaces, so swapping exactly those turns the
/// primary `Core` into the fork that owns the path — resolving from the
/// fork's root and notifying its session. Without this, every
/// path-addressed control request is resolved from the primary root, so
/// anything under a secondary on-demand mount is rejected as "not under the
/// mountpoint" and the user has no way to re-enumerate a folder they can see
/// (`docs/BUGS.md` B86).
///
/// `None` when no live mount covers `abs`.
fn rooted_at(&self, abs: &Path) -> Option<(Core, PathBuf)> {
let mount = self.states.covering(abs)?;
let rel = abs.strip_prefix(&mount.mountpoint).ok()?.to_path_buf();
let mut core = self.clone();
if !Arc::ptr_eq(&mount.notifier, &self.notifier) {
// Another mount, so adopt its per-mount half wholesale. `primary`
// is a property of the mount, and the only mount this `Core` can
// be the primary of is the one it was built for.
core.primary = false;
core.notifier = mount.notifier;
core.session_live = mount.session_live;
core.root = mount.root;
}
Some((core, rel))
}
/// Take the tree's lock.
///
/// The guard is what applies the mutation's write-throughs, after releasing
/// the lock — see [`StateGuard`]. Every caller takes it this way; nothing
/// locks `self.state` directly, or the writes it queues would be applied
/// only when some later caller happened to take the guard.
fn state(&self) -> StateGuard<'_> {
StateGuard::new(self.state.lock(), &self.db)
}
/// Run `apply` against the tree, then tell every mounted session about the
/// access changes it made.
///
/// For the drain and the sync engine, which are per-daemon: a `pending_op`
/// row records a uid, not the session that queued it, and any mount may be
/// showing the node.
fn for_each_state(&self, apply: impl FnOnce(&mut State)) {
let changed = {
let mut state = self.state();
apply(&mut state);
state.take_access_changes()
};
self.notify_access_changes(&changed);
}
/// Like [`Core::for_each_state`], but the closure also records what the
/// kernel needs told into a [`NotifyBatch`], which goes to every mounted
/// session.
///
/// Recorded rather than sent: notifications are flushed after the State
/// lock is released, the same rule [`Core::for_each_state`] and
/// [`Core::flush_access_changes`] follow.
fn for_each_mount(&self, apply: impl FnOnce(&mut State, &mut NotifyBatch)) {
let mut batch = NotifyBatch::default();
apply(&mut self.state(), &mut batch);
for mount in self.states.live() {
batch.flush(mount.notifier.get());
}
}
/// Publish access changes accumulated by an intern/root refresh. The State
/// lock is released before notifying the kernel because notifier callbacks
/// may synchronously provoke more filesystem work.
fn flush_access_changes(&self) {
let changed = self.state().take_access_changes();
self.notify_access_changes(&changed);
}
/// Tell every mounted session that `changed` have new permissions.
fn notify_access_changes(&self, changed: &[u64]) {
if changed.is_empty() {
return;
}
for mount in self.states.live() {
notify_access_changes(mount.notifier.get(), changed);
}
}
fn require_writable(&self, ino: u64) -> Result<(), Errno> {
self.state().require_writable(ino)
}
/// Mutation admission point. Persisted and every resident authority must
/// agree that the node is writable. A check completed before a downgrade is
/// admitted; later queued attempts are caught by drain. No global lock is
/// held across a network call.
///
/// The intersection closes both halves of an access-refresh race: a DB
/// downgrade cannot be hidden by stale live state, and a live downgrade
/// cannot be hidden by a restored DB row. Missing UIDs fail closed.
fn require_uid_writable(&self, uid: &NodeUid) -> Result<(), Errno> {
match self.uid_write_authority(uid) {
WriteAuthority::Writable => Ok(()),
// A uid nobody can speak for is refused here just as a denied one
// is: a syscall is a live request against a live tree, and a stale
// handle must not be admitted because the tree forgot the node.
// The drain, which replays intent recorded long ago, is the one
// caller that has to tell the two apart — see
// [`Core::uid_write_authority`].
WriteAuthority::Denied | WriteAuthority::Unknown => Err(Errno::EACCES),
}
}
/// The full answer behind [`Core::require_uid_writable`]: whether every
/// authority agrees this node is writable, refuses it, or has never heard
/// of it.
///
/// The third case is the one worth naming. A node absent from the local
/// tree yields no access row, and collapsing that into "denied" told the
/// drain to wait for a permission change that was never coming, because
/// nothing about the node's permissions was ever the problem (B83).
pub(crate) fn uid_write_authority(&self, uid: &NodeUid) -> WriteAuthority {
let live_access: Vec<Access> = self.state().access_by_uid(uid).into_iter().collect();
uid_write_authority(&self.db, uid, &live_access)
}
}
/// Whether a node may be written, as agreed by every authority that has an
/// opinion — or the fact that none of them does.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum WriteAuthority {
/// The persisted tree and every resident mount agree: writable.
Writable,
/// At least one authority refuses it, or could not be consulted. Fails
/// closed, so a lookup error lands here.
Denied,
/// The node is not in the local tree, so there is no authority to consult.
/// Not the same as a refusal, and callers that can ask the remote should.
Unknown,
}
fn uid_write_authority(db: &Db, uid: &NodeUid, live_access: &[Access]) -> WriteAuthority {
let persisted = match db.effective_node_access(uid) {
Ok(Some(access)) => access,
Ok(None) => return WriteAuthority::Unknown,
Err(error) => {
error!(%uid, %error, "db effective access lookup failed");
return WriteAuthority::Denied;
}
};
if persisted.writable() && live_access.iter().all(|access| access.writable()) {
WriteAuthority::Writable
} else {
WriteAuthority::Denied
}
}
/// Move `uid` to `name` in `new_parent` within one mount, for a rename or move
/// Drive has made, and record what the kernel has to forget. Returns the name
/// the mount knew the node by, and whether the node was moved rather than
/// forgotten, which writes its row through. `None` when the mount does not
/// hold the node.
///
/// The inode is kept wherever the mount shows the new parent, as the mount's
/// own rename keeps it (`State::relocate`). A mount that does not show the new
/// parent only forgets the node; to it, the node is gone. The row stays either
/// way (`docs/BUGS.md` B117).
fn relocate_in_mount(
st: &mut State,
notify: &mut NotifyBatch,
uid: &NodeUid,
new_parent: &NodeUid,
name: &str,
queued: bool,
) -> Option<(String, bool)> {
let ino = *st.by_uid.get(uid)?;
let entry = st.entries.get(&ino)?;
let (from_parent, from_name) = (entry.parent, entry.node.name.clone());
match st.by_uid.get(new_parent).copied() {
Some(to_parent) => {
if queued {
// Listing again would ask Drive, which still has the old name;
// the kernel's pages of both folders go instead.
st.rename_in_place(ino, to_parent, new_parent, name);
notify.inval_inode(from_parent);
if to_parent != from_parent {
notify.inval_inode(to_parent);
}
} else {
st.relocate(ino, from_parent, to_parent, new_parent, name);
}
notify.inval_entry(from_parent, from_name.clone());
notify.inval_entry(to_parent, name.to_string());
Some((from_name, true))
}
None => {
st.forget_mem(uid);
notify.delete(from_parent, ino, from_name.clone());
Some((from_name, false))
}
}
}
/// One kernel notification, recorded while the State lock is held so it can be
/// sent once it is not.
///
/// `inval_inode` calls into `invalidate_inode_pages2`, which can block waiting
/// on pages that this daemon's own workers have to fill — and those workers take
/// the State lock. Sending from inside the lock is that deadlock's shape; a
/// batch is the smallest way to keep the inode/mount pairing while moving the
/// send outside it.
enum KernelNotice {
InvalInode(u64),
InvalEntry {
parent: u64,
name: String,
},
Delete {
parent: u64,
child: u64,
name: String,
},
}
/// Notifications accumulated under the State lock. Flushed by
/// [`Core::for_each_mount`] to every session once the lock is released.
#[derive(Default)]
struct NotifyBatch(Vec<KernelNotice>);
impl NotifyBatch {
fn inval_inode(&mut self, ino: u64) {
self.0.push(KernelNotice::InvalInode(ino));
}
fn inval_entry(&mut self, parent: u64, name: String) {
self.0.push(KernelNotice::InvalEntry { parent, name });
}
fn delete(&mut self, parent: u64, child: u64, name: String) {
self.0.push(KernelNotice::Delete {
parent,
child,
name,
});
}
fn extend_inodes(&mut self, inodes: &[u64]) {
self.0
.extend(inodes.iter().map(|&ino| KernelNotice::InvalInode(ino)));
}
/// Send everything recorded to one session. A mount whose session has not
/// been spawned has nothing to tell the kernel. A session that never looked
/// a node up answers `ENOENT`, which is ignored like any other refusal.
fn flush(&self, notifier: Option<&KernelChannel>) {
let Some(notifier) = notifier else { return };
for notice in &self.0 {
let _ = match notice {
KernelNotice::InvalInode(ino) => notifier.inval_inode(*ino),
KernelNotice::InvalEntry { parent, name } => {
notifier.inval_entry(*parent, OsStr::new(name))
}
KernelNotice::Delete {
parent,
child,
name,
} => notifier.delete(*parent, *child, OsStr::new(name)),
};
}
}
}
/// One FUSE session's notification channel. The tree has one inode space for
/// every mount, and each session knows its own root as [`ROOT_INO`], so this
/// names the root that way and every other inode as it is.
pub(crate) struct KernelChannel {
notifier: Notifier,
/// The tree inode of the session's root.
root: u64,
}
impl KernelChannel {
fn new(notifier: Notifier, root: u64) -> Self {
Self { notifier, root }
}
pub(crate) fn inval_inode(&self, ino: u64) -> std::io::Result<()> {
let ino = filesystem::kernel_ino(self.root, ino);
off_the_runtime(|| self.notifier.inval_inode(ino, 0, 0))
}
/// Drop the cached attributes of `ino` but not its page cache.
fn inval_attrs(&self, ino: u64) -> std::io::Result<()> {
let ino = filesystem::kernel_ino(self.root, ino);
off_the_runtime(|| self.notifier.inval_inode(ino, -1, 0))
}
pub(crate) fn inval_entry(&self, parent: u64, name: &OsStr) -> std::io::Result<()> {
let parent = filesystem::kernel_ino(self.root, parent);
off_the_runtime(|| self.notifier.inval_entry(parent, name))
}
fn delete(&self, parent: u64, child: u64, name: &OsStr) -> std::io::Result<()> {
let parent = filesystem::kernel_ino(self.root, parent);
let child = filesystem::kernel_ino(self.root, child);
off_the_runtime(|| self.notifier.delete(parent, child, name))
}
}
/// Send a notice to the kernel without holding up the runtime.
///
/// The kernel answers an invalidation only once it has the pages' locks, and
/// a read holds those until the daemon answers it. Sent from a runtime worker,
/// the notice blocked that worker in the kernel; when it was the one driving
/// the timers and I/O, nothing drove them any more, so the read's fetch never
/// finished and the daemon hung (`docs/BUGS.md` B130). On a worker the notice
/// hands the worker's tasks to another thread first; anywhere else it is sent
/// as it is.
fn off_the_runtime<T>(notice: impl FnOnce() -> T) -> T {
match tokio::runtime::Handle::try_current() {
Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => {
tokio::task::block_in_place(notice)
}
_ => notice(),
}
}
fn notify_access_changes(notifier: Option<&KernelChannel>, changed: &[u64]) {
let Some(notifier) = notifier else { return };
for &ino in changed {
let _ = notifier.inval_inode(ino);
}
}
fn require_rename_access(
require: impl Fn(&NodeUid) -> Result<(), Errno>,
uid: &NodeUid,
old_parent_uid: &NodeUid,
new_parent_uid: &NodeUid,
) -> Result<(), Errno> {
for authority in [uid, old_parent_uid, new_parent_uid] {
require(authority)?;
}
Ok(())
}
fn require_node_parent_access(
require: impl Fn(&NodeUid) -> Result<(), Errno>,
uid: &NodeUid,
parent_uid: &NodeUid,
) -> Result<(), Errno> {
require(uid)?;
require(parent_uid)
}
fn preserve_on_access_denied(
access: Result<(), Errno>,
accepted_bytes: bool,
preserve: impl FnOnce(),
) -> Result<(), Errno> {
if let Err(error) = access {
if error.code() == libc::EACCES && accepted_bytes {
preserve();
}
return Err(error);
}
Ok(())
}
/// Fold the staged blob of an undrained write into the blob at `src` that is
/// about to supersede it, so the newer write inherits the older one's bytes
/// instead of losing them.
///
/// Only the ranges the older write authored and the newer one did not are
/// copied, so a small edit over a small edit moves almost nothing. What neither
/// wrote is untouched remote content, which is why `base_size`, `base_mtime` and
/// the baseline all come from the write being superseded: that is the revision
/// those ranges were last observed against, and the one the drain must gap-fill
/// and conflict-check them against.
fn merge_over_pending(
meta: &mut StagedWrite,
src: &Path,
previous: &PendingRevision,
) -> std::io::Result<()> {
let mut written = Intervals::default();
for &(s, e) in &meta.authored {
written.add(s, e);
}
let mut earlier = Intervals::default();
for &(s, e) in &previous.meta.authored {
earlier.add(s, e);
}
let blob = File::open(&previous.path)?;
let dst = std::fs::OpenOptions::new().write(true).open(src)?;
for (s, e, authored) in written.clone().segments(0, meta.len) {
if authored {
continue;
}
for (ps, pe, have) in earlier.segments(s, e.min(previous.meta.len)) {
if !have {
continue;
}
let mut buf = vec![0u8; (pe - ps) as usize];
blob.read_exact_at(&mut buf, ps)?;
dst.write_all_at(&buf, ps)?;
written.add(ps, pe);
}
}
dst.sync_all()?;
meta.base_size = previous.meta.base_size;
meta.base_mtime = previous.meta.base_mtime;
meta.based_on = previous.meta.based_on.clone();
meta.authored = written
.segments(0, meta.len)
.into_iter()
.filter(|&(_, _, authored)| authored)
.map(|(s, e, _)| (s, e))
.collect();
meta.complete = StagedWrite::covers(&meta.authored, meta.len);
Ok(())
}
/// Retire the last resident reference to an unlinked inode without touching its
/// persisted node row. The namespace path already chose whether that row is
/// disposable or retained as queued-trash authority.
fn release_unlinked_entry(state: &mut State, ino: u64) -> Option<NodeUid> {
let entry = state.entries.get_mut(&ino)?;
entry.open_count = entry.open_count.saturating_sub(1);
if entry.open_count != 0 || !entry.unlinked {
return None;
}
let uid = entry.uid.clone();
state.forget_mem(&uid);
Some(uid)
}
fn release_must_retain_queued_trash(db: &Db, uid: &NodeUid) -> pdfs_core::Result<bool> {
db.has_pending_op(&uid.to_string(), OP_TRASH)
}
/// The size of a staged blob that has outlived [`STAGING_ORPHAN_RETAIN`], or
/// `None` while it is younger than that — or while its age cannot be read at
/// all, which is never grounds for deleting bytes.
fn orphan_past_retention(blob: &Path) -> Option<u64> {
let meta = std::fs::metadata(blob).ok()?;
let age = meta.modified().ok()?.elapsed().ok()?;
(age >= STAGING_ORPHAN_RETAIN).then_some(meta.len())
}
fn release_can_discard_unlinked(db: &Db, uid: &NodeUid) -> bool {
match release_must_retain_queued_trash(db, uid) {
Ok(retain) => !retain,
Err(error) => {
warn!(%uid, %error, "queued-trash lookup failed; retaining unlinked state");
false
}
}
}
struct AcceptedShares {
uids: Vec<NodeUid>,
/// `None` means duplicate listing rows disagreed on provenance.
share_ids: HashMap<NodeUid, Option<ShareId>>,
}
fn accepted_share_provenance(items: Vec<SharedWithMeItem>) -> AcceptedShares {
let mut uids = Vec::new();
let mut share_ids = HashMap::new();
for item in items {
match share_ids.entry(item.uid.clone()) {
std::collections::hash_map::Entry::Vacant(entry) => {
uids.push(item.uid);
entry.insert(Some(item.share_id));
}
std::collections::hash_map::Entry::Occupied(mut entry) => {
if entry.get().as_ref() != Some(&item.share_id) {
entry.insert(None);
}
}
}
}
AcceptedShares { uids, share_ids }
}
fn shared_root_access(nodes: &[&Node], expected_share: Option<&ShareId>) -> Access {
let Some(expected_share) = expected_share else {
return Access::Viewer;
};
let mut observed = None;
for node in nodes {
let Some(membership) = node.membership.as_ref() else {
return Access::Viewer;
};
if &membership.share_id != expected_share {
return Access::Viewer;
}
let Some(role) = membership.role_exact() else {
return Access::Viewer;
};
let access = access_for(Some(role), Access::Viewer, true);
if observed.is_some_and(|prior| prior != access) {
return Access::Viewer;
}
observed = Some(access);
}
observed.unwrap_or(Access::Viewer)
}
fn prepare_shared_roots(
accepted: &AcceptedShares,
mut materialized: Vec<Node>,
parent: &NodeUid,
) -> Vec<PublishedSharedRoot> {
materialized.retain(|node| accepted.share_ids.contains_key(&node.uid));
materialized.sort_by(|a, b| {
a.uid
.to_string()
.cmp(&b.uid.to_string())
.then_with(|| a.name.cmp(&b.name))
});
let mut published = Vec::new();
let mut start = 0usize;
while start < materialized.len() {
let uid = materialized[start].uid.clone();
let mut end = start + 1;
while end < materialized.len() && materialized[end].uid == uid {
end += 1;
}
let access = shared_root_access(
&materialized[start..end].iter().collect::<Vec<_>>(),
accepted.share_ids.get(&uid).and_then(Option::as_ref),
);
let mut node = materialized[start].clone();
node.parent_uid = Some(parent.clone());
node.trashed = false;
published.push(PublishedSharedRoot { node, access });
start = end;
}
let mut named: Vec<Node> = published.iter().map(|root| root.node.clone()).collect();
disambiguate_shared_names(&mut named);
let names: HashMap<NodeUid, String> = named
.into_iter()
.map(|node| (node.uid, node.name))
.collect();
for root in &mut published {
if let Some(name) = names.get(&root.node.uid) {
root.node.name.clone_from(name);
}
}
published
}
impl Core {
/// Rename `uid` on Drive, riding out an "out of date" answer from a stale
/// name hash. See [`retry_out_of_date`].
pub(crate) fn rename_remote(&self, uid: &NodeUid, name: &str) -> Result<(), ProtonError> {
self.change_remote(uid, || {
self.block_on_bounded(self.drive.rename_node(uid, name, None))
})
}
/// Move `uid` into the folder `parent` on Drive, like [`Core::rename_remote`].
pub(crate) fn move_remote(&self, uid: &NodeUid, parent: &NodeUid) -> Result<(), ProtonError> {
self.change_remote(uid, || {
self.block_on_bounded(self.drive.move_node(uid, parent))
})
}
/// Move `uid` into the folder `parent` on Drive, renaming it to
/// `target_name` on the way when one is given, in one request.
///
/// Done as a rename and then a move, a sibling in the source folder that
/// held the new name made Drive refuse the first half, although the
/// destination was free (docs/BUGS.md B122). One request also leaves no
/// half-applied state between the two.
pub(crate) fn move_rename_remote(
&self,
uid: &NodeUid,
parent: &NodeUid,
target_name: Option<&str>,
) -> Result<(), ProtonError> {
self.change_remote(uid, || {
let item = NodeMoveItem {
uid: uid.clone(),
target_name: target_name.map(str::to_string),
};
self.block_on_bounded(async {
let mut outcomes =
std::pin::pin!(self.drive.move_nodes_streaming(vec![item], parent.clone()));
match outcomes.next().await {
Some(Ok((_, outcome))) => outcome,
Some(Err(e)) => Err(e),
None => Err(ProtonError::invalid_operation(format!(
"move of {uid} reported no outcome"
))),
}
})
})
}
/// Run a rename or move of `uid`, then drop the SDK's now stale copy of it.
fn change_remote(
&self,
uid: &NodeUid,
call: impl FnMut() -> Result<(), ProtonError>,
) -> Result<(), ProtonError> {
retry_out_of_date(call, || {
self.forget_sdk_node(uid);
std::thread::sleep(OUT_OF_DATE_RETRY_DELAY);
})
.inspect(|()| self.changed_remote(uid))
}
/// Bookkeeping after a rename or move of `uid` landed: the SDK's copy of it
/// is stale, and the feed will echo the change back. Without the second,
/// that echo reads as someone else's edit and evicts the cached bytes a
/// partial write is about to gap-fill from (docs/BUGS.md B123).
pub(crate) fn changed_remote(&self, uid: &NodeUid) {
self.forget_sdk_node(uid);
self.note_self_change(uid);
}
/// Drop the SDK's cached copy of `uid` after this daemon renamed or moved it.
///
/// The SDK sends a node's name hash from its entity cache, and nothing but
/// the event feed's echo of the change refreshes that entry. Until the next
/// poll, up to [`POLL_INTERVAL`] later, each rename or move of the node would
/// send the old hash and Drive would refuse it as out of date (docs/BUGS.md
/// B121). Feeding the SDK the echo now makes the next call read the link.
pub(crate) fn forget_sdk_node(&self, uid: &NodeUid) {
let changed = DriveEvent::NodeUpdated {
id: DriveEventId::from("pdfs-local-change"),
node_uid: uid.clone(),
parent_node_uid: None,
is_trashed: false,
is_shared: false,
};
if let Err(error) = self.block_on_bounded(self.drive.invalidate_caches_for_event(&changed))
{
debug!(%uid, %error, "could not drop the SDK's cached node");
}
}
/// Record that this daemon just changed `uid` on the remote, so the event
/// feed's echo of that change is recognised instead of re-applied. Called
/// from the drain once the remote has actually accepted the change and the
/// tree has been brought level with it. See [`Core::self_changes`].
pub(crate) fn note_self_change(&self, uid: &NodeUid) {
note_self_change(&mut self.self_changes.lock(), uid, now_millis());
}
/// Claim one outstanding echo for `uid`, reporting whether this event is one
/// this daemon caused. Consuming rather than merely testing is what bounds
/// the suppression: the next event for the node applies normally.
fn take_self_change(&self, uid: &NodeUid) -> bool {
take_self_change(&mut self.self_changes.lock(), uid, now_millis())
}
/// Rehydrate the in-memory `State` maps from the DB on mount, so a cold
/// start serves previously-seen metadata (stable inodes, instant listings)
/// without re-hitting the API. The root inode is already installed by
/// [`ProtonFs::new`]; this fills in every other persisted node and rebuilds
/// the child listings of folders the DB records as fully enumerated.
fn hydrate(&self) {
let stored = match self.db.load_all() {
Ok(s) => s,
Err(e) => {
warn!(error = %e, "db load_all failed; mounting cold");
return;
}
};
if stored.is_empty() {
return;
}
// A write queued by a previous run carries the optimistic size the file
// grew to, which the sealed remote revision does not yet reflect. The DB
// node still holds the pre-write `claimed_size` (often 0 for a fresh
// file), so without this a queued 500 MB file `stat`s as 0 bytes until
// the drain lands — reads still serve from the staged blob, but `ls`
// lying about the size reads as data loss. `hydrate_pending` ran before
// us, so the pending map is populated; snapshot it here rather than hold
// both locks at once.
let pending_sizes: HashMap<NodeUid, u64> = {
let pending = self.pending.lock();
pending
.iter()
.map(|(uid, pr)| (uid.clone(), pr.meta.len))
.collect()
};
// `hydrate_pending` runs first and restores queued-trash tombstones.
// Their node rows remain persisted as drain-time access authority, but
// must not reappear in the mounted namespace after a restart.
let hidden = self.hidden.lock().clone();
let mut st = self.state();
// Pass 1: give every uid its inode from its local id (the root is
// already mapped).
for sn in &stored {
if hidden.contains(&sn.node.uid) || st.by_uid.contains_key(&sn.node.uid) {
continue;
}
st.by_uid
.insert(sn.node.uid.clone(), state::lid_ino(sn.lid));
}
// Pass 2: materialize entries, resolving each parent's inode by uid.
// Track folders flagged complete so their listings rebuild in pass 3.
let mut listed_dirs: Vec<u64> = Vec::new();
for sn in stored {
let StoredNode {
mut node, listed, ..
} = sn;
if hidden.contains(&node.uid) {
continue;
}
let Some(&ino) = st.by_uid.get(&node.uid) else {
continue;
};
// Re-apply a queued write's optimistic size so `stat` matches what
// reads (served from the staged blob) already return.
if let Some(&len) = pending_sizes.get(&node.uid)
&& let NodeKind::File { claimed_size, .. } = &mut node.kind
{
*claimed_size = Some(len as i64);
}
if listed && node.is_folder() {
listed_dirs.push(ino);
}
// A root entry is owned by `ProtonFs::new`; don't overwrite it.
if st.roots.contains(&ino) {
continue;
}
// A node whose parent row never made it to disk must not be adopted
// by the root: it would surface as a phantom top-level entry.
let parent = node
.parent_uid
.as_ref()
.and_then(|p| st.by_uid.get(p).copied())
.unwrap_or(ORPHAN_INO);
let uid = node.uid.clone();
st.entries.insert(
ino,
Entry {
uid,
parent,
node,
access: pdfs_core::Access::Unknown,
lookup_count: 1,
open_count: 0,
unlinked: false,
},
);
}
st.hydrate_access();
// Pass 3: rebuild child listings for fully-enumerated folders.
let listings = listed_children(&st.entries, listed_dirs);
st.children.extend(listings);
info!(nodes = st.entries.len(), "hydrated metadata cache from db");
}
/// Rebuild the in-memory pending map from the `pending_op` table on mount.
///
/// A queued write survives a restart — that is the point of persisting it —
/// so until the drain worker gets to it, reads of that file must still come
/// from its staged blob rather than the remote's older revision.
///
/// A row whose blob has gone missing is dropped: there is nothing left to
/// upload, and keeping it would fail forever.
fn hydrate_pending(&self) {
let ops = match self.db.pending_ops() {
Ok(ops) => ops,
Err(e) => {
error!(error = %e, "loading pending uploads failed");
return;
}
};
let mut map = self.pending.lock();
let mut restored = 0usize;
for op in ops {
let Some(uid) = parse_node_uid(&op.uid) else {
error!(uid = %op.uid, id = op.id, "pending op has an unparseable uid; dropping");
let _ = self.db.delete_op(op.id);
continue;
};
if op.kind == OP_TRASH {
self.hidden.lock().insert(uid.clone());
}
restored += 1;
// Only a revision must have a blob. A create carries none until
// something is written to it (`touch` offline is a legitimate op
// with nothing to serve), and a rename never has one. A trash keeps
// the blob of a create it withdrew only to find the node by, not
// to serve. All still have to be replayed, so only the blob — if
// any — is checked.
if op.kind == OP_TRASH || (op.blob_path.is_none() && op.kind != OP_REVISION) {
continue;
}
let parsed = op
.meta_json
.as_deref()
.and_then(|j| serde_json::from_str::<StagedWrite>(j).ok())
.zip(op.blob_path.as_deref().map(PathBuf::from));
let Some((meta, path)) = parsed else {
error!(uid = %op.uid, id = op.id, "pending op is unreadable; dropping");
self.drop_unrecoverable_op(&op, &uid);
restored -= 1;
continue;
};
if !path.exists() {
error!(%uid, path = %path.display(), "staged blob is gone; dropping pending op");
self.drop_unrecoverable_op(&op, &uid);
restored -= 1;
continue;
}
map.insert(uid, PendingRevision { path, meta });
}
if restored > 0 {
info!(count = restored, "restored pending ops");
}
}
/// Re-attach staged blobs that no queued op refers to.
///
/// `staging/` holds the only copy of writes the kernel has already been
/// told succeeded. Three release-path failures put bytes there and return
/// `EIO` without leaving a `pending_op` row behind — an access change
/// mid-release, a write overlapping an undrained incomplete edit, a create
/// that drained out from under its own handle. Until now nothing ever
/// walked the directory again: `hydrate_pending` reads the op table and
/// [`recover_fsynced_writes`](Self::recover_fsynced_writes) reads
/// `recovery/`, so an orphaned staged blob was invisible to the daemon and
/// to the user alike, and stayed that way until someone found the file.
///
/// Runs once at mount, after `hydrate_pending`, so `pending` already
/// describes what the queue owns. An orphan whose sidecar names a real node
/// is queued as an ordinary revision — the condition that made it fail is
/// gone by definition, since the op it collided with has since drained.
/// Anything that cannot be addressed (no sidecar, an unparseable uid, a
/// `local~` placeholder whose create is gone) is left in place and named in
/// the log: unexplained bytes are never evidence that user data is
/// disposable.
fn reconcile_staging(&self) {
let staged = self.cache.staged_writes();
if staged.is_empty() {
return;
}
let claimed = match self.db.op_blob_paths() {
Ok(paths) => paths,
Err(e) => {
error!(error = %e, "cannot read queued blob paths; skipping staging reconcile");
return;
}
};
let mut queued = 0usize;
let mut stranded = 0usize;
let mut retired = 0usize;
let mut retired_bytes = 0u64;
let mut example: Option<PathBuf> = None;
for (blob, meta) in staged {
if claimed.contains(&blob.to_string_lossy().into_owned()) {
continue;
}
let addressable = meta.as_ref().and_then(|m| parse_node_uid(&m.uid));
let Some((meta, uid)) = meta.zip(addressable).filter(|(_, uid)| !is_local_uid(uid))
else {
if let Some(bytes) = orphan_past_retention(&blob) {
retired += 1;
retired_bytes += bytes;
warn!(blob = %blob.display(), bytes,
age_days = STAGING_ORPHAN_RETAIN.as_secs() / 86_400,
"retiring staged bytes that name no node and have outlived retention");
self.cache.discard_staged(&blob);
continue;
}
stranded += 1;
example.get_or_insert_with(|| blob.clone());
continue;
};
if self.pending.lock().contains_key(&uid) {
// A newer write for the same node is queued and carries its own
// blob. These bytes are older; keeping them is right, uploading
// them over the newer ones is not.
stranded += 1;
warn!(%uid, blob = %blob.display(),
"staged bytes are superseded by a queued write; kept for recovery");
continue;
}
if self
.db
.has_pending_op(&uid.to_string(), OP_TRASH)
.unwrap_or(false)
{
// The node was deleted while a handle still read these bytes,
// and the daemon stopped before its trash went out. Queued again,
// they lost their blob when the trash landed and retried on it
// for good (`docs/BUGS.md` B188).
info!(%uid, blob = %blob.display(),
"staged bytes belong to a node whose trash is queued; discarded");
self.cache.discard_staged(&blob);
continue;
}
let op = PendingOp {
id: 0,
kind: OP_REVISION.to_string(),
uid: uid.to_string(),
parent_uid: None,
name: None,
blob_path: Some(blob.to_string_lossy().into_owned()),
meta_json: Some(serde_json::to_string(&meta).unwrap_or_default()),
created_at: now_millis(),
attempts: 0,
last_error: None,
next_attempt_at: now_millis(),
};
match self.db.enqueue_op(&op) {
Ok((_, superseded)) => {
if let Some(old) = superseded {
self.cache.discard_staged(Path::new(&old));
}
self.pending.lock().insert(
uid.clone(),
PendingRevision {
path: blob.clone(),
meta,
},
);
queued += 1;
}
Err(e) => {
stranded += 1;
error!(%uid, blob = %blob.display(), error = %e,
"cannot queue an orphaned staged write; bytes kept");
}
}
}
if stranded > 0 {
// One line, not one per blob. A tree of them says exactly as much as
// a single count and an example does, and said it on every mount:
// 86 identical warnings for the same 86 files, boot after boot.
warn!(
count = stranded,
example = %example.map(|b| b.display().to_string()).unwrap_or_default(),
"staged bytes belong to no queued upload and name no node; kept for recovery"
);
}
if queued > 0 || stranded > 0 || retired > 0 {
info!(
queued,
stranded, retired, retired_bytes, "reconciled orphaned staged writes"
);
}
}
/// Queue the writes that an unclean shutdown caught between `fsync(2)` and
/// `close(2)`.
///
/// `fsync` promises the bytes survive a crash, but the queueing that makes a
/// write outlive the daemon happens at `release`. A crash in between used to
/// lose the data outright, because the scratch directory is cleared at open.
/// Now `fsync` leaves a sidecar, `ContentCache::open` moves those blobs to
/// `recovery/`, and this walks them into the same staging + queued-op path a
/// normal release takes.
///
/// Runs after [`hydrate_pending`](Self::hydrate_pending), which is what makes
/// the incomplete-blob check in [`enqueue_staged_write`](Self::enqueue_staged_write)
/// meaningful: a recovered partial write whose earlier write is still queued
/// must not gap-fill from a remote revision that no longer describes the
/// file. With `pending` already loaded, that case is detected and the bytes
/// are parked rather than mis-filled.
///
/// Failure is per-write and never fatal: a node that no longer exists, or an
/// op that cannot be queued, leaves its blob in `recovery/` for the next run
/// (and for a human) instead of taking the mount down.
fn recover_fsynced_writes(&self) {
let recovered = self.cache.recovered_writes();
if recovered.is_empty() {
return;
}
let mut queued = 0usize;
for (blob, meta) in recovered {
let Some(uid) = parse_node_uid(&meta.uid) else {
error!(uid = %meta.uid, "recovered write has an unparseable uid; keeping bytes");
continue;
};
// If the write is incomplete and an earlier edit is still queued, merging it
// now would use the wrong remote base. enqueue_staged_write would abandon it
// to staging; keep it in recovery instead so the drain loop can queue it
// once the queue clears.
if !meta.complete && self.pending.lock().contains_key(&uid) {
continue;
}
// The inode is only used to stamp the in-memory tree with the new
// size, and nothing is interned this early — `hydrate` reads the
// size back off `pending` when the node is first looked up, so 0
// (no such inode) is correct rather than merely tolerable.
match self.enqueue_staged_write(&uid, 0, &blob, meta) {
Ok(()) => {
self.cache.discard_recovered(&blob);
queued += 1;
}
Err(e) => {
error!(%uid, blob = %blob.display(), error = ?e,
"cannot queue a recovered write; bytes kept for the next run");
// Some failures (a partial write parked by
// `stage_orphaned_write`) still consume the blob. Its sidecar
// would then describe nothing, so retire it — the bytes are
// in staging, which is where a human looks for them.
if !blob.exists() {
self.cache.discard_recovered(&blob);
}
}
}
}
if queued > 0 {
info!(
count = queued,
"recovered fsynced writes from an unclean shutdown"
);
}
}
/// Discard an op that can never be performed, because the bytes it was to
/// upload are gone from staging (something outside the daemon deleted them).
///
/// For a node that only ever existed locally, the placeholder goes too. Its
/// content is unrecoverable and nothing will ever mint it a real uid, so
/// leaving the row would strand a file in the tree that can be listed but
/// never read and never uploaded.
fn drop_unrecoverable_op(&self, op: &PendingOp, uid: &NodeUid) {
let _ = self.db.delete_op(op.id);
if is_local_uid(uid) {
error!(%uid, name = op.name.as_deref().unwrap_or("?"),
"discarding a node whose only copy was lost");
if let Err(e) = self.db.delete_node(uid) {
warn!(%uid, error = %e, "db delete_node failed for lost local node");
}
}
}
/// Whether [`Core::ensure_children`] would answer for `ino` without
/// touching the network. Lets a handler decide between answering inline
/// and handing off to a worker.
///
/// A resident listing is not enough on its own. A shared folder's, or the
/// Shared with me root's, is fetched again once it is a minute old, and
/// that fetch ran on the dispatch loop, so every request on the mount
/// waited for Drive to answer (`docs/BUGS.md` B219).
fn children_cached(&self, ino: u64) -> bool {
let (uid, resident) = {
let st = self.state();
match st.entries.get(&ino) {
Some(e) => (e.uid.clone(), st.children.contains_key(&ino)),
// `list_children` answers ENOENT without the network.
None => return true,
}
};
if !resident {
return false;
}
let freshness = if is_virtual_uid(&uid) {
Some(SHARED_WITH_ME_SYNCED_MS.to_string())
} else if !self.is_own_or_virtual(&uid) && !is_local_uid(&uid) {
Some(Self::shared_folder_freshness_key(&uid))
} else {
None
};
freshness.is_none_or(|key| {
!listing_needs_refresh(
self.online.load(Ordering::Relaxed),
self.shared_listing_stale(&key),
)
})
}
/// Re-apply the optimistic size of any queued write to `nodes`.
///
/// A node that arrives from the remote (or from its DB row) carries the size
/// of the revision the *server* holds, which for a file with a write still
/// queued is the pre-write size — often 0 for a file created moments ago.
/// Interning it as-is silently reverts the optimistic size that
/// `record_pending_write` stamped, and a file that stats as 0 bytes is a file
/// the kernel will not issue a single `read` for: `cat` prints nothing and
/// the staged blob that `read_range` would have served is never asked for.
/// That reads as data loss even though nothing is lost (B11).
///
/// [`Core::hydrate`] does the same thing for the restart case. This covers
/// every *live* re-enumeration — which is what a rename or move triggers,
/// since both invalidate the listings they touch.
///
/// Snapshots the pending map and returns before any caller takes the state
/// lock: no site in the daemon holds `pending` and `state` at once, and this
/// is not the place to become the first.
fn stamp_pending_sizes(&self, nodes: &mut [Node]) {
let sizes: HashMap<NodeUid, u64> = {
let pending = self.pending.lock();
if pending.is_empty() {
return;
}
pending
.iter()
.map(|(uid, pr)| (uid.clone(), pr.meta.len))
.collect()
};
apply_pending_sizes(nodes, &sizes);
}
/// Read the inputs for virtual-root publication without changing SQLite.
///
/// The caller validates the listing snapshot after these reads. Keeping all
/// writes out of this phase prevents an invalidated/repopulated root listing
/// from changing the synthetic node's persisted visibility or FTS ancestry.
fn prepare_virtual_root(&self, real_names: &HashSet<String>) -> Result<VirtualRootPlan, Errno> {
let pinned = self.db.state_str(SHARED_WITH_ME_NAME).map_err(|error| {
error!(%error, "reading shared-root display name failed");
Errno::EIO
})?;
let (name, visible) = virtual_root_name(real_names, pinned.as_deref());
let uid = shared_with_me_uid();
let previous_node = self.db.node_by_uid(&uid.to_string()).map_err(|error| {
error!(%error, "reading synthetic shared root failed");
Errno::EIO
})?;
let mut node = previous_node.unwrap_or_else(|| {
virtual_node(self.primary_root_uid.clone(), name.clone(), now_secs())
});
node.parent_uid = Some(self.primary_root_uid.clone());
node.name = name;
node.kind = NodeKind::Folder;
// A later real-name collision suppresses both the dentry and its search
// hit. The row and pinned name remain, and clearing the collision
// restores both by writing `trashed = false` again.
node.trashed = !visible;
node.membership = None;
Ok(VirtualRootPlan { node, visible })
}
/// Reconcile the synthetic dentry against an already-known primary-root
/// listing. The node remains persisted and pinned even while a real folder
/// of the same name temporarily suppresses its dentry.
fn reconcile_virtual_root_dentry(
&self,
parent_ino: u64,
snapshot: RootListingSnapshot,
) -> Result<(), Errno> {
let _publication = self.shared_publication.lock();
let plan = self.prepare_virtual_root(&snapshot.real_names())?;
let mut st = self.state();
// State methods already establish the state -> DB lock order. No path
// holds the DB connection while acquiring state, including shared event
// publication, so keeping state locked across this one transaction
// closes the crash window without introducing an inverse order.
publish_virtual_root_in_listing(&self.db, &mut st, parent_ino, &snapshot, plan)?;
drop(st);
self.flush_access_changes();
Ok(())
}
fn resident_root_listing_snapshot(&self, ino: u64) -> Result<RootListingSnapshot, Errno> {
RootListingSnapshot::capture(&self.state(), ino).ok_or(Errno::EAGAIN)
}
fn shared_folder_freshness_key(uid: &NodeUid) -> String {
format!("{SHARED_FOLDER_SYNCED_PREFIX}{uid}")
}
fn shared_listing_stale(&self, key: &str) -> bool {
if self
.shared_refresh_deadlines
.lock()
.is_fresh(key, Instant::now())
{
return false;
}
self.listing_stale(key, SHARED_LISTING_TTL)
}
fn mark_shared_refresh_success(&self, key: &str) {
self.shared_refresh_deadlines
.lock()
.mark(key, Instant::now(), SHARED_LISTING_TTL);
if let Err(error) = self.db.set_state_i64(key, now_millis()) {
warn!(key, %error, "persisting shared-list freshness failed");
}
}
fn invalidate_shared_refreshes(&self) {
self.shared_generation.fetch_add(1, Ordering::SeqCst);
self.shared_refresh_deadlines.lock().clear();
}
fn is_own_or_virtual(&self, uid: &NodeUid) -> bool {
is_own_or_virtual_uid(uid, &self.primary_root_uid.volume_id)
}
fn ensure_shared_children(&self, ino: u64) -> Result<(), Errno> {
let online = self.online.load(Ordering::Relaxed);
let resident = self.state().children.contains_key(&ino);
match shared_listing_plan(
resident,
online,
self.shared_listing_stale(SHARED_WITH_ME_SYNCED_MS),
) {
SharedListingPlan::Resident => return Ok(()),
SharedListingPlan::Persisted => {
// A completed snapshot remains useful offline even after an
// event expired its freshness stamp.
let nodes = self
.db
.visible_children(&shared_with_me_uid())
.map_err(|error| {
error!(%error, "loading persisted shared roots failed");
Errno::EIO
})?;
let mut st = self.state();
if st.children.contains_key(&ino) {
return Ok(());
}
let children = nodes
.into_iter()
.map(|node| {
let access = st
.share_access
.get(&node.uid)
.copied()
.unwrap_or(Access::Viewer);
st.intern_published_share_root(ino, node, access)
})
.collect();
st.children.insert(ino, children);
drop(st);
self.flush_access_changes();
return Ok(());
}
SharedListingPlan::Refresh => {}
}
let generation = self.shared_generation.load(Ordering::SeqCst);
let accepted = accepted_share_provenance(
self.block_on_within(BATCH_CALL_TIMEOUT, self.client.enumerate_shared_with_me())
.map_err(|error| {
error!(%error, "enumerating shared roots failed");
Errno::EIO
})?,
);
let nodes = if accepted.uids.is_empty() {
Vec::new()
} else {
self.block_on_within(
BATCH_CALL_TIMEOUT,
self.drive.enumerate_nodes_light(&accepted.uids),
)
.map_err(|error| {
error!(%error, "materializing shared roots failed");
Errno::EIO
})?
};
let virtual_uid = shared_with_me_uid();
let published = prepare_shared_roots(&accepted, nodes, &virtual_uid);
let _publication = self.shared_publication.lock();
if !refresh_generation_is_current(generation, self.shared_generation.load(Ordering::SeqCst))
{
return Err(Errno::EAGAIN);
}
let removed = match self
.db
.publish_shared_roots(&virtual_uid, &accepted.uids, &published)
{
Ok(removed) => removed,
Err(error) => {
error!(%error, "publishing shared-root listing failed");
let mut st = self.state();
for uid in &accepted.uids {
st.downgrade_shared_subtree(uid);
}
drop(st);
self.flush_access_changes();
return Err(Errno::EIO);
}
};
let snapshot = self.db.visible_children(&virtual_uid).map_err(|error| {
error!(%error, "loading published shared-root listing failed");
Errno::EIO
})?;
let published_access: HashMap<NodeUid, Access> = published
.iter()
.map(|root| (root.node.uid.clone(), root.access))
.collect();
let mut st = self.state();
for uid in &removed {
st.hide_shared_root(uid);
}
let mut children = Vec::with_capacity(snapshot.len());
for node in snapshot {
let uid = node.uid.clone();
let access = published_access
.get(&uid)
.copied()
.unwrap_or(Access::Viewer);
let child = st.intern_published_share_root(ino, node, access);
children.push(child);
}
st.children.insert(ino, children);
drop(st);
self.flush_access_changes();
self.mark_shared_refresh_success(SHARED_WITH_ME_SYNCED_MS);
if let Some(notifier) = self.notifier.get() {
let _ = notifier.inval_inode(ino);
}
Ok(())
}
/// The nodes of `before`, queued in `folder_uid` when its listing was asked
/// of Drive, that are no longer queued in `after`, as the DB has them now.
///
/// Their create landed while the listing was on the wire, after Drive took
/// it, and so is in neither. One unlinked or moved away meanwhile is not
/// under the folder by that name any more and stays out.
fn landed_meanwhile(
&self,
folder_uid: &NodeUid,
before: Vec<Node>,
after: &[Node],
) -> Vec<Node> {
let still: HashSet<&str> = after.iter().map(|node| node.name.as_str()).collect();
let landed: HashSet<String> = before
.into_iter()
.map(|node| node.name)
.filter(|name| !still.contains(name.as_str()))
.collect();
if landed.is_empty() {
return Vec::new();
}
match self.db.known_children(folder_uid) {
Ok(known) => known
.into_iter()
.filter(|node| landed.contains(&node.name))
.collect(),
Err(e) => {
warn!(%folder_uid, error = %e, "db known_children failed");
Vec::new()
}
}
}
/// Serve a listing without the network: the one already in memory if any,
/// else whatever children the DB still knows. The fallback only answers
/// until the link is back, when [`Core::relist_offline_listings`] drops it:
/// a listing dropped as stale may lack a child another device made, and a
/// create over that name lands as a conflict copy rather than an `EIO` now.
fn adopt_stale_listing(
&self,
ino: u64,
folder_uid: &NodeUid,
cached: bool,
primary_root: bool,
) -> Result<(), Errno> {
if cached {
return Ok(());
}
let nodes = self.db.known_children(folder_uid).map_err(|e| {
warn!(%folder_uid, error = %e, "db known_children failed");
Errno::EIO
})?;
debug!(%folder_uid, count = nodes.len(), "offline: listing a folder from what the DB knows");
self.link.stale_listings.lock().insert(ino);
self.adopt_db_listing(ino, folder_uid, nodes, primary_root)
}
/// Relist every folder [`Core::adopt_stale_listing`] served while offline.
pub(crate) fn relist_offline_listings(&self) {
let stale = std::mem::take(&mut *self.link.stale_listings.lock());
if stale.is_empty() {
return;
}
let mut st = self.state();
for &ino in &stale {
st.invalidate_listing(ino);
}
drop(st);
for mount in self.states.live() {
if let Some(notifier) = mount.notifier.get() {
for &ino in &stale {
let _ = notifier.inval_inode(ino);
}
}
}
}
/// Install `nodes`, read from the DB, as `ino`'s listing.
fn adopt_db_listing(
&self,
ino: u64,
folder_uid: &NodeUid,
mut nodes: Vec<Node>,
primary_root: bool,
) -> Result<(), Errno> {
let folder_uid = folder_uid.clone();
if primary_root {
nodes.retain(|node| !is_virtual_uid(&node.uid));
}
// Before the lock: a DB row carries the size the server last
// sealed, which a queued write is ahead of (B11).
self.stamp_pending_sizes(&mut nodes);
let hidden = self.hidden.lock().clone();
let mut st = self.state();
if st.children.contains_key(&ino) {
return Ok(());
}
let mut child_inos = Vec::with_capacity(nodes.len());
let mut needs_size = Vec::new();
for node in nodes {
if !node_visible(&node, &folder_uid, &hidden) {
continue;
}
if matches!(
&node.kind,
NodeKind::File {
claimed_size: None,
..
}
) {
needs_size.push(node.uid.clone());
}
child_inos.push(st.intern_from_db(ino, node));
}
st.children.insert(ino, child_inos);
let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap());
drop(st);
self.flush_access_changes();
if let Some(snapshot) = root_snapshot {
self.reconcile_virtual_root_dentry(ino, snapshot)?;
}
// Rows persisted from a cheap enumeration whose upgrade never
// ran (a restart in between, say) still owe their real sizes.
self.spawn_size_upgrade(ino, needs_size);
Ok(())
}
/// Enumerate `ino`'s children from the remote and cache them. No-op if the
/// directory has already been listed. Network I/O happens without the lock
/// held so concurrent metadata reads aren't blocked behind a fetch.
///
/// A listing that changed while it was being published answers EAGAIN: the
/// root listed again after a restart, an event, a newer refresh of a shared
/// folder. The listing is still owed, so it is taken again rather than
/// handed to the lookup or open that asked (B159).
fn ensure_children(&self, ino: u64) -> Result<(), Errno> {
for _ in 1..LISTING_ATTEMPTS {
match self.list_children(ino) {
Err(e) if e.code() == Errno::EAGAIN.code() => {
debug!(ino, "listing changed while it was published; listing again");
}
done => return done,
}
}
self.list_children(ino)
}
fn list_children(&self, ino: u64) -> Result<(), Errno> {
let (folder_uid, cached) = {
let st = self.state();
match st.entries.get(&ino) {
Some(e) => (e.uid.clone(), st.children.contains_key(&ino)),
None => return Err(Errno::ENOENT),
}
};
if is_virtual_uid(&folder_uid) {
return self.ensure_shared_children(ino);
}
let primary_root =
is_primary_root_listing(self.primary, &folder_uid, &self.primary_root_uid);
let foreign = !self.is_own_or_virtual(&folder_uid) && !is_local_uid(&folder_uid);
let foreign_key = foreign.then(|| Self::shared_folder_freshness_key(&folder_uid));
let refresh_foreign = foreign_key.as_deref().is_some_and(|key| {
listing_needs_refresh(
self.online.load(Ordering::Relaxed),
self.shared_listing_stale(key),
)
});
if cached && !refresh_foreign {
if primary_root {
self.reconcile_virtual_root_dentry(ino, self.resident_root_listing_snapshot(ino)?)?;
}
return Ok(());
}
// Offline fast path: a folder the DB still records as fully enumerated
// can be rebuilt from disk without hitting the API, even if its listing
// was trimmed from the hot cache mid-run.
let cached_nodes = if refresh_foreign {
Ok(None)
} else {
self.db.children_if_listed(&folder_uid)
};
match cached_nodes {
Ok(Some(nodes)) => return self.adopt_db_listing(ino, &folder_uid, nodes, primary_root),
Ok(None) => {}
Err(e) => warn!(%folder_uid, error = %e, "db children_if_listed failed"),
}
// Drive knows nothing by a stand-in. Until its mkdir lands the folder
// is only here, and once it has, the DB holds what Drive does. A
// listing dropped meanwhile was asked of Drive, and every lookup in
// the folder failed with EIO (`docs/BUGS.md` B125).
if is_local_uid(&folder_uid) {
let nodes = self.db.known_children(&folder_uid).map_err(|e| {
warn!(%folder_uid, error = %e, "db known_children failed");
Errno::EIO
})?;
return self.adopt_db_listing(ino, &folder_uid, nodes, primary_root);
}
if !self.is_online() {
return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root);
}
let refresh_generation =
refresh_foreign.then(|| self.shared_generation.load(Ordering::SeqCst));
// What is queued before Drive is asked: a create that lands while the
// listing is on the wire is in neither Drive's answer nor the queue read
// after it (`docs/BUGS.md` B166).
let queued_before = (!foreign)
.then(|| self.db.queued_children(&folder_uid).ok())
.flatten();
let uids = match self
.rt
.block_on(self.drive.enumerate_folder_children_node_uids(&folder_uid))
{
Ok(uids) => uids,
Err(e) if self.lost_link(&e, "list folder") => {
return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root);
}
Err(e) => {
error!(%folder_uid, error = %e, "enumerate folder children failed");
return Err(Errno::EIO);
}
};
// Cheap enumeration: `Light` skips unlocking each *file's* node key,
// which is an S2K derivation per file and was ~74% of the cost of a cold
// listing (B12 — measured with `perf`, 64% of cycles in SHA-256 alone).
// Folders are unlocked either way; their keys are what the children are
// decrypted with, so the walk cannot proceed without them.
//
// The price is that files come back without a `claimed_size`, so
// `node_size` falls back to the *ciphertext* size until
// `spawn_size_upgrade` below fills the real one in.
let mut nodes = match self.rt.block_on(self.drive.enumerate_nodes_light(&uids)) {
Ok(nodes) => nodes,
Err(e) if self.lost_link(&e, "list folder") => {
return self.adopt_stale_listing(ino, &folder_uid, cached, primary_root);
}
Err(e) => {
error!(%folder_uid, error = %e, "enumerate nodes failed");
return Err(Errno::EIO);
}
};
// Drive shows none of what is still queued for the folder (B132).
// A shared folder's listing is published as Drive has it.
if !foreign {
match self.db.queued_children(&folder_uid) {
Ok((mut queued, gone)) => {
if let Some((before, _)) = queued_before {
queued.extend(self.landed_meanwhile(&folder_uid, before, &queued));
}
overlay_queued(&mut nodes, queued, &gone)
}
Err(e) => warn!(%folder_uid, error = %e, "db queued_children failed"),
}
}
// Same as the DB path above: the remote's size for a file with a write
// still queued is the pre-write one (B11).
self.stamp_pending_sizes(&mut nodes);
let hidden = self.hidden.lock().clone();
let mut filtered_nodes: Vec<Node> = nodes
.into_iter()
.filter(|node| node_visible(node, &folder_uid, &hidden))
.collect();
if foreign {
for node in &mut filtered_nodes {
node.parent_uid = Some(folder_uid.clone());
}
let _publication = self.shared_publication.lock();
if !refresh_generation_is_current(
refresh_generation.expect("foreign refresh has a generation"),
self.shared_generation.load(Ordering::SeqCst),
) {
return Err(Errno::EAGAIN);
}
let removed = self
.db
.publish_foreign_children(&folder_uid, &uids, &filtered_nodes)
.map_err(|error| {
error!(%folder_uid, %error, "publishing foreign-folder listing failed");
Errno::EIO
})?;
let mut snapshot = self.db.visible_children(&folder_uid).map_err(|error| {
error!(%folder_uid, %error, "loading published foreign-folder listing failed");
Errno::EIO
})?;
self.stamp_pending_sizes(&mut snapshot);
snapshot.retain(|node| node_visible(node, &folder_uid, &hidden));
let needs_size: Vec<NodeUid> = snapshot
.iter()
.filter(|node| {
matches!(
&node.kind,
NodeKind::File {
claimed_size: None,
..
}
)
})
.map(|node| node.uid.clone())
.collect();
let mut st = self.state();
for uid in &removed {
st.hide_foreign_subtree(uid);
}
let children = snapshot
.into_iter()
.map(|node| st.intern_from_db(ino, node))
.collect();
st.children.insert(ino, children);
drop(st);
self.flush_access_changes();
self.mark_shared_refresh_success(
foreign_key
.as_deref()
.expect("foreign refresh has a freshness key"),
);
self.spawn_size_upgrade(ino, needs_size);
return Ok(());
}
let preassigned = state::inos_for(&self.db, &filtered_nodes);
let mut st = self.state();
// Lost the race? Another thread already populated it.
if st.children.contains_key(&ino) {
return Ok(());
}
let mut child_inos = Vec::with_capacity(filtered_nodes.len());
// Files whose real size the cheap enumeration could not read. Collected
// before interning so the upgrade below has the uids without re-walking.
let needs_size: Vec<NodeUid> = filtered_nodes
.iter()
.filter(|n| {
matches!(
&n.kind,
NodeKind::File {
claimed_size: None,
..
}
)
})
.map(|n| n.uid.clone())
.collect();
let inos = st.intern_batch_with_inos(ino, filtered_nodes, preassigned);
child_inos.extend(inos);
st.children.insert(ino, child_inos);
let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap());
// Record the listing as complete so a later restart (or a trimmed hot
// cache) can rebuild it from the DB without the API. Written with the
// children's rows when the lock goes, not under it.
st.record_listed(folder_uid.clone());
drop(st);
self.flush_access_changes();
if let Some(snapshot) = root_snapshot {
self.reconcile_virtual_root_dentry(ino, snapshot)?;
}
self.spawn_size_upgrade(ino, needs_size);
Ok(())
}
/// Resolve the real size of every file in `parent`'s listing still carrying
/// a provisional one, returning when they are in `state`.
///
/// Called when a `stat` lands on such a file. That covers the paths
/// [`Core::ensure_children`] cannot: a listing rebuilt from the DB, or one
/// restored by [`Core::hydrate`] on mount, whose rows were persisted before
/// an earlier upgrade had a chance to run. Gathers the whole folder rather
/// than the one file asked for, because a `stat` of one entry in a listing
/// almost always means a `stat` of all of them.
fn upgrade_sizes_for_parent(
&self,
ino: u64,
uid: &NodeUid,
parent: u64,
) -> Option<Arc<SizeUpgrade>> {
let provisional = |e: &Entry| {
matches!(
&e.node.kind,
NodeKind::File {
claimed_size: None,
..
}
)
};
let (key, mut missing): (u64, Vec<NodeUid>) = {
let st = self.state();
match st.children.get(&parent) {
// The listing is resident: batch the whole folder under its
// inode, so the rest of an `ls -l` rides along on this fetch.
Some(kids) => (
parent,
kids.iter()
.filter_map(|k| st.entries.get(k))
.filter(|e| provisional(e))
.map(|e| e.uid.clone())
.collect(),
),
// It is not, and returning here is what let a provisional size
// reach the caller anyway: a rename invalidates its parents'
// listings, so a freshly renamed file always landed in this
// branch. Resolve the single node instead, keyed by its own
// inode — folder and file inodes share one space, so the two
// single-flight keys cannot collide. (Same shape as B4: an
// early return that assumed the hot cache was authoritative.)
None => (ino, Vec::new()),
}
};
// The node that was actually asked about is never optional, whichever
// branch produced the batch.
if !missing.iter().any(|u| u == uid) {
missing.push(uid.clone());
}
// Returned, not awaited here: the caller is a `getattr` that must not
// answer with a provisional size (`docs/BUGS.md` B14), and it parks its reply
// on this batch rather than holding a thread until it lands.
self.upgrade_sizes(key, missing)
}
/// Fill in the true sizes of files a `Light` enumeration returned without
/// one, on a worker, after the listing has already been served.
///
/// This is the other half of the split in [`Core::ensure_children`]. The
/// listing itself needs only names and parentage, so it is served from the
/// cheap enumeration immediately; the S2K-per-file work that produces
/// `claimed_size` happens here, off the path the user is waiting on.
///
/// **Sizes are provisional until this lands.** `node_size` falls back to
/// `total_size_on_storage`, the *ciphertext* size, which is slightly larger
/// than the real one. A read must not plan on it: since B84 a size the
/// revision disagrees with fails the read, so `read` settles the size first
/// ([`Core::resolve_provisional_size`], `docs/BUGS.md` B100). In `stat` it is a
/// discrepancy that closes within a round trip, not a repeat of B11 (which
/// reported **0** and so suppressed reads entirely).
///
/// Single-flight per folder: a `stat` of every entry in a fresh listing is
/// the normal case, and each one must not start its own upgrade.
fn spawn_size_upgrade(&self, folder_ino: u64, uids: Vec<NodeUid>) {
self.upgrade_sizes(folder_ino, uids);
}
/// Start (or join) the size upgrade for `key`, returning the batch a caller
/// can wait on — `None` when there was nothing to upgrade.
///
/// Single-flight per `key`. The fetch runs on its own thread rather than on
/// the [`Workers`] pool: callers wait on `Lane::Meta`, so a batch queued onto
/// that lane could have a wide enough `ls -l` fill it with threads waiting
/// for a job that can never be scheduled. `Lane::Transfer` would swap that
/// for starvation behind bulk reads. One short-lived thread per folder,
/// bounded by the single-flight, avoids both.
fn upgrade_sizes(&self, key: u64, uids: Vec<NodeUid>) -> Option<Arc<SizeUpgrade>> {
if uids.is_empty() {
return None;
}
let claim = claim_size_upgrade(&mut self.size_upgrades.lock(), key, uids);
match claim {
SizeUpgradeClaim::Joined(slot) | SizeUpgradeClaim::Queued(slot) => Some(slot),
SizeUpgradeClaim::Start(slot, uids) => {
let core = self.clone();
let worker = slot.clone();
std::thread::spawn(move || core.run_size_upgrade(key, uids, worker));
Some(slot)
}
}
}
/// Hold `respond` back until `ino` has a real size, `slot`'s batch ends, or
/// [`SizeUpgrade::WAIT`] elapses — then run it, on the queue's thread.
///
/// The caller returns immediately. That is the whole point: the thread it
/// was running on is a FUSE dispatch thread or a [`Workers`] one, and both
/// are needed by everything else on the mount (audit F4).
fn await_size(
&self,
ino: u64,
slot: Arc<SizeUpgrade>,
respond: impl FnOnce() + Send + 'static,
) {
let core = self.clone();
let waiter = SizeWaiter {
slot,
resolved: Box::new(move || core.size_is_real(ino)),
deadline: Instant::now() + SizeUpgrade::WAIT,
respond: Box::new(respond),
};
if !self.size_waiters.park(waiter) {
return;
}
let queue = self.size_waiters.clone();
if let Err(error) = std::thread::Builder::new()
.name("pdfs-size-waiters".into())
.spawn(move || queue.serve())
{
// Without a thread nothing would ever answer these. A provisional
// size is wrong; a `stat` that never returns wedges the caller.
warn!(%error, "no thread for parked size waiters; answering provisionally");
self.size_waiters.answer_all();
}
}
/// Fetch `uids` in chunks, applying and announcing each one as it lands.
///
/// Chunked so a waiter is released as soon as *its* file is resolved. A
/// single 793-node batch took ~80 s, which outran [`SizeUpgrade::WAIT`] and
/// put provisional sizes back in front of callers — the bug this was
/// supposed to fix (`docs/BUGS.md` B14).
///
/// When the batch ends, the thread takes over the oldest batch the cap held
/// back, so at most [`MAX_SIZE_UPGRADES`] threads run however many folders
/// a recursive walk touches (`docs/BUGS.md` B100).
fn run_size_upgrade(&self, key: u64, uids: Vec<NodeUid>, slot: Arc<SizeUpgrade>) {
let (mut key, mut uids, mut slot) = (key, uids, slot);
loop {
for chunk in uids.chunks(SIZE_UPGRADE_CHUNK) {
let result =
self.block_on_within(BATCH_CALL_TIMEOUT, self.drive.enumerate_nodes(chunk));
self.apply_size_upgrade(key, result);
slot.chunk_done();
}
// Both of these must happen however the loop ended: a folder whose
// upgrade failed has to be retryable, and its waiters released.
let next = next_size_upgrade(&mut self.size_upgrades.lock(), key);
slot.finish();
let Some(next) = next else {
return;
};
(key, uids, slot) = next;
}
}
/// Settle the size of a file whose listing only carries the provisional
/// one, before a read plans its blocks on it or a write handle starts from
/// it. Returns the real size, or `None` when the fetch did not produce one.
///
/// A provisional size is the ciphertext size. The read path checks the
/// size it is given against the revision's own block table and fails the
/// read on a mismatch (B84), so reading on the provisional size is an `EIO`
/// every time. `getattr` usually settles the size first, but not when its
/// wait times out, so the read cannot rely on it (`docs/BUGS.md` B100).
fn resolve_provisional_size(&self, ino: u64, uid: &NodeUid) -> Option<u64> {
let parent = self.state().entries.get(&ino)?.parent;
// Without a network the answer is `None` either way; the bound keeps a
// dead link from holding the open or read for the SDK's two minutes.
if !self.is_online() {
return None;
}
let result = self.block_on_bounded(self.drive.enumerate_nodes(std::slice::from_ref(uid)));
if let Err(e) = &result {
self.lost_link(e, "settle a provisional size");
}
self.apply_size_upgrade(parent, result);
let st = self.state();
match &st.entries.get(&ino)?.node.kind {
NodeKind::File {
claimed_size: Some(size),
..
} => Some((*size).max(0) as u64),
_ => None,
}
}
/// Wait, up to [`SizeUpgrade::WAIT`] as a `stat` would, for the real size
/// of every file in `ino`'s listing that still carries a provisional one.
///
/// For callers that read a whole listing's sizes straight from `state` and
/// do not run on a FUSE thread: `pdfs ls` and the app's file browser. A
/// listing just fetched has the ciphertext size of every file in it until
/// its upgrade lands, and they showed those (`docs/BUGS.md` B115).
fn settle_listing_sizes(&self, ino: u64) {
// Offline the fetch cannot succeed; the wait would only delay the
// answer it ends up giving anyway.
if !self.is_online() {
return;
}
let missing: Vec<NodeUid> = {
let st = self.state();
st.children
.get(&ino)
.map(|kids| {
kids.iter()
.filter_map(|k| st.entries.get(k))
.filter(|e| {
!is_local_uid(&e.uid)
&& matches!(
&e.node.kind,
NodeKind::File {
claimed_size: None,
..
}
)
})
.map(|e| e.uid.clone())
.collect()
})
.unwrap_or_default()
};
if let Some(slot) = self.upgrade_sizes(ino, missing) {
slot.wait_finished(Instant::now() + SizeUpgrade::WAIT);
}
}
/// Whether `ino` has a real size — the condition a waiter is waiting on.
/// A node that vanished counts as resolved; there is nothing left to wait
/// for, and its caller will find the `ENOENT` for itself.
fn size_is_real(&self, ino: u64) -> bool {
let st = self.state();
st.entries.get(&ino).is_none_or(|e| {
!matches!(
&e.node.kind,
NodeKind::File {
claimed_size: None,
..
}
)
})
}
/// Adopt the sizes a completed upgrade fetched. Split from
/// [`Core::upgrade_sizes`] so the single-flight bookkeeping there has one
/// exit path rather than one per early return.
fn apply_size_upgrade(&self, folder_ino: u64, result: Result<Vec<Node>, ProtonError>) {
let core = self;
{
let mut nodes = match result {
Ok(nodes) => nodes,
Err(e) => {
// Not fatal: the listing stands, sizes just stay provisional
// until something invalidates and re-enumerates it.
warn!(folder_ino, error = %e, "size upgrade failed; sizes stay provisional");
return;
}
};
// A queued write is ahead of anything the server can report, so its
// optimistic size must survive this just as it survives a re-listing
// (B11).
core.stamp_pending_sizes(&mut nodes);
let mut changed: Vec<u64> = Vec::new();
let mut st = core.state();
for node in nodes {
// Only adopt the size. Re-interning wholesale would also adopt a
// name or parent that a rename/move may have changed locally
// while this was in flight, undoing it.
let Some(&ino) = st.by_uid.get(&node.uid) else {
continue;
};
changed.push(ino);
// An open write handle is ahead of the server too (B109).
if st.active_writes.contains_key(&ino) {
continue;
}
let NodeKind::File { claimed_size, .. } = &node.kind else {
continue;
};
let (Some(size), Some(entry)) = (*claimed_size, st.entries.get_mut(&ino)) else {
continue;
};
if let NodeKind::File { claimed_size, .. } = &mut entry.node.kind {
*claimed_size = Some(size);
}
}
let updated: Vec<Node> = st
.children
.get(&folder_ino)
.map(|kids| {
kids.iter()
.filter_map(|k| st.entries.get(k).map(|e| e.node.clone()))
.collect()
})
.unwrap_or_default();
drop(st);
if let Err(e) = core.db.upsert_nodes(&updated) {
warn!(folder_ino, error = %e, "persisting upgraded sizes failed");
}
// Without this the corrected size is invisible for the length of the
// attr TTL: the kernel answers `stat` from the provisional attrs it
// cached while the listing was being served, so `ls -l` reports the
// ciphertext size for up to 30 s even though the daemon has had the
// real one all along. Notify *after* the DB write, so a re-`getattr`
// provoked by the invalidation cannot race the persistence.
//
// Attributes only (offset -1): the content did not change, and a
// page-cache invalidation deadlocks when this runs inside a `read`
// (B100 resolves sizes there). The kernel holds the page lock of
// that read until we reply, and the invalidation waits for the
// same lock, so the read never answered (docs/BUGS.md B110).
for mount in core.states.live() {
if let Some(notifier) = mount.notifier.get() {
for &ino in &changed {
let _ = notifier.inval_attrs(ino);
}
}
}
debug!(folder_ino, files = updated.len(), "filled in listing sizes");
}
}
/// Resolve a child `name` within `parent` to its `(inode, uid)`, ensuring
/// the parent's listing is cached first.
fn lookup_child(&self, parent: u64, name: &str) -> Result<(u64, NodeUid), Errno> {
self.ensure_children(parent)?;
let st = self.state();
st.children
.get(&parent)
.and_then(|kids| {
kids.iter().copied().find_map(|ino| {
st.entries
.get(&ino)
.filter(|e| e.node.name == name)
.map(|e| (ino, e.uid.clone()))
})
})
.ok_or(Errno::ENOENT)
}
/// Walk a mountpoint-relative path to its `(inode, uid)`, enumerating each
/// directory on the way as needed. Leading `/` and `.` components are
/// ignored; `..` is rejected.
fn resolve_path(&self, rel: &Path) -> Result<(u64, NodeUid), Errno> {
let mut ino = self.root_ino();
let mut uid = {
let st = self.state();
st.entries
.get(&ino)
.map(|e| e.uid.clone())
.ok_or(Errno::ENOENT)?
};
for comp in rel.components() {
match comp {
Component::RootDir | Component::CurDir => continue,
Component::Normal(name) => {
let (child_ino, child_uid) = self.lookup_child(ino, &name.to_string_lossy())?;
ino = child_ino;
uid = child_uid;
}
_ => return Err(Errno::EINVAL),
}
}
Ok((ino, uid))
}
/// [`resolve_path`](Self::resolve_path) for the request-serving side.
///
/// The kernel-facing resolver answers in `Errno` because that is what the
/// FUSE reply needs. A front-end needs the opposite: `{:?}` of a libc
/// constant is not something to show a user, and "does not exist" and "the
/// API is down" want different copy and different buttons. This is the one
/// place that knows enough to tell them apart, so it is the place that does.
fn resolve(&self, rel: &Path) -> CoreResult<(u64, NodeUid)> {
self.resolve_path(rel)
.map_err(|e| self.errno_error(e, &format!("could not resolve {}", rel.display())))
}
/// Resolve a wire-format uid only when one of this daemon's locations
/// proves that it owns the node.
///
/// The primary and on-demand mounts prove residency through the tree and
/// the roots in [`StateRegistry`]. Mirror locations have no tree, so their root
/// and last-synced descendants are resolved through `sync_folder` and
/// `sync_entry` instead.
pub(crate) fn resolve_anywhere(&self, uid: &str) -> CoreResult<NodeUid> {
resolve_anywhere_with(
uid,
|uid| self.owns_visible_uid(uid),
|uid| self.db.mirror_contains_uid(&uid.to_string()),
)
}
fn source_parent_uid(&self, ino: u64, rel: &Path) -> CoreResult<NodeUid> {
let state = self.state();
let entry = state
.entries
.get(&ino)
.ok_or_else(|| CoreError::not_found(format!("could not resolve {}", rel.display())))?;
entry
.node
.parent_uid
.clone()
.or_else(|| {
state
.entries
.get(&entry.parent)
.map(|parent| parent.uid.clone())
})
.ok_or_else(|| CoreError::invalid("the mount root cannot be mutated"))
}
/// Classify a failure that arrived as an `Errno`.
///
/// The internal paths speak `Errno` because they also serve FUSE, where a
/// number is the whole vocabulary. Everything crossing the control socket
/// has to be turned back into something a person can read, and this is the
/// one place that knows how — a call site holding only an `Errno` has
/// already lost the context needed to say what went wrong.
///
/// `Errno` is neither `PartialEq` nor structural-match, so this compares
/// raw codes rather than the `libc` constants.
fn errno_error(&self, e: Errno, what: &str) -> CoreError {
match e.code() {
libc::ENOENT => CoreError::not_found(format!("{what}: no such file or folder")),
libc::EACCES | libc::EPERM => CoreError::denied(format!("{what}: not allowed")),
// These paths walk the tree lazily, so a cold node needs the API.
// Offline that surfaces as EIO, which on its own would read to the
// user as a broken file rather than a missing network.
libc::EIO if !self.online.load(Ordering::Relaxed) => CoreError::offline(),
libc::EINVAL => CoreError::invalid(format!("{what}: not a usable path")),
libc::ENOSPC => CoreError::new(ErrorKind::Quota, format!("{what}: out of space")),
libc::EEXIST => CoreError::conflict(format!("{what}: already exists")),
libc::ENOTEMPTY => CoreError::conflict(format!("{what}: folder is not empty")),
_ => CoreError::internal(format!("{what}: {e:?}")),
}
}
/// Fetch a single node's current metadata from the remote.
fn fetch_node(&self, uid: &NodeUid) -> Result<Node, Errno> {
match self.fetch_node_remote(uid) {
Ok(Some(node)) => Ok(node),
Ok(None) => Err(Errno::ENOENT),
Err(e) => {
if !self.lost_link(&e, "fetch node") {
error!(%uid, error = %e, "enumerate node failed");
}
Err(Errno::EIO)
}
}
}
/// Read back a node just created on the remote.
///
/// Drive can answer a create before a lookup lists the new node, so an
/// empty answer is asked again for a moment. `None` means the node exists
/// but could not be read back. The mount then queues the create, and the
/// drain finds the node by name and adopts it: the application gets its
/// file or folder either way, never an `ENOENT` for something that exists.
fn fetch_minted_node(&self, uid: &NodeUid, name: &str) -> Option<Node> {
// The feed echoes the create. Unclaimed, it would take the echo the
// upload of the file's bytes records, and that one would then evict
// the bytes just uploaded (docs/BUGS.md B123).
self.note_self_change(uid);
let node = self.read_back(uid).ok();
if node.is_none() {
warn!(%uid, name, "Drive made the node but did not hand it back");
}
node
}
/// [`Core::fetch_node`] for a node Drive has just made, asked again for a
/// moment while Drive does not list it yet.
///
/// A file Drive lists without an active revision is not listed yet either:
/// the create commits one before it answers, so this answer is from before
/// the commit, with no size. Taken, it emptied the new file in the tree,
/// and a read of it right after found nothing (`docs/BUGS.md` B163).
pub(crate) fn read_back(&self, uid: &NodeUid) -> Result<Node, Errno> {
let mut delays = MINTED_READ_BACK_DELAYS.iter();
loop {
let fetched = match self.fetch_node(uid) {
Ok(node) if is_unrevised(&node) => Err(Errno::ENOENT),
fetched => fetched,
};
match fetched {
Err(e)
if e == Errno::ENOENT
&& let Some(delay) = delays.next() =>
{
std::thread::sleep(*delay);
}
fetched => return fetched,
}
}
}
/// [`Core::fetch_node`] without the collapse to an `Errno`, for the drain:
/// resolving a conflict turns on *why* a call failed, and "the node is not
/// there" (`Ok(None)`) is a different outcome from "we could not ask".
fn fetch_node_remote(&self, uid: &NodeUid) -> Result<Option<Node>, ProtonError> {
match self.block_on_bounded(self.drive.enumerate_nodes(std::slice::from_ref(uid))) {
Ok(nodes) => Ok(nodes.into_iter().next()),
// An unknown uid is reported either as an empty result or as an
// outright refusal, depending on the endpoint.
Err(e) if is_gone(&e) => Ok(None),
Err(e) => Err(e),
}
}
/// Point a handle closed while its file's create landed at the uid Drive
/// gave the node.
///
/// `Core::retire_create` moves the row to that uid before it repoints the
/// open handles. A handle closed in between still named the placeholder,
/// which no row has any more, and its write was refused as if access had
/// changed. The inode is the row's local id, which finds it either way.
pub(crate) fn follow_landed_create(&self, h: &mut WriteHandle) {
if !is_local_uid(&h.uid) {
return;
}
if let Some(real) = self.landed_row_uid(h.ino) {
debug!(local = %h.uid, %real, "write handle closed as its create landed");
h.uid = real;
}
}
/// The uid Drive gave the node at `ino`, once its create has landed, or
/// `None` while it is still queued. Read from the row, which has it before
/// the tree does.
pub(crate) fn landed_row_uid(&self, ino: u64) -> Option<NodeUid> {
state::ino_lid(ino).and_then(|lid| self.landed_lid_uid(lid))
}
/// [`Core::landed_row_uid`] for a node known by its stand-in `uid`.
pub(crate) fn landed_stand_in_uid(&self, uid: &NodeUid) -> Option<NodeUid> {
pdfs_core::db::local_lid(&uid.to_string()).and_then(|lid| self.landed_lid_uid(lid))
}
fn landed_lid_uid(&self, lid: i64) -> Option<NodeUid> {
self.db
.uid_of_lid(lid)
.ok()
.flatten()
.and_then(|uid| parse_node_uid(&uid))
.filter(|uid| !is_local_uid(uid))
}
/// Queue what the write handles the kernel never released hold.
///
/// `close(2)` returns before the kernel sends `release`, and the stop
/// aborts the FUSE connection with the releases still queued: a file a
/// program had just saved was never queued, and Drive kept it as it was
/// (`docs/BUGS.md` B190). Runs once every session has ended, so no release
/// can race it. A handle still open in a program goes the same way, with
/// what it wrote so far; one on an unlinked file only leaves its scratch.
pub(crate) fn queue_unreleased_writes(&self) {
let handles: Vec<(WriteHandle, bool)> = {
let mut st = self.state();
st.handles.clear();
let inos: Vec<u64> = st.active_writes.keys().copied().collect();
inos.into_iter()
.filter_map(|ino| {
let unlinked = st.entries.get(&ino).is_some_and(|e| e.unlinked);
st.active_writes.remove(&ino).map(|h| (h, unlinked))
})
.collect()
};
for (mut h, unlinked) in handles {
if unlinked {
self.cache.clear_scratch_durable(&h.path);
let _ = std::fs::remove_file(&h.path);
continue;
}
self.follow_landed_create(&mut h);
match self.queue_revision(&mut h) {
Ok(()) => info!(uid = %h.uid, "queued a write the kernel never released"),
Err(error) => {
warn!(uid = %h.uid, ?error, "cannot queue a write the kernel never released")
}
}
}
}
/// Accept a released write handle's bytes and queue their upload.
///
/// This is what makes a copy into the mount run at disk speed: the caller's
/// `close` returns once the bytes are staged on local disk and the intent is
/// in `pending_op`, instead of waiting out a full upload inside the FUSE
/// handler. It is also what makes an offline write succeed rather than EIO —
/// the queued op simply waits for the network.
///
/// The scratch file is *moved* into staging, never copied: it is the only
/// copy of what the user wrote.
fn queue_revision(&self, h: &mut WriteHandle) -> Result<(), Errno> {
// Closing an untouched writable handle is not a mutation. It remains
// valid after a downgrade and only owes cleanup of its empty scratch.
if !h.dirty {
self.cache.clear_scratch_durable(&h.path);
let _ = std::fs::remove_file(&h.path);
// A new file's create waits for this release to wake the drain
// (`Core::queue_local_node`). Closed empty, it attached nothing
// that would, and sat until the idle poll (`docs/BUGS.md` B192).
if is_local_uid(&h.uid) {
self.wake_drain();
}
return Ok(());
}
preserve_on_access_denied(self.require_uid_writable(&h.uid), h.dirty, || {
let meta = self.recovery_meta(h);
self.stage_orphaned_write(
&h.uid,
h.ino,
&h.path,
&meta,
"access changed before the write could be queued",
);
})?;
// The handle is being retired either way, so any durability sidecar an
// `fsync` left has done its job: from here the bytes are tracked as a
// staged write and a queued op, and a sidecar outliving them would offer
// recovery a second, stale copy of the same write.
self.cache.clear_scratch_durable(&h.path);
if is_local_uid(&h.uid) && !self.db.has_create_op(&h.uid.to_string()).unwrap_or(true) {
// A create that landed since the handle last looked left no op
// either. It moved the row to its uid in the same transaction, so
// the row says which it was (`docs/BUGS.md` B176).
self.follow_landed_create(h);
if is_local_uid(&h.uid) {
debug!(uid = %h.uid, "local node was unlinked before creation; dropping revision");
let _ = std::fs::remove_file(&h.path);
return Ok(());
}
}
// Materialize as much of the full content as is free to materialize. A
// complete blob is uploadable without the network and lets a later write
// to the same file supersede this one without any reconstruction at all.
//
// What is deliberately *not* done here is fetching the untouched ranges
// from the remote: that is a download of everything the edit did not
// touch, and this runs on the FUSE dispatch loop, so closing a large
// file after a small edit stopped every other request on the mount for
// the length of it (audit F1). An incomplete blob is first-class —
// `read_pending` resolves its gaps against the base, and `drain_revision`
// fills them on the drain thread — so the network half of the fill
// belongs there, and only there.
//
// A node that exists only locally has no remote base to fill from: its
// untouched ranges live in the blob queued by an earlier write, which
// `merge_over_pending` folds in below.
if let Err(e) = h.file.set_len(h.len) {
error!(uid = %h.uid, error = %e, "resize scratch file failed");
return Err(Errno::EIO);
}
let mut written = h.written.clone();
if !(is_local_uid(&h.uid) && h.base_size > 0) {
// A local node has no remote base, so nothing of it can be fetched
// or cached: passing zero says so, and leaves the fill doing only
// the part that still applies — claiming the zeroed tail.
let base_size = if is_local_uid(&h.uid) { 0 } else { h.base_size };
self.fill_gaps_cached(
&h.uid,
&h.file,
h.len,
h.base_mtime,
base_size,
&mut written,
);
}
let authored: Vec<(u64, u64)> = written
.segments(0, h.len)
.into_iter()
.filter(|&(_, _, authored)| authored)
.map(|(s, e, _)| (s, e))
.collect();
let meta = StagedWrite {
uid: h.uid.to_string(),
len: h.len,
base_size: h.base_size,
base_mtime: h.base_mtime,
complete: StagedWrite::covers(&authored, h.len),
authored,
based_on: self.remote_baseline(
&h.uid,
h.base_mtime,
h.base_size,
h.base_revision_id.clone(),
),
};
let complete = meta.complete;
self.enqueue_staged_write(&h.uid, h.ino, &h.path, meta)?;
debug!(uid = %h.uid, len = h.len, complete, "queued revision upload");
Ok(())
}
/// Describe the locally authored ranges without gap-filling them.
///
/// Used when a write was accepted by the kernel but a queue-time access
/// check now denies it. Building this metadata is side-effect free, so the
/// permission guard still precedes cache, database, and tree mutation.
fn recovery_meta(&self, h: &WriteHandle) -> StagedWrite {
let authored: Vec<(u64, u64)> = h
.written
.segments(0, h.len)
.into_iter()
.filter(|&(_, _, authored)| authored)
.map(|(start, end, _)| (start, end))
.collect();
StagedWrite {
uid: h.uid.to_string(),
len: h.len,
base_size: h.base_size,
base_mtime: h.base_mtime,
complete: StagedWrite::covers(&authored, h.len),
authored,
based_on: self.remote_baseline(
&h.uid,
h.base_mtime,
h.base_size,
h.base_revision_id.clone(),
),
}
}
/// The remote revision a change to `uid` is being made against, for
/// [`StagedWrite::based_on`].
///
/// Normally that is simply the base the handle opened over. The exception is
/// a write that supersedes a still-queued one: its "base" is the previous
/// *staged blob*, whose size and mtime are ours, not the server's — so the
/// baseline is inherited from the op being superseded, which is the last one
/// that actually observed the remote. Without that, chaining two writes
/// before the queue drains would leave the drain comparing the remote
/// against a revision it never had, and cutting a conflict copy over
/// nothing.
///
/// `None` for a node that has never existed remotely: there is no revision
/// to conflict with until its create drains.
fn remote_baseline(
&self,
uid: &NodeUid,
base_mtime: i64,
base_size: u64,
base_revision_id: Option<String>,
) -> Option<Baseline> {
if is_local_uid(uid) {
return None;
}
match self.pending.lock().get(uid) {
Some(p) => p.meta.based_on.clone(),
None => Some(Baseline {
mtime: base_mtime,
size: base_size,
hash: None,
revision_id: base_revision_id,
}),
}
}
/// Move a file holding a node's intended new content into staging and queue
/// the upload that will make it the remote's content. Shared by the release
/// of a write handle and by a path-based truncate.
///
/// `src` is consumed either way: on success it is *moved* into staging, and
/// on the refusal below it is moved there too, just without an op to upload
/// it. It is the only copy of what the user wrote, so nothing here may
/// simply delete it.
fn enqueue_staged_write(
&self,
uid: &NodeUid,
ino: u64,
src: &Path,
mut meta: StagedWrite,
) -> Result<(), Errno> {
preserve_on_access_denied(self.require_uid_writable(uid), true, || {
self.stage_orphaned_write(
uid,
ino,
src,
&meta,
"access changed before the staged write could be queued",
);
})?;
// An incomplete blob's gaps refer to the *remote* base. If an earlier
// write to this file is still queued, the remote no longer holds that
// base — the previous staged blob does — so taking those gaps at face
// value would fill them from the wrong revision. Fold the blob being
// superseded into this one instead: its authored bytes are a local copy
// of exactly the ranges this write did not touch, and what neither wrote
// is still the remote content that the earlier write was based on, which
// is why the baseline is inherited along with them.
let previous = self.pending.lock().get(uid).cloned();
if !meta.complete
&& let Some(previous) = previous
&& let Err(error) = merge_over_pending(&mut meta, src, &previous)
{
// Without the earlier bytes the gaps cannot be resolved against
// anything, so this is where the write stops being uploadable.
// Keep it recoverable rather than corrupting the file.
warn!(%uid, %error, "folding in the undrained edit failed");
self.stage_orphaned_write(
uid,
ino,
src,
&meta,
"write overlaps an undrained incomplete edit",
);
return Err(Errno::EIO);
}
let path = self.cache.stage_write(&meta, src).map_err(|e| {
error!(%uid, error = %e, "staging write failed");
// The scratch file is still the only copy of these bytes, and the
// caller is releasing the handle it belongs to — without a
// durability sidecar `discard_unmarked_scratch` deletes it at the
// next mount, the one place the "never delete on a failure path"
// invariant would break. Mark it first, whatever went wrong.
if let Err(mark) = self.cache.mark_scratch_durable(src, &meta) {
error!(%uid, error = %mark,
"could not mark an unstageable write durable; bytes at risk");
}
if e.is_disk_full() {
// Free cached blobs so the retry has somewhere to land, and tell
// the caller what is actually wrong.
self.cache.emergency_evict();
Errno::ENOSPC
} else {
Errno::EIO
}
})?;
// Whatever this write supersedes, it supersedes now — including an
// upload already on the wire, whose bytes describe a revision the row
// written just below replaces. Signalled before the enqueue so the
// in-flight reader stops at its next block rather than after however
// much of the file is left.
self.cancel_upload(uid);
let mut uid = uid.clone();
let len = meta.len;
let superseded = 'queued: {
if is_local_uid(&uid) {
// The node has no server-side identity to hang a revision on, so
// the bytes ride on the create that will mint it. Attached and
// recorded under one hold of `pending`, which is what the drain
// takes to retire a landed create: it either sees this blob and
// carries it over, or has already finished and this write
// queues against the real uid below (`Core::retire_create`).
let meta_json = serde_json::to_string(&meta).unwrap_or_default();
let mut pending = self.pending.lock();
let attached = self
.db
.attach_blob_to_create(&uid.to_string(), &path.to_string_lossy(), &meta_json)
.map_err(|e| {
error!(%uid, error = %e, "attaching write to queued create failed");
Errno::EIO
})?;
if let Some(attached) = attached {
pending.insert(uid.clone(), PendingRevision { path, meta });
break 'queued attached.superseded;
}
drop(pending);
// The create landed while this handle was open, and its row
// already has the uid Drive gave it.
let Some(real) = self.landed_row_uid(ino) else {
error!(%uid, staged = %path.display(),
"queued create vanished under a write; bytes kept in staging");
return Err(Errno::EIO);
};
debug!(local = %uid, %real, "create landed under an open write; queueing a revision");
uid = real;
meta.uid = uid.to_string();
let previous = self.pending.lock().get(&uid).cloned();
meta.based_on = previous.as_ref().and_then(|p| p.meta.based_on.clone());
if !meta.complete
&& let Some(previous) = previous
&& let Err(error) = merge_over_pending(&mut meta, &path, &previous)
{
error!(%uid, %error, staged = %path.display(),
"folding in the write the create carried failed; bytes kept in staging");
return Err(Errno::EIO);
}
}
let op = PendingOp {
id: 0,
kind: OP_REVISION.to_string(),
uid: uid.to_string(),
parent_uid: None,
name: None,
blob_path: Some(path.to_string_lossy().into_owned()),
meta_json: Some(serde_json::to_string(&meta).unwrap_or_default()),
created_at: now_millis(),
attempts: 0,
last_error: None,
next_attempt_at: now_millis() + self.revision_debounce(&uid).as_millis() as i64,
};
let (_id, superseded) = self.db.enqueue_op(&op).map_err(|e| {
error!(%uid, error = %e, "queueing upload failed");
Errno::EIO
})?;
self.pending
.lock()
.insert(uid.clone(), PendingRevision { path, meta });
superseded
};
if let Some(old) = superseded {
self.cache.discard_staged(Path::new(&old));
}
let uid = &uid;
// Reflect the write in the tree straight away: `ls` must show the new
// size and mtime even though the remote still holds the old revision.
let now = now_secs();
let updated = {
let mut st = self.state();
st.record_pending_write(ino, len, now)
};
// The only record that the file is as long as the caller was told. If it
// cannot be written the write did not fully take, and saying so beats
// acknowledging a `close(2)` and then serving the old size — the bytes
// themselves are already staged and queued either way. Written out here
// rather than inside `State` so the commit does not hold the inode lock.
let recorded = match updated {
Some(node) => self.db.upsert_node(&node),
None => Ok(()),
};
// Cached blobs and open readers describe the superseded revision. Reads
// come from the staged blob until the op drains, so just drop them.
// Done even when the row above failed: the staged blob is the file's
// content now either way, and leaving a stale one cached would serve it.
self.cache.evict(uid);
self.evict_reader(uid);
self.wake_drain();
if let Err(e) = recorded {
error!(%uid, error = %e, "recording a queued write's size failed");
return Err(if e.is_disk_full() {
Errno::ENOSPC
} else {
Errno::EIO
});
}
Ok(())
}
/// Queue the new content of a path-based truncate — `> file`, or any
/// `setattr(size=…)` arriving without a write handle.
///
/// No bytes have been authored at truncate time, which is why this path was
/// never staged and instead resized the remote content inline. That is also
/// why a shell redirect failed offline *before* the write that follows it:
/// the truncate itself needed the network. Staging a blob describing the
/// result puts it on the same queue as every other write.
///
/// The blob is a hole of the new length; what is real about it is `authored`:
///
/// - `> file` (size 0) is the whole point and needs nothing at all — an
/// empty file is complete content, so it queues and drains offline.
/// - Extending past the end authors the new tail (zeros, by definition) and
/// leaves the head to be gap-filled from the base.
/// - Shrinking authors nothing: every remaining byte still comes from the
/// base, so it is the drain that has to fetch it.
fn queue_truncate(&self, ino: u64, size: u64) -> Result<(), Errno> {
let (uid, base_mtime, base_size, base_revision_id) = {
let st = self.state();
match st.entries.get(&ino) {
Some(e) if e.node.is_file() => (
e.uid.clone(),
e.node.modification_time,
node_size(&e.node),
node_revision_id(&e.node),
),
Some(_) => return Err(Errno::EISDIR),
None => return Err(Errno::ENOENT),
}
};
self.require_uid_writable(&uid)?;
if size == base_size {
return Ok(());
}
// A close immediately followed by truncate is common (`write`, close,
// `truncate(2)`) and usually beats the write-back debounce. In that
// case the remote is *not* our base: the complete staged revision is.
// Compose from those local bytes so the truncate can supersede the
// queued upload without fetching stale remote content or returning EIO.
let pending_base = self.pending.lock().get(&uid).cloned();
if pending_base
.as_ref()
.is_some_and(|pending| !pending.meta.complete)
{
// An incomplete pending blob still has holes referring to the
// remote revision. Stacking another incomplete transform on it is
// not representable safely; preserve the existing refusal.
return Err(Errno::EIO);
}
let (authored, complete) = if size == 0 {
// An empty file has no content to be missing.
(Vec::new(), true)
} else if base_size == 0 {
// Nothing to gap-fill from: every byte is a zero this truncate
// defines.
(vec![(0, size)], true)
} else if size > base_size {
(vec![(base_size, size)], false)
} else {
(Vec::new(), false)
};
let (file, path) = self.cache.create_scratch().map_err(|e| {
error!(%uid, error = %e, "create scratch file for truncate failed");
Errno::EIO
})?;
if let Some(pending) = &pending_base
&& let Err(e) = copy_pending_for_truncate(pending, &file)
{
error!(%uid, source = %pending.path.display(), error = %e,
"copy pending revision for truncate failed");
let _ = std::fs::remove_file(&path);
return Err(Errno::EIO);
}
file.set_len(size).map_err(|e| {
error!(%uid, error = %e, "resize scratch file for truncate failed");
let _ = std::fs::remove_file(&path);
Errno::EIO
})?;
let meta = match pending_base {
Some(pending) => StagedWrite {
uid: uid.to_string(),
len: size,
base_size: pending.meta.len,
base_mtime: pending.meta.base_mtime,
authored: if size == 0 {
Vec::new()
} else {
vec![(0, size)]
},
complete: true,
// Both local revisions descend from the same last-observed
// remote revision. Never replace this with the optimistic
// mtime/size currently published by State.
based_on: pending.meta.based_on,
},
None => StagedWrite {
uid: uid.to_string(),
len: size,
base_size,
base_mtime,
authored,
complete,
based_on: self.remote_baseline(&uid, base_mtime, base_size, base_revision_id),
},
};
self.enqueue_staged_write(&uid, ino, &path, meta)?;
debug!(%uid, size, complete, "queued truncate");
Ok(())
}
/// Invent a node under `parent_uid` and queue the op that will make it real.
/// Returns the node to intern, exactly as the online
/// path returns the one the server minted.
///
/// The parent may itself be a placeholder — `mkdir -p` offline, or `cp -r` of
/// a tree. That is fine: the op records the parent it was made under by local
/// id, and is not sent until the parent's row has the uid it landed as.
fn queue_local_node(
&self,
parent_uid: &NodeUid,
name: &str,
is_dir: bool,
hold: bool,
) -> Result<Node, Errno> {
self.require_uid_writable(parent_uid)?;
// The row the tree would give it anyway, now: the node goes by its
// local id until it lands.
let lid = self
.db
.add_local_row(&parent_uid.to_string(), name, is_dir, now_secs())
.map_err(|e| {
error!(%parent_uid, name, error = %e, "giving a local node its row failed");
Errno::EIO
})?;
let uid = local_node_uid(lid);
let node = local_node(uid.clone(), parent_uid.clone(), name.to_string(), is_dir);
let op = PendingOp {
id: 0,
kind: if is_dir { OP_MKDIR } else { OP_CREATE }.to_string(),
uid: uid.to_string(),
parent_uid: Some(parent_uid.to_string()),
name: Some(name.to_string()),
blob_path: None,
meta_json: None,
created_at: now_millis(),
attempts: 0,
last_error: None,
// Parked when the name is transient: its bytes ride on this create
// (via `attach_blob_to_create`) and must not upload until a rename to
// the finished name un-parks it (docs/BUGS.md B70).
next_attempt_at: if hold { PARK_UNTIL } else { 0 },
};
self.db.enqueue_op(&op).map_err(|e| {
error!(%parent_uid, name, error = %e, "queueing local node failed");
Errno::EIO
})?;
debug!(%uid, %parent_uid, name, is_dir, "created node offline; queued");
// A folder is due now; without this the workers slept through it to
// the idle poll (B160). A file is not: its bytes ride on the create, and
// its release wakes the drain, with bytes or without (B192).
if is_dir {
self.wake_drain();
}
Ok(node)
}
/// Queue giving a node a new parent and/or name, and apply it to the tree
/// now.
///
/// The op records the desired end state rather than the step, so it both
/// supersedes any earlier queued rename and lets the drain skip whichever
/// half the remote already agrees with.
///
/// `new_parent_uid` may be a placeholder — moving a file into a folder
/// created offline — which is why this cannot simply be the online call with
/// a retry around it: the API would 404 on a `local~` parent. The op waits
/// for that folder's drain to rewrite it, exactly as a queued create does.
fn queue_rename(
&self,
ino: u64,
uid: &NodeUid,
old_parent_uid: &NodeUid,
new_parent_ino: u64,
new_parent_uid: &NodeUid,
new_name: &str,
) -> Result<(), Errno> {
self.require_uid_writable(uid)?;
self.require_uid_writable(old_parent_uid)?;
self.require_uid_writable(new_parent_uid)?;
self.queue_rename_authorized(
ino,
uid,
old_parent_uid,
new_parent_ino,
new_parent_uid,
new_name,
)
}
/// Enqueue a rename whose access was linearized immediately before a
/// replacement victim was removed. Rechecking here could reject after that
/// destructive half had already completed.
fn queue_rename_authorized(
&self,
ino: u64,
uid: &NodeUid,
old_parent_uid: &NodeUid,
new_parent_ino: u64,
new_parent_uid: &NodeUid,
new_name: &str,
) -> Result<(), Errno> {
self.enqueue_rename(ino, uid, old_parent_uid, new_parent_uid, new_name)?;
self.state
.lock()
.rename_in_place(ino, new_parent_ino, new_parent_uid, new_name);
self.wake_drain();
debug!(%uid, %new_parent_uid, new_name, "renamed offline; queued");
Ok(())
}
/// Record the queued rename op, and nothing else: the caller moves the node
/// in the tree, through the kernel or around it.
fn enqueue_rename(
&self,
ino: u64,
uid: &NodeUid,
old_parent_uid: &NodeUid,
new_parent_uid: &NodeUid,
new_name: &str,
) -> Result<(), Errno> {
// A rename that replaces a queued one keeps where the node started.
let meta = match self.db.pending_op_meta(&uid.to_string(), OP_RENAME) {
Ok(Some(json)) => serde_json::from_str::<RenameMeta>(&json).map_err(|error| {
error!(%uid, %error, "queued rename authority metadata is invalid");
Errno::EIO
})?,
Ok(None) => RenameMeta {
original_parent_uid: old_parent_uid.to_string(),
original_name: self
.state
.lock()
.entries
.get(&ino)
.map(|entry| entry.node.name.clone()),
},
Err(error) => {
error!(%uid, %error, "reading queued rename authority failed");
return Err(Errno::EIO);
}
};
let meta_json = serde_json::to_string(&meta).map_err(|error| {
error!(%uid, %error, "serializing queued rename authority failed");
Errno::EIO
})?;
let op = PendingOp {
id: 0,
kind: OP_RENAME.to_string(),
uid: uid.to_string(),
parent_uid: Some(new_parent_uid.to_string()),
name: Some(new_name.to_string()),
blob_path: None,
meta_json: Some(meta_json),
created_at: now_millis(),
attempts: 0,
last_error: None,
next_attempt_at: 0,
};
self.db.enqueue_op(&op).map_err(|e| {
error!(%uid, new_name, error = %e, "queueing rename failed");
Errno::EIO
})?;
Ok(())
}
/// Queue trashing a node the server knows about, and drop it from the tree
/// now.
///
/// Anything else queued for this node is discarded first: the user has said
/// the file should not exist, so uploading bytes to it or renaming it are
/// both work towards an outcome nobody wants any more. That does throw away
/// staged bytes that never landed — which is precisely what deleting an
/// un-uploaded file means, and the alternative (upload it, then trash it) is
/// worse in every way.
///
/// A file still open keeps the bytes its handle reads, the queued ones and
/// the cached ones, as when the trash is sent at once: a partial write
/// fills its gaps from them, offline too (`docs/BUGS.md` B186). The final
/// close or the trash landing, whichever comes last, lets go of them.
fn queue_trash(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> {
self.require_uid_writable(uid)?;
// The bytes below are about to be discarded, and a drain worker may be
// reading them onto the wire right now. Stop it first: with several
// workers the trash op this queues can be claimed while that upload is
// still running, and the two would race to decide whether the file
// exists.
self.cancel_upload(uid);
let (_, blobs) = self
.db
.replace_ops_with_trash(&uid.to_string(), name, now_millis())
.map_err(|e| {
error!(%uid, error = %e, "queueing trash failed");
Errno::EIO
})?;
let open_now = self.is_open_anywhere(uid);
if open_now {
let read_from = self.pending_blob(uid);
self.release_dropped_below(
blobs
.into_iter()
.filter(|blob| read_from.as_deref() != Some(Path::new(blob)))
.collect(),
);
} else {
self.release_dropped_ops(uid, blobs);
}
self.hidden.lock().insert(uid.clone());
// Withdrawn from the tree, which every mount serves from, so no other
// mount goes on serving a file the user just trashed (`docs/BUGS.md`
// B74).
self.for_each_state(|st| {
st.unlink_mem(uid);
});
if !open_now {
self.cache.evict(uid);
self.evict_reader(uid);
}
self.wake_drain();
debug!(%uid, name, "trashed offline; queued");
Ok(())
}
/// Drop every op queued against a node, and the staged bytes they own.
fn discard_queued_ops(&self, uid: &NodeUid) -> Result<(), Errno> {
// Same reason as `queue_trash`: an upload may be reading the very blobs
// this is about to unlink.
self.cancel_upload(uid);
let blobs = self.db.delete_ops_for_uid(&uid.to_string()).map_err(|e| {
error!(%uid, error = %e, "dropping queued ops failed");
Errno::EIO
})?;
self.release_dropped_ops(uid, blobs);
Ok(())
}
/// Drop every op queued against a node that just left the namespace while
/// a handle still has it open, but keep the bytes that handle reads.
///
/// The ops cannot wait for the final close: the node is already trashed, and
/// a revision the drain sends in the meantime lands on it after the trash —
/// an editor's save loop then found the name it was renaming onto still
/// taken (`docs/BUGS.md` B106). The pending entry and its staged blob are
/// what the open handle reads, so they stay until the final close drops
/// them.
fn withdraw_queued_ops(&self, uid: &NodeUid) -> Result<(), Errno> {
self.cancel_upload(uid);
let blobs = self.db.delete_ops_for_uid(&uid.to_string()).map_err(|e| {
error!(%uid, error = %e, "withdrawing queued ops failed");
Errno::EIO
})?;
let read_from = self.pending_blob(uid);
let blobs = blobs
.into_iter()
.filter(|blob| read_from.as_deref() != Some(Path::new(blob)))
.collect();
self.release_dropped_below(blobs);
Ok(())
}
/// Let go of what the ops just dropped for `uid` held: their pending
/// entries, any upload still reading their bytes, and the bytes.
///
/// Removing a folder also drops the queued revisions of the files below it
/// (`docs/BUGS.md` B102), so the entries to forget are found by blob, not
/// only by `uid`.
fn release_dropped_ops(&self, uid: &NodeUid, blobs: Vec<String>) {
self.pending.lock().remove(uid);
self.release_dropped_below(blobs);
}
/// [`Core::release_dropped_ops`] without touching the node's own pending
/// entry.
fn release_dropped_below(&self, blobs: Vec<String>) {
let below: Vec<NodeUid> = {
let mut pending = self.pending.lock();
let below: Vec<NodeUid> = pending
.iter()
.filter(|(_, p)| blobs.iter().any(|b| p.path == Path::new(b)))
.map(|(k, _)| k.clone())
.collect();
for k in &below {
pending.remove(k);
}
below
};
for k in &below {
self.cancel_upload(k);
}
for blob in blobs {
self.cache.discard_staged(Path::new(&blob));
}
}
/// Which staged blob is currently queued for `uid`, if any.
///
/// The path identifies the revision: publication stages a fresh file and
/// swaps the map entry to point at it, so an unchanged path means the base a
/// caller sampled earlier is still the base now (`docs/BUGS.md` B32).
fn pending_blob(&self, uid: &NodeUid) -> Option<PathBuf> {
self.pending.lock().get(uid).map(|p| p.path.clone())
}
/// Nudge the drain worker to re-examine the queue now.
fn wake_drain(&self) {
let (lock, cv) = &*self.drain_wake;
*lock.lock() = true;
cv.notify_all();
}
/// Keep a write we cannot safely queue, so the bytes are recoverable even
/// though the caller is getting an error. See [`Core::queue_revision`].
fn stage_orphaned_write(
&self,
uid: &NodeUid,
ino: u64,
src: &Path,
meta: &StagedWrite,
reason: &str,
) {
match self.cache.preserve_write(meta, src) {
Ok(staged) => {
self.cache.clear_scratch_durable(src);
error!(
%uid,
staged = %staged.display(),
%reason,
"cannot queue write; bytes kept in staging"
);
let name = {
let st = self.state();
st.entries
.get(&ino)
.map(|e| e.node.name.clone())
.unwrap_or_default()
};
self.log_activity(
ActivityKind::Upload,
&name,
format!(
"write not queued ({reason}); changes kept at {}",
staged.display()
),
false,
);
}
Err(e) => {
let surviving = e.surviving_path.clone();
let marker = e.marker_path.clone();
error!(
%uid,
error = %e,
surviving = %surviving.display(),
marker = ?marker,
%reason,
"write rescue was only partially published; bytes retained"
);
let name = {
let st = self.state();
st.entries
.get(&ino)
.map(|entry| entry.node.name.clone())
.unwrap_or_default()
};
self.log_activity(
ActivityKind::Upload,
&name,
format!(
"write not queued ({reason}); changes retained at {}",
surviving.display()
),
false,
);
}
}
}
/// Download a whole file's plaintext, registering the transfer so
/// `GetQueueStatus` can report its progress. `total` is the expected size
/// (`0` if unknown). Streams through [`download_file_to`] so each block ticks
/// the progress counter.
///
/// [`download_file_to`]: ProtonDriveClient::download_file_to
fn download_file_tracked(
&self,
uid: &NodeUid,
name: &str,
total: u64,
) -> std::result::Result<Vec<u8>, ProtonError> {
let guard = self
.transfers
.begin(name, uid.to_string(), TransferDirection::Download, total);
let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard);
self.rt
.block_on(self.drive.download_file_to(uid, &mut out))?;
Ok(out.into_inner())
}
/// Download a whole file directly into `out`, without retaining its
/// plaintext in memory. This is the disk-backed twin of
/// [`Core::download_file_tracked`], and preserves the same transfer
/// accounting while allowing callers that only need an on-disk file to
/// keep memory use independent of file size.
fn download_file_tracked_to<W: Write + Send>(
&self,
uid: &NodeUid,
name: &str,
total: u64,
out: W,
) -> std::result::Result<W, ProtonError> {
let guard = self
.transfers
.begin(name, uid.to_string(), TransferDirection::Download, total);
let mut out = CountingWriter::new(out, &guard);
self.rt
.block_on(self.drive.download_file_to(uid, &mut out))?;
Ok(out.into_inner())
}
/// Like [`download_file_tracked`] for a photo, streaming through the photos
/// client's [`download_photo_to`].
///
/// [`download_file_tracked`]: Core::download_file_tracked
/// [`download_photo_to`]: ProtonPhotosClient::download_photo_to
fn download_photo_tracked(
&self,
photos: &ProtonPhotosClient,
uid: &NodeUid,
name: &str,
total: u64,
) -> std::result::Result<Vec<u8>, ProtonError> {
let guard = self
.transfers
.begin(name, uid.to_string(), TransferDirection::Download, total);
let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard);
self.rt.block_on(photos.download_photo_to(uid, &mut out))?;
Ok(out.into_inner())
}
/// Pin the node at mountpoint-relative `rel`. A file downloads its full
/// plaintext into the content cache before this returns. A folder records a
/// recursive pin and downloads every descendant file on a background thread,
/// reported as a job: a big folder far outlasts the socket read timeout.
/// Returns a human message.
fn pin(&self, rel: &Path) -> CoreResult<String> {
let (ino, uid) = self.resolve(rel)?;
let (name, is_folder, mtime, size) = {
let st = self.state();
let e = st
.entries
.get(&ino)
.ok_or_else(|| CoreError::not_found("node vanished"))?;
(
e.node.name.clone(),
e.node.is_folder(),
e.node.modification_time,
node_size(&e.node),
)
};
if is_folder {
// Offline, every download would fail and the folder would only look
// kept. Refuse instead, as a file pin does.
if !self.is_online() {
return Err(CoreError::offline());
}
// Record the recursive pin first so every descendant is eviction-
// exempt before we start filling the cache with the subtree.
self.cache
.add_pin(&uid, rel, true)
.map_err(|e| CoreError::from_api(&e, "pin"))?;
let core = self.clone();
let folder = name.clone();
std::thread::Builder::new()
.name("pin-folder".into())
.spawn(move || core.pin_subtree(ino, &uid, &folder))
.map_err(|e| CoreError::internal(format!("spawn pin: {e}")))?;
return Ok(format!("{name}: downloading its files in the background"));
}
self.fetch_content(&uid, &name, mtime, size)?;
self.cache
.add_pin(&uid, rel, false)
.map_err(|e| CoreError::from_api(&e, "pin"))?;
Ok(name)
}
/// Download and cache every file in the subtree rooted at the pinned folder
/// `ino` (`folder`, named `name`), as a job the front ends show. Walks the
/// tree depth-first, enumerating each folder so a cold subtree is fully
/// discovered; the lock is dropped before each network download. Stops when
/// the folder is unpinned or the daemon stops.
fn pin_subtree(&self, ino: u64, folder: &NodeUid, name: &str) {
let job = self
.transfers
.begin_job(format!("Making {name} available offline"));
match self.pin_subtree_files(ino) {
Ok(files) => {
job.set_total(files.len() as u64);
let mut failed = 0usize;
for (uid, file, mtime, size) in files {
if self.shutdown.is_stopping() || !self.cache.is_pinned(folder) {
return;
}
job.detail(file.clone());
if let Err(e) = self.fetch_pinned(&uid, &file, mtime, size) {
warn!(%uid, error = %e, "pin folder: download failed");
failed += 1;
}
job.step();
}
info!(folder = %name, failed, "pin folder: done");
}
Err(e) => warn!(folder = %name, error = %e, "pin folder: enumerate failed"),
}
}
/// Cache the file `uid` for a folder pin. Its `mtime` and `size` were read
/// before the download, and a create landing or a remote edit can change
/// them while it runs. The blob is then stored under the old ones and reads
/// as stale, so the file is listed as kept but not available offline. Fetch
/// it again under the new ones, a few times at most.
fn fetch_pinned(&self, uid: &NodeUid, name: &str, mtime: i64, size: u64) -> CoreResult<()> {
let mut fetched = (mtime, size);
for _ in 0..3 {
self.fetch_content(uid, name, fetched.0, fetched.1)?;
let current = {
let st = self.state();
st.by_uid
.get(uid)
.and_then(|ino| st.entries.get(ino))
.map(|e| (e.node.modification_time, node_size(&e.node)))
};
match current {
Some(now) if now != fetched => fetched = now,
_ => break,
}
}
Ok(())
}
/// Every file in the subtree rooted at folder `ino`.
fn pin_subtree_files(&self, ino: u64) -> CoreResult<Vec<(NodeUid, String, i64, u64)>> {
let mut files: Vec<(NodeUid, String, i64, u64)> = Vec::new();
let mut stack = vec![ino];
while let Some(dir) = stack.pop() {
if self.shutdown.is_stopping() {
break;
}
self.ensure_children(dir)
.map_err(|e| self.errno_error(e, "enumerate"))?;
let st = self.state();
if let Some(kids) = st.children.get(&dir) {
for &k in kids {
if let Some(e) = st.entries.get(&k) {
if e.node.is_folder() {
stack.push(k);
} else {
files.push((
e.uid.clone(),
e.node.name.clone(),
e.node.modification_time,
node_size(&e.node),
));
}
}
}
}
}
Ok(files)
}
/// Fetch a thumbnail of `ttype` for the file at `ino`, served from the cache
/// when fresh and otherwise downloaded from the remote and cached. Returns
/// `Ok(None)` when the node is not a file or has no thumbnail of that type.
fn thumbnail(&self, ino: u64, ttype: ThumbnailType) -> Result<Option<Vec<u8>>, Errno> {
let (uid, mtime) = {
let st = self.state();
match st.entries.get(&ino) {
Some(e) if e.node.is_file() => (e.uid.clone(), e.node.modification_time),
Some(_) => return Ok(None),
None => return Err(Errno::ENOENT),
}
};
if let Some(bytes) = self.cache.read_thumbnail(&uid, ttype.as_i32(), mtime) {
return Ok(Some(bytes));
}
// "This file has no thumbnail" is an answer worth remembering: without
// it every listing pays a round trip per file to be told nothing (B5).
let key = (uid.clone(), ttype.as_i32());
if self.thumbnail_misses.lock().remote_contains(&key, mtime) {
return Ok(None);
}
// A file manager asks for one per file in a folder, on the metadata
// workers. Offline, or on a stalled link, each one held a worker for
// the SDK's whole retry budget, and every cold lookup, create and
// rename on the mount queued behind them (`docs/BUGS.md` B220).
if !self.is_online() {
return Ok(None);
}
let bytes = self
.block_on_bounded(self.client.download_thumbnail(&uid, ttype))
.map_err(|e| {
warn!(%uid, error = %e, "download thumbnail failed");
Errno::EIO
})?;
match &bytes {
Some(bytes) => {
let _ = self
.cache
.store_thumbnail(&uid, ttype.as_i32(), mtime, bytes);
}
None => {
self.thumbnail_misses.lock().remember_remote(key, mtime);
}
}
Ok(bytes)
}
/// Unpin the node at `rel`, evicting its cached content. For a folder, also
/// evicts every descendant's cached blob (the subtree is no longer kept).
/// A node kept only because a folder above it is pinned is refused: there
/// is no pin of its own to remove, and it would stay kept all the same.
fn unpin(&self, rel: &Path) -> CoreResult<String> {
let (ino, uid) = self.resolve(rel)?;
let (name, is_folder) = {
let st = self.state();
st.entries
.get(&ino)
.map(|e| (e.node.name.clone(), e.node.is_folder()))
.unwrap_or_default()
};
if self.pinned_by_folder(ino) {
return Err(CoreError::conflict(format!(
"{name} is kept offline because a folder above it is; unpin that folder instead"
)));
}
self.cache
.remove_pin(&uid)
.map_err(|e| CoreError::from_api(&e, "unpin"))?;
// A recursively-pinned folder's descendants were eviction-exempt; now
// that the pin is gone, reclaim their blobs eagerly instead of waiting
// for budget pressure. Descendants come from the DB node tree. One
// pinned on its own, or under a pinned subfolder, is still kept.
if is_folder && let Ok(uids) = self.db.descendants(&uid.to_string()) {
let still_pinned = self.pinned_set();
for s in uids {
if still_pinned.contains(&s) {
continue;
}
if let Some(u) = parse_uid(&s) {
self.cache.evict(&u);
self.evict_reader(&u);
}
}
}
Ok(name)
}
/// Whether the node at `ino` is kept offline because a folder above it is
/// pinned, whether or not it is also pinned on its own.
fn pinned_by_folder(&self, ino: u64) -> bool {
let parent = {
let st = self.state();
st.entries
.get(&ino)
.filter(|e| e.parent != ino)
.and_then(|e| st.entries.get(&e.parent))
.map(|p| p.uid.clone())
};
parent.is_some_and(|uid| self.cache.is_pinned(&uid))
}
/// A Photos API handle sharing this Core's Drive client and session, so it
/// reuses the daemon's single authenticated session rather than logging in
/// again (Proton refresh tokens are single-use). Cheap — the Drive client
/// is `Clone` over `Arc`-backed state.
fn photos(&self) -> ProtonPhotosClient {
ProtonPhotosClient::from_drive_client(self.client.clone())
}
/// List the directory at mountpoint-relative `rel` for the in-app browser:
/// the same lazy remote enumeration `readdir` uses, projected into
/// serializable [`DirEntry`]s (with per-file pin state).
fn list_dir(&self, rel: &Path) -> CoreResult<Vec<DirEntry>> {
let (ino, uid) = self.resolve(rel)?;
self.ensure_children(ino)
.map_err(|e| self.errno_error(e, "enumerate"))?;
self.settle_listing_sizes(ino);
// Every entry shares this folder as its parent, so one lookup says
// whether they are all kept through it.
let pinned_by_folder = self.cache.is_pinned(&uid);
// Snapshot the listing, then drop the lock before touching the on-disk
// pin registry so a slow disk read doesn't block FUSE metadata ops.
let rows: Vec<(String, bool, u64, i64, NodeUid, u64)> = {
let st = self.state();
st.children
.get(&ino)
.map(|kids| {
kids.iter()
.filter_map(|&k| st.entries.get(&k).map(|e| (k, e)))
.map(|(k, e)| {
(
e.node.name.clone(),
e.node.is_folder(),
node_size(&e.node),
e.node.modification_time,
e.uid.clone(),
k,
)
})
.collect()
})
.unwrap_or_default()
};
let issues = self.db.node_issues().unwrap_or_default();
Ok(rows
.into_iter()
.map(|(name, is_dir, size, modified, uid, ino)| DirEntry {
name,
is_dir,
size,
modified,
pinned: pinned_by_folder || self.cache.is_pinned(&uid),
pinned_by_folder,
cached: !is_dir && self.cache.is_cached(&uid, modified, size),
uid: uid.to_string(),
// Listing entries live in the requested dir; the caller derives
// the path from its name. Left empty.
path: String::new(),
// My own tree: a role only means something under a share, and
// this listing never crosses into one.
role: String::new(),
shared_by: String::new(),
shared_at: 0,
shared_by_unverified: false,
trashed_at: 0,
trashed_from: None,
issue: state::ino_lid(ino)
.and_then(|lid| issues.get(&lid))
.map(|issue| SyncIssue::parse(issue)),
})
.collect())
}
/// Every uid that is pinned right now, as one set.
///
/// `cache.is_pinned` is a direct lookup plus an ancestor walk; asking it per
/// candidate was a thousand walks per keystroke. The recursive expansion is
/// the same query, run once.
fn pinned_set(&self) -> HashSet<String> {
self.db
.pinned_uids()
.unwrap_or_default()
.into_iter()
.collect()
}
/// The local locations a search result can be opened through, resolved once
/// per search rather than once per hit.
///
/// An on-demand folder only answers while its mount is live — without the
/// mount its `local_path` is an ordinary empty directory. A mirror folder
/// needs no mount: its files are real local copies the sync loop pushes
/// back, so it is a valid open target whenever it is configured.
///
/// This used to be a per-hit call that re-read the sync-folder list and ran
/// an ancestor walk per folder, against a candidate pool of up to a
/// thousand: the whole cost of a keystroke was in here.
fn search_roots(&self) -> SearchRoots {
let live_mounts = self.mounts.lock();
let folders = self.db.sync_folder_list().unwrap_or_default();
SearchRoots {
roots: folders
.into_iter()
.filter(|folder| match folder.mode.as_str() {
"ondemand" => live_mounts.contains_key(&folder.id),
"mirror" => true,
_ => false,
})
.filter_map(|folder| {
let root = self.db.node_path(&folder.remote_uid).ok().flatten()?;
Some((root, PathBuf::from(folder.local_path)))
})
.collect(),
}
}
/// Full-text search node names against the local SQLite index, mapping each
/// DB hit to the wire [`SearchHit`] (resolving live pin state from the cache,
/// which the DB doesn't track). Pure local lookup — never hits the network.
/// Every pin, with `cached` saying whether a pinned file's content is on
/// disk, so a front-end listing pins can badge them without a round-trip
/// per file.
fn list_pins(&self) -> Vec<pdfs_core::cache::Pin> {
let mut pins = self.cache.list_pins();
for pin in pins.iter_mut().filter(|pin| pin.is_dir == Some(false)) {
if let Ok(Some(node)) = self.db.node_by_uid(&pin.uid) {
pin.cached =
self.cache
.is_cached(&node.uid, node.modification_time, node_size(&node));
}
}
pins
}
fn search(&self, query: &str, limit: usize, scope: Option<&str>) -> CoreResult<Vec<SearchHit>> {
let hits = self
.db
.search_in(query, limit, scope)
.map_err(|e| CoreError::from_api(&e, "search"))?;
let roots = self.search_roots();
let pinned = self.pinned_set();
Ok(hits
.into_iter()
.map(|h| SearchHit {
name: h.node.name.clone(),
is_dir: h.node.is_folder(),
size: node_size(&h.node),
modified: h.node.modification_time,
pinned: pinned.contains(&h.node.uid.to_string()),
cached: !h.node.is_folder()
&& self.cache.is_cached(
&h.node.uid,
h.node.modification_time,
node_size(&h.node),
),
uid: h.node.uid.to_string(),
mounted_path: roots.resolve(&h.path),
path: h.path,
score: 0,
})
.collect())
}
/// Search the index of files on this machine (outside Drive), built by the
/// background scanner in [`run_local_index`]. Pure local lookup, never hits
/// the network — and never touches the FUSE mount, which the scanner excludes.
fn search_local(&self, query: &str, limit: usize) -> CoreResult<Vec<LocalHit>> {
let hits = self
.db
.search_local(query, limit)
.map_err(|e| CoreError::from_api(&e, "local search"))?;
Ok(hits
.into_iter()
.map(|h| LocalHit {
name: h.name,
path: h.path,
is_dir: h.is_dir,
size: h.size.max(0) as u64,
modified: h.mtime,
score: 0,
})
.collect())
}
/// Search both prompt sources with identical query and per-source limits.
/// Keeping the composition in the daemon core gives every control surface
/// the same semantics while preserving the legacy source-specific methods.
fn search_v2(
&self,
query: &str,
limit: usize,
filters: &SearchFilters,
) -> CoreResult<(Vec<SearchHit>, Vec<LocalHit>)> {
if query.trim().is_empty() || limit == 0 {
return Ok((Vec::new(), Vec::new()));
}
// Candidate generation is intentionally broader than the final result
// cap: filtering and fuzzy rejection happen below, so limiting at the
// SQLite boundary to `limit` would recreate the old empty-filter bug.
let candidate_limit = limit.saturating_mul(10).clamp(100, 1_000);
let mut drive_hits = if filters.sources.contains(&SearchSource::Drive) {
let roots = self.search_roots();
let pinned = self.pinned_set();
self.db
.search_candidates(query, candidate_limit)
.map_err(|e| CoreError::from_api(&e, "search candidates"))?
.into_iter()
.filter(|hit| filters.kind.accepts(&hit.node.name, hit.node.is_folder()))
.filter_map(|hit| {
let parent = Path::new(&hit.path)
.parent()
.map_or_else(String::new, |path| path.display().to_string());
let uid = hit.node.uid.to_string();
let is_pinned = pinned.contains(&uid);
let score = relevance_score(query, &hit.node.name, &parent)?
+ if is_pinned { 250 } else { 0 };
Some(SearchHit {
name: hit.node.name.clone(),
is_dir: hit.node.is_folder(),
size: node_size(&hit.node),
modified: hit.node.modification_time,
pinned: is_pinned,
cached: !hit.node.is_folder()
&& self.cache.is_cached(
&hit.node.uid,
hit.node.modification_time,
node_size(&hit.node),
),
uid,
mounted_path: roots.resolve(&hit.path),
path: hit.path,
score,
})
})
.collect()
} else {
Vec::new()
};
let mut local_hits = if filters.sources.contains(&SearchSource::Local) {
self.db
.search_local_candidates(query, candidate_limit)
.map_err(|e| CoreError::from_api(&e, "local search candidates"))?
.into_iter()
.filter(|hit| filters.kind.accepts(&hit.name, hit.is_dir))
.filter_map(|hit| {
let parent = Path::new(&hit.path)
.parent()
.map_or_else(String::new, |path| path.display().to_string());
Some(LocalHit {
score: relevance_score(query, &hit.name, &parent)?,
name: hit.name,
path: hit.path,
is_dir: hit.is_dir,
size: hit.size.max(0) as u64,
modified: hit.mtime,
})
})
.collect()
} else {
Vec::new()
};
drive_hits.sort_by(|a, b| {
b.score
.cmp(&a.score)
.then_with(|| a.name.cmp(&b.name))
.then_with(|| a.path.cmp(&b.path))
});
local_hits.sort_by(|a, b| {
b.score
.cmp(&a.score)
.then_with(|| a.name.cmp(&b.name))
.then_with(|| a.path.cmp(&b.path))
});
drive_hits.truncate(limit);
local_hits.truncate(limit);
Ok((drive_hits, local_hits))
}
/// A page of the photos timeline (newest first), sliced out of the DB.
/// `Ok(None)` when the account has no photos volume.
///
/// Stale-while-revalidate: a persisted timeline is served *immediately*, and
/// refreshed on the runtime if it is older than [`TIMELINE_TTL`] — so opening
/// the app paints from disk rather than waiting on a full `enumerate_timeline`
/// (which returns the whole timeline, not a page). Only an empty DB blocks on
/// the network, i.e. the very first run.
///
/// Metadata only: a thumbnail path is attached for photos whose thumbnail is
/// already cached, and nothing is downloaded here — the front-end pulls the
/// thumbnails it actually paints via [`Core::photo_thumbs`].
fn listing_stale(&self, key: &str, ttl: Duration) -> bool {
match self.db.state_i64(key).ok().flatten() {
Some(ms) => now_ms().saturating_sub(ms) >= ttl.as_millis() as i64,
None => true,
}
}
/// Re-fetch the whole photos timeline and persist it. Returns whether the
/// account has a photos volume at all.
/// Download the full content of the Drive file at mountpoint-relative `rel`
/// into the content cache, returning its on-disk path (served from cache
/// when a fresh blob already exists). Lets a front-end open the file with
/// the user's default application without pinning it.
fn open_file(&self, rel: &Path) -> CoreResult<PathBuf> {
let (ino, uid) = self.resolve(rel)?;
let (name, mtime, size) = {
let st = self.state();
let e = st
.entries
.get(&ino)
.ok_or_else(|| CoreError::not_found("node vanished"))?;
if !e.node.is_file() {
return Err(CoreError::invalid("not a regular file"));
}
(
e.node.name.clone(),
e.node.modification_time,
node_size(&e.node),
)
};
self.fetch_content(&uid, &name, mtime, size)
}
/// [`open_file`](Self::open_file) addressed by uid instead of by path.
///
/// A search hit is a metadata-index row, and the index covers every
/// location this daemon knows — including on-demand sync folders and
/// mirrors, whose nodes have no path inside the primary mount's tree.
/// Walking such a hit's path from the mount root answers ENOENT even though
/// the node is perfectly reachable, so front-ends address hits by uid.
///
/// Metadata comes from the live tree, then the index, then the API — the
/// last covers a node the index knows but no location holds. Unlike the
/// mutating by-uid handlers this does not demand a
/// [`resolve_anywhere`](Self::resolve_anywhere) residency proof: reading
/// one's own file is what the API authorizes anyway, and requiring
/// residency is exactly what turned an openable search hit into "no such
/// file or folder". [`open_shared_file`](Self::open_shared_file) already
/// works this way. Reserved (local/virtual) uids name no remote node and
/// are still rejected.
fn open_file_uid(&self, raw_uid: &str) -> CoreResult<PathBuf> {
let uid = parse_uid(raw_uid)
.filter(|uid| !is_local_uid(uid) && !is_virtual_uid(uid))
.ok_or_else(|| CoreError::invalid(format!("invalid uid: {raw_uid}")))?;
let live = self
.state
.lock()
.entries
.values()
.find(|e| e.uid == uid)
.map(|e| e.node.clone());
let node = match live {
Some(node) => node,
None => match self
.db
.node_by_uid(&uid.to_string())
.map_err(|e| CoreError::from_api(&e, "load node"))?
{
Some(node) => node,
None => self
.rt
.block_on(self.drive.get_node(&uid))
.map_err(|e| CoreError::from_api(&e, "get node"))?
.ok_or_else(|| CoreError::not_found("node vanished"))?,
},
};
if !node.is_file() {
return Err(CoreError::invalid("not a regular file"));
}
self.fetch_content(&uid, &node.name, node.modification_time, node_size(&node))
}
/// Materialise a file's full content into the content cache, returning its
/// on-disk path. Serves a fresh cached blob without touching the network.
fn fetch_content(
&self,
uid: &NodeUid,
name: &str,
mtime: i64,
size: u64,
) -> CoreResult<PathBuf> {
if let Some(p) = self.cache.cached_content_path(uid, mtime, size) {
return Ok(p);
}
// Keep the temporary file beside the final cache blob. `store_file`
// can then adopt it with a hard link and atomic rename, while the SDK
// streams decrypted blocks to disk instead of building a file-sized
// `Vec<u8>`. A unique name also keeps concurrent opens of the same node
// from sharing a partially written staging file.
static OPEN_TEMP_SEQ: AtomicU64 = AtomicU64::new(0);
let seq = OPEN_TEMP_SEQ.fetch_add(1, Ordering::Relaxed);
let tmp = self.cache.content_path(uid).with_extension(format!(
"open-{}-{}-{seq}.tmp",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
));
let file = File::options()
.write(true)
.create_new(true)
.open(&tmp)
.map_err(|e| CoreError::internal(format!("create download temp: {e}")))?;
let file = match self.download_file_tracked_to(uid, name, size, file) {
Ok(file) => file,
Err(e) => {
let _ = std::fs::remove_file(&tmp);
return Err(CoreError::from_api(&e, "download"));
}
};
if let Err(e) = file.sync_all() {
let _ = std::fs::remove_file(&tmp);
return Err(CoreError::internal(format!("sync download temp: {e}")));
}
if let Err(e) = self.cache.store_file(uid, mtime, size, &tmp) {
let _ = std::fs::remove_file(&tmp);
return Err(CoreError::internal(format!("cache store: {e}")));
}
let _ = std::fs::remove_file(&tmp);
Ok(self.cache.content_path(uid))
}
/// Drop the cached child listing of `rel`'s parent directory so the next
/// `ListDir` re-enumerates it from the server. No-op when the parent can't be
/// resolved (e.g. `rel` is the root). Resolves the parent without holding the
/// state lock, then invalidates under it.
fn invalidate_parent_listing(&self, rel: &Path) {
let parent = rel.parent().unwrap_or_else(|| Path::new(""));
if let Ok((pino, _)) = self.resolve_path(parent) {
self.state().invalidate_listing(pino);
}
}
/// Drop the cached child listing of the folder `parent`, everywhere it is
/// held — naming the folder by uid rather than by inode.
///
/// [`Core::invalidate_parent_listing`] is the path-based sibling, and it
/// only reaches *this* mount's paths. A remote mutation made by the
/// sync engine needs more than that, for two reasons. The folder it changed
/// is usually not resident in the tree at the time — a mirror folder
/// has no FUSE session at all — so there is no inode to name. And the stale
/// listing that outlives the pass is not a hot-cache map but the `listed`
/// flag on the folder's DB row, which survives a daemon restart and is what
/// a later `ensure_children` trusts.
///
/// So the flag is cleared unconditionally, and the resident listing is
/// invalidated on top of it. That is what makes a mirror folder switched to
/// on-demand re-enumerate instead of serving a snapshot that predates the
/// engine's own uploads — `docs/BUGS.md` B86.
pub(crate) fn invalidate_children_of(&self, parent: &NodeUid) {
if let Err(e) = self.db.set_listed(parent, false) {
warn!(uid = %parent, error = ?e, "could not clear cached listing");
}
self.for_each_state(|st| {
if let Some(&ino) = st.by_uid.get(parent) {
st.invalidate_listing(ino);
}
});
}
/// Rename a file or folder to `new_name`. `rel` is mountpoint-relative.
/// Mirrors the FUSE `rename` write path: queued whenever the mount queues
/// ([`Core::queue_relocation`]), otherwise renamed on the remote, then
/// moved to its new name in every mount with the parent listing dropped
/// so the next `ListDir` re-enumerates.
fn rename(&self, rel: &Path, new_name: &str) -> CoreResult<String> {
if new_name.is_empty() || new_name.contains('/') {
return Err(CoreError::invalid(format!("invalid name: {new_name:?}")));
}
let (ino, uid) = self.resolve(rel)?;
let old_parent_uid = self.source_parent_uid(ino, rel)?;
require_rename_access(
|authority| self.require_uid_writable(authority),
&uid,
&old_parent_uid,
&old_parent_uid,
)
.map_err(|error| self.errno_error(error, "rename access"))?;
if self.queues_relocation(&uid, &old_parent_uid) {
self.queue_relocation(
ino,
&uid,
&old_parent_uid,
&old_parent_uid,
&rel.with_file_name(new_name),
new_name,
)?;
return Ok(new_name.to_string());
}
self.rename_remote(&uid, new_name)
.map_err(|e| CoreError::from_api(&e, "rename"))?;
self.relocate_everywhere(&uid, &old_parent_uid, new_name);
self.invalidate_parent_listing(rel);
Ok(new_name.to_string())
}
/// Whether a rename or move asked for over the control socket goes
/// through the queue. It does whenever the same change through the mount
/// would, and whenever the node has anything queued: sent straight to
/// Drive, it would land first, and the queued change landing after it
/// would undo it (`docs/BUGS.md` B149).
fn queues_relocation(&self, uid: &NodeUid, new_parent_uid: &NodeUid) -> bool {
!self.sends_inline()
|| is_local_uid(uid)
|| is_local_uid(new_parent_uid)
|| self.db.has_any_op(&uid.to_string()).unwrap_or(true)
}
/// Queue renaming or moving the node at `ino` to `dest`, whose last
/// component is `name`, and apply it to every mount now, as the mount's
/// own queued rename does.
///
/// The listings stay: Drive has not been told yet, so listing either
/// folder again would bring back the old name.
fn queue_relocation(
&self,
ino: u64,
uid: &NodeUid,
old_parent_uid: &NodeUid,
new_parent_uid: &NodeUid,
dest: &Path,
name: &str,
) -> CoreResult<()> {
// Drive refuses a taken name, but only once the queue sends the change.
match self.resolve_path(dest) {
Ok((found, _)) if found != ino => {
return Err(CoreError::conflict(format!(
"{} already exists",
dest.display()
)));
}
Ok(_) => {}
Err(e) if e.code() == libc::ENOENT => {}
Err(e) => {
return Err(self.errno_error(e, &format!("could not resolve {}", dest.display())));
}
}
let old_name = self
.state()
.entries
.get(&ino)
.map(|entry| entry.node.name.clone())
.unwrap_or_default();
let mut uid = uid.clone();
// A node whose create is still queued is that op: rewriting its target
// is the whole change, as through the mount.
if is_local_uid(&uid) {
match self
.db
.rewrite_op_target(&uid.to_string(), &new_parent_uid.to_string(), name)
{
Ok(true) => {
self.relocate_queued_everywhere(&uid, new_parent_uid, name);
// A parked transient file given a finished name is meant
// to reach Drive now (`docs/BUGS.md` B70).
if is_transient_name(&old_name)
&& !is_transient_name(name)
&& let Err(e) = self.db.set_create_hold(&uid.to_string(), false)
{
warn!(%uid, error = %e, "un-parking a finalized transient create failed");
}
self.wake_drain();
return Ok(());
}
// The create landed meanwhile: rename what Drive made.
Ok(false) => {
uid = self.landed_row_uid(ino).ok_or_else(|| {
CoreError::conflict(format!(
"{old_name} changed while it was being renamed; try again"
))
})?;
}
Err(e) => return Err(CoreError::internal(format!("db: {e}"))),
}
}
self.enqueue_rename(ino, &uid, old_parent_uid, new_parent_uid, name)
.map_err(|e| self.errno_error(e, "queue the change"))?;
self.relocate_queued_everywhere(&uid, new_parent_uid, name);
self.wake_drain();
debug!(%uid, %new_parent_uid, name, "renamed over the control socket; queued");
Ok(())
}
/// Bring every mount level with a rename or move Drive has made of `uid`,
/// to `name` in `new_parent`, and have each kernel session drop the name it
/// cached for it. Returns the name a mount knew the node by.
///
/// For changes that do not come through the mount. The kernel made a
/// rename through the mount itself, but one made over the control socket
/// leaves the old name cached for [`TTL`] (`docs/BUGS.md` B114).
///
/// The node is moved, not forgotten, and so is its row. Forgetting it, as
/// this used to, deleted the row, and until a listing brought it back
/// nothing local knew the node: `pdfs sync queue` showed its queued upload
/// by raw id, and the drain re-read the node and put the upload off
/// (`docs/BUGS.md` B117).
fn relocate_everywhere(
&self,
uid: &NodeUid,
new_parent: &NodeUid,
name: &str,
) -> Option<String> {
self.relocate_with(uid, new_parent, name, false)
}
/// [`Core::relocate_everywhere`] for a change the queue has not sent yet.
/// Each mount keeps its listings, which Drive cannot bring up to date
/// until the change lands.
fn relocate_queued_everywhere(&self, uid: &NodeUid, new_parent: &NodeUid, name: &str) {
self.relocate_with(uid, new_parent, name, true);
}
fn relocate_with(
&self,
uid: &NodeUid,
new_parent: &NodeUid,
name: &str,
queued: bool,
) -> Option<String> {
let mut old_name = None;
let mut moved_row = false;
self.for_each_mount(|st, notify| {
if let Some((from_name, relocated)) =
relocate_in_mount(st, notify, uid, new_parent, name, queued)
{
old_name = Some(from_name);
moved_row |= relocated;
}
});
if !moved_row {
self.move_row(uid, new_parent, name);
}
old_name
}
/// Point the persisted row of `uid` at `name` in `new_parent`, for a node
/// no mount holds where it now is.
fn move_row(&self, uid: &NodeUid, new_parent: &NodeUid, name: &str) {
let moved = self.db.node_by_uid(&uid.to_string()).and_then(|node| {
let Some(mut node) = node else {
return Ok(());
};
node.name = name.to_string();
node.parent_uid = Some(new_parent.clone());
self.db.upsert_node(&node)
});
if let Err(error) = moved {
warn!(%uid, ?error, "could not move the node's row along with it");
}
}
/// Forget `uid` in every mount, and have each kernel session drop the name
/// it cached for it. Returns the name a mount knew the node by.
///
/// For a removal that does not come through the mount. The kernel made an
/// unlink through the mount itself, but one made over the control socket
/// leaves the old name cached for [`TTL`], pointing at an inode the mount no
/// longer has (`docs/BUGS.md` B114). Every mount, not only this one, so a
/// fork showing the same node does not keep serving it (`docs/BUGS.md`
/// B74).
fn forget_everywhere(&self, uid: &NodeUid) -> Option<String> {
let mut name = None;
self.for_each_mount(|st, notify| {
if let Some(entry) = st.by_uid.get(uid).and_then(|ino| st.entries.get(ino)) {
name = Some(entry.node.name.clone());
}
forget_and_notify(st, notify, uid);
});
name
}
/// Move a file or folder into the folder at `new_parent_rel`. Both paths are
/// mountpoint-relative. Queued whenever the mount queues
/// ([`Core::queue_relocation`]); otherwise moved on the remote, then in
/// every mount, with both listings invalidated so each re-enumerates on
/// next access.
fn move_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult<String> {
let (ino, uid) = self.resolve(rel)?;
let old_parent_uid = self.source_parent_uid(ino, rel)?;
let (pino, new_parent_uid) = self
.resolve_path(new_parent_rel)
.map_err(|e| self.errno_error(e, "resolve new parent"))?;
require_rename_access(
|authority| self.require_uid_writable(authority),
&uid,
&old_parent_uid,
&new_parent_uid,
)
.map_err(|error| self.errno_error(error, "move access"))?;
let name = self
.state()
.entries
.get(&ino)
.map(|entry| entry.node.name.clone())
.unwrap_or_default();
if self.queues_relocation(&uid, &new_parent_uid) {
if self.state().is_ancestor_of(ino, pino) {
return Err(CoreError::invalid("cannot move a folder into itself"));
}
self.queue_relocation(
ino,
&uid,
&old_parent_uid,
&new_parent_uid,
&new_parent_rel.join(&name),
&name,
)?;
return Ok(name);
}
self.move_remote(&uid, &new_parent_uid)
.map_err(|e| CoreError::from_api(&e, "move"))?;
self.relocate_everywhere(&uid, &new_parent_uid, &name);
self.invalidate_parent_listing(rel);
self.state().invalidate_listing(pino);
Ok(name)
}
/// Copy a file or folder into `new_parent_rel` on Proton Drive, which copies
/// the content there: nothing is downloaded or uploaded. Both paths are
/// mountpoint-relative. A name already taken in the folder gets the next
/// free ` (n)` suffix. Returns the copy's name.
///
/// Drive copies what it holds, so the copy is refused offline and while the
/// mount still owes Drive part of it: a node or folder only queued locally,
/// a file with a queued upload or unsaved writes, or — for a folder, whose
/// queued changes could sit anywhere below it — any queued op at all.
fn copy_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult<String> {
let (ino, uid) = self.resolve(rel)?;
let (pino, new_parent_uid) = self
.resolve_path(new_parent_rel)
.map_err(|e| self.errno_error(e, "resolve new parent"))?;
self.ensure_children(pino)
.map_err(|e| self.errno_error(e, "enumerate"))?;
self.require_uid_writable(&new_parent_uid)
.map_err(|error| self.errno_error(error, "copy access"))?;
let (name, is_folder, unsaved) = {
let st = self.state();
let entry = st
.entries
.get(&ino)
.ok_or_else(|| CoreError::not_found(format!("{} is gone", rel.display())))?;
// Unsaved writes, or a release still queueing them, to the file or,
// for a folder, to any file below it.
let unsaved = st
.active_writes
.iter()
.filter(|(_, w)| w.dirty)
.map(|(&open, _)| open)
.chain(st.releasing.iter().copied())
.any(|open| relocate::ino_is_under(&st, open, ino));
(entry.node.name.clone(), entry.node.is_folder(), unsaved)
};
// Unlike a move, a copy cannot wait in the queue: it needs the node as
// Drive has it, so it goes now or not at all, local-first or not.
if !self.is_online() {
return Err(CoreError::offline());
}
let owed = unsaved
|| is_local_uid(&uid)
|| is_local_uid(&new_parent_uid)
|| self.db.has_any_op(&uid.to_string()).unwrap_or(true)
|| (is_folder
&& self
.db
.pending_op_counts()
.map(|counts| counts.uploads + counts.changes > 0)
.unwrap_or(true));
if owed {
return Err(CoreError::conflict(format!(
"{name} has changes that have not reached Proton Drive yet; \
copy it once they have synced"
)));
}
let target = if is_folder {
AvailableNameTarget::Folder
} else {
AvailableNameTarget::File
};
let copy_name = self
.rt
.block_on(
self.drive
.get_available_name(&new_parent_uid, &name, target),
)
.map_err(|e| CoreError::from_api(&e, "copy"))?;
let new_uid = self
.rt
.block_on(
self.drive
.copy_node(&uid, &new_parent_uid, Some(&copy_name)),
)
.map_err(|e| CoreError::from_api(&e, "copy"))?;
let node = self
.fetch_minted_node(&new_uid, &copy_name)
.ok_or_else(|| minted_but_unlisted(&copy_name))?;
let mut st = self.state();
let copy_ino = st.intern(pino, node);
if let Some(kids) = st.children.get_mut(&pino)
&& !kids.contains(&copy_ino)
{
kids.push(copy_ino);
}
drop(st);
self.flush_access_changes();
Ok(copy_name)
}
/// What the folder at `rel` holds on Proton Drive: bytes on storage and the
/// number of items below it. Drive adds it up, so nothing is listed here;
/// trashed items count, and the bytes are encrypted ones.
pub(crate) fn folder_size(&self, rel: &Path) -> CoreResult<proton_drive_rs::FolderSize> {
let (ino, uid) = self.resolve(rel)?;
let is_folder = self
.state()
.entries
.get(&ino)
.is_some_and(|entry| entry.node.is_folder());
if !is_folder {
return Err(CoreError::invalid(format!(
"{} is not a folder",
rel.display()
)));
}
self.folder_size_for_uid(&uid)
}
/// [`Core::folder_size`] for a folder addressed through any daemon location.
pub(crate) fn folder_size_by_uid(&self, uid: &str) -> CoreResult<proton_drive_rs::FolderSize> {
let uid = self.resolve_anywhere(uid)?;
self.folder_size_for_uid(&uid)
}
fn folder_size_for_uid(&self, uid: &NodeUid) -> CoreResult<proton_drive_rs::FolderSize> {
if is_local_uid(uid) {
return Err(CoreError::conflict("the folder is not on Proton Drive yet"));
}
if !self.is_online() {
return Err(CoreError::offline());
}
self.rt
.block_on(self.client.get_folder_size(uid))
.map_err(|e| CoreError::from_api(&e, "folder size"))
}
/// Trash a file or folder. `rel` is mountpoint-relative. Forgets the node,
/// evicts any cached content, and invalidates the parent listing.
fn delete(&self, rel: &Path) -> CoreResult<String> {
let (ino, uid) = self.resolve(rel)?;
let parent_uid = self.source_parent_uid(ino, rel)?;
require_node_parent_access(
|authority| self.require_uid_writable(authority),
&uid,
&parent_uid,
)
.map_err(|error| self.errno_error(error, "trash access"))?;
self.rt
.block_on(self.drive.trash_nodes(std::slice::from_ref(&uid)))
.and_then(batch::into_unit)
.map_err(|e| CoreError::from_api(&e, "trash"))?;
// Uploads queued for the node or anything below it would now only fail
// against the trash, forever, holding their bytes (`docs/BUGS.md` B102).
if let Err(error) = self.discard_queued_ops(&uid) {
warn!(%uid, ?error, "remote trash landed but queued-op cleanup failed");
}
let name = self.forget_everywhere(&uid).unwrap_or_default();
self.cache.evict(&uid);
self.evict_reader(&uid);
self.invalidate_parent_listing(rel);
self.invalidate_trash();
Ok(name)
}
/// Remove the node a `rename` is about to replace, so the new name is free
/// for the API call that follows.
///
/// `rename(2)` promises to replace an existing destination atomically. Proton
/// offers no such primitive — `rename_node` refuses a name that is already
/// taken — so this is the first half of an emulation that is *not* atomic:
/// see [`Core::restore_replaced`] for the other half (`docs/BUGS.md` B13).
///
/// A node whose own creation is still queued has never reached the server, so
/// dropping its queued ops is the whole removal; nothing goes to the wire and
/// it works offline. A create that landed before its ops were dropped left
/// none: Drive has the node, so it is trashed there like any other
/// (`docs/BUGS.md` B175).
///
/// A handle still open on the replaced node keeps reading its bytes, as
/// `rename(2)` promises: the inode stays as an unlinked entry and its
/// staged bytes, cache and reader go at the final close, exactly as for an
/// `unlink` under an open handle. Its queued ops go now, or the drain would
/// upload to the node after it was trashed (`docs/BUGS.md` B106).
fn remove_replaced(&self, ino: u64, uid: &NodeUid, name: &str) -> Result<(), Errno> {
let landed;
let uid = if is_local_uid(uid) {
if self.is_open_anywhere(uid) {
self.withdraw_queued_ops(uid)?;
} else {
self.discard_queued_ops(uid)?;
}
self.for_each_state(|st| {
st.forget_or_unlink(uid);
});
match self.landed_row_uid(ino) {
Some(real) => {
debug!(local = %uid, %real, name, "create landed under a replace");
landed = real;
&landed
}
None => {
debug!(%uid, name, "replaced a node whose create was still queued");
return Ok(());
}
}
} else {
uid
};
// Local-first or offline, or the network goes away under the call:
// queue the trash. The rename that follows queues as well, behind it.
if !self.sends_inline() {
return self.queue_trash(uid, name);
}
self.require_uid_writable(uid)?;
if let Err(e) = self
.block_on_bounded(self.drive.trash_nodes(std::slice::from_ref(uid)))
.and_then(batch::into_unit)
{
if self.lost_link(&e, "trash a replaced node") {
return self.queue_trash(uid, name);
}
error!(%uid, name, error = %e, "trashing the node a rename replaces failed");
self.log_activity(ActivityKind::Trash, name, e.to_string(), false);
return Err(Errno::EIO);
}
// The node was trashed on the server; withdraw it from every inode
// space that had it, not only the mount the rename came through.
let open_now = self.is_open_anywhere(uid);
self.for_each_state(|st| {
st.forget_or_unlink(uid);
});
self.hidden.lock().insert(uid.clone());
if open_now {
if let Err(error) = self.withdraw_queued_ops(uid) {
error!(%uid, "remote replacement landed but queued-op cleanup failed");
return Err(error);
}
} else {
if let Err(error) = self.discard_queued_ops(uid) {
error!(%uid, "remote replacement landed but queued-op cleanup failed");
return Err(error);
}
self.cache.evict(uid);
self.evict_reader(uid);
}
self.invalidate_trash();
// The node is recoverable from the trash, but only if the user knows it
// went there — a rename is not an operation anyone expects to trash
// something, so this is the only record that it happened.
self.log_activity(
ActivityKind::Trash,
name,
"replaced by a rename from the mount",
true,
);
Ok(())
}
/// Whether any mount holds an open handle on `uid`.
fn is_open_anywhere(&self, uid: &NodeUid) -> bool {
let mut open = false;
self.for_each_state(|st| {
open |= st
.by_uid
.get(uid)
.and_then(|ino| st.entries.get(ino))
.is_some_and(|entry| entry.open_count > 0);
});
open
}
/// Put back the node [`Core::remove_replaced`] trashed, after the rename it
/// was clearing the way for failed anyway.
///
/// Best-effort by construction: if the restore also fails there is nothing
/// further to try, and the node is still in the trash where `pdfs restore`
/// can reach it. Says so loudly in that case, because the alternative is a
/// file the user believes was only renamed quietly sitting in the trash.
///
/// A restored node comes back to the mount as the ordinary restore brings
/// it: out of `hidden`, and through a fresh listing of `parent` in every
/// mount that shows it (`docs/BUGS.md` B124).
fn restore_replaced(&self, victim: Option<&(u64, NodeUid)>, parent: &NodeUid, name: &str) {
let Some((_, uid)) = victim else { return };
if is_local_uid(uid) {
// Its queued create was discarded and cannot be reconstructed from
// here; the caller's error is what the user acts on.
warn!(%uid, name, "a rename failed after discarding a queued node it replaced");
return;
}
match self
.rt
.block_on(self.drive.restore_nodes(std::slice::from_ref(uid)))
.and_then(batch::into_unit)
{
Ok(()) => {
self.hidden.lock().remove(uid);
self.for_each_mount(|st, notify| {
if let Some(&ino) = st.by_uid.get(parent) {
st.invalidate_listing(ino);
notify.inval_entry(ino, name.to_string());
}
});
self.invalidate_trash();
debug!(%uid, name, "restored the node a failed rename had replaced");
}
Err(e) => {
error!(%uid, name, error = %e, "restoring a replaced node failed; it stays in the trash");
self.log_activity(ActivityKind::Restore, name, e.to_string(), false);
}
}
}
/// List the account's trash, from the DB. Trashed nodes are outside the
/// mounted tree — the FUSE side forgot them when they were trashed — so the
/// listing is persisted in its own table rather than derived from `State`, and
/// each entry is identified by its uid (its only remaining handle) with an
/// empty path.
///
/// Stale-while-revalidate, like the photos timeline: a persisted listing comes
/// back at DB speed and is refreshed in the background past [`TRASH_TTL`].
/// Our own trash mutations invalidate it outright (see
/// [`Core::invalidate_trash`]), so the TTL only covers changes made elsewhere.
///
/// The refresh always runs *off* the request: a never-fetched trash waits for
/// it, but only for [`TRASH_FIRST_WAIT`], and then answers with the batches
/// that have landed so far. Blocking the request on the whole refresh is what
/// made a large trash unlistable — the refresh outran the front-end's read
/// timeout, the user asked again, and each attempt left another full refresh
/// grinding in the daemon (`docs/BUGS.md` B72).
fn list_trash(&self) -> CoreResult<Vec<DirEntry>> {
let never_fetched = self.db.state_i64(TRASH_SYNCED_MS).ok().flatten().is_none();
if never_fetched || self.listing_stale(TRASH_SYNCED_MS, TRASH_TTL) {
self.spawn_trash_refresh();
}
if never_fetched {
self.await_trash_refresh(TRASH_FIRST_WAIT);
}
// Most of a trash comes from a handful of folders.
let mut origins: HashMap<String, Option<String>> = HashMap::new();
Ok(self
.db
.trash_list()
.map_err(CoreError::from)?
.into_iter()
.map(|item| DirEntry {
trashed_from: item.parent_uid.as_ref().and_then(|parent| {
origins
.entry(parent.clone())
.or_insert_with(|| self.db.node_path(parent).ok().flatten())
.clone()
}),
trashed_at: item.trashed_at.unwrap_or(0),
name: item.name,
is_dir: item.is_dir,
size: item.size.max(0) as u64,
modified: item.mtime,
// A trashed node can't be pinned or served from the mount, so its
// content is never current cache: report neither.
pinned: false,
pinned_by_folder: false,
cached: false,
uid: item.uid,
path: String::new(),
role: String::new(),
shared_by: String::new(),
shared_at: 0,
shared_by_unverified: false,
issue: None,
})
.collect())
}
/// Re-fetch the trash listing from the server and persist it.
///
/// Materialized in chunks of [`TRASH_MATERIALIZE_CHUNK`], each persisted as
/// it lands: decrypting a trashed node costs an S2K unlock, so a trash of a
/// few thousand nodes takes minutes, and an all-or-nothing write would throw
/// that away on the first failure and show the user nothing until the very
/// end. Every batch is a usable listing.
async fn refresh_trash(&self) -> CoreResult<()> {
let started = Instant::now();
let uids = within_trash_call("enumerate trash", self.drive.enumerate_trash_node_uids())
.await?
.map_err(|e| CoreError::from_api(&e, "enumerate trash"))?;
info!(
count = uids.len(),
elapsed_ms = started.elapsed().as_millis() as u64,
"trash refresh: enumerated"
);
// What the listing held before, and whether it was the whole trash: the
// chunks below overwrite it, so both are read up front.
let seen = self.db.trash_seen().unwrap_or_default();
let complete = self
.db
.state_i64(TRASH_COMPLETE_MS)
.ok()
.flatten()
.is_some();
let _ = self.db.clear_state(TRASH_COMPLETE_MS);
let now = now_secs();
if uids.is_empty() {
self.db.trash_replace(&[]).map_err(CoreError::from)?;
}
let mut items: Vec<StoredTrash> = Vec::with_capacity(uids.len());
for chunk in uids.chunks(TRASH_MATERIALIZE_CHUNK) {
let chunk_started = Instant::now();
let nodes = within_trash_call("enumerate nodes", self.drive.enumerate_nodes(chunk))
.await?
.map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?;
items.extend(nodes.into_iter().map(|node| StoredTrash {
uid: node.uid.to_string(),
name: node.name.clone(),
is_dir: node.is_folder(),
size: node_size(&node) as i64,
mtime: node.modification_time,
// The folder the node will go back to, which is also what tells
// a restore whether that folder is itself in the trash.
parent_uid: node.parent_uid.as_ref().map(|uid| uid.to_string()),
trashed_at: first_seen_in_trash(&seen, &node.uid.to_string(), complete, now),
}));
// Cumulative, so the table is always a prefix of the real trash
// rather than a mix of this refresh and the last one.
self.db.trash_replace(&items).map_err(CoreError::from)?;
self.trash_progress.notify_waiters();
debug!(
materialized = items.len(),
total = uids.len(),
chunk_ms = chunk_started.elapsed().as_millis() as u64,
"trash refresh: batch"
);
}
let _ = self.db.set_state_i64(TRASH_SYNCED_MS, now_ms());
let _ = self.db.set_state_i64(TRASH_COMPLETE_MS, now_ms());
info!(
count = items.len(),
elapsed_ms = started.elapsed().as_millis() as u64,
"trash refresh: done"
);
Ok(())
}
/// Refresh the trash off the request path. At most one refresh at a time —
/// including the one a first-ever [`Core::list_trash`] waits on, so a burst
/// of requests against an unfetched trash joins one refresh instead of
/// starting one apiece.
fn spawn_trash_refresh(&self) {
if self.trash_refreshing.swap(true, Ordering::SeqCst) {
return;
}
let core = self.clone();
self.rt.spawn(async move {
let refreshed = core.refresh_trash().await;
core.trash_refreshing.store(false, Ordering::SeqCst);
match refreshed {
Ok(_) => core.events.publish(&[Topic::Trash]),
Err(e) => warn!(error = %e, "background trash refresh failed"),
}
// After the flag, so a waiter that wakes on this sees it cleared.
core.trash_progress.notify_waiters();
});
}
/// Wait for the in-flight trash refresh to make progress, for at most
/// `budget`. Returns early when it finishes; never cancels it — the refresh
/// is a detached task, so giving up here costs the work nothing.
fn await_trash_refresh(&self, budget: Duration) {
let deadline = Instant::now() + budget;
self.rt.block_on(async {
while self.trash_refreshing.load(Ordering::SeqCst) {
let Some(left) = deadline.checked_duration_since(Instant::now()) else {
break;
};
let notified = self.trash_progress.notified();
tokio::pin!(notified);
// Register before re-checking the flag: a refresh that finishes
// in between must not leave us waiting for a notify that has
// already fired.
notified.as_mut().enable();
if !self.trash_refreshing.load(Ordering::SeqCst) {
break;
}
if tokio::time::timeout(left, notified).await.is_err() {
warn!(
budget_s = budget.as_secs(),
"trash refresh still running; answering with what has materialized"
);
break;
}
}
});
}
/// Drop the persisted trash listing's freshness stamp after we changed the
/// trash ourselves. The next Trash page then *waits* for a fresh listing
/// rather than painting one from before the mutation — the user just made the
/// change and is about to look straight at it.
fn invalidate_trash(&self) {
let _ = self.db.clear_state(TRASH_SYNCED_MS);
}
/// Drop the persisted photos timeline's freshness stamp, so the next timeline
/// read fetches rather than serving what it already has.
fn invalidate_photos(&self) {
let _ = self.db.clear_state(PHOTOS_SYNCED_MS);
}
/// Drop one folder's cached child listing (`rel` is mountpoint-relative), so
/// the next `ListDir`/`readdir` re-enumerates it from the server. Backs
/// [`CtlRequest::Refresh`] with a [`RefreshScope::Dir`] scope.
fn refresh_dir(&self, rel: &Path) -> CoreResult<()> {
let (ino, _uid) = self.resolve(rel)?;
self.state().invalidate_listing(ino);
Ok(())
}
/// Parse wire uids (`volume~link`) into [`NodeUid`]s, rejecting the whole
/// batch if any is malformed — a partial trash mutation is worse than none.
fn parse_uids(uids: &[String]) -> CoreResult<Vec<NodeUid>> {
if uids.is_empty() {
return Err(CoreError::invalid("no nodes given"));
}
uids.iter()
.map(|u| parse_uid(u).ok_or_else(|| CoreError::invalid(format!("invalid uid: {u}"))))
.collect()
}
/// Restore trashed nodes to the folders they were trashed from, together
/// with the rest of the trashed tree they belong to.
///
/// The trash is a flat list, but a user deletes a shape: a folder, and often
/// items inside it that were trashed separately beforehand. Restoring one of
/// those in isolation is not what anyone means by it — a folder would come
/// back without the contents that were deleted with it, and a file would come
/// back inside a folder that is still in the trash, which is to say nowhere
/// the user can see. So a restore takes the whole connected piece:
/// every trashed descendant of what was asked for, and every trashed ancestor
/// above it. See [`expand_restore`] for the exact rule.
///
/// The waves are restored shallowest first, because a node cannot be put back
/// under a parent that is still trashed.
///
/// The destination parents are read *before* the restore — a restored node
/// reappears in a listing the daemon may already have cached, so each one is
/// invalidated and re-enumerated on next access.
fn restore(&self, uids: &[String]) -> CoreResult<usize> {
let requested = Self::parse_uids(uids)?;
// A listing we could not read is not a reason to refuse the restore: it
// only costs the expansion, and the requested nodes still go back.
let relations = self.db.trash_parents().unwrap_or_default();
let waves = expand_restore(uids, &relations);
let waves: Vec<Vec<NodeUid>> = if waves.is_empty() {
vec![requested.clone()]
} else {
waves
.iter()
.map(|wave| {
wave.iter()
.filter_map(|uid| parse_uid(uid))
.collect::<Vec<_>>()
})
.filter(|wave: &Vec<NodeUid>| !wave.is_empty())
.collect()
};
let all: Vec<NodeUid> = waves.iter().flatten().cloned().collect();
let parents: Vec<NodeUid> = self
.rt
.block_on(self.drive.enumerate_nodes(&all))
.map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?
.into_iter()
.filter_map(|n| n.parent_uid)
.collect();
// Streamed per-node outcomes: a uid the server refuses (already restored,
// gone) leaves the rest of the batch restored, and each node is unhidden
// as *its* batch lands rather than after the last one — a daemon killed
// half way through a large restore leaves local state describing what the
// server actually did. Only a batch that restored nothing is an error the
// caller can act on.
let mut restored: Vec<NodeUid> = Vec::with_capacity(all.len());
let mut first_error: Option<ProtonError> = None;
for wave in &waves {
self.rt
.block_on(async {
let mut outcomes = std::pin::pin!(self.drive.restore_nodes_streaming(wave));
while let Some(item) = outcomes.next().await {
let (uid, outcome) = item?;
match outcome {
Ok(()) => {
self.hidden.lock().remove(&uid);
restored.push(uid);
}
Err(e) => {
warn!(%uid, error = %e, "restore failed for a node");
first_error.get_or_insert(e);
}
}
}
Ok::<(), ProtonError>(())
})
.map_err(|e| CoreError::from_api(&e, "restore"))?;
}
if restored.is_empty()
&& let Some(error) = first_error
{
return Err(CoreError::from_api(&error, "restore"));
}
// Keyed by uid, so it applies to every mount: a restored node reappears
// in whichever mounts show its parent, not only the primary one.
self.for_each_state(|st| {
for parent in &parents {
if let Some(&ino) = st.by_uid.get(parent) {
st.invalidate_listing(ino);
}
}
});
self.invalidate_trash();
Ok(restored.len())
}
/// Permanently delete trashed nodes. Irreversible on the server; locally it
/// drops any metadata and cached content the node still owns.
fn delete_forever(&self, uids: &[String]) -> CoreResult<usize> {
let parsed = Self::parse_uids(uids)?;
// Streamed, and each node's local state is dropped as its batch lands.
// A permanent delete is irreversible, so the useful property is that a
// daemon interrupted mid-batch has already forgotten exactly the nodes
// the server destroyed — never more, never fewer.
let mut deleted = 0usize;
let mut first_error: Option<ProtonError> = None;
self.rt
.block_on(async {
let mut outcomes = std::pin::pin!(self.drive.delete_nodes_streaming(&parsed));
while let Some(item) = outcomes.next().await {
let (uid, outcome) = item?;
match outcome {
Ok(()) => {
self.drop_local(std::slice::from_ref(&uid));
deleted += 1;
}
Err(e) => {
warn!(%uid, error = %e, "permanent delete failed for a node");
first_error.get_or_insert(e);
}
}
}
Ok::<(), ProtonError>(())
})
.map_err(|e| CoreError::from_api(&e, "delete"))?;
if deleted == 0
&& let Some(error) = first_error
{
return Err(CoreError::from_api(&error, "delete"));
}
self.invalidate_trash();
Ok(deleted)
}
/// Permanently delete everything in the trash. The uids are listed first so
/// the blobs of items trashed by *another* client — which this daemon may
/// still hold in its cache — are reclaimed too, not just the ones it trashed.
fn empty_trash(&self) -> CoreResult<usize> {
let uids = self
.rt
.block_on(self.drive.enumerate_trash_node_uids())
.map_err(|e| CoreError::from_api(&e, "enumerate trash"))?;
self.rt
.block_on(self.client.empty_trash())
.map_err(|e| CoreError::from_api(&e, "empty trash"))?;
self.drop_local(&uids);
self.invalidate_trash();
Ok(uids.len())
}
/// Forget every trace of nodes that no longer exist anywhere: their inode and
/// DB row, and their cached content.
fn drop_local(&self, uids: &[NodeUid]) {
// These nodes are gone from the server, and a sync-folder fork showing
// the same uid would otherwise keep serving them (`docs/BUGS.md` B74).
self.for_each_state(|st| {
for uid in uids {
st.forget(uid);
}
});
for uid in uids {
self.cache.evict(uid);
self.evict_reader(uid);
}
}
/// Create a folder named `name` under the mountpoint-relative `parent_rel`.
/// Interns the new node directly so it shows up without a re-enumeration.
fn create_folder(&self, parent_rel: &Path, name: &str) -> CoreResult<String> {
if name.is_empty() || name.contains('/') {
return Err(CoreError::invalid(format!("invalid name: {name:?}")));
}
let (pino, parent_uid) = self.resolve(parent_rel)?;
self.ensure_children(pino)
.map_err(|e| self.errno_error(e, "enumerate"))?;
self.require_uid_writable(&parent_uid)
.map_err(|error| self.errno_error(error, "create folder access"))?;
let new_uid = self
.rt
.block_on(
self.drive
.create_folder(&parent_uid, name, Some(now_secs())),
)
.map_err(|e| CoreError::from_api(&e, "create folder"))?;
let node = self
.fetch_minted_node(&new_uid, name)
.ok_or_else(|| minted_but_unlisted(name))?;
let mut st = self.state();
let ino = st.intern(pino, node);
if let Some(kids) = st.children.get_mut(&pino)
&& !kids.contains(&ino)
{
kids.push(ino);
}
drop(st);
self.flush_access_changes();
Ok(name.to_string())
}
// ---- activity log -----------------------------------------------------
/// Append one entry to the activity log. Callable from any thread (the sync
/// engine and the bulk uploader both log from background tasks). A failed
/// write is logged and dropped: the feed is a record of work, never a reason
/// to fail the work itself.
pub(crate) fn log_activity(
&self,
kind: ActivityKind,
target: impl Into<String>,
// `Display` rather than `Into<String>` so a classified `CoreError` can be
// logged as-is, without the caller flattening it first.
detail: impl std::fmt::Display,
ok: bool,
) {
let entry = ActivityEntry {
time: now_secs(),
kind,
target: target.into(),
detail: detail.to_string(),
ok,
};
if let Err(e) = self.db.activity_add(&entry) {
warn!(error = ?e, "could not record activity");
return;
}
// A sync pass is how a conflict copy reaches the listing, and the sweep
// reports the ones it finds as conflicts.
if matches!(entry.kind, ActivityKind::Conflict | ActivityKind::Sync) {
self.events.publish(&[Topic::Activity, Topic::Conflicts]);
} else {
self.events.publish(&[Topic::Activity]);
}
}
/// The recent activity, newest first, capped at `limit` entries.
fn list_activity(&self, limit: usize) -> Vec<ActivityEntry> {
match self.db.activity_list(limit) {
Ok(items) => items,
Err(e) => {
warn!(error = ?e, "could not read activity");
Vec::new()
}
}
}
// ---- live sync progress -----------------------------------------------
/// Start tracking a reconcile pass over `folder_id`, in [`SyncPhase::Scanning`].
pub(crate) fn progress_begin(&self, folder_id: i64) {
self.sync_progress.lock().insert(
folder_id,
SyncProgress {
phase: SyncPhase::Scanning,
done: 0,
total: 0,
current: String::new(),
},
);
}
/// Apply `f` to a folder's live progress, if a pass is running for it.
fn progress_update(&self, folder_id: i64, f: impl FnOnce(&mut SyncProgress)) {
if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) {
f(p);
}
self.sync_engine.moved();
}
/// Set the number of items the scan expects to check, from the size of the
/// last pass's baseline. Only an estimate — the folder may have grown since —
/// but it turns the scan from an indeterminate pulse into a bar that moves,
/// which is the difference between "it's stuck" and "it's working" on a folder
/// whose walk takes minutes.
pub(crate) fn progress_scan_total(&self, folder_id: i64, n: usize) {
self.progress_update(folder_id, |p| p.total = n);
}
/// Note that the scan has checked one more item, named `name`.
pub(crate) fn progress_scanned(&self, folder_id: i64, name: &str) {
self.progress_update(folder_id, |p| {
p.done += 1;
p.current = name.to_string();
});
}
/// Note that `n` more items have been queued for this pass, and that it has
/// moved on from scanning to applying the diff. The scan's counts are dropped:
/// they measured a different quantity (items checked, not items to apply), so
/// carrying them over would start the applying bar at a meaningless fraction.
pub(crate) fn progress_queued(&self, folder_id: i64, n: usize) {
self.progress_update(folder_id, |p| {
if p.phase == SyncPhase::Scanning {
p.phase = SyncPhase::Applying;
p.done = 0;
p.total = 0;
p.current.clear();
}
p.total += n;
});
}
/// Note that work has started on `name` (shown as the pass's current item).
pub(crate) fn progress_started(&self, folder_id: i64, name: &str) {
self.progress_update(folder_id, |p| p.current = name.to_string());
}
/// Note that one queued item finished, whether it succeeded or not.
pub(crate) fn progress_finished(&self, folder_id: i64) {
self.progress_update(folder_id, |p| {
p.done += 1;
p.current.clear();
});
}
/// Stop tracking a pass — no progress is reported for the folder until the
/// next [`progress_begin`](Self::progress_begin).
pub(crate) fn progress_end(&self, folder_id: i64) {
self.sync_progress.lock().remove(&folder_id);
}
}
fn publish_virtual_root_in_listing(
db: &Db,
state: &mut State,
parent_ino: u64,
snapshot: &RootListingSnapshot,
plan: VirtualRootPlan,
) -> Result<(), Errno> {
if !snapshot.is_current(state, parent_ino) {
return Err(Errno::EAGAIN);
}
db.publish_virtual_root(SHARED_WITH_ME_NAME, &plan.node)
.map_err(|error| {
error!(%error, "atomically publishing synthetic shared root failed");
Errno::EIO
})?;
let uid = plan.node.uid.clone();
state.share_access.insert(uid, Access::Viewer);
let virtual_ino = state.intern_mem(parent_ino, plan.node);
let published =
reconcile_virtual_root_in_listing(state, parent_ino, snapshot, virtual_ino, plan.visible);
debug_assert!(
published,
"snapshot cannot change while the state lock is held"
);
Ok(())
}
/// Update the synthetic dentry only while the parent listing is still known.
///
/// Event invalidation can remove the listing after its real names were captured.
/// Recreating it here would publish a synthetic-only partial snapshot as complete.
fn reconcile_virtual_root_in_listing(
state: &mut State,
parent_ino: u64,
snapshot: &RootListingSnapshot,
virtual_ino: u64,
visible: bool,
) -> bool {
if !snapshot.is_current(state, parent_ino) {
return false;
}
let Some(children) = state.children.get_mut(&parent_ino) else {
return false;
};
children.retain(|ino| *ino != virtual_ino);
if visible {
children.push(virtual_ino);
}
true
}
fn resolve_anywhere_with(
raw_uid: &str,
resident: impl FnOnce(&NodeUid) -> bool,
mirrored: impl FnOnce(&NodeUid) -> pdfs_core::Result<bool>,
) -> CoreResult<NodeUid> {
let uid =
parse_uid(raw_uid).ok_or_else(|| CoreError::invalid(format!("invalid uid: {raw_uid}")))?;
if is_local_uid(&uid) || is_virtual_uid(&uid) {
return Err(CoreError::invalid(format!("reserved uid: {raw_uid}")));
}
if resident(&uid) {
return Ok(uid);
}
if mirrored(&uid)? {
return Ok(uid);
}
Err(CoreError::not_found(format!(
"node is not present in any location: {raw_uid}"
)))
}
/// Map a [`MemberRole`] to its wire string.
fn role_to_str(role: MemberRole) -> &'static str {
match role {
MemberRole::Viewer => "viewer",
MemberRole::Editor => "editor",
MemberRole::Admin => "admin",
MemberRole::Inherited => "inherited",
}
}
/// Parse a wire role string into a [`MemberRole`]. "inherited" is read-only and
/// rejected here, since it cannot be sent when inviting or updating.
fn role_from_str(role: &str) -> CoreResult<MemberRole> {
match role.to_lowercase().as_str() {
"viewer" => Ok(MemberRole::Viewer),
"editor" => Ok(MemberRole::Editor),
"admin" => Ok(MemberRole::Admin),
other => Err(CoreError::invalid(format!("invalid role: {other}"))),
}
}
/// Map a device type to a display string.
fn device_type_str(t: proton_drive_rs::DeviceType) -> &'static str {
match t {
proton_drive_rs::DeviceType::Windows => "Windows",
proton_drive_rs::DeviceType::MacOs => "MacOs",
proton_drive_rs::DeviceType::Linux => "Linux",
}
}
/// Convert an SDK [`PublicLink`](proton_drive_rs::PublicLink) into the wire form.
fn public_link_info(link: proton_drive_rs::PublicLink) -> PublicLinkInfo {
PublicLinkInfo {
id: link.public_link_id,
url: link.url,
role: role_to_str(link.role).to_string(),
expires: link.expiration_time,
has_password: link.has_custom_password,
created: link.creation_time,
}
}
/// Parse a `volume~link` uid display string back into a [`NodeUid`]. Front-ends
/// receive uids as strings over the control socket and pass them back verbatim.
pub(crate) fn parse_uid(s: &str) -> Option<NodeUid> {
let (vol, link) = s.split_once('~')?;
if vol.is_empty() || link.is_empty() || link.contains('~') {
return None;
}
Some(NodeUid::new(VolumeId::from(vol), LinkId::from(link)))
}
/// Current wall-clock time as epoch seconds (0 if the clock is before the epoch).
fn now_secs() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
/// Current wall-clock time as epoch milliseconds, the resolution `pending_op`
/// timestamps and backoff deadlines are kept in.
fn now_millis() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0)
}
/// Parse a [`NodeUid`] back from its `Display` form (`volume~link`), which is
/// how one is persisted in `pending_op.uid` and a [`StagedWrite`] sidecar. The
/// SDK has no `FromStr`, and neither id contains a `~`.
fn parse_node_uid(s: &str) -> Option<NodeUid> {
parse_uid(s)
}
/// The uid a node made on this machine goes by until it lands, so it can be
/// interned, listed and written to before the server has ever heard of it:
/// its row's local id, as [`pdfs_core::db::local_uid`]
/// has it.
fn local_node_uid(lid: i64) -> NodeUid {
NodeUid::new(VolumeId::from(LOCAL_VOLUME), LinkId::from(lid.to_string()))
}
/// Whether this node exists only on this machine, so far. Such a uid is
/// meaningless to the API and must never be sent to it.
fn is_local_uid(uid: &NodeUid) -> bool {
uid.volume_id.as_str() == LOCAL_VOLUME
}
/// [`is_local_uid`] for a uid in its persisted `Display` form.
fn is_local_uid_str(s: &str) -> bool {
s.split_once('~')
.is_some_and(|(vol, _)| vol == LOCAL_VOLUME)
}
/// The API's response code for a failed call, when it failed *at* the API.
///
/// Takes `&dyn Error` so it reads a [`ProtonError`] equally well through the
/// boxes the drain deals in, where the concrete type survives but the static one
/// does not. `None` covers both "not an API error at all" (a transport failure,
/// which is what being offline looks like) and "not a `ProtonError`".
fn api_code(e: &(dyn std::error::Error + 'static)) -> Option<ResponseCode> {
match e.downcast_ref::<ProtonError>() {
Some(ProtonError::Api(api)) => Some(api.code),
_ => None,
}
}
/// How often [`Core::ensure_children`] takes a listing that changed while it
/// was being published.
const LISTING_ATTEMPTS: usize = 4;
/// How often a rename or move that Drive calls out of date is retried, and how
/// long apart.
const OUT_OF_DATE_RETRIES: u32 = 20;
const OUT_OF_DATE_RETRY_DELAY: Duration = Duration::from_millis(500);
/// Run a remote rename or move, retrying while Drive answers
/// `InvalidRequirements`.
///
/// Both send the node's current name hash. The SDK takes it from its entity
/// cache, which still holds the old hash when the node changed since it was
/// cached, and Drive then rejects the request as "out of date" (docs/BUGS.md
/// B111, B121). `wait` drops that entry before the next attempt. Any other
/// error returns at once.
fn retry_out_of_date(
mut call: impl FnMut() -> Result<(), ProtonError>,
mut wait: impl FnMut(),
) -> Result<(), ProtonError> {
let mut attempts = 0u32;
loop {
match call() {
Err(e)
if api_code(&e) == Some(ResponseCode::InvalidRequirements)
&& attempts < OUT_OF_DATE_RETRIES =>
{
attempts += 1;
debug!(attempts, error = %e, "Drive called the request out of date; retrying");
wait();
}
result => return result,
}
}
}
/// Whether a call failed because the name it asked for is already in use.
///
/// The queue makes this reachable in a way the synchronous path never was: a
/// mutation queued while offline is applied against a server that may have
/// gained a file of that name in the meantime.
fn is_already_exists(e: &(dyn std::error::Error + 'static)) -> bool {
api_code(e) == Some(ResponseCode::AlreadyExists)
}
/// Whether a call failed because the node it addressed is not there.
/// The error for a folder Drive created but did not hand back, where there is
/// no queue to adopt it later. Saying it was created keeps the caller from
/// taking a retry's "already exists" for someone else's folder.
fn minted_but_unlisted(name: &str) -> CoreError {
CoreError::remote(format!(
"created folder {name}, but Drive did not list it yet; it shows on the next refresh"
))
}
fn is_gone(e: &(dyn std::error::Error + 'static)) -> bool {
api_code(e) == Some(ResponseCode::DoesNotExist)
}
/// A variant of `name` to fall back on when the remote already has that name.
///
/// Deliberately the same shape the sync engine uses for its conflict copies
/// (`sync.rs`, `conflict_path`), so the two halves of the product name the same
/// situation the same way and a user only has to learn it once.
fn conflict_name(name: &str, stamp: i64) -> String {
conflict_name_as(name, conflict_label(), stamp)
}
/// [`conflict_name`] with the machine's label given.
fn conflict_name_as(name: &str, label: &str, stamp: i64) -> String {
let path = Path::new(name);
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(name);
let tag = conflict_tag(label, stamp, 0);
match path.extension().and_then(|s| s.to_str()) {
Some(ext) => format!("{stem} (sync-conflict {tag}).{ext}"),
None => format!("{stem} (sync-conflict {tag})"),
}
}
/// This machine's name as its conflict copies carry it (`docs/BUGS.md` B251).
///
/// Two computers syncing one folder that each keep a copy of the same file in
/// the same second would otherwise make the same name on Drive, and nobody
/// could tell which computer a copy came from. Read once, from the hostname.
fn conflict_label() -> &'static str {
static LABEL: OnceLock<String> = OnceLock::new();
LABEL.get_or_init(|| conflict_label_of(&this_hostname()))
}
/// `host` as one token: anything but ASCII letters, digits, `-`, `_` and `.`
/// becomes `-`, so the label holds no space for `conflict_base_name` to trip
/// on and no `/` or `)` to break the name.
fn conflict_label_of(host: &str) -> String {
let label: String = host
.chars()
.take(32)
.map(|c| {
if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') {
c
} else {
'-'
}
})
.collect();
label.trim_matches('-').to_string()
}
/// The inside of a conflict copy's marker: `<label> <stamp>`, with `-<n>`
/// when an earlier copy already holds that name. Without a label it is the
/// bare stamp that copies had before B251.
fn conflict_tag(label: &str, stamp: i64, suffix: u32) -> String {
let suffix = if suffix == 0 {
String::new()
} else {
format!("-{suffix}")
};
if label.is_empty() {
format!("{stamp}{suffix}")
} else {
format!("{label} {stamp}{suffix}")
}
}
/// Fabricate the node the server would have returned, for a `create`/`mkdir`
/// that could not reach it. Everything the kernel asks about a fresh node —
/// name, kind, size, times — is knowable locally; the uid is the only invention,
/// and the drain replaces it with the real one.
fn local_node(uid: NodeUid, parent_uid: NodeUid, name: String, is_dir: bool) -> Node {
let now = now_secs();
Node {
uid,
parent_uid: Some(parent_uid),
kind: if is_dir {
NodeKind::Folder
} else {
NodeKind::File {
media_type: media_type_for(&name).to_string(),
total_size_on_storage: 0,
// No revision has been sealed: nothing has been uploaded yet.
active_revision_state: None,
active_revision_id: None,
content_sha1: None,
claimed_size: Some(0),
claimed_modification_time: None,
}
},
name,
creation_time: now,
modification_time: now,
trashed: false,
is_shared: false,
is_shared_publicly: false,
signature_email: None,
membership: None,
// A locally fabricated node is on the main volume, never a photo/album.
photo: None,
album: None,
// Nothing signed it: it has never been near the crypto layer.
verification: Default::default(),
direct_role: None,
share_id: None,
}
}
/// This machine's hostname, used to name (and later recover) its Proton Drive
/// Device. Reads the live kernel hostname, falling back to a generic label.
fn this_hostname() -> String {
std::fs::read_to_string("/proc/sys/kernel/hostname")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "Linux device".to_string())
}
/// Whether `dir` has no entries.
fn dir_is_empty(dir: &Path) -> std::io::Result<bool> {
Ok(std::fs::read_dir(dir)?.next().is_none())
}
/// Delete everything inside `dir` but keep `dir` itself (it stays as the FUSE
/// mountpoint). Used when a `mirror` folder flips to `ondemand`: the local files
/// are the disk we're reclaiming.
fn evict_dir_contents(dir: &Path) -> std::io::Result<()> {
for entry in std::fs::read_dir(dir)? {
let path = entry?.path();
if path.is_dir() && !path.is_symlink() {
std::fs::remove_dir_all(&path)?;
} else {
std::fs::remove_file(&path)?;
}
}
Ok(())
}
/// Convert a stored synced folder into its wire form for the front-ends, with
/// the live progress of its pass when one is running.
fn sync_folder_info(
f: StoredSyncFolder,
progress: Option<SyncProgress>,
paused: bool,
) -> SyncFolderInfo {
SyncFolderInfo {
id: f.id,
local_path: f.local_path,
remote_uid: f.remote_uid,
mode: f.mode,
pending_mode: f.pending_mode,
state: f.state,
last_sync: f.last_sync,
progress,
paused,
}
}
/// A coarse MIME type guessed from a file name's extension; Proton stores this
/// on the node but an exact value is not required for correctness.
fn media_type_for(name: &str) -> &'static str {
let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase());
match ext.as_deref() {
Some("txt" | "md" | "log") => "text/plain",
Some("png") => "image/png",
Some("jpg" | "jpeg") => "image/jpeg",
Some("gif") => "image/gif",
Some("pdf") => "application/pdf",
Some("json") => "application/json",
Some("html" | "htm") => "text/html",
_ => "application/octet-stream",
}
}
/// The plaintext size, in bytes, that a node reports.
/// Overwrite each file node's `claimed_size` with the optimistic size of its
/// queued write, where it has one. Folders and nodes with nothing queued are
/// left alone. See [`Core::stamp_pending_sizes`] for why this exists.
fn apply_pending_sizes(nodes: &mut [Node], sizes: &HashMap<NodeUid, u64>) {
for node in nodes {
if let Some(&len) = sizes.get(&node.uid)
&& let NodeKind::File { claimed_size, .. } = &mut node.kind
{
*claimed_size = Some(len as i64);
}
}
}
/// A size upgrade in flight for one folder, so the `getattr`s that need its
/// result wait for the one batch instead of each fetching its own node.
///
/// A plain `Condvar` rather than a channel: the waiters do not want a value,
/// only the edge, and there may be hundreds of them for one folder.
#[derive(Default)]
struct Progress {
/// The whole batch has been applied (or failed). Nothing more is coming.
done: bool,
/// Bumped every time a chunk lands, so a waiter can tell "no progress since
/// I last looked" from "progress happened while I was looking".
generation: u64,
}
/// A size upgrade in flight for one folder, so the `getattr`s and `lookup`s that
/// need its result wait for the one batch instead of each fetching its own node.
///
/// Waiters are released **per chunk**, not once at the end: a waiter only cares
/// about its own file, and waiting for the other 792 is what let a large folder
/// outrun the timeout (`docs/BUGS.md` B14).
#[derive(Default)]
struct SizeUpgrade {
inner: Mutex<Progress>,
ready: Condvar,
/// The batch, while it waits for a thread because [`MAX_SIZE_UPGRADES`]
/// were already running, with when it was queued so the oldest goes first.
/// `None` once a thread has it.
queued: Mutex<Option<(Instant, Vec<NodeUid>)>>,
}
/// What [`Core::upgrade_sizes`] does with one folder's batch.
enum SizeUpgradeClaim {
/// A batch for this folder is already running or queued; wait on it.
Joined(Arc<SizeUpgrade>),
/// The cap is reached. The batch waits for a running thread to take it.
Queued(Arc<SizeUpgrade>),
/// Start a thread for the batch.
Start(Arc<SizeUpgrade>, Vec<NodeUid>),
}
/// Register the batch for `key` in `in_flight` and decide who runs it.
///
/// Past the cap this used to return nothing, and the caller answered with the
/// provisional size at once. Nothing retried the folder until the next `stat`,
/// so a recursive walk over more than [`MAX_SIZE_UPGRADES`] fresh folders showed
/// ciphertext sizes for most of them (`docs/BUGS.md` B100). Queued, the batch is still
/// fetched, and a waiter gets the real size unless [`SizeUpgrade::WAIT`] runs out.
fn claim_size_upgrade(
in_flight: &mut HashMap<u64, Arc<SizeUpgrade>>,
key: u64,
uids: Vec<NodeUid>,
) -> SizeUpgradeClaim {
if let Some(existing) = in_flight.get(&key) {
return SizeUpgradeClaim::Joined(existing.clone());
}
let running = in_flight
.values()
.filter(|slot| slot.queued.lock().is_none())
.count();
let slot = Arc::new(SizeUpgrade::default());
in_flight.insert(key, slot.clone());
if running >= MAX_SIZE_UPGRADES {
*slot.queued.lock() = Some((Instant::now(), uids));
return SizeUpgradeClaim::Queued(slot);
}
SizeUpgradeClaim::Start(slot, uids)
}
/// Retire the finished batch for `done` and hand its thread the oldest queued
/// batch, if there is one.
fn next_size_upgrade(
in_flight: &mut HashMap<u64, Arc<SizeUpgrade>>,
done: u64,
) -> Option<(u64, Vec<NodeUid>, Arc<SizeUpgrade>)> {
in_flight.remove(&done);
let (key, slot) = in_flight
.iter()
.filter_map(|(key, slot)| slot.queued.lock().as_ref().map(|(at, _)| (*at, *key, slot)))
.min_by_key(|(at, _, _)| *at)
.map(|(_, key, slot)| (key, slot.clone()))?;
let (_, uids) = slot.queued.lock().take()?;
Some((key, uids, slot))
}
impl SizeUpgrade {
/// How long a caller will wait for a real size before answering with the
/// provisional one.
///
/// A `stat` that never returns is far worse than one that is briefly wrong:
/// on timeout the caller falls back to the pre-fix behaviour rather than
/// wedging whatever is listing the directory.
const WAIT: Duration = Duration::from_secs(10);
/// Whether the batch has ended, however it ended. A waiter that sees this
/// has nothing left to wait for.
fn is_finished(&self) -> bool {
self.inner.lock().done
}
/// Announce that a chunk has been applied. Every waiter re-checks its own
/// node; the ones it resolved return, the rest go back to sleep.
fn chunk_done(&self) {
self.inner.lock().generation += 1;
self.ready.notify_all();
}
/// Block until the batch has ended or `deadline` has passed. Only for a
/// caller that is not on a FUSE or [`Workers`] thread; those park their
/// reply with [`Core::await_size`] instead.
fn wait_finished(&self, deadline: Instant) {
let mut progress = self.inner.lock();
while !progress.done {
if self.ready.wait_until(&mut progress, deadline).timed_out() {
return;
}
}
}
/// Release every waiter for good. The worker must reach this on all paths.
fn finish(&self) {
let mut progress = self.inner.lock();
progress.done = true;
progress.generation += 1;
drop(progress);
self.ready.notify_all();
}
}
/// How often [`SizeWaitQueue::serve`] re-tests its parked waiters.
///
/// The predicates read `state`, which the applying thread writes under its own
/// lock, so this polls rather than waiting on each slot's condvar: one thread
/// cannot wait on a hundred condvars, and the whole point of the queue is that
/// there is only one thread. Twenty milliseconds is far below the round trip a
/// waiter is waiting on and only ticks while something is actually parked.
const SIZE_WAIT_POLL: Duration = Duration::from_millis(20);
/// A reply held back until a size upgrade resolves the node it describes.
struct SizeWaiter {
/// The batch this reply is waiting on. When it ends, so does the wait.
slot: Arc<SizeUpgrade>,
/// Whether *this* waiter's node now has a real size.
///
/// Called with no queue lock held: it reaches into `state`, and the thread
/// applying a chunk holds `state` before it announces the chunk — taking
/// them in the other order would close the cycle.
resolved: Box<dyn Fn() -> bool + Send>,
/// When to answer with the provisional size anyway. A `stat` that never
/// returns is far worse than one that is briefly wrong.
deadline: Instant,
/// Sends the FUSE reply. Runs on the serving thread.
respond: Box<dyn FnOnce() + Send>,
}
#[derive(Default)]
struct Parked {
waiters: Vec<SizeWaiter>,
/// Whether a thread is currently serving `waiters`. The thread clears this
/// as it retires, so the next park starts a new one.
running: bool,
}
/// Every reply parked on a size upgrade, served by at most one thread.
///
/// This exists because waiting used to happen on the [`Workers`] pool. Sizes
/// are single-flighted per folder, so an `ls -l` across N cold folders parks a
/// waiter per entry — and each of those held a pool thread asleep on a condvar
/// for up to [`SizeUpgrade::WAIT`]. With enough folders that is every thread in
/// both lanes, including the ones reserved for metadata precisely so that this
/// could not happen (audit F4). Parking the `Reply` instead costs one thread
/// for the whole mount, and only while something is parked.
#[derive(Default)]
struct SizeWaitQueue {
parked: Mutex<Parked>,
/// Woken when a waiter is added, so a serving thread re-computes how long
/// it may sleep.
added: Condvar,
}
impl SizeWaitQueue {
/// Park `waiter`. Returns whether the caller must start a serving thread.
fn park(&self, waiter: SizeWaiter) -> bool {
let mut parked = self.parked.lock();
parked.waiters.push(waiter);
if parked.running {
drop(parked);
self.added.notify_all();
return false;
}
parked.running = true;
true
}
/// Answer every waiter whose node has resolved, whose batch has ended, or
/// whose deadline has passed; sleep until the next one might. Returns when
/// nothing is parked any more.
fn serve(&self) {
loop {
// Taken out of the lock wholesale: the predicates and the replies
// both take locks of their own, and none of them may be called
// while this one is held.
let taken = std::mem::take(&mut self.parked.lock().waiters);
let now = Instant::now();
let mut keep = Vec::with_capacity(taken.len());
for waiter in taken {
if now >= waiter.deadline || waiter.slot.is_finished() || (waiter.resolved)() {
(waiter.respond)();
} else {
keep.push(waiter);
}
}
let mut parked = self.parked.lock();
parked.waiters.append(&mut keep);
if parked.waiters.is_empty() {
parked.running = false;
return;
}
let until = parked
.waiters
.iter()
.map(|w| w.deadline)
.min()
.unwrap_or(now)
.min(Instant::now() + SIZE_WAIT_POLL);
self.added.wait_until(&mut parked, until);
}
}
/// Answer everything parked, immediately, and retire the serving thread.
/// Used when there is no thread to serve with: a provisional size now beats
/// a reply that never comes.
fn answer_all(&self) {
let taken = {
let mut parked = self.parked.lock();
parked.running = false;
std::mem::take(&mut parked.waiters)
};
for waiter in taken {
(waiter.respond)();
}
}
}
/// Whether a `rename` may replace an existing destination, per POSIX.
///
/// Split out from the handler because it is the part that is pure and the part
/// that is dangerous: every `Err` here is a refusal that happens *before*
/// anything is trashed, and getting one wrong turns a refusal into the
/// destruction of the destination (`docs/BUGS.md` B13).
///
/// `dst_empty` is only meaningful when `dst_dir`; pass `true` otherwise.
fn check_replaceable(src_dir: bool, dst_dir: bool, dst_empty: bool) -> Result<(), Errno> {
match (src_dir, dst_dir) {
// A non-directory may not replace a directory, or vice versa.
(false, true) => Err(Errno::EISDIR),
(true, false) => Err(Errno::ENOTDIR),
// Proton trashes a folder with its whole subtree, so replacing a
// non-empty one would silently take its contents with it. POSIX says
// ENOTEMPTY, which is also the safe answer.
(true, true) if !dst_empty => Err(Errno::ENOTEMPTY),
_ => Ok(()),
}
}
/// Whether `node` is a file Drive lists without an active revision, as it does
/// one whose create has not been committed.
fn is_unrevised(node: &Node) -> bool {
matches!(
node.kind,
NodeKind::File {
active_revision_id: None,
..
}
)
}
fn node_size(node: &Node) -> u64 {
match &node.kind {
NodeKind::Folder => 0,
NodeKind::File {
claimed_size,
total_size_on_storage,
..
} => claimed_size.unwrap_or(*total_size_on_storage).max(0) as u64,
}
}
/// `call`, failed with a remote error once it has run for
/// [`TRASH_CALL_TIMEOUT`].
async fn within_trash_call<T>(what: &str, call: impl Future<Output = T>) -> CoreResult<T> {
tokio::time::timeout(TRASH_CALL_TIMEOUT, call)
.await
.map_err(|_| {
CoreError::new(
ErrorKind::Remote,
format!("{what}: no answer within {}s", TRASH_CALL_TIMEOUT.as_secs()),
)
})
}
/// When a node a trash refresh found went to the trash, as far as the daemon
/// can tell. A node the stored listing already had keeps the time it had. A new
/// one went there since the last refresh, so now is close, but only when that
/// refresh saw the whole trash (`complete`): after a first or interrupted one,
/// a node missing from the listing may be years old, and no date beats a wrong
/// one.
fn first_seen_in_trash(
seen: &HashMap<String, Option<i64>>,
uid: &str,
complete: bool,
now: i64,
) -> Option<i64> {
match seen.get(uid) {
Some(at) => *at,
None if complete => Some(now),
None => None,
}
}
/// Expand a restore request across the trashed tree, and order it so a parent is
/// always restored before its children.
///
/// `relations` is the persisted trash listing as `(uid, parent_uid)` pairs; a
/// parent that is not itself in the listing is a live folder, which ends the
/// walk upward. The rule is:
///
/// - every trashed descendant of a requested node joins the restore, so a folder
/// comes back with the items that were trashed inside it;
/// - every trashed ancestor of a requested node joins it too, so a file never
/// lands in a folder that is still in the trash;
/// - a requested uid the listing does not know about is restored on its own,
/// because a stale or half-materialised listing must not silently drop it.
///
/// The result is one wave per depth, shallowest first. Waves are disjoint and
/// together contain each uid exactly once.
fn expand_restore(
requested: &[String],
relations: &[(String, Option<String>)],
) -> Vec<Vec<String>> {
let parents: HashMap<&str, Option<&str>> = relations
.iter()
.map(|(uid, parent)| (uid.as_str(), parent.as_deref()))
.collect();
let mut children: HashMap<&str, Vec<&str>> = HashMap::new();
for (uid, parent) in relations {
// Only a trashed parent: a live one keeps its child as a root here.
if let Some(parent) = parent.as_deref()
&& parents.contains_key(parent)
{
children.entry(parent).or_default().push(uid.as_str());
}
}
let mut selected: HashSet<&str> = HashSet::new();
for uid in requested {
let uid = uid.as_str();
if !parents.contains_key(uid) {
selected.insert(uid);
continue;
}
// Upward: every trashed ancestor, so the destination folder exists again.
let mut cursor = Some(uid);
while let Some(current) = cursor {
// A parent outside the listing is live: the walk stops under it.
if !parents.contains_key(current) || !selected.insert(current) {
break;
}
cursor = parents.get(current).copied().flatten();
}
// Downward: the whole trashed subtree, so the folder comes back full.
let mut stack = vec![uid];
while let Some(current) = stack.pop() {
for child in children.get(current).into_iter().flatten() {
if selected.insert(child) {
stack.push(child);
}
}
}
}
// Depth within the *trash*, not within the drive: a node whose parent is live
// (or unknown) is at depth 0 and can be restored straight away.
let mut waves: Vec<Vec<String>> = Vec::new();
for uid in &selected {
let mut depth = 0usize;
let mut cursor = parents.get(uid).copied().flatten();
// Only trashed ancestors count, and the walk is bounded by the listing's
// size: server data that somehow describes a cycle must not spin here.
while let Some(current) = cursor
&& parents.contains_key(current)
&& depth < parents.len()
{
depth += 1;
cursor = parents.get(current).copied().flatten();
}
if waves.len() <= depth {
waves.resize(depth + 1, Vec::new());
}
waves[depth].push((*uid).to_string());
}
// Stable output, so a restore of the same trash is the same sequence of calls
// every time — the batches are what shows up in the log and in the tests.
for wave in &mut waves {
wave.sort();
}
waves.retain(|wave| !wave.is_empty());
waves
}
/// The server revision id of a node's active revision, if it is a file that has
/// one. The stable identity the drain conflict-checks against (see [`Baseline`]).
fn node_revision_id(node: &Node) -> Option<String> {
match &node.kind {
NodeKind::Folder => None,
NodeKind::File {
active_revision_id, ..
} => active_revision_id.clone(),
}
}
/// The plaintext content SHA-1 of a file node, if its active revision carried one.
/// A download-free content fingerprint the conflict sweep uses to prove two files
/// hold identical bytes before it removes one.
fn node_content_sha1(node: &Node) -> Option<String> {
match &node.kind {
NodeKind::Folder => None,
NodeKind::File { content_sha1, .. } => content_sha1.clone(),
}
}
/// `"1 file"` / `"3 files"` — a count with a correctly pluralised noun, for
/// human-readable activity-log lines.
fn count_noun(n: usize, one: &str, many: &str) -> String {
format!("{n} {}", if n == 1 { one } else { many })
}
/// Seed a truncate scratch file from a complete queued revision.
///
/// `File` writes use an explicit offset so this is independent of either
/// descriptor's cursor and remains correct when the destination came from the
/// cache's scratch allocator.
fn copy_pending_for_truncate(pending: &PendingRevision, destination: &File) -> std::io::Result<()> {
let source = File::open(&pending.path)?;
destination.set_len(0)?;
let mut offset = 0u64;
let mut buffer = vec![0u8; 1024 * 1024];
loop {
let count = source.read_at(&mut buffer, offset)?;
if count == 0 {
break;
}
destination.write_all_at(&buffer[..count], offset)?;
offset += count as u64;
}
destination.set_len(offset)
}
/// Whether a rename must be represented as a durable desired end state.
/// Local-first, offline, or into a not-yet-uploaded directory, the API is not
/// called in the syscall. Only remote-to-remote renames sent inline
/// ([`Core::sends_inline`]) stay synchronous, so a successful kernel reply means
/// the remote namespace has already reached the same end state.
fn rename_needs_queue(
inline: bool,
destination_is_local: bool,
_parent_changed: bool,
_name_changed: bool,
) -> bool {
!inline || destination_is_local
}
/// Convert one kernel pathname component into the UTF-8 name accepted by Drive.
/// Linux filesystems conventionally cap a component at `NAME_MAX` (255 bytes),
/// but the FUSE kernel path can pass a longer component through to userspace.
fn fuse_name(name: &OsStr) -> Result<String, Errno> {
let bytes = name.as_bytes();
if bytes.len() > 255 {
return Err(Errno::ENAMETOOLONG);
}
if bytes.is_empty() || bytes == b"." || bytes == b".." {
return Err(Errno::EINVAL);
}
name.to_str().map(str::to_owned).ok_or(Errno::EILSEQ)
}
/// Whether a decrypted remote name can stand as one local path component.
///
/// Drive checks names only when a client creates or renames a node, and only
/// for length, so a name made by another client, or by a server that should
/// not be trusted, can hold anything. Joined onto a local path, a `/` or a `..`
/// in it walks out of the folder it belongs to, and the mirror engine would
/// then write or delete outside the sync root. In the mount, the kernel fails
/// the whole listing with EIO on a single such name (`docs/BUGS.md` B215).
pub(crate) fn remote_name_is_safe(name: &str) -> bool {
!name.is_empty()
&& name != "."
&& name != ".."
&& name.len() <= 255
&& !name.contains(['/', '\0'])
}
fn node_visible(node: &Node, folder_uid: &NodeUid, hidden: &HashSet<NodeUid>) -> bool {
if !remote_name_is_safe(&node.name) {
warn!(uid = %node.uid, "a remote name cannot be a local file name; leaving it out");
return false;
}
!node.trashed && node.uid != *folder_uid && !hidden.contains(&node.uid)
}
/// The children `entries` hold for each folder in `listed`, gathered in one
/// pass over them.
///
/// The root is its own parent (that is what `..` resolves to), so it would
/// count as its own child; a directory listed inside itself makes the kernel
/// fail the lookup with EIO, taking the whole listing down with it.
///
/// A pass per listed folder took 1.7 s on a tree of 37,000 nodes with 4,600 of
/// them listed, at every start before the mount came up (`docs/BUGS.md` B167).
fn listed_children(entries: &HashMap<u64, Entry>, listed: Vec<u64>) -> HashMap<u64, Vec<u64>> {
let mut children: HashMap<u64, Vec<u64>> =
listed.into_iter().map(|ino| (ino, Vec::new())).collect();
for (&ino, entry) in entries {
if ino != entry.parent
&& !entry.node.trashed
&& let Some(kids) = children.get_mut(&entry.parent)
{
kids.push(ino);
}
}
children
}
/// Lay what is queued for a folder over Drive's listing of it, from
/// `Db::queued_children`: a node a queued rename moved away is dropped, and a
/// node a queued create, mkdir or rename put there is listed as the DB has it.
///
/// That node holds its name here: a create Drive already has but whose answer
/// is still on its way, or which the drain is still reading back, is listed by
/// Drive under its new uid and by the queue under its stand-in. Both were
/// listed, and a 128-file folder listed 129 (`docs/BUGS.md` B164).
fn overlay_queued(nodes: &mut Vec<Node>, queued: Vec<Node>, gone: &HashSet<String>) {
let here: HashSet<String> = queued.iter().map(|node| node.uid.to_string()).collect();
let names: HashSet<&str> = queued.iter().map(|node| node.name.as_str()).collect();
nodes.retain(|node| {
let uid = node.uid.to_string();
!gone.contains(&uid) && !here.contains(&uid) && !names.contains(node.name.as_str())
});
nodes.extend(queued);
}
/// Bytes rendered with a binary unit and one decimal place (e.g. `"1.2 GB"`),
/// for the activity log. Uses 1024-based steps but the shorter SI labels.
fn human_bytes(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"];
let mut val = bytes as f64;
let mut unit = 0;
while val >= 1024.0 && unit < UNITS.len() - 1 {
val /= 1024.0;
unit += 1;
}
if unit == 0 {
format!("{bytes} B")
} else {
format!("{val:.1} {}", UNITS[unit])
}
}
/// A compact elapsed-time label for the activity log: `"820ms"`, `"43s"`, or
/// `"2m 5s"`.
fn human_duration(d: Duration) -> String {
let secs = d.as_secs();
if secs == 0 {
format!("{}ms", d.as_millis())
} else if secs < 60 {
format!("{secs}s")
} else {
format!("{}m {}s", secs / 60, secs % 60)
}
}
/// Wall clock in milliseconds since the epoch — the unit the persisted listings
/// stamp their freshness in.
fn now_ms() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0)
}
fn unix_secs(secs: i64) -> SystemTime {
if secs >= 0 {
UNIX_EPOCH + Duration::from_secs(secs as u64)
} else {
UNIX_EPOCH - Duration::from_secs(secs.unsigned_abs())
}
}
#[cfg(test)]
mod merge_over_pending_tests {
use super::{PendingRevision, merge_over_pending};
use pdfs_core::cache::{Baseline, StagedWrite};
use std::io::Write as _;
use std::path::PathBuf;
/// A unique temp directory removed on drop; avoids a dev-dependency, as in
/// `state.rs`.
struct TempDir(PathBuf);
impl TempDir {
fn new() -> Self {
use std::sync::atomic::{AtomicU64, Ordering};
static N: AtomicU64 = AtomicU64::new(0);
let p = std::env::temp_dir().join(format!(
"pdfs-merge-{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&p).unwrap();
TempDir(p)
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn staged(len: u64, base_size: u64, authored: &[(u64, u64)]) -> StagedWrite {
StagedWrite {
uid: "vol~link".into(),
len,
base_size,
base_mtime: 100,
complete: StagedWrite::covers(authored, len),
authored: authored.to_vec(),
based_on: Some(Baseline {
mtime: 100,
size: base_size,
hash: None,
revision_id: Some("r1".into()),
}),
}
}
/// Write a file of `len` bytes with `fill` at every offset in `ranges`.
fn blob(
dir: &std::path::Path,
name: &str,
len: u64,
ranges: &[(u64, u64)],
fill: u8,
) -> std::path::PathBuf {
let path = dir.join(name);
let mut bytes = vec![0u8; len as usize];
for &(s, e) in ranges {
bytes[s as usize..e as usize].fill(fill);
}
let mut f = std::fs::File::create(&path).unwrap();
f.write_all(&bytes).unwrap();
path
}
/// The case the merge exists for: two small edits to different parts of a
/// file, the second closing before the first has drained. The second blob
/// must come out holding both, or the first edit is silently lost.
#[test]
fn the_superseding_write_inherits_the_earlier_ones_bytes() {
let dir = TempDir::new();
let previous = PendingRevision {
path: blob(&dir.0, "old", 64, &[(0, 16)], 0xAA),
meta: staged(64, 64, &[(0, 16)]),
};
let src = blob(&dir.0, "new", 64, &[(32, 48)], 0xBB);
let mut meta = staged(64, 64, &[(32, 48)]);
merge_over_pending(&mut meta, &src, &previous).unwrap();
assert_eq!(meta.authored, vec![(0, 16), (32, 48)]);
assert!(
!meta.complete,
"the untouched middle still comes from the base"
);
let merged = std::fs::read(&src).unwrap();
assert!(
merged[0..16].iter().all(|&b| b == 0xAA),
"earlier edit kept"
);
assert!(merged[32..48].iter().all(|&b| b == 0xBB), "newer edit kept");
}
/// Where they overlap, the newer write wins — it is the later state of the
/// file, and the older bytes were never visible to anyone after it landed.
#[test]
fn the_newer_write_wins_the_ranges_it_authored() {
let dir = TempDir::new();
let previous = PendingRevision {
path: blob(&dir.0, "old", 32, &[(0, 32)], 0xAA),
meta: staged(32, 32, &[(0, 32)]),
};
let src = blob(&dir.0, "new", 32, &[(8, 16)], 0xBB);
let mut meta = staged(32, 32, &[(8, 16)]);
merge_over_pending(&mut meta, &src, &previous).unwrap();
assert_eq!(meta.authored, vec![(0, 32)]);
assert!(meta.complete, "between them they cover the file");
let merged = std::fs::read(&src).unwrap();
assert!(
merged[8..16].iter().all(|&b| b == 0xBB),
"newer edit survives"
);
assert!(merged[16..32].iter().all(|&b| b == 0xAA));
}
/// The baseline has to come from the write being superseded. This one's own
/// "base" is the earlier *staged blob*, whose size and mtime are ours, not
/// the server's — filling or conflict-checking against that would compare
/// the remote to a revision it never had.
#[test]
fn the_baseline_is_inherited_from_the_write_being_superseded() {
let dir = TempDir::new();
let mut earlier = staged(64, 64, &[(0, 16)]);
earlier.base_mtime = 42;
earlier.base_size = 50;
earlier.based_on = Some(Baseline {
mtime: 42,
size: 50,
hash: None,
revision_id: Some("original".into()),
});
let previous = PendingRevision {
path: blob(&dir.0, "old", 64, &[(0, 16)], 0xAA),
meta: earlier,
};
let src = blob(&dir.0, "new", 64, &[(32, 48)], 0xBB);
// Opened over the staged blob, so this write thinks its base is 64/100.
let mut meta = staged(64, 64, &[(32, 48)]);
merge_over_pending(&mut meta, &src, &previous).unwrap();
assert_eq!(meta.base_mtime, 42);
assert_eq!(meta.base_size, 50);
assert_eq!(
meta.based_on.unwrap().revision_id.as_deref(),
Some("original")
);
}
/// A shrink must not drag the earlier write's tail back in.
#[test]
fn a_truncating_write_does_not_inherit_past_its_own_length() {
let dir = TempDir::new();
let previous = PendingRevision {
path: blob(&dir.0, "old", 64, &[(0, 64)], 0xAA),
meta: staged(64, 64, &[(0, 64)]),
};
let src = blob(&dir.0, "new", 16, &[(8, 16)], 0xBB);
let mut meta = staged(16, 64, &[(8, 16)]);
merge_over_pending(&mut meta, &src, &previous).unwrap();
assert_eq!(meta.authored, vec![(0, 16)]);
assert!(meta.complete);
assert_eq!(std::fs::metadata(&src).unwrap().len(), 16);
}
}
#[cfg(test)]
mod size_upgrade_tests {
use super::{
LinkId, MAX_SIZE_UPGRADES, NodeUid, SizeUpgrade, SizeUpgradeClaim, SizeWaitQueue,
SizeWaiter, VolumeId, claim_size_upgrade, next_size_upgrade,
};
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
use std::time::{Duration, Instant};
/// Park a waiter the way [`Core::await_size`] does, and hand back the
/// channel its reply is sent down, so a test can time the release.
fn park(
queue: &Arc<SizeWaitQueue>,
slot: Arc<SizeUpgrade>,
resolved: impl Fn() -> bool + Send + 'static,
deadline: Instant,
) -> mpsc::Receiver<()> {
let (tx, rx) = mpsc::channel();
let waiter = SizeWaiter {
slot,
resolved: Box::new(resolved),
deadline,
respond: Box::new(move || {
let _ = tx.send(());
}),
};
if queue.park(waiter) {
let queue = queue.clone();
std::thread::spawn(move || queue.serve());
}
rx
}
/// Long enough that a test never trips the deadline by accident, short
/// enough that one which is *meant* to does not take ten seconds.
fn far_off() -> Instant {
Instant::now() + Duration::from_secs(30)
}
/// The point of the per-chunk design: a waiter is answered as soon as *its
/// own* file is resolved, without waiting for the rest of the folder. A
/// single batch over 793 nodes took ~80 s, which outran the timeout and
/// handed back the provisional size this is all meant to prevent.
#[test]
fn a_waiter_returns_on_the_chunk_that_resolves_it() {
let queue = Arc::new(SizeWaitQueue::default());
let slot = Arc::new(SizeUpgrade::default());
let mine = Arc::new(AtomicBool::new(false));
let worker = slot.clone();
let flag = mine.clone();
let t = std::thread::spawn(move || {
// Our node lands in the first chunk; two more follow.
std::thread::sleep(Duration::from_millis(30));
flag.store(true, Ordering::SeqCst);
worker.chunk_done();
std::thread::sleep(Duration::from_millis(300));
worker.chunk_done();
worker.finish();
});
let started = Instant::now();
let rx = park(&queue, slot, move || mine.load(Ordering::SeqCst), far_off());
rx.recv().expect("the waiter is answered");
// Released by the first chunk, not the last.
assert!(started.elapsed() < Duration::from_millis(200));
t.join().unwrap();
}
/// `pdfs ls` waits for the whole batch, so it lists no ciphertext size
/// (`docs/BUGS.md` B115), but no longer than a `stat` would.
#[test]
fn a_listing_waits_for_the_whole_size_batch_but_not_forever() {
let slot = Arc::new(SizeUpgrade::default());
let worker = slot.clone();
// Read before the spawn: the worker's sleeps can start before this
// thread runs again, which made the bound below fail on a busy runner.
let started = Instant::now();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(30));
worker.chunk_done();
std::thread::sleep(Duration::from_millis(60));
worker.finish();
});
slot.wait_finished(started + Duration::from_secs(5));
// Past the first chunk, up to the end of the batch.
assert!(started.elapsed() >= Duration::from_millis(90));
assert!(slot.is_finished());
t.join().unwrap();
let stuck = SizeUpgrade::default();
let started = Instant::now();
stuck.wait_finished(started + Duration::from_millis(50));
assert!(started.elapsed() < Duration::from_secs(1));
assert!(!stuck.is_finished());
}
/// A chunk that does not resolve this waiter must not answer it.
#[test]
fn an_unrelated_chunk_does_not_release_a_waiter() {
let queue = Arc::new(SizeWaitQueue::default());
let slot = Arc::new(SizeUpgrade::default());
let worker = slot.clone();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(30));
worker.chunk_done();
std::thread::sleep(Duration::from_millis(120));
worker.finish();
});
let started = Instant::now();
// Never resolved: only `finish` can end this wait.
let rx = park(&queue, slot, || false, far_off());
rx.recv().expect("the waiter is answered");
assert!(started.elapsed() >= Duration::from_millis(140));
t.join().unwrap();
}
/// A batch that ends without resolving the node — a failed fetch — still
/// answers its waiters, who fall back to the provisional size.
#[test]
fn finish_releases_a_waiter_that_was_never_resolved() {
let queue = Arc::new(SizeWaitQueue::default());
let slot = Arc::new(SizeUpgrade::default());
let worker = slot.clone();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(30));
worker.finish();
});
let started = Instant::now();
let rx = park(&queue, slot, || false, far_off());
rx.recv().expect("the waiter is answered");
assert!(started.elapsed() < SizeUpgrade::WAIT);
t.join().unwrap();
}
/// Already resolved before parking: answered on the first pass. This is the
/// follower that arrives after the chunk it needed has landed.
#[test]
fn an_already_resolved_waiter_does_not_block() {
let queue = Arc::new(SizeWaitQueue::default());
let started = Instant::now();
let rx = park(&queue, Arc::new(SizeUpgrade::default()), || true, far_off());
rx.recv().expect("the waiter is answered");
assert!(started.elapsed() < Duration::from_millis(100));
}
/// Finishing before anyone parks must not strand the late arrival — the
/// flag is what is checked, not the notification, which it would miss.
#[test]
fn waiting_after_finish_returns_at_once() {
let queue = Arc::new(SizeWaitQueue::default());
let slot = Arc::new(SizeUpgrade::default());
slot.finish();
let started = Instant::now();
let rx = park(&queue, slot, || false, far_off());
rx.recv().expect("the waiter is answered");
assert!(started.elapsed() < Duration::from_millis(100));
}
/// Every waiter is answered, not just the first: one `ls -l` puts one
/// waiter per file on the same folder.
#[test]
fn all_waiters_are_released() {
let queue = Arc::new(SizeWaitQueue::default());
let slot = Arc::new(SizeUpgrade::default());
let waiters: Vec<_> = (0..8)
.map(|_| park(&queue, slot.clone(), || false, far_off()))
.collect();
std::thread::sleep(Duration::from_millis(20));
slot.finish();
for rx in waiters {
rx.recv().expect("every waiter is answered");
}
}
/// The deadline is the backstop: a batch that never finishes must not wedge
/// the caller's `stat` forever.
#[test]
fn a_waiter_gives_up_at_the_deadline() {
let queue = Arc::new(SizeWaitQueue::default());
let started = Instant::now();
// Nothing will ever resolve or finish this.
let rx = park(
&queue,
Arc::new(SizeUpgrade::default()),
|| false,
Instant::now() + Duration::from_millis(120),
);
rx.recv().expect("the waiter is answered");
assert!(started.elapsed() >= Duration::from_millis(120));
}
/// The serving thread retires when nothing is parked, and a later park
/// starts a new one — otherwise the second `ls -l` of the session would
/// hang forever.
#[test]
fn the_queue_serves_again_after_it_has_drained() {
let queue = Arc::new(SizeWaitQueue::default());
for _ in 0..3 {
let rx = park(&queue, Arc::new(SizeUpgrade::default()), || true, far_off());
rx.recv().expect("the waiter is answered");
}
}
/// With no thread to serve them, waiters are answered on the spot rather
/// than never: a provisional size beats a `stat` that does not return.
#[test]
fn answer_all_releases_everything_parked() {
let queue = SizeWaitQueue::default();
let (tx, rx) = mpsc::channel();
for _ in 0..4 {
let tx = tx.clone();
queue.park(SizeWaiter {
slot: Arc::new(SizeUpgrade::default()),
resolved: Box::new(|| false),
deadline: far_off(),
respond: Box::new(move || {
let _ = tx.send(());
}),
});
}
drop(tx);
queue.answer_all();
assert_eq!(rx.into_iter().count(), 4);
}
fn uids(links: &[&str]) -> Vec<NodeUid> {
links
.iter()
.map(|link| NodeUid::new(VolumeId::from("vol"), LinkId::from(*link)))
.collect()
}
/// Past the cap a folder used to get no upgrade at all, so a recursive
/// walk reported ciphertext sizes for most of what it listed (`docs/BUGS.md` B100).
#[test]
fn a_folder_past_the_cap_is_queued_not_dropped() {
let mut in_flight = HashMap::new();
for key in 0..MAX_SIZE_UPGRADES as u64 {
let claim = claim_size_upgrade(&mut in_flight, key, uids(&["a"]));
assert!(matches!(claim, SizeUpgradeClaim::Start(..)));
}
let key = MAX_SIZE_UPGRADES as u64;
let claim = claim_size_upgrade(&mut in_flight, key, uids(&["late"]));
let SizeUpgradeClaim::Queued(slot) = claim else {
panic!("expected the batch to be queued");
};
assert!(Arc::ptr_eq(&slot, &in_flight[&key]));
// A second caller for the queued folder waits on the same batch.
let again = claim_size_upgrade(&mut in_flight, key, uids(&["late"]));
assert!(matches!(again, SizeUpgradeClaim::Joined(s) if Arc::ptr_eq(&s, &slot)));
}
#[test]
fn a_finished_batch_hands_its_thread_the_oldest_queued_one() {
let mut in_flight = HashMap::new();
for key in 0..MAX_SIZE_UPGRADES as u64 {
claim_size_upgrade(&mut in_flight, key, uids(&["a"]));
}
let first = MAX_SIZE_UPGRADES as u64;
claim_size_upgrade(&mut in_flight, first, uids(&["first"]));
std::thread::sleep(Duration::from_millis(2));
claim_size_upgrade(&mut in_flight, first + 1, uids(&["second"]));
let (key, taken, slot) = next_size_upgrade(&mut in_flight, 0).expect("a queued batch");
assert_eq!((key, taken), (first, uids(&["first"])));
assert!(slot.queued.lock().is_none());
assert!(!in_flight.contains_key(&0));
// Still at the cap: the handed-over batch counts as running.
let running = in_flight
.values()
.filter(|s| s.queued.lock().is_none())
.count();
assert_eq!(running, MAX_SIZE_UPGRADES);
let (key, _, _) = next_size_upgrade(&mut in_flight, 1).expect("the other one");
assert_eq!(key, first + 1);
assert!(next_size_upgrade(&mut in_flight, 2).is_none());
}
}
#[cfg(test)]
mod replace_tests {
use super::check_replaceable;
/// The case that motivated all of this: rsync renaming its temp file over
/// the real one. Two plain files, and it has to be allowed — refusing is
/// what made every rsync transfer fail at the last step (`docs/BUGS.md` B13).
#[test]
fn a_file_may_replace_a_file() {
assert!(check_replaceable(false, false, true).is_ok());
}
#[test]
fn an_empty_directory_may_be_replaced_by_a_directory() {
assert!(check_replaceable(true, true, true).is_ok());
}
/// Proton trashes a folder with its whole subtree, so allowing this would
/// discard every file under the destination without ever naming them.
#[test]
fn a_non_empty_directory_is_never_replaced() {
let e = check_replaceable(true, true, false).expect_err("must refuse");
assert_eq!(e.code(), libc::ENOTEMPTY);
}
#[test]
fn the_two_ends_must_agree_on_being_a_directory() {
assert_eq!(
check_replaceable(false, true, true)
.expect_err("a file may not replace a directory")
.code(),
libc::EISDIR
);
assert_eq!(
check_replaceable(true, false, true)
.expect_err("a directory may not replace a file")
.code(),
libc::ENOTDIR
);
}
/// `dst_empty` describes a directory; it must not leak into the file case,
/// where callers pass `true` by convention.
#[test]
fn emptiness_is_ignored_when_the_destination_is_a_file() {
assert!(check_replaceable(false, false, false).is_ok());
}
}
#[cfg(test)]
mod local_uid_tests {
use super::*;
#[test]
fn a_local_node_uid_is_recognisable_and_round_trips() {
let uid = local_node_uid(42);
assert!(is_local_uid(&uid));
assert!(is_local_uid_str(&uid.to_string()));
assert_eq!(uid.to_string(), pdfs_core::db::local_uid(42));
// It has to survive the trip through `pending_op.uid` as text, like any
// other uid does, and give back the row it stands for.
let parsed = parse_node_uid(&uid.to_string()).expect("parses back");
assert_eq!(parsed, uid);
assert_eq!(pdfs_core::db::local_lid(&uid.to_string()), Some(42));
}
#[test]
fn local_node_uids_are_distinct_per_row() {
// Two files created in the same millisecond must not collide — the whole
// queue is keyed by uid.
assert_ne!(local_node_uid(1), local_node_uid(2));
}
#[test]
fn a_real_uid_is_never_mistaken_for_a_placeholder() {
let real = NodeUid::new(VolumeId::from("vol1"), LinkId::from("link1"));
assert!(!is_local_uid(&real));
assert!(!is_local_uid_str("vol1~link1"));
// Not a uid at all.
assert!(!is_local_uid_str("local"));
// The sentinel is the *volume*; a link that merely says "local" is real.
assert!(!is_local_uid_str("vol1~local"));
}
#[test]
fn a_placeholder_file_reports_itself_as_empty_and_unsealed() {
let parent = NodeUid::new(VolumeId::from("vol1"), LinkId::from("dir"));
let node = local_node(local_node_uid(7), parent.clone(), "notes.txt".into(), false);
assert_eq!(node.name, "notes.txt");
assert_eq!(node.parent_uid, Some(parent));
assert!(!node.trashed);
match node.kind {
NodeKind::File {
claimed_size,
active_revision_state,
ref media_type,
..
} => {
assert_eq!(claimed_size, Some(0));
// Nothing has been uploaded, so there is no sealed revision.
assert!(active_revision_state.is_none());
assert_eq!(media_type, "text/plain");
}
NodeKind::Folder => panic!("expected a file"),
}
}
#[test]
fn a_placeholder_folder_is_a_folder() {
let parent = NodeUid::new(VolumeId::from("vol1"), LinkId::from("root"));
let node = local_node(local_node_uid(7), parent, "photos".into(), true);
assert!(node.is_folder());
}
}
#[cfg(test)]
mod pending_size_tests {
use super::{Node, NodeKind, NodeUid, apply_pending_sizes, node_size};
use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId};
use std::collections::HashMap;
fn uid(link: &str) -> NodeUid {
NodeUid::new(VolumeId::from("vol"), LinkId::from(link))
}
fn file(link: &str, claimed: i64) -> Node {
Node {
uid: uid(link),
parent_uid: Some(uid("parent")),
kind: NodeKind::File {
media_type: "text/plain".into(),
total_size_on_storage: 0,
active_revision_state: None,
active_revision_id: None,
content_sha1: None,
claimed_size: Some(claimed),
claimed_modification_time: None,
},
name: link.into(),
creation_time: 100,
modification_time: 100,
trashed: false,
is_shared: false,
is_shared_publicly: false,
signature_email: None,
membership: None,
photo: None,
album: None,
verification: Default::default(),
direct_role: None,
share_id: None,
}
}
fn folder(link: &str) -> Node {
Node {
kind: NodeKind::Folder,
..file(link, 0)
}
}
/// B11: a re-enumeration mid-write must not revert the size to the
/// server's. A file that stats as 0 gets no `read` from the kernel at all,
/// so the staged blob is never served and the file reads as empty.
#[test]
fn a_queued_write_keeps_its_optimistic_size_through_a_re_enumeration() {
let mut nodes = vec![file("queued", 0), file("settled", 4096)];
let sizes = HashMap::from([(uid("queued"), 3)]);
apply_pending_sizes(&mut nodes, &sizes);
assert_eq!(
node_size(&nodes[0]),
3,
"the remote's pre-write size must not win over the queued write's"
);
assert_eq!(
node_size(&nodes[1]),
4096,
"a file with nothing queued keeps the size the server reported"
);
}
/// The map is keyed by uid and says nothing about kind; a folder that
/// somehow collides must not grow a `claimed_size`.
#[test]
fn folders_are_left_alone() {
let mut nodes = vec![folder("dir")];
apply_pending_sizes(&mut nodes, &HashMap::from([(uid("dir"), 999)]));
assert!(matches!(nodes[0].kind, NodeKind::Folder));
assert_eq!(node_size(&nodes[0]), 0);
}
/// The common case: nothing queued, nothing touched.
#[test]
fn an_empty_pending_map_changes_nothing() {
let mut nodes = vec![file("a", 10), file("b", 20)];
apply_pending_sizes(&mut nodes, &HashMap::new());
assert_eq!(node_size(&nodes[0]), 10);
assert_eq!(node_size(&nodes[1]), 20);
}
}
#[cfg(test)]
mod tests {
use super::{
Access, AccessFlags, Errno, HashMap, Intervals, PendingRevision, RootListingSnapshot,
SELF_CHANGE_TTL_MS, ShareId, SharedWithMeItem, StateRegistry, VirtualRootPlan,
accepted_share_provenance, conflict_label, conflict_label_of, conflict_name,
conflict_name_as, copy_pending_for_truncate, expand_restore, first_seen_in_trash,
fuse_name, is_stale_mount, node_visible, note_self_change, off_the_runtime,
orphan_past_retention, parse_node_uid, prepare_shared_roots, preserve_on_access_denied,
publish_virtual_root_in_listing, reconcile_virtual_root_in_listing,
release_can_discard_unlinked, release_must_retain_queued_trash, release_unlinked_entry,
rename_needs_queue, require_node_parent_access, require_rename_access,
resolve_anywhere_with, shared_with_me_uid, take_self_change, uid_write_authority,
virtual_node,
};
use super::{Db, WriteAuthority};
use std::time::Duration;
/// Staged bytes that name no node are kept for a month, because the only
/// copy of a write must outlive the bug that stranded it. Past that, one
/// boot retires them instead of re-announcing them every boot forever.
#[test]
fn a_stranded_blob_is_retired_only_once_it_outlives_retention() {
let dir = std::env::temp_dir().join(format!("pdfs-orphan-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let blob = dir.join("blob");
std::fs::write(&blob, b"twelve bytes").unwrap();
// Written just now: still recoverable, still kept.
assert_eq!(orphan_past_retention(&blob), None);
set_mtime_ago(
&blob,
super::STAGING_ORPHAN_RETAIN + Duration::from_secs(60),
);
assert_eq!(orphan_past_retention(&blob), Some(12));
// A blob that is not there is not a blob to retire.
std::fs::remove_file(&blob).unwrap();
assert_eq!(orphan_past_retention(&blob), None);
let _ = std::fs::remove_dir_all(&dir);
}
/// Backdate a file's mtime, which is what the retention check reads.
fn set_mtime_ago(path: &std::path::Path, ago: Duration) {
let when = std::time::SystemTime::now() - ago;
let secs = when
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs() as libc::time_t;
let times = [
libc::timeval {
tv_sec: secs,
tv_usec: 0,
},
libc::timeval {
tv_sec: secs,
tv_usec: 0,
},
];
let c_path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
// SAFETY: both pointers are valid for the call and nothing keeps them.
let rc = unsafe { libc::utimes(c_path.as_ptr(), times.as_ptr()) };
assert_eq!(rc, 0, "utimes: {}", std::io::Error::last_os_error());
}
/// `(uid, parent)` pairs in the shape [`Db::trash_parents`] returns.
fn relations(pairs: &[(&str, Option<&str>)]) -> Vec<(String, Option<String>)> {
pairs
.iter()
.map(|(uid, parent)| (uid.to_string(), parent.map(str::to_string)))
.collect()
}
fn uids(values: &[&str]) -> Vec<String> {
values.iter().map(|v| v.to_string()).collect()
}
/// Restoring a folder brings back what was trashed inside it, and the folder
/// goes first: the server has nowhere to put a child whose parent is trashed.
#[test]
fn restoring_a_folder_restores_its_trashed_contents_parents_first() {
let trash = relations(&[
("dir", Some("live")),
("file", Some("dir")),
("deep", Some("sub")),
("sub", Some("dir")),
]);
assert_eq!(
expand_restore(&uids(&["dir"]), &trash),
vec![
vec!["dir".to_string()],
vec!["file".to_string(), "sub".to_string()],
vec!["deep".to_string()],
]
);
}
/// Restoring a file inside a trashed folder restores the folder too —
/// otherwise the file comes back somewhere the user cannot reach.
#[test]
fn restoring_content_restores_the_trashed_parents_above_it() {
let trash = relations(&[
("dir", Some("live")),
("sub", Some("dir")),
("file", Some("sub")),
("other", Some("dir")),
]);
// `other` is a sibling of `sub`, not an ancestor of `file`: it stays put.
assert_eq!(
expand_restore(&uids(&["file"]), &trash),
vec![
vec!["dir".to_string()],
vec!["sub".to_string()],
vec!["file".to_string()],
]
);
}
/// A node whose parent is not in the trash is already at the top: one wave,
/// no ancestors invented.
#[test]
fn a_node_under_a_live_parent_restores_on_its_own() {
let trash = relations(&[("file", Some("live")), ("elsewhere", None)]);
assert_eq!(
expand_restore(&uids(&["file"]), &trash),
vec![vec!["file".to_string()]]
);
}
/// The listing is materialised in chunks and can be stale, so a uid it does
/// not know about must still be restored rather than dropped.
#[test]
fn an_unlisted_uid_is_still_restored() {
let trash = relations(&[("dir", None)]);
assert_eq!(
expand_restore(&uids(&["missing"]), &trash),
vec![vec!["missing".to_string()]]
);
}
/// Asking for both a folder and something inside it restores each node once.
#[test]
fn overlapping_requests_do_not_duplicate_nodes() {
let trash = relations(&[("dir", None), ("file", Some("dir"))]);
assert_eq!(
expand_restore(&uids(&["dir", "file"]), &trash),
vec![vec!["dir".to_string()], vec!["file".to_string()]]
);
}
/// A node keeps the time it was first seen in the trash; a new one is dated
/// now only when the previous refresh saw the whole trash.
#[test]
fn trash_dates_are_kept_and_only_guessed_after_a_complete_refresh() {
let seen = HashMap::from([("old".to_string(), Some(5)), ("unknown".to_string(), None)]);
assert_eq!(first_seen_in_trash(&seen, "old", true, 9), Some(5));
assert_eq!(first_seen_in_trash(&seen, "old", false, 9), Some(5));
assert_eq!(first_seen_in_trash(&seen, "unknown", true, 9), None);
assert_eq!(first_seen_in_trash(&seen, "new", true, 9), Some(9));
assert_eq!(first_seen_in_trash(&seen, "new", false, 9), None);
}
/// A parent chain that loops back on itself must terminate.
#[test]
fn a_cyclic_listing_terminates() {
let trash = relations(&[("a", Some("b")), ("b", Some("a"))]);
let waves = expand_restore(&uids(&["a"]), &trash);
let total: usize = waves.iter().map(Vec::len).sum();
assert_eq!(total, 2);
}
/// The collapse [`Core::require_uid_writable`] performs, without a `Core`:
/// every non-writable authority is one `EACCES` on the syscall path.
fn require_uid_access(db: &Db, uid: &NodeUid, live: &[Access]) -> Result<(), Errno> {
match uid_write_authority(db, uid, live) {
WriteAuthority::Writable => Ok(()),
WriteAuthority::Denied | WriteAuthority::Unknown => Err(Errno::EACCES),
}
}
use crate::filesystem::access_allowed;
use pdfs_core::cache::{Baseline, StagedWrite};
use pdfs_core::db::{OP_REVISION, OP_TRASH, PendingOp};
fn session_flag(live: bool) -> std::sync::Arc<std::sync::atomic::AtomicBool> {
std::sync::Arc::new(std::sync::atomic::AtomicBool::new(live))
}
/// A mount as the registry sees it, with the cells its `Core` would own.
struct TestMount {
notifier: std::sync::Arc<std::sync::OnceLock<super::KernelChannel>>,
live: std::sync::Arc<std::sync::atomic::AtomicBool>,
root: std::sync::Arc<std::sync::OnceLock<u64>>,
}
impl TestMount {
fn register(registry: &StateRegistry, mountpoint: &str, root: Option<u64>) -> Self {
let mount = Self {
notifier: std::sync::Arc::new(std::sync::OnceLock::new()),
live: session_flag(false),
root: std::sync::Arc::new(std::sync::OnceLock::new()),
};
if let Some(root) = root {
mount.root.set(root).unwrap();
}
registry.register(
std::path::Path::new(mountpoint),
&mount.notifier,
mount.live.clone(),
mount.root.clone(),
);
mount
}
fn spawn(&self) {
self.live.store(true, std::sync::atomic::Ordering::Release);
}
}
/// Whether `uid` is visible from the root a single-mount test installs.
fn visible(state: &crate::state::State, uid: &NodeUid) -> bool {
state.owns_visible_uid(uid, &[super::ROOT_INO])
}
/// The registry is how background work reaches every mounted session's
/// kernel cache, so the walk has to see *every* registered mount — and
/// only the ones still mounted (`docs/BUGS.md` B74).
#[test]
fn state_registry_walks_every_live_mount_and_reaps_the_rest() {
let registry = StateRegistry::default();
assert!(registry.live().is_empty(), "a fresh registry has no mounts");
// The primary mount plus two on-demand forks.
let primary = TestMount::register(&registry, "/mnt/primary", None);
let fork_a = TestMount::register(&registry, "/mnt/fork-a", None);
let fork_b = TestMount::register(&registry, "/mnt/fork-b", None);
assert_eq!(registry.live().len(), 3, "every registered mount is walked");
// `live()` must not itself pin a mount alive, or an unmounted fork could
// never be reaped.
assert_eq!(
std::sync::Arc::strong_count(&fork_a.notifier),
1,
"the registry holds forks weakly"
);
// Unmounting a fork drops its `Core`; the dead entry goes on the next walk.
drop(fork_a);
assert_eq!(registry.live().len(), 2, "an unmounted fork is dropped");
assert_eq!(
registry.0.lock().len(),
2,
"and its slot is reaped, not merely skipped"
);
drop(primary);
drop(fork_b);
assert!(
registry.live().is_empty(),
"the last mount going away empties the registry"
);
}
#[test]
fn state_registry_covering_uses_longest_prefix_and_ignores_dead_forks() {
let registry = StateRegistry::default();
let primary_path = std::path::Path::new("/home/me/ProtonDrive");
let nested_path = std::path::Path::new("/home/me/ProtonDrive/Device");
let sibling_path = std::path::Path::new("/home/me/Archive");
let primary = TestMount::register(&registry, "/home/me/ProtonDrive", Some(2));
let nested = TestMount::register(&registry, "/home/me/ProtonDrive/Device", Some(3));
let sibling = TestMount::register(&registry, "/home/me/Archive", Some(4));
sibling.spawn();
assert_eq!(registry.live().len(), 3);
assert!(!registry.is_mounted_at(primary_path));
assert!(!registry.is_mounted_at(nested_path));
assert_eq!(
registry.live_roots(),
vec![4],
"an unspawned session's root must not authorize resident uids"
);
assert!(
registry
.covering(&nested_path.join("folder/file.txt"))
.is_none(),
"a registered but unspawned session must not win path covering"
);
primary.spawn();
nested.spawn();
let mut roots = registry.live_roots();
roots.sort();
assert_eq!(roots, vec![2, 3, 4]);
let covering = registry
.covering(&nested_path.join("folder/file.txt"))
.unwrap();
assert_eq!(covering.mountpoint, nested_path);
assert!(std::sync::Arc::ptr_eq(&covering.notifier, &nested.notifier));
drop(covering);
let covering = registry.covering(nested_path).unwrap().mountpoint;
assert_eq!(covering, nested_path, "an exact mountpoint covers itself");
let covering = registry
.covering(std::path::Path::new("/home/me/ProtonDrive/Device/folder/"))
.unwrap()
.mountpoint;
assert_eq!(
covering, nested_path,
"a trailing separator does not change component-prefix selection"
);
let covering = registry
.covering(std::path::Path::new(
"/home/me/ProtonDrive/DeviceBackup/file",
))
.unwrap()
.mountpoint;
assert_eq!(
covering, primary_path,
"/Device must not string-prefix-match /DeviceBackup"
);
let covering = registry.covering(&sibling_path.join("file")).unwrap();
assert_eq!(covering.mountpoint, sibling_path);
assert!(std::sync::Arc::ptr_eq(
&covering.notifier,
&sibling.notifier
));
drop(covering);
assert!(
registry
.covering(std::path::Path::new("/home/me/ProtonDriveBackup/file"))
.is_none(),
"/ProtonDrive must not string-prefix-match /ProtonDriveBackup"
);
assert!(
registry
.covering(std::path::Path::new("/tmp/outside"))
.is_none(),
"an outside path has no covering mount"
);
assert!(registry.is_mounted_at(primary_path));
assert!(registry.is_mounted_at(nested_path));
assert!(
!registry.is_mounted_at(&nested_path.join("folder")),
"a path covered by a session is not itself a mountpoint"
);
// This is also the failed-fork case: the caller registered the fork,
// session construction failed, and its last `Core` was dropped. It
// must not make ListLocations report mounted.
drop(nested);
assert!(!registry.is_mounted_at(nested_path));
assert_eq!(registry.live().len(), 2);
}
/// The routing half of `docs/BUGS.md` B86: a path under a secondary mount
/// has to resolve to *that* mount's root and a suffix relative to it, or
/// `pdfs ls` and `pdfs refresh` can only ever name the primary mount and a
/// stale on-demand folder has no escape hatch.
#[test]
fn covering_routes_a_path_to_the_mount_that_owns_it() {
let registry = StateRegistry::default();
let primary_path = std::path::Path::new("/home/me/ProtonDrive");
let nested_path = std::path::Path::new("/home/me/ProtonDrive/Device");
let primary = TestMount::register(&registry, "/home/me/ProtonDrive", Some(2));
let nested = TestMount::register(&registry, "/home/me/ProtonDrive/Device", Some(3));
primary.spawn();
nested.spawn();
let deep = nested_path.join("sub/file.txt");
let mount = registry.covering(&deep).expect("nested mount covers");
assert_eq!(mount.mountpoint, nested_path, "the nested mount wins");
assert_eq!(
mount.root.get(),
Some(&3),
"the request must be resolved from the nested root"
);
assert!(
std::sync::Arc::ptr_eq(&mount.notifier, &nested.notifier),
"and tell that mount's session, not the primary's"
);
assert_eq!(
deep.strip_prefix(&mount.mountpoint).unwrap(),
std::path::Path::new("sub/file.txt"),
"the suffix is relative to the mount that owns it, not the primary"
);
let shallow = primary_path.join("Documents");
let mount = registry.covering(&shallow).expect("primary mount covers");
assert_eq!(
mount.root.get(),
Some(&2),
"a path outside the nested root still routes to the primary"
);
assert!(
registry
.covering(std::path::Path::new("/tmp/outside"))
.is_none(),
"a path under no mount routes nowhere, and the caller reports that"
);
}
/// Every mount shares one tree, so a node under a fork's root and one under
/// the primary root are both found there — but only while the mount whose
/// root they sit under is mounted.
#[test]
fn a_uid_is_visible_under_the_root_of_any_mounted_session() {
let (mut state, _dir) = rooted_state("primary-volume", "primary");
let primary_uid = state.entries[&super::ROOT_INO].uid.clone();
let device_root = node_helper_in_volume("device-volume", "device", None, "device", true);
let fork_root = state.install_root(device_root);
let fork_node =
node_helper_in_volume("device-volume", "fork", Some("device"), "fork", true);
let fork_uid = fork_node.uid.clone();
let fork_ino = state.intern(fork_root, fork_node);
state.children.insert(fork_root, vec![fork_ino]);
let both = [super::ROOT_INO, fork_root];
assert!(state.owns_visible_uid(&primary_uid, &both));
assert!(state.owns_visible_uid(&fork_uid, &both));
assert!(
!state.owns_visible_uid(&super::parse_uid("device-volume~missing").unwrap(), &both)
);
assert!(
!state.owns_visible_uid(&fork_uid, &[super::ROOT_INO]),
"an unmounted on-demand root must stop authorizing its uids"
);
}
/// Each session calls its own root inode 1, and nothing else may come out
/// as 1 or come in from it.
#[test]
fn each_session_names_its_own_root_as_the_kernel_root() {
let fork_root = 41;
assert_eq!(
super::filesystem::kernel_ino(fork_root, fork_root).0,
super::ROOT_INO
);
assert_eq!(super::filesystem::kernel_ino(fork_root, 7).0, 7);
assert_eq!(super::filesystem::kernel_ino(super::ROOT_INO, 7).0, 7);
}
#[test]
fn a_missing_listing_is_unknown_not_absent() {
let (mut state, _dir) = rooted_state("device-volume", "device");
let child_node =
node_helper_in_volume("device-volume", "child", Some("device"), "child.txt", false);
let child_uid = child_node.uid.clone();
let child_ino = state.intern(super::ROOT_INO, child_node);
state.children.insert(super::ROOT_INO, vec![child_ino]);
assert!(visible(&state, &child_uid));
state.invalidate_listing(super::ROOT_INO);
assert!(
visible(&state, &child_uid),
"an invalidated listing is unknown and must not hide a valid resident child"
);
state.children.insert(super::ROOT_INO, Vec::new());
assert!(
!visible(&state, &child_uid),
"a present listing that omits the child proves it is no longer visible"
);
}
#[test]
fn virtual_root_reconcile_does_not_recreate_an_invalidated_listing() {
let (mut state, _dir) = rooted_state("primary-volume", "root");
let root_uid = state.entries[&super::ROOT_INO].uid.clone();
let child_node =
node_helper_in_volume("primary-volume", "child", Some("root"), "child.txt", false);
let child_uid = child_node.uid.clone();
let child_ino = state.intern(super::ROOT_INO, child_node);
state.children.insert(super::ROOT_INO, vec![child_ino]);
// Simulate ensure_children observing a complete primary-root listing.
let snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap();
let detected_names = snapshot.real_names();
assert_eq!(
detected_names,
std::collections::HashSet::from(["child.txt".into()])
);
// An event wins the race before virtual-root dentry publication.
state.invalidate_listing(super::ROOT_INO);
let virtual_ino = state.intern_mem(
super::ROOT_INO,
virtual_node(root_uid, "Shared with me".into(), 0),
);
assert!(
!reconcile_virtual_root_in_listing(
&mut state,
super::ROOT_INO,
&snapshot,
virtual_ino,
true,
),
"the invalidated snapshot must not publish"
);
assert!(
!state.children.contains_key(&super::ROOT_INO),
"reconciliation must leave an invalidated listing absent"
);
assert!(
visible(&state, &child_uid),
"the absent listing remains unknown until refresh, so a valid resident uid is allowed"
);
}
#[test]
fn virtual_root_reconcile_rejects_repopulated_changed_snapshot() {
let (mut state, _dir) = rooted_state("primary-volume", "root");
let root_uid = state.entries[&super::ROOT_INO].uid.clone();
let child =
node_helper_in_volume("primary-volume", "child", Some("root"), "child.txt", false);
let child_ino = state.intern(super::ROOT_INO, child);
state.children.insert(super::ROOT_INO, vec![child_ino]);
let stale_snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap();
let mut persisted_virtual = virtual_node(root_uid.clone(), "Shared with me".into(), 0);
persisted_virtual.trashed = true;
state
.flushed_db()
.publish_virtual_root(super::SHARED_WITH_ME_NAME, &persisted_virtual)
.unwrap();
let mut indexed_descendant = node_helper_in_volume(
"foreign-volume",
"inside",
None,
"SnapshotRaceFindable",
false,
);
indexed_descendant.parent_uid = Some(shared_with_me_uid());
state.flushed_db().upsert_node(&indexed_descendant).unwrap();
assert!(
state
.db
.search("SnapshotRaceFindable", 10)
.unwrap()
.is_empty(),
"the hidden synthetic ancestor must hide descendant search hits"
);
state.invalidate_listing(super::ROOT_INO);
state.entries.get_mut(&child_ino).unwrap().node.name = "Shared with me".into();
state.children.insert(super::ROOT_INO, vec![child_ino]);
let replacement_snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap();
assert_ne!(stale_snapshot, replacement_snapshot);
assert_eq!(
replacement_snapshot.real_names(),
std::collections::HashSet::from(["Shared with me".into()])
);
let mut stale_node = persisted_virtual.clone();
stale_node.trashed = false;
let stale_plan = VirtualRootPlan {
node: stale_node.clone(),
visible: true,
};
let db = state.flushed_db();
let error = publish_virtual_root_in_listing(
&db,
&mut state,
super::ROOT_INO,
&stale_snapshot,
stale_plan,
)
.unwrap_err();
assert_eq!(
error.code(),
Errno::EAGAIN.code(),
"stale work must fail before persistence or dentry publication"
);
assert_eq!(
state.children[&super::ROOT_INO],
vec![child_ino],
"the replacement listing must remain untouched"
);
assert!(
state
.db
.node_by_uid(&shared_with_me_uid().to_string())
.unwrap()
.unwrap()
.trashed,
"stale work must not change persisted visibility"
);
assert!(
state
.db
.search("SnapshotRaceFindable", 10)
.unwrap()
.is_empty(),
"stale work must not change FTS ancestry"
);
let virtual_ino = state.intern_mem(super::ROOT_INO, stale_node);
state
.children
.get_mut(&super::ROOT_INO)
.unwrap()
.push(virtual_ino);
let current_snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap();
state.entries.get_mut(&virtual_ino).unwrap().node.trashed = false;
assert!(
reconcile_virtual_root_in_listing(
&mut state,
super::ROOT_INO,
&current_snapshot,
virtual_ino,
false,
),
"the synthetic node's own visibility update must not stale the real-child snapshot"
);
assert_eq!(state.children[&super::ROOT_INO], vec![child_ino]);
}
#[test]
fn virtual_root_publication_failure_is_atomic_and_leaves_state_unchanged() {
let (mut state, _dir) = rooted_state("primary-volume", "root");
let root_uid = state.entries[&super::ROOT_INO].uid.clone();
let child =
node_helper_in_volume("primary-volume", "child", Some("root"), "child.txt", false);
let child_ino = state.intern(super::ROOT_INO, child);
state.children.insert(super::ROOT_INO, vec![child_ino]);
let snapshot = RootListingSnapshot::capture(&state, super::ROOT_INO).unwrap();
let mut descendant = node_helper_in_volume(
"foreign-volume",
"inside",
None,
"AtomicRollbackFindable",
false,
);
descendant.parent_uid = Some(shared_with_me_uid());
state.flushed_db().upsert_node(&descendant).unwrap();
// A node whose parent row is not cached — the synthetic root is not one
// until this publication commits — is indexed under the path it can
// actually be resolved to, which is its bare name. Publication reindexes
// the subtree beneath the synthetic ancestor, so that path is what the
// rollback has to restore.
let path_before = {
let hits = state
.flushed_db()
.search("AtomicRollbackFindable", 10)
.unwrap();
assert_eq!(hits.len(), 1, "the descendant starts out searchable");
hits[0].path.clone()
};
assert_eq!(path_before, "AtomicRollbackFindable");
// The pin is the transaction's final statement. Failing it proves the
// earlier access, node, and descendant FTS work rolls back with it.
state
.flushed_db()
.with_conn(|conn| {
conn.execute_batch(
"CREATE TRIGGER reject_virtual_root_pin
BEFORE INSERT ON sync_state
WHEN NEW.key = 'shared_with_me_name'
BEGIN
SELECT RAISE(ABORT, 'injected virtual-root pin failure');
END;",
)?;
Ok(())
})
.unwrap();
let entries_before = state.entries.len();
let by_uid_before = state.by_uid.len();
let next_ino_before = state.next_ino;
let access_before = state.share_access.clone();
let db = state.flushed_db();
let result = publish_virtual_root_in_listing(
&db,
&mut state,
super::ROOT_INO,
&snapshot,
VirtualRootPlan {
node: virtual_node(root_uid, "Shared with me".into(), 0),
visible: true,
},
);
assert_eq!(result.unwrap_err().code(), Errno::EIO.code());
assert_eq!(
RootListingSnapshot::capture(&state, super::ROOT_INO),
Some(snapshot),
"the resident listing must not change on DB failure"
);
assert_eq!(state.entries.len(), entries_before);
assert_eq!(state.by_uid.len(), by_uid_before);
assert_eq!(state.next_ino, next_ino_before);
assert_eq!(state.share_access, access_before);
assert!(
!state.by_uid.contains_key(&shared_with_me_uid()),
"the synthetic inode must not be interned before commit"
);
assert_eq!(
state
.flushed_db()
.state_str(super::SHARED_WITH_ME_NAME)
.unwrap(),
None
);
assert!(
state
.db
.node_by_uid(&shared_with_me_uid().to_string())
.unwrap()
.is_none()
);
assert_eq!(
state
.flushed_db()
.share_access(&shared_with_me_uid())
.unwrap(),
None
);
let hits = state
.flushed_db()
.search("AtomicRollbackFindable", 10)
.unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(
hits[0].path, path_before,
"the descendant's indexed path must roll back with the failed pin, \
not keep the synthetic ancestor the publication tried to add"
);
}
#[test]
fn open_unlinked_and_revoked_residents_are_not_visible() {
let (mut state, _dir) = rooted_state("device-volume", "device");
let open_node =
node_helper_in_volume("device-volume", "open", Some("device"), "open.txt", false);
let open_uid = open_node.uid.clone();
let open_ino = state.intern(super::ROOT_INO, open_node);
state.children.insert(super::ROOT_INO, vec![open_ino]);
state.entries.get_mut(&open_ino).unwrap().open_count = 1;
let revoked_node =
node_helper_in_volume("device-volume", "revoked", Some("device"), "revoked", true);
let revoked_uid = revoked_node.uid.clone();
let revoked_ino = state.intern(super::ROOT_INO, revoked_node);
let child_node = node_helper_in_volume(
"device-volume",
"revoked-child",
Some("revoked"),
"child.txt",
false,
);
let child_uid = child_node.uid.clone();
let child_ino = state.intern(revoked_ino, child_node);
state
.children
.get_mut(&super::ROOT_INO)
.unwrap()
.push(revoked_ino);
state.children.insert(revoked_ino, vec![child_ino]);
assert!(visible(&state, &open_uid));
assert!(visible(&state, &child_uid));
state.unlink_mem(&open_uid);
assert!(
!visible(&state, &open_uid),
"an open inode retained after unlink is not addressable"
);
state.hide_shared_root(&revoked_uid);
assert!(!visible(&state, &revoked_uid));
assert!(
!visible(&state, &child_uid),
"a retained descendant beneath a revoked root is not reachable"
);
}
#[test]
fn foreign_broken_and_cyclic_residents_are_not_visible() {
let (mut state, _dir) = rooted_state("own-volume", "root");
let foreign_node =
node_helper_in_volume("foreign-volume", "shared", Some("root"), "shared", true);
let foreign_uid = foreign_node.uid.clone();
let foreign_ino = state.intern(super::ROOT_INO, foreign_node);
let broken_node =
node_helper_in_volume("own-volume", "broken", Some("root"), "broken", true);
let broken_uid = broken_node.uid.clone();
let _broken_ino = state.intern(super::ROOT_INO, broken_node);
let cycle_a_node = node_helper_in_volume("own-volume", "cycle-a", Some("root"), "a", true);
let cycle_a_uid = cycle_a_node.uid.clone();
let cycle_a = state.intern(super::ROOT_INO, cycle_a_node);
let cycle_b_node =
node_helper_in_volume("own-volume", "cycle-b", Some("cycle-a"), "b", true);
let cycle_b = state.intern(cycle_a, cycle_b_node);
let cycle_b_uid = state.entries[&cycle_b].uid.clone();
let cycle_a_entry = state.entries.get_mut(&cycle_a).unwrap();
cycle_a_entry.parent = cycle_b;
cycle_a_entry.node.parent_uid = Some(cycle_b_uid);
state.children.insert(super::ROOT_INO, vec![foreign_ino]);
state.children.insert(cycle_a, vec![cycle_b]);
state.children.insert(cycle_b, vec![cycle_a]);
assert!(
!visible(&state, &foreign_uid),
"incoming-share residents are outside the mounted root's volume"
);
assert!(
!visible(&state, &broken_uid),
"an interned node without a visible parent dentry is not reachable"
);
assert!(
!visible(&state, &cycle_a_uid),
"a cyclic resident parent chain is not reachable"
);
}
#[test]
fn resolve_anywhere_prefers_resident_state_then_falls_back_to_mirror() {
let resident = resolve_anywhere_with(
"vol~resident",
|_| true,
|_| panic!("resident uid must not query mirror bookkeeping"),
)
.unwrap();
assert_eq!(resident.to_string(), "vol~resident");
let mirror = resolve_anywhere_with("vol~mirror", |_| false, |_| Ok(true)).unwrap();
assert_eq!(mirror.to_string(), "vol~mirror");
}
#[test]
fn resolve_anywhere_classifies_invalid_missing_and_db_errors() {
for raw_uid in ["not-a-uid", "~link", "vol~", "vol~link~extra"] {
let invalid = resolve_anywhere_with(
raw_uid,
|_| panic!("invalid uid must not query live state"),
|_| panic!("invalid uid must not query mirror bookkeeping"),
)
.unwrap_err();
assert_eq!(
invalid.kind,
pdfs_core::control::ErrorKind::Invalid,
"{raw_uid}"
);
}
for raw_uid in ["local~pending", "virtual~sharedwithme"] {
let reserved = resolve_anywhere_with(
raw_uid,
|_| panic!("reserved uid must not query live state"),
|_| panic!("reserved uid must not query mirror bookkeeping"),
)
.unwrap_err();
assert_eq!(
reserved.kind,
pdfs_core::control::ErrorKind::Invalid,
"{raw_uid}"
);
}
let missing = resolve_anywhere_with("vol~missing", |_| false, |_| Ok(false)).unwrap_err();
assert_eq!(missing.kind, pdfs_core::control::ErrorKind::NotFound);
let db_error = resolve_anywhere_with(
"vol~db",
|_| false,
|_| Err(pdfs_core::Error::Other("lookup failed".into())),
)
.unwrap_err();
assert_eq!(db_error.kind, pdfs_core::control::ErrorKind::Internal);
assert!(db_error.message.contains("lookup failed"));
}
/// An event feed carries no revision id, so the only thing that tells a
/// remote change apart from the echo of one this daemon just made is this
/// bookkeeping. Getting it wrong is expensive in both directions: too eager
/// and we evict the bytes we uploaded and re-download them from an API that
/// may still be serving the previous revision (a SQLite file on the mount
/// read back `malformed`); too generous and a real change from another device
/// is ignored.
#[test]
fn a_self_change_is_claimed_once_and_expires() {
let mut changes = HashMap::new();
let uid = parse_node_uid("vol~link").unwrap();
let other = parse_node_uid("vol~other").unwrap();
let t0 = 1_000_000;
assert!(
!take_self_change(&mut changes, &uid, t0),
"a node we never touched is never ours"
);
note_self_change(&mut changes, &uid, t0);
assert!(
take_self_change(&mut changes, &uid, t0 + 5_000),
"the echo of our own change is claimed"
);
assert!(
!take_self_change(&mut changes, &uid, t0 + 6_000),
"and only once, so a later foreign change still applies"
);
// A create followed by a revision upload is two changes to one node, and
// the feed reports both.
note_self_change(&mut changes, &uid, t0);
note_self_change(&mut changes, &uid, t0);
assert!(take_self_change(&mut changes, &uid, t0 + 1));
assert!(take_self_change(&mut changes, &uid, t0 + 2));
assert!(!take_self_change(&mut changes, &uid, t0 + 3));
// Past the window the attribution is a guess, and a wrong guess leaves
// the user looking at a stale file.
note_self_change(&mut changes, &uid, t0);
assert!(!take_self_change(
&mut changes,
&uid,
t0 + SELF_CHANGE_TTL_MS
));
assert!(
!changes.contains_key(&uid),
"an expired change is dropped, not left to be re-tested"
);
// An echo that never arrives must not accumulate for the life of the
// daemon; the next recorded change prunes it.
note_self_change(&mut changes, &uid, t0);
note_self_change(&mut changes, &other, t0 + SELF_CHANGE_TTL_MS);
assert_eq!(
changes.len(),
1,
"recording a change prunes the ones whose echo never came"
);
}
/// The predicate must answer *only* for a dead FUSE connection. A healthy
/// directory and an absent path are both "not stale" — widening it to any
/// `metadata` error would make the on-demand restore lazily unmount paths
/// that are simply missing.
#[test]
fn is_stale_mount_is_narrow() {
let dir = std::env::temp_dir().join(format!("pdfs-stale-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
assert!(!is_stale_mount(&dir));
assert!(!is_stale_mount(&dir.join("no-such-entry")));
let _ = std::fs::remove_dir_all(&dir);
}
/// Flatten `segments` into a readable form for assertions.
fn segs(iv: &Intervals, start: u64, end: u64) -> Vec<(u64, u64, bool)> {
iv.segments(start, end)
}
/// The conflict copy has to stay openable, so the extension survives — and
/// the shape matches the sync engine's `conflict_path` so the two features
/// name the same situation the same way.
#[test]
fn conflict_name_keeps_the_extension() {
assert_eq!(
conflict_name_as("notes.txt", "laptop", 1700),
"notes (sync-conflict laptop 1700).txt"
);
assert_eq!(
conflict_name_as("README", "laptop", 42),
"README (sync-conflict laptop 42)"
);
assert_eq!(
conflict_name_as("archive.tar.gz", "laptop", 7),
"archive.tar (sync-conflict laptop 7).gz",
"only the last extension is one, as everywhere else"
);
assert_eq!(
conflict_name_as("notes.txt", "", 1700),
"notes (sync-conflict 1700).txt"
);
assert_eq!(
conflict_name("notes.txt", 1700),
conflict_name_as("notes.txt", conflict_label(), 1700)
);
}
/// A hostname goes into a file name, so it is cut down to one safe token
/// (B251).
#[test]
fn a_hostname_becomes_one_safe_token() {
assert_eq!(conflict_label_of("laptop"), "laptop");
assert_eq!(conflict_label_of("my-pc.local"), "my-pc.local");
assert_eq!(conflict_label_of("Linux device"), "Linux-device");
assert_eq!(conflict_label_of("a/b) c"), "a-b--c");
assert_eq!(conflict_label_of(" "), "");
assert_eq!(conflict_label_of(&"x".repeat(40)).len(), 32);
}
#[test]
fn append_merges_into_one_run() {
// Sequential writes (an append) coalesce into a single authored range.
let mut iv = Intervals::default();
iv.add(0, 10);
iv.add(10, 20);
iv.add(20, 25);
assert_eq!(segs(&iv, 0, 25), vec![(0, 25, true)]);
}
#[test]
fn partial_overwrite_leaves_base_gap() {
// Author [0,4) and [8,12); [4,8) stays base. A read of [0,12) must stitch
// authored / base / authored in order.
let mut iv = Intervals::default();
iv.add(0, 4);
iv.add(8, 12);
assert_eq!(
segs(&iv, 0, 12),
vec![(0, 4, true), (4, 8, false), (8, 12, true)]
);
}
#[test]
fn overlapping_writes_coalesce() {
let mut iv = Intervals::default();
iv.add(0, 10);
iv.add(5, 15);
iv.add(14, 20);
assert_eq!(segs(&iv, 0, 20), vec![(0, 20, true)]);
}
#[test]
fn segments_clamp_to_request_window() {
let mut iv = Intervals::default();
iv.add(0, 100);
// A sub-window of one big authored range is a single authored segment.
assert_eq!(segs(&iv, 20, 50), vec![(20, 50, true)]);
// A window entirely outside any authored range is all base.
let empty = Intervals::default();
assert_eq!(segs(&empty, 0, 8), vec![(0, 8, false)]);
}
#[test]
fn truncate_shrink_drops_tail() {
// Grow-then-shrink: clip removes/truncates authored ranges past the end.
let mut iv = Intervals::default();
iv.add(0, 100);
iv.clip(40);
assert_eq!(segs(&iv, 0, 40), vec![(0, 40, true)]);
// Authored ranges wholly past the new end disappear.
let mut iv2 = Intervals::default();
iv2.add(0, 10);
iv2.add(50, 60);
iv2.clip(40);
assert_eq!(segs(&iv2, 0, 40), vec![(0, 10, true), (10, 40, false)]);
}
#[test]
fn truncate_extend_authors_zero_tail() {
// setattr grow claims the new tail as authored (defined zeros), so commit
// never pulls it from the base.
let mut iv = Intervals::default();
iv.add(0, 10); // base content authored over
let old_len = 10u64;
let new_len = 30u64;
iv.add(old_len, new_len);
assert_eq!(segs(&iv, 0, 30), vec![(0, 30, true)]);
}
fn pending_revision(dir: &TestDir, bytes: &[u8], complete: bool) -> PendingRevision {
let path = dir.0.join("pending");
std::fs::write(&path, bytes).unwrap();
PendingRevision {
path,
meta: StagedWrite {
uid: "vol~file".into(),
len: bytes.len() as u64,
base_size: 3,
base_mtime: 42,
authored: if complete {
vec![(0, bytes.len() as u64)]
} else {
vec![(3, bytes.len() as u64)]
},
complete,
based_on: Some(Baseline {
mtime: 42,
size: 3,
hash: None,
revision_id: None,
}),
},
}
}
#[test]
fn complete_pending_revision_can_seed_immediate_shrink() {
let (_, dir) = state_test_helper();
let pending = pending_revision(&dir, b"0123456789", true);
let destination_path = dir.0.join("shrink");
let destination = std::fs::File::create(&destination_path).unwrap();
copy_pending_for_truncate(&pending, &destination).unwrap();
destination.set_len(4).unwrap();
assert_eq!(std::fs::read(destination_path).unwrap(), b"0123");
}
#[test]
fn complete_pending_revision_can_seed_immediate_growth_and_zero() {
let (_, dir) = state_test_helper();
let pending = pending_revision(&dir, b"0123456789", true);
let destination_path = dir.0.join("grow");
let destination = std::fs::File::create(&destination_path).unwrap();
copy_pending_for_truncate(&pending, &destination).unwrap();
destination.set_len(14).unwrap();
assert_eq!(
std::fs::read(&destination_path).unwrap(),
b"0123456789\0\0\0\0"
);
destination.set_len(0).unwrap();
assert!(std::fs::read(destination_path).unwrap().is_empty());
}
#[test]
fn incomplete_pending_revision_is_not_a_safe_truncate_base() {
let (_, dir) = state_test_helper();
let pending = pending_revision(&dir, b"\0\0\0authored", false);
assert!(!pending.meta.complete);
assert_ne!(pending.meta.authored, vec![(0, pending.meta.len)]);
}
#[test]
fn online_combined_cross_directory_rename_stays_synchronous() {
assert!(!rename_needs_queue(true, false, true, true));
assert!(rename_needs_queue(false, false, false, true));
assert!(rename_needs_queue(true, true, true, false));
}
#[test]
fn one_step_online_namespace_changes_stay_synchronous() {
assert!(!rename_needs_queue(true, false, false, true));
assert!(!rename_needs_queue(true, false, true, false));
assert!(!rename_needs_queue(true, false, false, false));
}
#[test]
fn fuse_names_enforce_linux_component_limits() {
use std::ffi::OsStr;
assert_eq!(
fuse_name(OsStr::new("x".repeat(255).as_str()))
.unwrap()
.len(),
255
);
assert_eq!(
fuse_name(OsStr::new("x".repeat(256).as_str()))
.expect_err("256-byte component must be refused")
.code(),
libc::ENAMETOOLONG
);
assert_eq!(
fuse_name(OsStr::new("."))
.expect_err("dot is not an ordinary component")
.code(),
libc::EINVAL
);
assert_eq!(
fuse_name(OsStr::new("unicodé-文件")).unwrap(),
"unicodé-文件"
);
}
#[test]
fn local_trash_tombstones_hide_stale_remote_entries() {
let folder_uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("folder"));
let node = node_helper("child", "folder", "child", false);
assert!(node_visible(
&node,
&folder_uid,
&std::collections::HashSet::new()
));
assert!(!node_visible(
&node,
&folder_uid,
&std::collections::HashSet::from([node.uid.clone()])
));
}
#[test]
fn a_remote_name_that_is_not_one_path_component_is_left_out() {
let folder_uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("folder"));
let none = std::collections::HashSet::new();
for name in [
"a/b",
"../../.bashrc",
"..",
".",
"",
"a\0b",
&"x".repeat(256),
] {
let node = node_helper("child", "folder", name, false);
assert!(!node_visible(&node, &folder_uid, &none), "{name:?}");
}
for name in [".hidden", "a..b", "unicodé-文件", &"x".repeat(255)] {
let node = node_helper("child", "folder", name, false);
assert!(node_visible(&node, &folder_uid, &none), "{name:?}");
}
}
#[test]
fn denied_queue_guard_preserves_accepted_bytes_and_pending_count() {
use std::cell::Cell;
let (mut state, _dir) = state_test_helper();
let root = state.intern(0, node_helper("root", "none", "root", true));
let shared = state.intern(root, node_helper("shared", "root", "shared", true));
state.entries.get_mut(&shared).unwrap().access = Access::Viewer;
let uid = state.entries[&shared].uid.clone();
let db = state.flushed_db();
let before = db.pending_ops().unwrap().len();
let preserved = Cell::new(false);
let result = preserve_on_access_denied(
require_uid_access(&db, &uid, &[Access::Viewer]),
true,
|| preserved.set(true),
);
assert_eq!(result.unwrap_err().code(), libc::EACCES);
assert!(preserved.get(), "accepted bytes must enter recovery");
assert_eq!(db.pending_ops().unwrap().len(), before);
let untouched = Cell::new(false);
assert!(preserve_on_access_denied(Ok(()), true, || untouched.set(true)).is_ok());
assert!(
!untouched.get(),
"a permitted write must continue to the normal queue path"
);
// The same fail-closed rule applies before hydration and to a stale UID.
db.set_share_access(&uid, Access::Viewer).unwrap();
assert_eq!(
require_uid_access(&db, &uid, &[]).unwrap_err().code(),
libc::EACCES
);
let stale = NodeUid::new(VolumeId::from("vol"), LinkId::from("stale"));
assert_eq!(
require_uid_access(&db, &stale, &[]).unwrap_err().code(),
libc::EACCES
);
// Same refusal on the syscall path, but a distinguishable one
// underneath: the drain asks the remote about `Unknown` rather than
// waiting on a permission change (B83).
assert_eq!(
uid_write_authority(&db, &stale, &[]),
WriteAuthority::Unknown,
"a uid the tree has no row for is unknown, not denied"
);
assert_eq!(
uid_write_authority(&db, &uid, &[]),
WriteAuthority::Denied,
"a uid the tree knows and refuses is denied"
);
assert_eq!(db.pending_ops().unwrap().len(), before);
}
#[test]
fn persisted_and_live_queue_authorities_must_both_be_writable() {
let (mut state, _dir) = state_test_helper();
let root = state.intern(0, node_helper("root", "none", "root", true));
let node = state.intern(root, node_helper("shared", "root", "shared", true));
let uid = state.entries[&node].uid.clone();
let db = state.flushed_db();
db.set_share_access(&uid, Access::Viewer).unwrap();
assert_eq!(
require_uid_access(&db, &uid, &[Access::Editor])
.unwrap_err()
.code(),
libc::EACCES,
"a persisted downgrade must beat stale writable live state"
);
db.set_share_access(&uid, Access::Editor).unwrap();
assert_eq!(
require_uid_access(&db, &uid, &[Access::Viewer])
.unwrap_err()
.code(),
libc::EACCES,
"a live downgrade must beat restored writable persisted state"
);
assert!(require_uid_access(&db, &uid, &[Access::Editor]).is_ok());
}
#[test]
fn shared_listing_provenance_must_agree_before_editor_is_granted() {
use proton_drive_rs::ShareMembership;
use proton_drive_rs::proton_sdk::ids::ShareMembershipId;
let uid = NodeUid::new(VolumeId::from("foreign"), LinkId::from("root"));
let share_a = ShareId::from("share-a");
let share_b = ShareId::from("share-b");
let parent = shared_with_me_uid();
let item = |share_id: ShareId| SharedWithMeItem {
uid: uid.clone(),
share_id,
};
let node = |share_id: Option<ShareId>, permissions: i32| {
let mut node = node_helper("root", "none", "Shared", true);
node.uid = uid.clone();
node.membership = share_id.map(|share_id| ShareMembership {
share_id,
membership_id: ShareMembershipId::from("membership"),
permissions,
invite_time: None,
inviter_email: None,
inviter_verification: None,
});
node
};
let accepted =
accepted_share_provenance(vec![item(share_a.clone()), item(share_a.clone())]);
assert_eq!(
prepare_shared_roots(&accepted, vec![node(Some(share_a.clone()), 6)], &parent)[0]
.access,
Access::Editor
);
for materialized in [
node(None, 6),
node(Some(share_b.clone()), 6),
node(Some(share_a.clone()), 12345),
] {
assert_eq!(
prepare_shared_roots(&accepted, vec![materialized], &parent)[0].access,
Access::Viewer
);
}
let conflicting = accepted_share_provenance(vec![item(share_a), item(share_b)]);
assert_eq!(
prepare_shared_roots(
&conflicting,
vec![node(Some(ShareId::from("share-a")), 6)],
&parent
)[0]
.access,
Access::Viewer,
"duplicate UID provenance disagreement must fail closed"
);
}
#[test]
fn virtual_bulk_upload_destination_is_denied_by_runtime_authority() {
let (mut state, _dir) = state_test_helper();
let root = state.intern(0, node_helper("root", "none", "My Files", true));
let mut virtual_node = node_helper("sharedwithme", "root", "Shared with me", true);
virtual_node.uid = shared_with_me_uid();
state
.flushed_db()
.set_share_access(&virtual_node.uid, Access::Viewer)
.unwrap();
state
.share_access
.insert(virtual_node.uid.clone(), Access::Viewer);
let virtual_ino = state.intern(root, virtual_node);
let uid = state.entries[&virtual_ino].uid.clone();
assert_eq!(
require_uid_access(&state.db, &uid, &[Access::Viewer])
.unwrap_err()
.code(),
libc::EACCES
);
}
fn function_source<'a>(source: &'a str, signature: &str) -> &'a str {
let start = source.find(signature).expect("function signature exists");
let body = &source[start..];
let open = body.find('{').expect("function body starts");
let mut depth = 0usize;
for (offset, byte) in body.as_bytes()[open..].iter().enumerate() {
match byte {
b'{' => depth += 1,
b'}' => {
depth -= 1;
if depth == 0 {
return &body[..open + offset + 1];
}
}
_ => {}
}
}
panic!("function body closes")
}
fn assert_before(source: &str, first: &str, second: &str) {
let first = source.find(first).expect("first operation exists");
let second = source.find(second).expect("second operation exists");
assert!(first < second, "{first} must precede {second}");
}
#[test]
fn production_queue_guards_precede_mutating_work() {
let source = include_str!("lib.rs");
let revision = function_source(source, "fn queue_revision(");
assert_before(revision, "if !h.dirty", "preserve_on_access_denied");
assert_before(revision, "preserve_on_access_denied", "fill_gaps");
let staged = function_source(source, "fn enqueue_staged_write(");
assert_before(staged, "preserve_on_access_denied", "self.pending.lock()");
assert_before(
staged,
"preserve_on_access_denied",
"self.cache.stage_write",
);
let truncate = function_source(source, "fn queue_truncate(");
assert_before(truncate, "require_uid_writable", "self.pending.lock()");
assert_before(truncate, "require_uid_writable", "create_scratch");
for (signature, first_side_effect) in [
("fn queue_local_node(", "add_local_row"),
("fn queue_rename(", "queue_rename_authorized"),
("fn queue_trash(", "replace_ops_with_trash"),
] {
let function = function_source(source, signature);
assert_before(function, "require_uid_writable", first_side_effect);
}
let trash = function_source(source, "fn queue_trash(");
assert_before(trash, "replace_ops_with_trash", "release_dropped_ops");
// §5, parallel drain: every path that unlinks a staged blob or hands
// the same node's work to another worker has to stop the upload that
// may be reading those bytes *first*. Late is a superseded revision on
// the wire, or two workers deciding whether the file exists.
for (signature, mutation) in [
("fn enqueue_staged_write(", "attach_blob_to_create"),
("fn queue_trash(", "replace_ops_with_trash"),
("fn discard_queued_ops(", "delete_ops_for_uid"),
("fn withdraw_queued_ops(", "delete_ops_for_uid"),
] {
let function = function_source(source, signature);
assert_before(function, "cancel_upload", mutation);
}
let filesystem = include_str!("filesystem.rs");
for signature in ["fn serve_unlink(", "fn serve_rmdir("] {
let function = function_source(filesystem, signature);
assert_before(function, "require_writable", "lookup_child");
}
let rename = function_source(filesystem, "fn serve_rename(");
assert_before(rename, "require_rename_access", "remove_replaced");
assert_before(rename, "remove_replaced", "queue_rename_authorized");
for (signature, gate, remote_call) in [
("fn serve_create(", "require_uid_writable", "upload_file"),
("fn serve_mkdir(", "require_uid_writable", "create_folder"),
("fn trash_child(", "require_uid_writable", "trash_nodes"),
("fn serve_rename(", "require_rename_access", "rename_remote"),
] {
let function = function_source(filesystem, signature);
assert_before(function, gate, remote_call);
}
let control_mkdir = function_source(source, "fn create_folder(");
assert_before(
control_mkdir,
"require_uid_writable",
".create_folder(&parent_uid",
);
for (signature, gate, remote_call, settle) in [
(
"fn rename(&self, rel: &Path",
"require_rename_access",
".rename_remote",
"self.relocate_everywhere(&uid",
),
(
"fn move_to(&self, rel: &Path",
"require_rename_access",
".move_remote",
"self.relocate_everywhere(&uid",
),
(
"fn delete(&self, rel: &Path",
"require_node_parent_access",
".trash_nodes",
"self.forget_everywhere(&uid)",
),
] {
let function = function_source(source, signature);
assert_before(function, "source_parent_uid", gate);
assert_before(function, gate, remote_call);
// The kernel did not see this change, so it has to be told the
// old name is gone (`docs/BUGS.md` B114).
assert_before(function, remote_call, settle);
}
let upload = include_str!("upload.rs");
let run_uploads = function_source(upload, "async fn run_uploads(");
assert_before(run_uploads, "require_uid_writable", ".upload_file_from");
let bulk = function_source(upload, "pub(super) fn upload_paths(");
assert_before(bulk, "require_uid_writable", "collect_uploads");
let folder = function_source(upload, "fn ensure_remote_folder(");
assert_before(folder, "require_uid_writable(parent_uid)", ".create_folder");
}
#[test]
fn a_listing_for_the_cli_settles_sizes_before_reading_them() {
// A fresh listing holds ciphertext sizes until its upgrade lands
// (`docs/BUGS.md` B115).
let list = function_source(include_str!("lib.rs"), "fn list_dir(&self, rel: &Path)");
assert_before(
list,
"self.ensure_children(ino)",
"self.settle_listing_sizes(ino)",
);
assert_before(list, "self.settle_listing_sizes(ino)", "node_size(&e.node)");
}
#[test]
fn a_replaced_node_survives_its_open_handles() {
// A rename over a file someone is reading must leave that reader on the
// replaced bytes; forgetting the inode outright made its next read EIO.
let source = include_str!("lib.rs");
let replaced = function_source(source, "fn remove_replaced(");
assert!(replaced.contains("forget_or_unlink(uid)"));
assert!(!replaced.contains("st.forget(uid)"));
assert_before(
replaced,
"if self.is_open_anywhere(uid)",
"self.discard_queued_ops(uid)",
);
assert_before(replaced, "if open_now", "self.cache.evict(uid)");
// ...but its queued ops go at once, open or not: a revision drained
// after the trash landed on the trashed node and kept its name taken.
assert_eq!(replaced.matches("self.withdraw_queued_ops(uid)").count(), 2);
}
#[test]
fn a_write_closed_as_its_create_lands_is_queued() {
// The release followed a landing create, found none, and the create
// landed before it asked for the create op. Finding none, it took the
// file for one unlinked before it was created and dropped the write
// (B176). The landing removes the op and moves the row at once, so a
// missing op is checked against the row.
let revision = function_source(include_str!("lib.rs"), "fn queue_revision(");
assert_before(
revision,
"self.db.has_create_op(",
"self.follow_landed_create(h)",
);
assert_before(
revision,
"self.follow_landed_create(h)",
"dropping revision",
);
}
#[test]
fn a_replaced_node_put_back_shows_in_the_mount_again() {
// The restore after a failed rename left the node hidden and out of
// the listing, so a create by its name failed with ENOENT (B124).
let restore = function_source(include_str!("lib.rs"), "fn restore_replaced(");
assert_before(
restore,
"self.drive.restore_nodes(",
"self.hidden.lock().remove(uid)",
);
assert!(restore.contains("st.invalidate_listing(ino)"));
assert!(restore.contains("notify.inval_entry(ino, name.to_string())"));
}
#[test]
fn a_size_upgrade_leaves_the_page_cache_alone() {
// Reads resolve provisional sizes inline, while the kernel holds the
// page lock of that read; dropping pages here waited on it forever.
let upgrade = function_source(include_str!("lib.rs"), "fn apply_size_upgrade(");
assert!(upgrade.contains("notifier.inval_attrs(ino)"));
assert!(!upgrade.contains("notifier.inval_inode("));
let read = function_source(include_str!("filesystem.rs"), " fn read(");
assert!(read.contains("core.resolve_provisional_size(ino.0, &uid)"));
}
fn api_error(code: super::ResponseCode) -> super::ProtonError {
super::ProtonError::Api(proton_drive_rs::proton_sdk::error::ProtonApiError {
code,
http_status: 422,
message: "This file or folder was out of date, rename failed.".into(),
details: None,
})
}
#[test]
fn a_rename_called_out_of_date_is_retried_until_it_lands() {
let mut calls = 0;
let mut waits = 0;
let result = super::retry_out_of_date(
|| {
calls += 1;
if calls < 3 {
Err(api_error(super::ResponseCode::InvalidRequirements))
} else {
Ok(())
}
},
|| waits += 1,
);
assert!(result.is_ok());
assert_eq!((calls, waits), (3, 2));
}
#[test]
fn a_rename_gives_up_on_other_errors_and_after_its_retries() {
let mut calls = 0;
let taken = super::retry_out_of_date(
|| {
calls += 1;
Err(api_error(super::ResponseCode::AlreadyExists))
},
|| {},
);
assert!(taken.is_err());
assert_eq!(calls, 1, "a taken name is not a stale read");
let mut calls = 0;
let stale = super::retry_out_of_date(
|| {
calls += 1;
Err(api_error(super::ResponseCode::InvalidRequirements))
},
|| {},
);
assert!(stale.is_err());
assert_eq!(calls, super::OUT_OF_DATE_RETRIES + 1);
}
#[test]
fn every_remote_rename_and_move_rides_out_a_stale_hash() {
for source in [
include_str!("lib.rs"),
include_str!("filesystem.rs"),
include_str!("drain.rs"),
include_str!("relocate.rs"),
] {
assert_eq!(
source.matches(concat!("drive.rename_node", "(")).count(),
usize::from(source.contains("fn rename_remote(")),
"call Core::rename_remote instead of the SDK rename directly"
);
assert_eq!(
source.matches(concat!("drive.move_node", "(")).count(),
usize::from(source.contains("fn move_remote(")),
"call Core::move_remote instead of the SDK move directly"
);
assert_eq!(
source
.matches(concat!("drive.move_nodes_streaming", "("))
.count(),
usize::from(source.contains("fn move_rename_remote(")),
"call Core::move_rename_remote instead of the SDK move directly"
);
}
}
#[test]
fn a_rename_or_move_drops_the_sdk_copy_and_expects_its_echo() {
// The SDK would send the old name hash with the node's next rename or
// move until the event poll caught up (B121), and the echo of the
// change, taken for someone else's, evicted the cached bytes (B123).
let change = function_source(include_str!("lib.rs"), "fn change_remote(");
assert!(
change.contains("self.forget_sdk_node(uid)"),
"forget the node before each retry"
);
assert!(change.contains("self.changed_remote(uid)"));
let moved = function_source(include_str!("lib.rs"), "fn move_rename_remote(");
assert!(moved.contains("self.change_remote(uid"));
let changed = function_source(include_str!("lib.rs"), "fn changed_remote(");
assert!(changed.contains("self.forget_sdk_node(uid)"));
assert!(changed.contains("self.note_self_change(uid)"));
}
#[test]
fn a_listing_changed_while_published_is_taken_again() {
// The root listed again after a restart while a lookup was publishing
// the "Shared with me" entry, and the lookup answered EAGAIN (B159).
let ensure = function_source(include_str!("lib.rs"), "fn ensure_children(");
assert_before(
ensure,
"for _ in 1..LISTING_ATTEMPTS",
"Errno::EAGAIN.code()",
);
assert_eq!(ensure.matches("self.list_children(ino)").count(), 2);
assert!(
!ensure.contains("?"),
"an attempt's error must not skip the retry"
);
}
#[test]
fn a_folder_lists_from_the_db_while_the_link_is_down() {
// A listing dropped as stale mid-run, then a drop: every create in that
// folder looks the name up first, and answered EIO for want of it.
let ensure = function_source(include_str!("lib.rs"), "fn list_children(");
assert_before(
ensure,
"if !self.is_online()",
"enumerate_folder_children_node_uids",
);
assert_eq!(ensure.matches("adopt_stale_listing(").count(), 3);
let online = function_source(include_str!("link.rs"), "fn mark_online(");
assert!(online.contains("self.relist_offline_listings();"));
}
#[test]
fn a_stale_shared_listing_is_fetched_off_the_dispatch_loop() {
// A resident listing that `list_children` would fetch again anyway
// must not be answered inline (B219).
let cached = function_source(include_str!("lib.rs"), "fn children_cached(");
assert!(cached.contains("SHARED_WITH_ME_SYNCED_MS"));
assert!(cached.contains("Self::shared_folder_freshness_key(&uid)"));
assert!(cached.contains("listing_needs_refresh("));
assert!(cached.contains("self.shared_listing_stale(&key)"));
}
#[test]
fn a_handler_does_not_wait_out_the_sdk_retry_budget() {
// Thumbnails, size upgrades and the share list held mount threads for
// minutes on a stalled link (B220).
let lib = include_str!("lib.rs");
let thumbnail = function_source(lib, "fn thumbnail(");
assert_before(thumbnail, "if !self.is_online()", "download_thumbnail");
assert!(thumbnail.contains(".block_on_bounded(self.client.download_thumbnail("));
let upgrade = function_source(lib, "fn run_size_upgrade(");
assert!(upgrade.contains("block_on_within(BATCH_CALL_TIMEOUT"));
let shared = function_source(lib, "fn ensure_shared_children(");
assert!(shared.contains("block_on_within(BATCH_CALL_TIMEOUT"));
assert!(!shared.contains("self.rt"));
}
#[test]
fn a_remote_read_gives_up_at_its_deadline() {
// The SDK bounds requests, not reads; a stalled link held one `read`
// for 34 minutes.
let remote = function_source(include_str!("reads.rs"), "fn read_range_remote(");
assert_before(
remote,
"let fetch = async",
"tokio::time::timeout(budget, fetch)",
);
assert!(remote.contains("let deadline = Instant::now() + READ_FETCH_TIMEOUT;"));
// Built inside the runtime: a FUSE worker has no reactor, and a timer
// built there panicked every remote read.
assert!(remote.contains("async { tokio::time::timeout(budget, fetch).await }"));
}
#[test]
fn a_kernel_notice_sent_from_the_runtime_leaves_its_timers_running() {
// A notice waits in the kernel for a read whose fetch waits on a
// timer of the same runtime (B130). Here the runtime's one worker sends
// it, and the timer has to fire for the notice to return.
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.unwrap();
let (fetched, fetch) = std::sync::mpsc::channel();
rt.spawn(async move {
tokio::time::sleep(Duration::from_millis(10)).await;
let _ = fetched.send(());
});
let notice =
rt.spawn(async move { off_the_runtime(|| fetch.recv_timeout(Duration::from_secs(5))) });
assert!(rt.block_on(notice).unwrap().is_ok());
// Every notice to a session goes through it.
let lib = include_str!("lib.rs");
for method in [
"fn inval_inode(&self",
"fn inval_attrs(&self",
"fn inval_entry(&self",
"fn delete(&self, parent: u64",
] {
let source = function_source(lib, method);
assert_eq!(source.matches("self.notifier.").count(), 1, "{method}");
assert!(
source.contains("off_the_runtime(|| self.notifier."),
"{method}"
);
}
}
#[test]
fn a_deadline_built_under_block_on_needs_no_ambient_runtime() {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.unwrap();
let answered = std::thread::spawn(move || {
rt.block_on(async { tokio::time::timeout(Duration::from_secs(5), async { 7 }).await })
})
.join()
.expect("a worker thread outside the runtime can wait on the deadline");
assert_eq!(answered.unwrap(), 7);
}
#[test]
fn a_trashed_open_node_sends_nothing_more() {
// Trashing a file under an open handle withdraws its queued upload at
// once; the bytes the handle reads go with the last close.
let withdraw = function_source(include_str!("lib.rs"), "fn withdraw_queued_ops(");
assert_before(withdraw, "self.pending_blob(uid)", "release_dropped_below");
assert!(!withdraw.contains("self.pending.lock().remove(uid)"));
let filesystem = include_str!("filesystem.rs");
let trash = function_source(filesystem, "fn trash_child(");
assert_eq!(
trash.matches("self.core.withdraw_queued_ops(&uid)").count(),
2
);
let release = function_source(filesystem, "fn finish_release(");
assert_before(
release,
"let withdrawn = self.core.pending_blob(&uid)",
"discard_queued_ops",
);
assert_before(release, "discard_queued_ops", "discard_staged(&blob)");
// A drain worker that claimed the op before it was withdrawn checks
// again before any byte leaves.
let drain = function_source(include_str!("drain.rs"), "pub(crate) fn drain_revision(");
let upload = &drain[drain.find("begin_cancellable_upload").unwrap()..];
assert_before(upload, "self.db.op_exists(op.id)", "File::open(&blob)");
}
#[test]
fn an_online_rename_writes_the_move_through_before_it_answers() {
// Through the raw lock, the move stayed in the state's outbox, and the
// next lookup rebuilt the destination from a database without it
// (B135).
let rename: String = function_source(include_str!("filesystem.rs"), "fn serve_rename(")
.split_whitespace()
.collect();
assert!(rename.contains("self.core.state().relocate("));
assert!(!rename.contains(".lock().relocate("));
}
#[test]
fn a_paused_drain_claims_nothing() {
// A claimed create counts as sent, so trashing its folder kept it as
// a trash, although the paused worker only handed it back (B178).
let worker: String =
function_source(include_str!("drain.rs"), "pub(crate) fn run_pending_drain(")
.split_whitespace()
.collect();
assert!(worker.contains("(online&&!paused).then(||self.db.claim_next_due_op(now))"));
}
#[test]
fn a_landed_upload_stays_pending_until_its_bytes_are_cached() {
let drain = function_source(include_str!("drain.rs"), "pub(crate) fn drain_revision(");
assert_before(
drain,
"let _landing = LandingUpload::begin(&self.landing_uploads)",
"self.db.delete_op(op.id)",
);
assert_before(drain, "self.db.delete_op(op.id)", ".store_file(&uid,");
let status = function_source(include_str!("control.rs"), "fn status_response(");
assert_before(
status,
"core.db.pending_op_counts()",
"core.landing_uploads.load(",
);
assert!(status.contains("queued.uploads.max(0) as u64 + landing"));
}
#[test]
fn replacement_rename_authorizes_source_and_both_parents() {
use std::cell::RefCell;
let source = NodeUid::new(VolumeId::from("vol"), LinkId::from("source"));
let old_parent = NodeUid::new(VolumeId::from("vol"), LinkId::from("old"));
let new_parent = NodeUid::new(VolumeId::from("vol"), LinkId::from("new"));
let checked = RefCell::new(Vec::new());
require_rename_access(
|uid| {
checked.borrow_mut().push(uid.to_string());
Ok(())
},
&source,
&old_parent,
&new_parent,
)
.unwrap();
assert_eq!(
checked.into_inner(),
vec!["vol~source", "vol~old", "vol~new"]
);
}
#[test]
fn writable_nested_node_cannot_mutate_a_viewer_parent_namespace() {
let source = NodeUid::new(VolumeId::from("vol"), LinkId::from("nested-editor"));
let viewer_parent = NodeUid::new(VolumeId::from("vol"), LinkId::from("viewer-parent"));
let writable_destination = NodeUid::new(VolumeId::from("vol"), LinkId::from("destination"));
let access = |uid: &NodeUid| {
if uid == &viewer_parent {
Err(Errno::EACCES)
} else {
Ok(())
}
};
assert_eq!(
require_node_parent_access(access, &source, &viewer_parent)
.unwrap_err()
.code(),
libc::EACCES,
"delete must authorize the source namespace"
);
assert_eq!(
require_rename_access(access, &source, &viewer_parent, &writable_destination,)
.unwrap_err()
.code(),
libc::EACCES,
"rename and move must authorize the original parent"
);
}
#[test]
fn access_handler_logic_matches_advertised_permissions() {
assert!(access_allowed(false, Access::Viewer, AccessFlags::F_OK));
assert!(access_allowed(false, Access::Viewer, AccessFlags::R_OK));
assert!(!access_allowed(false, Access::Viewer, AccessFlags::W_OK));
assert!(!access_allowed(false, Access::Owner, AccessFlags::X_OK));
assert!(access_allowed(true, Access::Viewer, AccessFlags::X_OK));
assert!(access_allowed(
true,
Access::Editor,
AccessFlags::R_OK | AccessFlags::W_OK | AccessFlags::X_OK
));
}
use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId};
use proton_drive_rs::{Node, NodeKind};
#[test]
fn test_posix_unlink_and_rmdir_checks() {
let (mut st, _dir) = state_test_helper();
let parent = st.intern(0, node_helper("parent_uid", "none", "parent", true));
let folder_child = st.intern(
parent,
node_helper("folder_child", "parent_uid", "subfolder", true),
);
let file_child = st.intern(
parent,
node_helper("file_child", "parent_uid", "file.txt", false),
);
st.children.insert(parent, vec![folder_child, file_child]);
// unlink on folder -> EISDIR check
let folder_entry = st.entries.get(&folder_child).unwrap();
assert!(folder_entry.node.is_folder(), "folder_child is a folder");
// rmdir on file -> ENOTDIR check
let file_entry = st.entries.get(&file_child).unwrap();
assert!(!file_entry.node.is_folder(), "file_child is a file");
// rmdir on non-empty parent -> ENOTEMPTY check
assert!(st.has_children(parent), "parent is not empty");
}
struct TestDir(std::path::PathBuf);
impl Drop for TestDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn state_test_helper() -> (crate::state::State, TestDir) {
use std::sync::atomic::{AtomicU64, Ordering};
static N: AtomicU64 = AtomicU64::new(0);
let dir_path = std::env::temp_dir().join(format!(
"pdfs-lib-test-{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(&dir_path).unwrap();
let db = pdfs_core::db::Db::open(&dir_path.join("cache.db")).unwrap();
let share_access = db.all_share_access().unwrap();
let st = crate::state::State::new(std::sync::Arc::new(db), share_access);
(st, TestDir(dir_path))
}
fn rooted_state(volume: &str, root_link: &str) -> (crate::state::State, TestDir) {
let (mut state, dir) = state_test_helper();
let root = node_helper_in_volume(volume, root_link, None, "root", true);
// At the root inode, as `ProtonFs::new` puts it.
state.by_uid.insert(root.uid.clone(), super::ROOT_INO);
state.entries.insert(
super::ROOT_INO,
crate::state::Entry {
uid: root.uid.clone(),
parent: super::ROOT_INO,
node: root,
access: Access::Owner,
lookup_count: 1,
open_count: 0,
unlinked: false,
},
);
state.roots.insert(super::ROOT_INO);
(state, dir)
}
fn node_helper_in_volume(
volume: &str,
id: &str,
parent: Option<&str>,
name: &str,
is_dir: bool,
) -> Node {
let mut node = node_helper(id, "none", name, is_dir);
node.uid = NodeUid::new(VolumeId::from(volume), LinkId::from(id));
node.parent_uid =
parent.map(|parent| NodeUid::new(VolumeId::from(volume), LinkId::from(parent)));
node
}
/// A rename or move made over the control socket keeps the node and its
/// row, so the queue and the drain still know it (`docs/BUGS.md` B117),
/// and tells the kernel the old name is gone (`docs/BUGS.md` B114).
#[test]
fn a_control_move_keeps_the_node_and_its_row_and_drops_the_old_name() {
use super::{KernelNotice, NotifyBatch, relocate_in_mount};
let (mut state, _dir) = state_test_helper();
let src = state.intern(0, node_helper("src", "none", "src", true));
let dst = state.intern(0, node_helper("dst", "none", "dst", true));
let moved = node_helper("moved", "src", "a.txt", false);
let uid = moved.uid.clone();
let ino = state.intern(src, moved);
state.children.insert(src, vec![ino]);
state.children.insert(dst, vec![]);
let dst_uid = state.entries[&dst].uid.clone();
let mut notify = NotifyBatch::default();
let outcome = relocate_in_mount(&mut state, &mut notify, &uid, &dst_uid, "b.txt", false);
assert_eq!(outcome, Some(("a.txt".to_string(), true)));
assert_eq!(state.by_uid.get(&uid), Some(&ino), "the inode survives");
assert_eq!(state.entries[&ino].parent, dst);
let row = state
.flushed_db()
.node_by_uid(&uid.to_string())
.unwrap()
.expect("the row survives the move");
assert_eq!(row.name, "b.txt");
assert_eq!(row.parent_uid.as_ref(), Some(&dst_uid));
assert!(notify.0.iter().any(|notice| matches!(
notice,
KernelNotice::InvalEntry { parent, name } if *parent == src && name == "a.txt"
)));
// A mount that does not show the new parent forgets the node, and the
// kernel the name, but the row stays for the caller to move.
let elsewhere = node_helper("elsewhere", "none", "elsewhere", true).uid;
let mut notify = NotifyBatch::default();
let outcome = relocate_in_mount(&mut state, &mut notify, &uid, &elsewhere, "c.txt", false);
assert_eq!(outcome, Some(("b.txt".to_string(), false)));
assert!(!state.by_uid.contains_key(&uid));
assert!(notify.0.iter().any(|notice| matches!(
notice,
KernelNotice::Delete { parent, child, name } if *parent == dst && *child == ino && name == "b.txt"
)));
assert!(
state
.flushed_db()
.node_by_uid(&uid.to_string())
.unwrap()
.is_some()
);
}
/// A control-socket move that is queued keeps both listings: Drive still
/// has the old name, so listing again would bring it back (B149).
#[test]
fn a_queued_control_move_keeps_both_listings() {
use super::{KernelNotice, NotifyBatch, relocate_in_mount};
let (mut state, _dir) = state_test_helper();
let src = state.intern(0, node_helper("src", "none", "src", true));
let dst = state.intern(0, node_helper("dst", "none", "dst", true));
let moved = node_helper("moved", "src", "a.txt", false);
let uid = moved.uid.clone();
let ino = state.intern(src, moved);
state.children.insert(src, vec![ino]);
state.children.insert(dst, vec![]);
let dst_uid = state.entries[&dst].uid.clone();
let mut notify = NotifyBatch::default();
let outcome = relocate_in_mount(&mut state, &mut notify, &uid, &dst_uid, "b.txt", true);
assert_eq!(outcome, Some(("a.txt".to_string(), true)));
assert_eq!(state.children.get(&src), Some(&vec![]));
assert_eq!(state.children.get(&dst), Some(&vec![ino]));
assert_eq!(state.entries[&ino].node.name, "b.txt");
for folder in [src, dst] {
assert!(
notify.0.iter().any(
|notice| matches!(notice, KernelNotice::InvalInode(ino) if *ino == folder)
)
);
}
assert!(notify.0.iter().any(|notice| matches!(
notice,
KernelNotice::InvalEntry { parent, name } if *parent == src && name == "a.txt"
)));
}
#[test]
fn queued_trash_of_open_file_survives_last_release_until_atomic_completion() {
let (mut state, _dir) = state_test_helper();
let parent = state.intern(0, node_helper("parent", "none", "parent", true));
let file_node = node_helper("open-trash", "parent", "dirty.txt", false);
let uid = file_node.uid.clone();
let ino = state.intern(parent, file_node);
state.children.insert(parent, vec![ino]);
state.entries.get_mut(&ino).unwrap().open_count = 1;
// The typed DB operation atomically replaces the old revision with the
// one intent that must survive the open handle's final release.
state
.flushed_db()
.enqueue_op(&PendingOp {
id: 0,
kind: OP_REVISION.to_string(),
uid: uid.to_string(),
parent_uid: None,
name: None,
blob_path: None,
meta_json: None,
created_at: 1,
attempts: 0,
last_error: None,
next_attempt_at: 0,
})
.unwrap();
let (_, blobs) = state
.flushed_db()
.replace_ops_with_trash(&uid.to_string(), "dirty.txt", 2)
.unwrap();
assert!(blobs.is_empty());
let queued_counts = state.flushed_db().pending_op_counts().unwrap();
state.unlink_mem(&uid);
assert!(state.entries[&ino].unlinked);
assert!(!state.children[&parent].contains(&ino));
assert!(
state
.flushed_db()
.node_by_uid(&uid.to_string())
.unwrap()
.is_some()
);
let released = release_unlinked_entry(&mut state, ino).unwrap();
assert_eq!(released, uid);
assert!(!state.by_uid.contains_key(&uid));
assert!(release_must_retain_queued_trash(&state.db, &uid).unwrap());
assert!(
state
.flushed_db()
.node_by_uid(&uid.to_string())
.unwrap()
.is_some(),
"last release keeps persisted drain authority"
);
let pending = state.flushed_db().pending_ops().unwrap();
assert_eq!(pending.len(), 1, "release must not add a revision");
assert_eq!(pending[0].kind, OP_TRASH);
let after_release = state.flushed_db().pending_op_counts().unwrap();
assert_eq!(after_release.uploads, 0);
assert_eq!(after_release.changes, 1);
assert_eq!(after_release.uploads, queued_counts.uploads);
assert_eq!(after_release.changes, queued_counts.changes);
state
.flushed_db()
.complete_trash_op(pending[0].id, &uid)
.unwrap();
assert!(state.flushed_db().pending_ops().unwrap().is_empty());
assert!(
state
.flushed_db()
.node_by_uid(&uid.to_string())
.unwrap()
.is_none()
);
}
#[test]
fn queued_trash_lookup_failure_conservatively_retains_unlinked_state() {
let (mut state, _dir) = state_test_helper();
let uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("uncertain-trash"));
state
.flushed_db()
.with_conn(|conn| {
conn.execute_batch("DROP TABLE pending_op")?;
Ok(())
})
.unwrap();
assert!(release_must_retain_queued_trash(&state.db, &uid).is_err());
assert!(
!release_can_discard_unlinked(&state.db, &uid),
"database uncertainty must retain the tombstone and authority row"
);
}
#[test]
fn hydrated_listings_hold_each_folders_live_children() {
let entry = |id: &str, parent: u64, is_dir: bool, trashed: bool| {
let mut node = node_helper(id, "none", id, is_dir);
node.trashed = trashed;
crate::state::Entry {
uid: node.uid.clone(),
parent,
node,
access: Access::Owner,
lookup_count: 1,
open_count: 0,
unlinked: false,
}
};
let entries: HashMap<u64, crate::state::Entry> = [
(1, entry("root", 1, true, false)),
(2, entry("a", 1, true, false)),
(3, entry("b", 1, true, false)),
(4, entry("a1", 2, false, false)),
(5, entry("a2", 2, false, true)),
(6, entry("b1", 3, false, false)),
]
.into_iter()
.collect();
let mut listings = super::listed_children(&entries, vec![1, 2]);
listings.values_mut().for_each(|kids| kids.sort());
assert_eq!(listings, HashMap::from([(1, vec![2, 3]), (2, vec![4])]));
}
fn node_helper(id: &str, parent: &str, name: &str, is_dir: bool) -> Node {
let uid = NodeUid::new(VolumeId::from("vol"), LinkId::from(id));
let parent_uid = if parent == "none" {
None
} else {
Some(NodeUid::new(VolumeId::from("vol"), LinkId::from(parent)))
};
Node {
uid,
parent_uid,
name: name.to_string(),
kind: if is_dir {
NodeKind::Folder
} else {
NodeKind::File {
media_type: "text/plain".into(),
total_size_on_storage: 0,
active_revision_state: None,
active_revision_id: None,
content_sha1: None,
claimed_size: Some(0),
claimed_modification_time: None,
}
},
creation_time: 100,
modification_time: 100,
trashed: false,
is_shared: false,
is_shared_publicly: false,
signature_email: None,
membership: None,
photo: None,
album: None,
verification: Default::default(),
direct_role: None,
share_id: None,
}
}
}
#[cfg(test)]
mod thumbnail_miss_tests {
use super::*;
#[test]
fn remote_misses_never_suppress_local_generation() {
let uid = parse_uid("volume~node").expect("uid");
let key = (uid, ThumbnailType::Thumbnail.as_i32());
let mut misses = ThumbnailMissCaches::default();
misses.remember_remote(key.clone(), 42);
assert!(misses.remote_contains(&key, 42));
assert!(!misses.local_contains(&key, 42));
misses.remember_local(key.clone(), 42);
assert!(misses.local_contains(&key, 42));
misses.forget_local(&key);
assert!(!misses.local_contains(&key, 42));
assert!(misses.remote_contains(&key, 42));
}
#[test]
fn local_miss_cache_is_bounded_on_gui_only_paths() {
let mut misses = ThumbnailMissCaches::default();
for index in 0..=MAX_THUMBNAIL_MISSES {
let uid = parse_uid(&format!("volume~node-{index}")).expect("uid");
misses.remember_local((uid, ThumbnailType::Thumbnail.as_i32()), 42);
}
assert_eq!(misses.local.len(), 1);
}
}
#[cfg(test)]
mod search_root_tests {
use super::*;
fn roots(pairs: &[(&str, &str)]) -> SearchRoots {
SearchRoots {
roots: pairs
.iter()
.map(|(root, local)| ((*root).to_string(), PathBuf::from(local)))
.collect(),
}
}
#[test]
fn a_hit_below_a_root_resolves_through_it() {
let roots = roots(&[("Videos/Anime", "/home/u/anime")]);
assert_eq!(
roots.resolve("Videos/Anime/s01/e01.mkv").as_deref(),
Some("/home/u/anime/s01/e01.mkv")
);
}
#[test]
fn a_hit_outside_every_root_resolves_nowhere() {
let roots = roots(&[("Videos/Anime", "/home/u/anime")]);
assert_eq!(roots.resolve("Documents/tax.pdf"), None);
}
#[test]
fn a_sibling_sharing_a_name_prefix_is_not_below_the_root() {
// `Videos/Anime2` starts with `Videos/Anime` as a string and is a
// different folder; only a `/` boundary makes it a descendant.
let roots = roots(&[("Videos/Anime", "/home/u/anime")]);
assert_eq!(roots.resolve("Videos/Anime2/e01.mkv"), None);
}
#[test]
fn the_most_specific_root_wins() {
let roots = roots(&[
("Videos", "/home/u/videos"),
("Videos/Anime", "/home/u/anime"),
]);
assert_eq!(
roots.resolve("Videos/Anime/e01.mkv").as_deref(),
Some("/home/u/anime/e01.mkv"),
"the root leaving the shortest descendant path is the one the user \
configured for this content"
);
}
#[test]
fn a_root_covering_the_whole_drive_resolves_everything() {
let roots = roots(&[("", "/home/u/drive")]);
assert_eq!(
roots.resolve("Documents/tax.pdf").as_deref(),
Some("/home/u/drive/Documents/tax.pdf")
);
}
#[test]
fn the_root_folder_itself_resolves_to_its_local_directory() {
let roots = roots(&[("Videos/Anime", "/home/u/anime")]);
assert_eq!(
roots.resolve("Videos/Anime").as_deref(),
Some("/home/u/anime/")
);
}
}