mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-04 00:47:00 +01:00
6059 lines
247 KiB
Rust
6059 lines
247 KiB
Rust
//! On-demand FUSE filesystem for Proton Drive.
|
||
//!
|
||
//! Phase 1 is a read-only placeholder mount: directories are enumerated lazily
|
||
//! from the remote on first access and file content is hydrated on `read` via
|
||
//! [`ProtonDriveClient::download_range`]. Nothing is mirrored to local disk;
|
||
//! every byte is fetched on demand.
|
||
//!
|
||
//! Phase 2 adds live remote sync: a background task polls the volume event
|
||
//! cursor ([`ProtonDriveClient::enumerate_events`]) and pushes invalidations
|
||
//! into the kernel via a fuser [`Notifier`], so the cache TTL can be long while
|
||
//! remote changes still show up promptly.
|
||
//!
|
||
//! Phase 3 makes the mount writable. Each file opened for writing gets a
|
||
//! [`WriteHandle`] whose buffer accumulates the full plaintext; on flush/release
|
||
//! the buffer is sealed as a new revision via
|
||
//! [`ProtonDriveClient::upload_new_revision`] (the SDK uploads whole revisions,
|
||
//! not byte ranges). New files are created empty up front so they get a real
|
||
//! uid; namespace ops map to `create_folder`, `trash_nodes`, `rename_node` and
|
||
//! `move_node`.
|
||
//!
|
||
//! Phase 4 adds Files-On-Demand pinning. A control socket (see [`control`])
|
||
//! lets the CLI pin/unpin files; a pinned file's plaintext is downloaded once
|
||
//! into the on-disk [`ContentCache`] and every later `read` is served from disk
|
||
//! instead of the network. Writes and remote events evict the cache so it never
|
||
//! goes stale.
|
||
//!
|
||
//! Reads of unpinned files no longer hit the network per call: [`Core::read_range`]
|
||
//! fetches and caches [`BLOCK_SIZE`]-aligned blocks, so sequential/sparse reads
|
||
//! reuse the on-disk block cache. Writes are disk-backed: each [`WriteHandle`]
|
||
//! stages authored bytes in a scratch file and tracks them with an [`Intervals`]
|
||
//! set, so a multi-GiB write never buffers in RAM and only the untouched
|
||
//! remainder of the file is pulled from the remote — lazily, at commit.
|
||
|
||
use parking_lot::{Condvar, Mutex};
|
||
use std::collections::{HashMap, HashSet, VecDeque};
|
||
use std::ffi::OsStr;
|
||
use std::fs::File;
|
||
use std::os::unix::fs::FileExt;
|
||
use std::os::unix::net::UnixListener;
|
||
use std::path::{Component, Path, PathBuf};
|
||
use std::sync::Arc;
|
||
use std::sync::OnceLock;
|
||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||
|
||
use fuser::ReplyXattr;
|
||
use fuser::{
|
||
BackgroundSession, BsdFileFlags, Config, Errno, FileAttr, FileHandle, FileType, Filesystem,
|
||
FopenFlags, Generation, INodeNo, LockOwner, MountOption, Notifier, OpenAccMode, OpenFlags,
|
||
RenameFlags, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry,
|
||
ReplyOpen, ReplyWrite, Request, Session, TimeOrNow, WriteFlags,
|
||
};
|
||
use pdfs_core::cache::{BLOCK_SIZE, Baseline, ContentCache, StagedWrite};
|
||
use pdfs_core::config::AppDirs;
|
||
use pdfs_core::control::{
|
||
ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo,
|
||
SearchHit, SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection,
|
||
};
|
||
use pdfs_core::db::{
|
||
Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingOp, StoredNode,
|
||
StoredSyncFolder, StoredTrash,
|
||
};
|
||
use pdfs_core::localindex;
|
||
use pdfs_core::{CoreError, CoreResult};
|
||
use proton_drive_rs::proton_sdk::api::ResponseCode;
|
||
use proton_drive_rs::proton_sdk::error::ProtonError;
|
||
use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, VolumeId};
|
||
use proton_drive_rs::{
|
||
DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, ProtonDriveClient,
|
||
ProtonPhotosClient, RevisionReader, ThumbnailType,
|
||
};
|
||
|
||
mod control;
|
||
use control::run_control_socket;
|
||
mod devices;
|
||
mod drain;
|
||
mod photos;
|
||
mod profile;
|
||
mod sharing;
|
||
mod state;
|
||
mod sync;
|
||
mod transfers;
|
||
mod workers;
|
||
use state::{Entry, Intervals, PendingRevision, State, WriteHandle};
|
||
use tracing::{debug, error, info, warn};
|
||
use transfers::{CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry};
|
||
use workers::{FUSE_WORKERS, Lane, Workers};
|
||
|
||
/// Attribute/entry cache lifetime handed back to the kernel. Long because the
|
||
/// Phase 2 event poller actively invalidates changed inodes; without a remote
|
||
/// change this is how long the kernel may serve stale metadata.
|
||
const TTL: Duration = Duration::from_secs(30);
|
||
|
||
/// How often the background task polls the remote event cursor.
|
||
const POLL_INTERVAL: Duration = Duration::from_secs(10);
|
||
/// First and longest delay between probes for the network coming back after an
|
||
/// offline mount (offline.md Phase 1). Doubles from min to max: a laptop shut in
|
||
/// a bag is the common case, so the steady state must be cheap, while a brief
|
||
/// blip should still recover in seconds.
|
||
/// Retry backoff for a queued upload, doubling per attempt between these. The
|
||
/// floor is short because the common failure is a brief network blip; the
|
||
/// ceiling keeps a persistently failing op from spinning.
|
||
const DRAIN_BACKOFF_MIN: Duration = Duration::from_secs(2);
|
||
const DRAIN_BACKOFF_MAX: Duration = Duration::from_secs(300);
|
||
/// How long the drain worker sleeps when it has nothing due. It is woken
|
||
/// directly on a new write or a reconnect, so this only bounds how late a
|
||
/// backoff can fire.
|
||
const DRAIN_IDLE_POLL: Duration = Duration::from_secs(30);
|
||
|
||
const ONLINE_PROBE_MIN: Duration = Duration::from_secs(5);
|
||
const ONLINE_PROBE_MAX: Duration = Duration::from_secs(300);
|
||
/// How long the persisted photos timeline stays good before a page request
|
||
/// revalidates it. The SDK hands back the whole timeline at once, so it is stored
|
||
/// in the DB and every page is sliced from there; a stale one is still served
|
||
/// immediately and refreshed in the background.
|
||
const TIMELINE_TTL: Duration = Duration::from_secs(5 * 60);
|
||
/// How many photo nodes are resolved per [`ProtonPhotosClient::enumerate_nodes`]
|
||
/// call when enriching a refreshed timeline with names and media types (for the
|
||
/// Photos / Videos / Raw split). Batched so a large library is a handful of
|
||
/// round-trips rather than one request per photo, and bounded so a single call
|
||
/// never asks the server to decrypt the whole library at once.
|
||
const TIMELINE_ENRICH_CHUNK: usize = 200;
|
||
/// The same, for the persisted trash listing. Shorter, because the trash is the
|
||
/// one listing a user changes and then immediately looks at — though our own
|
||
/// mutations also invalidate it outright, so this only covers other clients.
|
||
const TRASH_TTL: Duration = Duration::from_secs(60);
|
||
|
||
/// `sync_state` keys for the freshness stamps of the two persisted listings, and
|
||
/// for whether the account has a photos volume at all (so an account without one
|
||
/// doesn't re-ask the server on every page request).
|
||
const PHOTOS_SYNCED_MS: &str = "photos_synced_ms";
|
||
const PHOTOS_AVAILABLE: &str = "photos_available";
|
||
const TRASH_SYNCED_MS: &str = "trash_synced_ms";
|
||
|
||
/// Longest edge, in px, of a thumbnail generated locally for a photo the server
|
||
/// has none for. Matches the server's own thumbnail scale closely enough that a
|
||
/// tile can't tell them apart.
|
||
const THUMB_EDGE: u32 = 512;
|
||
/// JPEG quality of a locally generated thumbnail.
|
||
const THUMB_QUALITY: u8 = 82;
|
||
/// How many photos may be downloaded at once to generate their missing
|
||
/// thumbnails. Bounded: a screenful of 20 MB digicam JPEGs would otherwise
|
||
/// saturate the link and starve the rest of the daemon.
|
||
const THUMB_GEN_CONCURRENCY: usize = 4;
|
||
|
||
/// A read of an unpinned video at least this large streams *without* persisting
|
||
/// its blocks to the on-disk cache. Playing a 2 GB film would otherwise pour it
|
||
/// through the block LRU and evict everything else the user actually wants kept —
|
||
/// and it re-streams cheaply enough on a rewatch that keeping it was never worth
|
||
/// that. Pinned videos (kept offline on purpose) and anything smaller cache as
|
||
/// usual.
|
||
const STREAM_BYPASS_MIN: u64 = 256 * 1024 * 1024;
|
||
|
||
/// How many bytes of streamed blocks the in-memory ring keeps. Bypassing the
|
||
/// on-disk cache must not mean re-fetching: the kernel asks for a streamed file
|
||
/// in reads far smaller than [`BLOCK_SIZE`], so without a ring every 128 KiB the
|
||
/// player consumes would download and decrypt a whole 4 MiB block again. Sized
|
||
/// for a handful of blocks per concurrently-streamed file.
|
||
const STREAM_RING_BYTES: u64 = 128 * 1024 * 1024;
|
||
|
||
/// Blocks fetched *past* the one a sequential streaming read needed, warmed into
|
||
/// the ring in the background so the player's next read is already in memory
|
||
/// instead of waiting a round-trip. Kept small so it cannot crowd out the
|
||
/// in-flight block governor.
|
||
const STREAM_READAHEAD: u64 = 4;
|
||
|
||
/// How stale the local-file index may get before the background scanner rebuilds
|
||
/// it. A rescan is a full walk of `$HOME`, so this trades index freshness against
|
||
/// disk churn; the scanner also always rebuilds once per daemon start when the
|
||
/// index is older than this.
|
||
const LOCAL_INDEX_TTL: Duration = Duration::from_secs(15 * 60);
|
||
|
||
/// How often the scanner thread wakes to check whether the index went stale.
|
||
const LOCAL_INDEX_CHECK: Duration = Duration::from_secs(60);
|
||
|
||
/// The FUSE root inode is always 1.
|
||
const ROOT_INO: u64 = 1;
|
||
|
||
/// Parent inode for a persisted node whose parent row is missing from the DB.
|
||
/// No folder carries this inode, so such a node is listed by nobody and stays
|
||
/// inert until a live enumeration re-parents it.
|
||
const ORPHAN_INO: u64 = 0;
|
||
|
||
/// Extended attribute exposing the small server-side thumbnail of a file.
|
||
const XATTR_THUMBNAIL: &str = "user.proton.thumbnail";
|
||
/// Extended attribute exposing the larger server-side preview image of a file.
|
||
const XATTR_PREVIEW: &str = "user.proton.preview";
|
||
|
||
/// How many "this file has no thumbnail" answers [`Core::thumbnail`] remembers
|
||
/// before dropping the lot and re-learning them. Sized to cover a large browsing
|
||
/// session; each entry is a uid, a type tag and an mtime.
|
||
const MAX_THUMBNAIL_MISSES: usize = 8192;
|
||
|
||
/// How many nodes one size-upgrade request covers.
|
||
///
|
||
/// Matches the SDK's own `MAX_BATCH_COUNT`, so a chunk is exactly one request:
|
||
/// chunking smaller would add round trips, larger would be split anyway and
|
||
/// delay the waiters this chunking exists to release (bugs.md B14).
|
||
const SIZE_UPGRADE_CHUNK: usize = 150;
|
||
|
||
/// How many files the bulk uploader ships at once. Overlaps the per-file network
|
||
/// round-trips without letting an unbounded number of block buffers pile up.
|
||
const UPLOAD_CONCURRENCY: usize = 4;
|
||
|
||
/// One file queued for bulk upload, resolved during the directory walk so the
|
||
/// concurrent phase carries everything it needs (no shared state, no `block_on`).
|
||
struct UploadTask {
|
||
/// Inode of the (already-created) remote parent folder, for interning the
|
||
/// uploaded node afterwards.
|
||
parent_ino: u64,
|
||
parent_uid: NodeUid,
|
||
name: String,
|
||
/// Local filesystem path to stream from.
|
||
path: PathBuf,
|
||
size: u64,
|
||
}
|
||
|
||
/// Tally of a completed [`Core::upload_paths`] batch, for the daemon log.
|
||
#[derive(Default)]
|
||
struct UploadStats {
|
||
uploaded: usize,
|
||
failed: usize,
|
||
/// Total plaintext bytes of the files that uploaded successfully.
|
||
bytes: u64,
|
||
/// Folders created (or reused) to mirror the local tree.
|
||
folders: usize,
|
||
}
|
||
|
||
/// Upload every [`UploadTask`] with at most `limit` in flight at once, each
|
||
/// streamed straight from disk and ticking its own transfer-registry guard.
|
||
/// Returns, per file, either `(parent_ino, new_uid)` for the caller to intern or
|
||
/// `(name, error)` to log — one failure never sinks the batch.
|
||
///
|
||
/// `job` counts files finished (either way: a failure is still one file the batch
|
||
/// no longer waits on), so a front-end can show "12 of 40" over the per-file bars.
|
||
async fn run_uploads(
|
||
core: Core,
|
||
tasks: Vec<UploadTask>,
|
||
limit: usize,
|
||
job: Arc<JobGuard>,
|
||
) -> Vec<Result<(u64, NodeUid, u64), (String, String)>> {
|
||
let sem = Arc::new(tokio::sync::Semaphore::new(limit));
|
||
let mut set = tokio::task::JoinSet::new();
|
||
for t in tasks {
|
||
let core = core.clone();
|
||
let sem = sem.clone();
|
||
set.spawn(async move {
|
||
let _permit = sem.acquire_owned().await.expect("semaphore not closed");
|
||
let file = match std::fs::File::open(&t.path) {
|
||
Ok(f) => f,
|
||
Err(e) => return Err((t.name, format!("open {}: {e}", t.path.display()))),
|
||
};
|
||
let mtime = std::fs::metadata(&t.path)
|
||
.and_then(|m| m.modified())
|
||
.ok()
|
||
.and_then(|mt| mt.duration_since(std::time::UNIX_EPOCH).ok())
|
||
.map(|d| d.as_secs() as i64);
|
||
let guard = core
|
||
.transfers
|
||
.begin(&t.name, "", TransferDirection::Upload, t.size);
|
||
let reader = OwnedCountingReader::new(file, guard);
|
||
match core
|
||
.client
|
||
.upload_file_from(
|
||
&t.parent_uid,
|
||
&t.name,
|
||
media_type_for(&t.name),
|
||
reader,
|
||
t.size as i64,
|
||
Vec::new(),
|
||
mtime,
|
||
false,
|
||
)
|
||
.await
|
||
{
|
||
Ok(uid) => Ok((t.parent_ino, uid, t.size)),
|
||
Err(e) => Err((t.name, format!("upload: {e}"))),
|
||
}
|
||
});
|
||
}
|
||
let mut out = Vec::new();
|
||
while let Some(joined) = set.join_next().await {
|
||
job.step();
|
||
match joined {
|
||
Ok(result) => out.push(result),
|
||
Err(e) => warn!(error = %e, "upload task panicked"),
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Shared engine behind both the FUSE callbacks and the control socket: the
|
||
/// How many [`RevisionReader`]s stay open at once.
|
||
///
|
||
/// A reader holds its revision's content key and block table — a few KB even for
|
||
/// a large file, so this is bounded for tidiness and staleness rather than
|
||
/// memory. Evicted least-recently-used; a dropped reader costs one re-open
|
||
/// (two API calls and a node-key unlock) the next time that file is read.
|
||
const MAX_OPEN_READERS: usize = 64;
|
||
|
||
/// An open reader plus the node metadata it was opened against.
|
||
///
|
||
/// The SDK pins a reader to the revision that was active at `open_revision`, so
|
||
/// a reader is only reusable while the node still reports the same
|
||
/// `(mtime, size)` — the same validity pair the content cache uses (a new
|
||
/// revision bumps mtime). On a mismatch the reader is dropped and reopened.
|
||
struct CachedReader {
|
||
reader: Arc<RevisionReader>,
|
||
mtime: i64,
|
||
size: u64,
|
||
/// For LRU eviction.
|
||
last_used: Instant,
|
||
}
|
||
|
||
/// What the `readers` map holds for a node: an open reader, or the fact that
|
||
/// someone is already opening one.
|
||
///
|
||
/// The `Pending` arm is what makes the map single-flight rather than merely a
|
||
/// cache. Opening resolves link details, ancestor keys, an S2K node-key unlock
|
||
/// and the block table — five round-trips — and read-ahead on a cold file puts
|
||
/// several workers into the open at the same moment, so without this each
|
||
/// racer would redo all of it and discard every result but one.
|
||
enum ReaderSlot {
|
||
Ready(CachedReader),
|
||
Pending(PendingOpen),
|
||
}
|
||
|
||
/// An `open_revision` in flight. Racers clone `rx` and await the leader's result
|
||
/// instead of opening their own.
|
||
struct PendingOpen {
|
||
/// Identifies this attempt so the leader can tell "my slot is still there"
|
||
/// from "someone evicted or replaced it while I was on the network", and
|
||
/// only publish its reader in the first case.
|
||
id: u64,
|
||
/// What the leader is opening against. A racer wanting a different revision
|
||
/// must not join this open — it would get a reader for the wrong bytes.
|
||
mtime: i64,
|
||
size: u64,
|
||
rx: tokio::sync::watch::Receiver<Option<Result<Arc<RevisionReader>, Errno>>>,
|
||
}
|
||
|
||
/// Distinguishes `PendingOpen`s. Only uniqueness matters, so a plain counter
|
||
/// does; ids are never compared for order.
|
||
static NEXT_OPEN_ID: AtomicU64 = AtomicU64::new(0);
|
||
|
||
/// Drive client, a Tokio handle to bridge the synchronous FUSE/socket threads
|
||
/// to the async SDK, the inode bookkeeping, and the on-disk content cache.
|
||
///
|
||
/// Cheaply cloneable (every field is a handle/`Arc`), so the control-socket task
|
||
/// gets its own copy while the FUSE session keeps another.
|
||
/// In-memory LRU of blocks belonging to a file streaming past the on-disk cache
|
||
/// (see [`STREAM_BYPASS_MIN`]). Bypassing the disk stops one film evicting the
|
||
/// block LRU; it must not also mean the same 4 MiB block is downloaded once per
|
||
/// 128 KiB the player reads out of it.
|
||
///
|
||
/// Validated by the same `(mtime, size)` pair the on-disk caches use: a node
|
||
/// whose tag no longer matches has its blocks dropped rather than served, so a
|
||
/// new revision can never be stitched together from the old one.
|
||
#[derive(Default)]
|
||
struct StreamRing {
|
||
blocks: HashMap<(NodeUid, u64), Arc<Vec<u8>>>,
|
||
/// Keys oldest-first, for eviction.
|
||
order: VecDeque<(NodeUid, u64)>,
|
||
/// Per-node validity tag, `(mtime, size)`.
|
||
tags: HashMap<NodeUid, (i64, u64)>,
|
||
bytes: u64,
|
||
}
|
||
|
||
impl StreamRing {
|
||
fn get(&mut self, uid: &NodeUid, mtime: i64, size: u64, idx: u64) -> Option<Arc<Vec<u8>>> {
|
||
if self.tags.get(uid) != Some(&(mtime, size)) {
|
||
return None;
|
||
}
|
||
self.blocks.get(&(uid.clone(), idx)).cloned()
|
||
}
|
||
|
||
fn insert(&mut self, uid: &NodeUid, mtime: i64, size: u64, idx: u64, bytes: Arc<Vec<u8>>) {
|
||
if self.tags.get(uid) != Some(&(mtime, size)) {
|
||
// Revision changed under us (or first sight): anything held for this
|
||
// node describes the old one.
|
||
self.drop_node(uid);
|
||
self.tags.insert(uid.clone(), (mtime, size));
|
||
}
|
||
let key = (uid.clone(), idx);
|
||
if self.blocks.contains_key(&key) {
|
||
return;
|
||
}
|
||
self.bytes += bytes.len() as u64;
|
||
self.blocks.insert(key.clone(), bytes);
|
||
self.order.push_back(key);
|
||
while self.bytes > STREAM_RING_BYTES {
|
||
let Some(victim) = self.order.pop_front() else {
|
||
break;
|
||
};
|
||
if let Some(dropped) = self.blocks.remove(&victim) {
|
||
self.bytes -= dropped.len() as u64;
|
||
}
|
||
}
|
||
}
|
||
|
||
fn drop_node(&mut self, uid: &NodeUid) {
|
||
self.order.retain(|(u, _)| u != uid);
|
||
let mut freed = 0u64;
|
||
self.blocks.retain(|(u, _), bytes| {
|
||
if u == uid {
|
||
freed += bytes.len() as u64;
|
||
false
|
||
} else {
|
||
true
|
||
}
|
||
});
|
||
self.bytes -= freed;
|
||
self.tags.remove(uid);
|
||
}
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
struct Core {
|
||
client: ProtonDriveClient,
|
||
rt: tokio::runtime::Handle,
|
||
state: Arc<Mutex<State>>,
|
||
cache: Arc<ContentCache>,
|
||
/// Open [`RevisionReader`]s keyed by node, so the block fetches of a file
|
||
/// resolve its keys and block table once instead of once per block.
|
||
/// Validated by `(mtime, size)` exactly like the content cache, and bounded
|
||
/// by [`MAX_OPEN_READERS`].
|
||
readers: Arc<Mutex<HashMap<NodeUid, ReaderSlot>>>,
|
||
/// Blocks of files streaming past the on-disk cache, held in memory so a
|
||
/// player's small sequential reads are served from the last few 4 MiB blocks
|
||
/// instead of re-downloading each one. See [`StreamRing`].
|
||
stream_ring: Arc<Mutex<StreamRing>>,
|
||
/// Threads that serve the FUSE handlers which touch the network, so the
|
||
/// session's dispatch loop stays free to answer cheap metadata calls while a
|
||
/// cold read is on the wire. See [`Workers`].
|
||
workers: Arc<Workers>,
|
||
/// Unified SQLite metadata cache: the persistence layer behind the in-memory
|
||
/// `State` maps. Every mutation writes through here, and the maps rehydrate
|
||
/// from it on mount (plan.md P1).
|
||
db: Arc<Db>,
|
||
/// False while the API is unreachable and we are serving the cached tree
|
||
/// (offline.md Phase 1). Set by the probe thread; read by front-ends through
|
||
/// `Response::Status` so the UI can say so rather than leaving the user to
|
||
/// infer it from a wall of EIO.
|
||
online: Arc<AtomicBool>,
|
||
/// Writes accepted from the kernel but not yet uploaded, keyed by node
|
||
/// (offline.md Phase 3). The in-memory face of the `pending_op` table, from
|
||
/// which it is rebuilt on mount.
|
||
///
|
||
/// Two things read it: [`Core::read_range`], because until the op drains the
|
||
/// staged blob *is* the file's content and the remote still holds the old
|
||
/// revision; and the drain worker, which performs the uploads.
|
||
pending: Arc<Mutex<HashMap<NodeUid, PendingRevision>>>,
|
||
/// Nudges the drain worker: set true and notify to have it re-examine the
|
||
/// queue instead of waiting out its backoff.
|
||
drain_wake: Arc<(Mutex<bool>, Condvar)>,
|
||
/// True while a background refresh of the photos timeline (resp. the trash) is
|
||
/// already running, so a burst of page requests against a stale listing kicks
|
||
/// off one refresh rather than one per request.
|
||
timeline_refreshing: Arc<AtomicBool>,
|
||
trash_refreshing: Arc<AtomicBool>,
|
||
/// Photos whose missing thumbnail is being generated right now. A tile that is
|
||
/// still on screen asks for its thumbnail again every few seconds, and each of
|
||
/// those downloads is a full-size photo — so an in-flight uid is never started
|
||
/// twice.
|
||
thumb_gen: Arc<Mutex<HashSet<NodeUid>>>,
|
||
/// Nodes the remote has told us have *no* thumbnail of a given type, keyed by
|
||
/// `(uid, thumbnail type)` and holding the mtime the answer was learned at.
|
||
///
|
||
/// Absence has to be cached or it costs a round trip every time it is asked
|
||
/// for, and it is asked for constantly: an `ls -l` from an xattr-aware lister
|
||
/// issues a `getxattr` per advertised name per entry, so a 65-file directory
|
||
/// of videos re-probed 130 times per listing at ~186 ms each (B5). The mtime
|
||
/// is the validity tag — a new revision may well have a thumbnail — matching
|
||
/// how [`ContentCache::read_thumbnail`] validates the positive side.
|
||
no_thumbnail: Arc<Mutex<HashMap<(NodeUid, i32), i64>>>,
|
||
/// Folders whose listing was enumerated cheaply and is having its file sizes
|
||
/// filled in right now, so a burst of `stat`s over a fresh listing starts one
|
||
/// upgrade rather than one per entry. See [`Core::spawn_size_upgrade`].
|
||
/// Size upgrades currently running, per folder inode. A `getattr` that
|
||
/// needs a real size waits on the entry rather than issuing its own fetch —
|
||
/// `ls -l` of a folder is one `getattr` per file, and they must collapse
|
||
/// onto a single batch (bugs.md B14).
|
||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||
/// This mount's kernel notification channel, for telling the kernel to drop
|
||
/// metadata it has cached. Set once the session exists — which is *after*
|
||
/// the `Core` it is built from, hence the cell.
|
||
///
|
||
/// Per mount, not per daemon: each on-demand fork runs its own session over
|
||
/// its own inode space, so notifying through the primary mount's channel
|
||
/// would name inodes that session has never heard of. [`Core::fork_state`]
|
||
/// gives each fork an empty cell of its own.
|
||
notifier: Arc<OnceLock<Notifier>>,
|
||
/// In-flight upload/download progress, served to `GetQueueStatus`. Shared
|
||
/// across the FUSE session and the control-socket task.
|
||
transfers: Arc<TransferRegistry>,
|
||
/// True while the background scanner is rebuilding the local-file index, so
|
||
/// `SearchLocal` can tell a front-end "still indexing" apart from "no match".
|
||
indexing: Arc<AtomicBool>,
|
||
/// Live per-folder sync progress, keyed by sync-folder id, so `ListSyncFolders`
|
||
/// can say what a pass is doing rather than just "syncing". An entry exists
|
||
/// only while that folder's reconcile pass is running.
|
||
sync_progress: Arc<Mutex<HashMap<i64, SyncProgress>>>,
|
||
/// Channel to the folder-sync engine (devices.md Phase 2): nudges it to
|
||
/// reconcile a folder, reconcile everything, or re-scan its watch set.
|
||
sync_tx: std::sync::mpsc::Sender<sync::SyncMsg>,
|
||
/// Secondary FUSE sessions for `ondemand` sync folders, keyed by sync-folder
|
||
/// id (devices.md Phase 3). Each is a `ProtonFs` rooted at the folder's remote
|
||
/// node, mounted over its local path, sharing this Core's client/cache/db but
|
||
/// with its own inode space (`fork_state`). Held so we can unmount on toggle
|
||
/// back to `mirror` and on daemon shutdown. The `u32` is the FUSE connection
|
||
/// id (see [`fuse_connection_id`]), captured at mount time so teardown can
|
||
/// abort a mid-transfer connection instead of blocking on it.
|
||
mounts: Arc<Mutex<HashMap<i64, SecondaryMount>>>,
|
||
/// Per-sync-folder locks, held for a whole reconcile pass and for a whole
|
||
/// mode switch. A `mirror→ondemand` flip evicts the local tree and mounts
|
||
/// FUSE over it, so it must never overlap a pass that is walking and
|
||
/// uploading that same tree — the engine would upload files as they vanish
|
||
/// and then walk the FUSE mount as if it were local.
|
||
sync_locks: Arc<Mutex<HashMap<i64, Arc<Mutex<()>>>>>,
|
||
}
|
||
|
||
/// Why [`Core::apply_sync_folder_mode`] did not switch a folder. The two cases are
|
||
/// answered very differently: `NotNow` is the normal state of a folder that is busy
|
||
/// or has local changes still to push, and the caller queues the request; `Failed`
|
||
/// is a real fault the user has to hear about.
|
||
enum SwitchBlocked {
|
||
/// The folder is mid-pass, or not yet safe to switch. Try again after a pass.
|
||
NotNow,
|
||
/// The switch was attempted and broke.
|
||
Failed(String),
|
||
}
|
||
|
||
impl Core {
|
||
/// Rehydrate the in-memory `State` maps from the DB on mount, so a cold
|
||
/// start serves previously-seen metadata (stable inodes, instant listings)
|
||
/// without re-hitting the API. The root inode is already installed by
|
||
/// [`ProtonFs::new`]; this fills in every other persisted node and rebuilds
|
||
/// the child listings of folders the DB records as fully enumerated.
|
||
fn hydrate(&self) {
|
||
let stored = match self.db.load_all() {
|
||
Ok(s) => s,
|
||
Err(e) => {
|
||
warn!(error = %e, "db load_all failed; mounting cold");
|
||
return;
|
||
}
|
||
};
|
||
if stored.is_empty() {
|
||
return;
|
||
}
|
||
// A write queued by a previous run carries the optimistic size the file
|
||
// grew to, which the sealed remote revision does not yet reflect. The DB
|
||
// node still holds the pre-write `claimed_size` (often 0 for a fresh
|
||
// file), so without this a queued 500 MB file `stat`s as 0 bytes until
|
||
// the drain lands — reads still serve from the staged blob, but `ls`
|
||
// lying about the size reads as data loss. `hydrate_pending` ran before
|
||
// us, so the pending map is populated; snapshot it here rather than hold
|
||
// both locks at once.
|
||
let pending_sizes: HashMap<NodeUid, u64> = {
|
||
let pending = self.pending.lock();
|
||
pending
|
||
.iter()
|
||
.map(|(uid, pr)| (uid.clone(), pr.meta.len))
|
||
.collect()
|
||
};
|
||
let mut st = self.state.lock();
|
||
|
||
// Pass 1: assign a stable inode to every uid (root is already mapped).
|
||
for sn in &stored {
|
||
if st.by_uid.contains_key(&sn.node.uid) {
|
||
continue;
|
||
}
|
||
let ino = st.next_ino;
|
||
st.next_ino += 1;
|
||
st.by_uid.insert(sn.node.uid.clone(), ino);
|
||
}
|
||
|
||
// Pass 2: materialize entries, resolving each parent's inode by uid.
|
||
// Track folders flagged complete so their listings rebuild in pass 3.
|
||
let mut listed_dirs: Vec<u64> = Vec::new();
|
||
for sn in stored {
|
||
let StoredNode { mut node, listed } = sn;
|
||
let Some(&ino) = st.by_uid.get(&node.uid) else {
|
||
continue;
|
||
};
|
||
// Re-apply a queued write's optimistic size so `stat` matches what
|
||
// reads (served from the staged blob) already return.
|
||
if let Some(&len) = pending_sizes.get(&node.uid)
|
||
&& let NodeKind::File { claimed_size, .. } = &mut node.kind
|
||
{
|
||
*claimed_size = Some(len as i64);
|
||
}
|
||
if listed && node.is_folder() {
|
||
listed_dirs.push(ino);
|
||
}
|
||
// The root entry is owned by `ProtonFs::new`; don't overwrite it.
|
||
if ino == ROOT_INO {
|
||
continue;
|
||
}
|
||
// A node whose parent row never made it to disk must not be adopted
|
||
// by the root: it would surface as a phantom top-level entry.
|
||
let parent = node
|
||
.parent_uid
|
||
.as_ref()
|
||
.and_then(|p| st.by_uid.get(p).copied())
|
||
.unwrap_or(ORPHAN_INO);
|
||
let uid = node.uid.clone();
|
||
st.entries.insert(ino, Entry { uid, parent, node });
|
||
}
|
||
|
||
// Pass 3: rebuild child listings for fully-enumerated folders. The root
|
||
// is its own parent (that is what `..` resolves to), so it would match
|
||
// its own filter; a directory listed inside itself makes the kernel fail
|
||
// the lookup with EIO, taking the whole listing down with it.
|
||
for dir_ino in listed_dirs {
|
||
let kids: Vec<u64> = st
|
||
.entries
|
||
.iter()
|
||
.filter(|&(&ino, e)| ino != dir_ino && e.parent == dir_ino && !e.node.trashed)
|
||
.map(|(&ino, _)| ino)
|
||
.collect();
|
||
st.children.insert(dir_ino, kids);
|
||
}
|
||
info!(nodes = st.entries.len(), "hydrated metadata cache from db");
|
||
}
|
||
|
||
/// Rebuild the in-memory pending map from the `pending_op` table on mount
|
||
/// (offline.md Phase 3).
|
||
///
|
||
/// A queued write survives a restart — that is the point of persisting it —
|
||
/// so until the drain worker gets to it, reads of that file must still come
|
||
/// from its staged blob rather than the remote's older revision.
|
||
///
|
||
/// A row whose blob has gone missing is dropped: there is nothing left to
|
||
/// upload, and keeping it would fail forever.
|
||
fn hydrate_pending(&self) {
|
||
let ops = match self.db.pending_ops() {
|
||
Ok(ops) => ops,
|
||
Err(e) => {
|
||
error!(error = %e, "loading pending uploads failed");
|
||
return;
|
||
}
|
||
};
|
||
let mut map = self.pending.lock();
|
||
let mut restored = 0usize;
|
||
for op in ops {
|
||
let Some(uid) = parse_node_uid(&op.uid) else {
|
||
error!(uid = %op.uid, id = op.id, "pending op has an unparseable uid; dropping");
|
||
let _ = self.db.delete_op(op.id);
|
||
continue;
|
||
};
|
||
restored += 1;
|
||
// Only a revision must have a blob. A create carries none until
|
||
// something is written to it (`touch` offline is a legitimate op
|
||
// with nothing to serve), and a rename or trash never has one. All
|
||
// still have to be replayed, so only the blob — if any — is checked.
|
||
if op.blob_path.is_none() && op.kind != OP_REVISION {
|
||
continue;
|
||
}
|
||
let parsed = op
|
||
.meta_json
|
||
.as_deref()
|
||
.and_then(|j| serde_json::from_str::<StagedWrite>(j).ok())
|
||
.zip(op.blob_path.as_deref().map(PathBuf::from));
|
||
let Some((meta, path)) = parsed else {
|
||
error!(uid = %op.uid, id = op.id, "pending op is unreadable; dropping");
|
||
self.drop_unrecoverable_op(&op, &uid);
|
||
restored -= 1;
|
||
continue;
|
||
};
|
||
if !path.exists() {
|
||
error!(%uid, path = %path.display(), "staged blob is gone; dropping pending op");
|
||
self.drop_unrecoverable_op(&op, &uid);
|
||
restored -= 1;
|
||
continue;
|
||
}
|
||
map.insert(uid, PendingRevision { path, meta });
|
||
}
|
||
if restored > 0 {
|
||
info!(count = restored, "restored pending ops");
|
||
}
|
||
}
|
||
|
||
/// Queue the writes that an unclean shutdown caught between `fsync(2)` and
|
||
/// `close(2)`.
|
||
///
|
||
/// `fsync` promises the bytes survive a crash, but the queueing that makes a
|
||
/// write outlive the daemon happens at `release`. A crash in between used to
|
||
/// lose the data outright, because the scratch directory is cleared at open.
|
||
/// Now `fsync` leaves a sidecar, `ContentCache::open` moves those blobs to
|
||
/// `recovery/`, and this walks them into the same staging + queued-op path a
|
||
/// normal release takes.
|
||
///
|
||
/// Runs after [`hydrate_pending`](Self::hydrate_pending), which is what makes
|
||
/// the incomplete-blob check in [`enqueue_staged_write`](Self::enqueue_staged_write)
|
||
/// meaningful: a recovered partial write whose earlier write is still queued
|
||
/// must not gap-fill from a remote revision that no longer describes the
|
||
/// file. With `pending` already loaded, that case is detected and the bytes
|
||
/// are parked rather than mis-filled.
|
||
///
|
||
/// Failure is per-write and never fatal: a node that no longer exists, or an
|
||
/// op that cannot be queued, leaves its blob in `recovery/` for the next run
|
||
/// (and for a human) instead of taking the mount down.
|
||
fn recover_fsynced_writes(&self) {
|
||
let recovered = self.cache.recovered_writes();
|
||
if recovered.is_empty() {
|
||
return;
|
||
}
|
||
let mut queued = 0usize;
|
||
for (blob, meta) in recovered {
|
||
let Some(uid) = parse_node_uid(&meta.uid) else {
|
||
error!(uid = %meta.uid, "recovered write has an unparseable uid; keeping bytes");
|
||
continue;
|
||
};
|
||
// The inode is only used to stamp the in-memory tree with the new
|
||
// size, and nothing is interned this early — `hydrate` reads the
|
||
// size back off `pending` when the node is first looked up, so 0
|
||
// (no such inode) is correct rather than merely tolerable.
|
||
match self.enqueue_staged_write(&uid, 0, &blob, meta) {
|
||
Ok(()) => {
|
||
self.cache.discard_recovered(&blob);
|
||
queued += 1;
|
||
}
|
||
Err(e) => {
|
||
error!(%uid, blob = %blob.display(), error = ?e,
|
||
"cannot queue a recovered write; bytes kept for the next run");
|
||
// Some failures (a partial write parked by
|
||
// `stage_orphaned_write`) still consume the blob. Its sidecar
|
||
// would then describe nothing, so retire it — the bytes are
|
||
// in staging, which is where a human looks for them.
|
||
if !blob.exists() {
|
||
self.cache.discard_recovered(&blob);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if queued > 0 {
|
||
info!(
|
||
count = queued,
|
||
"recovered fsynced writes from an unclean shutdown"
|
||
);
|
||
}
|
||
}
|
||
|
||
/// Discard an op that can never be performed, because the bytes it was to
|
||
/// upload are gone from staging (something outside the daemon deleted them).
|
||
///
|
||
/// For a node that only ever existed locally, the placeholder goes too. Its
|
||
/// content is unrecoverable and nothing will ever mint it a real uid, so
|
||
/// leaving the row would strand a file in the tree that can be listed but
|
||
/// never read and never uploaded.
|
||
fn drop_unrecoverable_op(&self, op: &PendingOp, uid: &NodeUid) {
|
||
let _ = self.db.delete_op(op.id);
|
||
if is_local_uid(uid) {
|
||
error!(%uid, name = op.name.as_deref().unwrap_or("?"),
|
||
"discarding a node whose only copy was lost");
|
||
if let Err(e) = self.db.delete_node(uid) {
|
||
warn!(%uid, error = %e, "db delete_node failed for lost local node");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Poll for the API becoming reachable again after an offline mount, then
|
||
/// flip `online` and refresh the root (offline.md Phase 1). Runs on its own
|
||
/// thread and returns once we are back online: nothing sets `online` false
|
||
/// again, because a mount that has been online once keeps its live event
|
||
/// sync, which does its own retrying.
|
||
///
|
||
/// Backs off to [`ONLINE_PROBE_MAX`] rather than hammering a fixed interval —
|
||
/// a laptop can sit offline for days, and each probe is a real API round trip.
|
||
fn run_online_probe(&self) {
|
||
let mut delay = ONLINE_PROBE_MIN;
|
||
loop {
|
||
std::thread::sleep(delay);
|
||
match self.rt.block_on(self.client.get_my_files_folder()) {
|
||
Ok(root) => {
|
||
{
|
||
let mut st = self.state.lock();
|
||
if let Some(e) = st.entries.get_mut(&ROOT_INO) {
|
||
e.node = root.clone();
|
||
}
|
||
}
|
||
if let Err(e) = self.db.upsert_node(&root) {
|
||
warn!(error = %e, "refresh root after reconnect failed");
|
||
}
|
||
self.online.store(true, Ordering::Relaxed);
|
||
// Anything written while offline is queued and waiting on
|
||
// exactly this.
|
||
self.wake_drain();
|
||
info!("back online");
|
||
return;
|
||
}
|
||
Err(e) => {
|
||
debug!(error = %e, ?delay, "online probe failed; still offline");
|
||
delay = (delay * 2).min(ONLINE_PROBE_MAX);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Whether `ino`'s listing is already in memory, i.e. whether
|
||
/// [`Core::ensure_children`] would return without touching the network.
|
||
/// Lets a handler decide between answering inline and handing off to a
|
||
/// worker, at the cost of one uncontended map lookup.
|
||
fn children_cached(&self, ino: u64) -> bool {
|
||
self.state.lock().children.contains_key(&ino)
|
||
}
|
||
|
||
/// Re-apply the optimistic size of any queued write to `nodes`.
|
||
///
|
||
/// A node that arrives from the remote (or from its DB row) carries the size
|
||
/// of the revision the *server* holds, which for a file with a write still
|
||
/// queued is the pre-write size — often 0 for a file created moments ago.
|
||
/// Interning it as-is silently reverts the optimistic size that
|
||
/// `record_pending_write` stamped, and a file that stats as 0 bytes is a file
|
||
/// the kernel will not issue a single `read` for: `cat` prints nothing and
|
||
/// the staged blob that `read_range` would have served is never asked for.
|
||
/// That reads as data loss even though nothing is lost (B11).
|
||
///
|
||
/// [`Core::hydrate`] does the same thing for the restart case. This covers
|
||
/// every *live* re-enumeration — which is what a rename or move triggers,
|
||
/// since both invalidate the listings they touch.
|
||
///
|
||
/// Snapshots the pending map and returns before any caller takes the state
|
||
/// lock: no site in the daemon holds `pending` and `state` at once, and this
|
||
/// is not the place to become the first.
|
||
fn stamp_pending_sizes(&self, nodes: &mut [Node]) {
|
||
let sizes: HashMap<NodeUid, u64> = {
|
||
let pending = self.pending.lock();
|
||
if pending.is_empty() {
|
||
return;
|
||
}
|
||
pending
|
||
.iter()
|
||
.map(|(uid, pr)| (uid.clone(), pr.meta.len))
|
||
.collect()
|
||
};
|
||
apply_pending_sizes(nodes, &sizes);
|
||
}
|
||
|
||
/// Enumerate `ino`'s children from the remote and cache them. No-op if the
|
||
/// directory has already been listed. Network I/O happens without the lock
|
||
/// held so concurrent metadata reads aren't blocked behind a fetch.
|
||
fn ensure_children(&self, ino: u64) -> Result<(), Errno> {
|
||
let folder_uid = {
|
||
let st = self.state.lock();
|
||
if st.children.contains_key(&ino) {
|
||
return Ok(());
|
||
}
|
||
match st.entries.get(&ino) {
|
||
Some(e) => e.uid.clone(),
|
||
None => return Err(Errno::ENOENT),
|
||
}
|
||
};
|
||
|
||
// Offline fast path: a folder the DB still records as fully enumerated
|
||
// can be rebuilt from disk without hitting the API, even if its listing
|
||
// was trimmed from the hot cache mid-run.
|
||
match self.db.children_if_listed(&folder_uid) {
|
||
Ok(Some(mut nodes)) => {
|
||
// Before the lock: a DB row carries the size the server last
|
||
// sealed, which a queued write is ahead of (B11).
|
||
self.stamp_pending_sizes(&mut nodes);
|
||
let mut st = self.state.lock();
|
||
if st.children.contains_key(&ino) {
|
||
return Ok(());
|
||
}
|
||
let mut child_inos = Vec::with_capacity(nodes.len());
|
||
let mut needs_size = Vec::new();
|
||
for node in nodes {
|
||
if node.trashed || node.uid == folder_uid {
|
||
continue;
|
||
}
|
||
if matches!(
|
||
&node.kind,
|
||
NodeKind::File {
|
||
claimed_size: None,
|
||
..
|
||
}
|
||
) {
|
||
needs_size.push(node.uid.clone());
|
||
}
|
||
child_inos.push(st.intern_from_db(ino, node));
|
||
}
|
||
st.children.insert(ino, child_inos);
|
||
drop(st);
|
||
// Rows persisted from a cheap enumeration whose upgrade never
|
||
// ran (a restart in between, say) still owe their real sizes.
|
||
self.spawn_size_upgrade(ino, needs_size);
|
||
return Ok(());
|
||
}
|
||
Ok(None) => {}
|
||
Err(e) => warn!(%folder_uid, error = %e, "db children_if_listed failed"),
|
||
}
|
||
|
||
let uids = self
|
||
.rt
|
||
.block_on(self.client.enumerate_folder_children_node_uids(&folder_uid))
|
||
.map_err(|e| {
|
||
error!(%folder_uid, error = %e, "enumerate folder children failed");
|
||
Errno::EIO
|
||
})?;
|
||
// Cheap enumeration: `Light` skips unlocking each *file's* node key,
|
||
// which is an S2K derivation per file and was ~74% of the cost of a cold
|
||
// listing (B12 — measured with `perf`, 64% of cycles in SHA-256 alone).
|
||
// Folders are unlocked either way; their keys are what the children are
|
||
// decrypted with, so the walk cannot proceed without them.
|
||
//
|
||
// The price is that files come back without a `claimed_size`, so
|
||
// `node_size` falls back to the *ciphertext* size until
|
||
// `spawn_size_upgrade` below fills the real one in.
|
||
let mut nodes = self
|
||
.rt
|
||
.block_on(self.client.enumerate_nodes_light(&uids))
|
||
.map_err(|e| {
|
||
error!(%folder_uid, error = %e, "enumerate nodes failed");
|
||
Errno::EIO
|
||
})?;
|
||
// Same as the DB path above: the remote's size for a file with a write
|
||
// still queued is the pre-write one (B11).
|
||
self.stamp_pending_sizes(&mut nodes);
|
||
|
||
let mut st = self.state.lock();
|
||
// Lost the race? Another thread already populated it.
|
||
if st.children.contains_key(&ino) {
|
||
return Ok(());
|
||
}
|
||
let mut child_inos = Vec::with_capacity(nodes.len());
|
||
let filtered_nodes: Vec<Node> = nodes
|
||
.into_iter()
|
||
.filter(|node| !node.trashed && node.uid != folder_uid)
|
||
.collect();
|
||
// Files whose real size the cheap enumeration could not read. Collected
|
||
// before interning so the upgrade below has the uids without re-walking.
|
||
let needs_size: Vec<NodeUid> = filtered_nodes
|
||
.iter()
|
||
.filter(|n| {
|
||
matches!(
|
||
&n.kind,
|
||
NodeKind::File {
|
||
claimed_size: None,
|
||
..
|
||
}
|
||
)
|
||
})
|
||
.map(|n| n.uid.clone())
|
||
.collect();
|
||
let inos = st.intern_batch(ino, filtered_nodes);
|
||
child_inos.extend(inos);
|
||
st.children.insert(ino, child_inos);
|
||
// Record the listing as complete so a later restart (or a trimmed hot
|
||
// cache) can rebuild it from the DB without the API.
|
||
if let Err(e) = self.db.set_listed(&folder_uid, true) {
|
||
warn!(%folder_uid, error = %e, "db set_listed(true) failed");
|
||
}
|
||
drop(st);
|
||
self.spawn_size_upgrade(ino, needs_size);
|
||
Ok(())
|
||
}
|
||
|
||
/// Resolve the real size of every file in `parent`'s listing still carrying
|
||
/// a provisional one, returning when they are in `state`.
|
||
///
|
||
/// Called when a `stat` lands on such a file. That covers the paths
|
||
/// [`Core::ensure_children`] cannot: a listing rebuilt from the DB, or one
|
||
/// restored by [`Core::hydrate`] on mount, whose rows were persisted before
|
||
/// an earlier upgrade had a chance to run. Gathers the whole folder rather
|
||
/// than the one file asked for, because a `stat` of one entry in a listing
|
||
/// almost always means a `stat` of all of them.
|
||
fn upgrade_sizes_for_parent(&self, ino: u64, uid: &NodeUid, parent: u64) {
|
||
let provisional = |e: &Entry| {
|
||
matches!(
|
||
&e.node.kind,
|
||
NodeKind::File {
|
||
claimed_size: None,
|
||
..
|
||
}
|
||
)
|
||
};
|
||
let (key, mut missing): (u64, Vec<NodeUid>) = {
|
||
let st = self.state.lock();
|
||
match st.children.get(&parent) {
|
||
// The listing is resident: batch the whole folder under its
|
||
// inode, so the rest of an `ls -l` rides along on this fetch.
|
||
Some(kids) => (
|
||
parent,
|
||
kids.iter()
|
||
.filter_map(|k| st.entries.get(k))
|
||
.filter(|e| provisional(e))
|
||
.map(|e| e.uid.clone())
|
||
.collect(),
|
||
),
|
||
// It is not, and returning here is what let a provisional size
|
||
// reach the caller anyway: a rename invalidates its parents'
|
||
// listings, so a freshly renamed file always landed in this
|
||
// branch. Resolve the single node instead, keyed by its own
|
||
// inode — folder and file inodes share one space, so the two
|
||
// single-flight keys cannot collide. (Same shape as B4: an
|
||
// early return that assumed the hot cache was authoritative.)
|
||
None => (ino, Vec::new()),
|
||
}
|
||
};
|
||
// The node that was actually asked about is never optional, whichever
|
||
// branch produced the batch.
|
||
if !missing.iter().any(|u| u == uid) {
|
||
missing.push(uid.clone());
|
||
}
|
||
// Blocking, not spawning: the caller is a `getattr` that must not answer
|
||
// with a provisional size (bugs.md B14).
|
||
self.upgrade_sizes(key, missing, Some(ino));
|
||
}
|
||
|
||
/// Fill in the true sizes of files a `Light` enumeration returned without
|
||
/// one, on a worker, after the listing has already been served.
|
||
///
|
||
/// This is the other half of the split in [`Core::ensure_children`]. The
|
||
/// listing itself needs only names and parentage, so it is served from the
|
||
/// cheap enumeration immediately; the S2K-per-file work that produces
|
||
/// `claimed_size` happens here, off the path the user is waiting on.
|
||
///
|
||
/// **Sizes are provisional until this lands.** `node_size` falls back to
|
||
/// `total_size_on_storage`, the *ciphertext* size, which is slightly larger
|
||
/// than the real one. Reads are unaffected — the revision reader carries its
|
||
/// own authoritative size — so this is a cosmetic discrepancy in `stat` that
|
||
/// closes within a round trip, not a repeat of B11 (which reported **0** and
|
||
/// so suppressed reads entirely).
|
||
///
|
||
/// Single-flight per folder: a `stat` of every entry in a fresh listing is
|
||
/// the normal case, and each one must not start its own upgrade.
|
||
fn spawn_size_upgrade(&self, folder_ino: u64, uids: Vec<NodeUid>) {
|
||
self.upgrade_sizes(folder_ino, uids, None);
|
||
}
|
||
|
||
/// Start (or join) the size upgrade for `key`, and — when `waiting_for` names
|
||
/// an inode — block until that node's real size has landed.
|
||
///
|
||
/// Single-flight per `key`. The fetch runs on its own thread rather than on
|
||
/// the [`Workers`] pool: callers wait on `Lane::Meta`, so a batch queued onto
|
||
/// that lane could have a wide enough `ls -l` fill it with threads waiting
|
||
/// for a job that can never be scheduled. `Lane::Transfer` would swap that
|
||
/// for starvation behind bulk reads. One short-lived thread per folder,
|
||
/// bounded by the single-flight, avoids both.
|
||
fn upgrade_sizes(&self, key: u64, uids: Vec<NodeUid>, waiting_for: Option<u64>) {
|
||
if uids.is_empty() {
|
||
return;
|
||
}
|
||
let slot = {
|
||
let mut in_flight = self.size_upgrades.lock();
|
||
match in_flight.get(&key) {
|
||
// Someone else is already fetching this folder; their batch
|
||
// covers us, so just wait on it.
|
||
Some(existing) => existing.clone(),
|
||
None => {
|
||
let slot = Arc::new(SizeUpgrade::default());
|
||
in_flight.insert(key, slot.clone());
|
||
let core = self.clone();
|
||
let worker = slot.clone();
|
||
std::thread::spawn(move || {
|
||
core.run_size_upgrade(key, uids, &worker);
|
||
});
|
||
slot
|
||
}
|
||
}
|
||
};
|
||
let Some(ino) = waiting_for else {
|
||
return;
|
||
};
|
||
slot.wait_for(|| self.size_is_real(ino));
|
||
}
|
||
|
||
/// Fetch `uids` in chunks, applying and announcing each one as it lands.
|
||
///
|
||
/// Chunked so a waiter is released as soon as *its* file is resolved. A
|
||
/// single 793-node batch took ~80 s, which outran [`SizeUpgrade::WAIT`] and
|
||
/// put provisional sizes back in front of callers — the bug this was
|
||
/// supposed to fix (bugs.md B14).
|
||
fn run_size_upgrade(&self, key: u64, uids: Vec<NodeUid>, slot: &SizeUpgrade) {
|
||
for chunk in uids.chunks(SIZE_UPGRADE_CHUNK) {
|
||
let result = self.rt.block_on(self.client.enumerate_nodes(chunk));
|
||
self.apply_size_upgrade(key, result);
|
||
slot.chunk_done();
|
||
}
|
||
// Both of these must happen however the loop ended: a folder whose
|
||
// upgrade failed has to be retryable, and its waiters released.
|
||
self.size_upgrades.lock().remove(&key);
|
||
slot.finish();
|
||
}
|
||
|
||
/// Whether `ino` has a real size — the condition a waiter is waiting on.
|
||
/// A node that vanished counts as resolved; there is nothing left to wait
|
||
/// for, and its caller will find the `ENOENT` for itself.
|
||
fn size_is_real(&self, ino: u64) -> bool {
|
||
let st = self.state.lock();
|
||
st.entries.get(&ino).is_none_or(|e| {
|
||
!matches!(
|
||
&e.node.kind,
|
||
NodeKind::File {
|
||
claimed_size: None,
|
||
..
|
||
}
|
||
)
|
||
})
|
||
}
|
||
|
||
/// Adopt the sizes a completed upgrade fetched. Split from
|
||
/// [`Core::upgrade_sizes`] so the single-flight bookkeeping there has one
|
||
/// exit path rather than one per early return.
|
||
fn apply_size_upgrade(&self, folder_ino: u64, result: Result<Vec<Node>, ProtonError>) {
|
||
let core = self;
|
||
{
|
||
let mut nodes = match result {
|
||
Ok(nodes) => nodes,
|
||
Err(e) => {
|
||
// Not fatal: the listing stands, sizes just stay provisional
|
||
// until something invalidates and re-enumerates it.
|
||
warn!(folder_ino, error = %e, "size upgrade failed; sizes stay provisional");
|
||
return;
|
||
}
|
||
};
|
||
// A queued write is ahead of anything the server can report, so its
|
||
// optimistic size must survive this just as it survives a re-listing
|
||
// (B11).
|
||
core.stamp_pending_sizes(&mut nodes);
|
||
let mut changed: Vec<u64> = Vec::new();
|
||
let mut st = core.state.lock();
|
||
for node in nodes {
|
||
// Only adopt the size. Re-interning wholesale would also adopt a
|
||
// name or parent that a rename/move may have changed locally
|
||
// while this was in flight, undoing it.
|
||
let Some(&ino) = st.by_uid.get(&node.uid) else {
|
||
continue;
|
||
};
|
||
changed.push(ino);
|
||
let NodeKind::File { claimed_size, .. } = &node.kind else {
|
||
continue;
|
||
};
|
||
let (Some(size), Some(entry)) = (*claimed_size, st.entries.get_mut(&ino)) else {
|
||
continue;
|
||
};
|
||
if let NodeKind::File { claimed_size, .. } = &mut entry.node.kind {
|
||
*claimed_size = Some(size);
|
||
}
|
||
}
|
||
let updated: Vec<Node> = st
|
||
.children
|
||
.get(&folder_ino)
|
||
.map(|kids| {
|
||
kids.iter()
|
||
.filter_map(|k| st.entries.get(k).map(|e| e.node.clone()))
|
||
.collect()
|
||
})
|
||
.unwrap_or_default();
|
||
drop(st);
|
||
if let Err(e) = core.db.upsert_nodes(&updated) {
|
||
warn!(folder_ino, error = %e, "persisting upgraded sizes failed");
|
||
}
|
||
// Without this the corrected size is invisible for the length of the
|
||
// attr TTL: the kernel answers `stat` from the provisional attrs it
|
||
// cached while the listing was being served, so `ls -l` reports the
|
||
// ciphertext size for up to 30 s even though the daemon has had the
|
||
// real one all along. Notify *after* the DB write, so a re-`getattr`
|
||
// provoked by the invalidation cannot race the persistence.
|
||
if let Some(notifier) = core.notifier.get() {
|
||
for ino in changed {
|
||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||
}
|
||
}
|
||
debug!(folder_ino, files = updated.len(), "filled in listing sizes");
|
||
}
|
||
}
|
||
|
||
/// Resolve a child `name` within `parent` to its `(inode, uid)`, ensuring
|
||
/// the parent's listing is cached first.
|
||
fn lookup_child(&self, parent: u64, name: &str) -> Result<(u64, NodeUid), Errno> {
|
||
self.ensure_children(parent)?;
|
||
let st = self.state.lock();
|
||
st.children
|
||
.get(&parent)
|
||
.and_then(|kids| {
|
||
kids.iter().copied().find_map(|ino| {
|
||
st.entries
|
||
.get(&ino)
|
||
.filter(|e| e.node.name == name)
|
||
.map(|e| (ino, e.uid.clone()))
|
||
})
|
||
})
|
||
.ok_or(Errno::ENOENT)
|
||
}
|
||
|
||
/// Walk a mountpoint-relative path to its `(inode, uid)`, enumerating each
|
||
/// directory on the way as needed. Leading `/` and `.` components are
|
||
/// ignored; `..` is rejected.
|
||
fn resolve_path(&self, rel: &Path) -> Result<(u64, NodeUid), Errno> {
|
||
let mut ino = ROOT_INO;
|
||
let mut uid = {
|
||
let st = self.state.lock();
|
||
st.entries
|
||
.get(&ROOT_INO)
|
||
.map(|e| e.uid.clone())
|
||
.ok_or(Errno::ENOENT)?
|
||
};
|
||
for comp in rel.components() {
|
||
match comp {
|
||
Component::RootDir | Component::CurDir => continue,
|
||
Component::Normal(name) => {
|
||
let (child_ino, child_uid) = self.lookup_child(ino, &name.to_string_lossy())?;
|
||
ino = child_ino;
|
||
uid = child_uid;
|
||
}
|
||
_ => return Err(Errno::EINVAL),
|
||
}
|
||
}
|
||
Ok((ino, uid))
|
||
}
|
||
|
||
/// [`resolve_path`](Self::resolve_path) for the request-serving side.
|
||
///
|
||
/// The kernel-facing resolver answers in `Errno` because that is what the
|
||
/// FUSE reply needs. A front-end needs the opposite: `{:?}` of a libc
|
||
/// constant is not something to show a user, and "does not exist" and "the
|
||
/// API is down" want different copy and different buttons. This is the one
|
||
/// place that knows enough to tell them apart, so it is the place that does.
|
||
fn resolve(&self, rel: &Path) -> CoreResult<(u64, NodeUid)> {
|
||
self.resolve_path(rel)
|
||
.map_err(|e| self.errno_error(e, &format!("could not resolve {}", rel.display())))
|
||
}
|
||
|
||
/// Classify a failure that arrived as an `Errno`.
|
||
///
|
||
/// The internal paths speak `Errno` because they also serve FUSE, where a
|
||
/// number is the whole vocabulary. Everything crossing the control socket
|
||
/// has to be turned back into something a person can read, and this is the
|
||
/// one place that knows how — a call site holding only an `Errno` has
|
||
/// already lost the context needed to say what went wrong.
|
||
///
|
||
/// `Errno` is neither `PartialEq` nor structural-match, so this compares
|
||
/// raw codes rather than the `libc` constants.
|
||
fn errno_error(&self, e: Errno, what: &str) -> CoreError {
|
||
match e.code() {
|
||
libc::ENOENT => CoreError::not_found(format!("{what}: no such file or folder")),
|
||
libc::EACCES | libc::EPERM => CoreError::denied(format!("{what}: not allowed")),
|
||
// These paths walk the tree lazily, so a cold node needs the API.
|
||
// Offline that surfaces as EIO, which on its own would read to the
|
||
// user as a broken file rather than a missing network.
|
||
libc::EIO if !self.online.load(Ordering::Relaxed) => CoreError::offline(),
|
||
libc::EINVAL => CoreError::invalid(format!("{what}: not a usable path")),
|
||
libc::ENOSPC => CoreError::new(ErrorKind::Quota, format!("{what}: out of space")),
|
||
libc::EEXIST => CoreError::conflict(format!("{what}: already exists")),
|
||
libc::ENOTEMPTY => CoreError::conflict(format!("{what}: folder is not empty")),
|
||
_ => CoreError::internal(format!("{what}: {e:?}")),
|
||
}
|
||
}
|
||
|
||
/// Fetch a single node's current metadata from the remote.
|
||
fn fetch_node(&self, uid: &NodeUid) -> Result<Node, Errno> {
|
||
match self.fetch_node_remote(uid) {
|
||
Ok(Some(node)) => Ok(node),
|
||
Ok(None) => Err(Errno::ENOENT),
|
||
Err(e) => {
|
||
error!(%uid, error = %e, "enumerate node failed");
|
||
Err(Errno::EIO)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// [`Core::fetch_node`] without the collapse to an `Errno`, for the drain:
|
||
/// resolving a conflict turns on *why* a call failed, and "the node is not
|
||
/// there" (`Ok(None)`) is a different outcome from "we could not ask".
|
||
fn fetch_node_remote(&self, uid: &NodeUid) -> Result<Option<Node>, ProtonError> {
|
||
match self
|
||
.rt
|
||
.block_on(self.client.enumerate_nodes(std::slice::from_ref(uid)))
|
||
{
|
||
Ok(nodes) => Ok(nodes.into_iter().next()),
|
||
// An unknown uid is reported either as an empty result or as an
|
||
// outright refusal, depending on the endpoint.
|
||
Err(e) if is_gone(&e) => Ok(None),
|
||
Err(e) => Err(e),
|
||
}
|
||
}
|
||
|
||
/// An open [`RevisionReader`] for `uid`, reusing the cached one when it is
|
||
/// still valid for `(mtime, fsize)` and opening a fresh one otherwise.
|
||
///
|
||
/// Opening resolves the file's link details, ancestor keys, node key (an S2K
|
||
/// unlock) and block table. Doing that once per file rather than once per
|
||
/// block is the whole point: a cold 100 MB read is 25 block misses, which
|
||
/// used to mean 25 full resolutions (50 API calls, 25 unlocks) and now means
|
||
/// one.
|
||
///
|
||
/// Racing reads of one cold file open it **once**: the first caller claims
|
||
/// the slot and the rest await its result. That matters now that `read` is
|
||
/// served off the dispatch loop, because read-ahead on a first touch issues
|
||
/// several parallel reads of the same file as a matter of course.
|
||
async fn revision_reader(
|
||
&self,
|
||
uid: &NodeUid,
|
||
mtime: i64,
|
||
fsize: u64,
|
||
) -> Result<Arc<RevisionReader>, Errno> {
|
||
loop {
|
||
// Decide under the lock, act outside it — network I/O must never run
|
||
// under a std `Mutex`, and an await must never hold one.
|
||
enum Act {
|
||
Hit(Arc<RevisionReader>),
|
||
Join(tokio::sync::watch::Receiver<Option<Result<Arc<RevisionReader>, Errno>>>),
|
||
Lead(
|
||
u64,
|
||
tokio::sync::watch::Sender<Option<Result<Arc<RevisionReader>, Errno>>>,
|
||
),
|
||
}
|
||
let act = {
|
||
let mut readers = self.readers.lock();
|
||
match readers.get_mut(uid) {
|
||
Some(ReaderSlot::Ready(entry))
|
||
if entry.mtime == mtime && entry.size == fsize =>
|
||
{
|
||
entry.last_used = Instant::now();
|
||
Act::Hit(entry.reader.clone())
|
||
}
|
||
Some(ReaderSlot::Pending(p)) if p.mtime == mtime && p.size == fsize => {
|
||
Act::Join(p.rx.clone())
|
||
}
|
||
// Either nothing here, or something for a revision we no
|
||
// longer want (a stale reader, or an open for one). Ours
|
||
// replaces it: the newer `(mtime, size)` is the truth.
|
||
_ => {
|
||
let id = NEXT_OPEN_ID.fetch_add(1, Ordering::Relaxed);
|
||
let (tx, rx) = tokio::sync::watch::channel(None);
|
||
readers.insert(
|
||
uid.clone(),
|
||
ReaderSlot::Pending(PendingOpen {
|
||
id,
|
||
mtime,
|
||
size: fsize,
|
||
rx,
|
||
}),
|
||
);
|
||
Act::Lead(id, tx)
|
||
}
|
||
}
|
||
};
|
||
|
||
match act {
|
||
Act::Hit(reader) => return Ok(reader),
|
||
Act::Join(mut rx) => {
|
||
// `changed()` errors only if the leader dropped its sender
|
||
// without publishing — it panicked. Retry from the top
|
||
// rather than fail: the slot it left is gone, so this pass
|
||
// leads its own open.
|
||
loop {
|
||
if let Some(result) = rx.borrow_and_update().clone() {
|
||
return result;
|
||
}
|
||
if rx.changed().await.is_err() {
|
||
break;
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
Act::Lead(id, tx) => {
|
||
debug!(%uid, mtime, fsize, "opening revision");
|
||
let result = match self.client.open_revision(uid).await {
|
||
Ok(reader) => Ok(Arc::new(reader)),
|
||
Err(e) => {
|
||
warn!(%uid, error = %e, "open_revision failed");
|
||
Err(Errno::EIO)
|
||
}
|
||
};
|
||
{
|
||
let mut readers = self.readers.lock();
|
||
// Publish only into the slot we claimed. If it is gone
|
||
// or belongs to a later attempt, an eviction or a newer
|
||
// revision overtook us: our reader still answers the
|
||
// callers waiting on it, but it must not be cached.
|
||
let ours = matches!(
|
||
readers.get(uid),
|
||
Some(ReaderSlot::Pending(p)) if p.id == id
|
||
);
|
||
if ours {
|
||
match &result {
|
||
Ok(reader) => {
|
||
readers.insert(
|
||
uid.clone(),
|
||
ReaderSlot::Ready(CachedReader {
|
||
reader: reader.clone(),
|
||
mtime,
|
||
size: fsize,
|
||
last_used: Instant::now(),
|
||
}),
|
||
);
|
||
}
|
||
// Never cache a failure: the next read retries.
|
||
Err(_) => {
|
||
readers.remove(uid);
|
||
}
|
||
}
|
||
}
|
||
Self::trim_readers(&mut readers);
|
||
}
|
||
// After the map is updated, so a racer that wakes and
|
||
// re-enters finds the finished slot rather than racing us.
|
||
let _ = tx.send(Some(result.clone()));
|
||
return result;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Bound the reader map: drop least-recently-used entries until it is back
|
||
/// at [`MAX_OPEN_READERS`]. Only `Ready` slots are eviction candidates — a
|
||
/// `Pending` has callers waiting on it and no reader to drop yet.
|
||
fn trim_readers(readers: &mut HashMap<NodeUid, ReaderSlot>) {
|
||
while readers.len() > MAX_OPEN_READERS {
|
||
let victim = readers
|
||
.iter()
|
||
.filter_map(|(uid, slot)| match slot {
|
||
ReaderSlot::Ready(entry) => Some((uid, entry.last_used)),
|
||
ReaderSlot::Pending(_) => None,
|
||
})
|
||
.min_by_key(|(_, last_used)| *last_used)
|
||
.map(|(uid, _)| uid.clone());
|
||
let Some(victim) = victim else {
|
||
break; // every slot is an open in flight; nothing to drop
|
||
};
|
||
readers.remove(&victim);
|
||
}
|
||
}
|
||
|
||
/// Drop any open reader for `uid`, so the next read reopens against the
|
||
/// current revision. Called wherever cached content is evicted.
|
||
///
|
||
/// Drops an in-flight open too: its leader finds the slot gone and declines
|
||
/// to cache what it opened, which is what we want — the eviction says that
|
||
/// revision is no longer the truth.
|
||
fn evict_reader(&self, uid: &NodeUid) {
|
||
self.readers.lock().remove(uid);
|
||
}
|
||
|
||
/// Serve bytes `[offset, offset + len)` of `uid`'s active revision, hitting
|
||
/// the on-disk caches before the network: a whole-file blob (pinned files)
|
||
/// first, then the block cache — fetching only the [`BLOCK_SIZE`]-aligned
|
||
/// blocks that overlap the request and caching each. `mtime`/`fsize` validate
|
||
/// both caches. Network I/O runs without any lock held.
|
||
fn read_range(
|
||
&self,
|
||
uid: &NodeUid,
|
||
mtime: i64,
|
||
fsize: u64,
|
||
offset: u64,
|
||
len: u64,
|
||
cache_blocks: bool,
|
||
) -> Result<Vec<u8>, Errno> {
|
||
// A queued write has not reached the remote yet, so the remote's current
|
||
// revision is stale and the staged blob is the truth. Serve from it until
|
||
// the drain worker lands the upload (offline.md Phase 3).
|
||
if let Some(pending) = self.pending.lock().get(uid).cloned() {
|
||
return self.read_pending(&pending, offset, len);
|
||
}
|
||
// A node created offline and never written has no blob and no remote: it
|
||
// is an empty file, and asking the API about a `local~` uid would only
|
||
// earn a 404 (offline.md Phase 3b).
|
||
if is_local_uid(uid) {
|
||
return Ok(Vec::new());
|
||
}
|
||
self.read_range_remote(uid, mtime, fsize, offset, len, cache_blocks)
|
||
}
|
||
|
||
/// Serve a read from a staged blob, falling back to the remote base for any
|
||
/// range the write did not author (an incomplete [`StagedWrite`] holds zeros
|
||
/// there, which must never be handed out as content).
|
||
fn read_pending(
|
||
&self,
|
||
pending: &PendingRevision,
|
||
offset: u64,
|
||
len: u64,
|
||
) -> Result<Vec<u8>, Errno> {
|
||
let m = &pending.meta;
|
||
if offset >= m.len || len == 0 {
|
||
return Ok(Vec::new());
|
||
}
|
||
let uid = parse_node_uid(&m.uid).ok_or(Errno::EIO)?;
|
||
let file = File::open(&pending.path).map_err(|e| {
|
||
error!(%uid, path = %pending.path.display(), error = %e, "open staged blob failed");
|
||
Errno::EIO
|
||
})?;
|
||
let mut written = Intervals::default();
|
||
for &(s, e) in &m.authored {
|
||
written.add(s, e);
|
||
}
|
||
// Same merge as `serve_open_read`, but resolving gaps against the remote
|
||
// rather than through `read_range` — going through `read_range` would find
|
||
// this very pending op and recurse.
|
||
let end = offset.saturating_add(len).min(m.len);
|
||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||
for (s, e, authored) in written.segments(offset, end) {
|
||
if authored {
|
||
let mut buf = vec![0u8; (e - s) as usize];
|
||
file.read_exact_at(&mut buf, s).map_err(|err| {
|
||
warn!(%uid, error = %err, "staged blob read failed");
|
||
Errno::EIO
|
||
})?;
|
||
out.extend_from_slice(&buf);
|
||
continue;
|
||
}
|
||
let bend = e.min(m.base_size);
|
||
if s < bend {
|
||
out.extend_from_slice(&self.read_range_remote(
|
||
&uid,
|
||
m.base_mtime,
|
||
m.base_size,
|
||
s,
|
||
bend - s,
|
||
true,
|
||
)?);
|
||
}
|
||
// Past the base: a hole the write extended over.
|
||
out.resize(out.len() + e.saturating_sub(s.max(m.base_size)) as usize, 0);
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// Warm the [`STREAM_READAHEAD`] blocks after `last` into the ring, in the
|
||
/// background, so a player reading forward finds its next block already in
|
||
/// memory rather than paying a fetch + decrypt round-trip at each boundary.
|
||
///
|
||
/// Fire-and-forget: a failure here is not an error, it only means the read
|
||
/// that needs those bytes will fetch them itself.
|
||
fn stream_readahead(&self, uid: &NodeUid, mtime: i64, fsize: u64, last: u64) {
|
||
let last_block = (fsize.saturating_sub(1)) / BLOCK_SIZE;
|
||
let wanted: Vec<u64> = ((last + 1)..=(last + STREAM_READAHEAD).min(last_block))
|
||
.filter(|&bidx| {
|
||
self.stream_ring
|
||
.lock()
|
||
.get(uid, mtime, fsize, bidx)
|
||
.is_none()
|
||
})
|
||
.collect();
|
||
if wanted.is_empty() {
|
||
return;
|
||
}
|
||
let core = self.clone();
|
||
let uid = uid.clone();
|
||
self.rt.spawn(async move {
|
||
let Ok(reader) = core.revision_reader(&uid, mtime, fsize).await else {
|
||
return;
|
||
};
|
||
// Concurrently, for the same reason the demand path fetches its
|
||
// misses concurrently: serially awaited blocks cost the sum of their
|
||
// round-trips, and the player catches up with the readahead before it
|
||
// finishes. The SDK's in-flight block governor bounds the fan-out.
|
||
let mut set = tokio::task::JoinSet::new();
|
||
for bidx in wanted {
|
||
let reader = reader.clone();
|
||
let bstart = bidx * BLOCK_SIZE;
|
||
let blen = BLOCK_SIZE.min(fsize - bstart);
|
||
set.spawn(async move { (bidx, reader.read_at(bstart, blen).await) });
|
||
}
|
||
while let Some(joined) = set.join_next().await {
|
||
let Ok((bidx, read)) = joined else { continue };
|
||
match read {
|
||
Ok(bytes) => {
|
||
core.stream_ring
|
||
.lock()
|
||
.insert(&uid, mtime, fsize, bidx, Arc::new(bytes))
|
||
}
|
||
Err(e) => debug!(%uid, bidx, error = %e, "stream readahead failed"),
|
||
}
|
||
}
|
||
});
|
||
}
|
||
|
||
/// Read from the content cache, else the remote. The base-content path, with
|
||
/// no awareness of queued writes — callers wanting the file's *current*
|
||
/// content want [`Core::read_range`]. Gap-filling a staged blob is the one
|
||
/// caller that genuinely means "the base", since that is what its zeroed
|
||
/// ranges have to be filled from.
|
||
fn read_range_remote(
|
||
&self,
|
||
uid: &NodeUid,
|
||
mtime: i64,
|
||
fsize: u64,
|
||
offset: u64,
|
||
len: u64,
|
||
cache_blocks: bool,
|
||
) -> Result<Vec<u8>, Errno> {
|
||
if let Some(bytes) = self.cache.read_range(uid, mtime, fsize, offset, len) {
|
||
return Ok(bytes);
|
||
}
|
||
if offset >= fsize || len == 0 {
|
||
return Ok(Vec::new());
|
||
}
|
||
let end = offset.saturating_add(len).min(fsize);
|
||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||
let first = offset / BLOCK_SIZE;
|
||
let last = (end - 1) / BLOCK_SIZE;
|
||
|
||
// Collect the blocks overlapping the request, serving any already cached
|
||
// and fetching the rest concurrently. A multi-block read (e.g. a media
|
||
// player buffering, or a large sequential read split into one FUSE call)
|
||
// would otherwise stall on each block round-trip in turn; downloading the
|
||
// misses in parallel saturates the connection and bounds latency at the
|
||
// slowest single block instead of their sum.
|
||
let mut blocks: Vec<Option<Arc<Vec<u8>>>> = Vec::with_capacity((last - first + 1) as usize);
|
||
let mut misses: Vec<u64> = Vec::new();
|
||
for bidx in first..=last {
|
||
// A streaming read keeps its blocks only in memory, so the ring is
|
||
// the only cache it has — check it before the disk (which will not
|
||
// hold them) and before the network (which would refetch the same
|
||
// 4 MiB block for every 128 KiB the player consumes).
|
||
let hit = if cache_blocks {
|
||
self.cache
|
||
.cached_block(uid, mtime, fsize, bidx)
|
||
.map(Arc::new)
|
||
} else {
|
||
self.stream_ring.lock().get(uid, mtime, fsize, bidx)
|
||
};
|
||
match hit {
|
||
Some(b) => blocks.push(Some(b)),
|
||
None => {
|
||
blocks.push(None);
|
||
misses.push(bidx);
|
||
}
|
||
}
|
||
}
|
||
|
||
if !misses.is_empty() {
|
||
let fetched = self.rt.block_on(async {
|
||
// Resolve the file's keys and block table once, then read every
|
||
// missing block through the shared reader. Previously each block
|
||
// called `download_range`, which redid that resolution per block.
|
||
let reader = self.revision_reader(uid, mtime, fsize).await?;
|
||
|
||
let mut set = tokio::task::JoinSet::new();
|
||
for &bidx in &misses {
|
||
let reader = reader.clone();
|
||
let uid = uid.clone();
|
||
let bstart = bidx * BLOCK_SIZE;
|
||
let blen = BLOCK_SIZE.min(fsize - bstart);
|
||
set.spawn(async move {
|
||
reader
|
||
.read_at(bstart, blen)
|
||
.await
|
||
.map(|bytes| (bidx, bytes))
|
||
.map_err(|e| {
|
||
warn!(%uid, bstart, blen, error = %e, "block read failed");
|
||
Errno::EIO
|
||
})
|
||
});
|
||
}
|
||
let mut out = Vec::with_capacity(misses.len());
|
||
while let Some(joined) = set.join_next().await {
|
||
// A join error means the task panicked; surface it as EIO.
|
||
out.push(joined.map_err(|_| Errno::EIO)??);
|
||
}
|
||
Ok::<_, Errno>(out)
|
||
})?;
|
||
for (bidx, bytes) in fetched {
|
||
// A streaming read (large unpinned video) skips the on-disk cache
|
||
// so it can't evict the rest of it; it keeps the block in the
|
||
// in-memory ring instead, which is bounded and dies with the read.
|
||
let bytes = Arc::new(bytes);
|
||
if cache_blocks {
|
||
let _ = self.cache.store_block(uid, mtime, fsize, bidx, &bytes);
|
||
} else {
|
||
self.stream_ring
|
||
.lock()
|
||
.insert(uid, mtime, fsize, bidx, bytes.clone());
|
||
}
|
||
blocks[(bidx - first) as usize] = Some(bytes);
|
||
}
|
||
if !cache_blocks {
|
||
self.stream_readahead(uid, mtime, fsize, last);
|
||
}
|
||
}
|
||
|
||
for (i, block) in blocks.into_iter().enumerate() {
|
||
let bidx = first + i as u64;
|
||
let bstart = bidx * BLOCK_SIZE;
|
||
// Every slot is populated: cache hits up front, misses by the fetch above.
|
||
let block = block.expect("block fetched or cached");
|
||
let s = (offset.max(bstart) - bstart) as usize;
|
||
let e = (end.min(bstart + block.len() as u64) - bstart) as usize;
|
||
if s < e {
|
||
out.extend_from_slice(&block[s..e]);
|
||
}
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// Serve a read against an open write handle: stitch authored ranges (from
|
||
/// the scratch file) and untouched ranges (from the remote base via the
|
||
/// block cache) in order, zero-filling any region past the base.
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn serve_open_read(
|
||
&self,
|
||
file: &Arc<File>,
|
||
len: u64,
|
||
uid: &NodeUid,
|
||
base_mtime: i64,
|
||
base_size: u64,
|
||
written: &Intervals,
|
||
offset: u64,
|
||
size: u64,
|
||
) -> Result<Vec<u8>, Errno> {
|
||
if offset >= len || size == 0 {
|
||
return Ok(Vec::new());
|
||
}
|
||
let end = offset.saturating_add(size).min(len);
|
||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||
for (s, e, authored) in written.segments(offset, end) {
|
||
if authored {
|
||
let mut buf = vec![0u8; (e - s) as usize];
|
||
file.read_exact_at(&mut buf, s).map_err(|err| {
|
||
warn!(%uid, error = %err, "scratch read failed");
|
||
Errno::EIO
|
||
})?;
|
||
out.extend_from_slice(&buf);
|
||
} else {
|
||
let bend = e.min(base_size);
|
||
if s < bend {
|
||
out.extend_from_slice(&self.read_range(
|
||
uid,
|
||
base_mtime,
|
||
base_size,
|
||
s,
|
||
bend - s,
|
||
true,
|
||
)?);
|
||
}
|
||
// Anything past the base is a hole: zero-fill.
|
||
let zeros = e.saturating_sub(s.max(base_size));
|
||
out.resize(out.len() + zeros as usize, 0);
|
||
}
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// Fill every unauthored range of a scratch/staged file that overlaps its
|
||
/// base with the base's bytes, so the file becomes the complete new content.
|
||
///
|
||
/// This is the step a partial overwrite cannot skip: only the authored bytes
|
||
/// were ever written to disk, and a revision upload sends the whole file.
|
||
/// The gaps come from the *remote base* (through the block cache, so a small
|
||
/// edit of a large file does not pull all of it), which is exactly why this
|
||
/// can fail with no network — see `StagedWrite`.
|
||
fn fill_gaps(
|
||
&self,
|
||
uid: &NodeUid,
|
||
file: &File,
|
||
len: u64,
|
||
base_mtime: i64,
|
||
base_size: u64,
|
||
written: &Intervals,
|
||
) -> Result<(), Errno> {
|
||
file.set_len(len).map_err(|e| {
|
||
error!(%uid, error = %e, "resize scratch file failed");
|
||
Errno::EIO
|
||
})?;
|
||
for (s, e, authored) in written.segments(0, len) {
|
||
if authored {
|
||
continue;
|
||
}
|
||
let bend = e.min(base_size);
|
||
if s >= bend {
|
||
continue; // wholly past the base: already zero-filled on disk
|
||
}
|
||
let bytes = self.read_range_remote(uid, base_mtime, base_size, s, bend - s, true)?;
|
||
file.write_all_at(&bytes, s).map_err(|err| {
|
||
error!(%uid, error = %err, "scratch gap-fill write failed");
|
||
Errno::EIO
|
||
})?;
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Accept a released write handle's bytes and queue their upload
|
||
/// (offline.md Phase 3).
|
||
///
|
||
/// This is what makes a copy into the mount run at disk speed: the caller's
|
||
/// `close` returns once the bytes are staged on local disk and the intent is
|
||
/// in `pending_op`, instead of waiting out a full upload inside the FUSE
|
||
/// handler. It is also what makes an offline write succeed rather than EIO —
|
||
/// the queued op simply waits for the network.
|
||
///
|
||
/// The scratch file is *moved* into staging, never copied: it is the only
|
||
/// copy of what the user wrote.
|
||
fn queue_revision(&self, h: &WriteHandle) -> Result<(), Errno> {
|
||
// The handle is being retired either way, so any durability sidecar an
|
||
// `fsync` left has done its job: from here the bytes are tracked as a
|
||
// staged write and a queued op, and a sidecar outliving them would offer
|
||
// recovery a second, stale copy of the same write.
|
||
self.cache.clear_scratch_durable(&h.path);
|
||
if !h.dirty {
|
||
let _ = std::fs::remove_file(&h.path);
|
||
return Ok(());
|
||
}
|
||
// Materialize the full content now if we can. A complete blob is
|
||
// uploadable without the network and, crucially, lets a later write to
|
||
// the same file supersede this one safely.
|
||
//
|
||
// A node that exists only locally has no remote base to fill from: its
|
||
// untouched ranges live in the blob queued by an earlier write, not on
|
||
// any server. Filling is only safe — and only needed — while it is still
|
||
// empty, which is exactly when `fill_gaps` skips the network anyway.
|
||
let filled = if is_local_uid(&h.uid) && h.base_size > 0 {
|
||
if let Err(e) = h.file.set_len(h.len) {
|
||
error!(uid = %h.uid, error = %e, "resize scratch file failed");
|
||
return Err(Errno::EIO);
|
||
}
|
||
false
|
||
} else {
|
||
self.fill_gaps(
|
||
&h.uid,
|
||
&h.file,
|
||
h.len,
|
||
h.base_mtime,
|
||
h.base_size,
|
||
&h.written,
|
||
)
|
||
.is_ok()
|
||
};
|
||
let authored: Vec<(u64, u64)> = if filled {
|
||
vec![(0, h.len)]
|
||
} else {
|
||
h.written
|
||
.segments(0, h.len)
|
||
.into_iter()
|
||
.filter(|&(_, _, authored)| authored)
|
||
.map(|(s, e, _)| (s, e))
|
||
.collect()
|
||
};
|
||
let meta = StagedWrite {
|
||
uid: h.uid.to_string(),
|
||
len: h.len,
|
||
base_size: h.base_size,
|
||
base_mtime: h.base_mtime,
|
||
complete: authored == [(0, h.len)],
|
||
authored,
|
||
based_on: self.remote_baseline(&h.uid, h.base_mtime, h.base_size),
|
||
};
|
||
self.enqueue_staged_write(&h.uid, h.ino, &h.path, meta)?;
|
||
debug!(uid = %h.uid, len = h.len, complete = filled, "queued revision upload");
|
||
Ok(())
|
||
}
|
||
|
||
/// The remote revision a change to `uid` is being made against, for
|
||
/// [`StagedWrite::based_on`].
|
||
///
|
||
/// Normally that is simply the base the handle opened over. The exception is
|
||
/// a write that supersedes a still-queued one: its "base" is the previous
|
||
/// *staged blob*, whose size and mtime are ours, not the server's — so the
|
||
/// baseline is inherited from the op being superseded, which is the last one
|
||
/// that actually observed the remote. Without that, chaining two writes
|
||
/// before the queue drains would leave the drain comparing the remote
|
||
/// against a revision it never had, and cutting a conflict copy over
|
||
/// nothing.
|
||
///
|
||
/// `None` for a node that has never existed remotely: there is no revision
|
||
/// to conflict with until its create drains.
|
||
fn remote_baseline(&self, uid: &NodeUid, base_mtime: i64, base_size: u64) -> Option<Baseline> {
|
||
if is_local_uid(uid) {
|
||
return None;
|
||
}
|
||
match self.pending.lock().get(uid) {
|
||
Some(p) => p.meta.based_on,
|
||
None => Some(Baseline {
|
||
mtime: base_mtime,
|
||
size: base_size,
|
||
}),
|
||
}
|
||
}
|
||
|
||
/// Move a file holding a node's intended new content into staging and queue
|
||
/// the upload that will make it the remote's content. Shared by the release
|
||
/// of a write handle and by a path-based truncate.
|
||
///
|
||
/// `src` is consumed either way: on success it is *moved* into staging, and
|
||
/// on the refusal below it is moved there too, just without an op to upload
|
||
/// it. It is the only copy of what the user wrote, so nothing here may
|
||
/// simply delete it.
|
||
fn enqueue_staged_write(
|
||
&self,
|
||
uid: &NodeUid,
|
||
ino: u64,
|
||
src: &Path,
|
||
meta: StagedWrite,
|
||
) -> Result<(), Errno> {
|
||
// An incomplete blob's gaps refer to the *remote* base. If an earlier
|
||
// write to this file is still queued, the remote no longer holds that
|
||
// base — the previous staged blob does — so superseding it would fill
|
||
// those gaps from the wrong revision. Rather than corrupt the file,
|
||
// refuse the write and keep the bytes recoverable (Phase 2 behaviour).
|
||
// Only reachable offline, editing in place a file whose previous edit
|
||
// has not drained and whose base is not cached.
|
||
if !meta.complete && self.pending.lock().contains_key(uid) {
|
||
self.stage_orphaned_write(uid, ino, src, &meta);
|
||
return Err(Errno::EIO);
|
||
}
|
||
let path = self.cache.stage_write(&meta, src).map_err(|e| {
|
||
error!(%uid, error = %e, "staging write failed");
|
||
Errno::EIO
|
||
})?;
|
||
let meta_json = serde_json::to_string(&meta).unwrap_or_default();
|
||
let superseded = if is_local_uid(uid) {
|
||
// The node has no server-side identity to hang a revision on, so the
|
||
// bytes ride on the create that will mint it.
|
||
let attached = self
|
||
.db
|
||
.attach_blob_to_create(&uid.to_string(), &path.to_string_lossy(), &meta_json)
|
||
.map_err(|e| {
|
||
error!(%uid, error = %e, "attaching write to queued create failed");
|
||
Errno::EIO
|
||
})?;
|
||
match attached {
|
||
Some(a) => a.superseded,
|
||
None => {
|
||
// The create drained between `release` and here, so the node
|
||
// has a real uid now and this handle's is stale. The bytes are
|
||
// safe in staging, but nothing here can address them.
|
||
error!(%uid, staged = %path.display(),
|
||
"queued create vanished under a write; bytes kept in staging");
|
||
return Err(Errno::EIO);
|
||
}
|
||
}
|
||
} else {
|
||
let op = PendingOp {
|
||
id: 0,
|
||
kind: OP_REVISION.to_string(),
|
||
uid: uid.to_string(),
|
||
parent_uid: None,
|
||
name: None,
|
||
blob_path: Some(path.to_string_lossy().into_owned()),
|
||
meta_json: Some(meta_json),
|
||
created_at: now_millis(),
|
||
attempts: 0,
|
||
last_error: None,
|
||
next_attempt_at: 0,
|
||
};
|
||
let (_id, superseded) = self.db.enqueue_op(&op).map_err(|e| {
|
||
error!(%uid, error = %e, "queueing upload failed");
|
||
Errno::EIO
|
||
})?;
|
||
superseded
|
||
};
|
||
if let Some(old) = superseded {
|
||
self.cache.discard_staged(Path::new(&old));
|
||
}
|
||
let len = meta.len;
|
||
self.pending
|
||
.lock()
|
||
.insert(uid.clone(), PendingRevision { path, meta });
|
||
// Reflect the write in the tree straight away: `ls` must show the new
|
||
// size and mtime even though the remote still holds the old revision.
|
||
let now = now_secs();
|
||
{
|
||
let mut st = self.state.lock();
|
||
st.record_pending_write(ino, len, now);
|
||
}
|
||
// Cached blobs and open readers describe the superseded revision. Reads
|
||
// come from the staged blob until the op drains, so just drop them.
|
||
self.cache.evict(uid);
|
||
self.evict_reader(uid);
|
||
self.wake_drain();
|
||
Ok(())
|
||
}
|
||
|
||
/// Queue the new content of a path-based truncate — `> file`, or any
|
||
/// `setattr(size=…)` arriving without a write handle.
|
||
///
|
||
/// No bytes have been authored at truncate time, which is why this path was
|
||
/// never staged and instead resized the remote content inline. That is also
|
||
/// why a shell redirect failed offline *before* the write that follows it:
|
||
/// the truncate itself needed the network. Staging a blob describing the
|
||
/// result puts it on the same queue as every other write.
|
||
///
|
||
/// The blob is a hole of the new length; what is real about it is `authored`:
|
||
///
|
||
/// - `> file` (size 0) is the whole point and needs nothing at all — an
|
||
/// empty file is complete content, so it queues and drains offline.
|
||
/// - Extending past the end authors the new tail (zeros, by definition) and
|
||
/// leaves the head to be gap-filled from the base.
|
||
/// - Shrinking authors nothing: every remaining byte still comes from the
|
||
/// base, so it is the drain that has to fetch it.
|
||
fn queue_truncate(&self, ino: u64, size: u64) -> Result<(), Errno> {
|
||
let (uid, base_mtime, base_size) = {
|
||
let st = self.state.lock();
|
||
match st.entries.get(&ino) {
|
||
Some(e) if e.node.is_file() => {
|
||
(e.uid.clone(), e.node.modification_time, node_size(&e.node))
|
||
}
|
||
Some(_) => return Err(Errno::EISDIR),
|
||
None => return Err(Errno::ENOENT),
|
||
}
|
||
};
|
||
if size == base_size {
|
||
return Ok(());
|
||
}
|
||
let (authored, complete) = if size == 0 {
|
||
// An empty file has no content to be missing.
|
||
(Vec::new(), true)
|
||
} else if base_size == 0 {
|
||
// Nothing to gap-fill from: every byte is a zero this truncate
|
||
// defines.
|
||
(vec![(0, size)], true)
|
||
} else if size > base_size {
|
||
(vec![(base_size, size)], false)
|
||
} else {
|
||
(Vec::new(), false)
|
||
};
|
||
let (file, path) = self.cache.create_scratch().map_err(|e| {
|
||
error!(%uid, error = %e, "create scratch file for truncate failed");
|
||
Errno::EIO
|
||
})?;
|
||
file.set_len(size).map_err(|e| {
|
||
error!(%uid, error = %e, "resize scratch file for truncate failed");
|
||
let _ = std::fs::remove_file(&path);
|
||
Errno::EIO
|
||
})?;
|
||
let meta = StagedWrite {
|
||
uid: uid.to_string(),
|
||
len: size,
|
||
base_size,
|
||
base_mtime,
|
||
authored,
|
||
complete,
|
||
based_on: self.remote_baseline(&uid, base_mtime, base_size),
|
||
};
|
||
self.enqueue_staged_write(&uid, ino, &path, meta)?;
|
||
debug!(%uid, size, complete, "queued truncate");
|
||
Ok(())
|
||
}
|
||
|
||
/// Invent a node under `parent_uid` and queue the op that will make it real
|
||
/// (offline.md Phase 3b). Returns the node to intern, exactly as the online
|
||
/// path returns the one the server minted.
|
||
///
|
||
/// The parent may itself be a placeholder — `mkdir -p` offline, or `cp -r` of
|
||
/// a tree. That is fine: the op records the parent it was made under, and the
|
||
/// parent's own drain rewrites it to the real uid before this op can run.
|
||
fn queue_local_node(
|
||
&self,
|
||
parent_uid: &NodeUid,
|
||
name: &str,
|
||
is_dir: bool,
|
||
) -> Result<Node, Errno> {
|
||
let uid = mint_local_uid();
|
||
let op = PendingOp {
|
||
id: 0,
|
||
kind: if is_dir { OP_MKDIR } else { OP_CREATE }.to_string(),
|
||
uid: uid.to_string(),
|
||
parent_uid: Some(parent_uid.to_string()),
|
||
name: Some(name.to_string()),
|
||
blob_path: None,
|
||
meta_json: None,
|
||
created_at: now_millis(),
|
||
attempts: 0,
|
||
last_error: None,
|
||
next_attempt_at: 0,
|
||
};
|
||
self.db.enqueue_op(&op).map_err(|e| {
|
||
error!(%parent_uid, name, error = %e, "queueing local node failed");
|
||
Errno::EIO
|
||
})?;
|
||
debug!(%uid, %parent_uid, name, is_dir, "created node offline; queued");
|
||
Ok(local_node(
|
||
uid,
|
||
parent_uid.clone(),
|
||
name.to_string(),
|
||
is_dir,
|
||
))
|
||
}
|
||
|
||
/// Queue giving a node a new parent and/or name, and apply it to the tree
|
||
/// now (offline.md Phase 3b).
|
||
///
|
||
/// The op records the desired end state rather than the step, so it both
|
||
/// supersedes any earlier queued rename and lets the drain skip whichever
|
||
/// half the remote already agrees with.
|
||
///
|
||
/// `new_parent_uid` may be a placeholder — moving a file into a folder
|
||
/// created offline — which is why this cannot simply be the online call with
|
||
/// a retry around it: the API would 404 on a `local~` parent. The op waits
|
||
/// for that folder's drain to rewrite it, exactly as a queued create does.
|
||
fn queue_rename(
|
||
&self,
|
||
ino: u64,
|
||
uid: &NodeUid,
|
||
new_parent_ino: u64,
|
||
new_parent_uid: &NodeUid,
|
||
new_name: &str,
|
||
) -> Result<(), Errno> {
|
||
let op = PendingOp {
|
||
id: 0,
|
||
kind: OP_RENAME.to_string(),
|
||
uid: uid.to_string(),
|
||
parent_uid: Some(new_parent_uid.to_string()),
|
||
name: Some(new_name.to_string()),
|
||
blob_path: None,
|
||
meta_json: None,
|
||
created_at: now_millis(),
|
||
attempts: 0,
|
||
last_error: None,
|
||
next_attempt_at: 0,
|
||
};
|
||
self.db.enqueue_op(&op).map_err(|e| {
|
||
error!(%uid, new_name, error = %e, "queueing rename failed");
|
||
Errno::EIO
|
||
})?;
|
||
self.state
|
||
.lock()
|
||
.rename_in_place(ino, new_parent_ino, new_parent_uid, new_name);
|
||
self.wake_drain();
|
||
debug!(%uid, %new_parent_uid, new_name, "renamed offline; queued");
|
||
Ok(())
|
||
}
|
||
|
||
/// Queue trashing a node the server knows about, and drop it from the tree
|
||
/// now (offline.md Phase 3b).
|
||
///
|
||
/// Anything else queued for this node is discarded first: the user has said
|
||
/// the file should not exist, so uploading bytes to it or renaming it are
|
||
/// both work towards an outcome nobody wants any more. That does throw away
|
||
/// staged bytes that never landed — which is precisely what deleting an
|
||
/// un-uploaded file means, and the alternative (upload it, then trash it) is
|
||
/// worse in every way.
|
||
fn queue_trash(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> {
|
||
self.discard_queued_ops(uid);
|
||
let op = PendingOp {
|
||
id: 0,
|
||
kind: OP_TRASH.to_string(),
|
||
uid: uid.to_string(),
|
||
parent_uid: None,
|
||
name: Some(name.to_string()),
|
||
blob_path: None,
|
||
meta_json: None,
|
||
created_at: now_millis(),
|
||
attempts: 0,
|
||
last_error: None,
|
||
next_attempt_at: 0,
|
||
};
|
||
self.db.enqueue_op(&op).map_err(|e| {
|
||
error!(%uid, error = %e, "queueing trash failed");
|
||
Errno::EIO
|
||
})?;
|
||
self.state.lock().forget(uid);
|
||
self.cache.evict(uid);
|
||
self.evict_reader(uid);
|
||
self.wake_drain();
|
||
debug!(%uid, name, "trashed offline; queued");
|
||
Ok(())
|
||
}
|
||
|
||
/// Drop every op queued against a node, and the staged bytes they own.
|
||
fn discard_queued_ops(&self, uid: &NodeUid) {
|
||
match self.db.delete_ops_for_uid(&uid.to_string()) {
|
||
Ok(blobs) => {
|
||
for blob in blobs {
|
||
self.cache.discard_staged(Path::new(&blob));
|
||
}
|
||
}
|
||
Err(e) => error!(%uid, error = %e, "dropping queued ops failed"),
|
||
}
|
||
self.pending.lock().remove(uid);
|
||
}
|
||
|
||
/// Nudge the drain worker to re-examine the queue now.
|
||
fn wake_drain(&self) {
|
||
let (lock, cv) = &*self.drain_wake;
|
||
*lock.lock() = true;
|
||
cv.notify_all();
|
||
}
|
||
|
||
/// Keep a write we cannot safely queue, so the bytes are recoverable even
|
||
/// though the caller is getting an error. See [`Core::queue_revision`].
|
||
fn stage_orphaned_write(&self, uid: &NodeUid, ino: u64, src: &Path, meta: &StagedWrite) {
|
||
match self.cache.stage_write(meta, src) {
|
||
Ok(staged) => {
|
||
error!(
|
||
%uid,
|
||
staged = %staged.display(),
|
||
"cannot queue write over an undrained edit; bytes kept in staging"
|
||
);
|
||
let name = {
|
||
let st = self.state.lock();
|
||
st.entries
|
||
.get(&ino)
|
||
.map(|e| e.node.name.clone())
|
||
.unwrap_or_default()
|
||
};
|
||
self.log_activity(
|
||
ActivityKind::Upload,
|
||
&name,
|
||
format!("write not queued; changes kept at {}", staged.display()),
|
||
false,
|
||
);
|
||
}
|
||
Err(e) => {
|
||
error!(%uid, error = %e, "staging write failed; bytes lost");
|
||
let _ = std::fs::remove_file(src);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Download a whole file's plaintext, registering the transfer so
|
||
/// `GetQueueStatus` can report its progress. `total` is the expected size
|
||
/// (`0` if unknown). Streams through [`download_file_to`] so each block ticks
|
||
/// the progress counter.
|
||
///
|
||
/// [`download_file_to`]: ProtonDriveClient::download_file_to
|
||
fn download_file_tracked(
|
||
&self,
|
||
uid: &NodeUid,
|
||
name: &str,
|
||
total: u64,
|
||
) -> std::result::Result<Vec<u8>, ProtonError> {
|
||
let guard = self
|
||
.transfers
|
||
.begin(name, uid.to_string(), TransferDirection::Download, total);
|
||
let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard);
|
||
self.rt
|
||
.block_on(self.client.download_file_to(uid, &mut out))?;
|
||
Ok(out.into_inner())
|
||
}
|
||
|
||
/// Like [`download_file_tracked`] for a photo, streaming through the photos
|
||
/// client's [`download_photo_to`].
|
||
///
|
||
/// [`download_file_tracked`]: Core::download_file_tracked
|
||
/// [`download_photo_to`]: ProtonPhotosClient::download_photo_to
|
||
fn download_photo_tracked(
|
||
&self,
|
||
photos: &ProtonPhotosClient,
|
||
uid: &NodeUid,
|
||
name: &str,
|
||
total: u64,
|
||
) -> std::result::Result<Vec<u8>, ProtonError> {
|
||
let guard = self
|
||
.transfers
|
||
.begin(name, uid.to_string(), TransferDirection::Download, total);
|
||
let mut out = CountingWriter::new(Vec::with_capacity(total as usize), &guard);
|
||
self.rt.block_on(photos.download_photo_to(uid, &mut out))?;
|
||
Ok(out.into_inner())
|
||
}
|
||
|
||
/// Pin the node at mountpoint-relative `rel`. A file downloads its full
|
||
/// plaintext into the content cache; a folder records a recursive pin and
|
||
/// downloads every descendant file (selective sync). Returns a human message.
|
||
fn pin(&self, rel: &Path) -> CoreResult<String> {
|
||
let (ino, uid) = self.resolve(rel)?;
|
||
let (name, is_folder, mtime, size) = {
|
||
let st = self.state.lock();
|
||
let e = st
|
||
.entries
|
||
.get(&ino)
|
||
.ok_or_else(|| CoreError::not_found("node vanished"))?;
|
||
(
|
||
e.node.name.clone(),
|
||
e.node.is_folder(),
|
||
e.node.modification_time,
|
||
node_size(&e.node),
|
||
)
|
||
};
|
||
if is_folder {
|
||
// Record the recursive pin first so every descendant is eviction-
|
||
// exempt before we start filling the cache with the subtree.
|
||
self.cache
|
||
.add_pin(&uid, rel, true)
|
||
.map_err(|e| CoreError::from_api(&e, "pin"))?;
|
||
let n = self.pin_subtree(ino)?;
|
||
return Ok(format!("{name} ({n} files)"));
|
||
}
|
||
let bytes = self
|
||
.download_file_tracked(&uid, &name, size)
|
||
.map_err(|e| CoreError::from_api(&e, "download"))?;
|
||
self.cache
|
||
.store(&uid, mtime, size, &bytes)
|
||
.map_err(|e| CoreError::internal(format!("cache store: {e}")))?;
|
||
self.cache
|
||
.add_pin(&uid, rel, false)
|
||
.map_err(|e| CoreError::from_api(&e, "pin"))?;
|
||
Ok(name)
|
||
}
|
||
|
||
/// Download and cache every file in the subtree rooted at folder `ino`,
|
||
/// returning the count cached (already-fresh blobs counted, not re-fetched).
|
||
/// Walks the tree depth-first, enumerating each folder so a cold subtree is
|
||
/// fully discovered; the lock is dropped before each network download.
|
||
fn pin_subtree(&self, ino: u64) -> CoreResult<usize> {
|
||
let mut files: Vec<(NodeUid, String, i64, u64)> = Vec::new();
|
||
let mut stack = vec![ino];
|
||
while let Some(dir) = stack.pop() {
|
||
self.ensure_children(dir)
|
||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||
let st = self.state.lock();
|
||
if let Some(kids) = st.children.get(&dir) {
|
||
for &k in kids {
|
||
if let Some(e) = st.entries.get(&k) {
|
||
if e.node.is_folder() {
|
||
stack.push(k);
|
||
} else {
|
||
files.push((
|
||
e.uid.clone(),
|
||
e.node.name.clone(),
|
||
e.node.modification_time,
|
||
node_size(&e.node),
|
||
));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let mut count = 0;
|
||
for (uid, name, mtime, size) in files {
|
||
if self.cache.is_cached(&uid, mtime, size) {
|
||
count += 1;
|
||
continue;
|
||
}
|
||
match self.download_file_tracked(&uid, &name, size) {
|
||
Ok(bytes) => {
|
||
if self.cache.store(&uid, mtime, size, &bytes).is_ok() {
|
||
count += 1;
|
||
}
|
||
}
|
||
Err(e) => warn!(%uid, error = %e, "pin subtree: download failed"),
|
||
}
|
||
}
|
||
Ok(count)
|
||
}
|
||
|
||
/// Fetch a thumbnail of `ttype` for the file at `ino`, served from the cache
|
||
/// when fresh and otherwise downloaded from the remote and cached. Returns
|
||
/// `Ok(None)` when the node is not a file or has no thumbnail of that type.
|
||
fn thumbnail(&self, ino: u64, ttype: ThumbnailType) -> Result<Option<Vec<u8>>, Errno> {
|
||
let (uid, mtime) = {
|
||
let st = self.state.lock();
|
||
match st.entries.get(&ino) {
|
||
Some(e) if e.node.is_file() => (e.uid.clone(), e.node.modification_time),
|
||
Some(_) => return Ok(None),
|
||
None => return Err(Errno::ENOENT),
|
||
}
|
||
};
|
||
if let Some(bytes) = self.cache.read_thumbnail(&uid, ttype.as_i32(), mtime) {
|
||
return Ok(Some(bytes));
|
||
}
|
||
// "This file has no thumbnail" is an answer worth remembering: without
|
||
// it every listing pays a round trip per file to be told nothing (B5).
|
||
let key = (uid.clone(), ttype.as_i32());
|
||
if self.no_thumbnail.lock().get(&key) == Some(&mtime) {
|
||
return Ok(None);
|
||
}
|
||
let bytes = self
|
||
.rt
|
||
.block_on(self.client.download_thumbnail(&uid, ttype))
|
||
.map_err(|e| {
|
||
warn!(%uid, error = %e, "download thumbnail failed");
|
||
Errno::EIO
|
||
})?;
|
||
match &bytes {
|
||
Some(bytes) => {
|
||
let _ = self
|
||
.cache
|
||
.store_thumbnail(&uid, ttype.as_i32(), mtime, bytes);
|
||
}
|
||
None => {
|
||
let mut misses = self.no_thumbnail.lock();
|
||
// Bounded by clearing rather than by LRU: the entries are two
|
||
// words each, the cap is far above any plausible working set,
|
||
// and re-learning a miss costs one round trip. Not worth a
|
||
// second data structure to order them.
|
||
if misses.len() >= MAX_THUMBNAIL_MISSES {
|
||
misses.clear();
|
||
}
|
||
misses.insert(key, mtime);
|
||
}
|
||
}
|
||
Ok(bytes)
|
||
}
|
||
|
||
/// Unpin the node at `rel`, evicting its cached content. For a folder, also
|
||
/// evicts every descendant's cached blob (the subtree is no longer kept).
|
||
fn unpin(&self, rel: &Path) -> CoreResult<String> {
|
||
let (ino, uid) = self.resolve(rel)?;
|
||
let (name, is_folder) = {
|
||
let st = self.state.lock();
|
||
st.entries
|
||
.get(&ino)
|
||
.map(|e| (e.node.name.clone(), e.node.is_folder()))
|
||
.unwrap_or_default()
|
||
};
|
||
self.cache
|
||
.remove_pin(&uid)
|
||
.map_err(|e| CoreError::from_api(&e, "unpin"))?;
|
||
// A recursively-pinned folder's descendants were eviction-exempt; now
|
||
// that the pin is gone, reclaim their blobs eagerly instead of waiting
|
||
// for budget pressure. Descendants come from the DB node tree.
|
||
if is_folder && let Ok(uids) = self.db.descendants(&uid.to_string()) {
|
||
for s in uids {
|
||
if let Some(u) = parse_uid(&s) {
|
||
self.cache.evict(&u);
|
||
self.evict_reader(&u);
|
||
}
|
||
}
|
||
}
|
||
Ok(name)
|
||
}
|
||
|
||
/// A Photos API handle sharing this Core's Drive client and session, so it
|
||
/// reuses the daemon's single authenticated session rather than logging in
|
||
/// again (Proton refresh tokens are single-use). Cheap — the Drive client
|
||
/// is `Clone` over `Arc`-backed state.
|
||
fn photos(&self) -> ProtonPhotosClient {
|
||
ProtonPhotosClient::from_drive_client(self.client.clone())
|
||
}
|
||
|
||
/// List the directory at mountpoint-relative `rel` for the in-app browser:
|
||
/// the same lazy remote enumeration `readdir` uses, projected into
|
||
/// serializable [`DirEntry`]s (with per-file pin state).
|
||
fn list_dir(&self, rel: &Path) -> CoreResult<Vec<DirEntry>> {
|
||
let (ino, _uid) = self.resolve(rel)?;
|
||
self.ensure_children(ino)
|
||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||
// Snapshot the listing, then drop the lock before touching the on-disk
|
||
// pin registry so a slow disk read doesn't block FUSE metadata ops.
|
||
let rows: Vec<(String, bool, u64, i64, NodeUid)> = {
|
||
let st = self.state.lock();
|
||
st.children
|
||
.get(&ino)
|
||
.map(|kids| {
|
||
kids.iter()
|
||
.filter_map(|k| st.entries.get(k))
|
||
.map(|e| {
|
||
(
|
||
e.node.name.clone(),
|
||
e.node.is_folder(),
|
||
node_size(&e.node),
|
||
e.node.modification_time,
|
||
e.uid.clone(),
|
||
)
|
||
})
|
||
.collect()
|
||
})
|
||
.unwrap_or_default()
|
||
};
|
||
Ok(rows
|
||
.into_iter()
|
||
.map(|(name, is_dir, size, modified, uid)| DirEntry {
|
||
name,
|
||
is_dir,
|
||
size,
|
||
modified,
|
||
pinned: self.cache.is_pinned(&uid),
|
||
cached: !is_dir && self.cache.is_cached(&uid, modified, size),
|
||
uid: uid.to_string(),
|
||
// Listing entries live in the requested dir; the caller derives
|
||
// the path from its name. Left empty.
|
||
path: String::new(),
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// Full-text search node names against the local SQLite index, mapping each
|
||
/// DB hit to the wire [`SearchHit`] (resolving live pin state from the cache,
|
||
/// which the DB doesn't track). Pure local lookup — never hits the network.
|
||
fn search(&self, query: &str, limit: usize) -> CoreResult<Vec<SearchHit>> {
|
||
let hits = self
|
||
.db
|
||
.search(query, limit)
|
||
.map_err(|e| CoreError::from_api(&e, "search"))?;
|
||
Ok(hits
|
||
.into_iter()
|
||
.map(|h| SearchHit {
|
||
name: h.node.name.clone(),
|
||
path: h.path,
|
||
is_dir: h.node.is_folder(),
|
||
size: node_size(&h.node),
|
||
modified: h.node.modification_time,
|
||
pinned: self.cache.is_pinned(&h.node.uid),
|
||
uid: h.node.uid.to_string(),
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// Search the index of files on this machine (outside Drive), built by the
|
||
/// background scanner in [`run_local_index`]. Pure local lookup, never hits
|
||
/// the network — and never touches the FUSE mount, which the scanner excludes.
|
||
fn search_local(&self, query: &str, limit: usize) -> CoreResult<Vec<LocalHit>> {
|
||
let hits = self
|
||
.db
|
||
.search_local(query, limit)
|
||
.map_err(|e| CoreError::from_api(&e, "local search"))?;
|
||
Ok(hits
|
||
.into_iter()
|
||
.map(|h| LocalHit {
|
||
name: h.name,
|
||
path: h.path,
|
||
is_dir: h.is_dir,
|
||
size: h.size.max(0) as u64,
|
||
modified: h.mtime,
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// A page of the photos timeline (newest first), sliced out of the DB.
|
||
/// `Ok(None)` when the account has no photos volume.
|
||
///
|
||
/// Stale-while-revalidate: a persisted timeline is served *immediately*, and
|
||
/// refreshed on the runtime if it is older than [`TIMELINE_TTL`] — so opening
|
||
/// the app paints from disk rather than waiting on a full `enumerate_timeline`
|
||
/// (which returns the whole timeline, not a page). Only an empty DB blocks on
|
||
/// the network, i.e. the very first run.
|
||
///
|
||
/// Metadata only: a thumbnail path is attached for photos whose thumbnail is
|
||
/// already cached, and nothing is downloaded here — the front-end pulls the
|
||
/// thumbnails it actually paints via [`Core::photo_thumbs`].
|
||
fn listing_stale(&self, key: &str, ttl: Duration) -> bool {
|
||
match self.db.state_i64(key).ok().flatten() {
|
||
Some(ms) => now_ms().saturating_sub(ms) >= ttl.as_millis() as i64,
|
||
None => true,
|
||
}
|
||
}
|
||
|
||
/// Re-fetch the whole photos timeline and persist it. Returns whether the
|
||
/// account has a photos volume at all.
|
||
/// Download the full content of the Drive file at mountpoint-relative `rel`
|
||
/// into the content cache, returning its on-disk path (served from cache
|
||
/// when a fresh blob already exists). Lets a front-end open the file with
|
||
/// the user's default application without pinning it.
|
||
fn open_file(&self, rel: &Path) -> CoreResult<PathBuf> {
|
||
let (ino, uid) = self.resolve(rel)?;
|
||
let (name, mtime, size) = {
|
||
let st = self.state.lock();
|
||
let e = st
|
||
.entries
|
||
.get(&ino)
|
||
.ok_or_else(|| CoreError::not_found("node vanished"))?;
|
||
if !e.node.is_file() {
|
||
return Err(CoreError::invalid("not a regular file"));
|
||
}
|
||
(
|
||
e.node.name.clone(),
|
||
e.node.modification_time,
|
||
node_size(&e.node),
|
||
)
|
||
};
|
||
if let Some(p) = self.cache.cached_content_path(&uid, mtime, size) {
|
||
return Ok(p);
|
||
}
|
||
let bytes = self
|
||
.download_file_tracked(&uid, &name, size)
|
||
.map_err(|e| CoreError::from_api(&e, "download"))?;
|
||
self.cache
|
||
.store(&uid, mtime, size, &bytes)
|
||
.map_err(|e| CoreError::internal(format!("cache store: {e}")))?;
|
||
Ok(self.cache.content_path(&uid))
|
||
}
|
||
|
||
/// Drop the cached child listing of `rel`'s parent directory so the next
|
||
/// `ListDir` re-enumerates it from the server. No-op when the parent can't be
|
||
/// resolved (e.g. `rel` is the root). Resolves the parent without holding the
|
||
/// state lock, then invalidates under it.
|
||
fn invalidate_parent_listing(&self, rel: &Path) {
|
||
let parent = rel.parent().unwrap_or_else(|| Path::new(""));
|
||
if let Ok((pino, _)) = self.resolve_path(parent) {
|
||
self.state.lock().invalidate_listing(pino);
|
||
}
|
||
}
|
||
|
||
/// Rename a file or folder to `new_name`. `rel` is mountpoint-relative.
|
||
/// Mirrors the FUSE `rename` write path: rename on the remote, forget the
|
||
/// node so it re-interns under its new name, and drop the parent listing so
|
||
/// the next `ListDir` re-enumerates.
|
||
fn rename(&self, rel: &Path, new_name: &str) -> CoreResult<String> {
|
||
if new_name.is_empty() || new_name.contains('/') {
|
||
return Err(CoreError::invalid(format!("invalid name: {new_name:?}")));
|
||
}
|
||
let (_ino, uid) = self.resolve(rel)?;
|
||
self.rt
|
||
.block_on(self.client.rename_node(&uid, new_name, None))
|
||
.map_err(|e| CoreError::from_api(&e, "rename"))?;
|
||
self.state.lock().forget(&uid);
|
||
self.invalidate_parent_listing(rel);
|
||
Ok(new_name.to_string())
|
||
}
|
||
|
||
/// Move a file or folder into the folder at `new_parent_rel`. Both paths are
|
||
/// mountpoint-relative. Forgets the node and invalidates both the source and
|
||
/// destination listings so each re-enumerates on next access.
|
||
fn move_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult<String> {
|
||
let (_ino, uid) = self.resolve(rel)?;
|
||
let (pino, new_parent_uid) = self
|
||
.resolve_path(new_parent_rel)
|
||
.map_err(|e| self.errno_error(e, "resolve new parent"))?;
|
||
self.rt
|
||
.block_on(self.client.move_node(&uid, &new_parent_uid))
|
||
.map_err(|e| CoreError::from_api(&e, "move"))?;
|
||
let name = self
|
||
.state
|
||
.lock()
|
||
.forget(&uid)
|
||
.map(|(_, n)| n)
|
||
.unwrap_or_default();
|
||
self.invalidate_parent_listing(rel);
|
||
self.state.lock().invalidate_listing(pino);
|
||
Ok(name)
|
||
}
|
||
|
||
/// Trash a file or folder. `rel` is mountpoint-relative. Forgets the node,
|
||
/// evicts any cached content, and invalidates the parent listing.
|
||
fn delete(&self, rel: &Path) -> CoreResult<String> {
|
||
let (_ino, uid) = self.resolve(rel)?;
|
||
self.rt
|
||
.block_on(self.client.trash_nodes(std::slice::from_ref(&uid)))
|
||
.map_err(|e| CoreError::from_api(&e, "trash"))?;
|
||
let name = self
|
||
.state
|
||
.lock()
|
||
.forget(&uid)
|
||
.map(|(_, n)| n)
|
||
.unwrap_or_default();
|
||
self.cache.evict(&uid);
|
||
self.evict_reader(&uid);
|
||
self.invalidate_parent_listing(rel);
|
||
self.invalidate_trash();
|
||
Ok(name)
|
||
}
|
||
|
||
/// Remove the node a `rename` is about to replace, so the new name is free
|
||
/// for the API call that follows.
|
||
///
|
||
/// `rename(2)` promises to replace an existing destination atomically. Proton
|
||
/// offers no such primitive — `rename_node` refuses a name that is already
|
||
/// taken — so this is the first half of an emulation that is *not* atomic:
|
||
/// see [`Core::restore_replaced`] for the other half (bugs.md B13).
|
||
///
|
||
/// A node whose own creation is still queued has never reached the server, so
|
||
/// dropping its queued ops is the whole removal; nothing goes to the wire and
|
||
/// it works offline.
|
||
fn remove_replaced(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> {
|
||
if is_local_uid(uid) {
|
||
self.discard_queued_ops(uid);
|
||
self.state.lock().forget(uid);
|
||
debug!(%uid, name, "replaced a node whose create was still queued");
|
||
return Ok(());
|
||
}
|
||
if let Err(e) = self
|
||
.rt
|
||
.block_on(self.client.trash_nodes(std::slice::from_ref(uid)))
|
||
{
|
||
error!(%uid, name, error = %e, "trashing the node a rename replaces failed");
|
||
self.log_activity(ActivityKind::Trash, name, e.to_string(), false);
|
||
return Err(Errno::EIO);
|
||
}
|
||
self.discard_queued_ops(uid);
|
||
self.state.lock().forget(uid);
|
||
self.cache.evict(uid);
|
||
self.evict_reader(uid);
|
||
self.invalidate_trash();
|
||
// The node is recoverable from the trash, but only if the user knows it
|
||
// went there — a rename is not an operation anyone expects to trash
|
||
// something, so this is the only record that it happened.
|
||
self.log_activity(
|
||
ActivityKind::Trash,
|
||
name,
|
||
"replaced by a rename from the mount",
|
||
true,
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
/// Put back the node [`Core::remove_replaced`] trashed, after the rename it
|
||
/// was clearing the way for failed anyway.
|
||
///
|
||
/// Best-effort by construction: if the restore also fails there is nothing
|
||
/// further to try, and the node is still in the trash where `pdfs restore`
|
||
/// can reach it. Says so loudly in that case, because the alternative is a
|
||
/// file the user believes was only renamed quietly sitting in the trash.
|
||
fn restore_replaced(&self, victim: Option<&(u64, NodeUid)>, name: &str) {
|
||
let Some((_, uid)) = victim else { return };
|
||
if is_local_uid(uid) {
|
||
// Its queued create was discarded and cannot be reconstructed from
|
||
// here; the caller's error is what the user acts on.
|
||
warn!(%uid, name, "a rename failed after discarding a queued node it replaced");
|
||
return;
|
||
}
|
||
match self
|
||
.rt
|
||
.block_on(self.client.restore_nodes(std::slice::from_ref(uid)))
|
||
{
|
||
Ok(()) => {
|
||
self.invalidate_trash();
|
||
debug!(%uid, name, "restored the node a failed rename had replaced");
|
||
}
|
||
Err(e) => {
|
||
error!(%uid, name, error = %e, "restoring a replaced node failed; it stays in the trash");
|
||
self.log_activity(ActivityKind::Restore, name, e.to_string(), false);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// List the account's trash, from the DB. Trashed nodes are outside the
|
||
/// mounted tree — the FUSE side forgot them when they were trashed — so the
|
||
/// listing is persisted in its own table rather than derived from `State`, and
|
||
/// each entry is identified by its uid (its only remaining handle) with an
|
||
/// empty path.
|
||
///
|
||
/// Stale-while-revalidate, like the photos timeline: a persisted listing comes
|
||
/// back at DB speed and is refreshed in the background past [`TRASH_TTL`].
|
||
/// Our own trash mutations invalidate it outright (see
|
||
/// [`Core::invalidate_trash`]), so the TTL only covers changes made elsewhere.
|
||
fn list_trash(&self) -> CoreResult<Vec<DirEntry>> {
|
||
let stale = self.listing_stale(TRASH_SYNCED_MS, TRASH_TTL);
|
||
// Never fetched: this request has to wait for it.
|
||
if self.db.state_i64(TRASH_SYNCED_MS).ok().flatten().is_none() {
|
||
self.rt.block_on(self.refresh_trash())?;
|
||
} else if stale {
|
||
self.spawn_trash_refresh();
|
||
}
|
||
|
||
Ok(self
|
||
.db
|
||
.trash_list()
|
||
.map_err(CoreError::from)?
|
||
.into_iter()
|
||
.map(|item| DirEntry {
|
||
name: item.name,
|
||
is_dir: item.is_dir,
|
||
size: item.size.max(0) as u64,
|
||
modified: item.mtime,
|
||
// A trashed node can't be pinned or served from the mount, so its
|
||
// content is never current cache: report neither.
|
||
pinned: false,
|
||
cached: false,
|
||
uid: item.uid,
|
||
path: String::new(),
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// Re-fetch the trash listing from the server and persist it.
|
||
async fn refresh_trash(&self) -> CoreResult<()> {
|
||
let uids = self
|
||
.client
|
||
.enumerate_trash_node_uids()
|
||
.await
|
||
.map_err(|e| CoreError::from_api(&e, "enumerate trash"))?;
|
||
let nodes = if uids.is_empty() {
|
||
Vec::new()
|
||
} else {
|
||
self.client
|
||
.enumerate_nodes(&uids)
|
||
.await
|
||
.map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?
|
||
};
|
||
let items: Vec<StoredTrash> = nodes
|
||
.into_iter()
|
||
.map(|node| StoredTrash {
|
||
uid: node.uid.to_string(),
|
||
name: node.name.clone(),
|
||
is_dir: node.is_folder(),
|
||
size: node_size(&node) as i64,
|
||
mtime: node.modification_time,
|
||
})
|
||
.collect();
|
||
self.db.trash_replace(&items).map_err(CoreError::from)?;
|
||
let _ = self.db.set_state_i64(TRASH_SYNCED_MS, now_ms());
|
||
Ok(())
|
||
}
|
||
|
||
/// Refresh the trash off the request path. At most one refresh at a time.
|
||
fn spawn_trash_refresh(&self) {
|
||
if self.trash_refreshing.swap(true, Ordering::SeqCst) {
|
||
return;
|
||
}
|
||
let core = self.clone();
|
||
self.rt.spawn(async move {
|
||
if let Err(e) = core.refresh_trash().await {
|
||
warn!(error = %e, "background trash refresh failed");
|
||
}
|
||
core.trash_refreshing.store(false, Ordering::SeqCst);
|
||
});
|
||
}
|
||
|
||
/// Drop the persisted trash listing's freshness stamp after we changed the
|
||
/// trash ourselves. The next Trash page then *waits* for a fresh listing
|
||
/// rather than painting one from before the mutation — the user just made the
|
||
/// change and is about to look straight at it.
|
||
fn invalidate_trash(&self) {
|
||
let _ = self.db.clear_state(TRASH_SYNCED_MS);
|
||
}
|
||
|
||
/// Drop the persisted photos timeline's freshness stamp, so the next timeline
|
||
/// read fetches rather than serving what it already has.
|
||
fn invalidate_photos(&self) {
|
||
let _ = self.db.clear_state(PHOTOS_SYNCED_MS);
|
||
}
|
||
|
||
/// Drop one folder's cached child listing (`rel` is mountpoint-relative), so
|
||
/// the next `ListDir`/`readdir` re-enumerates it from the server. Backs
|
||
/// [`CtlRequest::Refresh`] with a [`RefreshScope::Dir`] scope.
|
||
fn refresh_dir(&self, rel: &Path) -> CoreResult<()> {
|
||
let (ino, _uid) = self.resolve(rel)?;
|
||
self.state.lock().invalidate_listing(ino);
|
||
Ok(())
|
||
}
|
||
|
||
/// Parse wire uids (`volume~link`) into [`NodeUid`]s, rejecting the whole
|
||
/// batch if any is malformed — a partial trash mutation is worse than none.
|
||
fn parse_uids(uids: &[String]) -> CoreResult<Vec<NodeUid>> {
|
||
if uids.is_empty() {
|
||
return Err(CoreError::invalid("no nodes given"));
|
||
}
|
||
uids.iter()
|
||
.map(|u| parse_uid(u).ok_or_else(|| CoreError::invalid(format!("invalid uid: {u}"))))
|
||
.collect()
|
||
}
|
||
|
||
/// Restore trashed nodes to the folders they were trashed from. The parents
|
||
/// are read *before* the restore — a restored node reappears in a listing the
|
||
/// daemon may already have cached, so each destination folder is invalidated
|
||
/// and re-enumerated on next access.
|
||
fn restore(&self, uids: &[String]) -> CoreResult<usize> {
|
||
let parsed = Self::parse_uids(uids)?;
|
||
let parents: Vec<NodeUid> = self
|
||
.rt
|
||
.block_on(self.client.enumerate_nodes(&parsed))
|
||
.map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?
|
||
.into_iter()
|
||
.filter_map(|n| n.parent_uid)
|
||
.collect();
|
||
self.rt
|
||
.block_on(self.client.restore_nodes(&parsed))
|
||
.map_err(|e| CoreError::from_api(&e, "restore"))?;
|
||
{
|
||
let mut st = self.state.lock();
|
||
for parent in parents {
|
||
if let Some(&ino) = st.by_uid.get(&parent) {
|
||
st.invalidate_listing(ino);
|
||
}
|
||
}
|
||
}
|
||
self.invalidate_trash();
|
||
Ok(parsed.len())
|
||
}
|
||
|
||
/// Permanently delete trashed nodes. Irreversible on the server; locally it
|
||
/// drops any metadata and cached content the node still owns.
|
||
fn delete_forever(&self, uids: &[String]) -> CoreResult<usize> {
|
||
let parsed = Self::parse_uids(uids)?;
|
||
self.rt
|
||
.block_on(self.client.delete_nodes(&parsed))
|
||
.map_err(|e| CoreError::from_api(&e, "delete"))?;
|
||
self.drop_local(&parsed);
|
||
self.invalidate_trash();
|
||
Ok(parsed.len())
|
||
}
|
||
|
||
/// Permanently delete everything in the trash. The uids are listed first so
|
||
/// the blobs of items trashed by *another* client — which this daemon may
|
||
/// still hold in its cache — are reclaimed too, not just the ones it trashed.
|
||
fn empty_trash(&self) -> CoreResult<usize> {
|
||
let uids = self
|
||
.rt
|
||
.block_on(self.client.enumerate_trash_node_uids())
|
||
.map_err(|e| CoreError::from_api(&e, "enumerate trash"))?;
|
||
self.rt
|
||
.block_on(self.client.empty_trash())
|
||
.map_err(|e| CoreError::from_api(&e, "empty trash"))?;
|
||
self.drop_local(&uids);
|
||
self.invalidate_trash();
|
||
Ok(uids.len())
|
||
}
|
||
|
||
/// Forget every trace of nodes that no longer exist anywhere: their inode and
|
||
/// DB row, and their cached content.
|
||
fn drop_local(&self, uids: &[NodeUid]) {
|
||
let mut st = self.state.lock();
|
||
for uid in uids {
|
||
st.forget(uid);
|
||
}
|
||
drop(st);
|
||
for uid in uids {
|
||
self.cache.evict(uid);
|
||
self.evict_reader(uid);
|
||
}
|
||
}
|
||
|
||
/// Create a folder named `name` under the mountpoint-relative `parent_rel`.
|
||
/// Interns the new node directly so it shows up without a re-enumeration.
|
||
fn create_folder(&self, parent_rel: &Path, name: &str) -> CoreResult<String> {
|
||
if name.is_empty() || name.contains('/') {
|
||
return Err(CoreError::invalid(format!("invalid name: {name:?}")));
|
||
}
|
||
let (pino, parent_uid) = self.resolve(parent_rel)?;
|
||
self.ensure_children(pino)
|
||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||
let new_uid = self
|
||
.rt
|
||
.block_on(
|
||
self.client
|
||
.create_folder(&parent_uid, name, Some(now_secs())),
|
||
)
|
||
.map_err(|e| CoreError::from_api(&e, "create folder"))?;
|
||
let node = self
|
||
.fetch_node(&new_uid)
|
||
.map_err(|e| self.errno_error(e, "fetch node"))?;
|
||
let mut st = self.state.lock();
|
||
let ino = st.intern(pino, node);
|
||
if let Some(kids) = st.children.get_mut(&pino)
|
||
&& !kids.contains(&ino)
|
||
{
|
||
kids.push(ino);
|
||
}
|
||
Ok(name.to_string())
|
||
}
|
||
|
||
/// Bulk-upload local files and directory trees under `sources` into the
|
||
/// mountpoint-relative `parent_rel` folder. Directories are recreated (or
|
||
/// merged into an existing same-named folder) and walked; the resulting flat
|
||
/// set of files is uploaded with bounded concurrency, each ticking the
|
||
/// transfer registry so a front-end sees live progress. Runs on a background
|
||
/// thread — a large tree far outlasts the control socket's read timeout — so
|
||
/// it reports only a summary for the log. Individual failures are counted and
|
||
/// logged rather than aborting the whole batch.
|
||
fn upload_paths(&self, parent_rel: &Path, sources: &[PathBuf]) -> CoreResult<UploadStats> {
|
||
let (pino, parent_uid) = self.resolve(parent_rel)?;
|
||
self.ensure_children(pino)
|
||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||
|
||
// Phase 1 (sequential): build the remote folder skeleton and collect the
|
||
// flat list of files to upload. Folders must exist before their children,
|
||
// so this can't be parallelised. On a deep tree this is a folder-creation
|
||
// round-trip per directory before a single byte moves — long enough that
|
||
// it needs a job of its own, or the daemon looks idle for minutes.
|
||
let mut tasks = Vec::new();
|
||
let mut folders = 0usize;
|
||
{
|
||
let job = self.transfers.begin_job("Preparing upload");
|
||
for src in sources {
|
||
if let Err(e) =
|
||
self.collect_uploads(pino, &parent_uid, src, &mut tasks, &mut folders, &job)
|
||
{
|
||
warn!(source = %src.display(), error = %e, "skipping source");
|
||
}
|
||
}
|
||
}
|
||
|
||
// Phase 2 (concurrent): upload the files, up to UPLOAD_CONCURRENCY at once.
|
||
// Each file reports its own bytes; this job is the batch's "N of M files",
|
||
// which is the number a user actually waits on.
|
||
let job = Arc::new(self.transfers.begin_job(match tasks.len() {
|
||
1 => "Uploading 1 file".to_string(),
|
||
n => format!("Uploading {n} files"),
|
||
}));
|
||
job.set_total(tasks.len() as u64);
|
||
let outcomes = self.rt.block_on(run_uploads(
|
||
self.clone(),
|
||
tasks,
|
||
UPLOAD_CONCURRENCY,
|
||
job.clone(),
|
||
));
|
||
drop(job);
|
||
|
||
// Phase 3 (sequential): intern each uploaded node so it shows up in the
|
||
// listing without a re-enumeration. fetch_node uses `block_on`, so it must
|
||
// run here rather than inside the async batch — and it is a round-trip per
|
||
// file, so it too gets a job rather than a silent tail.
|
||
let mut stats = UploadStats {
|
||
folders,
|
||
..UploadStats::default()
|
||
};
|
||
let job = self.transfers.begin_job("Finishing upload");
|
||
job.set_total(outcomes.len() as u64);
|
||
for outcome in outcomes {
|
||
job.step();
|
||
match outcome {
|
||
Ok((parent_ino, uid, size)) => {
|
||
stats.uploaded += 1;
|
||
stats.bytes += size;
|
||
match self.fetch_node(&uid) {
|
||
Ok(node) => {
|
||
let mut st = self.state.lock();
|
||
let ino = st.intern(parent_ino, node);
|
||
if let Some(kids) = st.children.get_mut(&parent_ino)
|
||
&& !kids.contains(&ino)
|
||
{
|
||
kids.push(ino);
|
||
}
|
||
}
|
||
// Uploaded fine but the metadata refresh failed; it will
|
||
// appear on the next directory enumeration regardless.
|
||
Err(e) => warn!(%uid, error = ?e, "uploaded node metadata refresh failed"),
|
||
}
|
||
}
|
||
Err((name, msg)) => {
|
||
stats.failed += 1;
|
||
warn!(name, error = %msg, "file upload failed");
|
||
}
|
||
}
|
||
}
|
||
info!(
|
||
uploaded = stats.uploaded,
|
||
failed = stats.failed,
|
||
"bulk upload finished"
|
||
);
|
||
Ok(stats)
|
||
}
|
||
|
||
/// Resolve a remote child folder named `name` under `pino`, creating it if it
|
||
/// doesn't exist, and return its `(inode, uid)`. Reusing an existing same-named
|
||
/// folder makes re-uploading a directory merge into it rather than fail on a
|
||
/// duplicate name.
|
||
fn ensure_remote_folder(
|
||
&self,
|
||
pino: u64,
|
||
parent_uid: &NodeUid,
|
||
name: &str,
|
||
) -> CoreResult<(u64, NodeUid)> {
|
||
if name.is_empty() || name.contains('/') {
|
||
return Err(CoreError::invalid(format!("invalid folder name: {name:?}")));
|
||
}
|
||
self.ensure_children(pino)
|
||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||
{
|
||
let st = self.state.lock();
|
||
if let Some(kids) = st.children.get(&pino) {
|
||
for &ino in kids {
|
||
if let Some(e) = st.entries.get(&ino)
|
||
&& e.node.is_folder()
|
||
&& e.node.name == name
|
||
{
|
||
return Ok((ino, e.uid.clone()));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
let new_uid = self
|
||
.rt
|
||
.block_on(
|
||
self.client
|
||
.create_folder(parent_uid, name, Some(now_secs())),
|
||
)
|
||
.map_err(|e| CoreError::from_api(&e, &format!("create folder {name}")))?;
|
||
let node = self
|
||
.fetch_node(&new_uid)
|
||
.map_err(|e| self.errno_error(e, "fetch node"))?;
|
||
let mut st = self.state.lock();
|
||
let ino = st.intern(pino, node);
|
||
if let Some(kids) = st.children.get_mut(&pino)
|
||
&& !kids.contains(&ino)
|
||
{
|
||
kids.push(ino);
|
||
}
|
||
Ok((ino, new_uid))
|
||
}
|
||
|
||
/// Walk one local source path, appending its files to `tasks`. A file becomes
|
||
/// one task; a directory is recreated remotely and recursed into (children
|
||
/// sorted for a stable order). Symlinks and other special files are skipped.
|
||
///
|
||
/// `job` narrates the walk with the folder currently being mirrored. It stays
|
||
/// indeterminate: nothing knows the size of the tree until the walk has ended.
|
||
fn collect_uploads(
|
||
&self,
|
||
pino: u64,
|
||
parent_uid: &NodeUid,
|
||
src: &Path,
|
||
tasks: &mut Vec<UploadTask>,
|
||
folders: &mut usize,
|
||
job: &JobGuard,
|
||
) -> CoreResult<()> {
|
||
let meta = std::fs::symlink_metadata(src)
|
||
.map_err(|e| CoreError::internal(format!("stat {}: {e}", src.display())))?;
|
||
let name = src
|
||
.file_name()
|
||
.and_then(|n| n.to_str())
|
||
.ok_or_else(|| CoreError::invalid(format!("unusable name: {}", src.display())))?
|
||
.to_string();
|
||
|
||
if meta.is_dir() {
|
||
job.detail(&name);
|
||
let (child_ino, child_uid) = self.ensure_remote_folder(pino, parent_uid, &name)?;
|
||
*folders += 1;
|
||
let mut entries: Vec<PathBuf> = std::fs::read_dir(src)
|
||
.map_err(|e| CoreError::internal(format!("read dir {}: {e}", src.display())))?
|
||
.filter_map(|e| e.ok())
|
||
.map(|e| e.path())
|
||
.collect();
|
||
entries.sort();
|
||
for entry in entries {
|
||
if let Err(e) =
|
||
self.collect_uploads(child_ino, &child_uid, &entry, tasks, folders, job)
|
||
{
|
||
warn!(source = %entry.display(), error = %e, "skipping entry");
|
||
}
|
||
}
|
||
// Deeper folders have retitled the job by now; put this one back so the
|
||
// line tracks the walk's position rather than its deepest leaf.
|
||
job.detail(&name);
|
||
} else if meta.is_file() {
|
||
if name.contains('/') {
|
||
return Err(CoreError::invalid(format!("invalid file name: {name:?}")));
|
||
}
|
||
tasks.push(UploadTask {
|
||
parent_ino: pino,
|
||
parent_uid: parent_uid.clone(),
|
||
name,
|
||
path: src.to_path_buf(),
|
||
size: meta.len(),
|
||
});
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
// ---- activity log -----------------------------------------------------
|
||
|
||
/// Append one entry to the activity log. Callable from any thread (the sync
|
||
/// engine and the bulk uploader both log from background tasks). A failed
|
||
/// write is logged and dropped: the feed is a record of work, never a reason
|
||
/// to fail the work itself.
|
||
pub(crate) fn log_activity(
|
||
&self,
|
||
kind: ActivityKind,
|
||
target: impl Into<String>,
|
||
// `Display` rather than `Into<String>` so a classified `CoreError` can be
|
||
// logged as-is, without the caller flattening it first.
|
||
detail: impl std::fmt::Display,
|
||
ok: bool,
|
||
) {
|
||
let entry = ActivityEntry {
|
||
time: now_secs(),
|
||
kind,
|
||
target: target.into(),
|
||
detail: detail.to_string(),
|
||
ok,
|
||
};
|
||
if let Err(e) = self.db.activity_add(&entry) {
|
||
warn!(error = ?e, "could not record activity");
|
||
}
|
||
}
|
||
|
||
/// The recent activity, newest first, capped at `limit` entries.
|
||
fn list_activity(&self, limit: usize) -> Vec<ActivityEntry> {
|
||
match self.db.activity_list(limit) {
|
||
Ok(items) => items,
|
||
Err(e) => {
|
||
warn!(error = ?e, "could not read activity");
|
||
Vec::new()
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---- live sync progress -----------------------------------------------
|
||
|
||
/// Start tracking a reconcile pass over `folder_id`, in [`SyncPhase::Scanning`].
|
||
pub(crate) fn progress_begin(&self, folder_id: i64) {
|
||
self.sync_progress.lock().insert(
|
||
folder_id,
|
||
SyncProgress {
|
||
phase: SyncPhase::Scanning,
|
||
done: 0,
|
||
total: 0,
|
||
current: String::new(),
|
||
},
|
||
);
|
||
}
|
||
|
||
/// Apply `f` to a folder's live progress, if a pass is running for it.
|
||
fn progress_update(&self, folder_id: i64, f: impl FnOnce(&mut SyncProgress)) {
|
||
if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) {
|
||
f(p);
|
||
}
|
||
}
|
||
|
||
/// Set the number of items the scan expects to check, from the size of the
|
||
/// last pass's baseline. Only an estimate — the folder may have grown since —
|
||
/// but it turns the scan from an indeterminate pulse into a bar that moves,
|
||
/// which is the difference between "it's stuck" and "it's working" on a folder
|
||
/// whose walk takes minutes.
|
||
pub(crate) fn progress_scan_total(&self, folder_id: i64, n: usize) {
|
||
self.progress_update(folder_id, |p| p.total = n);
|
||
}
|
||
|
||
/// Note that the scan has checked one more item, named `name`.
|
||
pub(crate) fn progress_scanned(&self, folder_id: i64, name: &str) {
|
||
self.progress_update(folder_id, |p| {
|
||
p.done += 1;
|
||
p.current = name.to_string();
|
||
});
|
||
}
|
||
|
||
/// Note that `n` more items have been queued for this pass, and that it has
|
||
/// moved on from scanning to applying the diff. The scan's counts are dropped:
|
||
/// they measured a different quantity (items checked, not items to apply), so
|
||
/// carrying them over would start the applying bar at a meaningless fraction.
|
||
pub(crate) fn progress_queued(&self, folder_id: i64, n: usize) {
|
||
self.progress_update(folder_id, |p| {
|
||
if p.phase == SyncPhase::Scanning {
|
||
p.phase = SyncPhase::Applying;
|
||
p.done = 0;
|
||
p.total = 0;
|
||
p.current.clear();
|
||
}
|
||
p.total += n;
|
||
});
|
||
}
|
||
|
||
/// Note that work has started on `name` (shown as the pass's current item).
|
||
pub(crate) fn progress_started(&self, folder_id: i64, name: &str) {
|
||
self.progress_update(folder_id, |p| p.current = name.to_string());
|
||
}
|
||
|
||
/// Note that one queued item finished, whether it succeeded or not.
|
||
pub(crate) fn progress_finished(&self, folder_id: i64) {
|
||
self.progress_update(folder_id, |p| {
|
||
p.done += 1;
|
||
p.current.clear();
|
||
});
|
||
}
|
||
|
||
/// Stop tracking a pass — no progress is reported for the folder until the
|
||
/// next [`progress_begin`](Self::progress_begin).
|
||
pub(crate) fn progress_end(&self, folder_id: i64) {
|
||
self.sync_progress.lock().remove(&folder_id);
|
||
}
|
||
}
|
||
|
||
/// Map a [`MemberRole`] to its wire string.
|
||
fn role_to_str(role: MemberRole) -> &'static str {
|
||
match role {
|
||
MemberRole::Viewer => "viewer",
|
||
MemberRole::Editor => "editor",
|
||
MemberRole::Admin => "admin",
|
||
MemberRole::Inherited => "inherited",
|
||
}
|
||
}
|
||
|
||
/// Parse a wire role string into a [`MemberRole`]. "inherited" is read-only and
|
||
/// rejected here, since it cannot be sent when inviting or updating.
|
||
fn role_from_str(role: &str) -> CoreResult<MemberRole> {
|
||
match role.to_lowercase().as_str() {
|
||
"viewer" => Ok(MemberRole::Viewer),
|
||
"editor" => Ok(MemberRole::Editor),
|
||
"admin" => Ok(MemberRole::Admin),
|
||
other => Err(CoreError::invalid(format!("invalid role: {other}"))),
|
||
}
|
||
}
|
||
|
||
/// Map a device type to a display string.
|
||
fn device_type_str(t: proton_drive_rs::DeviceType) -> &'static str {
|
||
match t {
|
||
proton_drive_rs::DeviceType::Windows => "Windows",
|
||
proton_drive_rs::DeviceType::MacOs => "MacOs",
|
||
proton_drive_rs::DeviceType::Linux => "Linux",
|
||
}
|
||
}
|
||
|
||
/// Convert an SDK [`PublicLink`](proton_drive_rs::PublicLink) into the wire form.
|
||
fn public_link_info(link: proton_drive_rs::PublicLink) -> PublicLinkInfo {
|
||
PublicLinkInfo {
|
||
id: link.public_link_id,
|
||
url: link.url,
|
||
role: role_to_str(link.role).to_string(),
|
||
expires: link.expiration_time,
|
||
has_password: link.has_custom_password,
|
||
}
|
||
}
|
||
|
||
/// Parse a `volume~link` uid display string back into a [`NodeUid`]. Front-ends
|
||
/// receive uids as strings over the control socket and pass them back verbatim.
|
||
fn parse_uid(s: &str) -> Option<NodeUid> {
|
||
let (vol, link) = s.split_once('~')?;
|
||
Some(NodeUid::new(VolumeId::from(vol), LinkId::from(link)))
|
||
}
|
||
|
||
/// The Proton Drive VFS. FUSE callbacks are synchronous, so the Tokio handle
|
||
/// bridges each one to the async SDK via [`Handle::block_on`]; the fuser
|
||
/// session thread is not a runtime worker, so blocking on it is sound.
|
||
/// Cloneable so a handler can move a copy onto a [`Workers`] thread and answer
|
||
/// from there; every field is a handle or a plain id.
|
||
#[derive(Clone)]
|
||
pub struct ProtonFs {
|
||
core: Core,
|
||
uid: u32,
|
||
gid: u32,
|
||
}
|
||
|
||
impl ProtonFs {
|
||
/// Build the filesystem rooted at `root` (the user's My Files folder).
|
||
fn new(core: Core, root: Node) -> Self {
|
||
{
|
||
let mut st = core.state.lock();
|
||
if let Err(e) = st.db.upsert_node(&root) {
|
||
warn!(uid = %root.uid, error = %e, "db upsert root failed");
|
||
}
|
||
st.by_uid.insert(root.uid.clone(), ROOT_INO);
|
||
st.entries.insert(
|
||
ROOT_INO,
|
||
Entry {
|
||
uid: root.uid.clone(),
|
||
parent: ROOT_INO,
|
||
node: root,
|
||
},
|
||
);
|
||
}
|
||
// SAFETY: geteuid/getegid are infallible and have no preconditions.
|
||
let (uid, gid) = unsafe { (libc::geteuid(), libc::getegid()) };
|
||
Self { core, uid, gid }
|
||
}
|
||
|
||
/// The body of [`Filesystem::lookup`], on whichever thread ends up serving it.
|
||
/// `off_loop` says whether the caller is already on a worker, and so whether
|
||
/// this may block. `lookup` answers a cached parent on the dispatch loop,
|
||
/// where it may not.
|
||
fn serve_lookup(&self, parent: u64, name: &str, reply: ReplyEntry, off_loop: bool) {
|
||
if let Err(e) = self.core.ensure_children(parent) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
let hit = {
|
||
let st = self.core.state.lock();
|
||
st.children.get(&parent).and_then(|kids| {
|
||
kids.iter().copied().find_map(|ino| {
|
||
st.entries
|
||
.get(&ino)
|
||
.filter(|e| e.node.name == name)
|
||
.map(|e| {
|
||
// A `lookup` reply carries attrs with the same TTL a
|
||
// `getattr` would, so `ls -l` — which is one `lookup`
|
||
// per entry and no `getattr` at all — takes its sizes
|
||
// from here. Resolving only in `getattr` left the whole
|
||
// listing provisional (bugs.md B14).
|
||
let provisional = matches!(
|
||
&e.node.kind,
|
||
NodeKind::File {
|
||
claimed_size: None,
|
||
..
|
||
}
|
||
)
|
||
.then(|| (e.parent, e.uid.clone()));
|
||
(ino, self.attr(ino, &e.node), provisional)
|
||
})
|
||
})
|
||
})
|
||
};
|
||
let Some((ino, attr, provisional)) = hit else {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
};
|
||
let Some((grandparent, uid)) = provisional else {
|
||
reply.entry(&TTL, &attr, Generation(0));
|
||
return;
|
||
};
|
||
// Resolving goes to the network. On the dispatch loop that is not
|
||
// allowed (the B5 lesson), so hand off — but only from the loop: doing
|
||
// it while already on a worker would queue a job onto the lane this
|
||
// thread occupies.
|
||
if !off_loop {
|
||
let fs = self.clone();
|
||
let name = name.to_string();
|
||
self.core.workers.run(Lane::Meta, move || {
|
||
fs.serve_lookup(parent, &name, reply, true)
|
||
});
|
||
return;
|
||
}
|
||
self.core.upgrade_sizes_for_parent(ino, &uid, grandparent);
|
||
// Re-read: on timeout or failure this is the provisional attr again,
|
||
// which is no worse than what this path used to always send.
|
||
let attr = {
|
||
let st = self.core.state.lock();
|
||
st.entries
|
||
.get(&ino)
|
||
.map_or(attr, |e| self.attr(ino, &e.node))
|
||
};
|
||
reply.entry(&TTL, &attr, Generation(0));
|
||
}
|
||
|
||
/// The body of [`Filesystem::readdir`], on whichever thread ends up serving it.
|
||
fn serve_readdir(&self, ino: u64, offset: u64, mut reply: ReplyDirectory) {
|
||
if let Err(e) = self.core.ensure_children(ino) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
let st = self.core.state.lock();
|
||
let parent = st.entries.get(&ino).map_or(ROOT_INO, |e| e.parent);
|
||
|
||
// "." and ".." occupy offsets 0 and 1; real children follow.
|
||
let mut listing: Vec<(u64, FileType, String)> = vec![
|
||
(ino, FileType::Directory, ".".to_string()),
|
||
(parent, FileType::Directory, "..".to_string()),
|
||
];
|
||
if let Some(kids) = st.children.get(&ino) {
|
||
for &kid in kids {
|
||
if let Some(e) = st.entries.get(&kid) {
|
||
let ft = if e.node.is_folder() {
|
||
FileType::Directory
|
||
} else {
|
||
FileType::RegularFile
|
||
};
|
||
listing.push((kid, ft, e.node.name.clone()));
|
||
}
|
||
}
|
||
}
|
||
drop(st);
|
||
|
||
for (i, (ino, ft, name)) in listing.into_iter().enumerate().skip(offset as usize) {
|
||
// The stored offset is that of the *next* entry to resume at.
|
||
if reply.add(INodeNo(ino), (i + 1) as u64, ft, &name) {
|
||
break;
|
||
}
|
||
}
|
||
reply.ok();
|
||
}
|
||
|
||
fn attr(&self, ino: u64, node: &Node) -> FileAttr {
|
||
let (kind, perm) = match node.kind {
|
||
NodeKind::Folder => (FileType::Directory, 0o755),
|
||
NodeKind::File { .. } => (FileType::RegularFile, 0o644),
|
||
};
|
||
let size = node_size(node);
|
||
let mtime = unix_secs(node.modification_time);
|
||
let crtime = unix_secs(node.creation_time);
|
||
FileAttr {
|
||
ino: INodeNo(ino),
|
||
size,
|
||
blocks: size.div_ceil(512),
|
||
atime: mtime,
|
||
mtime,
|
||
ctime: mtime,
|
||
crtime,
|
||
kind,
|
||
perm,
|
||
nlink: if kind == FileType::Directory { 2 } else { 1 },
|
||
uid: self.uid,
|
||
gid: self.gid,
|
||
rdev: 0,
|
||
blksize: 512,
|
||
flags: 0,
|
||
}
|
||
}
|
||
|
||
/// Trash the child `name` under `parent` on the remote, then drop it from the
|
||
/// local cache. Backs both `unlink` and `rmdir` (Proton trashes whole
|
||
/// subtrees, so an `rmdir` of a non-empty dir behaves the same).
|
||
fn trash_child(&self, parent: u64, name: &str, reply: ReplyEmpty) {
|
||
let (_ino, uid) = match self.core.lookup_child(parent, name) {
|
||
Ok(x) => x,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
};
|
||
// A node the server has never heard of cannot be trashed there; deleting
|
||
// it just means its queued creation is no longer wanted. This works
|
||
// offline, which the remote path below cannot (offline.md Phase 3b).
|
||
if is_local_uid(&uid) {
|
||
self.core.discard_queued_ops(&uid);
|
||
self.core.state.lock().forget(&uid);
|
||
debug!(%uid, name, "deleted a node that had not been created remotely yet");
|
||
reply.ok();
|
||
return;
|
||
}
|
||
// Offline: queue it. Trashing is the one mutation a user expects to work
|
||
// regardless — the file is gone from their point of view the moment the
|
||
// command returns (offline.md Phase 3b).
|
||
if !self.core.online.load(Ordering::Relaxed) {
|
||
match self.core.queue_trash(&uid, name) {
|
||
Ok(()) => reply.ok(),
|
||
Err(e) => reply.error(e),
|
||
}
|
||
return;
|
||
}
|
||
if let Err(e) = self
|
||
.core
|
||
.rt
|
||
.block_on(self.core.client.trash_nodes(std::slice::from_ref(&uid)))
|
||
{
|
||
error!(%uid, error = %e, "trash failed");
|
||
self.core
|
||
.log_activity(ActivityKind::Trash, name, e.to_string(), false);
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
self.core.discard_queued_ops(&uid);
|
||
self.core.state.lock().forget(&uid);
|
||
self.core.cache.evict(&uid);
|
||
self.core.evict_reader(&uid);
|
||
self.core.invalidate_trash();
|
||
// Every other trash site records itself; this one did not, which made a
|
||
// file found in the trash impossible to attribute after the fact — the
|
||
// activity log was the only record and it showed nothing (bugs.md B2).
|
||
self.core
|
||
.log_activity(ActivityKind::Trash, name, "trashed from the mount", true);
|
||
reply.ok();
|
||
}
|
||
}
|
||
|
||
/// Current wall-clock time as epoch seconds (0 if the clock is before the epoch).
|
||
fn now_secs() -> i64 {
|
||
SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.map(|d| d.as_secs() as i64)
|
||
.unwrap_or(0)
|
||
}
|
||
|
||
/// Current wall-clock time as epoch milliseconds, the resolution `pending_op`
|
||
/// timestamps and backoff deadlines are kept in.
|
||
fn now_millis() -> i64 {
|
||
SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.map(|d| d.as_millis() as i64)
|
||
.unwrap_or(0)
|
||
}
|
||
|
||
/// Parse a [`NodeUid`] back from its `Display` form (`volume~link`), which is
|
||
/// how one is persisted in `pending_op.uid` and a [`StagedWrite`] sidecar. The
|
||
/// SDK has no `FromStr`, and neither id contains a `~`.
|
||
fn parse_node_uid(s: &str) -> Option<NodeUid> {
|
||
let (vol, link) = s.split_once('~')?;
|
||
Some(NodeUid::new(VolumeId::from(vol), LinkId::from(link)))
|
||
}
|
||
|
||
/// Distinguishes placeholder uids minted by [`mint_local_uid`] within one run.
|
||
static LOCAL_UID_SEQ: AtomicU64 = AtomicU64::new(0);
|
||
|
||
/// Invent a uid for a node created while offline, so it can be interned, listed
|
||
/// and written to before the server has ever heard of it (offline.md Phase 3b).
|
||
///
|
||
/// Uniqueness only has to hold among this machine's undrained ops, so the clock
|
||
/// (which separates runs) plus a counter (which separates nodes within a run) is
|
||
/// enough without taking on a uuid dependency.
|
||
fn mint_local_uid() -> NodeUid {
|
||
let seq = LOCAL_UID_SEQ.fetch_add(1, Ordering::Relaxed);
|
||
NodeUid::new(
|
||
VolumeId::from(LOCAL_VOLUME),
|
||
LinkId::from(format!("{}-{seq}", now_millis())),
|
||
)
|
||
}
|
||
|
||
/// Whether this node exists only on this machine, so far. Such a uid is
|
||
/// meaningless to the API and must never be sent to it.
|
||
fn is_local_uid(uid: &NodeUid) -> bool {
|
||
uid.volume_id.as_str() == LOCAL_VOLUME
|
||
}
|
||
|
||
/// [`is_local_uid`] for a uid in its persisted `Display` form.
|
||
fn is_local_uid_str(s: &str) -> bool {
|
||
s.split_once('~')
|
||
.is_some_and(|(vol, _)| vol == LOCAL_VOLUME)
|
||
}
|
||
|
||
/// The API's response code for a failed call, when it failed *at* the API.
|
||
///
|
||
/// Takes `&dyn Error` so it reads a [`ProtonError`] equally well through the
|
||
/// boxes the drain deals in, where the concrete type survives but the static one
|
||
/// does not. `None` covers both "not an API error at all" (a transport failure,
|
||
/// which is what being offline looks like) and "not a `ProtonError`".
|
||
fn api_code(e: &(dyn std::error::Error + 'static)) -> Option<ResponseCode> {
|
||
match e.downcast_ref::<ProtonError>() {
|
||
Some(ProtonError::Api(api)) => Some(api.code),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
/// Whether a call failed because the name it asked for is already in use.
|
||
///
|
||
/// The queue makes this reachable in a way the synchronous path never was: a
|
||
/// mutation queued while offline is applied against a server that may have
|
||
/// gained a file of that name in the meantime.
|
||
fn is_already_exists(e: &(dyn std::error::Error + 'static)) -> bool {
|
||
api_code(e) == Some(ResponseCode::AlreadyExists)
|
||
}
|
||
|
||
/// Whether a call failed because the node it addressed is not there.
|
||
fn is_gone(e: &(dyn std::error::Error + 'static)) -> bool {
|
||
api_code(e) == Some(ResponseCode::DoesNotExist)
|
||
}
|
||
|
||
/// A variant of `name` to fall back on when the remote already has that name.
|
||
///
|
||
/// Deliberately the same shape the sync engine uses for its conflict copies
|
||
/// (`sync.rs`, `conflict_path`), so the two halves of the product name the same
|
||
/// situation the same way and a user only has to learn it once.
|
||
fn conflict_name(name: &str, stamp: i64) -> String {
|
||
let path = Path::new(name);
|
||
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(name);
|
||
match path.extension().and_then(|s| s.to_str()) {
|
||
Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"),
|
||
None => format!("{stem} (sync-conflict {stamp})"),
|
||
}
|
||
}
|
||
|
||
/// Fabricate the node the server would have returned, for a `create`/`mkdir`
|
||
/// that could not reach it. Everything the kernel asks about a fresh node —
|
||
/// name, kind, size, times — is knowable locally; the uid is the only invention,
|
||
/// and the drain replaces it with the real one.
|
||
fn local_node(uid: NodeUid, parent_uid: NodeUid, name: String, is_dir: bool) -> Node {
|
||
let now = now_secs();
|
||
Node {
|
||
uid,
|
||
parent_uid: Some(parent_uid),
|
||
kind: if is_dir {
|
||
NodeKind::Folder
|
||
} else {
|
||
NodeKind::File {
|
||
media_type: media_type_for(&name).to_string(),
|
||
total_size_on_storage: 0,
|
||
// No revision has been sealed: nothing has been uploaded yet.
|
||
active_revision_state: None,
|
||
claimed_size: Some(0),
|
||
claimed_modification_time: None,
|
||
}
|
||
},
|
||
name,
|
||
creation_time: now,
|
||
modification_time: now,
|
||
trashed: false,
|
||
is_shared: false,
|
||
is_shared_publicly: false,
|
||
signature_email: None,
|
||
// Nothing signed it: it has never been near the crypto layer.
|
||
verification: Default::default(),
|
||
}
|
||
}
|
||
|
||
/// This machine's hostname, used to name (and later recover) its Proton Drive
|
||
/// Device. Reads the live kernel hostname, falling back to a generic label.
|
||
fn this_hostname() -> String {
|
||
std::fs::read_to_string("/proc/sys/kernel/hostname")
|
||
.ok()
|
||
.map(|s| s.trim().to_string())
|
||
.filter(|s| !s.is_empty())
|
||
.unwrap_or_else(|| "Linux device".to_string())
|
||
}
|
||
|
||
/// Whether `dir` has no entries.
|
||
fn dir_is_empty(dir: &Path) -> std::io::Result<bool> {
|
||
Ok(std::fs::read_dir(dir)?.next().is_none())
|
||
}
|
||
|
||
/// Delete everything inside `dir` but keep `dir` itself (it stays as the FUSE
|
||
/// mountpoint). Used when a `mirror` folder flips to `ondemand`: the local files
|
||
/// are the disk we're reclaiming (devices.md Phase 3).
|
||
fn evict_dir_contents(dir: &Path) -> std::io::Result<()> {
|
||
for entry in std::fs::read_dir(dir)? {
|
||
let path = entry?.path();
|
||
if path.is_dir() && !path.is_symlink() {
|
||
std::fs::remove_dir_all(&path)?;
|
||
} else {
|
||
std::fs::remove_file(&path)?;
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Convert a stored synced folder into its wire form for the front-ends, with
|
||
/// the live progress of its pass when one is running.
|
||
fn sync_folder_info(f: StoredSyncFolder, progress: Option<SyncProgress>) -> SyncFolderInfo {
|
||
SyncFolderInfo {
|
||
id: f.id,
|
||
local_path: f.local_path,
|
||
remote_uid: f.remote_uid,
|
||
mode: f.mode,
|
||
pending_mode: f.pending_mode,
|
||
state: f.state,
|
||
last_sync: f.last_sync,
|
||
progress,
|
||
}
|
||
}
|
||
|
||
/// A coarse MIME type guessed from a file name's extension; Proton stores this
|
||
/// on the node but an exact value is not required for correctness.
|
||
fn media_type_for(name: &str) -> &'static str {
|
||
let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase());
|
||
match ext.as_deref() {
|
||
Some("txt" | "md" | "log") => "text/plain",
|
||
Some("png") => "image/png",
|
||
Some("jpg" | "jpeg") => "image/jpeg",
|
||
Some("gif") => "image/gif",
|
||
Some("pdf") => "application/pdf",
|
||
Some("json") => "application/json",
|
||
Some("html" | "htm") => "text/html",
|
||
_ => "application/octet-stream",
|
||
}
|
||
}
|
||
|
||
/// The plaintext size, in bytes, that a node reports.
|
||
/// Overwrite each file node's `claimed_size` with the optimistic size of its
|
||
/// queued write, where it has one. Folders and nodes with nothing queued are
|
||
/// left alone. See [`Core::stamp_pending_sizes`] for why this exists.
|
||
fn apply_pending_sizes(nodes: &mut [Node], sizes: &HashMap<NodeUid, u64>) {
|
||
for node in nodes {
|
||
if let Some(&len) = sizes.get(&node.uid)
|
||
&& let NodeKind::File { claimed_size, .. } = &mut node.kind
|
||
{
|
||
*claimed_size = Some(len as i64);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// A size upgrade in flight for one folder, so the `getattr`s that need its
|
||
/// result wait for the one batch instead of each fetching its own node.
|
||
///
|
||
/// A plain `Condvar` rather than a channel: the waiters do not want a value,
|
||
/// only the edge, and there may be hundreds of them for one folder.
|
||
#[derive(Default)]
|
||
struct Progress {
|
||
/// The whole batch has been applied (or failed). Nothing more is coming.
|
||
done: bool,
|
||
/// Bumped every time a chunk lands, so a waiter can tell "no progress since
|
||
/// I last looked" from "progress happened while I was looking".
|
||
generation: u64,
|
||
}
|
||
|
||
/// A size upgrade in flight for one folder, so the `getattr`s and `lookup`s that
|
||
/// need its result wait for the one batch instead of each fetching its own node.
|
||
///
|
||
/// Waiters are released **per chunk**, not once at the end: a waiter only cares
|
||
/// about its own file, and waiting for the other 792 is what let a large folder
|
||
/// outrun the timeout (bugs.md B14).
|
||
#[derive(Default)]
|
||
struct SizeUpgrade {
|
||
inner: Mutex<Progress>,
|
||
ready: Condvar,
|
||
}
|
||
|
||
impl SizeUpgrade {
|
||
/// How long a caller will wait for a real size before answering with the
|
||
/// provisional one.
|
||
///
|
||
/// A `stat` that never returns is far worse than one that is briefly wrong:
|
||
/// on timeout the caller falls back to the pre-fix behaviour rather than
|
||
/// wedging whatever is listing the directory.
|
||
const WAIT: Duration = Duration::from_secs(10);
|
||
|
||
/// Block until `resolved` reports the caller's own node has a real size, the
|
||
/// batch finishes without producing one, or [`SizeUpgrade::WAIT`] elapses.
|
||
///
|
||
/// `resolved` is called with **no lock of ours held**. It reaches into
|
||
/// `state`, and the thread applying a chunk holds `state` before it signals
|
||
/// here — taking them in the other order would close the cycle.
|
||
fn wait_for(&self, resolved: impl Fn() -> bool) {
|
||
let deadline = Instant::now() + Self::WAIT;
|
||
loop {
|
||
let seen = {
|
||
let progress = self.inner.lock();
|
||
if progress.done {
|
||
return;
|
||
}
|
||
progress.generation
|
||
};
|
||
if resolved() {
|
||
return;
|
||
}
|
||
let mut progress = self.inner.lock();
|
||
if progress.done {
|
||
return;
|
||
}
|
||
// A chunk landed while we were checking, so re-check rather than
|
||
// sleeping through the answer we were waiting for.
|
||
if progress.generation == seen
|
||
&& self.ready.wait_until(&mut progress, deadline).timed_out()
|
||
{
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Announce that a chunk has been applied. Every waiter re-checks its own
|
||
/// node; the ones it resolved return, the rest go back to sleep.
|
||
fn chunk_done(&self) {
|
||
self.inner.lock().generation += 1;
|
||
self.ready.notify_all();
|
||
}
|
||
|
||
/// Release every waiter for good. The worker must reach this on all paths.
|
||
fn finish(&self) {
|
||
let mut progress = self.inner.lock();
|
||
progress.done = true;
|
||
progress.generation += 1;
|
||
drop(progress);
|
||
self.ready.notify_all();
|
||
}
|
||
}
|
||
|
||
/// Whether a `rename` may replace an existing destination, per POSIX.
|
||
///
|
||
/// Split out from the handler because it is the part that is pure and the part
|
||
/// that is dangerous: every `Err` here is a refusal that happens *before*
|
||
/// anything is trashed, and getting one wrong turns a refusal into the
|
||
/// destruction of the destination (bugs.md B13).
|
||
///
|
||
/// `dst_empty` is only meaningful when `dst_dir`; pass `true` otherwise.
|
||
fn check_replaceable(src_dir: bool, dst_dir: bool, dst_empty: bool) -> Result<(), Errno> {
|
||
match (src_dir, dst_dir) {
|
||
// A non-directory may not replace a directory, or vice versa.
|
||
(false, true) => Err(Errno::EISDIR),
|
||
(true, false) => Err(Errno::ENOTDIR),
|
||
// Proton trashes a folder with its whole subtree, so replacing a
|
||
// non-empty one would silently take its contents with it. POSIX says
|
||
// ENOTEMPTY, which is also the safe answer.
|
||
(true, true) if !dst_empty => Err(Errno::ENOTEMPTY),
|
||
_ => Ok(()),
|
||
}
|
||
}
|
||
|
||
fn node_size(node: &Node) -> u64 {
|
||
match &node.kind {
|
||
NodeKind::Folder => 0,
|
||
NodeKind::File {
|
||
claimed_size,
|
||
total_size_on_storage,
|
||
..
|
||
} => claimed_size.unwrap_or(*total_size_on_storage).max(0) as u64,
|
||
}
|
||
}
|
||
|
||
/// `"1 file"` / `"3 files"` — a count with a correctly pluralised noun, for
|
||
/// human-readable activity-log lines.
|
||
fn count_noun(n: usize, one: &str, many: &str) -> String {
|
||
format!("{n} {}", if n == 1 { one } else { many })
|
||
}
|
||
|
||
/// Bytes rendered with a binary unit and one decimal place (e.g. `"1.2 GB"`),
|
||
/// for the activity log. Uses 1024-based steps but the shorter SI labels.
|
||
fn human_bytes(bytes: u64) -> String {
|
||
const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"];
|
||
let mut val = bytes as f64;
|
||
let mut unit = 0;
|
||
while val >= 1024.0 && unit < UNITS.len() - 1 {
|
||
val /= 1024.0;
|
||
unit += 1;
|
||
}
|
||
if unit == 0 {
|
||
format!("{bytes} B")
|
||
} else {
|
||
format!("{val:.1} {}", UNITS[unit])
|
||
}
|
||
}
|
||
|
||
/// A compact elapsed-time label for the activity log: `"820ms"`, `"43s"`, or
|
||
/// `"2m 5s"`.
|
||
fn human_duration(d: Duration) -> String {
|
||
let secs = d.as_secs();
|
||
if secs == 0 {
|
||
format!("{}ms", d.as_millis())
|
||
} else if secs < 60 {
|
||
format!("{secs}s")
|
||
} else {
|
||
format!("{}m {}s", secs / 60, secs % 60)
|
||
}
|
||
}
|
||
|
||
/// Wall clock in milliseconds since the epoch — the unit the persisted listings
|
||
/// stamp their freshness in.
|
||
fn now_ms() -> i64 {
|
||
SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.map(|d| d.as_millis() as i64)
|
||
.unwrap_or(0)
|
||
}
|
||
|
||
/// The aspect ratio (w/h) of an encoded image, read from its header alone — no
|
||
/// pixels are decoded. `None` when the format is unknown or the header is torn.
|
||
fn ratio_of(bytes: &[u8]) -> Option<f64> {
|
||
let (width, height) = image::ImageReader::new(std::io::Cursor::new(bytes))
|
||
.with_guessed_format()
|
||
.ok()?
|
||
.into_dimensions()
|
||
.ok()?;
|
||
(height > 0).then(|| f64::from(width) / f64::from(height))
|
||
}
|
||
|
||
/// A thumbnail the daemon made itself, and the aspect ratio of the photo it was
|
||
/// made from (free at that point — the full image had to be decoded anyway).
|
||
struct GeneratedThumb {
|
||
bytes: Vec<u8>,
|
||
ratio: f64,
|
||
}
|
||
|
||
/// How one attempt at generating a missing thumbnail ended. The distinction that
|
||
/// matters is *permanent* versus *transient*: only bytes we cannot decode prove
|
||
/// the photo will never have a thumbnail, and only that verdict is persisted.
|
||
enum ThumbAttempt {
|
||
Made(GeneratedThumb),
|
||
/// Decoded nothing — a format this build has no decoder for. Permanent.
|
||
Undecodable,
|
||
/// The photo couldn't be downloaded. Transient: try again next time.
|
||
Unavailable,
|
||
}
|
||
|
||
/// Scale a full-size photo down to a thumbnail: at most [`THUMB_EDGE`] on its
|
||
/// longest side, JPEG, aspect ratio preserved. `None` when the bytes aren't an
|
||
/// image this build can decode — the caller then writes the photo off as
|
||
/// un-thumbnailable.
|
||
///
|
||
/// CPU-bound (a 20 MP JPEG is real work), so callers run it on the blocking pool.
|
||
fn scale_thumbnail(bytes: &[u8]) -> Option<GeneratedThumb> {
|
||
let image = image::ImageReader::new(std::io::Cursor::new(bytes))
|
||
.with_guessed_format()
|
||
.ok()?
|
||
.decode()
|
||
.ok()?;
|
||
let (width, height) = (image.width(), image.height());
|
||
if width == 0 || height == 0 {
|
||
return None;
|
||
}
|
||
let ratio = f64::from(width) / f64::from(height);
|
||
|
||
// `thumbnail` fits the image *inside* the box, so the longest edge lands on
|
||
// THUMB_EDGE and the ratio is untouched.
|
||
let thumb = image.thumbnail(THUMB_EDGE, THUMB_EDGE).to_rgb8();
|
||
let mut bytes = Vec::new();
|
||
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, THUMB_QUALITY)
|
||
.encode_image(&thumb)
|
||
.ok()?;
|
||
Some(GeneratedThumb { bytes, ratio })
|
||
}
|
||
|
||
/// Convert a Unix timestamp (seconds, possibly negative) to a `SystemTime`.
|
||
fn unix_secs(secs: i64) -> SystemTime {
|
||
if secs >= 0 {
|
||
UNIX_EPOCH + Duration::from_secs(secs as u64)
|
||
} else {
|
||
UNIX_EPOCH - Duration::from_secs(secs.unsigned_abs())
|
||
}
|
||
}
|
||
|
||
impl Filesystem for ProtonFs {
|
||
fn lookup(&self, _req: &Request, parent: INodeNo, name: &OsStr, reply: ReplyEntry) {
|
||
let parent = parent.0;
|
||
let name = name.to_string_lossy().into_owned();
|
||
// A folder that has not been listed yet is enumerated from the remote,
|
||
// so serve it from a worker rather than stalling the dispatch loop. A
|
||
// listed folder — the common case — is a map hit, and answering it
|
||
// inline costs less than the handoff would.
|
||
if self.core.children_cached(parent) {
|
||
self.serve_lookup(parent, &name, reply, false);
|
||
return;
|
||
}
|
||
let fs = self.clone();
|
||
self.core.workers.run(Lane::Meta, move || {
|
||
fs.serve_lookup(parent, &name, reply, true)
|
||
});
|
||
}
|
||
|
||
fn getattr(&self, _req: &Request, ino: INodeNo, _fh: Option<FileHandle>, reply: ReplyAttr) {
|
||
let (attr, provisional_in) = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&ino.0) {
|
||
Some(e) => {
|
||
// A file still carrying the cheap enumeration's placeholder
|
||
// size (B12).
|
||
let provisional = matches!(
|
||
&e.node.kind,
|
||
NodeKind::File {
|
||
claimed_size: None,
|
||
..
|
||
}
|
||
)
|
||
.then(|| (e.parent, e.uid.clone()));
|
||
(self.attr(ino.0, &e.node), provisional)
|
||
}
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
// The common case: the size is real, answer from the lock we just held.
|
||
let Some((parent, uid)) = provisional_in else {
|
||
reply.attr(&TTL, &attr);
|
||
return;
|
||
};
|
||
// A provisional size is the *ciphertext* size, which is larger than the
|
||
// file. Publishing it makes every reader that trusts `st_size` — rsync,
|
||
// mmap, sendfile, a sized read loop — run off the end of the file and
|
||
// fail. So resolve it before answering rather than after (bugs.md B14).
|
||
//
|
||
// The cost is one batched round trip for the whole folder, not one per
|
||
// file: `ls -l` is one `getattr` per entry and they collapse onto a
|
||
// single upgrade. This is why B12's split is still worth having — the
|
||
// plain listing never comes here at all.
|
||
//
|
||
// Off the dispatch loop, because it goes to the network: blocking there
|
||
// would stall every other operation on the mount (the B5 lesson).
|
||
let fs = self.clone();
|
||
self.core.workers.run(Lane::Meta, move || {
|
||
fs.core.upgrade_sizes_for_parent(ino.0, &uid, parent);
|
||
// Re-read: the upgrade writes through `state`, and on timeout or
|
||
// failure this is simply the provisional attr we already had.
|
||
let attr = {
|
||
let st = fs.core.state.lock();
|
||
match st.entries.get(&ino.0) {
|
||
Some(e) => fs.attr(ino.0, &e.node),
|
||
// Forgotten while we waited.
|
||
None => attr,
|
||
}
|
||
};
|
||
reply.attr(&TTL, &attr);
|
||
});
|
||
}
|
||
|
||
fn readdir(
|
||
&self,
|
||
_req: &Request,
|
||
ino: INodeNo,
|
||
_fh: FileHandle,
|
||
offset: u64,
|
||
reply: ReplyDirectory,
|
||
) {
|
||
let ino = ino.0;
|
||
// Same split as `lookup`: only the cold, remote-enumerating path pays
|
||
// for a worker.
|
||
if self.core.children_cached(ino) {
|
||
self.serve_readdir(ino, offset, reply);
|
||
return;
|
||
}
|
||
let fs = self.clone();
|
||
self.core
|
||
.workers
|
||
.run(Lane::Meta, move || fs.serve_readdir(ino, offset, reply));
|
||
}
|
||
|
||
fn open(&self, _req: &Request, ino: INodeNo, flags: OpenFlags, reply: ReplyOpen) {
|
||
let (uid, base_mtime, base_size) = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&ino.0) {
|
||
Some(e) if e.node.is_file() => {
|
||
(e.uid.clone(), e.node.modification_time, node_size(&e.node))
|
||
}
|
||
Some(_) => {
|
||
reply.error(Errno::EISDIR);
|
||
return;
|
||
}
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
debug!(ino = ino.0, ?flags, base_size, base_mtime, "open");
|
||
// Read-only opens stay stateless (fh 0). A write open allocates a
|
||
// disk-backed handle; bytes are authored into a scratch file and the
|
||
// untouched remainder is pulled from the base lazily (on read / commit).
|
||
if flags.acc_mode() == OpenAccMode::O_RDONLY {
|
||
reply.opened(FileHandle(0), FopenFlags::empty());
|
||
return;
|
||
}
|
||
let (file, path) = match self.core.cache.create_scratch() {
|
||
Ok(x) => x,
|
||
Err(e) => {
|
||
error!(%uid, error = %e, "create scratch file failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
};
|
||
let mut st = self.core.state.lock();
|
||
let fh = st.next_fh;
|
||
st.next_fh += 1;
|
||
st.handles.insert(
|
||
fh,
|
||
WriteHandle {
|
||
ino: ino.0,
|
||
uid,
|
||
file: Arc::new(file),
|
||
path,
|
||
written: Intervals::default(),
|
||
// Starts at the current size; reads in [0, base_size) come from
|
||
// the base until overwritten.
|
||
len: base_size,
|
||
base_size,
|
||
base_mtime,
|
||
dirty: false,
|
||
},
|
||
);
|
||
reply.opened(FileHandle(fh), FopenFlags::empty());
|
||
}
|
||
|
||
fn read(
|
||
&self,
|
||
_req: &Request,
|
||
ino: INodeNo,
|
||
_fh: FileHandle,
|
||
offset: u64,
|
||
size: u32,
|
||
_flags: OpenFlags,
|
||
_lock_owner: Option<LockOwner>,
|
||
reply: ReplyData,
|
||
) {
|
||
// A file open for writing is served from its handle so reads see the
|
||
// in-flight (possibly unsaved) content: authored bytes come from the
|
||
// scratch file, untouched bytes from the base.
|
||
//
|
||
// This path stays on the dispatch loop. `write` runs there too, so
|
||
// keeping its reads there as well preserves the ordering the write path
|
||
// has always had between a write and a read of the same handle; a mostly
|
||
// local read is not worth reasoning about that for.
|
||
let handle = {
|
||
let st = self.core.state.lock();
|
||
st.handles.values().find(|h| h.ino == ino.0).map(|h| {
|
||
(
|
||
h.file.clone(),
|
||
h.len,
|
||
h.uid.clone(),
|
||
h.base_mtime,
|
||
h.base_size,
|
||
h.written.clone(),
|
||
)
|
||
})
|
||
};
|
||
if let Some((file, len, uid, base_mtime, base_size, written)) = handle {
|
||
match self.core.serve_open_read(
|
||
&file,
|
||
len,
|
||
&uid,
|
||
base_mtime,
|
||
base_size,
|
||
&written,
|
||
offset,
|
||
size as u64,
|
||
) {
|
||
Ok(bytes) => reply.data(&bytes),
|
||
Err(e) => reply.error(e),
|
||
}
|
||
return;
|
||
}
|
||
let (uid, mtime, fsize, is_video) = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&ino.0) {
|
||
Some(e) if e.node.is_file() => {
|
||
let media_type = match &e.node.kind {
|
||
NodeKind::File { media_type, .. } => Some(media_type.as_str()),
|
||
NodeKind::Folder => None,
|
||
};
|
||
let is_video =
|
||
PhotoKind::classify(Some(&e.node.name), media_type) == PhotoKind::Video;
|
||
(
|
||
e.uid.clone(),
|
||
e.node.modification_time,
|
||
node_size(&e.node),
|
||
is_video,
|
||
)
|
||
}
|
||
Some(_) => {
|
||
reply.error(Errno::EISDIR);
|
||
return;
|
||
}
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
// A large unpinned video streams without polluting the block cache (see
|
||
// [`STREAM_BYPASS_MIN`]); everything else caches its blocks as usual.
|
||
let cache_blocks =
|
||
!(is_video && fsize >= STREAM_BYPASS_MIN && !self.core.cache.is_pinned(&uid));
|
||
// Off the dispatch loop: this is the one handler that routinely goes to
|
||
// the network (block fetch + decrypt, hundreds of ms on a cold file),
|
||
// and until it returns the kernel's next request for this mount is not
|
||
// even read off the FUSE device. It only reads `state`, so moving it
|
||
// races with nothing. FUSE does not require replies in request order.
|
||
let core = self.core.clone();
|
||
self.core.workers.run(Lane::Transfer, move || {
|
||
match core.read_range(&uid, mtime, fsize, offset, size as u64, cache_blocks) {
|
||
Ok(bytes) => reply.data(&bytes),
|
||
Err(e) => reply.error(e),
|
||
}
|
||
});
|
||
}
|
||
|
||
fn create(
|
||
&self,
|
||
_req: &Request,
|
||
parent: INodeNo,
|
||
name: &OsStr,
|
||
_mode: u32,
|
||
_umask: u32,
|
||
_flags: i32,
|
||
reply: ReplyCreate,
|
||
) {
|
||
let parent = parent.0;
|
||
if let Err(e) = self.core.ensure_children(parent) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
let name = name.to_string_lossy().into_owned();
|
||
let parent_uid = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&parent) {
|
||
Some(e) => e.uid.clone(),
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
// Offline the server cannot mint a uid, so invent one and queue the
|
||
// create. The file is real to the caller either way; only its identity is
|
||
// provisional until the drain (offline.md Phase 3b).
|
||
//
|
||
// A parent that is itself still queued forces the same path even when we
|
||
// are online: the API has no folder to put this in yet.
|
||
let node = if self.core.online.load(Ordering::Relaxed) && !is_local_uid(&parent_uid) {
|
||
// Create an empty file on the remote so it has a real uid immediately;
|
||
// written bytes are buffered and sealed as a new revision on close.
|
||
let new_uid = match self.core.rt.block_on(self.core.client.upload_file(
|
||
&parent_uid,
|
||
&name,
|
||
media_type_for(&name),
|
||
b"",
|
||
)) {
|
||
Ok(u) => u,
|
||
Err(e) => {
|
||
error!(%parent_uid, name, error = %e, "create file failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
};
|
||
match self.core.fetch_node(&new_uid) {
|
||
Ok(n) => n,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
}
|
||
} else {
|
||
match self.core.queue_local_node(&parent_uid, &name, false) {
|
||
Ok(n) => n,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
let new_uid = node.uid.clone();
|
||
// The base this handle writes over is the node as it actually exists —
|
||
// the empty file the server just minted — so its modification time comes
|
||
// from the node, never from the local clock. `queue_revision` turns this
|
||
// into `StagedWrite::based_on` and the drain compares that against the
|
||
// remote: a `now_secs()` here differs from the server's stamp by however
|
||
// long the create round-trip took, so the first write to a brand-new file
|
||
// conflicts with its own create whenever the second happens to tick in
|
||
// between.
|
||
let base_mtime = node.modification_time;
|
||
let (file, path) = match self.core.cache.create_scratch() {
|
||
Ok(x) => x,
|
||
Err(e) => {
|
||
error!(%new_uid, error = %e, "create scratch file failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
};
|
||
let mut st = self.core.state.lock();
|
||
let ino = st.intern(parent, node);
|
||
if let Some(kids) = st.children.get_mut(&parent)
|
||
&& !kids.contains(&ino)
|
||
{
|
||
kids.push(ino);
|
||
}
|
||
let fh = st.next_fh;
|
||
st.next_fh += 1;
|
||
st.handles.insert(
|
||
fh,
|
||
// A brand-new file: empty base, everything written is authored.
|
||
WriteHandle {
|
||
ino,
|
||
uid: new_uid,
|
||
file: Arc::new(file),
|
||
path,
|
||
written: Intervals::default(),
|
||
len: 0,
|
||
base_size: 0,
|
||
base_mtime,
|
||
dirty: false,
|
||
},
|
||
);
|
||
let attr = self.attr(ino, &st.entries.get(&ino).unwrap().node);
|
||
reply.created(
|
||
&TTL,
|
||
&attr,
|
||
Generation(0),
|
||
FileHandle(fh),
|
||
FopenFlags::empty(),
|
||
);
|
||
}
|
||
|
||
fn write(
|
||
&self,
|
||
_req: &Request,
|
||
ino: INodeNo,
|
||
fh: FileHandle,
|
||
offset: u64,
|
||
data: &[u8],
|
||
_write_flags: WriteFlags,
|
||
_flags: OpenFlags,
|
||
_lock_owner: Option<LockOwner>,
|
||
reply: ReplyWrite,
|
||
) {
|
||
let fh = fh.0;
|
||
// Stage the bytes straight into the scratch file (no base download): only
|
||
// the untouched remainder is pulled from the remote, and only at commit.
|
||
let file = {
|
||
let st = self.core.state.lock();
|
||
match st.handles.get(&fh) {
|
||
Some(h) => h.file.clone(),
|
||
None => {
|
||
reply.error(Errno::EBADF);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
if let Err(e) = file.write_all_at(data, offset) {
|
||
error!(ino = ino.0, fh, error = %e, "scratch write failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
let new_len = {
|
||
let mut st = self.core.state.lock();
|
||
let Some(h) = st.handles.get_mut(&fh) else {
|
||
reply.error(Errno::EBADF);
|
||
return;
|
||
};
|
||
let end = offset + data.len() as u64;
|
||
h.written.add(offset, end);
|
||
h.len = h.len.max(end);
|
||
h.dirty = true;
|
||
let len = h.len;
|
||
st.set_size(ino.0, len);
|
||
len
|
||
};
|
||
debug!(
|
||
ino = ino.0,
|
||
fh,
|
||
offset,
|
||
len = data.len(),
|
||
new_len,
|
||
"staged write"
|
||
);
|
||
reply.written(data.len() as u32);
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn setattr(
|
||
&self,
|
||
_req: &Request,
|
||
ino: INodeNo,
|
||
_mode: Option<u32>,
|
||
_uid: Option<u32>,
|
||
_gid: Option<u32>,
|
||
size: Option<u64>,
|
||
_atime: Option<TimeOrNow>,
|
||
_mtime: Option<TimeOrNow>,
|
||
_ctime: Option<SystemTime>,
|
||
fh: Option<FileHandle>,
|
||
_crtime: Option<SystemTime>,
|
||
_chgtime: Option<SystemTime>,
|
||
_bkuptime: Option<SystemTime>,
|
||
_flags: Option<BsdFileFlags>,
|
||
reply: ReplyAttr,
|
||
) {
|
||
// Only resizes change remote state; everything else (mode/owner/times) is
|
||
// accepted and ignored so tools that chmod/utimes after writing succeed.
|
||
if let Some(size) = size {
|
||
let fh = fh.map(|f| f.0).filter(|&f| f != 0);
|
||
let handled = {
|
||
let mut st = self.core.state.lock();
|
||
// The kernel does not put `O_TRUNC` in the `open` flags unless
|
||
// the mount enables `atomic_o_trunc`; it opens, then truncates
|
||
// with a *separate* `setattr` carrying no file handle. So an fh
|
||
// is a hint, not a precondition — fall back to the write handle
|
||
// for this inode, the same lookup `read` does and for the same
|
||
// reason. Without it a `cp` over an existing file becomes two
|
||
// independent queued ops: a truncate, plus a release that still
|
||
// believes the file is its old length. They then conflict with
|
||
// each other and the write is diverted into a conflict copy.
|
||
let target = match fh {
|
||
Some(fh) if st.handles.contains_key(&fh) => Some(fh),
|
||
_ => st
|
||
.handles
|
||
.iter()
|
||
.find(|(_, h)| h.ino == ino.0)
|
||
.map(|(&fh, _)| fh),
|
||
};
|
||
match target.and_then(|fh| st.handles.get_mut(&fh)) {
|
||
Some(h) => {
|
||
if size < h.len {
|
||
// Shrink: drop authored ranges past the new end.
|
||
h.written.clip(size);
|
||
} else if size > h.len {
|
||
// Grow: the new tail is defined as zeros, so claim
|
||
// it as authored rather than base content.
|
||
h.written.add(h.len, size);
|
||
}
|
||
let _ = h.file.set_len(size);
|
||
h.len = size;
|
||
h.dirty = true;
|
||
true
|
||
}
|
||
None => false,
|
||
}
|
||
};
|
||
debug!(ino = ino.0, size, fh = ?fh, handled, "setattr resize");
|
||
if !handled {
|
||
// Path-based truncate with no open write handle — a shell's
|
||
// `> file`. This is the second write path into the API, and
|
||
// queueing it is what lets a redirect work offline at all: it
|
||
// never reaches `release`, so without this it failed before any
|
||
// byte was written (offline.md Phase 2/3b).
|
||
if let Err(e) = self.core.queue_truncate(ino.0, size) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
} else {
|
||
self.core.state.lock().set_size(ino.0, size);
|
||
}
|
||
}
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&ino.0) {
|
||
Some(e) => {
|
||
let attr = self.attr(ino.0, &e.node);
|
||
reply.attr(&TTL, &attr);
|
||
}
|
||
None => reply.error(Errno::ENOENT),
|
||
}
|
||
}
|
||
|
||
/// `close(2)` calls flush before release. The upload is queued in `release`
|
||
/// and performed in the background (offline.md Phase 3), so there is nothing
|
||
/// to push here — the written bytes are already in the scratch file.
|
||
fn flush(
|
||
&self,
|
||
_req: &Request,
|
||
_ino: INodeNo,
|
||
_fh: FileHandle,
|
||
_lock_owner: LockOwner,
|
||
reply: ReplyEmpty,
|
||
) {
|
||
reply.ok();
|
||
}
|
||
|
||
/// Durability here means the scratch file, not the remote: a caller asking
|
||
/// for fsync wants its bytes to survive a crash, and blocking it on an upload
|
||
/// (which offline would never finish) buys nothing the queue does not already
|
||
/// guarantee.
|
||
fn fsync(
|
||
&self,
|
||
_req: &Request,
|
||
_ino: INodeNo,
|
||
fh: FileHandle,
|
||
_datasync: bool,
|
||
reply: ReplyEmpty,
|
||
) {
|
||
// Everything the durability sidecar needs, copied out so no lock is held
|
||
// across the sync.
|
||
let handle = self.core.state.lock().handles.get(&fh.0).map(|h| {
|
||
(
|
||
h.file.clone(),
|
||
h.path.clone(),
|
||
h.uid.clone(),
|
||
h.dirty,
|
||
h.written.clone(),
|
||
h.len,
|
||
h.base_size,
|
||
h.base_mtime,
|
||
)
|
||
});
|
||
let Some((f, path, uid, dirty, written, len, base_size, base_mtime)) = handle else {
|
||
reply.error(Errno::EBADF);
|
||
return;
|
||
};
|
||
if let Err(e) = f.sync_all() {
|
||
error!(fh = fh.0, error = %e, "fsync of scratch file failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
// A clean handle has nothing on it that the remote does not already
|
||
// hold, so there is nothing for a crash to lose.
|
||
if !dirty {
|
||
reply.ok();
|
||
return;
|
||
}
|
||
// The bytes are on stable storage, but only the scratch file knows that,
|
||
// and the scratch directory is cleared at open. Record what the blob is
|
||
// so a restart can hand it to the upload queue instead of deleting it.
|
||
//
|
||
// Deliberately *not* the full release path: staging the blob here would
|
||
// move the file out from under a handle the application still has open,
|
||
// and queueing an upload per `fsync` would push a revision for every
|
||
// barrier in a write loop. This only makes the existing file findable.
|
||
let authored: Vec<(u64, u64)> = written
|
||
.segments(0, len)
|
||
.into_iter()
|
||
.filter(|&(_, _, authored)| authored)
|
||
.map(|(s, e, _)| (s, e))
|
||
.collect();
|
||
let meta = StagedWrite {
|
||
uid: uid.to_string(),
|
||
len,
|
||
base_size,
|
||
base_mtime,
|
||
complete: authored == [(0, len)],
|
||
authored,
|
||
based_on: self.core.remote_baseline(&uid, base_mtime, base_size),
|
||
};
|
||
// A failed sidecar means the write is not durable, and `fsync` promising
|
||
// otherwise is the defect this exists to fix — so it is an error, not a
|
||
// warning.
|
||
if let Err(e) = self.core.cache.mark_scratch_durable(&path, &meta) {
|
||
error!(%uid, error = %e, "recording fsync durability failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
reply.ok();
|
||
}
|
||
|
||
fn release(
|
||
&self,
|
||
_req: &Request,
|
||
_ino: INodeNo,
|
||
fh: FileHandle,
|
||
_flags: OpenFlags,
|
||
_lock_owner: Option<LockOwner>,
|
||
_flush: bool,
|
||
reply: ReplyEmpty,
|
||
) {
|
||
let handle = self.core.state.lock().handles.remove(&fh.0);
|
||
// Hand the bytes to the queue rather than uploading them here: the
|
||
// scratch file is the only copy of what was just written, and blocking
|
||
// the caller on the network is what made a copy into the mount run at
|
||
// upload speed (and fail outright offline).
|
||
match handle {
|
||
Some(h) => match self.core.queue_revision(&h) {
|
||
Ok(()) => reply.ok(),
|
||
Err(e) => reply.error(e),
|
||
},
|
||
None => reply.ok(),
|
||
}
|
||
}
|
||
|
||
fn unlink(&self, _req: &Request, parent: INodeNo, name: &OsStr, reply: ReplyEmpty) {
|
||
self.trash_child(parent.0, &name.to_string_lossy(), reply);
|
||
}
|
||
|
||
fn rmdir(&self, _req: &Request, parent: INodeNo, name: &OsStr, reply: ReplyEmpty) {
|
||
self.trash_child(parent.0, &name.to_string_lossy(), reply);
|
||
}
|
||
|
||
fn mkdir(
|
||
&self,
|
||
_req: &Request,
|
||
parent: INodeNo,
|
||
name: &OsStr,
|
||
_mode: u32,
|
||
_umask: u32,
|
||
reply: ReplyEntry,
|
||
) {
|
||
let parent = parent.0;
|
||
if let Err(e) = self.core.ensure_children(parent) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
let name = name.to_string_lossy().into_owned();
|
||
let parent_uid = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&parent) {
|
||
Some(e) => e.uid.clone(),
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
let now = SystemTime::now()
|
||
.duration_since(UNIX_EPOCH)
|
||
.map(|d| d.as_secs() as i64)
|
||
.ok();
|
||
// As in `create`: offline — or under a parent that is itself still
|
||
// queued — the folder becomes a placeholder that the drain turns into a
|
||
// real one (offline.md Phase 3b).
|
||
let node = if self.core.online.load(Ordering::Relaxed) && !is_local_uid(&parent_uid) {
|
||
let new_uid =
|
||
match self
|
||
.core
|
||
.rt
|
||
.block_on(self.core.client.create_folder(&parent_uid, &name, now))
|
||
{
|
||
Ok(u) => u,
|
||
Err(e) => {
|
||
error!(%parent_uid, name, error = %e, "create folder failed");
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
};
|
||
match self.core.fetch_node(&new_uid) {
|
||
Ok(n) => n,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
}
|
||
} else {
|
||
match self.core.queue_local_node(&parent_uid, &name, true) {
|
||
Ok(n) => n,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
let mut st = self.core.state.lock();
|
||
let local = is_local_uid(&node.uid);
|
||
let uid = node.uid.clone();
|
||
let ino = st.intern(parent, node);
|
||
if let Some(kids) = st.children.get_mut(&parent)
|
||
&& !kids.contains(&ino)
|
||
{
|
||
kids.push(ino);
|
||
}
|
||
// A folder that exists only here has nothing to enumerate, and asking the
|
||
// API to enumerate a `local~` uid would 404. Record it as fully listed and
|
||
// empty, which it is, so reads stay offline-clean across a restart too.
|
||
if local {
|
||
st.children.insert(ino, Vec::new());
|
||
if let Err(e) = self.core.db.set_listed(&uid, true) {
|
||
warn!(%uid, error = %e, "db set_listed(true) failed for local folder");
|
||
}
|
||
}
|
||
let attr = self.attr(ino, &st.entries.get(&ino).unwrap().node);
|
||
reply.entry(&TTL, &attr, Generation(0));
|
||
}
|
||
|
||
fn rename(
|
||
&self,
|
||
_req: &Request,
|
||
parent: INodeNo,
|
||
name: &OsStr,
|
||
newparent: INodeNo,
|
||
newname: &OsStr,
|
||
flags: RenameFlags,
|
||
reply: ReplyEmpty,
|
||
) {
|
||
let parent = parent.0;
|
||
let newparent = newparent.0;
|
||
let name = name.to_string_lossy().into_owned();
|
||
let newname = newname.to_string_lossy().into_owned();
|
||
let (ino, uid) = match self.core.lookup_child(parent, &name) {
|
||
Ok(x) => x,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
};
|
||
// The destination has to be listed either way: the queued path pushes
|
||
// the node into that listing, and the online one drops it to force a
|
||
// re-enumeration.
|
||
if newparent != parent
|
||
&& let Err(e) = self.core.ensure_children(newparent)
|
||
{
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
let new_parent_uid = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&newparent) {
|
||
Some(e) => e.uid.clone(),
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
// `rename(2)` replaces an existing destination; Proton has no replacing
|
||
// rename, so the victim has to be removed first and the operation stops
|
||
// being atomic. Without this the 422 surfaced as a blanket EIO and every
|
||
// write-to-temp-then-rename tool — rsync, atomic editor saves — failed
|
||
// at the very end of its transfer (bugs.md B13).
|
||
//
|
||
// `RENAME_EXCHANGE` has no Proton primitive and cannot be emulated
|
||
// without a window in which one of the two names does not exist.
|
||
if flags.contains(RenameFlags::RENAME_EXCHANGE) {
|
||
reply.error(Errno::EINVAL);
|
||
return;
|
||
}
|
||
let victim = match self.core.lookup_child(newparent, &newname) {
|
||
// Renaming a node onto its own name is a no-op, never a self-replace.
|
||
Ok((_, vuid)) if vuid == uid => None,
|
||
Ok(x) => Some(x),
|
||
// Nothing to replace: the overwhelmingly common case.
|
||
Err(e) if e.code() == libc::ENOENT => None,
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
};
|
||
if let Some((victim_ino, victim_uid)) = &victim {
|
||
if flags.contains(RenameFlags::RENAME_NOREPLACE) {
|
||
reply.error(Errno::EEXIST);
|
||
return;
|
||
}
|
||
// POSIX requires both ends to agree on being a directory, and the
|
||
// check has to happen before anything is trashed: getting it wrong
|
||
// turns a refusal into the destruction of the destination.
|
||
let (src_dir, dst_dir) = {
|
||
let st = self.core.state.lock();
|
||
let is_dir = |i: &u64| {
|
||
st.entries
|
||
.get(i)
|
||
.map(|e| matches!(e.node.kind, NodeKind::Folder))
|
||
};
|
||
match (is_dir(&ino), is_dir(victim_ino)) {
|
||
(Some(a), Some(b)) => (a, b),
|
||
_ => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
// Replacing a directory means trashing it, and Proton trashes a
|
||
// folder with its whole subtree — so the destination's emptiness has
|
||
// to be known before the decision is made.
|
||
let dst_empty = if dst_dir {
|
||
if let Err(e) = self.core.ensure_children(*victim_ino) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
self.core
|
||
.state
|
||
.lock()
|
||
.children
|
||
.get(victim_ino)
|
||
.is_none_or(|kids| kids.is_empty())
|
||
} else {
|
||
true
|
||
};
|
||
if let Err(e) = check_replaceable(src_dir, dst_dir, dst_empty) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
if let Err(e) = self.core.remove_replaced(victim_uid, &newname) {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
}
|
||
// A node whose own creation is still queued has no server-side identity
|
||
// to rename: the queued op *is* the node, so rewriting its target is the
|
||
// whole rename. Nothing reaches the API, which is why this works offline
|
||
// (offline.md Phase 3b).
|
||
if is_local_uid(&uid) {
|
||
match self.core.db.rewrite_op_target(
|
||
&uid.to_string(),
|
||
&new_parent_uid.to_string(),
|
||
&newname,
|
||
) {
|
||
Ok(true) => {
|
||
self.core.state.lock().rename_in_place(
|
||
ino,
|
||
newparent,
|
||
&new_parent_uid,
|
||
&newname,
|
||
);
|
||
debug!(%uid, newname, "renamed a node whose create is still queued");
|
||
reply.ok();
|
||
}
|
||
// The create drained underneath us, so the node has a real uid
|
||
// now and this handle's is stale. A retry resolves it.
|
||
Ok(false) => {
|
||
warn!(%uid, name, newname, "queued create vanished under a rename");
|
||
self.core.restore_replaced(victim.as_ref(), &newname);
|
||
reply.error(Errno::EBUSY);
|
||
}
|
||
Err(e) => {
|
||
error!(%uid, error = %e, "rewriting a queued create's target failed");
|
||
self.core.restore_replaced(victim.as_ref(), &newname);
|
||
reply.error(Errno::EIO);
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
// Offline, or into a folder that does not exist remotely yet: queue the
|
||
// rename rather than 404 or fail. Online moves into a real parent still
|
||
// take the synchronous path, so a genuine API refusal (permissions, a
|
||
// name clash) surfaces to the caller instead of becoming a queued op
|
||
// that can only ever conflict.
|
||
if !self.core.online.load(Ordering::Relaxed) || is_local_uid(&new_parent_uid) {
|
||
match self
|
||
.core
|
||
.queue_rename(ino, &uid, newparent, &new_parent_uid, &newname)
|
||
{
|
||
Ok(()) => reply.ok(),
|
||
Err(e) => {
|
||
self.core.restore_replaced(victim.as_ref(), &newname);
|
||
reply.error(e);
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
// Move first if the parent changed, then rename if the name changed.
|
||
// A failure past this point has already trashed any node the rename was
|
||
// replacing, so put it back rather than leaving the caller with neither
|
||
// the source moved nor the destination intact.
|
||
if newparent != parent
|
||
&& let Err(e) = self
|
||
.core
|
||
.rt
|
||
.block_on(self.core.client.move_node(&uid, &new_parent_uid))
|
||
{
|
||
error!(%uid, error = %e, "move failed");
|
||
self.core.restore_replaced(victim.as_ref(), &newname);
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
if newname != name
|
||
&& let Err(e) = self
|
||
.core
|
||
.rt
|
||
.block_on(self.core.client.rename_node(&uid, &newname, None))
|
||
{
|
||
error!(%uid, error = %e, "rename failed");
|
||
self.core.restore_replaced(victim.as_ref(), &newname);
|
||
reply.error(Errno::EIO);
|
||
return;
|
||
}
|
||
self.core
|
||
.state
|
||
.lock()
|
||
.relocate(ino, parent, newparent, &new_parent_uid, &newname);
|
||
reply.ok();
|
||
}
|
||
|
||
/// Expose a file's server-side thumbnail/preview as an extended attribute, so
|
||
/// a previewing client can fetch it without downloading the whole file. The
|
||
/// bytes are fetched on demand and cached; absence yields `ENODATA`.
|
||
fn getxattr(&self, _req: &Request, ino: INodeNo, name: &OsStr, size: u32, reply: ReplyXattr) {
|
||
// Always off the dispatch loop: a miss goes to the wire, and a lister
|
||
// that stats a directory issues one of these per file per advertised
|
||
// name — inline, that serialized the whole mount behind ~186 ms of
|
||
// network per call (B5).
|
||
let fs = self.clone();
|
||
let name = name.to_os_string();
|
||
self.core.workers.run(Lane::Meta, move || {
|
||
fs.serve_getxattr(ino, &name, size, reply)
|
||
});
|
||
}
|
||
|
||
fn listxattr(&self, _req: &Request, ino: INodeNo, size: u32, reply: ReplyXattr) {
|
||
self.serve_listxattr(ino, size, reply);
|
||
}
|
||
}
|
||
|
||
impl ProtonFs {
|
||
/// The body of [`Filesystem::getxattr`], on a worker thread.
|
||
fn serve_getxattr(&self, ino: INodeNo, name: &OsStr, size: u32, reply: ReplyXattr) {
|
||
let ttype = match name.to_str() {
|
||
Some(XATTR_THUMBNAIL) => ThumbnailType::Thumbnail,
|
||
Some(XATTR_PREVIEW) => ThumbnailType::Preview,
|
||
// Any other attribute simply does not exist on this filesystem.
|
||
_ => {
|
||
reply.error(Errno::ENODATA);
|
||
return;
|
||
}
|
||
};
|
||
let bytes = match self.core.thumbnail(ino.0, ttype) {
|
||
Ok(Some(b)) => b,
|
||
Ok(None) => {
|
||
reply.error(Errno::ENODATA);
|
||
return;
|
||
}
|
||
Err(e) => {
|
||
reply.error(e);
|
||
return;
|
||
}
|
||
};
|
||
let len = bytes.len() as u32;
|
||
// The xattr protocol: a zero `size` is a probe for the length; otherwise
|
||
// the caller's buffer must be large enough or it gets `ERANGE`.
|
||
if size == 0 {
|
||
reply.size(len);
|
||
} else if size < len {
|
||
reply.error(Errno::ERANGE);
|
||
} else {
|
||
reply.data(&bytes);
|
||
}
|
||
}
|
||
|
||
/// Advertise the thumbnail/preview attribute names, but only for files whose
|
||
/// media type can actually carry one.
|
||
///
|
||
/// Listing them for every file is what made `ls -l` expensive: an xattr-aware
|
||
/// lister asks for each advertised name, and each ask that misses is a network
|
||
/// round trip (B5). Proton only ever generates thumbnails for images and
|
||
/// videos, so advertising them on a `.mkv` or a `.pdf` is an invitation to do
|
||
/// work that can only end in `ENODATA`. `getxattr` still serves an explicit
|
||
/// request for an unadvertised name, so nothing becomes unreachable.
|
||
fn serve_listxattr(&self, ino: INodeNo, size: u32, reply: ReplyXattr) {
|
||
let thumbnailable = {
|
||
let st = self.core.state.lock();
|
||
match st.entries.get(&ino.0) {
|
||
Some(e) => match &e.node.kind {
|
||
NodeKind::File { media_type, .. } => {
|
||
media_type.starts_with("image/") || media_type.starts_with("video/")
|
||
}
|
||
NodeKind::Folder => false,
|
||
},
|
||
None => {
|
||
reply.error(Errno::ENOENT);
|
||
return;
|
||
}
|
||
}
|
||
};
|
||
// xattr names are returned as a NUL-terminated, concatenated list.
|
||
let mut buf = Vec::new();
|
||
if thumbnailable {
|
||
for name in [XATTR_THUMBNAIL, XATTR_PREVIEW] {
|
||
buf.extend_from_slice(name.as_bytes());
|
||
buf.push(0);
|
||
}
|
||
}
|
||
let len = buf.len() as u32;
|
||
if size == 0 {
|
||
reply.size(len);
|
||
} else if size < len {
|
||
reply.error(Errno::ERANGE);
|
||
} else {
|
||
reply.data(&buf);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Apply one remote event to the local cache and notify the kernel so it drops
|
||
/// any stale cached metadata/data for the affected inodes.
|
||
///
|
||
/// The cache is authoritative-by-absence: dropping a directory's `children`
|
||
/// entry forces the next `lookup`/`readdir` to re-enumerate from the remote, so
|
||
/// most events only need to invalidate listings rather than re-fetch eagerly.
|
||
fn apply_event(
|
||
state: &Mutex<State>,
|
||
content: &ContentCache,
|
||
pending: &Mutex<HashMap<NodeUid, PendingRevision>>,
|
||
notifier: &Notifier,
|
||
event: &DriveEvent,
|
||
) {
|
||
match event {
|
||
DriveEvent::NodeUpdated {
|
||
node_uid,
|
||
parent_node_uid,
|
||
is_trashed,
|
||
..
|
||
} => {
|
||
let mut st = state.lock();
|
||
if *is_trashed {
|
||
// Trashing makes a node vanish from its parent listing.
|
||
let child = st.by_uid.get(node_uid).copied();
|
||
if let Some((parent, name)) = st.forget(node_uid) {
|
||
content.evict(node_uid);
|
||
match child {
|
||
Some(child) => {
|
||
let _ =
|
||
notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
||
}
|
||
None => {
|
||
let _ = notifier.inval_entry(INodeNo(parent), OsStr::new(&name));
|
||
}
|
||
}
|
||
}
|
||
} else if pending.lock().contains_key(node_uid) {
|
||
// A node we owe an upload for is *ahead* of the remote, not
|
||
// behind it: this event is almost always the echo of our own
|
||
// empty-file create, and re-fetching would replace the size and
|
||
// mtime of the write we just accepted with the stale revision's
|
||
// — making a file that was copied in seconds ago read as empty
|
||
// until its upload lands (offline.md Phase 3).
|
||
debug!(uid = %node_uid, "ignoring remote event for a node with a queued write");
|
||
} else if let Some(&ino) = st.by_uid.get(node_uid) {
|
||
// Known node changed: drop its cached attrs/data (and listing if
|
||
// it is a directory) so the next access re-fetches. Its content
|
||
// blob may now be stale, so evict it too.
|
||
st.invalidate_listing(ino);
|
||
content.evict(node_uid);
|
||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||
}
|
||
// A create (or move-in) shows up as a change to the parent listing;
|
||
// drop it so the new child is picked up on the next readdir.
|
||
if let Some(parent_uid) = parent_node_uid
|
||
&& let Some(&parent) = st.by_uid.get(parent_uid)
|
||
{
|
||
st.invalidate_listing(parent);
|
||
let _ = notifier.inval_inode(INodeNo(parent), 0, 0);
|
||
}
|
||
}
|
||
DriveEvent::NodeDeleted { node_uid, .. } => {
|
||
let mut st = state.lock();
|
||
// Capture the inode before `forget` clears the uid mapping.
|
||
let child = st.by_uid.get(node_uid).copied();
|
||
content.evict(node_uid);
|
||
if let Some((parent, name)) = st.forget(node_uid) {
|
||
match child {
|
||
Some(child) => {
|
||
let _ = notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
||
}
|
||
None => {
|
||
let _ = notifier.inval_entry(INodeNo(parent), OsStr::new(&name));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// Continuity or scope was lost: our cached listings may be arbitrarily
|
||
// stale, so drop every listing and tell the kernel to forget all
|
||
// metadata. Inodes stay stable; dirs simply re-enumerate on next access.
|
||
DriveEvent::ContinuityLost { .. } | DriveEvent::ScopeAccessLost { .. } => {
|
||
warn!("event continuity lost; dropping all cached listings, resyncing lazily");
|
||
let mut st = state.lock();
|
||
let dirs: Vec<u64> = st.children.keys().copied().collect();
|
||
for &ino in &dirs {
|
||
st.invalidate_listing(ino);
|
||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||
}
|
||
}
|
||
// No substantive local change; the cursor advance is handled by the
|
||
// caller persisting the event id.
|
||
DriveEvent::CursorAdvanced { .. } | DriveEvent::SharedWithMeUpdated { .. } => {}
|
||
}
|
||
}
|
||
|
||
/// Poll the remote event cursor forever, applying each batch to the shared
|
||
/// state. Resumes from the cursor persisted in the DB so changes made while
|
||
/// unmounted are applied; only a first-ever mount seeds from the server head.
|
||
/// The cursor is persisted after every batch. Runs as a Tokio task; returns
|
||
/// only on fatal error.
|
||
async fn run_event_sync(
|
||
client: ProtonDriveClient,
|
||
scope: DriveEventScopeId,
|
||
state: Arc<Mutex<State>>,
|
||
content: Arc<ContentCache>,
|
||
db: Arc<Db>,
|
||
pending: Arc<Mutex<HashMap<NodeUid, PendingRevision>>>,
|
||
notifier: Notifier,
|
||
) {
|
||
let mut cursor: Option<DriveEventId> = match db.get_event_cursor() {
|
||
// Resume: pick up exactly where the last run left off.
|
||
Ok(Some(saved)) => Some(DriveEventId::from(saved)),
|
||
// First mount: a `None` cursor yields a single `CursorAdvanced` at the
|
||
// server head; persist it so the next restart resumes instead of
|
||
// reseeding (which would skip everything that changed offline).
|
||
// Seeding needs the network, and this task also runs on mounts that
|
||
// started offline (offline.md Phase 1) — so retry rather than giving up,
|
||
// which used to disable live sync for the life of the daemon.
|
||
Ok(None) => {
|
||
let mut delay = ONLINE_PROBE_MIN;
|
||
loop {
|
||
match client.enumerate_events(&scope, None).await {
|
||
Ok(events) => {
|
||
let head = events.last().map(|e| e.id().clone());
|
||
if let Some(c) = &head
|
||
&& let Err(e) = db.set_event_cursor(c.as_str())
|
||
{
|
||
warn!(error = %e, "persist seed cursor failed");
|
||
}
|
||
break head;
|
||
}
|
||
Err(e) => {
|
||
warn!(error = %e, ?delay, "seed event cursor failed; retrying");
|
||
tokio::time::sleep(delay).await;
|
||
delay = (delay * 2).min(ONLINE_PROBE_MAX);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
error!(error = %e, "read persisted cursor failed; live sync disabled");
|
||
return;
|
||
}
|
||
};
|
||
info!(?cursor, "event sync started");
|
||
|
||
loop {
|
||
tokio::time::sleep(POLL_INTERVAL).await;
|
||
let events = match client.enumerate_events(&scope, cursor.as_ref()).await {
|
||
Ok(events) => events,
|
||
Err(e) => {
|
||
warn!(error = %e, "event poll failed; retrying after interval");
|
||
continue;
|
||
}
|
||
};
|
||
if events.is_empty() {
|
||
continue;
|
||
}
|
||
debug!(count = events.len(), "applying remote events");
|
||
for event in &events {
|
||
// Converge the SDK's own caches (folder keys, entity cache) on the
|
||
// server before applying the event to our tree. Without this, a node
|
||
// re-keyed/moved by another client keeps a stale key in the SDK for
|
||
// the life of the daemon (SDK plan #9). `apply_event` only touches
|
||
// our FUSE state, so nothing else does this.
|
||
if let Err(e) = client.invalidate_caches_for_event(event).await {
|
||
warn!(error = %e, "sdk cache invalidation for event failed");
|
||
}
|
||
apply_event(&state, &content, &pending, ¬ifier, event);
|
||
}
|
||
cursor = events.last().map(|e| e.id().clone());
|
||
if let Some(c) = &cursor
|
||
&& let Err(e) = db.set_event_cursor(c.as_str())
|
||
{
|
||
warn!(error = %e, "persist event cursor failed");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Keep the local-file index fresh for the launcher prompt's "This computer"
|
||
/// results. Rebuilds the index whenever it is older than [`LOCAL_INDEX_TTL`],
|
||
/// then sleeps; runs on its own thread for the life of the daemon.
|
||
///
|
||
/// The walk is the one part of the daemon that touches the wider filesystem, so
|
||
/// it is deliberately kept off every hot path: it never runs on a FUSE or
|
||
/// control-socket thread, and it excludes the mountpoint (walking it would fault
|
||
/// every remote node in through FUSE, defeating on-demand hydration).
|
||
fn run_local_index(
|
||
db: Arc<Db>,
|
||
indexing: Arc<AtomicBool>,
|
||
transfers: Arc<TransferRegistry>,
|
||
mountpoint: PathBuf,
|
||
) {
|
||
loop {
|
||
let age = db.local_indexed_at().ok().flatten();
|
||
let stale =
|
||
age.is_none_or(|at| now_secs().saturating_sub(at) >= LOCAL_INDEX_TTL.as_secs() as i64);
|
||
if stale {
|
||
scan_local_once(&db, &indexing, &transfers, &mountpoint);
|
||
}
|
||
std::thread::sleep(LOCAL_INDEX_CHECK);
|
||
}
|
||
}
|
||
|
||
/// Walk `$HOME` once and replace the local-file index with what it finds.
|
||
/// Batches stream straight into SQLite, so peak memory is one batch — not the
|
||
/// whole home directory.
|
||
///
|
||
/// Reports itself as a job: the first scan after a fresh install walks the whole
|
||
/// home directory, and `indexing` alone only tells the launcher prompt to say
|
||
/// "still indexing" — nothing else showed that the daemon was busy.
|
||
fn scan_local_once(
|
||
db: &Db,
|
||
indexing: &AtomicBool,
|
||
transfers: &Arc<TransferRegistry>,
|
||
mountpoint: &Path,
|
||
) {
|
||
let dirs = match AppDirs::new() {
|
||
Ok(d) => d,
|
||
Err(e) => {
|
||
warn!(error = %e, "local index: cannot resolve app dirs");
|
||
return;
|
||
}
|
||
};
|
||
let Some(home) = dirs.home_dir() else {
|
||
warn!("local index: cannot resolve home directory");
|
||
return;
|
||
};
|
||
let generation = match db.local_begin_scan() {
|
||
Ok(g) => g,
|
||
Err(e) => {
|
||
warn!(error = %e, "local index: cannot open scan generation");
|
||
return;
|
||
}
|
||
};
|
||
|
||
let excludes = localindex::default_excludes(mountpoint, &dirs.state_dir(), &dirs.cache_dir());
|
||
indexing.store(true, Ordering::Relaxed);
|
||
let started = Instant::now();
|
||
|
||
// The walk has no idea how many files it will find, so the job counts what it
|
||
// has seen and stays indeterminate.
|
||
let job = transfers.begin_job("Indexing this computer");
|
||
job.detail("Scanning your files");
|
||
let walked = localindex::scan(&[home], &excludes, |batch| {
|
||
if let Err(e) = db.local_upsert_batch(generation, &batch) {
|
||
warn!(error = %e, "local index: batch write failed");
|
||
}
|
||
});
|
||
|
||
// Prune what this scan did not see and rebuild the FTS index over the rest,
|
||
// even if some batches failed — a partial index still beats none.
|
||
job.detail("Building the search index");
|
||
match db.local_finish_scan(generation, now_secs()) {
|
||
Ok(indexed) => info!(
|
||
walked,
|
||
indexed,
|
||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||
"local index rebuilt"
|
||
),
|
||
Err(e) => warn!(error = %e, "local index: finish failed"),
|
||
}
|
||
indexing.store(false, Ordering::Relaxed);
|
||
}
|
||
|
||
/// Why a [`mount`] call returned. Lets the daemon decide whether to exit (clean
|
||
/// shutdown) or remount (the mount went away under it).
|
||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
pub enum MountOutcome {
|
||
/// The daemon was asked to stop (SIGTERM/SIGINT) and we lazily unmounted
|
||
/// ourselves. The caller should exit cleanly.
|
||
Shutdown,
|
||
/// The kernel mount ended on its own (e.g. an external `fusermount -u`).
|
||
/// The caller may want to remount.
|
||
Unmounted,
|
||
}
|
||
|
||
/// Whether `path` is a mountpoint whose FUSE connection is dead — the state a
|
||
/// daemon killed before it could unmount leaves behind.
|
||
///
|
||
/// The kernel answers every operation on such a path with `ENOTCONN`, which the
|
||
/// ordinary existence checks (`is_dir`, `exists`) report as plain `false`: the
|
||
/// path looks *absent* rather than broken. Callers that would otherwise treat
|
||
/// that as "nothing to mount here" use this to tell the two apart.
|
||
pub(crate) fn is_stale_mount(path: &Path) -> bool {
|
||
matches!(
|
||
std::fs::metadata(path).map_err(|e| e.raw_os_error()),
|
||
Err(Some(libc::ENOTCONN))
|
||
)
|
||
}
|
||
|
||
/// A secondary (on-demand sync folder) FUSE session, paired with its FUSE
|
||
/// connection id so teardown can abort a mid-transfer connection rather than
|
||
/// block on it. See [`Core::mounts`] and [`umount_session_unblocked`].
|
||
type SecondaryMount = (BackgroundSession, Option<u32>);
|
||
|
||
/// The kernel's id for the FUSE connection backing the mount at `mountpoint` —
|
||
/// the directory name under `/sys/fs/fuse/connections`, which is the minor
|
||
/// number of the mountpoint's device. Must be read *while still mounted*; after
|
||
/// unmount the path resolves to the underlying directory on another device.
|
||
/// `None` when the path can't be stat'd.
|
||
fn fuse_connection_id(mountpoint: &Path) -> Option<u32> {
|
||
use std::os::unix::fs::MetadataExt;
|
||
let dev = std::fs::metadata(mountpoint).ok()?.dev();
|
||
Some(libc::minor(dev))
|
||
}
|
||
|
||
/// Force the kernel to abort FUSE connection `id`, erroring every in-flight
|
||
/// request.
|
||
///
|
||
/// On a stop signal we unmount lazily (`MNT_DETACH`) so the call succeeds even
|
||
/// mid-transfer — but detach only removes the mountpoint; it does *not* end the
|
||
/// connection while a request is still in flight. fuser's session loop then
|
||
/// blocks on `/dev/fuse` waiting for a next request that never comes, so `join`
|
||
/// hangs — long enough during a transfer that systemd's stop timeout SIGKILLs
|
||
/// the daemon mid-unmount, stranding the on-demand mounts as dead endpoints.
|
||
/// Writing the connection's `abort` file makes the pending reads fail with
|
||
/// `ENODEV`, so the loop returns and `join` completes at once. Best-effort:
|
||
/// there is nothing more to do on the shutdown path if it fails.
|
||
fn abort_fuse_connection(id: u32) {
|
||
let path = format!("/sys/fs/fuse/connections/{id}/abort");
|
||
if std::fs::write(&path, b"1").is_ok() {
|
||
info!(id, "aborted FUSE connection to unblock unmount");
|
||
}
|
||
}
|
||
|
||
/// Unmount a background session that may be mid-transfer without wedging: abort
|
||
/// its connection first (so the session loop exits promptly), then lazily
|
||
/// unmount and join. `conn` is the id captured at mount time; `None` skips the
|
||
/// abort and just unmounts (a healthy idle mount joins on its own `Destroy`).
|
||
fn umount_session_unblocked(session: BackgroundSession, conn: Option<u32>) -> std::io::Result<()> {
|
||
if let Some(id) = conn {
|
||
abort_fuse_connection(id);
|
||
}
|
||
session.umount_and_join()
|
||
}
|
||
|
||
/// Best-effort teardown of a stale mount left behind by a crashed daemon. A
|
||
/// previous run that died without unmounting leaves the kernel mount in place,
|
||
/// so the fresh `Session::new` below would fail with EBUSY ("Device or resource
|
||
/// busy"). `fusermount3 -u -z` is the lazy (detach) unmount, which succeeds even
|
||
/// when the old mount is still busy. Swallow all output/errors: if there is no
|
||
/// stale mount this is simply a no-op.
|
||
fn clear_stale_mount(mountpoint: &Path) {
|
||
for bin in ["fusermount3", "fusermount"] {
|
||
let ok = std::process::Command::new(bin)
|
||
.arg("-u")
|
||
.arg("-z")
|
||
.arg(mountpoint)
|
||
.stdout(std::process::Stdio::null())
|
||
.stderr(std::process::Stdio::null())
|
||
.status()
|
||
.map(|s| s.success())
|
||
.unwrap_or(false);
|
||
if ok {
|
||
info!(mountpoint = %mountpoint.display(), "cleared stale mount before remount");
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
/// `sync_state` key holding the uid of the My Files root, so a later run can
|
||
/// recover it from `nodes` without the API (offline.md Phase 1).
|
||
const ROOT_UID_KEY: &str = "root_uid";
|
||
|
||
/// The My Files root, and whether we got it from the API (`true`) or from the
|
||
/// cache because the API was unreachable (`false`).
|
||
///
|
||
/// A successful fetch also records the root's uid, which is what makes the
|
||
/// fallback possible on a later run: `nodes` is keyed by uid, so without this we
|
||
/// would have the root's row on disk and no way to tell which one it is.
|
||
///
|
||
/// Failing to fetch is only fatal on a genuinely cold start — no cached root
|
||
/// means an empty tree, and mounting that would show the user an empty Drive
|
||
/// rather than an honest error.
|
||
fn fetch_or_recall_root(
|
||
client: &ProtonDriveClient,
|
||
rt: &tokio::runtime::Handle,
|
||
db: &Db,
|
||
) -> std::io::Result<(Node, bool)> {
|
||
let err = match rt.block_on(client.get_my_files_folder()) {
|
||
Ok(root) => {
|
||
if let Err(e) = db.set_state_str(ROOT_UID_KEY, &root.uid.to_string()) {
|
||
warn!(error = %e, "persist root uid failed");
|
||
}
|
||
return Ok((root, true));
|
||
}
|
||
Err(e) => e,
|
||
};
|
||
let cached = db
|
||
.state_str(ROOT_UID_KEY)
|
||
.ok()
|
||
.flatten()
|
||
.and_then(|uid| db.node_by_uid(&uid).ok().flatten());
|
||
match cached {
|
||
Some(root) => {
|
||
warn!(error = %err, "fetch My Files root failed; mounting from cache (offline)");
|
||
Ok((root, false))
|
||
}
|
||
None => Err(std::io::Error::other(format!("fetch My Files root: {err}"))),
|
||
}
|
||
}
|
||
|
||
/// Mount the filesystem at `mountpoint` and block until it is unmounted or the
|
||
/// daemon is asked to stop.
|
||
///
|
||
/// Resolves the My Files root up front — from the API, or from the cached tree
|
||
/// when the API is unreachable ([`fetch_or_recall_root`]) — then spawns the
|
||
/// Phase 2 event-sync task, the
|
||
/// Phase 4 control socket, and runs the FUSE session on its own thread while
|
||
/// this thread waits for either a stop signal (SIGTERM/SIGINT) or the kernel
|
||
/// mount ending on its own. On a stop signal we lazily unmount ourselves
|
||
/// (`umount_and_join`, the MNT_DETACH path that succeeds even while downloads
|
||
/// are in flight), so `systemctl --user stop` is always a clean teardown.
|
||
/// `rt` must be a handle to a *running* multi-threaded runtime.
|
||
pub fn mount(
|
||
client: ProtonDriveClient,
|
||
rt: tokio::runtime::Handle,
|
||
mountpoint: &Path,
|
||
cache: ContentCache,
|
||
control_socket: &Path,
|
||
db: Arc<Db>,
|
||
username: String,
|
||
) -> std::io::Result<MountOutcome> {
|
||
let (root, online) = fetch_or_recall_root(&client, &rt, &db)?;
|
||
let scope = root.tree_event_scope_id();
|
||
|
||
// The folder-sync engine (devices.md Phase 2) runs on its own thread and is
|
||
// nudged over this channel; the sender lives in Core so control-socket
|
||
// handlers can trigger reconciles.
|
||
let (sync_tx, sync_rx) = std::sync::mpsc::channel::<sync::SyncMsg>();
|
||
|
||
let core = Core {
|
||
client: client.clone(),
|
||
rt: rt.clone(),
|
||
state: Arc::new(Mutex::new(State {
|
||
entries: HashMap::new(),
|
||
by_uid: HashMap::new(),
|
||
children: HashMap::new(),
|
||
next_ino: 2,
|
||
handles: HashMap::new(),
|
||
next_fh: 1,
|
||
db: db.clone(),
|
||
})),
|
||
cache: Arc::new(cache),
|
||
readers: Arc::new(Mutex::new(HashMap::new())),
|
||
stream_ring: Arc::new(Mutex::new(StreamRing::default())),
|
||
workers: Arc::new(Workers::new(FUSE_WORKERS)?),
|
||
db,
|
||
online: Arc::new(AtomicBool::new(online)),
|
||
pending: Arc::new(Mutex::new(HashMap::new())),
|
||
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
||
timeline_refreshing: Arc::new(AtomicBool::new(false)),
|
||
trash_refreshing: Arc::new(AtomicBool::new(false)),
|
||
thumb_gen: Arc::new(Mutex::new(HashSet::new())),
|
||
no_thumbnail: Arc::new(Mutex::new(HashMap::new())),
|
||
size_upgrades: Arc::new(Mutex::new(HashMap::new())),
|
||
notifier: Arc::new(OnceLock::new()),
|
||
transfers: TransferRegistry::new(),
|
||
indexing: Arc::new(AtomicBool::new(false)),
|
||
sync_progress: Arc::new(Mutex::new(HashMap::new())),
|
||
sync_tx,
|
||
mounts: Arc::new(Mutex::new(HashMap::new())),
|
||
sync_locks: Arc::new(Mutex::new(HashMap::new())),
|
||
};
|
||
|
||
// Writes queued by a previous run (or left behind by a crash) are still owed
|
||
// an upload, and reads must be served from their staged blobs until they land.
|
||
core.hydrate_pending();
|
||
// Then the writes that were fsynced but never closed, which the cache moved
|
||
// aside at open. After `hydrate_pending`, so a recovered partial write can
|
||
// see an already-queued write to the same node.
|
||
core.recover_fsynced_writes();
|
||
{
|
||
let core = core.clone();
|
||
std::thread::Builder::new()
|
||
.name("pdfs-drain".into())
|
||
.spawn(move || core.run_pending_drain())?;
|
||
}
|
||
|
||
// Start the folder-sync engine. It watches every mirror folder, polls the
|
||
// remotes, and reconciles on its own thread — never in front of a FUSE call.
|
||
sync::spawn(core.clone(), sync_rx);
|
||
|
||
// Mounted from the cache: watch for the network coming back so the mount can
|
||
// stop being read-only-ish without the user restarting the daemon.
|
||
if !online {
|
||
let core = core.clone();
|
||
std::thread::Builder::new()
|
||
.name("pdfs-online-probe".into())
|
||
.spawn(move || core.run_online_probe())?;
|
||
}
|
||
|
||
// Re-establish on-demand mounts left over from a previous run (devices.md
|
||
// Phase 4). On its own thread: each remount fetches a remote node, and we
|
||
// must not block the main mount below on the network.
|
||
{
|
||
let core = core.clone();
|
||
std::thread::Builder::new()
|
||
.name("pdfs-restore-ondemand".into())
|
||
.spawn(move || core.restore_ondemand_mounts())?;
|
||
}
|
||
|
||
// Keep the launcher prompt's "This computer" index fresh. Its own thread:
|
||
// the walk is I/O-heavy and must never sit in front of a FUSE callback.
|
||
{
|
||
let db = core.db.clone();
|
||
let indexing = core.indexing.clone();
|
||
let transfers = core.transfers.clone();
|
||
let mountpoint = mountpoint.to_path_buf();
|
||
std::thread::Builder::new()
|
||
.name("pdfs-localindex".into())
|
||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint))?;
|
||
}
|
||
|
||
let mut config = Config::default();
|
||
config.mount_options = vec![
|
||
MountOption::FSName("protondrive".to_string()),
|
||
MountOption::Subtype("protondrive".to_string()),
|
||
MountOption::DefaultPermissions,
|
||
];
|
||
|
||
// A crashed previous run can leave the kernel mount in place, which makes
|
||
// the fresh mount below fail with EBUSY. Lazily detach any leftover first.
|
||
clear_stale_mount(mountpoint);
|
||
info!(mountpoint = %mountpoint.display(), "mounting Proton Drive");
|
||
|
||
// Bind the control socket before the FUSE session takes over the thread. A
|
||
// stale socket file from a previous run would block the bind, so clear it.
|
||
let _ = std::fs::remove_file(control_socket);
|
||
let listener = UnixListener::bind(control_socket)?;
|
||
// Owner-only before anything can connect: a peer on this socket commands the
|
||
// daemon's authenticated session without a credential of its own (B6).
|
||
if let Err(e) = pdfs_core::config::restrict_socket(control_socket) {
|
||
error!(error = %e, "cannot restrict control socket permissions; refusing to serve");
|
||
let _ = std::fs::remove_file(control_socket);
|
||
return Err(std::io::Error::other(format!(
|
||
"control socket permissions: {e}"
|
||
)));
|
||
}
|
||
{
|
||
let core = core.clone();
|
||
let username = username.clone();
|
||
let mountpoint = mountpoint.to_path_buf();
|
||
std::thread::Builder::new()
|
||
.name("pdfs-control".into())
|
||
.spawn(move || run_control_socket(core, username, mountpoint, listener))?;
|
||
}
|
||
|
||
let fs = ProtonFs::new(core.clone(), root);
|
||
// Warm the in-memory maps from the DB so a cold start serves previously
|
||
// seen metadata without re-hitting the API (plan.md P1).
|
||
core.hydrate();
|
||
|
||
// Build the session explicitly (not `mount2`) so we can grab a `Notifier`
|
||
// for the event task. `spawn` runs the session loop on its own background
|
||
// thread; we then wait here for either a stop signal or the mount ending.
|
||
let bg = Session::new(fs, mountpoint, &config)?.spawn()?;
|
||
// The connection id, captured now while the mount is live, so a stop signal
|
||
// mid-transfer can abort it rather than block `join` (see `abort_fuse_connection`).
|
||
let main_conn = fuse_connection_id(mountpoint);
|
||
let notifier = bg.notifier();
|
||
// Same channel, kept on the `Core` so background work (a size upgrade, say)
|
||
// can invalidate kernel-cached metadata without threading a handle through.
|
||
let _ = core.notifier.set(notifier.clone());
|
||
rt.spawn(run_event_sync(
|
||
client,
|
||
scope,
|
||
core.state,
|
||
core.cache,
|
||
core.db,
|
||
core.pending,
|
||
notifier,
|
||
));
|
||
|
||
// Stop signals (SIGTERM from `systemctl --user stop`, SIGINT from Ctrl-C)
|
||
// are delivered onto the async runtime; bridge them onto a sync channel so
|
||
// the loop below can react without blocking a worker thread. A bounded
|
||
// channel of 1 is enough — we only need to know that *a* stop arrived.
|
||
let (sig_tx, sig_rx) = std::sync::mpsc::sync_channel::<()>(1);
|
||
rt.spawn(async move {
|
||
let mut sigterm =
|
||
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
|
||
Ok(s) => s,
|
||
Err(e) => {
|
||
warn!(error = %e, "install SIGTERM handler failed");
|
||
return;
|
||
}
|
||
};
|
||
let mut sigint =
|
||
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) {
|
||
Ok(s) => s,
|
||
Err(e) => {
|
||
warn!(error = %e, "install SIGINT handler failed");
|
||
return;
|
||
}
|
||
};
|
||
tokio::select! {
|
||
_ = sigterm.recv() => info!("received SIGTERM"),
|
||
_ = sigint.recv() => info!("received SIGINT"),
|
||
}
|
||
let _ = sig_tx.try_send(());
|
||
});
|
||
|
||
// Wait for whichever happens first: a stop signal (→ we unmount ourselves
|
||
// via the lazy MNT_DETACH path, clean even mid-download), or the kernel
|
||
// mount ending on its own (→ the session thread finishes). Poll instead of
|
||
// blocking on `join` so we can also notice the signal.
|
||
let outcome = loop {
|
||
match sig_rx.recv_timeout(Duration::from_millis(500)) {
|
||
Ok(()) => {
|
||
info!("stop requested; unmounting");
|
||
if let Err(e) = umount_session_unblocked(bg, main_conn) {
|
||
warn!(error = %e, "umount_and_join failed");
|
||
}
|
||
break MountOutcome::Shutdown;
|
||
}
|
||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
|
||
if bg.guard.is_finished() {
|
||
info!("mount ended externally");
|
||
if let Err(e) = bg.join() {
|
||
warn!(error = %e, "session join failed");
|
||
}
|
||
break MountOutcome::Unmounted;
|
||
}
|
||
}
|
||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
|
||
// Signal task gone (failed to install); fall back to join.
|
||
let _ = bg.join();
|
||
break MountOutcome::Unmounted;
|
||
}
|
||
}
|
||
};
|
||
|
||
// Unmount every on-demand sync folder too, or the kernel mounts linger as
|
||
// stale and the next start fails with EBUSY (devices.md Phase 3).
|
||
let secondaries: Vec<_> = core.mounts.lock().drain().collect();
|
||
for (id, (session, conn)) in secondaries {
|
||
if let Err(e) = umount_session_unblocked(session, conn) {
|
||
warn!(id, error = %e, "unmount on-demand folder failed");
|
||
}
|
||
}
|
||
|
||
let _ = std::fs::remove_file(control_socket);
|
||
Ok(outcome)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod size_upgrade_tests {
|
||
use super::SizeUpgrade;
|
||
use std::sync::Arc;
|
||
use std::sync::atomic::{AtomicBool, Ordering};
|
||
use std::time::{Duration, Instant};
|
||
|
||
/// The point of the per-chunk design: a waiter returns as soon as *its own*
|
||
/// file is resolved, without waiting for the rest of the folder. A single
|
||
/// batch over 793 nodes took ~80 s, which outran the timeout and handed back
|
||
/// the provisional size this is all meant to prevent.
|
||
#[test]
|
||
fn a_waiter_returns_on_the_chunk_that_resolves_it() {
|
||
let slot = Arc::new(SizeUpgrade::default());
|
||
let mine = Arc::new(AtomicBool::new(false));
|
||
let worker = slot.clone();
|
||
let flag = mine.clone();
|
||
let t = std::thread::spawn(move || {
|
||
// Our node lands in the first chunk; two more follow.
|
||
std::thread::sleep(Duration::from_millis(30));
|
||
flag.store(true, Ordering::SeqCst);
|
||
worker.chunk_done();
|
||
std::thread::sleep(Duration::from_millis(200));
|
||
worker.chunk_done();
|
||
worker.finish();
|
||
});
|
||
let started = Instant::now();
|
||
slot.wait_for(|| mine.load(Ordering::SeqCst));
|
||
// Released by the first chunk, not the last.
|
||
assert!(started.elapsed() < Duration::from_millis(150));
|
||
t.join().unwrap();
|
||
}
|
||
|
||
/// A chunk that does not resolve this waiter must not wake it for good.
|
||
#[test]
|
||
fn an_unrelated_chunk_does_not_release_a_waiter() {
|
||
let slot = Arc::new(SizeUpgrade::default());
|
||
let worker = slot.clone();
|
||
let t = std::thread::spawn(move || {
|
||
std::thread::sleep(Duration::from_millis(30));
|
||
worker.chunk_done();
|
||
std::thread::sleep(Duration::from_millis(60));
|
||
worker.finish();
|
||
});
|
||
let started = Instant::now();
|
||
// Never resolved: only `finish` can end this wait.
|
||
slot.wait_for(|| false);
|
||
assert!(started.elapsed() >= Duration::from_millis(80));
|
||
t.join().unwrap();
|
||
}
|
||
|
||
/// A batch that ends without resolving the node — a failed fetch — still
|
||
/// releases its waiters, who fall back to the provisional size.
|
||
#[test]
|
||
fn finish_releases_a_waiter_that_was_never_resolved() {
|
||
let slot = Arc::new(SizeUpgrade::default());
|
||
let worker = slot.clone();
|
||
let t = std::thread::spawn(move || {
|
||
std::thread::sleep(Duration::from_millis(30));
|
||
worker.finish();
|
||
});
|
||
let started = Instant::now();
|
||
slot.wait_for(|| false);
|
||
assert!(started.elapsed() < SizeUpgrade::WAIT);
|
||
t.join().unwrap();
|
||
}
|
||
|
||
/// Already resolved before waiting: must not block at all. This is the
|
||
/// follower that arrives after the chunk it needed has landed.
|
||
#[test]
|
||
fn an_already_resolved_waiter_does_not_block() {
|
||
let slot = SizeUpgrade::default();
|
||
let started = Instant::now();
|
||
slot.wait_for(|| true);
|
||
assert!(started.elapsed() < Duration::from_millis(100));
|
||
}
|
||
|
||
/// Finishing before anyone waits must not strand the late arrival — the
|
||
/// flag is what is checked, not the notification, which it would miss.
|
||
#[test]
|
||
fn waiting_after_finish_returns_at_once() {
|
||
let slot = SizeUpgrade::default();
|
||
slot.finish();
|
||
let started = Instant::now();
|
||
slot.wait_for(|| false);
|
||
assert!(started.elapsed() < Duration::from_millis(100));
|
||
}
|
||
|
||
/// Every waiter is released, not just the first: one `ls -l` puts one
|
||
/// waiter per file on the same folder.
|
||
#[test]
|
||
fn all_waiters_are_released() {
|
||
let slot = Arc::new(SizeUpgrade::default());
|
||
let waiters: Vec<_> = (0..8)
|
||
.map(|_| {
|
||
let slot = slot.clone();
|
||
std::thread::spawn(move || slot.wait_for(|| false))
|
||
})
|
||
.collect();
|
||
std::thread::sleep(Duration::from_millis(20));
|
||
slot.finish();
|
||
for w in waiters {
|
||
w.join().expect("every waiter returns");
|
||
}
|
||
}
|
||
|
||
/// The timeout is the backstop: a batch that never finishes must not wedge
|
||
/// the caller's `stat` forever.
|
||
#[test]
|
||
fn a_waiter_gives_up_at_the_timeout() {
|
||
let slot = SizeUpgrade::default();
|
||
let started = Instant::now();
|
||
// Nothing will ever resolve or finish this.
|
||
slot.wait_for(|| false);
|
||
assert!(started.elapsed() >= SizeUpgrade::WAIT);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod replace_tests {
|
||
use super::check_replaceable;
|
||
|
||
/// The case that motivated all of this: rsync renaming its temp file over
|
||
/// the real one. Two plain files, and it has to be allowed — refusing is
|
||
/// what made every rsync transfer fail at the last step (bugs.md B13).
|
||
#[test]
|
||
fn a_file_may_replace_a_file() {
|
||
assert!(check_replaceable(false, false, true).is_ok());
|
||
}
|
||
|
||
#[test]
|
||
fn an_empty_directory_may_be_replaced_by_a_directory() {
|
||
assert!(check_replaceable(true, true, true).is_ok());
|
||
}
|
||
|
||
/// Proton trashes a folder with its whole subtree, so allowing this would
|
||
/// discard every file under the destination without ever naming them.
|
||
#[test]
|
||
fn a_non_empty_directory_is_never_replaced() {
|
||
let e = check_replaceable(true, true, false).expect_err("must refuse");
|
||
assert_eq!(e.code(), libc::ENOTEMPTY);
|
||
}
|
||
|
||
#[test]
|
||
fn the_two_ends_must_agree_on_being_a_directory() {
|
||
assert_eq!(
|
||
check_replaceable(false, true, true)
|
||
.expect_err("a file may not replace a directory")
|
||
.code(),
|
||
libc::EISDIR
|
||
);
|
||
assert_eq!(
|
||
check_replaceable(true, false, true)
|
||
.expect_err("a directory may not replace a file")
|
||
.code(),
|
||
libc::ENOTDIR
|
||
);
|
||
}
|
||
|
||
/// `dst_empty` describes a directory; it must not leak into the file case,
|
||
/// where callers pass `true` by convention.
|
||
#[test]
|
||
fn emptiness_is_ignored_when_the_destination_is_a_file() {
|
||
assert!(check_replaceable(false, false, false).is_ok());
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod thumb_tests {
|
||
use super::{THUMB_EDGE, ratio_of, scale_thumbnail};
|
||
|
||
/// A `width`×`height` JPEG, standing in for a camera photo the server never
|
||
/// generated a thumbnail for.
|
||
fn jpeg(width: u32, height: u32) -> Vec<u8> {
|
||
let image = image::RgbImage::from_fn(width, height, |x, y| {
|
||
image::Rgb([(x % 256) as u8, (y % 256) as u8, 128])
|
||
});
|
||
let mut bytes = Vec::new();
|
||
image::codecs::jpeg::JpegEncoder::new(&mut bytes)
|
||
.encode_image(&image)
|
||
.unwrap();
|
||
bytes
|
||
}
|
||
|
||
#[test]
|
||
fn scaling_fits_the_long_edge_and_keeps_the_aspect_ratio() {
|
||
let photo = jpeg(4000, 3000);
|
||
let thumb = scale_thumbnail(&photo).expect("a JPEG scales");
|
||
|
||
let (width, height) = image::ImageReader::new(std::io::Cursor::new(&thumb.bytes))
|
||
.with_guessed_format()
|
||
.unwrap()
|
||
.into_dimensions()
|
||
.unwrap();
|
||
assert_eq!(width, THUMB_EDGE, "the long edge lands on the target");
|
||
assert_eq!(height, THUMB_EDGE * 3 / 4, "and nothing is cropped");
|
||
assert!((thumb.ratio - 4.0 / 3.0).abs() < 1e-6);
|
||
assert!(
|
||
thumb.bytes.len() < photo.len(),
|
||
"a thumbnail that isn't smaller than its photo is no thumbnail"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn a_portrait_photo_fits_its_long_edge_too() {
|
||
let thumb = scale_thumbnail(&jpeg(1000, 2000)).expect("a JPEG scales");
|
||
assert!(thumb.ratio < 1.0, "portrait stays portrait");
|
||
assert_eq!(ratio_of(&thumb.bytes).map(|r| r < 1.0), Some(true));
|
||
}
|
||
|
||
#[test]
|
||
fn undecodable_bytes_are_not_a_thumbnail() {
|
||
// What a photo in a format this build has no decoder for looks like: the
|
||
// caller writes it off as un-thumbnailable rather than retrying forever.
|
||
assert!(scale_thumbnail(b"not an image at all").is_none());
|
||
assert!(ratio_of(b"not an image at all").is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn ratio_is_read_from_the_header_alone() {
|
||
assert_eq!(ratio_of(&jpeg(300, 200)), Some(1.5));
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod local_uid_tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn a_minted_uid_is_recognisable_and_round_trips() {
|
||
let uid = mint_local_uid();
|
||
assert!(is_local_uid(&uid));
|
||
assert!(is_local_uid_str(&uid.to_string()));
|
||
|
||
// It has to survive the trip through `pending_op.uid` as text, like any
|
||
// other uid does.
|
||
let parsed = parse_node_uid(&uid.to_string()).expect("parses back");
|
||
assert_eq!(parsed, uid);
|
||
}
|
||
|
||
#[test]
|
||
fn minted_uids_are_distinct_within_a_run() {
|
||
// Two files created in the same millisecond must not collide — the whole
|
||
// queue is keyed by uid.
|
||
let a = mint_local_uid();
|
||
let b = mint_local_uid();
|
||
assert_ne!(a, b);
|
||
}
|
||
|
||
#[test]
|
||
fn a_real_uid_is_never_mistaken_for_a_placeholder() {
|
||
let real = NodeUid::new(VolumeId::from("vol1"), LinkId::from("link1"));
|
||
assert!(!is_local_uid(&real));
|
||
assert!(!is_local_uid_str("vol1~link1"));
|
||
// Not a uid at all.
|
||
assert!(!is_local_uid_str("local"));
|
||
// The sentinel is the *volume*; a link that merely says "local" is real.
|
||
assert!(!is_local_uid_str("vol1~local"));
|
||
}
|
||
|
||
#[test]
|
||
fn a_placeholder_file_reports_itself_as_empty_and_unsealed() {
|
||
let parent = NodeUid::new(VolumeId::from("vol1"), LinkId::from("dir"));
|
||
let node = local_node(mint_local_uid(), parent.clone(), "notes.txt".into(), false);
|
||
|
||
assert_eq!(node.name, "notes.txt");
|
||
assert_eq!(node.parent_uid, Some(parent));
|
||
assert!(!node.trashed);
|
||
match node.kind {
|
||
NodeKind::File {
|
||
claimed_size,
|
||
active_revision_state,
|
||
ref media_type,
|
||
..
|
||
} => {
|
||
assert_eq!(claimed_size, Some(0));
|
||
// Nothing has been uploaded, so there is no sealed revision.
|
||
assert!(active_revision_state.is_none());
|
||
assert_eq!(media_type, "text/plain");
|
||
}
|
||
NodeKind::Folder => panic!("expected a file"),
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn a_placeholder_folder_is_a_folder() {
|
||
let parent = NodeUid::new(VolumeId::from("vol1"), LinkId::from("root"));
|
||
let node = local_node(mint_local_uid(), parent, "photos".into(), true);
|
||
assert!(node.is_folder());
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod pending_size_tests {
|
||
use super::{Node, NodeKind, NodeUid, apply_pending_sizes, node_size};
|
||
use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId};
|
||
use std::collections::HashMap;
|
||
|
||
fn uid(link: &str) -> NodeUid {
|
||
NodeUid::new(VolumeId::from("vol"), LinkId::from(link))
|
||
}
|
||
|
||
fn file(link: &str, claimed: i64) -> Node {
|
||
Node {
|
||
uid: uid(link),
|
||
parent_uid: Some(uid("parent")),
|
||
kind: NodeKind::File {
|
||
media_type: "text/plain".into(),
|
||
total_size_on_storage: 0,
|
||
active_revision_state: None,
|
||
claimed_size: Some(claimed),
|
||
claimed_modification_time: None,
|
||
},
|
||
name: link.into(),
|
||
creation_time: 100,
|
||
modification_time: 100,
|
||
trashed: false,
|
||
is_shared: false,
|
||
is_shared_publicly: false,
|
||
signature_email: None,
|
||
verification: Default::default(),
|
||
}
|
||
}
|
||
|
||
fn folder(link: &str) -> Node {
|
||
Node {
|
||
kind: NodeKind::Folder,
|
||
..file(link, 0)
|
||
}
|
||
}
|
||
|
||
/// B11: a re-enumeration mid-write must not revert the size to the
|
||
/// server's. A file that stats as 0 gets no `read` from the kernel at all,
|
||
/// so the staged blob is never served and the file reads as empty.
|
||
#[test]
|
||
fn a_queued_write_keeps_its_optimistic_size_through_a_re_enumeration() {
|
||
let mut nodes = vec![file("queued", 0), file("settled", 4096)];
|
||
let sizes = HashMap::from([(uid("queued"), 3)]);
|
||
|
||
apply_pending_sizes(&mut nodes, &sizes);
|
||
|
||
assert_eq!(
|
||
node_size(&nodes[0]),
|
||
3,
|
||
"the remote's pre-write size must not win over the queued write's"
|
||
);
|
||
assert_eq!(
|
||
node_size(&nodes[1]),
|
||
4096,
|
||
"a file with nothing queued keeps the size the server reported"
|
||
);
|
||
}
|
||
|
||
/// The map is keyed by uid and says nothing about kind; a folder that
|
||
/// somehow collides must not grow a `claimed_size`.
|
||
#[test]
|
||
fn folders_are_left_alone() {
|
||
let mut nodes = vec![folder("dir")];
|
||
apply_pending_sizes(&mut nodes, &HashMap::from([(uid("dir"), 999)]));
|
||
assert!(matches!(nodes[0].kind, NodeKind::Folder));
|
||
assert_eq!(node_size(&nodes[0]), 0);
|
||
}
|
||
|
||
/// The common case: nothing queued, nothing touched.
|
||
#[test]
|
||
fn an_empty_pending_map_changes_nothing() {
|
||
let mut nodes = vec![file("a", 10), file("b", 20)];
|
||
apply_pending_sizes(&mut nodes, &HashMap::new());
|
||
assert_eq!(node_size(&nodes[0]), 10);
|
||
assert_eq!(node_size(&nodes[1]), 20);
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::{Intervals, conflict_name, is_stale_mount};
|
||
|
||
/// The predicate must answer *only* for a dead FUSE connection. A healthy
|
||
/// directory and an absent path are both "not stale" — widening it to any
|
||
/// `metadata` error would make the on-demand restore lazily unmount paths
|
||
/// that are simply missing.
|
||
#[test]
|
||
fn is_stale_mount_is_narrow() {
|
||
let dir = std::env::temp_dir().join(format!("pdfs-stale-{}", std::process::id()));
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
assert!(!is_stale_mount(&dir));
|
||
assert!(!is_stale_mount(&dir.join("no-such-entry")));
|
||
let _ = std::fs::remove_dir_all(&dir);
|
||
}
|
||
|
||
/// Flatten `segments` into a readable form for assertions.
|
||
fn segs(iv: &Intervals, start: u64, end: u64) -> Vec<(u64, u64, bool)> {
|
||
iv.segments(start, end)
|
||
}
|
||
|
||
/// The conflict copy has to stay openable, so the extension survives — and
|
||
/// the shape matches the sync engine's `conflict_path` so the two features
|
||
/// name the same situation the same way.
|
||
#[test]
|
||
fn conflict_name_keeps_the_extension() {
|
||
assert_eq!(
|
||
conflict_name("notes.txt", 1700),
|
||
"notes (sync-conflict 1700).txt"
|
||
);
|
||
assert_eq!(conflict_name("README", 42), "README (sync-conflict 42)");
|
||
assert_eq!(
|
||
conflict_name("archive.tar.gz", 7),
|
||
"archive.tar (sync-conflict 7).gz",
|
||
"only the last extension is one, as everywhere else"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn append_merges_into_one_run() {
|
||
// Sequential writes (an append) coalesce into a single authored range.
|
||
let mut iv = Intervals::default();
|
||
iv.add(0, 10);
|
||
iv.add(10, 20);
|
||
iv.add(20, 25);
|
||
assert_eq!(segs(&iv, 0, 25), vec![(0, 25, true)]);
|
||
}
|
||
|
||
#[test]
|
||
fn partial_overwrite_leaves_base_gap() {
|
||
// Author [0,4) and [8,12); [4,8) stays base. A read of [0,12) must stitch
|
||
// authored / base / authored in order.
|
||
let mut iv = Intervals::default();
|
||
iv.add(0, 4);
|
||
iv.add(8, 12);
|
||
assert_eq!(
|
||
segs(&iv, 0, 12),
|
||
vec![(0, 4, true), (4, 8, false), (8, 12, true)]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn overlapping_writes_coalesce() {
|
||
let mut iv = Intervals::default();
|
||
iv.add(0, 10);
|
||
iv.add(5, 15);
|
||
iv.add(14, 20);
|
||
assert_eq!(segs(&iv, 0, 20), vec![(0, 20, true)]);
|
||
}
|
||
|
||
#[test]
|
||
fn segments_clamp_to_request_window() {
|
||
let mut iv = Intervals::default();
|
||
iv.add(0, 100);
|
||
// A sub-window of one big authored range is a single authored segment.
|
||
assert_eq!(segs(&iv, 20, 50), vec![(20, 50, true)]);
|
||
// A window entirely outside any authored range is all base.
|
||
let empty = Intervals::default();
|
||
assert_eq!(segs(&empty, 0, 8), vec![(0, 8, false)]);
|
||
}
|
||
|
||
#[test]
|
||
fn truncate_shrink_drops_tail() {
|
||
// Grow-then-shrink: clip removes/truncates authored ranges past the end.
|
||
let mut iv = Intervals::default();
|
||
iv.add(0, 100);
|
||
iv.clip(40);
|
||
assert_eq!(segs(&iv, 0, 40), vec![(0, 40, true)]);
|
||
// Authored ranges wholly past the new end disappear.
|
||
let mut iv2 = Intervals::default();
|
||
iv2.add(0, 10);
|
||
iv2.add(50, 60);
|
||
iv2.clip(40);
|
||
assert_eq!(segs(&iv2, 0, 40), vec![(0, 10, true), (10, 40, false)]);
|
||
}
|
||
|
||
#[test]
|
||
fn truncate_extend_authors_zero_tail() {
|
||
// setattr grow claims the new tail as authored (defined zeros), so commit
|
||
// never pulls it from the base.
|
||
let mut iv = Intervals::default();
|
||
iv.add(0, 10); // base content authored over
|
||
let old_len = 10u64;
|
||
let new_len = 30u64;
|
||
iv.add(old_len, new_len);
|
||
assert_eq!(segs(&iv, 0, 30), vec![(0, 30, true)]);
|
||
}
|
||
}
|