mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-05 17:02:11 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
078774f6a1 | ||
|
|
326a9b4e92 | ||
|
|
78dc72a34d |
No files matched your search
Generated
+4
-4
@@ -3538,7 +3538,7 @@ checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-cli"
|
||||
version = "1.8.0"
|
||||
version = "1.8.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap 4.6.4",
|
||||
@@ -3553,7 +3553,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-core"
|
||||
version = "1.8.0"
|
||||
version = "1.8.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"directories",
|
||||
@@ -3574,7 +3574,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-fuse"
|
||||
version = "1.8.0"
|
||||
version = "1.8.2"
|
||||
dependencies = [
|
||||
"fuser",
|
||||
"futures",
|
||||
@@ -3591,7 +3591,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-gui"
|
||||
version = "1.8.0"
|
||||
version = "1.8.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-channel",
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ resolver = "2"
|
||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.8.0"
|
||||
version = "1.8.2"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||
|
||||
+893
-60
File diff suppressed because it is too large.
Load diff
@@ -9,7 +9,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
/// Current schema version. Bump on every forward migration added below.
|
||||
pub(super) const SCHEMA_VERSION: i64 = 27;
|
||||
pub(super) const SCHEMA_VERSION: i64 = 28;
|
||||
|
||||
impl Db {
|
||||
pub(super) fn migrate(&self) -> Result<()> {
|
||||
@@ -201,6 +201,23 @@ impl Db {
|
||||
tx.execute_batch(MIGRATION_V27)?;
|
||||
}
|
||||
}
|
||||
if current < 28 {
|
||||
// Same guards as V26/V27, for the same two reasons.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('sync_entry') \
|
||||
WHERE name = 'local_mtime_ns'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_entries: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_entry'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_entries && !has_column {
|
||||
tx.execute_batch(MIGRATION_V28)?;
|
||||
}
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -808,3 +825,24 @@ CREATE INDEX IF NOT EXISTS idx_pending_op_claimed
|
||||
const MIGRATION_V27: &str = "
|
||||
ALTER TABLE pending_op ADD COLUMN access_deferred_since INTEGER NOT NULL DEFAULT 0;
|
||||
";
|
||||
|
||||
/// Schema v28: sub-second precision for a mirror folder's local baseline.
|
||||
///
|
||||
/// A reconcile pass decides whether the local side of a path changed by
|
||||
/// comparing `(local_mtime, local_size)` against the baseline. Both sides of
|
||||
/// that comparison were whole seconds, so an edit that landed in the same second
|
||||
/// as the last sync and left the file the same length — a fixed-size record
|
||||
/// rewritten, an image re-exported, a database page updated in place — read as
|
||||
/// *unchanged* and was never uploaded. The remote copy silently stayed at the
|
||||
/// older content until something else about the file moved (bugs.md B25).
|
||||
///
|
||||
/// Nullable rather than a converted `local_mtime * 1_000_000_000`: the
|
||||
/// nanosecond part of an existing baseline is genuinely unknown, and inventing
|
||||
/// zero for it would make every already-synced file compare as changed and
|
||||
/// re-upload an entire mirror on first start. `NULL` means "this row predates
|
||||
/// the column", which the comparison reads as "fall back to whole seconds" —
|
||||
/// exactly the behaviour that row was written under. Each row gains a real value
|
||||
/// the next time its path is synced.
|
||||
const MIGRATION_V28: &str = "
|
||||
ALTER TABLE sync_entry ADD COLUMN local_mtime_ns INTEGER;
|
||||
";
|
||||
@@ -35,6 +35,14 @@ pub struct StoredSyncEntry {
|
||||
pub remote_uid: Option<String>,
|
||||
pub local_mtime: i64,
|
||||
pub local_size: i64,
|
||||
/// The same modification time to nanosecond precision, when it is known.
|
||||
///
|
||||
/// `local_mtime` alone cannot tell an edit that landed inside the same
|
||||
/// second as the last sync — and left the file the same length — from no
|
||||
/// edit at all, so those never uploaded (bugs.md B25). `None` on a row
|
||||
/// written before schema 28, which the comparison reads as "compare whole
|
||||
/// seconds", so upgrading does not make an entire mirror look changed.
|
||||
pub local_mtime_ns: Option<i64>,
|
||||
pub remote_rev: Option<String>,
|
||||
pub remote_hash: Option<String>,
|
||||
}
|
||||
@@ -168,7 +176,8 @@ impl Db {
|
||||
pub fn sync_entries(&self, folder_id: i64) -> Result<HashMap<String, StoredSyncEntry>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash,
|
||||
local_mtime_ns
|
||||
FROM sync_entry WHERE folder_id = ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![folder_id], |r| {
|
||||
@@ -179,6 +188,7 @@ impl Db {
|
||||
local_size: r.get(3)?,
|
||||
remote_rev: r.get(4)?,
|
||||
remote_hash: r.get(5)?,
|
||||
local_mtime_ns: r.get(6)?,
|
||||
})
|
||||
})?;
|
||||
let mut map = HashMap::new();
|
||||
@@ -216,14 +226,16 @@ impl Db {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO sync_entry
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev,
|
||||
remote_hash, local_mtime_ns)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
ON CONFLICT(folder_id, rel_path) DO UPDATE SET
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash",
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash,
|
||||
local_mtime_ns = excluded.local_mtime_ns",
|
||||
params![
|
||||
folder_id,
|
||||
e.rel_path,
|
||||
@@ -232,6 +244,7 @@ impl Db {
|
||||
e.local_size,
|
||||
e.remote_rev,
|
||||
e.remote_hash,
|
||||
e.local_mtime_ns,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
|
||||
@@ -1895,6 +1895,123 @@ fn failed_superseding_insert_keeps_the_old_pending_op() {
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||
}
|
||||
|
||||
/// B53's remaining half. The insert-failure regression above proves the *row*
|
||||
/// rolls back; this proves the two things that make the rollback usable — the
|
||||
/// old blob is not reported as superseded (so the caller does not delete the
|
||||
/// bytes the surviving row still points at), and the rollback survives a
|
||||
/// restart rather than living only in the connection that failed.
|
||||
#[test]
|
||||
fn a_rolled_back_supersede_keeps_its_blob_and_survives_reopen() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-db-b53-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("queue.sqlite");
|
||||
let op = |blob: &str| PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("durable").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some(blob.to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch(
|
||||
"CREATE TRIGGER reject_replacement BEFORE INSERT ON pending_op
|
||||
WHEN NEW.blob_path = '/staging/replacement'
|
||||
BEGIN SELECT RAISE(ABORT, 'injected insert failure'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// The failure must not hand back the old blob path. That value is the
|
||||
// caller's instruction to delete those bytes, and the row that owns
|
||||
// them is still queued.
|
||||
assert!(db.enqueue_op(&op("/staging/replacement")).is_err());
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER reject_replacement")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1, "old or new — never neither");
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||
|
||||
// And once the injected failure is gone, superseding works and *does*
|
||||
// report the blob it retired.
|
||||
let (_, superseded) = db.enqueue_op(&op("/staging/replacement")).unwrap();
|
||||
assert_eq!(superseded.as_deref(), Some("/staging/original"));
|
||||
assert_eq!(db.pending_ops().unwrap().len(), 1);
|
||||
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
/// A full disk, as SQLite reports it. `max_page_count` makes the database
|
||||
/// refuse to grow, which is the same `SQLITE_FULL` an out-of-space filesystem
|
||||
/// produces — and the queue must come out of it with the acknowledged work
|
||||
/// still in it (B53).
|
||||
#[test]
|
||||
fn a_full_database_rolls_the_supersede_back_rather_than_erasing_it() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let op = |blob: &str| PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("full").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some(blob.to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||
|
||||
// Cap the file at its current size, so any page the insert needs fails.
|
||||
let pages: i64 = db
|
||||
.with_conn(|conn| Ok(conn.query_row("PRAGMA page_count", [], |r| r.get(0))?))
|
||||
.unwrap();
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch(&format!("PRAGMA max_page_count = {pages}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// A payload far larger than one page, so growth is unavoidable.
|
||||
let mut fat = op("/staging/replacement");
|
||||
fat.meta_json = Some("x".repeat(512 * 1024));
|
||||
let result = db.enqueue_op(&fat);
|
||||
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch("PRAGMA max_page_count = 1073741823")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(result.is_err(), "a database that cannot grow must refuse");
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(
|
||||
ops[0].blob_path.as_deref(),
|
||||
Some("/staging/original"),
|
||||
"the acknowledged upload is still queued and still owns its bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_trash_replacement_returns_owned_blobs_after_commit() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -2408,6 +2525,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -2424,6 +2542,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -3545,6 +3664,78 @@ fn migration_v27_adds_the_access_deferral_column_to_a_v26_queue() {
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// A mirror baseline carried over from V27 keeps its rows, and each one reads
|
||||
/// back with no nanosecond time. That `None` is the whole point: it makes the
|
||||
/// comparison fall back to whole seconds for rows written before the column
|
||||
/// existed, instead of asserting a sub-second time of zero and re-uploading
|
||||
/// every already-synced file in the folder (bugs.md B25).
|
||||
#[test]
|
||||
fn migration_v28_adds_sub_second_local_times_without_disturbing_a_v27_baseline() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v27-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
let folder = db
|
||||
.sync_folder_add("/home/me/Mirror", "vol~root", "share")
|
||||
.unwrap();
|
||||
db.sync_entry_upsert(
|
||||
folder,
|
||||
&StoredSyncEntry {
|
||||
rel_path: "notes.txt".into(),
|
||||
remote_uid: Some("vol~notes".into()),
|
||||
local_mtime: 1_700_000_000,
|
||||
local_mtime_ns: Some(1_700_000_000_123_456_789),
|
||||
local_size: 42,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("42".into()),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V27 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE sync_entry DROP COLUMN local_mtime_ns;
|
||||
UPDATE sync_state SET value = '27' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let entries = db.sync_entries(1).unwrap();
|
||||
let entry = entries.get("notes.txt").expect("the baseline row survives");
|
||||
assert_eq!(entry.local_mtime, 1_700_000_000);
|
||||
assert_eq!(entry.local_size, 42);
|
||||
assert_eq!(
|
||||
entry.local_mtime_ns, None,
|
||||
"a carried-over row must not claim a sub-second time it never recorded"
|
||||
);
|
||||
|
||||
// And the column is live: a refreshed row keeps the precision it is given.
|
||||
db.sync_entry_upsert(
|
||||
1,
|
||||
&StoredSyncEntry {
|
||||
local_mtime_ns: Some(1_700_000_001_000_000_500),
|
||||
..entry.clone()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.sync_entries(1).unwrap()["notes.txt"].local_mtime_ns,
|
||||
Some(1_700_000_001_000_000_500)
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
// --- read-only connection pool (DB2) ---------------------------------------
|
||||
|
||||
/// A read must not queue behind whatever happens to be writing.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Remote-event invalidation and local-index background workers.
|
||||
|
||||
use super::*;
|
||||
use crate::shutdown::Shutdown;
|
||||
|
||||
/// Drop `uid` from one mount's tree and tell that mount's kernel session the
|
||||
/// entry is gone. Shared by the trash and delete paths, which differ only in
|
||||
@@ -437,6 +438,7 @@ pub(super) fn run_local_index(
|
||||
indexing: Arc<AtomicBool>,
|
||||
transfers: Arc<TransferRegistry>,
|
||||
mountpoint: PathBuf,
|
||||
shutdown: Arc<Shutdown>,
|
||||
) {
|
||||
loop {
|
||||
let age = db.local_indexed_at().ok().flatten();
|
||||
@@ -445,7 +447,11 @@ pub(super) fn run_local_index(
|
||||
if stale {
|
||||
scan_local_once(&db, &indexing, &transfers, &mountpoint);
|
||||
}
|
||||
std::thread::sleep(LOCAL_INDEX_CHECK);
|
||||
// Interruptible, so teardown joins this thread instead of leaving it to
|
||||
// keep rewriting the index for a mount that is gone (bugs.md B44).
|
||||
if !shutdown.sleep(LOCAL_INDEX_CHECK) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -104,6 +104,31 @@ fn read_request_line(reader: &mut impl BufRead) -> std::io::Result<Option<String
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
|
||||
/// Turn a CLI-supplied path into the mount that owns it and the path relative
|
||||
/// to *that* mount's root.
|
||||
///
|
||||
/// [`rel_to_mount`] is the older, primary-only reading, and it is what every
|
||||
/// mutating request still uses. It rejects anything outside the primary
|
||||
/// mountpoint, which leaves a secondary on-demand mount — a synced folder
|
||||
/// switched out of mirror mode — with no way to be named at all: `pdfs ls` and
|
||||
/// `pdfs refresh` both answered "is not under the mountpoint", so a folder
|
||||
/// serving a stale listing had no manual escape hatch (`docs/BUGS.md` B86).
|
||||
///
|
||||
/// A relative path keeps the old meaning, relative to the primary mount, since
|
||||
/// there is nothing in it to choose a mount by.
|
||||
fn route_to_mount(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Core, PathBuf)> {
|
||||
let p = Path::new(path);
|
||||
if !p.is_absolute() {
|
||||
return Ok((core.clone(), p.to_path_buf()));
|
||||
}
|
||||
if let Some(routed) = core.rooted_at(p) {
|
||||
return Ok(routed);
|
||||
}
|
||||
// Fall back to the primary reading so the error text stays the familiar one
|
||||
// when the path is simply outside everything this daemon has mounted.
|
||||
rel_to_mount(mountpoint, path).map(|rel| (core.clone(), rel))
|
||||
}
|
||||
|
||||
/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path
|
||||
/// must live under `mountpoint`; a relative path is taken as already relative to
|
||||
/// the mount root.
|
||||
@@ -193,8 +218,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Ok(CtlRequest::ListPins) => CtlResponse::Pins {
|
||||
pins: core.cache.list_pins(),
|
||||
},
|
||||
Ok(CtlRequest::ListDir { path }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => match core.list_dir(&rel) {
|
||||
Ok(CtlRequest::ListDir { path }) => match route_to_mount(core, mountpoint, &path) {
|
||||
Ok((core, rel)) => match core.list_dir(&rel) {
|
||||
Ok(entries) => CtlResponse::Entries { entries },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
@@ -202,9 +227,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Ok(CtlRequest::Refresh { scope }) => {
|
||||
let result = match &scope {
|
||||
RefreshScope::Dir { path } => {
|
||||
rel_to_mount(mountpoint, path).and_then(|rel| core.refresh_dir(&rel))
|
||||
}
|
||||
RefreshScope::Dir { path } => route_to_mount(core, mountpoint, path)
|
||||
.and_then(|(core, rel)| core.refresh_dir(&rel)),
|
||||
RefreshScope::Trash => {
|
||||
core.invalidate_trash();
|
||||
Ok(())
|
||||
@@ -1084,6 +1108,12 @@ pub(crate) fn run_control_socket(
|
||||
) {
|
||||
info!("control socket listening");
|
||||
for conn in listener.incoming() {
|
||||
// Checked on every accept, including the deliberate poke teardown sends
|
||||
// to wake this thread out of a blocking accept (bugs.md B44).
|
||||
if core.shutdown.is_stopping() {
|
||||
info!("control socket closing");
|
||||
return;
|
||||
}
|
||||
match conn {
|
||||
Ok(stream) => {
|
||||
let Some(permit) = ControlHandlerPermit::acquire() else {
|
||||
|
||||
@@ -260,6 +260,13 @@ impl Core {
|
||||
/// [`Db::claim_next_due_op`]: pdfs_core::db::Db::claim_next_due_op
|
||||
pub(crate) fn run_pending_drain(&self, primary: bool) {
|
||||
loop {
|
||||
// Between ops, never inside one: an op that has started is either
|
||||
// retired or released, so stopping here cannot leave the queue with
|
||||
// a row claimed by a worker that no longer exists (bugs.md B44).
|
||||
if self.shutdown.is_stopping() {
|
||||
debug!(primary, "drain worker stopping");
|
||||
return;
|
||||
}
|
||||
let now = now_millis();
|
||||
// One row, chosen and *claimed* by the database. Reading the whole
|
||||
// queue to pick one op made a long queue quadratic to drain, and
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Kernel-facing FUSE callback adapter.
|
||||
|
||||
use super::*;
|
||||
use crate::state::DirListing;
|
||||
|
||||
impl Filesystem for ProtonFs {
|
||||
/// Negotiate kernel-side behaviour once, before any other callback.
|
||||
@@ -162,15 +163,46 @@ impl Filesystem for ProtonFs {
|
||||
}
|
||||
}
|
||||
|
||||
/// Freeze this directory's listing for the life of the handle.
|
||||
///
|
||||
/// `readdir` is paged, and its continuation offset only means anything
|
||||
/// against a listing that does not move underneath it. Taking the snapshot
|
||||
/// here — once per `opendir`, which the kernel pairs exactly with one
|
||||
/// `releasedir` — is what makes the offsets stable, so a create or a trash
|
||||
/// during an enumeration can no longer make the caller skip or repeat an
|
||||
/// entry (bugs.md B43).
|
||||
///
|
||||
/// Enumerating a cold folder is a remote call, so it goes to a worker for
|
||||
/// the same reason `lookup` and `readdir` do.
|
||||
fn opendir(&self, _req: &Request, ino: INodeNo, _flags: OpenFlags, reply: ReplyOpen) {
|
||||
let ino = ino.0;
|
||||
if self.core.children_cached(ino) {
|
||||
self.serve_opendir(ino, reply);
|
||||
return;
|
||||
}
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Meta, move || fs.serve_opendir(ino, reply));
|
||||
}
|
||||
|
||||
fn readdir(
|
||||
&self,
|
||||
_req: &Request,
|
||||
ino: INodeNo,
|
||||
_fh: FileHandle,
|
||||
fh: FileHandle,
|
||||
offset: u64,
|
||||
reply: ReplyDirectory,
|
||||
) {
|
||||
let ino = ino.0;
|
||||
// The snapshot is the whole point: served from it, this is a pure
|
||||
// in-memory walk with no chance of a remote call, so it never needs a
|
||||
// worker. Only a handle whose `opendir` failed to snapshot (or a kernel
|
||||
// that issued `readdir` without one) falls back to the live path.
|
||||
if let Some(listing) = self.core.state().dir_snapshots.get(&fh.0).cloned() {
|
||||
reply_listing(&listing, offset, reply);
|
||||
return;
|
||||
}
|
||||
// Same split as `lookup`: only the cold, remote-enumerating path pays
|
||||
// for a worker.
|
||||
if self.core.children_cached(ino) {
|
||||
@@ -183,6 +215,18 @@ impl Filesystem for ProtonFs {
|
||||
.run(Lane::Meta, move || fs.serve_readdir(ino, offset, reply));
|
||||
}
|
||||
|
||||
fn releasedir(
|
||||
&self,
|
||||
_req: &Request,
|
||||
_ino: INodeNo,
|
||||
fh: FileHandle,
|
||||
_flags: OpenFlags,
|
||||
reply: ReplyEmpty,
|
||||
) {
|
||||
self.core.state().dir_snapshots.remove(&fh.0);
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
/// A read-only open is a map lookup and answers inline; a write open has to
|
||||
/// allocate a scratch file and, when a queued revision is the base, duplicate
|
||||
/// a blob that is the whole file — so it is served from a worker rather than
|
||||
@@ -912,6 +956,58 @@ pub struct ProtonFs {
|
||||
gid: u32,
|
||||
}
|
||||
|
||||
/// How many times a write `open` will re-read its base after a `release`
|
||||
/// published a newer queued revision underneath it (bugs.md B32).
|
||||
///
|
||||
/// One retry covers the race, which needs a close and an open of the same file
|
||||
/// to overlap on different threads. More than that is not contention, it is a
|
||||
/// file being republished continuously, and looping on it would keep a FUSE
|
||||
/// worker busy indefinitely.
|
||||
const OPEN_BASE_ATTEMPTS: u32 = 3;
|
||||
|
||||
/// One directory's entries as `readdir` reports them: `.` and `..` first, then
|
||||
/// the interned children in `children` order.
|
||||
///
|
||||
/// Split out because it is built from two places that must agree — `opendir`
|
||||
/// freezes it, and the fallback `readdir` builds it live — and a divergence
|
||||
/// between them would show up as entries appearing or vanishing depending on
|
||||
/// whether the kernel happened to send an `opendir`.
|
||||
fn build_listing(st: &State, ino: u64) -> DirListing {
|
||||
let parent = st.entries.get(&ino).map_or(ROOT_INO, |e| e.parent);
|
||||
// "." and ".." occupy offsets 0 and 1; real children follow.
|
||||
let mut listing = vec![
|
||||
(ino, true, ".".to_string()),
|
||||
(parent, true, "..".to_string()),
|
||||
];
|
||||
if let Some(kids) = st.children.get(&ino) {
|
||||
for &kid in kids {
|
||||
if let Some(e) = st.entries.get(&kid) {
|
||||
listing.push((kid, e.node.is_folder(), e.node.name.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
listing
|
||||
}
|
||||
|
||||
/// Fill a `readdir` reply from `listing`, resuming at `offset`.
|
||||
///
|
||||
/// The cookie handed back is the index of the *next* entry, so a resumed call
|
||||
/// continues exactly where the last one stopped — which is only true because
|
||||
/// the listing behind those indexes is a snapshot (bugs.md B43).
|
||||
fn reply_listing(listing: &DirListing, offset: u64, mut reply: ReplyDirectory) {
|
||||
for (i, (ino, is_dir, name)) in listing.iter().enumerate().skip(offset as usize) {
|
||||
let ft = if *is_dir {
|
||||
FileType::Directory
|
||||
} else {
|
||||
FileType::RegularFile
|
||||
};
|
||||
if reply.add(INodeNo(*ino), (i + 1) as u64, ft, name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
}
|
||||
|
||||
pub(crate) fn access_allowed(is_dir: bool, access: Access, mask: AccessFlags) -> bool {
|
||||
(!mask.contains(AccessFlags::W_OK) || access.writable())
|
||||
&& (!mask.contains(AccessFlags::X_OK) || is_dir)
|
||||
@@ -1027,40 +1123,39 @@ impl ProtonFs {
|
||||
}
|
||||
|
||||
/// The body of [`Filesystem::readdir`], on whichever thread ends up serving it.
|
||||
fn serve_readdir(&self, ino: u64, offset: u64, mut reply: ReplyDirectory) {
|
||||
/// The body of [`Filesystem::opendir`]: enumerate the folder if it is cold,
|
||||
/// then publish one frozen listing under a fresh handle.
|
||||
///
|
||||
/// A failure to enumerate is reported as a failed `opendir` rather than
|
||||
/// deferred to the first `readdir` — the caller gets the errno at the syscall
|
||||
/// that can still act on it.
|
||||
fn serve_opendir(&self, ino: u64, reply: ReplyOpen) {
|
||||
if let Err(e) = self.core.ensure_children(ino) {
|
||||
reply.error(e);
|
||||
return;
|
||||
}
|
||||
let mut st = self.core.state();
|
||||
let listing = Arc::new(build_listing(&st, ino));
|
||||
// Shares the counter with write handles, so a directory handle can never
|
||||
// collide with a file handle (both are `fh` values in one kernel space).
|
||||
let fh = st.next_fh;
|
||||
st.next_fh += 1;
|
||||
st.dir_snapshots.insert(fh, listing);
|
||||
drop(st);
|
||||
reply.opened(FileHandle(fh), FopenFlags::empty());
|
||||
}
|
||||
|
||||
/// The live, un-snapshotted `readdir`. Reached only by a handle that has no
|
||||
/// snapshot — see [`Filesystem::readdir`].
|
||||
fn serve_readdir(&self, ino: u64, offset: u64, reply: ReplyDirectory) {
|
||||
if let Err(e) = self.core.ensure_children(ino) {
|
||||
reply.error(e);
|
||||
return;
|
||||
}
|
||||
let st = self.core.state();
|
||||
let parent = st.entries.get(&ino).map_or(ROOT_INO, |e| e.parent);
|
||||
|
||||
// "." and ".." occupy offsets 0 and 1; real children follow.
|
||||
let mut listing: Vec<(u64, FileType, String)> = vec![
|
||||
(ino, FileType::Directory, ".".to_string()),
|
||||
(parent, FileType::Directory, "..".to_string()),
|
||||
];
|
||||
if let Some(kids) = st.children.get(&ino) {
|
||||
for &kid in kids {
|
||||
if let Some(e) = st.entries.get(&kid) {
|
||||
let ft = if e.node.is_folder() {
|
||||
FileType::Directory
|
||||
} else {
|
||||
FileType::RegularFile
|
||||
};
|
||||
listing.push((kid, ft, e.node.name.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
let listing = build_listing(&st, ino);
|
||||
drop(st);
|
||||
|
||||
for (i, (ino, ft, name)) in listing.into_iter().enumerate().skip(offset as usize) {
|
||||
// The stored offset is that of the *next* entry to resume at.
|
||||
if reply.add(INodeNo(ino), (i + 1) as u64, ft, &name) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
reply.ok();
|
||||
reply_listing(&listing, offset, reply);
|
||||
}
|
||||
|
||||
fn attr(&self, ino: u64, node: &Node, access: Access) -> FileAttr {
|
||||
@@ -1142,66 +1237,101 @@ impl ProtonFs {
|
||||
// any network access. An incomplete blob still has gaps referring to
|
||||
// the remote base; stacking another write on it cannot be represented
|
||||
// safely by WriteHandle, so fail the open rather than risk corruption.
|
||||
let pending_base = self.core.pending.lock().get(&uid).cloned();
|
||||
if pending_base.as_ref().is_some_and(|p| !p.meta.complete) {
|
||||
if let Some(entry) = self.core.state().entries.get_mut(&ino.0) {
|
||||
entry.open_count = entry.open_count.saturating_sub(1);
|
||||
}
|
||||
error!(%uid, "refusing write over incomplete queued revision");
|
||||
reply.error(Errno::EIO);
|
||||
return;
|
||||
}
|
||||
let (file, path) = match self.core.cache.create_scratch() {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
if let Some(entry) = self.core.state().entries.get_mut(&ino.0) {
|
||||
entry.open_count = entry.open_count.saturating_sub(1);
|
||||
//
|
||||
// Reading that blob and installing the handle cannot be done under one
|
||||
// lock — the copy is the size of the file — so the base is re-checked at
|
||||
// the moment the handle goes in, and a `release` that published a newer
|
||||
// revision in between sends this around again. Without the re-check the
|
||||
// new handle carried the *older* base, and its own release published
|
||||
// content that silently discarded the revision closed a moment earlier
|
||||
// (bugs.md B32). A second attempt is already unlikely and a third would
|
||||
// mean something is republishing continuously, so give up rather than
|
||||
// spin: the caller's `open` fails instead of its data being lost.
|
||||
//
|
||||
// `reply` is consumed by whichever arm answers, so the loop yields an
|
||||
// outcome and the answer is sent once, after it.
|
||||
let outcome = 'install: {
|
||||
for attempt in 0..OPEN_BASE_ATTEMPTS {
|
||||
let pending_base = self.core.pending.lock().get(&uid).cloned();
|
||||
if pending_base.as_ref().is_some_and(|p| !p.meta.complete) {
|
||||
error!(%uid, "refusing write over incomplete queued revision");
|
||||
break 'install Err(Errno::EIO);
|
||||
}
|
||||
error!(%uid, error = %e, "create scratch file failed");
|
||||
reply.error(Errno::EIO);
|
||||
return;
|
||||
let (file, path) = match self.core.cache.create_scratch() {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
error!(%uid, error = %e, "create scratch file failed");
|
||||
break 'install Err(Errno::EIO);
|
||||
}
|
||||
};
|
||||
let mut initial_written = Intervals::default();
|
||||
if let Some(pending) = &pending_base {
|
||||
// Reflink where the filesystem can: this blob is the whole
|
||||
// file, and a byte-for-byte copy of a multi-GB pending
|
||||
// revision is the kind of work `open` must not do
|
||||
// (`ContentCache::clone_or_copy`).
|
||||
if let Err(e) = ContentCache::clone_or_copy(&pending.path, &path) {
|
||||
let _ = std::fs::remove_file(&path);
|
||||
error!(%uid, source = %pending.path.display(), error = %e,
|
||||
"copy queued revision into write scratch failed");
|
||||
break 'install Err(Errno::EIO);
|
||||
}
|
||||
initial_written.add(0, pending.meta.len);
|
||||
}
|
||||
// Sampled *before* the state lock, never under it: no site in
|
||||
// the daemon holds `pending` and `state` at once (see
|
||||
// `stamp_pending_sizes`) and this must not become the first.
|
||||
// What that costs is a window of a few instructions instead of
|
||||
// a lock-tight comparison — against the multi-gigabyte copy
|
||||
// above, which is the window that actually loses data.
|
||||
let current = self.core.pending_blob(&uid);
|
||||
let mut st = self.core.state();
|
||||
// Only a handle this open is about to *create* can carry a stale
|
||||
// base; if one already exists, that handle is authoritative and
|
||||
// the scratch prepared above is discarded either way.
|
||||
let creating = !st.active_writes.contains_key(&ino.0);
|
||||
if creating && current != pending_base.map(|p| p.path) {
|
||||
drop(st);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
warn!(%uid, attempt, "queued revision changed while opening for write; retrying");
|
||||
continue;
|
||||
}
|
||||
let fh = st.next_fh;
|
||||
st.next_fh += 1;
|
||||
let aw = st.active_writes.entry(ino.0).or_insert_with(|| {
|
||||
WriteHandle {
|
||||
ino: ino.0,
|
||||
uid: uid.clone(),
|
||||
file: Arc::new(file),
|
||||
path,
|
||||
written: initial_written,
|
||||
// Starts at the current size; reads in [0, base_size)
|
||||
// come from the base until overwritten.
|
||||
len: base_size,
|
||||
base_size,
|
||||
base_mtime,
|
||||
base_revision_id,
|
||||
dirty: false,
|
||||
open_count: 0,
|
||||
}
|
||||
});
|
||||
aw.open_count += 1;
|
||||
st.handles.insert(fh, ino.0);
|
||||
break 'install Ok(fh);
|
||||
}
|
||||
error!(%uid, "queued revision kept changing while opening for write");
|
||||
Err(Errno::EIO)
|
||||
};
|
||||
let mut initial_written = Intervals::default();
|
||||
if let Some(pending) = &pending_base {
|
||||
// Reflink where the filesystem can: this blob is the whole file, and
|
||||
// a byte-for-byte copy of a multi-GB pending revision is the kind of
|
||||
// work `open` must not do (`ContentCache::clone_or_copy`).
|
||||
if let Err(e) = ContentCache::clone_or_copy(&pending.path, &path) {
|
||||
match outcome {
|
||||
Ok(fh) => reply.opened(FileHandle(fh), FopenFlags::empty()),
|
||||
Err(error) => {
|
||||
// The lookup count was raised before any of this could fail.
|
||||
if let Some(entry) = self.core.state().entries.get_mut(&ino.0) {
|
||||
entry.open_count = entry.open_count.saturating_sub(1);
|
||||
}
|
||||
let _ = std::fs::remove_file(&path);
|
||||
error!(%uid, source = %pending.path.display(), error = %e,
|
||||
"copy queued revision into write scratch failed");
|
||||
reply.error(Errno::EIO);
|
||||
return;
|
||||
reply.error(error);
|
||||
}
|
||||
initial_written.add(0, pending.meta.len);
|
||||
}
|
||||
let mut st = self.core.state();
|
||||
let fh = st.next_fh;
|
||||
st.next_fh += 1;
|
||||
let aw = st.active_writes.entry(ino.0).or_insert_with(|| {
|
||||
WriteHandle {
|
||||
ino: ino.0,
|
||||
uid,
|
||||
file: Arc::new(file),
|
||||
path,
|
||||
written: initial_written,
|
||||
// Starts at the current size; reads in [0, base_size) come from
|
||||
// the base until overwritten.
|
||||
len: base_size,
|
||||
base_size,
|
||||
base_mtime,
|
||||
base_revision_id,
|
||||
dirty: false,
|
||||
open_count: 0,
|
||||
}
|
||||
});
|
||||
aw.open_count += 1;
|
||||
st.handles.insert(fh, ino.0);
|
||||
reply.opened(FileHandle(fh), FopenFlags::empty());
|
||||
}
|
||||
|
||||
/// The body of [`Filesystem::fsync`], run off the dispatch loop.
|
||||
|
||||
+222
-15
@@ -55,7 +55,7 @@ use fuser::{
|
||||
};
|
||||
use futures::StreamExt as _;
|
||||
use pdfs_core::batch;
|
||||
use pdfs_core::cache::{BLOCK_SIZE, Baseline, ContentCache, StagedWrite};
|
||||
use pdfs_core::cache::{BLOCK_SIZE, Baseline, BlockGeometry, BlockSpan, ContentCache, StagedWrite};
|
||||
use pdfs_core::config::{AppDirs, SweepMode};
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo,
|
||||
@@ -92,6 +92,7 @@ mod profile;
|
||||
mod reads;
|
||||
mod revisions;
|
||||
mod sharing;
|
||||
mod shutdown;
|
||||
mod state;
|
||||
mod sweep;
|
||||
mod sync;
|
||||
@@ -410,6 +411,10 @@ struct Core {
|
||||
/// kernel read-ahead, our own prefetch) share a single download and decrypt.
|
||||
/// See [`BlockFlight`].
|
||||
block_flight: Arc<BlockFlight>,
|
||||
/// One lock per file being repaired by [`Core::repair_block`], so the
|
||||
/// concurrent block reads that all discover the same untrustworthy block
|
||||
/// table share a single whole-file download instead of one each.
|
||||
content_repairs: Arc<tokio::sync::Mutex<HashMap<NodeUid, Arc<tokio::sync::Mutex<()>>>>>,
|
||||
/// Per-file sequential-read detection driving [`Core::prefetch`].
|
||||
prefetch: Arc<Prefetch>,
|
||||
/// Permits bounding speculative block fetches, so prefetch cannot queue in
|
||||
@@ -450,6 +455,9 @@ struct Core {
|
||||
/// Nudges the drain workers: set true and notify to have them re-examine
|
||||
/// the queue instead of waiting out their backoff.
|
||||
drain_wake: Arc<(Mutex<bool>, Condvar)>,
|
||||
/// The daemon's stop signal, shared with every background loop this `Core`
|
||||
/// was cloned into so teardown can end and join them (bugs.md B44).
|
||||
shutdown: Arc<shutdown::Shutdown>,
|
||||
/// How long this node's last upload actually took, in milliseconds, with
|
||||
/// the time it was measured for eviction. Feeds
|
||||
/// [`Core::revision_debounce`]: a file that takes 30 seconds to send needs
|
||||
@@ -707,6 +715,26 @@ struct MountedState {
|
||||
/// [`Core::notifier`], still empty until its session is spawned.
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
/// This mount's in-flight size upgrades, keyed by *its* folder inodes — so
|
||||
/// a `Core` rebuilt onto this inode space by [`Core::rooted_at`] shares the
|
||||
/// batch the mount's own session is already running rather than starting a
|
||||
/// second one against inode numbers that mean nothing to it.
|
||||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||||
}
|
||||
|
||||
/// The per-mount half of a [`Core`] — every field [`Core::fork_state`] gives a
|
||||
/// fork a fresh copy of — recovered from the registry.
|
||||
///
|
||||
/// This is what lets a control request naming a path under a secondary mount be
|
||||
/// answered *in that mount's inode space* instead of being rejected for not
|
||||
/// being under the primary mountpoint (`docs/BUGS.md` B86). See
|
||||
/// [`Core::rooted_at`].
|
||||
struct MountParts {
|
||||
mountpoint: PathBuf,
|
||||
state: Arc<Mutex<State>>,
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||||
}
|
||||
|
||||
/// Every mounted inode space in the daemon, shared by the primary `Core` and
|
||||
@@ -729,6 +757,7 @@ impl StateRegistry {
|
||||
state: &Arc<Mutex<State>>,
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||||
) {
|
||||
let mut states = self.0.lock();
|
||||
states.retain(|m| m.state.strong_count() > 0);
|
||||
@@ -743,6 +772,7 @@ impl StateRegistry {
|
||||
state: Arc::downgrade(state),
|
||||
notifier,
|
||||
session_live,
|
||||
size_upgrades,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -769,9 +799,9 @@ impl StateRegistry {
|
||||
///
|
||||
/// On-demand roots can be nested below another location. Selecting by the
|
||||
/// longest component prefix ensures the nested session wins rather than the
|
||||
/// broader primary mount. Callers currently need the selected mount only;
|
||||
/// a future path router can derive a relative suffix with
|
||||
/// `path.strip_prefix(mountpoint)` without storing a second path value here.
|
||||
/// broader primary mount. Callers that need the relative suffix derive it
|
||||
/// with `path.strip_prefix(mountpoint)`; [`StateRegistry::covering_parts`]
|
||||
/// is the variant that also hands back enough to *serve* a request there.
|
||||
fn covering(&self, path: &Path) -> Option<LiveMount> {
|
||||
self.live()
|
||||
.into_iter()
|
||||
@@ -781,6 +811,48 @@ impl StateRegistry {
|
||||
.max_by_key(|(mountpoint, _, _, _)| mountpoint.components().count())
|
||||
}
|
||||
|
||||
/// [`StateRegistry::register`] with an empty size-upgrade map, for the
|
||||
/// tests that exercise the reap-and-route rules and nothing else.
|
||||
#[cfg(test)]
|
||||
fn register_bare(
|
||||
&self,
|
||||
mountpoint: &Path,
|
||||
state: &Arc<Mutex<State>>,
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
) {
|
||||
self.register(
|
||||
mountpoint,
|
||||
state,
|
||||
notifier,
|
||||
session_live,
|
||||
Arc::new(Mutex::new(HashMap::new())),
|
||||
);
|
||||
}
|
||||
|
||||
/// [`StateRegistry::covering`], but returning every per-mount field a
|
||||
/// [`Core`] needs to be re-rooted onto that inode space.
|
||||
///
|
||||
/// Kept separate from `covering` so the common callers — which only ask
|
||||
/// *whether* a path is mounted — keep the cheaper tuple.
|
||||
fn covering_parts(&self, path: &Path) -> Option<MountParts> {
|
||||
let mut states = self.0.lock();
|
||||
states.retain(|m| m.state.strong_count() > 0);
|
||||
states
|
||||
.iter()
|
||||
.filter(|m| m.session_live.load(Ordering::Acquire) && path.starts_with(&m.mountpoint))
|
||||
.max_by_key(|m| m.mountpoint.components().count())
|
||||
.and_then(|m| {
|
||||
Some(MountParts {
|
||||
mountpoint: m.mountpoint.clone(),
|
||||
state: m.state.upgrade()?,
|
||||
notifier: m.notifier.clone(),
|
||||
session_live: m.session_live.clone(),
|
||||
size_upgrades: m.size_upgrades.clone(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn is_mounted_at(&self, path: &Path) -> bool {
|
||||
self.covering(path).is_some_and(|(mountpoint, _, _, live)| {
|
||||
mountpoint == path && live.load(Ordering::Acquire)
|
||||
@@ -900,9 +972,41 @@ impl Core {
|
||||
&self.state,
|
||||
self.notifier.clone(),
|
||||
self.session_live.clone(),
|
||||
self.size_upgrades.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
/// This `Core`, re-rooted onto whichever live mount most specifically covers
|
||||
/// the absolute path `abs`, together with `abs` relative to that mount's
|
||||
/// root.
|
||||
///
|
||||
/// Everything a `Core` holds is per-daemon except the five fields
|
||||
/// [`Core::fork_state`] replaces, so swapping exactly those turns the
|
||||
/// primary `Core` into the fork that owns the path — sharing the fork's
|
||||
/// inode space, notification channel and size-upgrade batches rather than
|
||||
/// shadowing them. Without this, every path-addressed control request is
|
||||
/// answered against the primary inode space, so anything under a secondary
|
||||
/// on-demand mount is rejected as "not under the mountpoint" and the user
|
||||
/// has no way to re-enumerate a folder they can see (`docs/BUGS.md` B86).
|
||||
///
|
||||
/// `None` when no live mount covers `abs`.
|
||||
fn rooted_at(&self, abs: &Path) -> Option<(Core, PathBuf)> {
|
||||
let parts = self.states.covering_parts(abs)?;
|
||||
let rel = abs.strip_prefix(&parts.mountpoint).ok()?.to_path_buf();
|
||||
let mut core = self.clone();
|
||||
if !Arc::ptr_eq(&parts.state, &self.state) {
|
||||
// A different inode space, so adopt its per-mount half wholesale.
|
||||
// `primary` is a property of the mount, and the only mount this
|
||||
// `Core` can be the primary of is the one it was built for.
|
||||
core.primary = false;
|
||||
core.state = parts.state;
|
||||
core.notifier = parts.notifier;
|
||||
core.session_live = parts.session_live;
|
||||
core.size_upgrades = parts.size_upgrades;
|
||||
}
|
||||
Some((core, rel))
|
||||
}
|
||||
|
||||
/// Run `apply` against every live inode space — this daemon's primary mount
|
||||
/// and each on-demand fork — one lock at a time.
|
||||
///
|
||||
@@ -1708,7 +1812,9 @@ impl Core {
|
||||
fn run_online_probe(&self) {
|
||||
let mut delay = ONLINE_PROBE_MIN;
|
||||
loop {
|
||||
std::thread::sleep(delay);
|
||||
if !self.shutdown.sleep(delay) {
|
||||
return;
|
||||
}
|
||||
match self.rt.block_on(self.client.get_my_files_folder()) {
|
||||
Ok(root) => {
|
||||
{
|
||||
@@ -3225,6 +3331,15 @@ impl Core {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Which staged blob is currently queued for `uid`, if any.
|
||||
///
|
||||
/// The path identifies the revision: publication stages a fresh file and
|
||||
/// swaps the map entry to point at it, so an unchanged path means the base a
|
||||
/// caller sampled earlier is still the base now (bugs.md B32).
|
||||
fn pending_blob(&self, uid: &NodeUid) -> Option<PathBuf> {
|
||||
self.pending.lock().get(uid).map(|p| p.path.clone())
|
||||
}
|
||||
|
||||
/// Nudge the drain worker to re-examine the queue now.
|
||||
fn wake_drain(&self) {
|
||||
let (lock, cv) = &*self.drain_wake;
|
||||
@@ -3915,6 +4030,33 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the cached child listing of the folder `parent`, everywhere it is
|
||||
/// held — naming the folder by uid rather than by inode.
|
||||
///
|
||||
/// [`Core::invalidate_parent_listing`] is the path-based sibling, and it
|
||||
/// only reaches *this* mount's inode space. A remote mutation made by the
|
||||
/// sync engine needs more than that, for two reasons. The folder it changed
|
||||
/// is usually not resident in any inode space at the time — a mirror folder
|
||||
/// has no FUSE session at all — so there is no inode to name. And the stale
|
||||
/// listing that outlives the pass is not a hot-cache map but the `listed`
|
||||
/// flag on the folder's DB row, which survives a daemon restart and is what
|
||||
/// a later `ensure_children` trusts.
|
||||
///
|
||||
/// So the flag is cleared unconditionally, and the resident inode spaces are
|
||||
/// invalidated on top of it. That is what makes a mirror folder switched to
|
||||
/// on-demand re-enumerate instead of serving a snapshot that predates the
|
||||
/// engine's own uploads — `docs/BUGS.md` B86.
|
||||
pub(crate) fn invalidate_children_of(&self, parent: &NodeUid) {
|
||||
if let Err(e) = self.db.set_listed(parent, false) {
|
||||
warn!(uid = %parent, error = ?e, "could not clear cached listing");
|
||||
}
|
||||
self.for_each_state(|st| {
|
||||
if let Some(&ino) = st.by_uid.get(parent) {
|
||||
st.invalidate_listing(ino);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Rename a file or folder to `new_name`. `rel` is mountpoint-relative.
|
||||
/// Mirrors the FUSE `rename` write path: rename on the remote, forget the
|
||||
/// node so it re-interns under its new name, and drop the parent listing so
|
||||
@@ -5834,7 +5976,7 @@ mod tests {
|
||||
("/mnt/fork-a", &fork_a),
|
||||
("/mnt/fork-b", &fork_b),
|
||||
] {
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new(mountpoint),
|
||||
state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -5885,19 +6027,19 @@ mod tests {
|
||||
let nested_live = session_flag(false);
|
||||
let sibling_live = session_flag(true);
|
||||
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
primary_path,
|
||||
&primary,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
primary_live.clone(),
|
||||
);
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
nested_path,
|
||||
&nested,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
nested_live.clone(),
|
||||
);
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
sibling_path,
|
||||
&sibling,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -5975,6 +6117,71 @@ mod tests {
|
||||
assert_eq!(registry.live().len(), 2);
|
||||
}
|
||||
|
||||
/// The routing half of `docs/BUGS.md` B86: a path under a secondary mount
|
||||
/// has to resolve to *that* mount's inode space and a suffix relative to
|
||||
/// *its* root, or `pdfs ls` and `pdfs refresh` can only ever name the
|
||||
/// primary mount and a stale on-demand folder has no escape hatch.
|
||||
#[test]
|
||||
fn covering_parts_route_a_path_to_the_mount_that_owns_it() {
|
||||
let registry = StateRegistry::default();
|
||||
let (primary, _primary_dir) = rooted_state("primary-volume", "primary");
|
||||
let (nested, _nested_dir) = rooted_state("nested-volume", "nested");
|
||||
let primary = std::sync::Arc::new(parking_lot::Mutex::new(primary));
|
||||
let nested = std::sync::Arc::new(parking_lot::Mutex::new(nested));
|
||||
let primary_path = std::path::Path::new("/home/me/ProtonDrive");
|
||||
let nested_path = std::path::Path::new("/home/me/ProtonDrive/Device");
|
||||
let primary_upgrades = std::sync::Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
let nested_upgrades = std::sync::Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
registry.register(
|
||||
primary_path,
|
||||
&primary,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
session_flag(true),
|
||||
primary_upgrades.clone(),
|
||||
);
|
||||
registry.register(
|
||||
nested_path,
|
||||
&nested,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
session_flag(true),
|
||||
nested_upgrades.clone(),
|
||||
);
|
||||
|
||||
let deep = nested_path.join("sub/file.txt");
|
||||
let parts = registry.covering_parts(&deep).expect("nested mount covers");
|
||||
assert_eq!(parts.mountpoint, nested_path, "the nested mount wins");
|
||||
assert!(
|
||||
std::sync::Arc::ptr_eq(&parts.state, &nested),
|
||||
"the request must be answered in the nested inode space"
|
||||
);
|
||||
assert!(
|
||||
std::sync::Arc::ptr_eq(&parts.size_upgrades, &nested_upgrades),
|
||||
"and share that mount's in-flight size upgrades, not the primary's"
|
||||
);
|
||||
assert_eq!(
|
||||
deep.strip_prefix(&parts.mountpoint).unwrap(),
|
||||
std::path::Path::new("sub/file.txt"),
|
||||
"the suffix is relative to the mount that owns it, not the primary"
|
||||
);
|
||||
|
||||
let shallow = primary_path.join("Documents");
|
||||
let parts = registry
|
||||
.covering_parts(&shallow)
|
||||
.expect("primary mount covers");
|
||||
assert!(
|
||||
std::sync::Arc::ptr_eq(&parts.state, &primary),
|
||||
"a path outside the nested root still routes to the primary"
|
||||
);
|
||||
|
||||
assert!(
|
||||
registry
|
||||
.covering_parts(std::path::Path::new("/tmp/outside"))
|
||||
.is_none(),
|
||||
"a path under no mount routes nowhere, and the caller reports that"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_registry_resolves_uids_from_any_resident_mount() {
|
||||
let registry = StateRegistry::default();
|
||||
@@ -5988,13 +6195,13 @@ mod tests {
|
||||
fork.children.insert(super::ROOT_INO, vec![fork_ino]);
|
||||
let primary = std::sync::Arc::new(parking_lot::Mutex::new(primary));
|
||||
let fork = std::sync::Arc::new(parking_lot::Mutex::new(fork));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/primary"),
|
||||
&primary,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
session_flag(true),
|
||||
);
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&fork,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6022,7 +6229,7 @@ mod tests {
|
||||
let child_ino = state.intern(super::ROOT_INO, child_node);
|
||||
state.children.insert(super::ROOT_INO, vec![child_ino]);
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6085,7 +6292,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6353,7 +6560,7 @@ mod tests {
|
||||
state.children.insert(revoked_ino, vec![child_ino]);
|
||||
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6407,7 +6614,7 @@ mod tests {
|
||||
state.children.insert(cycle_b, vec![cycle_a]);
|
||||
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
|
||||
@@ -365,6 +365,7 @@ pub fn mount(
|
||||
readers: Arc::new(Mutex::new(HashMap::new())),
|
||||
block_ring: Arc::new(Mutex::new(BlockRing::default())),
|
||||
block_flight: Arc::new(BlockFlight::default()),
|
||||
content_repairs: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
prefetch: Arc::new(Prefetch::default()),
|
||||
prefetch_budget: Arc::new(tokio::sync::Semaphore::new(PREFETCH_BUDGET)),
|
||||
workers: Arc::new(Workers::new(FUSE_WORKERS)?),
|
||||
@@ -376,6 +377,7 @@ pub fn mount(
|
||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||
hidden: Arc::new(Mutex::new(HashSet::new())),
|
||||
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
||||
shutdown: Arc::new(crate::shutdown::Shutdown::default()),
|
||||
upload_times: Arc::new(Mutex::new(HashMap::new())),
|
||||
upload_cancel: Arc::new(Mutex::new(HashMap::new())),
|
||||
timeline_refreshing: Arc::new(AtomicBool::new(false)),
|
||||
@@ -420,16 +422,24 @@ pub fn mount(
|
||||
// long as it ran; the claim column (`Db::claim_next_due_op`) keeps the
|
||||
// workers off each other's nodes, which is the only ordering the queue
|
||||
// actually needs. Worker 0 additionally runs the queue's idle chores.
|
||||
//
|
||||
// Every long-lived thread started from here on is kept in `workers` and
|
||||
// joined at the end of this function. Leaving them running turned an
|
||||
// in-process remount into a second generation of drain workers, sync engines
|
||||
// and sweeps all operating on a mount that no longer exists (bugs.md B44).
|
||||
let mut workers: Vec<std::thread::JoinHandle<()>> = Vec::new();
|
||||
for worker in 0..DRAIN_WORKERS {
|
||||
let core = core.clone();
|
||||
std::thread::Builder::new()
|
||||
.name(format!("pdfs-drain-{worker}"))
|
||||
.spawn(move || core.run_pending_drain(worker == 0))?;
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name(format!("pdfs-drain-{worker}"))
|
||||
.spawn(move || core.run_pending_drain(worker == 0))?,
|
||||
);
|
||||
}
|
||||
|
||||
// Start the folder-sync engine. It watches every mirror folder, polls the
|
||||
// remotes, and reconciles on its own thread — never in front of a FUSE call.
|
||||
sync::spawn(core.clone(), sync_rx);
|
||||
workers.extend(sync::spawn(core.clone(), sync_rx));
|
||||
|
||||
// Reconcile leftover `(sync-conflict …)` copies: drop the ones that turned
|
||||
// out identical to the live file, surface the ones that genuinely diverge.
|
||||
@@ -441,18 +451,22 @@ pub fn mount(
|
||||
} else {
|
||||
info!(mode = ?sweep_mode, "conflict sweep enabled");
|
||||
let core = core.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-conflict-sweep".into())
|
||||
.spawn(move || core.run_conflict_sweep_loop())?;
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-conflict-sweep".into())
|
||||
.spawn(move || core.run_conflict_sweep_loop())?,
|
||||
);
|
||||
}
|
||||
|
||||
// Mounted from the cache: watch for the network coming back so the mount can
|
||||
// stop being read-only-ish without the user restarting the daemon.
|
||||
if !online {
|
||||
let core = core.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-online-probe".into())
|
||||
.spawn(move || core.run_online_probe())?;
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-online-probe".into())
|
||||
.spawn(move || core.run_online_probe())?,
|
||||
);
|
||||
}
|
||||
|
||||
// Keep the launcher prompt's "This computer" index fresh. Its own thread:
|
||||
@@ -462,9 +476,12 @@ pub fn mount(
|
||||
let indexing = core.indexing.clone();
|
||||
let transfers = core.transfers.clone();
|
||||
let mountpoint = mountpoint.to_path_buf();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-localindex".into())
|
||||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint))?;
|
||||
let shutdown = core.shutdown.clone();
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-localindex".into())
|
||||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint, shutdown))?,
|
||||
);
|
||||
}
|
||||
|
||||
// Bind the control socket before the FUSE session takes over the thread. A
|
||||
@@ -501,23 +518,29 @@ pub fn mount(
|
||||
let control_core = core.clone();
|
||||
let username = username.clone();
|
||||
let mountpoint = mountpoint.to_path_buf();
|
||||
if let Err(error) = std::thread::Builder::new()
|
||||
match std::thread::Builder::new()
|
||||
.name("pdfs-control".into())
|
||||
.spawn(move || run_control_socket(control_core, username, mountpoint, listener))
|
||||
{
|
||||
let secondaries = core.mounts.lock().close_and_drain();
|
||||
for (id, mount) in secondaries {
|
||||
if let Err(unmount_error) = mount.teardown() {
|
||||
warn!(id, error = %unmount_error, "secondary teardown after control startup failure failed");
|
||||
Ok(handle) => workers.push(handle),
|
||||
Err(error) => {
|
||||
// A failure here abandons the mount, so the background threads
|
||||
// started above have to be told to stop with it.
|
||||
stop_workers(&core, control_socket, workers);
|
||||
let secondaries = core.mounts.lock().close_and_drain();
|
||||
for (id, mount) in secondaries {
|
||||
if let Err(unmount_error) = mount.teardown() {
|
||||
warn!(id, error = %unmount_error, "secondary teardown after control startup failure failed");
|
||||
}
|
||||
}
|
||||
if let Err(unmount_error) = teardown_session(&core.session_live, || {
|
||||
umount_session_unblocked(bg, main_conn)
|
||||
}) {
|
||||
warn!(error = %unmount_error, "unmount after control startup failure failed");
|
||||
}
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
return Err(error);
|
||||
}
|
||||
if let Err(unmount_error) = teardown_session(&core.session_live, || {
|
||||
umount_session_unblocked(bg, main_conn)
|
||||
}) {
|
||||
warn!(error = %unmount_error, "unmount after control startup failure failed");
|
||||
}
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
// Re-establish on-demand mounts only after the primary session is live.
|
||||
@@ -626,10 +649,43 @@ pub fn mount(
|
||||
}
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
// Last, because the workers may legitimately be finishing an upload, and
|
||||
// nothing they can still do is harmful once the mounts are down.
|
||||
stop_workers(&core, control_socket, workers);
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Signal every background loop, wake the ones that are blocked, and join them.
|
||||
///
|
||||
/// The mount used to return with its drain workers, sync engine, sweep, online
|
||||
/// probe, indexer and control listener all still running on `Core` clones. The
|
||||
/// process usually exited straight afterwards and hid it; an in-process remount
|
||||
/// did not, and started a second full set on top of the first (bugs.md B44).
|
||||
///
|
||||
/// Two of the waits need waking rather than merely flagging: the drain workers
|
||||
/// block on their own condvar (`wake_drain`), and the control listener blocks
|
||||
/// inside `accept`, which only returns when a connection arrives — so it gets
|
||||
/// one, from us, after the flag is already set. A join that takes a while is
|
||||
/// expected and correct: a worker part-way through an upload finishes it rather
|
||||
/// than abandoning the user's bytes mid-flight.
|
||||
fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
||||
core.shutdown.stop();
|
||||
core.wake_drain();
|
||||
let _ = core.sync_tx.send(sync::SyncMsg::Stop);
|
||||
// Connect while the socket still exists — that is the whole point of the
|
||||
// poke — then unlink it so nothing else can arrive behind us. The listener
|
||||
// re-checks the flag after every accept, so even a connection that races the
|
||||
// unlink cannot put it back to sleep.
|
||||
let _ = std::os::unix::net::UnixStream::connect(control_socket);
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
for worker in workers {
|
||||
if worker.join().is_err() {
|
||||
warn!("a background worker panicked before shutdown");
|
||||
}
|
||||
}
|
||||
debug!("background workers stopped");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
+306
-71
@@ -26,6 +26,11 @@ const PREFETCH_MAX: u64 = 8;
|
||||
/// Window a sequential read starts at, and returns to after a seek.
|
||||
const PREFETCH_MIN: u64 = 2;
|
||||
|
||||
/// Largest file [`Core::repair_block`] will pull down whole to work around a
|
||||
/// revision whose block table understates it. Past this a single read would cost
|
||||
/// gigabytes of download, so the read fails instead and says to pin the file.
|
||||
pub(super) const REPAIR_MAX: u64 = 512 * 1024 * 1024;
|
||||
|
||||
/// Blocks that may be prefetching at once, across every file. Prefetch takes a
|
||||
/// permit and gives up rather than queueing, so it can never sit in front of a
|
||||
/// demand read in the SDK's flat block semaphore.
|
||||
@@ -97,22 +102,38 @@ static NEXT_OPEN_ID: AtomicU64 = AtomicU64::new(0);
|
||||
/// Validated by the same `(mtime, size)` pair the on-disk caches use: a node
|
||||
/// whose tag no longer matches has its blocks dropped rather than served, so a
|
||||
/// new revision can never be stitched together from the old one.
|
||||
/// A ring entry's identity: the node, and the plaintext range the block covers.
|
||||
///
|
||||
/// The range, rather than the block index, because the index alone stopped
|
||||
/// naming one range when the geometry started coming from the revision — a
|
||||
/// block cached under the [`BlockGeometry::uniform`] fallback and the block at
|
||||
/// the same index under the real geometry are different bytes
|
||||
/// (`docs/BUGS.md` B85). A range is the same identity under either plan, so a
|
||||
/// re-planned read misses rather than being served the wrong bytes.
|
||||
type RingKey = (NodeUid, BlockSpan);
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct BlockRing {
|
||||
blocks: HashMap<(NodeUid, u64), Arc<Vec<u8>>>,
|
||||
blocks: HashMap<RingKey, Arc<Vec<u8>>>,
|
||||
/// Keys least-recently-used first, for eviction.
|
||||
order: VecDeque<(NodeUid, u64)>,
|
||||
order: VecDeque<RingKey>,
|
||||
/// Per-node validity tag, `(mtime, size)`.
|
||||
tags: HashMap<NodeUid, (i64, u64)>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
impl BlockRing {
|
||||
fn get(&mut self, uid: &NodeUid, mtime: i64, size: u64, idx: u64) -> Option<Arc<Vec<u8>>> {
|
||||
fn get(
|
||||
&mut self,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: u64,
|
||||
span: BlockSpan,
|
||||
) -> Option<Arc<Vec<u8>>> {
|
||||
if self.tags.get(uid) != Some(&(mtime, size)) {
|
||||
return None;
|
||||
}
|
||||
let key = (uid.clone(), idx);
|
||||
let key = (uid.clone(), span);
|
||||
let hit = self.blocks.get(&key).cloned()?;
|
||||
// Move to the young end: a block being read repeatedly (the whole reason
|
||||
// this exists) must not age out under a sequential read passing through.
|
||||
@@ -123,14 +144,21 @@ impl BlockRing {
|
||||
Some(hit)
|
||||
}
|
||||
|
||||
fn insert(&mut self, uid: &NodeUid, mtime: i64, size: u64, idx: u64, bytes: Arc<Vec<u8>>) {
|
||||
fn insert(
|
||||
&mut self,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: u64,
|
||||
span: BlockSpan,
|
||||
bytes: Arc<Vec<u8>>,
|
||||
) {
|
||||
if self.tags.get(uid) != Some(&(mtime, size)) {
|
||||
// Revision changed under us (or first sight): anything held for this
|
||||
// node describes the old one.
|
||||
self.drop_node(uid);
|
||||
self.tags.insert(uid.clone(), (mtime, size));
|
||||
}
|
||||
let key = (uid.clone(), idx);
|
||||
let key = (uid.clone(), span);
|
||||
if self.blocks.contains_key(&key) {
|
||||
return;
|
||||
}
|
||||
@@ -171,8 +199,10 @@ impl BlockRing {
|
||||
/// The result of fetching one block, shared between everyone who wanted it.
|
||||
type BlockResult = Result<Arc<Vec<u8>>, Errno>;
|
||||
|
||||
/// Identifies a block *of a revision*: node, validity tag, index.
|
||||
type BlockKey = (NodeUid, i64, u64, u64);
|
||||
/// Identifies a block *of a revision*: node, validity tag, and the plaintext
|
||||
/// range the block covers — see [`RingKey`] for why it is the range and not the
|
||||
/// index.
|
||||
type BlockKey = (NodeUid, i64, u64, BlockSpan);
|
||||
|
||||
/// A claim on a block key: who owns the fetch, and where its result is published.
|
||||
type Claim = (u64, tokio::sync::watch::Receiver<Option<BlockResult>>);
|
||||
@@ -412,11 +442,30 @@ impl Core {
|
||||
self.prefetch.streams.lock().remove(uid);
|
||||
}
|
||||
|
||||
/// How this revision's plaintext is divided into content blocks.
|
||||
///
|
||||
/// The revision's own geometry when a previous read has learned and recorded
|
||||
/// it, and the [`BLOCK_SIZE`] assumption when it has not — which is every
|
||||
/// first read, because learning it means opening a `RevisionReader`, and
|
||||
/// opening one per read is precisely the per-file key derivation the block
|
||||
/// cache exists to avoid (bugs.md B12, B85).
|
||||
///
|
||||
/// Being wrong here is not a correctness problem: `read_at` plans over the
|
||||
/// real sizes and clamps to what was asked for either way. It costs a
|
||||
/// straddling fetch and a cached block the next read will not ask for, and
|
||||
/// both settle once the geometry is known.
|
||||
fn block_geometry(&self, uid: &NodeUid, mtime: i64, fsize: u64) -> BlockGeometry {
|
||||
self.cache
|
||||
.block_geometry(uid, mtime, fsize)
|
||||
.unwrap_or_else(|| BlockGeometry::uniform(fsize))
|
||||
}
|
||||
|
||||
/// Serve bytes `[offset, offset + len)` of `uid`'s active revision, hitting
|
||||
/// the on-disk caches before the network: a whole-file blob (pinned files)
|
||||
/// first, then the block cache — fetching only the [`BLOCK_SIZE`]-aligned
|
||||
/// blocks that overlap the request and caching each. `mtime`/`fsize` validate
|
||||
/// both caches. Network I/O runs without any lock held.
|
||||
/// first, then the block cache — fetching only the blocks that overlap the
|
||||
/// request, on the revision's own boundaries ([`Core::block_geometry`]), and
|
||||
/// caching each. `mtime`/`fsize` validate both caches. Network I/O runs
|
||||
/// without any lock held.
|
||||
pub(super) fn read_range(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
@@ -495,7 +544,7 @@ impl Core {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Fetch block `bidx`, joining a fetch already in flight for it rather than
|
||||
/// Fetch the block at `span`, joining a fetch already in flight rather than
|
||||
/// issuing a second one, and publish the result to everyone who joined.
|
||||
///
|
||||
/// See [`BlockFlight`]. A follower whose leader disappears (a panic, or a
|
||||
@@ -507,10 +556,10 @@ impl Core {
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
bidx: u64,
|
||||
span: BlockSpan,
|
||||
cache_blocks: bool,
|
||||
) -> BlockResult {
|
||||
let key: BlockKey = (uid.clone(), mtime, fsize, bidx);
|
||||
let key: BlockKey = (uid.clone(), mtime, fsize, span);
|
||||
let claim = {
|
||||
let mut inflight = self.block_flight.inflight.lock();
|
||||
match inflight.get(&key) {
|
||||
@@ -542,12 +591,12 @@ impl Core {
|
||||
}
|
||||
}
|
||||
return self
|
||||
.read_block(reader, uid, mtime, fsize, bidx, cache_blocks)
|
||||
.read_block(reader, uid, mtime, fsize, span, cache_blocks)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let result = self
|
||||
.read_block(reader, uid, mtime, fsize, bidx, cache_blocks)
|
||||
.read_block(reader, uid, mtime, fsize, span, cache_blocks)
|
||||
.await;
|
||||
// Retire the claim before publishing, so a racer that wakes and re-enters
|
||||
// finds no stale entry to join.
|
||||
@@ -563,36 +612,193 @@ impl Core {
|
||||
/// The disk store is handed to a blocking thread — it is a 4 MiB write plus
|
||||
/// a cache-index update, and neither the reactor nor the kernel read waiting
|
||||
/// on these bytes should be held up by it.
|
||||
///
|
||||
/// A block that comes back shorter than the file's size says it must be is
|
||||
/// never served, cached or promoted: it is repaired from the whole-file
|
||||
/// download, or the read fails. See [`Core::repair_block`] (bugs.md B84).
|
||||
async fn read_block(
|
||||
&self,
|
||||
reader: &Arc<RevisionReader>,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
bidx: u64,
|
||||
span: BlockSpan,
|
||||
cache_blocks: bool,
|
||||
) -> BlockResult {
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
let blen = BLOCK_SIZE.min(fsize - bstart);
|
||||
let bytes = reader.read_at(bstart, blen).await.map_err(|e| {
|
||||
warn!(%uid, bstart, blen, error = %e, "block read failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
// The reader derives every block boundary from the revision's recorded
|
||||
// block sizes. If they do not add up to the size the node reports, this
|
||||
// file's boundaries are wrong, and the failure is not confined to short
|
||||
// reads: sizes that *overstate* a block shift every block after it and
|
||||
// return full-length reads of the wrong bytes. So a disagreement of any
|
||||
// kind takes the file off the range path entirely (bugs.md B84).
|
||||
let bytes = if reader.size() != fsize {
|
||||
warn!(
|
||||
%uid, bidx = span.idx, fsize,
|
||||
reader_size = reader.size(),
|
||||
reader_blocks = reader.block_sizes().len(),
|
||||
"the revision's block table disagrees with the node size"
|
||||
);
|
||||
self.repair_block(uid, mtime, fsize, span).await?
|
||||
} else {
|
||||
// Learned here rather than on a path of its own: this is where a
|
||||
// reader is open anyway, and recording the real geometry is what
|
||||
// lets the *next* read plan its blocks on the revision's own
|
||||
// boundaries instead of the 4 MiB fallback (bugs.md B85).
|
||||
if cache_blocks {
|
||||
self.cache
|
||||
.store_block_geometry(uid, mtime, fsize, reader.block_sizes());
|
||||
}
|
||||
let bytes = reader.read_at(span.start, span.len).await.map_err(|e| {
|
||||
warn!(%uid, start = span.start, len = span.len, error = %e, "block read failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
if bytes.len() as u64 == span.len {
|
||||
bytes
|
||||
} else {
|
||||
warn!(
|
||||
%uid, bidx = span.idx, blen = span.len, got = bytes.len(), fsize,
|
||||
"block came back shorter than the range asked for"
|
||||
);
|
||||
self.repair_block(uid, mtime, fsize, span).await?
|
||||
}
|
||||
};
|
||||
let bytes = Arc::new(bytes);
|
||||
if cache_blocks {
|
||||
let cache = self.cache.clone();
|
||||
let uid = uid.clone();
|
||||
let bytes = bytes.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _ = cache.store_block(&uid, mtime, fsize, bidx, &bytes);
|
||||
let _ = cache.store_block(&uid, mtime, fsize, span, &bytes);
|
||||
});
|
||||
}
|
||||
self.block_ring
|
||||
.lock()
|
||||
.insert(uid, mtime, fsize, bidx, bytes.clone());
|
||||
.insert(uid, mtime, fsize, span, bytes.clone());
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Recover the block at `span` of a file whose per-block reads cannot be
|
||||
/// by fetching the whole revision through the manifest-verified download and
|
||||
/// serving the block out of that.
|
||||
///
|
||||
/// The range path derives every block boundary from the revision's recorded
|
||||
/// block sizes. When those understate the file, reads come back short with
|
||||
/// no error anywhere, and the caller hands userspace a truncated file that
|
||||
/// looks like a clean EOF (bugs.md B84). `download_file_to` does not do that
|
||||
/// arithmetic — it streams the block list — so it returns the whole file
|
||||
/// where the range path cannot, which is why pinning was the workaround.
|
||||
/// This does the same thing on the user's behalf, once, and the resulting
|
||||
/// blob answers every later read of the file before the block path is
|
||||
/// consulted at all.
|
||||
///
|
||||
/// Bounded by [`REPAIR_MAX`]: past that, one unlucky read would pull down
|
||||
/// gigabytes, and a clear `EIO` plus a `pdfs pin` is the better trade.
|
||||
async fn repair_block(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
span: BlockSpan,
|
||||
) -> Result<Vec<u8>, Errno> {
|
||||
let (bstart, blen) = (span.start, span.len);
|
||||
if fsize > REPAIR_MAX {
|
||||
warn!(
|
||||
%uid, fsize,
|
||||
"file is too large to repair from a whole-file download; pin it to read it"
|
||||
);
|
||||
return Err(Errno::EIO);
|
||||
}
|
||||
// One download per file, not per block: every block of a file with a bad
|
||||
// block table discovers the same problem at once.
|
||||
let lock = {
|
||||
let mut locks = self.content_repairs.lock().await;
|
||||
locks.entry(uid.clone()).or_default().clone()
|
||||
};
|
||||
let guard = lock.clone().lock_owned().await;
|
||||
let downloaded = match self.cache.read_range(uid, mtime, fsize, bstart, blen) {
|
||||
// The first arrival did the work; the rest just read its blob.
|
||||
Some(bytes) if bytes.len() as u64 == blen => Ok(Some(bytes)),
|
||||
_ => self
|
||||
.download_whole_revision(uid, mtime, fsize)
|
||||
.await
|
||||
.map(|()| None),
|
||||
};
|
||||
// Before releasing, so the map entry is only dropped once no waiter is
|
||||
// still holding it.
|
||||
drop(guard);
|
||||
self.release_repair_lock(uid, lock).await;
|
||||
if let Some(bytes) = downloaded? {
|
||||
return Ok(bytes);
|
||||
}
|
||||
match self.cache.read_range(uid, mtime, fsize, bstart, blen) {
|
||||
Some(bytes) if bytes.len() as u64 == blen => {
|
||||
info!(%uid, bidx = span.idx, "served a short block from a repaired whole-file download");
|
||||
Ok(bytes)
|
||||
}
|
||||
_ => {
|
||||
error!(%uid, bidx = span.idx, fsize, "repairing a short block did not produce the block");
|
||||
Err(Errno::EIO)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Download the whole of `uid`'s active revision to disk and adopt it as the
|
||||
/// cached blob, so the range path is bypassed entirely for this file.
|
||||
///
|
||||
/// Streamed into a scratch file rather than memory: this runs for files up
|
||||
/// to [`REPAIR_MAX`], and buffering that per concurrent reader is not a cost
|
||||
/// a read should carry. A download that does not produce exactly `fsize`
|
||||
/// bytes is discarded — a short blob would fail the cache's own validity
|
||||
/// check anyway, and storing it would only hide the disagreement one layer
|
||||
/// further down.
|
||||
async fn download_whole_revision(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
) -> Result<(), Errno> {
|
||||
let (mut file, path) = self.cache.create_scratch().map_err(|error| {
|
||||
error!(%uid, %error, "creating a scratch file for a repair download failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
let outcome = self.client.download_file_to(uid, &mut file).await;
|
||||
let stored = (|| {
|
||||
outcome.map_err(|error| {
|
||||
warn!(%uid, %error, "repair download failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
file.sync_all().map_err(|error| {
|
||||
warn!(%uid, %error, "syncing a repair download failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
let got = file.metadata().map(|m| m.len()).unwrap_or(0);
|
||||
if got != fsize {
|
||||
warn!(%uid, fsize, got, "repair download did not match the node size");
|
||||
return Err(Errno::EIO);
|
||||
}
|
||||
self.cache
|
||||
.store_file(uid, mtime, fsize, &path)
|
||||
.map_err(|error| {
|
||||
error!(%uid, %error, "adopting a repair download as cached content failed");
|
||||
Errno::EIO
|
||||
})
|
||||
})();
|
||||
// The blob is a hardlink to the same inode, so unlinking the scratch
|
||||
// name leaves it intact.
|
||||
let _ = std::fs::remove_file(&path);
|
||||
stored
|
||||
}
|
||||
|
||||
/// Drop a repair lock's map entry once nobody else holds it, so a file
|
||||
/// repaired long ago does not keep a mutex alive forever.
|
||||
async fn release_repair_lock(&self, uid: &NodeUid, lock: Arc<tokio::sync::Mutex<()>>) {
|
||||
let mut locks = self.content_repairs.lock().await;
|
||||
// Two: the map's and ours. Anything more means a waiter still needs it.
|
||||
if Arc::strong_count(&lock) <= 2 {
|
||||
locks.remove(uid);
|
||||
}
|
||||
}
|
||||
|
||||
/// Warm the blocks after a sequential read into the caches, in the
|
||||
/// background, so the next read finds them instead of paying a round-trip.
|
||||
///
|
||||
@@ -619,17 +825,19 @@ impl Core {
|
||||
if depth == 0 {
|
||||
return;
|
||||
}
|
||||
let last_block = fsize.saturating_sub(1) / BLOCK_SIZE;
|
||||
let wanted: Vec<u64> = ((last + 1)..=(last + depth).min(last_block))
|
||||
.filter(|&bidx| {
|
||||
let geometry = self.block_geometry(uid, mtime, fsize);
|
||||
let last_block = geometry.blocks().saturating_sub(1);
|
||||
let wanted: Vec<BlockSpan> = ((last + 1)..=(last + depth).min(last_block))
|
||||
.filter_map(|bidx| geometry.span(bidx))
|
||||
.filter(|&span| {
|
||||
// Blocks already in memory need nothing; blocks already on disk
|
||||
// are cheap enough to serve that spending a round-trip to move
|
||||
// them into the ring would be the wrong trade.
|
||||
self.block_ring
|
||||
.lock()
|
||||
.get(uid, mtime, fsize, bidx)
|
||||
.get(uid, mtime, fsize, span)
|
||||
.is_none()
|
||||
&& !(cache_blocks && self.cache.has_block(uid, mtime, fsize, bidx))
|
||||
&& !(cache_blocks && self.cache.has_block(uid, mtime, fsize, span))
|
||||
})
|
||||
.collect();
|
||||
if wanted.is_empty() {
|
||||
@@ -646,7 +854,7 @@ impl Core {
|
||||
// round-trips, and the reader catches up with the window before it
|
||||
// finishes.
|
||||
let mut set = tokio::task::JoinSet::new();
|
||||
for bidx in wanted {
|
||||
for span in wanted {
|
||||
let Ok(permit) = core.prefetch_budget.clone().try_acquire_owned() else {
|
||||
break; // the budget is spent; the demand path can have it
|
||||
};
|
||||
@@ -654,10 +862,10 @@ impl Core {
|
||||
set.spawn(async move {
|
||||
let _permit = permit;
|
||||
if let Err(error) = core
|
||||
.fetch_block(&reader, &uid, mtime, fsize, bidx, cache_blocks)
|
||||
.fetch_block(&reader, &uid, mtime, fsize, span, cache_blocks)
|
||||
.await
|
||||
{
|
||||
debug!(%uid, bidx, ?error, "prefetch failed");
|
||||
debug!(%uid, bidx = span.idx, ?error, "prefetch failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -687,8 +895,13 @@ impl Core {
|
||||
}
|
||||
let end = offset.saturating_add(len).min(fsize);
|
||||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||||
let first = offset / BLOCK_SIZE;
|
||||
let last = (end - 1) / BLOCK_SIZE;
|
||||
// On the revision's own boundaries where they are known, and on the
|
||||
// 4 MiB fallback where they are not (bugs.md B85).
|
||||
let wanted = self.block_geometry(uid, mtime, fsize).spans(offset, end);
|
||||
let (Some(&head), Some(&tail)) = (wanted.first(), wanted.last()) else {
|
||||
return Ok(out);
|
||||
};
|
||||
let (first, last) = (head.idx, tail.idx);
|
||||
|
||||
// Collect the blocks overlapping the request, serving any already cached
|
||||
// and fetching the rest concurrently. A multi-block read (e.g. a media
|
||||
@@ -696,31 +909,31 @@ impl Core {
|
||||
// would otherwise stall on each block round-trip in turn; downloading the
|
||||
// misses in parallel saturates the connection and bounds latency at the
|
||||
// slowest single block instead of their sum.
|
||||
let mut blocks: Vec<Option<Arc<Vec<u8>>>> = Vec::with_capacity((last - first + 1) as usize);
|
||||
let mut misses: Vec<u64> = Vec::new();
|
||||
for bidx in first..=last {
|
||||
let mut blocks: Vec<Option<Arc<Vec<u8>>>> = Vec::with_capacity(wanted.len());
|
||||
let mut misses: Vec<BlockSpan> = Vec::new();
|
||||
for &span in &wanted {
|
||||
// The in-memory ring comes first on both paths: a streaming read has
|
||||
// no other cache at all, and a cached one would otherwise pay two
|
||||
// opens, a JSON parse and a 4 MiB read+alloc for every 128 KiB the
|
||||
// kernel asks for (audit T3).
|
||||
let hit = self.block_ring.lock().get(uid, mtime, fsize, bidx);
|
||||
let hit = self.block_ring.lock().get(uid, mtime, fsize, span);
|
||||
let hit = hit.or_else(|| {
|
||||
if !cache_blocks {
|
||||
return None;
|
||||
}
|
||||
let bytes = Arc::new(self.cache.cached_block(uid, mtime, fsize, bidx)?);
|
||||
let bytes = Arc::new(self.cache.cached_block(uid, mtime, fsize, span)?);
|
||||
// Promote it, so the other ~31 reads of this block are answered
|
||||
// from memory.
|
||||
self.block_ring
|
||||
.lock()
|
||||
.insert(uid, mtime, fsize, bidx, bytes.clone());
|
||||
.insert(uid, mtime, fsize, span, bytes.clone());
|
||||
Some(bytes)
|
||||
});
|
||||
match hit {
|
||||
Some(b) => blocks.push(Some(b)),
|
||||
None => {
|
||||
blocks.push(None);
|
||||
misses.push(bidx);
|
||||
misses.push(span);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -733,12 +946,12 @@ impl Core {
|
||||
let reader = self.revision_reader(uid, mtime, fsize).await?;
|
||||
|
||||
let mut set = tokio::task::JoinSet::new();
|
||||
for &bidx in &misses {
|
||||
for &span in &misses {
|
||||
let (core, reader, uid) = (self.clone(), reader.clone(), uid.clone());
|
||||
set.spawn(async move {
|
||||
core.fetch_block(&reader, &uid, mtime, fsize, bidx, cache_blocks)
|
||||
core.fetch_block(&reader, &uid, mtime, fsize, span, cache_blocks)
|
||||
.await
|
||||
.map(|bytes| (bidx, bytes))
|
||||
.map(|bytes| (span.idx, bytes))
|
||||
});
|
||||
}
|
||||
let mut out = Vec::with_capacity(misses.len());
|
||||
@@ -756,11 +969,22 @@ impl Core {
|
||||
// ahead of a sequential reader instead of stalling once it catches up.
|
||||
self.prefetch_ahead(uid, mtime, fsize, first, last, cache_blocks);
|
||||
|
||||
for (i, block) in blocks.into_iter().enumerate() {
|
||||
let bidx = first + i as u64;
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
for (span, block) in wanted.into_iter().zip(blocks) {
|
||||
let bstart = span.start;
|
||||
// Every slot is populated: cache hits up front, misses by the fetch above.
|
||||
let block = block.expect("block fetched or cached");
|
||||
// Both sources guarantee the length the geometry says the block has
|
||||
// — the cache validates it on read, the network path in `read_block`
|
||||
// — so this refuses rather than quietly contributing fewer bytes
|
||||
// than the range covers, which is what turned a bad block into a
|
||||
// truncated file (bugs.md B84).
|
||||
if block.len() as u64 != span.len {
|
||||
error!(
|
||||
%uid, bidx = span.idx, len = block.len(), expected = span.len, fsize,
|
||||
"assembling a read hit a short block"
|
||||
);
|
||||
return Err(Errno::EIO);
|
||||
}
|
||||
let s = (offset.max(bstart) - bstart) as usize;
|
||||
let e = (end.min(bstart + block.len() as u64) - bstart) as usize;
|
||||
if s < e {
|
||||
@@ -850,13 +1074,13 @@ impl Core {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||||
for bidx in (offset / BLOCK_SIZE)..=((end - 1) / BLOCK_SIZE) {
|
||||
let ring = self.block_ring.lock().get(uid, mtime, fsize, bidx);
|
||||
for span in self.block_geometry(uid, mtime, fsize).spans(offset, end) {
|
||||
let ring = self.block_ring.lock().get(uid, mtime, fsize, span);
|
||||
let block = match ring {
|
||||
Some(bytes) => bytes,
|
||||
None => Arc::new(self.cache.cached_block(uid, mtime, fsize, bidx)?),
|
||||
None => Arc::new(self.cache.cached_block(uid, mtime, fsize, span)?),
|
||||
};
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
let bstart = span.start;
|
||||
let s = (offset.max(bstart) - bstart) as usize;
|
||||
let e = (end.min(bstart + block.len() as u64) - bstart) as usize;
|
||||
if s >= e {
|
||||
@@ -976,31 +1200,42 @@ mod block_ring_tests {
|
||||
RING_BYTES / BLOCK_SIZE
|
||||
}
|
||||
|
||||
/// Block `idx` under the uniform geometry — the shape these tests are
|
||||
/// written against, since what they exercise is tagging and eviction rather
|
||||
/// than block layout.
|
||||
fn span(idx: u64) -> BlockSpan {
|
||||
BlockSpan {
|
||||
idx,
|
||||
start: idx * BLOCK_SIZE,
|
||||
len: BLOCK_SIZE,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stored_block_is_served_back() {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
ring.insert(&node, 7, 100, 3, block(0xab));
|
||||
assert_eq!(ring.get(&node, 7, 100, 3).map(|b| b[0]), Some(0xab));
|
||||
ring.insert(&node, 7, 100, span(3), block(0xab));
|
||||
assert_eq!(ring.get(&node, 7, 100, span(3)).map(|b| b[0]), Some(0xab));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_block_of_another_revision_is_never_served() {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
ring.insert(&node, 7, 100, 0, block(1));
|
||||
ring.insert(&node, 7, 100, span(0), block(1));
|
||||
// Same node, newer revision: the tag moved, so the old bytes go.
|
||||
assert!(ring.get(&node, 8, 100, 0).is_none());
|
||||
assert!(ring.get(&node, 7, 101, 0).is_none());
|
||||
assert!(ring.get(&node, 8, 100, span(0)).is_none());
|
||||
assert!(ring.get(&node, 7, 101, span(0)).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_revision_drops_what_the_old_one_left() {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
ring.insert(&node, 7, 100, 0, block(1));
|
||||
ring.insert(&node, 8, 100, 0, block(2));
|
||||
assert_eq!(ring.get(&node, 8, 100, 0).map(|b| b[0]), Some(2));
|
||||
ring.insert(&node, 7, 100, span(0), block(1));
|
||||
ring.insert(&node, 8, 100, span(0), block(2));
|
||||
assert_eq!(ring.get(&node, 8, 100, span(0)).map(|b| b[0]), Some(2));
|
||||
assert_eq!(ring.bytes, BLOCK_SIZE);
|
||||
}
|
||||
|
||||
@@ -1009,13 +1244,13 @@ mod block_ring_tests {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
for idx in 0..capacity() {
|
||||
ring.insert(&node, 1, u64::MAX, idx, block(idx as u8));
|
||||
ring.insert(&node, 1, u64::MAX, span(idx), block(idx as u8));
|
||||
}
|
||||
// Re-reading block 0 makes block 1 the oldest instead.
|
||||
assert!(ring.get(&node, 1, u64::MAX, 0).is_some());
|
||||
ring.insert(&node, 1, u64::MAX, capacity(), block(0xff));
|
||||
assert!(ring.get(&node, 1, u64::MAX, 0).is_some());
|
||||
assert!(ring.get(&node, 1, u64::MAX, 1).is_none());
|
||||
assert!(ring.get(&node, 1, u64::MAX, span(0)).is_some());
|
||||
ring.insert(&node, 1, u64::MAX, span(capacity()), block(0xff));
|
||||
assert!(ring.get(&node, 1, u64::MAX, span(0)).is_some());
|
||||
assert!(ring.get(&node, 1, u64::MAX, span(1)).is_none());
|
||||
assert_eq!(ring.bytes, RING_BYTES);
|
||||
}
|
||||
|
||||
@@ -1023,9 +1258,9 @@ mod block_ring_tests {
|
||||
fn a_node_whose_last_block_ages_out_keeps_no_tag() {
|
||||
let mut ring = BlockRing::default();
|
||||
let (old, new) = (uid("old"), uid("new"));
|
||||
ring.insert(&old, 1, u64::MAX, 0, block(1));
|
||||
ring.insert(&old, 1, u64::MAX, span(0), block(1));
|
||||
for idx in 0..capacity() {
|
||||
ring.insert(&new, 1, u64::MAX, idx, block(2));
|
||||
ring.insert(&new, 1, u64::MAX, span(idx), block(2));
|
||||
}
|
||||
assert!(!ring.tags.contains_key(&old));
|
||||
assert!(ring.tags.contains_key(&new));
|
||||
@@ -1035,11 +1270,11 @@ mod block_ring_tests {
|
||||
fn dropping_a_node_frees_only_its_own_bytes() {
|
||||
let mut ring = BlockRing::default();
|
||||
let (a, b) = (uid("a"), uid("b"));
|
||||
ring.insert(&a, 1, 100, 0, block(1));
|
||||
ring.insert(&b, 1, 100, 0, block(2));
|
||||
ring.insert(&a, 1, 100, span(0), block(1));
|
||||
ring.insert(&b, 1, 100, span(0), block(2));
|
||||
ring.drop_node(&a);
|
||||
assert!(ring.get(&a, 1, 100, 0).is_none());
|
||||
assert_eq!(ring.get(&b, 1, 100, 0).map(|x| x[0]), Some(2));
|
||||
assert!(ring.get(&a, 1, 100, span(0)).is_none());
|
||||
assert_eq!(ring.get(&b, 1, 100, span(0)).map(|x| x[0]), Some(2));
|
||||
assert_eq!(ring.bytes, BLOCK_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
//! One shutdown signal shared by every long-lived worker in the daemon.
|
||||
//!
|
||||
//! The mount used to end by unmounting and returning, leaving the drain
|
||||
//! workers, the sync engine, the conflict sweep, the online probe and the local
|
||||
//! indexer running on `Core` clones. In a process that exits immediately that is
|
||||
//! invisible; in an in-process remount — which is how the daemon recovers from a
|
||||
//! mount going away underneath it — it leaks a whole generation of threads that
|
||||
//! keep writing to the database and calling the API on behalf of a mount that no
|
||||
//! longer exists (bugs.md B44).
|
||||
//!
|
||||
//! The primitive is deliberately small: a flag plus a condvar. Every loop that
|
||||
//! would otherwise call `thread::sleep` calls [`Shutdown::sleep`] instead, which
|
||||
//! returns early — and reports `false` — the moment [`Shutdown::stop`] is
|
||||
//! called, so teardown does not have to wait out a five-minute sweep interval to
|
||||
//! join a thread.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use parking_lot::{Condvar, Mutex};
|
||||
|
||||
/// The daemon's stop signal. Set once, never cleared: a `Core` that has been
|
||||
/// told to stop belongs to a mount that is going away.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Shutdown {
|
||||
stopping: Mutex<bool>,
|
||||
changed: Condvar,
|
||||
}
|
||||
|
||||
impl Shutdown {
|
||||
/// Whether teardown has begun. Loops check this where they would otherwise
|
||||
/// start another unit of work.
|
||||
pub(crate) fn is_stopping(&self) -> bool {
|
||||
*self.stopping.lock()
|
||||
}
|
||||
|
||||
/// Begin teardown and wake everything waiting in [`sleep`](Self::sleep).
|
||||
/// Idempotent, because more than one path can reach teardown (a signal, the
|
||||
/// kernel mount ending, a failed startup).
|
||||
pub(crate) fn stop(&self) {
|
||||
let mut stopping = self.stopping.lock();
|
||||
*stopping = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
|
||||
/// Wait up to `duration`, or until teardown starts.
|
||||
///
|
||||
/// Returns `false` if the daemon is stopping — including when it was already
|
||||
/// stopping on entry, so `while shutdown.sleep(interval)` is a complete and
|
||||
/// correct loop condition and no separate check is needed at the top.
|
||||
pub(crate) fn sleep(&self, duration: Duration) -> bool {
|
||||
let mut stopping = self.stopping.lock();
|
||||
if !*stopping {
|
||||
self.changed.wait_for(&mut stopping, duration);
|
||||
}
|
||||
!*stopping
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
#[test]
|
||||
fn sleeping_returns_early_once_stopped() {
|
||||
let shutdown = Arc::new(Shutdown::default());
|
||||
assert!(!shutdown.is_stopping());
|
||||
|
||||
let waker = shutdown.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
waker.stop();
|
||||
});
|
||||
|
||||
let started = Instant::now();
|
||||
// An hour's worth of interval, cut short by the stop.
|
||||
assert!(!shutdown.sleep(Duration::from_secs(3600)));
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(5),
|
||||
"stop must interrupt the wait, not wait it out"
|
||||
);
|
||||
assert!(shutdown.is_stopping());
|
||||
// Already stopping: the next wait does not sleep at all.
|
||||
let started = Instant::now();
|
||||
assert!(!shutdown.sleep(Duration::from_secs(3600)));
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleeping_runs_to_the_timeout_while_running() {
|
||||
let shutdown = Shutdown::default();
|
||||
assert!(shutdown.sleep(Duration::from_millis(10)));
|
||||
assert!(!shutdown.is_stopping());
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,11 @@ pub(crate) struct PendingRevision {
|
||||
pub(crate) meta: StagedWrite,
|
||||
}
|
||||
|
||||
/// One directory's entries as `readdir` reports them: `(ino, is_dir, name)` per
|
||||
/// entry, in reply order. The kernel's file-type enum is mapped in at reply
|
||||
/// time so this module stays free of `fuser`.
|
||||
pub(crate) type DirListing = Vec<(u64, bool, String)>;
|
||||
|
||||
/// Mutable inode bookkeeping, guarded by a mutex because fuser drives the
|
||||
/// `Filesystem` trait through `&self`.
|
||||
pub(crate) struct State {
|
||||
@@ -158,6 +163,16 @@ pub(crate) struct State {
|
||||
/// have no entry here.
|
||||
pub(crate) handles: HashMap<u64, u64>,
|
||||
pub(crate) next_fh: u64,
|
||||
/// One frozen listing per open directory handle, taken at `opendir`.
|
||||
///
|
||||
/// `readdir` resumes from a byte offset into the listing it is walking, so
|
||||
/// that listing has to stay the same across the calls of one `getdents`
|
||||
/// sequence. Rebuilding it live meant a create or a trash landing between
|
||||
/// two pages shifted every later entry, and the caller silently skipped or
|
||||
/// repeated whatever crossed the page boundary. Each entry is
|
||||
/// `(ino, is_dir, name)` — the kernel type is mapped at reply time so this
|
||||
/// module stays free of `fuser` (bugs.md B43).
|
||||
pub(crate) dir_snapshots: HashMap<u64, Arc<DirListing>>,
|
||||
/// Resident inode attributes whose effective access changed during an
|
||||
/// intern/root refresh. Core drains this set and notifies the matching
|
||||
/// kernel only after releasing the State lock.
|
||||
@@ -317,6 +332,7 @@ impl State {
|
||||
active_writes: HashMap::new(),
|
||||
handles: HashMap::new(),
|
||||
next_fh: 1,
|
||||
dir_snapshots: HashMap::new(),
|
||||
access_changes: HashSet::new(),
|
||||
share_access,
|
||||
db,
|
||||
|
||||
@@ -67,10 +67,16 @@ impl Core {
|
||||
/// Sweep for stale conflict copies forever, on its own thread. See the module
|
||||
/// docs for the identical/divergent policy.
|
||||
pub(crate) fn run_conflict_sweep_loop(&self) {
|
||||
std::thread::sleep(CONFLICT_SWEEP_WARMUP);
|
||||
// Both waits are interruptible, so teardown does not have to sit out a
|
||||
// five-minute interval to join this thread (bugs.md B44).
|
||||
if !self.shutdown.sleep(CONFLICT_SWEEP_WARMUP) {
|
||||
return;
|
||||
}
|
||||
loop {
|
||||
self.sweep_conflicts_once();
|
||||
std::thread::sleep(CONFLICT_SWEEP_INTERVAL);
|
||||
if !self.shutdown.sleep(CONFLICT_SWEEP_INTERVAL) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+585
-77
@@ -54,7 +54,7 @@ use engine::{classify, settle_with};
|
||||
#[cfg(test)]
|
||||
use local_scan::is_write_mode;
|
||||
use local_scan::open_for_write_set;
|
||||
use model::{LocalItem, Outcome, PassAbort, Pending, RemoteItem};
|
||||
use model::{LocalItem, LocalSig, Outcome, PassAbort, Pending, RemoteItem};
|
||||
pub(crate) use planner::base_name;
|
||||
#[cfg(test)]
|
||||
use planner::conflict_path;
|
||||
@@ -525,6 +525,9 @@ impl Core {
|
||||
continue;
|
||||
}
|
||||
trashed.push(rel.clone());
|
||||
if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -687,6 +690,7 @@ impl Core {
|
||||
rel_path: rel.clone(),
|
||||
remote_uid: Some(remote.uid.to_string()),
|
||||
local_mtime: local.mtime,
|
||||
local_mtime_ns: local.mtime_ns,
|
||||
local_size: local.size,
|
||||
remote_rev: Some(remote.mtime.to_string()),
|
||||
remote_hash: Some(remote.size.to_string()),
|
||||
@@ -717,6 +721,9 @@ impl Core {
|
||||
uid: remote.uid.clone(),
|
||||
mtime: remote.mtime,
|
||||
size: remote.size,
|
||||
// What the walk saw here, so the apply step can tell a
|
||||
// stale plan from a live one before it overwrites.
|
||||
local: l.map(LocalSig::from),
|
||||
});
|
||||
}
|
||||
FilePlan::Conflict => {
|
||||
@@ -739,7 +746,9 @@ impl Core {
|
||||
}
|
||||
},
|
||||
FilePlan::DeleteLocal => {
|
||||
if let Err(e) = std::fs::remove_file(local_root.join(rel_to_path(rel))) {
|
||||
if let Err(e) =
|
||||
removed_locally(std::fs::remove_file(local_root.join(rel_to_path(rel))))
|
||||
{
|
||||
warn!(rel, error = %e, "sync: remove local file failed");
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -771,6 +780,9 @@ impl Core {
|
||||
outcome.errors += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -797,7 +809,9 @@ impl Core {
|
||||
// Deferred folder deletions, deepest first.
|
||||
delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count()));
|
||||
for rel in delete_local_dirs {
|
||||
if let Err(e) = std::fs::remove_dir_all(local_root.join(rel_to_path(&rel))) {
|
||||
if let Err(e) =
|
||||
removed_locally(std::fs::remove_dir_all(local_root.join(rel_to_path(&rel))))
|
||||
{
|
||||
warn!(rel, error = %e, "sync: remove local folder failed");
|
||||
self.log_activity(ActivityKind::Trash, base_name(&rel), e.to_string(), false);
|
||||
outcome.errors += 1;
|
||||
@@ -824,6 +838,9 @@ impl Core {
|
||||
outcome.errors += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = parent_uid_of(&rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, &rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -849,68 +866,9 @@ impl Core {
|
||||
writing: &HashSet<PathBuf>,
|
||||
out: &mut HashMap<String, LocalItem>,
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||
format!(
|
||||
"local path {} escaped sync root {}: {e}",
|
||||
path.display(),
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let rel = stripped.to_str().ok_or_else(|| {
|
||||
format!(
|
||||
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
// Ignore our own in-flight download temp files.
|
||||
if rel.contains(".pdfs-tmp-") {
|
||||
continue;
|
||||
}
|
||||
// Ignored paths are dropped here as well as from the classification
|
||||
// union — not for correctness (the union filter is what makes this
|
||||
// safe) but so an ignored `node_modules/` is never descended into or
|
||||
// counted against scan progress.
|
||||
if rules.is_ignored(rel, meta.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
if meta.is_dir() {
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
open_for_write: false,
|
||||
},
|
||||
);
|
||||
self.progress_scanned(folder_id, base_name(rel));
|
||||
self.walk_local(folder_id, root, &path, rules, writing, out)?;
|
||||
} else if meta.is_file() {
|
||||
// Check whether any process holds this file open for writing.
|
||||
// The canonical path is used because /proc/*/fd links resolve
|
||||
// to canonical targets.
|
||||
let ofw = std::fs::canonicalize(&path)
|
||||
.map(|canon| writing.contains(&canon))
|
||||
.unwrap_or(false);
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: false,
|
||||
mtime: system_mtime(&meta),
|
||||
size: meta.len() as i64,
|
||||
open_for_write: ofw,
|
||||
},
|
||||
);
|
||||
self.progress_scanned(folder_id, base_name(rel));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
walk_local_tree(root, dir, rules, writing, out, &mut |rel| {
|
||||
self.progress_scanned(folder_id, base_name(rel))
|
||||
})
|
||||
}
|
||||
|
||||
/// Map every remote folder under `folder` to its uid by rel path, ignoring
|
||||
@@ -1069,6 +1027,7 @@ impl Core {
|
||||
rel_path: rel.to_string(),
|
||||
remote_uid: Some(uid.to_string()),
|
||||
local_mtime: 0,
|
||||
local_mtime_ns: None,
|
||||
local_size: 0,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
@@ -1175,6 +1134,9 @@ impl Core {
|
||||
.create_folder(parent, base_name(rel), Some(now_secs()))
|
||||
.await
|
||||
.map_err(|e| format!("create remote folder {rel}: {e}"))?;
|
||||
// The parent has gained a child that no cached listing of it
|
||||
// knows about (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
self.baseline_dir(folder_id, rel, &uid)?;
|
||||
Ok(Applied::Dir(rel.clone(), uid))
|
||||
}
|
||||
@@ -1192,8 +1154,9 @@ impl Core {
|
||||
uid,
|
||||
mtime,
|
||||
size,
|
||||
local,
|
||||
} => {
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size)
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size, *local)
|
||||
.await?;
|
||||
Ok(Applied::Downloaded)
|
||||
}
|
||||
@@ -1211,7 +1174,11 @@ impl Core {
|
||||
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
||||
info!(rel, "sync: kept local changes as a conflict copy");
|
||||
}
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size)
|
||||
// The local copy has just been moved aside, so the state this
|
||||
// download expects to find at the path is "nothing" — anything
|
||||
// that appears while it runs is a *new* racing write, and gets
|
||||
// its own conflict copy rather than being overwritten (B39).
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size, None)
|
||||
.await?;
|
||||
Ok(Applied::Conflict)
|
||||
}
|
||||
@@ -1250,6 +1217,9 @@ impl Core {
|
||||
let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?;
|
||||
let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?;
|
||||
let mtime = system_mtime(&meta);
|
||||
// Taken from the descriptor being streamed, so it names the content that
|
||||
// actually goes up rather than whatever is at the path when we finish.
|
||||
let streamed = LocalSig::from(&meta);
|
||||
// Count the bytes as they are read, so this upload shows up in
|
||||
// `GetQueueStatus` next to a manual one. The uid isn't known until the
|
||||
// draft is sealed, so the transfer registers without one.
|
||||
@@ -1276,7 +1246,13 @@ impl Core {
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("upload {rel}: {e}"))?;
|
||||
self.record_file_baseline(folder_id, rel, &path, &uid).await
|
||||
// The parent has gained a child that no cached listing of it knows
|
||||
// about. Without this, a folder later switched to on-demand serves a
|
||||
// snapshot that predates this upload (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), &uid)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Upload a changed local file as a new revision of an existing remote node.
|
||||
@@ -1291,6 +1267,7 @@ impl Core {
|
||||
let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?;
|
||||
let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?;
|
||||
let mtime = system_mtime(&meta);
|
||||
let streamed = LocalSig::from(&meta);
|
||||
let reader = OwnedCountingReader::new(
|
||||
file,
|
||||
self.transfers.begin(
|
||||
@@ -1304,11 +1281,20 @@ impl Core {
|
||||
.upload_new_revision_from(uid, reader, meta.len() as i64, Vec::new(), Some(mtime))
|
||||
.await
|
||||
.map_err(|e| format!("upload revision {rel}: {e}"))?;
|
||||
self.record_file_baseline(folder_id, rel, &path, uid).await
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), uid)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Download a remote file to its local path (atomically via a temp file),
|
||||
/// stamp the local mtime to match the remote, then record baseline.
|
||||
///
|
||||
/// `expected_local` is the identity planning saw at the destination. The
|
||||
/// rename below is the destructive step, and the download in between can run
|
||||
/// for minutes, so the destination is re-examined right before it: a local
|
||||
/// file that no longer matches the plan is set aside as a conflict copy
|
||||
/// instead of being replaced (bugs.md B39).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn download_file(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
@@ -1317,6 +1303,7 @@ impl Core {
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
expected_local: Option<LocalSig>,
|
||||
) -> Result<(), String> {
|
||||
let path = local_root.join(rel_to_path(rel));
|
||||
if let Some(parent) = path.parent() {
|
||||
@@ -1351,6 +1338,13 @@ impl Core {
|
||||
out.flush().map_err(|e| format!("flush tmp {rel}: {e}"))?;
|
||||
out.sync_all().map_err(|e| format!("sync tmp {rel}: {e}"))?;
|
||||
}
|
||||
// The last moment at which the destination can still be saved. Anything
|
||||
// other than the state planning classified means a writer got there
|
||||
// first, and its bytes are the ones nothing else has a copy of.
|
||||
if let Err(e) = keep_racing_local_edit(&path, expected_local) {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(format!("set aside racing local edit for {rel}: {e}"));
|
||||
}
|
||||
std::fs::rename(&tmp, &path).map_err(|e| format!("place {rel}: {e}"))?;
|
||||
// Match local mtime to the remote's so neither side looks "changed" next pass.
|
||||
let f = std::fs::File::options()
|
||||
@@ -1366,23 +1360,36 @@ impl Core {
|
||||
.and_then(|dir| dir.sync_all())
|
||||
.map_err(|e| format!("sync directory for {rel}: {e}"))?;
|
||||
}
|
||||
self.record_file_baseline(folder_id, rel, &path, uid).await
|
||||
// Here the file on disk *is* the transferred content: it was published
|
||||
// by a rename and stamped just above, so its stat is the identity.
|
||||
self.record_file_baseline(folder_id, rel, local_sig(&path), uid)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Refresh a file's baseline from ground truth: the local stat and the
|
||||
/// remote node's reported signature. Called after every upload/download so
|
||||
/// the very next reconcile sees the path as unchanged.
|
||||
/// Refresh a file's baseline from ground truth: the local identity these
|
||||
/// bytes came from and the remote node's reported signature. Called after
|
||||
/// every upload/download so the very next reconcile sees the path as
|
||||
/// unchanged.
|
||||
///
|
||||
/// `local` is the identity the *transferred content* corresponds to, which
|
||||
/// is not always the file's current stat. An upload streams the live file,
|
||||
/// so it passes the identity it read the first byte at: if a writer changed
|
||||
/// the file mid-stream, the remote now holds a torn revision, and recording
|
||||
/// the pre-upload identity is what makes the next pass see the path as
|
||||
/// locally changed and upload a whole, clean revision over it. Recording the
|
||||
/// post-upload stat instead declared the torn revision synchronized and left
|
||||
/// it as the file's content on Drive (bugs.md B40).
|
||||
async fn record_file_baseline(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
rel: &str,
|
||||
local_path: &Path,
|
||||
local: Option<LocalSig>,
|
||||
uid: &NodeUid,
|
||||
) -> Result<(), String> {
|
||||
let (lmtime, lsize) = match std::fs::metadata(local_path) {
|
||||
Ok(m) => (system_mtime(&m), m.len() as i64),
|
||||
Err(e) => return Err(format!("stat {rel}: {e}")),
|
||||
let Some(local) = local else {
|
||||
return Err(format!("stat {rel}: no local identity to baseline"));
|
||||
};
|
||||
let (lmtime, lsize) = (local.mtime, local.size);
|
||||
// Re-fetch the node so the stored remote signature is exactly what a walk
|
||||
// will report next time.
|
||||
let (rmtime, rsize) = match self.client.enumerate_nodes(std::slice::from_ref(uid)).await {
|
||||
@@ -1402,6 +1409,7 @@ impl Core {
|
||||
rel_path: rel.to_string(),
|
||||
remote_uid: Some(uid.to_string()),
|
||||
local_mtime: lmtime,
|
||||
local_mtime_ns: local.mtime_ns,
|
||||
local_size: lsize,
|
||||
remote_rev: Some(rmtime.to_string()),
|
||||
remote_hash: Some(rsize.to_string()),
|
||||
@@ -1411,6 +1419,177 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Say so when a file changed under an upload that was streaming it.
|
||||
///
|
||||
/// Purely diagnostic — the correction is already in place, because the baseline
|
||||
/// records the identity the upload *read* rather than the one on disk now, so
|
||||
/// the next pass classifies the path as locally changed and replaces the torn
|
||||
/// revision. Without the log there is nothing in the journal to explain the
|
||||
/// extra revision (bugs.md B40).
|
||||
fn warn_if_torn(rel: &str, path: &Path, streamed: LocalSig) {
|
||||
let now = local_sig(path);
|
||||
if !now.is_some_and(|now| now.same_content(&streamed)) {
|
||||
warn!(
|
||||
rel,
|
||||
?streamed,
|
||||
?now,
|
||||
"sync: file changed while uploading; the remote revision may be torn \
|
||||
and will be replaced on the next pass"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The identity a reconcile pass uses to recognise a local file it has already
|
||||
/// looked at: `None` when nothing is at the path.
|
||||
fn local_sig(path: &Path) -> Option<LocalSig> {
|
||||
std::fs::metadata(path).ok().map(|m| LocalSig {
|
||||
mtime: system_mtime(&m),
|
||||
mtime_ns: system_mtime_ns(&m),
|
||||
size: m.len() as i64,
|
||||
})
|
||||
}
|
||||
|
||||
/// Guard the destination of a download: if what is at `path` is no longer what
|
||||
/// planning classified, move it aside as a conflict copy before the caller
|
||||
/// renames over it.
|
||||
///
|
||||
/// A reconcile pass decides what to do from a walk that finished before the
|
||||
/// transfer started, and a download of a large file can run for minutes. Without
|
||||
/// this, a local edit completed inside that window was replaced by the remote
|
||||
/// version with no copy of it kept anywhere — the one shape of loss a sync
|
||||
/// engine must not have (bugs.md B39). Keeping the copy makes it re-upload as a
|
||||
/// new file on the next pass, which is the same resolution a both-sides-changed
|
||||
/// conflict already gets.
|
||||
///
|
||||
/// A path that is empty when it was expected to be is not a conflict: the
|
||||
/// download is about to recreate it, and there is nothing to preserve.
|
||||
fn keep_racing_local_edit(path: &Path, expected: Option<LocalSig>) -> std::io::Result<()> {
|
||||
let actual = local_sig(path);
|
||||
let unchanged = match (actual, expected) {
|
||||
(Some(a), Some(e)) => a.same_content(&e),
|
||||
// Nothing there: the download is about to create it, and there is
|
||||
// nothing to preserve either way.
|
||||
(None, _) => true,
|
||||
(Some(_), None) => false,
|
||||
};
|
||||
if unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
warn!(
|
||||
path = %path.display(),
|
||||
?expected,
|
||||
?actual,
|
||||
"sync: local file changed while downloading; keeping it as a conflict copy"
|
||||
);
|
||||
preserve_conflict_copy(path, now_secs()).map(|_| ())
|
||||
}
|
||||
|
||||
/// Read a local removal as the sync engine has to read it: the *state* it was
|
||||
/// asking for is "this path is gone", so a path that was already gone is a
|
||||
/// success, not a failure.
|
||||
///
|
||||
/// It matters because the baseline is only dropped on success. Treating
|
||||
/// `NotFound` as an error left the baseline in place for a file that no longer
|
||||
/// exists, so the same removal was re-attempted and re-reported as an error on
|
||||
/// every pass, forever. Every other error is still an error, and keeping the
|
||||
/// baseline is exactly right there: the surviving file must not be mistaken for
|
||||
/// a new local creation on the next pass and uploaded back (bugs.md B42).
|
||||
fn removed_locally(result: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match result {
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// The body of [`Core::walk_local`], without a `Core`: recursively collect the
|
||||
/// local tree under `dir` into `out`, reporting each entry to `scanned`.
|
||||
///
|
||||
/// **Every failure here is fatal to the pass, deliberately.** A `read_dir` that
|
||||
/// fails, an entry that cannot be stat-ed, a name that is not UTF-8 — each of
|
||||
/// them would, if skipped, leave `out` describing a *smaller* tree than the one
|
||||
/// on disk. Reconciliation reads a path present in the baseline and absent from
|
||||
/// the scan as "the user deleted this", and propagates that to Drive. So an
|
||||
/// unreadable subdirectory would trash its remote counterpart, and the deeper
|
||||
/// the failure, the more it takes with it (`docs/BUGS.md` B55). Refusing the
|
||||
/// whole pass keeps the previous baseline and leaves the remote alone; the next
|
||||
/// pass retries once the transient condition clears.
|
||||
///
|
||||
/// Split out from the method so exactly that can be tested against a real tree
|
||||
/// with a real unreadable directory in it, which needs no `Core` at all.
|
||||
fn walk_local_tree(
|
||||
root: &Path,
|
||||
dir: &Path,
|
||||
rules: &IgnoreRules,
|
||||
writing: &HashSet<PathBuf>,
|
||||
out: &mut HashMap<String, LocalItem>,
|
||||
scanned: &mut dyn FnMut(&str),
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||
format!(
|
||||
"local path {} escaped sync root {}: {e}",
|
||||
path.display(),
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let rel = stripped.to_str().ok_or_else(|| {
|
||||
format!(
|
||||
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
// Ignore our own in-flight download temp files.
|
||||
if rel.contains(".pdfs-tmp-") {
|
||||
continue;
|
||||
}
|
||||
// Ignored paths are dropped here as well as from the classification
|
||||
// union — not for correctness (the union filter is what makes this
|
||||
// safe) but so an ignored `node_modules/` is never descended into or
|
||||
// counted against scan progress.
|
||||
if rules.is_ignored(rel, meta.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
if meta.is_dir() {
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
mtime_ns: None,
|
||||
size: 0,
|
||||
open_for_write: false,
|
||||
},
|
||||
);
|
||||
scanned(rel);
|
||||
walk_local_tree(root, &path, rules, writing, out, scanned)?;
|
||||
} else if meta.is_file() {
|
||||
// Check whether any process holds this file open for writing.
|
||||
// The canonical path is used because /proc/*/fd links resolve
|
||||
// to canonical targets.
|
||||
let ofw = std::fs::canonicalize(&path)
|
||||
.map(|canon| writing.contains(&canon))
|
||||
.unwrap_or(false);
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: false,
|
||||
mtime: system_mtime(&meta),
|
||||
mtime_ns: system_mtime_ns(&meta),
|
||||
size: meta.len() as i64,
|
||||
open_for_write: ofw,
|
||||
},
|
||||
);
|
||||
scanned(rel);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publish a conflict copy without overwriting an earlier conflict bearing the
|
||||
/// same timestamp. The original is removed only after the new copy is durable.
|
||||
fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||
@@ -1444,6 +1623,26 @@ fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||
unreachable!("u32 conflict suffix space exhausted")
|
||||
}
|
||||
|
||||
/// The remote uid of `rel`'s parent folder: from the pass's folder-uid map
|
||||
/// first, and from the baseline second for a parent the map never covered.
|
||||
///
|
||||
/// Used to name the folder whose cached listing a remote mutation has just
|
||||
/// invalidated (B86). `None` is not an error — it only means this pass cannot
|
||||
/// say which folder to invalidate, and the caller skips it.
|
||||
fn parent_uid_of(
|
||||
rel: &str,
|
||||
remote_dirs: &HashMap<String, NodeUid>,
|
||||
baseline: &HashMap<String, StoredSyncEntry>,
|
||||
) -> Option<NodeUid> {
|
||||
let parent = parent_rel(rel);
|
||||
remote_dirs.get(parent).cloned().or_else(|| {
|
||||
baseline
|
||||
.get(parent)
|
||||
.and_then(|e| e.remote_uid.as_deref())
|
||||
.and_then(parse_uid)
|
||||
})
|
||||
}
|
||||
|
||||
fn sync_parent_dir(path: &Path) -> std::io::Result<()> {
|
||||
match path.parent() {
|
||||
Some(parent) => std::fs::File::open(parent)?.sync_all(),
|
||||
@@ -1476,10 +1675,77 @@ fn system_mtime(meta: &std::fs::Metadata) -> i64 {
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// The same modification time in nanoseconds, when the filesystem reports one.
|
||||
///
|
||||
/// Whole seconds cannot distinguish an edit made in the same second as the last
|
||||
/// sync from no edit at all, and if the size did not change either, that file
|
||||
/// was never uploaded (bugs.md B25). `None` for a clock or filesystem that
|
||||
/// cannot answer, which falls the comparison back to seconds rather than
|
||||
/// asserting an mtime of zero.
|
||||
fn system_mtime_ns(meta: &std::fs::Metadata) -> Option<i64> {
|
||||
meta.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok())
|
||||
.and_then(|d| i64::try_from(d.as_nanos()).ok())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A removal of something already gone has produced the state the sync
|
||||
/// engine wanted, so its baseline is dropped rather than retried forever.
|
||||
/// Anything else stays an error and keeps the baseline (bugs.md B42).
|
||||
#[test]
|
||||
fn an_already_absent_path_counts_as_removed() {
|
||||
let root = std::env::temp_dir().join(format!("pdfs-b42-{}", std::process::id()));
|
||||
let missing = root.join("never-existed");
|
||||
assert!(removed_locally(std::fs::remove_file(&missing)).is_ok());
|
||||
assert!(removed_locally(std::fs::remove_dir_all(&missing)).is_ok());
|
||||
|
||||
// A directory in the way of `remove_file` is a real failure, and must
|
||||
// stay one: the file survives, and its baseline has to survive with it.
|
||||
let occupied = root.join("a-directory");
|
||||
std::fs::create_dir_all(&occupied).unwrap();
|
||||
assert!(removed_locally(std::fs::remove_file(&occupied)).is_err());
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// The download guard: a destination that still matches the plan is
|
||||
/// replaced silently, one that has moved on since is kept (bugs.md B39).
|
||||
#[test]
|
||||
fn a_download_keeps_a_local_edit_it_did_not_plan_for() {
|
||||
let root = std::env::temp_dir().join(format!("pdfs-b39-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
let path = root.join("report.txt");
|
||||
|
||||
// Nothing there, nothing expected: the download simply creates it.
|
||||
assert!(keep_racing_local_edit(&path, None).is_ok());
|
||||
assert!(!path.exists());
|
||||
|
||||
// Unchanged since planning: no copy, the caller overwrites it.
|
||||
std::fs::write(&path, b"planned").unwrap();
|
||||
let planned = local_sig(&path).unwrap();
|
||||
keep_racing_local_edit(&path, Some(planned)).unwrap();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"planned");
|
||||
|
||||
// Changed since planning: the racing bytes are the only copy of
|
||||
// themselves, so they must survive the overwrite as a conflict copy.
|
||||
std::fs::write(&path, b"a much longer local edit").unwrap();
|
||||
keep_racing_local_edit(&path, Some(planned)).unwrap();
|
||||
let kept: Vec<_> = std::fs::read_dir(&root)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| std::fs::read(e.path()).unwrap_or_default())
|
||||
.collect();
|
||||
assert!(
|
||||
kept.iter()
|
||||
.any(|bytes| bytes == b"a much longer local edit"),
|
||||
"the racing edit must still exist somewhere: {kept:?}"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// Feed `n` `Reconcile` events `gap` apart on another thread, starting
|
||||
/// immediately. Returns the sender's join handle so the test can keep it
|
||||
/// alive (dropping the sender early would look like a disconnect).
|
||||
@@ -1565,6 +1831,7 @@ mod tests {
|
||||
|
||||
fn local_item(is_dir: bool) -> LocalItem {
|
||||
LocalItem {
|
||||
mtime_ns: None,
|
||||
is_dir,
|
||||
mtime: 1,
|
||||
size: 1,
|
||||
@@ -1722,6 +1989,44 @@ mod tests {
|
||||
assert!(guard_local_wipe(&base, &fresh).is_err());
|
||||
}
|
||||
|
||||
/// B54's remaining case, end to end: a folder whose local root is still
|
||||
/// *readable* but is no longer the tree it was — an empty directory left
|
||||
/// where a mount used to be, or a different device mounted over it.
|
||||
///
|
||||
/// Nothing fails here. The scan succeeds and honestly reports an empty
|
||||
/// directory, so the only thing standing between the user and a trashed
|
||||
/// remote folder is the guard, and it has to fire for a one-file baseline
|
||||
/// exactly as it does for a large one.
|
||||
#[test]
|
||||
fn a_replaced_mountpoint_trips_the_guard_at_every_baseline_size() {
|
||||
let root = sync_test_dir("wipe-replaced-mount");
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut local: HashMap<String, LocalItem> = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap();
|
||||
assert!(local.is_empty(), "the replacement root is genuinely empty");
|
||||
|
||||
for size in 1..=3 {
|
||||
let baseline: HashMap<String, ()> =
|
||||
(0..size).map(|i| (format!("file{i}.txt"), ())).collect();
|
||||
assert!(
|
||||
guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_err(),
|
||||
"a {size}-path baseline against an empty root is not a deletion"
|
||||
);
|
||||
}
|
||||
|
||||
// And the guard steps aside the moment the real tree is back, so a
|
||||
// remounted folder resumes rather than staying wedged.
|
||||
std::fs::write(root.join("file0.txt"), b"back").unwrap();
|
||||
let mut local: HashMap<String, LocalItem> = HashMap::new();
|
||||
walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap();
|
||||
let baseline: HashMap<String, ()> = [("file0.txt".to_string(), ())].into_iter().collect();
|
||||
assert!(guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_ok());
|
||||
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
/// A restored folder (features.md 5.2) starts with an empty baseline against
|
||||
/// an empty local directory and a full remote. That must reconcile as
|
||||
/// "download everything" rather than tripping the wipe guard — otherwise
|
||||
@@ -1764,6 +2069,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 0,
|
||||
local_size: 0,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: Some(mtime.to_string()),
|
||||
remote_hash: Some(size.to_string()),
|
||||
},
|
||||
@@ -1801,6 +2107,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 0,
|
||||
local_size: 0,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
},
|
||||
@@ -1815,6 +2122,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 10,
|
||||
local_size: 20,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: Some("1700".into()),
|
||||
remote_hash: Some("4096".into()),
|
||||
};
|
||||
@@ -1828,6 +2136,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 0,
|
||||
local_size: 0,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
};
|
||||
@@ -1863,6 +2172,7 @@ mod tests {
|
||||
uid: uid(),
|
||||
mtime: 0,
|
||||
size: 10,
|
||||
local: None,
|
||||
};
|
||||
assert_eq!(op.kind(), ActivityKind::Download);
|
||||
}
|
||||
@@ -1918,6 +2228,204 @@ mod tests {
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
/// B49's remaining cases. Preservation is the step that makes a download
|
||||
/// safe to publish, so every way it can fail has to leave the local file
|
||||
/// where it was — the download is aborted, not the file replaced.
|
||||
#[test]
|
||||
fn conflict_preservation_failures_all_keep_the_local_file() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
// An unwritable directory: the conflict copy cannot be created at all.
|
||||
let dir = sync_test_dir("conflict-eacces");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"the only copy").unwrap();
|
||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o500)).unwrap();
|
||||
let denied = preserve_conflict_copy(&source, 1700);
|
||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(denied.is_err(), "a read-only directory must fail the copy");
|
||||
assert_eq!(
|
||||
std::fs::read(&source).unwrap(),
|
||||
b"the only copy",
|
||||
"the local file survives byte-identical"
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
// A source that vanished between planning and preservation. The pass
|
||||
// must report it rather than publish the download over the gap.
|
||||
let dir = sync_test_dir("conflict-enoent");
|
||||
assert!(preserve_conflict_copy(&dir.join("gone.txt"), 1700).is_err());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
// A write error partway through the copy — induced by making the
|
||||
// destination name a directory, so `create_new` fails as something
|
||||
// other than `AlreadyExists`.
|
||||
let dir = sync_test_dir("conflict-eisdir");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"still the only copy").unwrap();
|
||||
std::fs::create_dir(conflict_path(&source, 1700)).unwrap();
|
||||
// Suffix 0 is taken by a directory; `create_new` on it reports
|
||||
// `AlreadyExists`, so preservation moves on to suffix 1 and succeeds.
|
||||
// What must not happen is the source being removed without a copy.
|
||||
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||
assert!(preserved.is_file());
|
||||
assert_eq!(std::fs::read(&preserved).unwrap(), b"still the only copy");
|
||||
assert!(!source.exists());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
/// A conflict copy keeps the bytes *and* the metadata that identifies them,
|
||||
/// so the next pass uploads the user's version rather than a file that looks
|
||||
/// freshly written.
|
||||
#[test]
|
||||
fn a_conflict_copy_is_byte_and_mtime_identical() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let dir = sync_test_dir("conflict-identity");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"local edit that must survive").unwrap();
|
||||
let before = std::fs::metadata(&source).unwrap();
|
||||
|
||||
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||
let after = std::fs::metadata(&preserved).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read(&preserved).unwrap(),
|
||||
b"local edit that must survive"
|
||||
);
|
||||
assert_eq!(after.len(), before.len());
|
||||
assert_eq!(after.modified().unwrap(), before.modified().unwrap());
|
||||
assert_eq!(after.permissions().mode(), before.permissions().mode());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
// ---- B55: an incomplete local scan is never a set of deletions ---------
|
||||
|
||||
fn no_progress() -> impl FnMut(&str) {
|
||||
|_: &str| {}
|
||||
}
|
||||
|
||||
/// A directory the pass cannot read must fail the pass. Omitting it would
|
||||
/// hand reconciliation a baseline-minus-subtree, which it reads as the user
|
||||
/// having deleted that subtree and propagates to Drive (B55).
|
||||
#[test]
|
||||
fn an_unreadable_subdirectory_fails_the_scan_instead_of_shrinking_it() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-eacces");
|
||||
std::fs::write(root.join("keep.txt"), b"a").unwrap();
|
||||
let locked = root.join("locked");
|
||||
std::fs::create_dir(&locked).unwrap();
|
||||
std::fs::write(locked.join("inner.txt"), b"b").unwrap();
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let error = result.expect_err("an unreadable directory must abort the pass");
|
||||
assert!(
|
||||
error.contains("locked"),
|
||||
"the error should name the subtree it could not read: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The same rule one level up: if the sync root itself cannot be read, the
|
||||
/// pass sees nothing, and "nothing" must never mean "delete everything".
|
||||
#[test]
|
||||
fn an_unreadable_root_fails_the_scan() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-root-eacces");
|
||||
std::fs::write(root.join("keep.txt"), b"a").unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
/// A root that is gone entirely — an unmounted device folder, a removed
|
||||
/// directory — is the sharpest version of the same case: every baseline path
|
||||
/// is "missing", and the scan must refuse rather than report an empty tree.
|
||||
#[test]
|
||||
fn an_absent_root_fails_the_scan() {
|
||||
let root = sync_test_dir("scan-absent");
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
assert!(walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress).is_err());
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
/// A name that is not valid UTF-8 cannot be keyed into the scan, so it
|
||||
/// cannot be compared against the baseline either. Refusing the pass is the
|
||||
/// only safe reading: skipping it would classify the file as deleted.
|
||||
#[test]
|
||||
fn a_non_utf8_name_fails_the_scan() {
|
||||
use std::ffi::OsStr;
|
||||
use std::os::unix::ffi::OsStrExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-non-utf8");
|
||||
std::fs::write(root.join(OsStr::from_bytes(b"bad\xffname")), b"x").unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let error = result.expect_err("a name the scan cannot key must abort the pass");
|
||||
assert!(error.contains("UTF-8"), "{error}");
|
||||
}
|
||||
|
||||
/// The complement: a scan that *can* read everything reports everything, so
|
||||
/// the strictness above does not come at the cost of a working pass. A
|
||||
/// symlink is skipped by design rather than by failure, and a dangling one
|
||||
/// stats fine through `symlink_metadata`.
|
||||
#[test]
|
||||
fn a_readable_tree_scans_completely() {
|
||||
let root = sync_test_dir("scan-complete");
|
||||
std::fs::write(root.join("a.txt"), b"aa").unwrap();
|
||||
std::fs::create_dir(root.join("sub")).unwrap();
|
||||
std::fs::write(root.join("sub/b.txt"), b"bbb").unwrap();
|
||||
std::os::unix::fs::symlink("nowhere", root.join("dangling")).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut |rel| {
|
||||
seen.push(rel.to_string())
|
||||
});
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
result.unwrap();
|
||||
let mut keys: Vec<&str> = out.keys().map(String::as_str).collect();
|
||||
keys.sort_unstable();
|
||||
assert_eq!(keys, vec!["a.txt", "sub", "sub/b.txt"]);
|
||||
assert_eq!(out["a.txt"].size, 2);
|
||||
assert_eq!(out["sub/b.txt"].size, 3);
|
||||
assert!(out["sub"].is_dir);
|
||||
assert_eq!(seen.len(), 3, "every reported entry ticks scan progress");
|
||||
}
|
||||
|
||||
fn sync_test_dir(label: &str) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
|
||||
@@ -32,17 +32,24 @@ pub(crate) enum SyncMsg {
|
||||
ReconcileAll,
|
||||
/// Re-read the folder set and adjust the filesystem watches (after add/remove).
|
||||
Rewatch,
|
||||
/// The daemon is tearing down: finish the current pass and end the engine
|
||||
/// thread, so the mount can join it (bugs.md B44).
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Start the sync engine: a control thread that owns the filesystem watcher and
|
||||
/// serialises reconcile passes, plus a periodic poll thread. The engine receives
|
||||
/// on `rx`; senders live in [`Core::sync_tx`].
|
||||
pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) {
|
||||
if let Err(e) = std::thread::Builder::new()
|
||||
pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) -> Option<std::thread::JoinHandle<()>> {
|
||||
match std::thread::Builder::new()
|
||||
.name("pdfs-sync".into())
|
||||
.spawn(move || engine_loop(core, rx))
|
||||
{
|
||||
warn!(error = %e, "failed to start sync engine");
|
||||
Ok(handle) => Some(handle),
|
||||
Err(e) => {
|
||||
warn!(error = %e, "failed to start sync engine");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,11 +81,11 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
// Periodic remote poll.
|
||||
{
|
||||
let tx = core.sync_tx.clone();
|
||||
let shutdown = core.shutdown.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-sync-poll".into())
|
||||
.spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(POLL_INTERVAL);
|
||||
while shutdown.sleep(POLL_INTERVAL) {
|
||||
if tx.send(SyncMsg::ReconcileAll).is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -94,8 +101,12 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
let mut ids: HashSet<i64> = HashSet::new();
|
||||
let mut all = false;
|
||||
let mut do_rewatch = false;
|
||||
classify(msg, &mut ids, &mut all, &mut do_rewatch);
|
||||
settle(&rx, &mut ids, &mut all, &mut do_rewatch);
|
||||
if !classify(msg, &mut ids, &mut all, &mut do_rewatch)
|
||||
|| !settle(&rx, &mut ids, &mut all, &mut do_rewatch)
|
||||
{
|
||||
info!("sync engine stopping");
|
||||
return;
|
||||
}
|
||||
|
||||
if do_rewatch {
|
||||
rewatch(&core, &mut watcher, &watched);
|
||||
@@ -116,9 +127,9 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
||||
{
|
||||
let tx = core.sync_tx.clone();
|
||||
let shutdown = core.shutdown.clone();
|
||||
std::thread::spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(POLL_INTERVAL);
|
||||
while shutdown.sleep(POLL_INTERVAL) {
|
||||
if tx.send(SyncMsg::ReconcileAll).is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -130,8 +141,12 @@ fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
||||
let mut ids: HashSet<i64> = HashSet::new();
|
||||
let mut all = false;
|
||||
let mut do_rewatch = false;
|
||||
classify(msg, &mut ids, &mut all, &mut do_rewatch);
|
||||
settle(&rx, &mut ids, &mut all, &mut do_rewatch);
|
||||
if !classify(msg, &mut ids, &mut all, &mut do_rewatch)
|
||||
|| !settle(&rx, &mut ids, &mut all, &mut do_rewatch)
|
||||
{
|
||||
info!("sync engine stopping");
|
||||
return;
|
||||
}
|
||||
if all {
|
||||
reconcile_all(core);
|
||||
} else {
|
||||
@@ -158,7 +173,7 @@ pub(super) fn settle_with(
|
||||
do_rewatch: &mut bool,
|
||||
quiet: Duration,
|
||||
cap: Duration,
|
||||
) {
|
||||
) -> bool {
|
||||
let deadline = Instant::now() + cap;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
@@ -170,28 +185,49 @@ pub(super) fn settle_with(
|
||||
match rx.recv_timeout(wait) {
|
||||
// Another event: the burst is still going, so the quiet window
|
||||
// starts again from here.
|
||||
Ok(m) => classify(m, ids, all, do_rewatch),
|
||||
// A stop lands mid-burst: give up the rest of the burst rather
|
||||
// than settling for it, and let the caller end the loop.
|
||||
Ok(m) => {
|
||||
if !classify(m, ids, all, do_rewatch) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Quiet for a full window: whatever was being written has stopped.
|
||||
Err(RecvTimeoutError::Timeout) => break,
|
||||
// Every sender is gone; the caller's loop is about to end anyway.
|
||||
Err(RecvTimeoutError::Disconnected) => break,
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// [`settle_with`] at the real timings.
|
||||
fn settle(rx: &Receiver<SyncMsg>, ids: &mut HashSet<i64>, all: &mut bool, do_rewatch: &mut bool) {
|
||||
settle_with(rx, ids, all, do_rewatch, DEBOUNCE, MAX_SETTLE);
|
||||
fn settle(
|
||||
rx: &Receiver<SyncMsg>,
|
||||
ids: &mut HashSet<i64>,
|
||||
all: &mut bool,
|
||||
do_rewatch: &mut bool,
|
||||
) -> bool {
|
||||
settle_with(rx, ids, all, do_rewatch, DEBOUNCE, MAX_SETTLE)
|
||||
}
|
||||
|
||||
pub(super) fn classify(msg: SyncMsg, ids: &mut HashSet<i64>, all: &mut bool, rewatch: &mut bool) {
|
||||
/// Fold one message into the pending work of the current burst. Returns `false`
|
||||
/// for [`SyncMsg::Stop`], which is not work but an instruction to end the loop.
|
||||
pub(super) fn classify(
|
||||
msg: SyncMsg,
|
||||
ids: &mut HashSet<i64>,
|
||||
all: &mut bool,
|
||||
rewatch: &mut bool,
|
||||
) -> bool {
|
||||
match msg {
|
||||
SyncMsg::Reconcile(id) => {
|
||||
ids.insert(id);
|
||||
}
|
||||
SyncMsg::ReconcileAll => *all = true,
|
||||
SyncMsg::Rewatch => *rewatch = true,
|
||||
SyncMsg::Stop => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Reconcile every mirror folder in turn.
|
||||
|
||||
@@ -2,9 +2,73 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
/// What a reconcile pass remembers about the local side of one path: enough to
|
||||
/// recognise the same content again, and no more.
|
||||
///
|
||||
/// `mtime_ns` is the whole reason this is a struct rather than a pair — see
|
||||
/// [`StoredSyncEntry::local_mtime_ns`](pdfs_core::db::StoredSyncEntry).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) struct LocalSig {
|
||||
pub(super) mtime: i64,
|
||||
pub(super) mtime_ns: Option<i64>,
|
||||
pub(super) size: i64,
|
||||
}
|
||||
|
||||
impl LocalSig {
|
||||
/// Whether these two observations describe the same local content.
|
||||
///
|
||||
/// Nanoseconds decide it when both sides have them; otherwise the comparison
|
||||
/// falls back to whole seconds, which is what a baseline row written before
|
||||
/// schema 28 — or a filesystem that reports no sub-second time — can support
|
||||
/// (bugs.md B25).
|
||||
pub(super) fn same_content(&self, other: &Self) -> bool {
|
||||
if self.size != other.size {
|
||||
return false;
|
||||
}
|
||||
match (self.mtime_ns, other.mtime_ns) {
|
||||
(Some(a), Some(b)) => a == b,
|
||||
_ => self.mtime == other.mtime,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&LocalItem> for LocalSig {
|
||||
fn from(item: &LocalItem) -> Self {
|
||||
Self {
|
||||
mtime: item.mtime,
|
||||
mtime_ns: item.mtime_ns,
|
||||
size: item.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&std::fs::Metadata> for LocalSig {
|
||||
fn from(meta: &std::fs::Metadata) -> Self {
|
||||
Self {
|
||||
mtime: system_mtime(meta),
|
||||
mtime_ns: system_mtime_ns(meta),
|
||||
size: meta.len() as i64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The baseline's record of the local side, as [`LocalSig`] compares it.
|
||||
impl From<&StoredSyncEntry> for LocalSig {
|
||||
fn from(entry: &StoredSyncEntry) -> Self {
|
||||
Self {
|
||||
mtime: entry.local_mtime,
|
||||
mtime_ns: entry.local_mtime_ns,
|
||||
size: entry.local_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct LocalItem {
|
||||
pub(super) is_dir: bool,
|
||||
pub(super) mtime: i64,
|
||||
/// [`LocalItem::mtime`] to nanosecond precision, when the filesystem
|
||||
/// reports one.
|
||||
pub(super) mtime_ns: Option<i64>,
|
||||
pub(super) size: i64,
|
||||
/// True when another process holds this file open for writing (detected via
|
||||
/// `/proc/*/fd`). An open-for-write file is kept in the map — so it is not
|
||||
@@ -91,11 +155,18 @@ pub(super) enum Pending {
|
||||
UploadRevision { rel: String, uid: NodeUid },
|
||||
/// Download remote `uid` to the local path, stamping `mtime`. `size` is the
|
||||
/// remote's reported size, used as the transfer's expected total.
|
||||
///
|
||||
/// `local` is what the planning walk saw at this path, or `None` if nothing
|
||||
/// was there. A download can run for minutes, so the apply step revalidates
|
||||
/// against it immediately before it renames over the destination: a writer
|
||||
/// that finished an edit in the meantime must not be overwritten without a
|
||||
/// conflict copy (bugs.md B39).
|
||||
Download {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
local: Option<LocalSig>,
|
||||
},
|
||||
/// Both sides changed: set the local copy aside, then download remote `uid`.
|
||||
Conflict {
|
||||
|
||||
@@ -218,7 +218,10 @@ pub(super) fn plan_file(
|
||||
}
|
||||
|
||||
let local_changed = local.is_some_and(|item| {
|
||||
baseline.is_none_or(|base| base.local_mtime != item.mtime || base.local_size != item.size)
|
||||
// Nanoseconds where both sides have them (bugs.md B25): whole seconds
|
||||
// could not see an edit made in the same second as the last sync that
|
||||
// left the file the same length, so it was never uploaded.
|
||||
baseline.is_none_or(|base| !LocalSig::from(item).same_content(&LocalSig::from(base)))
|
||||
});
|
||||
let remote_changed = remote.is_some_and(|item| {
|
||||
baseline.is_none_or(|base| remote_sig(base) != Some((item.mtime, item.size)))
|
||||
@@ -252,6 +255,7 @@ mod file_plan_tests {
|
||||
|
||||
fn local(mtime: i64, size: i64) -> LocalItem {
|
||||
LocalItem {
|
||||
mtime_ns: None,
|
||||
is_dir: false,
|
||||
mtime,
|
||||
size,
|
||||
@@ -278,12 +282,49 @@ mod file_plan_tests {
|
||||
rel_path: "file".into(),
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime,
|
||||
local_mtime_ns: None,
|
||||
local_size,
|
||||
remote_rev: Some(remote_mtime.to_string()),
|
||||
remote_hash: Some(remote_size.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whole seconds cannot see an edit made inside the same second that leaves the
|
||||
/// file the same length, so a mirror silently stopped uploading it. With a
|
||||
/// nanosecond time on both sides it is an ordinary local change (bugs.md B25).
|
||||
#[test]
|
||||
fn a_same_second_same_size_edit_is_still_a_local_change() {
|
||||
let mut base = baseline(10, 20, 30, 40);
|
||||
base.local_mtime_ns = Some(10_000_000_000);
|
||||
let edited = LocalItem {
|
||||
mtime_ns: Some(10_400_000_000),
|
||||
..local(10, 20)
|
||||
};
|
||||
let remote = remote(30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&edited), Some(&remote), Some(&base)),
|
||||
FilePlan::UploadRevision
|
||||
);
|
||||
|
||||
// The same file untouched still reads as unchanged.
|
||||
let same = LocalItem {
|
||||
mtime_ns: Some(10_000_000_000),
|
||||
..local(10, 20)
|
||||
};
|
||||
assert_eq!(
|
||||
plan_file(Some(&same), Some(&remote), Some(&base)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
|
||||
// And a row from before the column exists compares on seconds, exactly
|
||||
// as it did when it was written — no mass re-upload on upgrade.
|
||||
let old = baseline(10, 20, 30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&edited), Some(&remote), Some(&old)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_plan_covers_every_presence_and_change_shape() {
|
||||
let same_local = local(10, 20);
|
||||
|
||||
+802
-19
File diff suppressed because it is too large.
Load diff
@@ -9,6 +9,166 @@ release ships. Migrations are forward-only — a database written by a newer cli
|
||||
refuse-to-open, not a downgrade, so rolling back a release means restoring the cache from
|
||||
scratch (user data in `staging/` and `recovery/` is never touched by this).
|
||||
|
||||
## [1.8.2] — 2026-08-17
|
||||
|
||||
Closes the two findings 1.8.1's live verification produced (B85, B86) and finishes the data-safety
|
||||
audit entries that had been sitting as "fixed in code, verification pending" since 1.0 (B49–B55).
|
||||
No schema change. Schema: **28**.
|
||||
|
||||
The read path no longer assumes a revision is a series of 4 MiB blocks; it takes the geometry from
|
||||
the revision. The B85 investigation also **contradicted its own bug report** — see Notes, because
|
||||
the conclusion recorded there was that reads of such files were losing data, and they were not.
|
||||
|
||||
### Fixed
|
||||
- A folder switched from mirror to on-demand now shows the files the mirror pass itself uploaded.
|
||||
The sync engine created nodes on the remote without invalidating the parent's cached child
|
||||
listing, and that listing is a database flag rather than an in-memory cache — so it survived a
|
||||
daemon restart, and the files stayed invisible until the folder was switched back. Every remote
|
||||
mutation the engine makes (folder create, file upload, and the three trash paths) now drops the
|
||||
parent's cached listing by uid, which reaches a folder that no mount has resident — the case a
|
||||
mirror folder always is (`docs/BUGS.md` B86).
|
||||
- `pdfs ls` and `pdfs refresh` accept a path under any mount the daemon owns, not only the primary
|
||||
one. Both previously answered "is not under the mountpoint" for anything inside a secondary
|
||||
on-demand mount, which left a folder serving a stale listing with no manual way to re-enumerate
|
||||
it. A path under nothing this daemon has mounted still gets the same error as before.
|
||||
- A file whose blocks are not 4 MiB is no longer read twice over. Block offsets came from
|
||||
`index * 4 MiB`, so on a revision laid out differently every block the client asked for
|
||||
straddled two of the revision's own, and each was fetched and decrypted to answer for one — then
|
||||
cached at boundaries the next read would not ask about. Offsets now come from the revision's
|
||||
recorded block sizes, and the on-disk cache, the in-memory ring and the in-flight fetch key
|
||||
address a block by the byte range it covers rather than by its index (`docs/BUGS.md` B85).
|
||||
|
||||
### Changed
|
||||
- The block cache records each block's plaintext start alongside its `(mtime, size)` tag, and each
|
||||
revision's block table in a small sidecar beside its blocks. Blocks cached by an earlier release
|
||||
carry no start; they are honoured wherever the old 4 MiB assumption and the new plan agree on
|
||||
the offset, which is every file this client uploaded — so the existing cache survives the
|
||||
upgrade intact and is quietly refetched only for the files it was placing wrongly.
|
||||
|
||||
### Notes
|
||||
- **B85's diagnosis was wrong, and the entry has been corrected rather than closed.** It read a
|
||||
set of journal lines as proof that this client was requesting wrong byte ranges and being saved
|
||||
only by B84's repair path. `RevisionReader::read_at` does not work that way: it plans over the
|
||||
revision's real block sizes and clamps every block to the range asked for, identically in the
|
||||
0.6.0 the client builds against and in every earlier version. A read can therefore never return
|
||||
more bytes than it requested — yet every quoted line shows exactly that (`blen=4194304
|
||||
got=4939506`), and the one place in the client that logs that shape can only reach it on the
|
||||
opposite condition. The numbers were either transcribed with two fields swapped or captured from
|
||||
a build not in the history; they want re-capturing from the journal before anything is concluded
|
||||
from them. What was real is the assumption itself, and that is what this release removes — a
|
||||
cost fix, not a correctness one. Nothing here changes what a read returns.
|
||||
- The first read of a file still plans on the 4 MiB assumption, and learns the real geometry as a
|
||||
side effect of opening the reader it needs anyway. Resolving a reader merely to *plan* would
|
||||
reintroduce the per-read key derivation the block cache exists to avoid, so the first read of a
|
||||
non-4-MiB file pays one straddling fetch and then self-corrects.
|
||||
- Seven data-safety audit entries (B49–B55) moved from "fixed in code, verification pending" to
|
||||
verified. The fixes were all genuinely present and needed no change; what they lacked was the
|
||||
injected-failure testing each entry asked for, and that is what was written — unwritable
|
||||
directories and `chmod 000` subtrees for permission failures, a real `/dev/shm`↔`/tmp` device
|
||||
boundary for the cross-filesystem fallbacks, SQLite's `max_page_count` for a full disk, an abort
|
||||
trigger for a failed insert, and truncated and unpublished sidecars for the crash boundaries.
|
||||
Two seams were added to make failure arrangeable at all: `stage_write_at`, mirroring the
|
||||
`preserve_write_at` that already existed, and `walk_local_tree` lifted out of the sync engine so
|
||||
the local scan can be driven against a real tree without a daemon.
|
||||
- **Two of those entries are not fully closed, and say so.** B51 and B52 concern crash consistency,
|
||||
and nothing short of a machine actually losing power tests that; the crash *boundaries* are
|
||||
covered by reconstructing on disk exactly what a crash at each point leaves. B54 also diverges
|
||||
from its required fix on purpose: it has no "deliberate delete" confirmation path, so a total
|
||||
local wipe is refused outright rather than offered for confirmation. That is stricter than asked
|
||||
for, and the cost is that a user who really did delete everything must remove and re-add the
|
||||
sync folder.
|
||||
- B86's fix routes only `ls` and `refresh` to the owning mount. The other path-addressed control
|
||||
requests — pin, rename, move, delete, sharing — still resolve against the primary mount only.
|
||||
That is a larger change and was not what blocked the repro; it is recorded as an open follow-up
|
||||
(`docs/BUGS.md` B88).
|
||||
- Three findings were opened rather than fixed: the geometry above is learned one read late, so a
|
||||
non-4-MiB file pays a single wasted fetch on its first read (B87); the remaining path-addressed
|
||||
control requests are still primary-only (B88); and the container-level `CLAUDE.md` still
|
||||
describes the SDK dependency as 0.1.11 when 1.7.1 bumped it to 0.6.0, which is what sent the
|
||||
B85 investigation to the wrong source tree to begin with (B89).
|
||||
|
||||
## [1.8.1] — 2026-08-16
|
||||
|
||||
Some files read short through the mount and reported success — an application got a truncated
|
||||
copy with nothing anywhere saying so (`docs/BUGS.md` B84). Found because GNOME's file indexer
|
||||
kept complaining that PNGs under `~/ProtonDrive/Pictures` were corrupt; it was right. The rest
|
||||
of this release is the long-standing architectural backlog from the July concurrency audits
|
||||
(B25, B32, B39, B40, B43, B44), each verified against the current code before being fixed.
|
||||
Schema: **28** — migration V28 adds `sync_entry.local_mtime_ns`.
|
||||
|
||||
### Fixed
|
||||
- A read no longer trusts a revision whose recorded block sizes disagree with the file's size.
|
||||
Such a file leaves the block/range path entirely: it is fetched through the whole-file,
|
||||
manifest-verified download — the same path `pdfs pin` uses, and the reason pinning was the
|
||||
workaround — and served from the resulting cached blob. One download per file however many
|
||||
reads discover the problem at once, and files above 512 MiB fail `EIO` with a note to pin
|
||||
them rather than pulling down gigabytes for one read.
|
||||
- A block that comes back shorter than the file's size implies is never served, cached or kept
|
||||
in memory. Assembling a read out of short blocks is what quietly produced the truncation:
|
||||
the missing bytes were simply not appended, the kernel handed userspace a short read, and
|
||||
every tool reads a short read as end-of-file.
|
||||
- A synced folder no longer reports the same deletion as an error on every pass. Deleting a
|
||||
local file that something else had already deleted counted as a failure, which kept its
|
||||
baseline and made the next pass try again — forever. A path that is already gone is the
|
||||
state the sync pass wanted (`docs/BUGS.md` B42).
|
||||
- A mirror folder no longer misses an edit made in the same second as its last sync. Change
|
||||
detection compared whole seconds, so rewriting a file in place without changing its length
|
||||
read as "unchanged" and never uploaded — the remote silently stayed behind until something
|
||||
else about the file moved. Baselines now record the modification time to nanoseconds
|
||||
(`docs/BUGS.md` B25).
|
||||
- A sync download no longer overwrites an edit that finished while it was running. The
|
||||
destination is re-checked in the last moment before the download is published, and a local
|
||||
file that has moved on since the pass planned for it is kept as a conflict copy — the same
|
||||
resolution a both-sides-changed conflict gets (`docs/BUGS.md` B39).
|
||||
- An upload interrupted by a writer no longer reports itself as synchronized. The baseline now
|
||||
records the content that actually went up rather than whatever is on disk when the transfer
|
||||
ends, so a revision torn by a concurrent write is replaced by a clean one on the next pass
|
||||
instead of standing as the file's content on Drive (`docs/BUGS.md` B40).
|
||||
- Listing a directory that changes while it is being read no longer skips or repeats entries.
|
||||
`readdir` resumed by index into a listing rebuilt on every call, so a create or a trash
|
||||
between two pages shifted everything after it. The listing is now frozen for the life of the
|
||||
directory handle (`docs/BUGS.md` B43).
|
||||
- Opening a file for writing while a previous close is still publishing no longer discards that
|
||||
close's revision. The base is re-checked at the moment the write handle is installed, rather
|
||||
than trusted across a copy that can be the size of the whole file (`docs/BUGS.md` B32).
|
||||
- Unmounting now stops and joins the daemon's background workers instead of leaving them
|
||||
running. The drain, sync engine, conflict sweep, online probe, local indexer and control
|
||||
listener share a stop signal and interruptible waits, so an in-process remount no longer
|
||||
starts a second set on top of the first (`docs/BUGS.md` B44).
|
||||
|
||||
### Notes
|
||||
- The size `stat` reports was always the correct one; it is the reader's belief about where
|
||||
blocks end that was wrong, so nothing about reported sizes changes.
|
||||
- Verified live on the affected account: four multi-block PNGs that had been reading short now
|
||||
deliver their full size, and the journal shows each one repaired. The diagnostics also
|
||||
answered the open question, and not as expected — the revision metadata is fine; those files
|
||||
simply do not have 4 MiB blocks, and this client assumes they do. Filed as `docs/BUGS.md`
|
||||
B85. Until that is fixed, a file with non-4-MiB blocks is downloaded whole on its first read.
|
||||
- Migration V28's new column is nullable rather than converted from the existing seconds. A
|
||||
baseline row written before this release has no sub-second time to recover, and inventing
|
||||
zero for it would make every already-synced file compare as changed and re-upload an entire
|
||||
mirror on first start; `NULL` keeps those rows comparing exactly as they did, and each gains
|
||||
a real value the next time its path syncs.
|
||||
- Three audit entries closed without a code change, having been fixed already or being correct
|
||||
as they stand: sync-folder removal is serialized against reconciliation (B41), the drain's
|
||||
database contention was removed by the read-only connection pool and the claim column (B30),
|
||||
and the refusals for symlinks, hard links, device nodes and `chmod` are deliberate — Drive
|
||||
cannot represent them, and a local-only representation no other Proton client can read would
|
||||
be worse than a clean refusal (B31).
|
||||
- Six of this release's fixes were then driven live against the installed 1.8.1 on a real
|
||||
account: the same-second edit uploads (B25), three rapid rewrites of a 64 MiB file publish the
|
||||
third one byte for byte (B32), a write racing a download survives as a conflict copy (B39), an
|
||||
upload torn by a concurrent write heals itself on the following pass (B40), a `getdents64`
|
||||
walk with a 160-byte buffer against a directory mutated between pages returns every entry
|
||||
exactly once (B43), and a restart of a daemon with six mounts and a busy drain stops in 110 ms
|
||||
and comes back at the same thread count (B44). B42 could not be driven from a single machine —
|
||||
see B86 below.
|
||||
- One new bug came out of the verification: an on-demand mount serves a folder listing that
|
||||
predates the sync engine's own uploads, so files the mirror pass created are invisible until
|
||||
the folder is switched back (`docs/BUGS.md` B86). Nothing is lost — they are on the server and
|
||||
come back on the next mirror pass — but they cannot be seen or named while the folder is
|
||||
on-demand, which is also what blocks B42's live verification.
|
||||
|
||||
## [1.8.0] — 2026-08-16
|
||||
|
||||
A queued write whose node had left the local tree was retried every five seconds forever and
|
||||
@@ -279,6 +439,9 @@ First stable release: FUSE files-on-demand mount, sync daemon under `proton-driv
|
||||
- The outstanding FUSE defects tracked in `docs/BUGS.md`, plus a truncate defect, validated
|
||||
by a new POSIX compliance suite for the filesystem.
|
||||
|
||||
[1.8.2]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.2
|
||||
[1.8.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.1
|
||||
[1.8.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.0
|
||||
[1.7.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.7.1
|
||||
[1.7.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.7.0
|
||||
[1.6.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.6.0
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
# Build + install: cd packaging && makepkg -fi
|
||||
|
||||
pkgname=proton-drive-linux
|
||||
pkgver=1.8.0
|
||||
pkgver=1.8.2
|
||||
pkgrel=1
|
||||
pkgdesc="Proton Drive client for Linux: FUSE files-on-demand mount, CLI, GTK4 app + tray"
|
||||
arch=('x86_64')
|
||||
|
||||
Reference in new issue
Block a user