mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-18 17:55:03 +01:00
fixed some conflict bugs + ui fix
This commit is contained in:
1 parent
1d5fdfb6b9
commit
190ea3e9cb
8 files changed
+569
-112
No files matched your search
@@ -52,14 +52,17 @@ impl Db {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop the whole-file blob row for `key` plus every block row for the same
|
||||
/// uid (`<key>.b<idx>`). Called by `ContentCache::evict`, which removes the
|
||||
/// blob and all of a uid's cached blocks at once.
|
||||
/// Drop the whole-file blob row for `key` plus every derived row for the
|
||||
/// same uid — blocks (`<key>.b<idx>`) and thumbnails (`<key>.t<type>`).
|
||||
/// Called by `ContentCache::evict`, which removes the blob, all of a uid's
|
||||
/// cached blocks and all of its thumbnails at once; a row left behind here
|
||||
/// would keep counting bytes that are no longer on disk.
|
||||
pub fn cache_remove_all(&self, key: &str) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"DELETE FROM cache_entries WHERE cache_key = ?1 OR cache_key LIKE ?2",
|
||||
params![key, format!("{key}.b%")],
|
||||
"DELETE FROM cache_entries
|
||||
WHERE cache_key = ?1 OR cache_key LIKE ?2 OR cache_key LIKE ?3",
|
||||
params![key, format!("{key}.b%"), format!("{key}.t%")],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -52,6 +52,13 @@ pub use trash::StoredTrash;
|
||||
/// outliers rather than fighting the normal write path for disk.
|
||||
const WAL_SIZE_LIMIT: i64 = 64 * 1024 * 1024;
|
||||
|
||||
/// How long a statement waits for a database lock before giving up.
|
||||
///
|
||||
/// Long enough to outlast any transaction this daemon writes (the largest, a
|
||||
/// full-index FTS rebuild, is well under a second), short enough that a
|
||||
/// genuinely stuck holder surfaces as an error rather than an apparent hang.
|
||||
const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Handle to the unified metadata database.
|
||||
///
|
||||
/// Cheap to wrap in an `Arc`; clone the `Arc`, not this. All access goes through
|
||||
@@ -77,6 +84,17 @@ impl Db {
|
||||
// forever — a multi-GB file next to a database two orders of magnitude
|
||||
// smaller. Checkpointing is unaffected; only the file is truncated back.
|
||||
conn.pragma_update(None, "journal_size_limit", WAL_SIZE_LIMIT)?;
|
||||
// Wait for a lock rather than failing instantly on SQLITE_BUSY.
|
||||
//
|
||||
// This is deliberately explicit and deliberately redundant: rusqlite
|
||||
// already applies a five-second busy timeout of its own, so this changes
|
||||
// nothing today. It is here so the value is *ours* — a behaviour the
|
||||
// daemon depends on should not be an inherited default that a dependency
|
||||
// bump can silently change. `open_bounds_the_wal_size` pins it.
|
||||
//
|
||||
// (Bare SQLite does default to 0, which is where the belief that this
|
||||
// was unset came from. That default is not what we get.)
|
||||
conn.busy_timeout(BUSY_TIMEOUT)?;
|
||||
|
||||
let db = Self {
|
||||
conn: Mutex::new(conn),
|
||||
|
||||
@@ -188,6 +188,29 @@ impl Db {
|
||||
Ok(Some(AttachedBlob { id, superseded }))
|
||||
}
|
||||
|
||||
/// Replace the sidecar of a queued op, for a baseline that has moved under
|
||||
/// it.
|
||||
///
|
||||
/// Exists for one caller: when *our own* upload seals a new revision while a
|
||||
/// further write to the same node is already queued, that queued write's
|
||||
/// [`StagedWrite::based_on`](crate::cache::StagedWrite::based_on) still
|
||||
/// names the revision we just replaced. Left alone it would read as "another
|
||||
/// device changed this file" and divert the write into a conflict copy over
|
||||
/// nothing — a self-conflict. Restamping it is what keeps a chain of writes
|
||||
/// to one file a chain rather than a pile of copies.
|
||||
///
|
||||
/// Deliberately narrow: only the sidecar changes, so the blob, the attempt
|
||||
/// count and the backoff all survive. Returns false when no such op is
|
||||
/// queued, which is the ordinary case.
|
||||
pub fn update_op_meta(&self, uid: &str, kind: &str, meta_json: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let n = conn.execute(
|
||||
"UPDATE pending_op SET meta_json = ?3 WHERE uid = ?1 AND kind = ?2",
|
||||
params![uid, kind, meta_json],
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Point a queued create at a new parent and name, for a node renamed or
|
||||
/// moved before it ever reached the server.
|
||||
///
|
||||
|
||||
@@ -26,10 +26,60 @@ fn open_bounds_the_wal_size() {
|
||||
.unwrap();
|
||||
assert_eq!(limit, WAL_SIZE_LIMIT);
|
||||
|
||||
// Without a busy timeout the default is 0: a lock held by anyone else fails
|
||||
// the statement instantly instead of waiting for it to clear.
|
||||
let busy: i64 = conn
|
||||
.query_row("PRAGMA busy_timeout", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(busy, BUSY_TIMEOUT.as_millis() as i64);
|
||||
|
||||
drop(conn);
|
||||
drop(db);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
/// A second connection to the same file — a hand-started `pdfs mount` alongside
|
||||
/// the systemd service, a backup tool, a stray `sqlite3` session — holds a write
|
||||
/// lock briefly. Our write must wait for it rather than failing on the spot.
|
||||
///
|
||||
/// **This passed before the busy timeout was set explicitly**, because rusqlite
|
||||
/// already applies one. It is kept as a characterisation test, not as evidence
|
||||
/// of a fix: it states the behaviour the daemon relies on, so that losing it
|
||||
/// (dropping the pragma, or a dependency changing its default) fails here rather
|
||||
/// than in front of a user.
|
||||
#[test]
|
||||
fn a_write_waits_for_a_competing_writer() {
|
||||
use std::time::Duration;
|
||||
|
||||
let path = std::env::temp_dir().join(format!("pdfs-db-busy-{}.db", std::process::id()));
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let db = Db::open(&path).unwrap();
|
||||
|
||||
// A second process's connection takes an exclusive write lock and holds it
|
||||
// for a moment, as any real transaction would.
|
||||
let holder = {
|
||||
let path = path.clone();
|
||||
std::thread::spawn(move || {
|
||||
let other = rusqlite::Connection::open(&path).unwrap();
|
||||
other.busy_timeout(Duration::from_secs(5)).unwrap();
|
||||
other.execute_batch("BEGIN IMMEDIATE").unwrap();
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
other.execute_batch("COMMIT").unwrap();
|
||||
})
|
||||
};
|
||||
// Let the holder actually acquire the lock before we contend for it.
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
|
||||
// This is the assertion: it blocks until the holder commits, rather than
|
||||
// returning SQLITE_BUSY.
|
||||
db.set_state_i64("busy_probe", 7).unwrap();
|
||||
|
||||
holder.join().unwrap();
|
||||
assert_eq!(db.state_i64("busy_probe").unwrap(), Some(7));
|
||||
|
||||
drop(db);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId};
|
||||
|
||||
fn uid(link: &str) -> NodeUid {
|
||||
@@ -541,6 +591,56 @@ fn opens_and_migrates() {
|
||||
assert_eq!(version, SCHEMA_VERSION.to_string());
|
||||
}
|
||||
|
||||
/// A queued write whose baseline is restamped must keep everything else.
|
||||
///
|
||||
/// The restamp happens after our *own* upload seals a new revision under a
|
||||
/// still-queued write. If it took the blob or the retry state with it, the fix
|
||||
/// for a spurious conflict copy would cost the bytes that conflict copy was
|
||||
/// there to protect.
|
||||
#[test]
|
||||
fn restamping_a_baseline_leaves_the_blob_and_retry_state_alone() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let op = PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("a").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some("/staging/blob".to_string()),
|
||||
meta_json: Some(r#"{"based_on":"old"}"#.to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
db.enqueue_op(&op).unwrap();
|
||||
db.record_op_failure(db.pending_ops().unwrap()[0].id, "offline", 999)
|
||||
.unwrap();
|
||||
|
||||
let updated = db
|
||||
.update_op_meta(&uid("a").to_string(), OP_REVISION, r#"{"based_on":"new"}"#)
|
||||
.unwrap();
|
||||
assert!(updated, "the queued write is there to restamp");
|
||||
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(ops[0].meta_json.as_deref(), Some(r#"{"based_on":"new"}"#));
|
||||
assert_eq!(
|
||||
ops[0].blob_path.as_deref(),
|
||||
Some("/staging/blob"),
|
||||
"the only copy of the user's bytes"
|
||||
);
|
||||
assert_eq!(ops[0].attempts, 1, "backoff survives a restamp");
|
||||
assert_eq!(ops[0].next_attempt_at, 999);
|
||||
|
||||
// No queued write for that node is the ordinary case, not an error: most
|
||||
// uploads are the last one for their file.
|
||||
assert!(
|
||||
!db.update_op_meta(&uid("b").to_string(), OP_REVISION, "{}")
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_second_write_supersedes_the_first_pending_op() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -1182,9 +1282,11 @@ fn next_due_op_respects_backoff() {
|
||||
/// under budget" — under the shared connection lock, so it also stalled FUSE
|
||||
/// metadata calls.
|
||||
///
|
||||
/// Asserts the shape: the under-budget check must not scale with cache size.
|
||||
/// Asserts correctness — the aggregate must agree with summing the rows, and
|
||||
/// must count the kinds apart — and *reports* the timing without asserting on
|
||||
/// it. See the note at the measurement for why.
|
||||
#[test]
|
||||
fn cache_total_bytes_does_not_scale_with_cache_size() {
|
||||
fn cache_total_bytes_agrees_with_summing_the_rows() {
|
||||
use std::time::Instant;
|
||||
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -1223,17 +1325,14 @@ fn cache_total_bytes_does_not_scale_with_cache_size() {
|
||||
}
|
||||
let aggregate = t1.elapsed();
|
||||
|
||||
// Reported, not asserted. The SUM is cheaper than reading every row — no
|
||||
// materialization, no allocation, no sort — but it is still O(rows), so this
|
||||
// is a constant factor, and an assertion on a constant factor measured
|
||||
// alongside the rest of the suite tests the machine's load as much as the
|
||||
// query. The claim that matters — that the *common* path does not touch the
|
||||
// database at all — is pinned by `budget_check_is_free_when_under_budget`,
|
||||
// which compares against itself and so is load-independent.
|
||||
println!("B4: {rounds} budget checks over {N} entries — scan {scan:?}, aggregate {aggregate:?}");
|
||||
// The SUM is much cheaper — no row materialization, no allocation, no sort —
|
||||
// but it is still O(rows): summing N values reads N index entries. That is
|
||||
// why `ContentCache` keeps a running total on top of this and only falls
|
||||
// back to the query when it has to evict. See
|
||||
// `budget_check_is_free_when_under_budget`.
|
||||
assert!(
|
||||
aggregate * 2 < scan,
|
||||
"expected the SUM to be materially cheaper than reading every row; \
|
||||
scan={scan:?} aggregate={aggregate:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Victims still come out least-recently-accessed first, now in bounded batches
|
||||
|
||||
Reference in new issue
Block a user