mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-03 16:41:26 +01:00
fix(fuse,core): keep a mirrored folder safe when another computer syncs it
A mirrored folder assumed it was the only client writing to its folder on Drive. With a second computer syncing the same folder, edits could be lost: - The first sync took a same-size file at the same path for the same file. It now records the baseline only if the local file's SHA-1 matches Drive's digest, and otherwise keeps a conflict copy (B248). - An upload replaced a revision another client had committed since the pass looked. The upload now checks Drive's active revision first, and an edit that read back with someone else's digest is left for the next pass to keep as a conflict copy (B249). - A download recorded a revision committed while it ran as synced, so that edit never arrived. The baseline now records the revision the pass planned to download (B250). - Conflict copies now carry the computer's hostname, so two computers do not make the same name on Drive (B251). The baseline gains the revision id it was synced at (schema 44), and a new revision counts as a remote change even at the same mtime and size.
This commit is contained in:
1 parent
7cd516a164
commit
05b84bf796
10 files changed
+998
-168
No files matched your search
@@ -11,7 +11,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
/// Current schema version. Bump on every forward migration added below.
|
||||
pub(super) const SCHEMA_VERSION: i64 = 43;
|
||||
pub(super) const SCHEMA_VERSION: i64 = 44;
|
||||
|
||||
impl Db {
|
||||
pub(super) fn migrate(&self) -> Result<()> {
|
||||
@@ -450,6 +450,23 @@ impl Db {
|
||||
tx.execute_batch(MIGRATION_V43)?;
|
||||
}
|
||||
}
|
||||
if current < 44 {
|
||||
// Same guards as V26-V42.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('sync_entry') \
|
||||
WHERE name = 'remote_revision_id'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_entries: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_entry'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_entries && !has_column {
|
||||
tx.execute_batch(MIGRATION_V44)?;
|
||||
}
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -1351,3 +1368,17 @@ ALTER TABLE pending_op ADD COLUMN sent_to TEXT;
|
||||
const MIGRATION_V43: &str = "
|
||||
CREATE INDEX IF NOT EXISTS pending_op_kind_name ON pending_op(kind, name);
|
||||
";
|
||||
|
||||
/// Schema v44: a synced file's baseline remembers the revision it was synced
|
||||
/// at.
|
||||
///
|
||||
/// A mirrored file edited here is uploaded as a new revision of whatever is
|
||||
/// current on Drive, and another computer syncing the same folder may have
|
||||
/// committed one since this pass looked. The revision id is what the upload
|
||||
/// checks first, so that edit becomes a conflict copy instead of being
|
||||
/// replaced (B249). `NULL` means "this row predates the column", and the check
|
||||
/// falls back to the modification time and size; each row gains a real value
|
||||
/// the next time its path is synced.
|
||||
const MIGRATION_V44: &str = "
|
||||
ALTER TABLE sync_entry ADD COLUMN remote_revision_id TEXT;
|
||||
";
|
||||
@@ -45,6 +45,12 @@ pub struct StoredSyncEntry {
|
||||
pub local_mtime_ns: Option<i64>,
|
||||
pub remote_rev: Option<String>,
|
||||
pub remote_hash: Option<String>,
|
||||
/// The id of the remote revision this row was synced at. A mirror upload
|
||||
/// checks it before it supersedes what Drive holds, because another client
|
||||
/// may have committed a revision since (`docs/BUGS.md` B249). `None` on a
|
||||
/// row written before schema 44, or by a pass that did not learn it; the
|
||||
/// check then falls back to `remote_rev` and `remote_hash`.
|
||||
pub remote_revision_id: Option<String>,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
@@ -188,7 +194,7 @@ impl Db {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash,
|
||||
local_mtime_ns
|
||||
local_mtime_ns, remote_revision_id
|
||||
FROM sync_entry WHERE folder_id = ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![folder_id], |r| {
|
||||
@@ -200,6 +206,7 @@ impl Db {
|
||||
remote_rev: r.get(4)?,
|
||||
remote_hash: r.get(5)?,
|
||||
local_mtime_ns: r.get(6)?,
|
||||
remote_revision_id: r.get(7)?,
|
||||
})
|
||||
})?;
|
||||
let mut map = HashMap::new();
|
||||
@@ -238,15 +245,16 @@ impl Db {
|
||||
conn.execute(
|
||||
"INSERT INTO sync_entry
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev,
|
||||
remote_hash, local_mtime_ns)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
remote_hash, local_mtime_ns, remote_revision_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
||||
ON CONFLICT(folder_id, rel_path) DO UPDATE SET
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash,
|
||||
local_mtime_ns = excluded.local_mtime_ns",
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash,
|
||||
local_mtime_ns = excluded.local_mtime_ns,
|
||||
remote_revision_id = excluded.remote_revision_id",
|
||||
params![
|
||||
folder_id,
|
||||
e.rel_path,
|
||||
@@ -256,6 +264,7 @@ impl Db {
|
||||
e.remote_rev,
|
||||
e.remote_hash,
|
||||
e.local_mtime_ns,
|
||||
e.remote_revision_id,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
@@ -306,7 +315,7 @@ impl Db {
|
||||
{
|
||||
let mut select = tx.prepare(
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash,
|
||||
local_mtime_ns
|
||||
local_mtime_ns, remote_revision_id
|
||||
FROM sync_entry
|
||||
WHERE folder_id = ?1
|
||||
AND (rel_path = ?2 OR substr(rel_path, 1, length(?2) + 1) = ?2 || '/')",
|
||||
@@ -321,6 +330,7 @@ impl Db {
|
||||
remote_rev: r.get(4)?,
|
||||
remote_hash: r.get(5)?,
|
||||
local_mtime_ns: r.get(6)?,
|
||||
remote_revision_id: r.get(7)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
@@ -332,8 +342,8 @@ impl Db {
|
||||
let mut insert = tx.prepare(
|
||||
"INSERT OR REPLACE INTO sync_entry
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev,
|
||||
remote_hash, local_mtime_ns)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
remote_hash, local_mtime_ns, remote_revision_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
)?;
|
||||
for row in rows {
|
||||
let rel = format!("{to_rel}{}", &row.rel_path[from_rel.len()..]);
|
||||
@@ -346,6 +356,7 @@ impl Db {
|
||||
row.remote_rev,
|
||||
row.remote_hash,
|
||||
row.local_mtime_ns,
|
||||
row.remote_revision_id,
|
||||
])?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1327,6 +1327,7 @@ fn a_rebuild_keeps_the_queue_baselines_and_pins_of_the_damaged_file() {
|
||||
local_size: 6,
|
||||
remote_rev: Some("7".into()),
|
||||
remote_hash: Some("8".into()),
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -5343,6 +5344,7 @@ fn baseline_row(rel: &str) -> StoredSyncEntry {
|
||||
local_size: 7,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("7".into()),
|
||||
remote_revision_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5437,6 +5439,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
remote_revision_id: None,
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
@@ -5454,6 +5457,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
remote_revision_id: None,
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
@@ -6631,6 +6635,71 @@ fn migration_v27_adds_the_access_deferral_column_to_a_v26_queue() {
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// V44 adds the remote revision a baseline row was synced at. A row carried
|
||||
/// over from V43 has none, and the mirror then compares the modification time
|
||||
/// and size as before, so upgrading changes nothing about what is in step
|
||||
/// (`docs/BUGS.md` B249).
|
||||
#[test]
|
||||
fn migration_v44_adds_the_synced_revision_without_disturbing_a_v43_baseline() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v43-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let row = StoredSyncEntry {
|
||||
rel_path: "notes.txt".into(),
|
||||
remote_uid: Some("vol~notes".into()),
|
||||
local_mtime: 1_700_000_000,
|
||||
local_mtime_ns: Some(1_700_000_000_123_456_789),
|
||||
local_size: 42,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("42".into()),
|
||||
remote_revision_id: Some("rev-1".into()),
|
||||
};
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
let folder = db
|
||||
.sync_folder_add("/home/me/Mirror", "vol~root", "share")
|
||||
.unwrap();
|
||||
db.sync_entry_upsert(folder, &row).unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V43 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE sync_entry DROP COLUMN remote_revision_id;
|
||||
UPDATE sync_state SET value = '43' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let entries = db.sync_entries(1).unwrap();
|
||||
let entry = entries.get("notes.txt").expect("the baseline row survives");
|
||||
assert_eq!(
|
||||
entry,
|
||||
&StoredSyncEntry {
|
||||
remote_revision_id: None,
|
||||
..row.clone()
|
||||
}
|
||||
);
|
||||
|
||||
// And the column is live, including through a move of the row.
|
||||
db.sync_entry_upsert(1, &row).unwrap();
|
||||
db.sync_entries_move(1, "notes.txt", 1, "moved.txt")
|
||||
.unwrap();
|
||||
let entries = db.sync_entries(1).unwrap();
|
||||
assert_eq!(
|
||||
entries["moved.txt"].remote_revision_id.as_deref(),
|
||||
Some("rev-1")
|
||||
);
|
||||
drop(db);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
/// A mirror baseline carried over from V27 keeps its rows, and each one reads
|
||||
/// back with no nanosecond time. That `None` is the whole point: it makes the
|
||||
/// comparison fall back to whole seconds for rows written before the column
|
||||
@@ -6661,6 +6730,7 @@ fn migration_v28_adds_sub_second_local_times_without_disturbing_a_v27_baseline()
|
||||
local_size: 42,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("42".into()),
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
+89
-15
@@ -6165,11 +6165,61 @@ fn is_gone(e: &(dyn std::error::Error + 'static)) -> bool {
|
||||
/// (`sync.rs`, `conflict_path`), so the two halves of the product name the same
|
||||
/// situation the same way and a user only has to learn it once.
|
||||
fn conflict_name(name: &str, stamp: i64) -> String {
|
||||
conflict_name_as(name, conflict_label(), stamp)
|
||||
}
|
||||
|
||||
/// [`conflict_name`] with the machine's label given.
|
||||
fn conflict_name_as(name: &str, label: &str, stamp: i64) -> String {
|
||||
let path = Path::new(name);
|
||||
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(name);
|
||||
let tag = conflict_tag(label, stamp, 0);
|
||||
match path.extension().and_then(|s| s.to_str()) {
|
||||
Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {stamp})"),
|
||||
Some(ext) => format!("{stem} (sync-conflict {tag}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {tag})"),
|
||||
}
|
||||
}
|
||||
|
||||
/// This machine's name as its conflict copies carry it (`docs/BUGS.md` B251).
|
||||
///
|
||||
/// Two computers syncing one folder that each keep a copy of the same file in
|
||||
/// the same second would otherwise make the same name on Drive, and nobody
|
||||
/// could tell which computer a copy came from. Read once, from the hostname.
|
||||
fn conflict_label() -> &'static str {
|
||||
static LABEL: OnceLock<String> = OnceLock::new();
|
||||
LABEL.get_or_init(|| conflict_label_of(&this_hostname()))
|
||||
}
|
||||
|
||||
/// `host` as one token: anything but ASCII letters, digits, `-`, `_` and `.`
|
||||
/// becomes `-`, so the label holds no space for `conflict_base_name` to trip
|
||||
/// on and no `/` or `)` to break the name.
|
||||
fn conflict_label_of(host: &str) -> String {
|
||||
let label: String = host
|
||||
.chars()
|
||||
.take(32)
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') {
|
||||
c
|
||||
} else {
|
||||
'-'
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
label.trim_matches('-').to_string()
|
||||
}
|
||||
|
||||
/// The inside of a conflict copy's marker: `<label> <stamp>`, with `-<n>`
|
||||
/// when an earlier copy already holds that name. Without a label it is the
|
||||
/// bare stamp that copies had before B251.
|
||||
fn conflict_tag(label: &str, stamp: i64, suffix: u32) -> String {
|
||||
let suffix = if suffix == 0 {
|
||||
String::new()
|
||||
} else {
|
||||
format!("-{suffix}")
|
||||
};
|
||||
if label.is_empty() {
|
||||
format!("{stamp}{suffix}")
|
||||
} else {
|
||||
format!("{label} {stamp}{suffix}")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7548,14 +7598,15 @@ mod tests {
|
||||
use super::{
|
||||
Access, AccessFlags, Errno, HashMap, Intervals, PendingRevision, RootListingSnapshot,
|
||||
SELF_CHANGE_TTL_MS, ShareId, SharedWithMeItem, StateRegistry, VirtualRootPlan,
|
||||
accepted_share_provenance, conflict_name, copy_pending_for_truncate, expand_restore,
|
||||
first_seen_in_trash, fuse_name, is_stale_mount, node_visible, note_self_change,
|
||||
off_the_runtime, orphan_past_retention, parse_node_uid, prepare_shared_roots,
|
||||
preserve_on_access_denied, publish_virtual_root_in_listing,
|
||||
reconcile_virtual_root_in_listing, release_can_discard_unlinked,
|
||||
release_must_retain_queued_trash, release_unlinked_entry, rename_needs_queue,
|
||||
require_node_parent_access, require_rename_access, resolve_anywhere_with,
|
||||
shared_with_me_uid, take_self_change, uid_write_authority, virtual_node,
|
||||
accepted_share_provenance, conflict_label, conflict_label_of, conflict_name,
|
||||
conflict_name_as, copy_pending_for_truncate, expand_restore, first_seen_in_trash,
|
||||
fuse_name, is_stale_mount, node_visible, note_self_change, off_the_runtime,
|
||||
orphan_past_retention, parse_node_uid, prepare_shared_roots, preserve_on_access_denied,
|
||||
publish_virtual_root_in_listing, reconcile_virtual_root_in_listing,
|
||||
release_can_discard_unlinked, release_must_retain_queued_trash, release_unlinked_entry,
|
||||
rename_needs_queue, require_node_parent_access, require_rename_access,
|
||||
resolve_anywhere_with, shared_with_me_uid, take_self_change, uid_write_authority,
|
||||
virtual_node,
|
||||
};
|
||||
use super::{Db, WriteAuthority};
|
||||
use std::time::Duration;
|
||||
@@ -8510,15 +8561,38 @@ mod tests {
|
||||
#[test]
|
||||
fn conflict_name_keeps_the_extension() {
|
||||
assert_eq!(
|
||||
conflict_name("notes.txt", 1700),
|
||||
"notes (sync-conflict 1700).txt"
|
||||
conflict_name_as("notes.txt", "laptop", 1700),
|
||||
"notes (sync-conflict laptop 1700).txt"
|
||||
);
|
||||
assert_eq!(conflict_name("README", 42), "README (sync-conflict 42)");
|
||||
assert_eq!(
|
||||
conflict_name("archive.tar.gz", 7),
|
||||
"archive.tar (sync-conflict 7).gz",
|
||||
conflict_name_as("README", "laptop", 42),
|
||||
"README (sync-conflict laptop 42)"
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_name_as("archive.tar.gz", "laptop", 7),
|
||||
"archive.tar (sync-conflict laptop 7).gz",
|
||||
"only the last extension is one, as everywhere else"
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_name_as("notes.txt", "", 1700),
|
||||
"notes (sync-conflict 1700).txt"
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_name("notes.txt", 1700),
|
||||
conflict_name_as("notes.txt", conflict_label(), 1700)
|
||||
);
|
||||
}
|
||||
|
||||
/// A hostname goes into a file name, so it is cut down to one safe token
|
||||
/// (B251).
|
||||
#[test]
|
||||
fn a_hostname_becomes_one_safe_token() {
|
||||
assert_eq!(conflict_label_of("laptop"), "laptop");
|
||||
assert_eq!(conflict_label_of("my-pc.local"), "my-pc.local");
|
||||
assert_eq!(conflict_label_of("Linux device"), "Linux-device");
|
||||
assert_eq!(conflict_label_of("a/b) c"), "a-b--c");
|
||||
assert_eq!(conflict_label_of(" "), "");
|
||||
assert_eq!(conflict_label_of(&"x".repeat(40)).len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1401,6 +1401,216 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A folder `mirrored` on Drive, synced in mirror mode with `dir/local`,
|
||||
/// which holds `files` before the first pass. Returns the daemon, the
|
||||
/// folder's id and the link it uses.
|
||||
fn start_mirror(
|
||||
drive: &Arc<FakeDrive>,
|
||||
dir: &Path,
|
||||
files: &[(&str, &[u8])],
|
||||
) -> (Daemon, i64, FakeClient) {
|
||||
let daemon = Daemon::start(dir, drive.client(1, Faults::lan())).unwrap();
|
||||
std::fs::create_dir(daemon.mountpoint.join("mirrored")).unwrap();
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
daemon.pending().is_ok_and(|items| items.is_empty())
|
||||
&& drive.lookup("mirrored").is_some()
|
||||
}));
|
||||
let remote = drive.lookup("mirrored").unwrap();
|
||||
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||
let local = dir.join("local");
|
||||
std::fs::create_dir_all(&local).unwrap();
|
||||
for (name, bytes) in files {
|
||||
std::fs::write(local.join(name), bytes).unwrap();
|
||||
}
|
||||
let folder = reopen(dir)
|
||||
.sync_folder_add(local.to_str().unwrap(), &remote.to_string(), "")
|
||||
.unwrap();
|
||||
let client = drive.client(1, Faults::lan());
|
||||
let daemon = Daemon::start(dir, client.clone()).unwrap();
|
||||
(daemon, folder, client)
|
||||
}
|
||||
|
||||
/// Wait until a pass of the mirror that began after `since` (epoch
|
||||
/// seconds) is over without errors.
|
||||
fn wait_for_pass(daemon: &Daemon, folder: i64, since: i64) -> bool {
|
||||
wait_until(Duration::from_secs(30), || {
|
||||
let Ok(Response::SyncFolders { items }) = daemon.request(&Request::ListSyncFolders)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
items.iter().any(|f| {
|
||||
f.id == folder && f.state == "idle" && f.progress.is_none() && f.last_sync >= since
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn conflict_copies(dir: &Path, stem: &str) -> Vec<PathBuf> {
|
||||
let prefix = format!("{stem} (sync-conflict ");
|
||||
std::fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.is_some_and(|n| n.starts_with(&prefix))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn on_drive(drive: &FakeDrive, path: &str) -> Option<Vec<u8>> {
|
||||
match drive.tree().get(path) {
|
||||
Some(Entry::File(content)) => Some(content.to_vec()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
|
||||
fn a_joining_mirror_keeps_a_same_size_different_file_as_a_conflict() {
|
||||
// A computer joining a folder another one syncs found a file of the
|
||||
// same size at the same path and took it for the same file, so the two
|
||||
// went on holding different content (B248). One that is the same is
|
||||
// still taken as it is.
|
||||
let drive = FakeDrive::new();
|
||||
let dir = scratch("join-same-size");
|
||||
let theirs: &[u8] = b"from the other computer";
|
||||
let ours: &[u8] = b"written on this machine";
|
||||
assert_eq!(theirs.len(), ours.len());
|
||||
let daemon = {
|
||||
let daemon = Daemon::start(&dir, drive.client(1, Faults::lan())).unwrap();
|
||||
std::fs::create_dir(daemon.mountpoint.join("mirrored")).unwrap();
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
daemon.pending().is_ok_and(|items| items.is_empty())
|
||||
&& drive.lookup("mirrored").is_some()
|
||||
}));
|
||||
daemon
|
||||
};
|
||||
drive.device().write("mirrored/notes.txt", theirs);
|
||||
drive.device().write("mirrored/same.txt", b"identical");
|
||||
let remote = drive.lookup("mirrored").unwrap();
|
||||
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||
let local = dir.join("local");
|
||||
std::fs::create_dir_all(&local).unwrap();
|
||||
std::fs::write(local.join("notes.txt"), ours).unwrap();
|
||||
std::fs::write(local.join("same.txt"), b"identical").unwrap();
|
||||
reopen(&dir)
|
||||
.sync_folder_add(local.to_str().unwrap(), &remote.to_string(), "")
|
||||
.unwrap();
|
||||
let daemon = Daemon::start(&dir, drive.client(1, Faults::lan())).unwrap();
|
||||
|
||||
assert!(wait_until(Duration::from_secs(60), || {
|
||||
conflict_copies(&local, "notes").len() == 1
|
||||
&& std::fs::read(local.join("notes.txt")).is_ok_and(|b| b == theirs)
|
||||
&& drive
|
||||
.tree()
|
||||
.keys()
|
||||
.any(|k| k.starts_with("mirrored/notes (sync-conflict "))
|
||||
}));
|
||||
let copy = &conflict_copies(&local, "notes")[0];
|
||||
assert_eq!(std::fs::read(copy).unwrap(), ours);
|
||||
assert_eq!(on_drive(&drive, "mirrored/notes.txt").unwrap(), theirs);
|
||||
assert!(conflict_copies(&local, "same").is_empty());
|
||||
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
|
||||
fn a_mirror_edit_does_not_replace_a_revision_another_computer_made_since() {
|
||||
// A local edit went up as a new revision of whatever Drive held at the
|
||||
// time of the upload, from a plan made before another computer saved
|
||||
// its own edit. Its edit was left only in the version history (B249).
|
||||
let drive = FakeDrive::new();
|
||||
let dir = scratch("mirror-revision-race");
|
||||
let (daemon, folder, client) =
|
||||
start_mirror(&drive, &dir, &[("a.txt", b"one"), ("b.txt", b"two")]);
|
||||
let local = dir.join("local");
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
drive.lookup("mirrored/a.txt").is_some() && drive.lookup("mirrored/b.txt").is_some()
|
||||
}));
|
||||
assert!(wait_for_pass(&daemon, folder, 0));
|
||||
// The echoes of the first uploads read the files too.
|
||||
std::thread::sleep(Duration::from_secs(2));
|
||||
|
||||
let a = drive.lookup("mirrored/a.txt").unwrap();
|
||||
let check = client.hold_next_read(&a);
|
||||
std::fs::write(local.join("a.txt"), b"edited here").unwrap();
|
||||
assert!(wait_until(Duration::from_secs(30), || check.reached()));
|
||||
drive
|
||||
.device()
|
||||
.write("mirrored/a.txt", b"edited on the other computer");
|
||||
drop(check);
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
take_logged()
|
||||
.iter()
|
||||
.any(|line| line.contains("changed on Drive since this pass looked"))
|
||||
}));
|
||||
daemon
|
||||
.request(&Request::SyncNow { id: Some(folder) })
|
||||
.unwrap();
|
||||
|
||||
assert!(wait_until(Duration::from_secs(60), || {
|
||||
std::fs::read(local.join("a.txt")).is_ok_and(|b| b == b"edited on the other computer")
|
||||
&& conflict_copies(&local, "a").len() == 1
|
||||
&& drive
|
||||
.tree()
|
||||
.keys()
|
||||
.any(|k| k.starts_with("mirrored/a (sync-conflict "))
|
||||
}));
|
||||
assert_eq!(
|
||||
std::fs::read(&conflict_copies(&local, "a")[0]).unwrap(),
|
||||
b"edited here"
|
||||
);
|
||||
assert_eq!(
|
||||
on_drive(&drive, "mirrored/a.txt").unwrap(),
|
||||
b"edited on the other computer"
|
||||
);
|
||||
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
|
||||
fn a_revision_made_during_a_mirror_download_still_arrives() {
|
||||
// The baseline after a download read the node again, and recorded a
|
||||
// revision another computer made while the download ran as synced.
|
||||
// The disk held the older one, and the newer never arrived (B250).
|
||||
let drive = FakeDrive::new();
|
||||
let dir = scratch("mirror-download-race");
|
||||
let (daemon, folder, client) =
|
||||
start_mirror(&drive, &dir, &[("a.txt", b"one"), ("b.txt", b"two")]);
|
||||
let local = dir.join("local");
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
drive.lookup("mirrored/a.txt").is_some() && drive.lookup("mirrored/b.txt").is_some()
|
||||
}));
|
||||
assert!(wait_for_pass(&daemon, folder, 0));
|
||||
|
||||
let a = drive.lookup("mirrored/a.txt").unwrap();
|
||||
drive.device().write("mirrored/a.txt", b"second");
|
||||
let download = client.hold_next_download(&a);
|
||||
daemon
|
||||
.request(&Request::SyncNow { id: Some(folder) })
|
||||
.unwrap();
|
||||
assert!(wait_until(Duration::from_secs(30), || download.reached()));
|
||||
drive.device().write("mirrored/a.txt", b"thirdd");
|
||||
drop(download);
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
std::fs::read(local.join("a.txt")).is_ok_and(|b| b == b"second")
|
||||
}));
|
||||
assert!(wait_for_pass(&daemon, folder, 0));
|
||||
|
||||
daemon
|
||||
.request(&Request::SyncNow { id: Some(folder) })
|
||||
.unwrap();
|
||||
assert!(wait_until(Duration::from_secs(30), || {
|
||||
std::fs::read(local.join("a.txt")).is_ok_and(|b| b == b"thirdd")
|
||||
}));
|
||||
assert!(conflict_copies(&local, "a").is_empty());
|
||||
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
|
||||
fn a_file_removed_with_its_folder_while_uploading_stays_removed() {
|
||||
|
||||
@@ -37,10 +37,12 @@ const CONFLICT_SWEEP_INTERVAL: Duration = Duration::from_secs(300);
|
||||
/// fresh mount does at startup (hydrate, recover, initial enumeration).
|
||||
const CONFLICT_SWEEP_WARMUP: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Strip a ` (sync-conflict <stamp>)` (optionally `-<n>`) marker from `name`,
|
||||
/// returning the original file name it was a copy of. `None` when `name` is not a
|
||||
/// conflict copy. Inverse of the `conflict_name` / `conflict_path_with_suffix`
|
||||
/// formats, so the two halves of the product agree on what a conflict is.
|
||||
/// Strip a ` (sync-conflict <label> <stamp>)` (optionally `-<n>`) marker from
|
||||
/// `name`, returning the original file name it was a copy of. The label names
|
||||
/// the computer that made the copy (B251); copies made before it have the bare
|
||||
/// stamp. `None` when `name` is not a conflict copy. Inverse of the
|
||||
/// `conflict_name` / `conflict_path_with_suffix` formats, so the two halves of
|
||||
/// the product agree on what a conflict is.
|
||||
pub(crate) fn conflict_base_name(name: &str) -> Option<String> {
|
||||
const MARKER: &str = " (sync-conflict ";
|
||||
let start = name.find(MARKER)?;
|
||||
@@ -48,6 +50,20 @@ pub(crate) fn conflict_base_name(name: &str) -> Option<String> {
|
||||
let close_rel = name[after..].find(')')?;
|
||||
let close = after + close_rel;
|
||||
let inner = &name[after..close];
|
||||
// A label is one token of the characters `conflict_label_of` keeps.
|
||||
let inner = match inner.rsplit_once(' ') {
|
||||
Some((label, token)) => {
|
||||
let label_ok = !label.is_empty()
|
||||
&& label
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
|
||||
if !label_ok {
|
||||
return None;
|
||||
}
|
||||
token
|
||||
}
|
||||
None => inner,
|
||||
};
|
||||
// The token is `<digits>` or `<digits>-<digits>`; reject anything else so a
|
||||
// file that merely happens to contain the phrase is not mistaken for a copy.
|
||||
let mut parts = inner.splitn(2, '-');
|
||||
@@ -486,6 +502,24 @@ mod tests {
|
||||
conflict_base_name("notes (sync-conflict 1700-3).txt").as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
// Named after the computer that made it (B251).
|
||||
assert_eq!(
|
||||
conflict_base_name("notes (sync-conflict laptop 1700).txt").as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_base_name("notes (sync-conflict my-pc.local 1700-2).txt").as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_base_name(&crate::conflict_name("notes.txt", 1700)).as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_base_name("x (sync-conflict two words 1).txt"),
|
||||
None
|
||||
);
|
||||
assert_eq!(conflict_base_name("x (sync-conflict 1).txt"), None);
|
||||
// Not a conflict copy.
|
||||
assert_eq!(conflict_base_name("report.xml"), None);
|
||||
// Looks close but the token is not a stamp.
|
||||
|
||||
+335
-104
@@ -34,6 +34,7 @@ use std::time::{Duration, Instant, SystemTime};
|
||||
use notify::{RecursiveMode, Watcher};
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use proton_drive_rs::{Node, NodeKind};
|
||||
use sha1::{Digest as _, Sha1};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::transfers::{CountingWriter, OwnedCountingReader};
|
||||
@@ -54,14 +55,16 @@ use engine::{classify, event_target, settle_with};
|
||||
#[cfg(test)]
|
||||
use local_scan::is_write_mode;
|
||||
use local_scan::open_for_write_set;
|
||||
use model::{LocalItem, LocalSig, Outcome, PassAbort, Pending, RemoteItem};
|
||||
use model::{LocalItem, LocalSig, Outcome, PassAbort, Pending, RemoteItem, RemoteSeen};
|
||||
pub(crate) use planner::base_name;
|
||||
#[cfg(test)]
|
||||
use planner::conflict_path;
|
||||
#[cfg(test)]
|
||||
use planner::conflict_path_as;
|
||||
use planner::{
|
||||
FilePlan, RemoteGaps, Unscanned, classification_order, conflict_path_with_suffix,
|
||||
filter_baseline, guard_local_wipe, guard_remote_wipe, join_rel, local_deletions, parent_rel,
|
||||
plan_file, rel_to_path, remote_sig, remote_subtree_changed, tracked_baseline,
|
||||
plan_file, rel_to_path, remote_sig, remote_subtree_changed, revision_moved, tracked_baseline,
|
||||
unchanged_remote_size,
|
||||
};
|
||||
/// What a [`Pending`] op did, so the reconcile can update shared state on the
|
||||
@@ -75,6 +78,9 @@ enum Applied {
|
||||
Downloaded,
|
||||
/// A conflict copy was kept.
|
||||
Conflict,
|
||||
/// A path already the same on both sides got its first baseline. Nothing
|
||||
/// moved, so it is neither counted nor logged.
|
||||
Adopted,
|
||||
/// The local file to upload was deleted after the scan planned it. There is
|
||||
/// nothing left to send, and the next pass sees the deletion (B108).
|
||||
Vanished,
|
||||
@@ -410,15 +416,16 @@ impl Core {
|
||||
.collect();
|
||||
|
||||
// One light batch tells us which of those also changed remotely. A light
|
||||
// node carries the modification time without unlocking the node key, and
|
||||
// mtime alone is what the baseline compares against.
|
||||
// node carries the modification time and the revision id without
|
||||
// unlocking the node key, and those are what the baseline compares
|
||||
// against.
|
||||
let check: Vec<NodeUid> = changed
|
||||
.iter()
|
||||
.filter_map(|rel| baseline.get(*rel))
|
||||
.filter_map(|e| e.remote_uid.as_deref())
|
||||
.filter_map(parse_uid)
|
||||
.collect();
|
||||
let remote_mtimes: HashMap<String, i64> = if check.is_empty() {
|
||||
let remote_now: HashMap<String, (i64, Option<String>)> = if check.is_empty() {
|
||||
HashMap::new()
|
||||
} else {
|
||||
self.rt
|
||||
@@ -426,7 +433,10 @@ impl Core {
|
||||
.map_err(|e| format!("resolve changed nodes: {e}"))?
|
||||
.into_iter()
|
||||
.filter(|n| !n.trashed)
|
||||
.map(|n| (n.uid.to_string(), n.modification_time))
|
||||
.map(|n| {
|
||||
let revision = crate::node_revision_id(&n);
|
||||
(n.uid.to_string(), (n.modification_time, revision))
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let changed: HashSet<&String> = changed.into_iter().collect();
|
||||
@@ -507,8 +517,18 @@ impl Core {
|
||||
}
|
||||
},
|
||||
Some(uid) => {
|
||||
let remote_moved = match remote_mtimes.get(&uid.to_string()) {
|
||||
Some(mtime) => base.and_then(remote_sig).is_none_or(|(m, _)| m != *mtime),
|
||||
let (remote_moved, seen) = match remote_now.get(&uid.to_string()) {
|
||||
Some((mtime, revision)) => {
|
||||
let recorded = base.and_then(remote_sig);
|
||||
let moved = recorded.is_none_or(|(m, _)| m != *mtime)
|
||||
|| revision_moved(base, revision.as_deref());
|
||||
let seen = RemoteSeen {
|
||||
mtime: *mtime,
|
||||
size: recorded.map_or(0, |(_, size)| size),
|
||||
revision: revision.clone(),
|
||||
};
|
||||
(moved, seen)
|
||||
}
|
||||
// Trashed or unreadable remotely — treat as gone and re-upload
|
||||
// the local copy as a new file rather than losing it.
|
||||
None => {
|
||||
@@ -540,6 +560,7 @@ impl Core {
|
||||
batch.push(Pending::UploadRevision {
|
||||
rel: rel.clone(),
|
||||
uid,
|
||||
seen,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -757,26 +778,22 @@ impl Core {
|
||||
match plan_file(l, r, baseline.get(rel)) {
|
||||
FilePlan::Unchanged => {}
|
||||
FilePlan::AdoptBaseline => {
|
||||
// First sync of an already-identical file: record the baseline
|
||||
// in place so future divergence is detected, without any
|
||||
// transfer or a spurious conflict copy.
|
||||
// First sync of a file the same size on both sides: the apply
|
||||
// step hashes the local copy and records the baseline only if
|
||||
// it matches Drive's digest (B248).
|
||||
let local = l.expect("adopt baseline requires a local file");
|
||||
let remote = r.expect("adopt baseline requires a remote file");
|
||||
if let Err(e) = self.db.sync_entry_upsert(
|
||||
folder_id,
|
||||
&StoredSyncEntry {
|
||||
rel_path: rel.clone(),
|
||||
remote_uid: Some(remote.uid.to_string()),
|
||||
local_mtime: local.mtime,
|
||||
local_mtime_ns: local.mtime_ns,
|
||||
local_size: local.size,
|
||||
remote_rev: Some(remote.mtime.to_string()),
|
||||
remote_hash: Some(remote.size.to_string()),
|
||||
},
|
||||
) {
|
||||
warn!(rel, error = ?e, "sync: adopt baseline failed");
|
||||
outcome.errors += 1;
|
||||
}
|
||||
let sha1 = remote
|
||||
.sha1
|
||||
.clone()
|
||||
.expect("adopt baseline requires a remote digest");
|
||||
batch.push(Pending::Adopt {
|
||||
rel: rel.clone(),
|
||||
uid: remote.uid.clone(),
|
||||
seen: RemoteSeen::from(remote),
|
||||
sha1,
|
||||
local: LocalSig::from(local),
|
||||
});
|
||||
}
|
||||
FilePlan::Deferred => {
|
||||
debug!(
|
||||
@@ -790,6 +807,7 @@ impl Core {
|
||||
batch.push(Pending::UploadRevision {
|
||||
rel: rel.clone(),
|
||||
uid: remote.uid.clone(),
|
||||
seen: RemoteSeen::from(remote),
|
||||
});
|
||||
}
|
||||
FilePlan::Download => {
|
||||
@@ -797,8 +815,7 @@ impl Core {
|
||||
batch.push(Pending::Download {
|
||||
rel: rel.clone(),
|
||||
uid: remote.uid.clone(),
|
||||
mtime: remote.mtime,
|
||||
size: remote.size,
|
||||
seen: RemoteSeen::from(remote),
|
||||
// What the walk saw here, so the apply step can tell a
|
||||
// stale plan from a live one before it overwrites.
|
||||
local: l.map(LocalSig::from),
|
||||
@@ -809,8 +826,7 @@ impl Core {
|
||||
batch.push(Pending::Conflict {
|
||||
rel: rel.clone(),
|
||||
uid: remote.uid.clone(),
|
||||
mtime: remote.mtime,
|
||||
size: remote.size,
|
||||
seen: RemoteSeen::from(remote),
|
||||
});
|
||||
}
|
||||
FilePlan::UploadNew => match remote_dirs.get(parent_rel(rel)) {
|
||||
@@ -1107,19 +1123,31 @@ impl Core {
|
||||
.iter()
|
||||
.filter(|n| !n.trashed && !n.is_folder())
|
||||
.filter(|n| {
|
||||
unchanged_remote_size(baseline, &join_rel(prefix, &n.name), n.modification_time)
|
||||
.is_none()
|
||||
unchanged_remote_size(
|
||||
baseline,
|
||||
&join_rel(prefix, &n.name),
|
||||
n.modification_time,
|
||||
crate::node_revision_id(n).as_deref(),
|
||||
)
|
||||
.is_none()
|
||||
})
|
||||
.map(|n| n.uid.clone())
|
||||
.collect();
|
||||
let sized: HashMap<String, i64> = if stale.is_empty() {
|
||||
// The full enumeration also decrypts the content digest, which the
|
||||
// first sync of a path compares against the local file (B248).
|
||||
let sized: HashMap<String, (i64, Option<String>)> = if stale.is_empty() {
|
||||
HashMap::new()
|
||||
} else {
|
||||
self.rt
|
||||
.block_on(self.drive.enumerate_nodes(&stale))
|
||||
.map_err(|e| format!("resolve nodes: {e}"))?
|
||||
.iter()
|
||||
.map(|n| (n.uid.to_string(), remote_file_sig(n).1))
|
||||
.map(|n| {
|
||||
(
|
||||
n.uid.to_string(),
|
||||
(remote_file_sig(n).1, crate::node_content_sha1(n)),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
let stale: HashSet<String> = stale.iter().map(|u| u.to_string()).collect();
|
||||
@@ -1139,13 +1167,15 @@ impl Core {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
revision: None,
|
||||
sha1: None,
|
||||
},
|
||||
);
|
||||
self.walk_remote(folder_id, &node.uid, &rel, walk, baseline)?;
|
||||
} else {
|
||||
let size = if stale.contains(&node.uid.to_string()) {
|
||||
let (size, sha1) = if stale.contains(&node.uid.to_string()) {
|
||||
match sized.get(&node.uid.to_string()) {
|
||||
Some(size) => *size,
|
||||
Some(sized) => sized.clone(),
|
||||
// Its size was needed but could not be read (the full
|
||||
// enumeration skipped it as undecryptable). Leave it out
|
||||
// rather than record a stale size, and say so: missing,
|
||||
@@ -1156,18 +1186,25 @@ impl Core {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match unchanged_remote_size(baseline, &rel, node.modification_time) {
|
||||
Some(size) => size,
|
||||
match unchanged_remote_size(
|
||||
baseline,
|
||||
&rel,
|
||||
node.modification_time,
|
||||
crate::node_revision_id(&node).as_deref(),
|
||||
) {
|
||||
Some(size) => (size, None),
|
||||
None => continue,
|
||||
}
|
||||
};
|
||||
walk.out.insert(
|
||||
rel,
|
||||
RemoteItem {
|
||||
revision: crate::node_revision_id(&node),
|
||||
uid: node.uid,
|
||||
is_dir: false,
|
||||
mtime: node.modification_time,
|
||||
size,
|
||||
sha1,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -1188,6 +1225,7 @@ impl Core {
|
||||
local_size: 0,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("baseline dir {rel}: {e:?}"))
|
||||
@@ -1289,6 +1327,7 @@ impl Core {
|
||||
self.progress_finished(folder_id);
|
||||
let name = base_name(op.rel());
|
||||
match &result {
|
||||
Ok(Applied::Adopted) => {}
|
||||
Ok(_) => self.log_activity(op.kind(), name, op.detail(), true),
|
||||
Err(e) => self.log_activity(op.kind(), name, e.clone(), false),
|
||||
}
|
||||
@@ -1318,41 +1357,68 @@ impl Core {
|
||||
Pending::UploadNew { rel, parent } => {
|
||||
self.upload_new(folder_id, local_root, rel, parent).await
|
||||
}
|
||||
Pending::UploadRevision { rel, uid } => {
|
||||
self.upload_revision(folder_id, local_root, rel, uid).await
|
||||
Pending::UploadRevision { rel, uid, seen } => {
|
||||
self.upload_revision(folder_id, local_root, rel, uid, seen)
|
||||
.await
|
||||
}
|
||||
Pending::Download {
|
||||
rel,
|
||||
uid,
|
||||
mtime,
|
||||
size,
|
||||
seen,
|
||||
local,
|
||||
} => {
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size, *local)
|
||||
self.download_file(folder_id, local_root, rel, uid, seen, *local)
|
||||
.await?;
|
||||
Ok(Applied::Downloaded)
|
||||
}
|
||||
Pending::Conflict {
|
||||
Pending::Adopt {
|
||||
rel,
|
||||
uid,
|
||||
mtime,
|
||||
size,
|
||||
seen: remote,
|
||||
sha1,
|
||||
local,
|
||||
} => {
|
||||
// Set the local copy aside (it re-uploads as a new file next
|
||||
// pass), then take the remote version as the shared truth.
|
||||
let path = local_root.join(rel_to_path(rel));
|
||||
if path.exists() {
|
||||
preserve_conflict_copy(&path, now_secs())
|
||||
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
||||
info!(rel, "sync: kept local changes as a conflict copy");
|
||||
let hashed = tokio::task::spawn_blocking(move || local_sha1(&path))
|
||||
.await
|
||||
.map_err(|e| format!("hash {rel}: {e}"))?
|
||||
.map_err(|e| format!("hash {rel}: {e}"))?;
|
||||
let Some((seen, digest)) = hashed else {
|
||||
return Ok(Applied::Vanished);
|
||||
};
|
||||
// Edited since the scan: the next pass plans it again.
|
||||
if !seen.same_content(local) {
|
||||
return Ok(Applied::Vanished);
|
||||
}
|
||||
// The local copy has just been moved aside, so the state this
|
||||
// download expects to find at the path is "nothing" — anything
|
||||
// that appears while it runs is a *new* racing write, and gets
|
||||
// its own conflict copy rather than being overwritten (B39).
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size, None)
|
||||
.await?;
|
||||
Ok(Applied::Conflict)
|
||||
if !digest.eq_ignore_ascii_case(sha1) {
|
||||
info!(
|
||||
rel,
|
||||
"sync: same size as on Drive but different content; keeping a conflict copy"
|
||||
);
|
||||
return self
|
||||
.take_remote_keeping_local(folder_id, local_root, rel, uid, remote)
|
||||
.await;
|
||||
}
|
||||
self.db
|
||||
.sync_entry_upsert(
|
||||
folder_id,
|
||||
&StoredSyncEntry {
|
||||
rel_path: rel.clone(),
|
||||
remote_uid: Some(uid.to_string()),
|
||||
local_mtime: seen.mtime,
|
||||
local_mtime_ns: seen.mtime_ns,
|
||||
local_size: seen.size,
|
||||
remote_rev: Some(remote.mtime.to_string()),
|
||||
remote_hash: Some(remote.size.to_string()),
|
||||
remote_revision_id: remote.revision.clone(),
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("adopt baseline {rel}: {e:?}"))?;
|
||||
Ok(Applied::Adopted)
|
||||
}
|
||||
Pending::Conflict { rel, uid, seen } => {
|
||||
self.take_remote_keeping_local(folder_id, local_root, rel, uid, seen)
|
||||
.await
|
||||
}
|
||||
Pending::PushConflict { rel, parent } => {
|
||||
// The ordinary conflict resolution — rename the local copy aside and
|
||||
@@ -1376,6 +1442,31 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the local copy aside (it re-uploads as a new file next pass), then
|
||||
/// take the remote version as the shared truth.
|
||||
async fn take_remote_keeping_local(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
local_root: &Path,
|
||||
rel: &str,
|
||||
uid: &NodeUid,
|
||||
seen: &RemoteSeen,
|
||||
) -> Result<Applied, String> {
|
||||
let path = local_root.join(rel_to_path(rel));
|
||||
if path.exists() {
|
||||
preserve_conflict_copy(&path, now_secs())
|
||||
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
||||
info!(rel, "sync: kept local changes as a conflict copy");
|
||||
}
|
||||
// The local copy has just been moved aside, so the state this
|
||||
// download expects to find at the path is "nothing" — anything
|
||||
// that appears while it runs is a *new* racing write, and gets
|
||||
// its own conflict copy rather than being overwritten (B39).
|
||||
self.download_file(folder_id, local_root, rel, uid, seen, None)
|
||||
.await?;
|
||||
Ok(Applied::Conflict)
|
||||
}
|
||||
|
||||
/// Upload a brand-new local file to its remote parent, then record baseline.
|
||||
async fn upload_new(
|
||||
&self,
|
||||
@@ -1400,11 +1491,11 @@ impl Core {
|
||||
// Count the bytes as they are read, so this upload shows up in
|
||||
// `GetQueueStatus` next to a manual one. The uid isn't known until the
|
||||
// draft is sealed, so the transfer registers without one.
|
||||
let reader = OwnedCountingReader::new(
|
||||
let (reader, sent) = Sha1Tap::new(OwnedCountingReader::new(
|
||||
file,
|
||||
self.transfers
|
||||
.begin(name, "", TransferDirection::Upload, meta.len()),
|
||||
);
|
||||
));
|
||||
let uid = self
|
||||
.drive
|
||||
// Mirror push: the local file is authoritative. If a prior attempt was
|
||||
@@ -1428,30 +1519,60 @@ impl Core {
|
||||
// snapshot that predates this upload (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), &uid)
|
||||
let remote = self
|
||||
.landed_as_sent(rel, &uid, &sent.digest(), streamed)
|
||||
.await?;
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), &uid, &remote)?;
|
||||
Ok(Applied::Uploaded)
|
||||
}
|
||||
|
||||
/// Upload a changed local file as a new revision of an existing remote node.
|
||||
///
|
||||
/// The revision supersedes whatever is current on Drive, and the plan comes
|
||||
/// from a walk that may be minutes old. Another computer syncing the same
|
||||
/// folder may have committed a revision since, and uploading over it would
|
||||
/// leave that edit only in the revision history. So Drive is asked first,
|
||||
/// and a file that moved on is left for the next pass, which keeps the
|
||||
/// local edit as a conflict copy (B249). A revision committed between this
|
||||
/// check and the upload's own draft is not caught here; the read-back after
|
||||
/// the upload catches one committed after it.
|
||||
async fn upload_revision(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
local_root: &Path,
|
||||
rel: &str,
|
||||
uid: &NodeUid,
|
||||
seen: &RemoteSeen,
|
||||
) -> Result<Applied, String> {
|
||||
let path = local_root.join(rel_to_path(rel));
|
||||
let Some(file) = open_to_upload(&path, rel)? else {
|
||||
return Ok(Applied::Vanished);
|
||||
};
|
||||
let current = self
|
||||
.drive
|
||||
.enumerate_nodes(std::slice::from_ref(uid))
|
||||
.await
|
||||
.map_err(|e| format!("check {rel} on Drive: {e}"))?;
|
||||
match current.first() {
|
||||
Some(node) if !node.trashed && seen.still_current(node) => {}
|
||||
Some(node) if !node.trashed => {
|
||||
return Err(format!(
|
||||
"{rel} changed on Drive since this pass looked; it is compared again on the next pass"
|
||||
));
|
||||
}
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"{rel} is no longer on Drive where this pass saw it; it is compared again on the next pass"
|
||||
));
|
||||
}
|
||||
}
|
||||
let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?;
|
||||
let mtime = system_mtime(&meta);
|
||||
let streamed = LocalSig::from(&meta);
|
||||
let thumbnails = self
|
||||
.upload_thumbnails_async(path.clone(), base_name(rel).to_string())
|
||||
.await;
|
||||
let reader = OwnedCountingReader::new(
|
||||
let (reader, sent) = Sha1Tap::new(OwnedCountingReader::new(
|
||||
file,
|
||||
self.transfers.begin(
|
||||
base_name(rel),
|
||||
@@ -1459,14 +1580,16 @@ impl Core {
|
||||
TransferDirection::Upload,
|
||||
meta.len(),
|
||||
),
|
||||
);
|
||||
));
|
||||
self.drive
|
||||
.upload_new_revision_from(uid, reader, meta.len() as i64, thumbnails, Some(mtime))
|
||||
.await
|
||||
.map_err(|e| format!("upload revision {rel}: {e}"))?;
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), uid)
|
||||
let remote = self
|
||||
.landed_as_sent(rel, uid, &sent.digest(), streamed)
|
||||
.await?;
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), uid, &remote)?;
|
||||
Ok(Applied::Uploaded)
|
||||
}
|
||||
|
||||
@@ -1478,17 +1601,23 @@ impl Core {
|
||||
/// for minutes, so the destination is re-examined right before it: a local
|
||||
/// file that no longer matches the plan is set aside as a conflict copy
|
||||
/// instead of being replaced (`docs/BUGS.md` B39).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
///
|
||||
/// The baseline records `seen`, the revision the pass planned to download,
|
||||
/// and not a read of the node afterwards. Another computer may commit a
|
||||
/// revision while the download runs, and a read-back would record that
|
||||
/// revision as synced while the disk holds the older one, so its edit would
|
||||
/// never arrive (B250). If the download did fetch the newer revision, the
|
||||
/// next pass fetches it once more, which costs a transfer and loses nothing.
|
||||
async fn download_file(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
local_root: &Path,
|
||||
rel: &str,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
seen: &RemoteSeen,
|
||||
expected_local: Option<LocalSig>,
|
||||
) -> Result<(), String> {
|
||||
let (mtime, size) = (seen.mtime, seen.size);
|
||||
let path = local_root.join(rel_to_path(rel));
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("mkdir for {rel}: {e}"))?;
|
||||
@@ -1560,14 +1689,60 @@ impl Core {
|
||||
}
|
||||
// Here the file on disk *is* the transferred content: it was published
|
||||
// by a rename and stamped just above, so its stat is the identity.
|
||||
self.record_file_baseline(folder_id, rel, local_sig(&path), uid)
|
||||
.await
|
||||
self.record_file_baseline(folder_id, rel, local_sig(&path), uid, seen)
|
||||
}
|
||||
|
||||
/// Refresh a file's baseline from ground truth: the local identity these
|
||||
/// bytes came from and the remote node's reported signature. Called after
|
||||
/// every upload/download so the very next reconcile sees the path as
|
||||
/// unchanged.
|
||||
/// What Drive holds for `uid` after this daemon uploaded content with SHA-1
|
||||
/// `sent` and identity `streamed` to it, as the baseline should record it.
|
||||
///
|
||||
/// Read back so the stored signature is exactly what a walk reports next
|
||||
/// time. A digest on Drive that is not `sent` means another computer
|
||||
/// committed a revision right after this upload. Recording that revision
|
||||
/// as synced would hide it, and the edit here would be lost to it on the
|
||||
/// next download. So the baseline is left as it was, and the next pass sees
|
||||
/// both sides changed and keeps a conflict copy (B250).
|
||||
async fn landed_as_sent(
|
||||
&self,
|
||||
rel: &str,
|
||||
uid: &NodeUid,
|
||||
sent: &str,
|
||||
streamed: LocalSig,
|
||||
) -> Result<RemoteSeen, String> {
|
||||
let node = match self.drive.enumerate_nodes(std::slice::from_ref(uid)).await {
|
||||
Ok(nodes) => nodes.into_iter().next(),
|
||||
Err(e) => {
|
||||
warn!(rel, error = %e, "sync: baseline refetch failed; using local mtime");
|
||||
None
|
||||
}
|
||||
};
|
||||
let Some(node) = node else {
|
||||
// Drive does not list a node it has just made at once. The upload
|
||||
// stamped it with the local modification time, so that and the
|
||||
// size sent are what a walk reports.
|
||||
return Ok(RemoteSeen {
|
||||
mtime: streamed.mtime,
|
||||
size: streamed.size,
|
||||
revision: None,
|
||||
});
|
||||
};
|
||||
if let Some(digest) = crate::node_content_sha1(&node)
|
||||
&& !digest.eq_ignore_ascii_case(sent)
|
||||
{
|
||||
return Err(format!(
|
||||
"{rel} was changed on Drive by another client right after this upload; it is compared again on the next pass"
|
||||
));
|
||||
}
|
||||
let (mtime, size) = remote_file_sig(&node);
|
||||
Ok(RemoteSeen {
|
||||
mtime,
|
||||
size,
|
||||
revision: crate::node_revision_id(&node),
|
||||
})
|
||||
}
|
||||
|
||||
/// Record a file's baseline: the local identity these bytes came from and
|
||||
/// the remote revision they were synced with. Called after every
|
||||
/// upload/download so the very next reconcile sees the path as unchanged.
|
||||
///
|
||||
/// `local` is the identity the *transferred content* corresponds to, which
|
||||
/// is not always the file's current stat. An upload streams the live file,
|
||||
@@ -1577,46 +1752,81 @@ impl Core {
|
||||
/// locally changed and upload a whole, clean revision over it. Recording the
|
||||
/// post-upload stat instead declared the torn revision synchronized and left
|
||||
/// it as the file's content on Drive (`docs/BUGS.md` B40).
|
||||
async fn record_file_baseline(
|
||||
fn record_file_baseline(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
rel: &str,
|
||||
local: Option<LocalSig>,
|
||||
uid: &NodeUid,
|
||||
remote: &RemoteSeen,
|
||||
) -> Result<(), String> {
|
||||
let Some(local) = local else {
|
||||
return Err(format!("stat {rel}: no local identity to baseline"));
|
||||
};
|
||||
let (lmtime, lsize) = (local.mtime, local.size);
|
||||
// Re-fetch the node so the stored remote signature is exactly what a walk
|
||||
// will report next time.
|
||||
let (rmtime, rsize) = match self.drive.enumerate_nodes(std::slice::from_ref(uid)).await {
|
||||
Ok(nodes) => nodes
|
||||
.first()
|
||||
.map(remote_file_sig)
|
||||
.unwrap_or((lmtime, lsize)),
|
||||
Err(e) => {
|
||||
warn!(rel, error = %e, "sync: baseline refetch failed; using local mtime");
|
||||
(lmtime, lsize)
|
||||
}
|
||||
};
|
||||
self.db
|
||||
.sync_entry_upsert(
|
||||
folder_id,
|
||||
&StoredSyncEntry {
|
||||
rel_path: rel.to_string(),
|
||||
remote_uid: Some(uid.to_string()),
|
||||
local_mtime: lmtime,
|
||||
local_mtime: local.mtime,
|
||||
local_mtime_ns: local.mtime_ns,
|
||||
local_size: lsize,
|
||||
remote_rev: Some(rmtime.to_string()),
|
||||
remote_hash: Some(rsize.to_string()),
|
||||
local_size: local.size,
|
||||
remote_rev: Some(remote.mtime.to_string()),
|
||||
remote_hash: Some(remote.size.to_string()),
|
||||
remote_revision_id: remote.revision.clone(),
|
||||
},
|
||||
)
|
||||
.map_err(|e| format!("baseline {rel}: {e:?}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`Read`] that hashes what an upload reads through it, so the digest Drive
|
||||
/// reports afterwards can be compared with what was sent.
|
||||
struct Sha1Tap<R> {
|
||||
inner: R,
|
||||
hasher: Arc<Mutex<Sha1>>,
|
||||
}
|
||||
|
||||
/// The digest of what went through a [`Sha1Tap`], read once the upload is in.
|
||||
struct SentDigest(Arc<Mutex<Sha1>>);
|
||||
|
||||
impl<R: std::io::Read> Sha1Tap<R> {
|
||||
fn new(inner: R) -> (Self, SentDigest) {
|
||||
let hasher = Arc::new(Mutex::new(Sha1::new()));
|
||||
let digest = SentDigest(hasher.clone());
|
||||
(Self { inner, hasher }, digest)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: std::io::Read> std::io::Read for Sha1Tap<R> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let n = self.inner.read(buf)?;
|
||||
self.hasher.lock().update(&buf[..n]);
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
impl SentDigest {
|
||||
fn digest(&self) -> String {
|
||||
crate::takeout::hex_lower(&self.0.lock().clone().finalize())
|
||||
}
|
||||
}
|
||||
|
||||
/// A local file's identity and content SHA-1, both read from one descriptor,
|
||||
/// or `None` when it is gone. The identity is taken after the hash, so a file
|
||||
/// written to while it was hashed no longer matches the scan's.
|
||||
fn local_sha1(path: &Path) -> std::io::Result<Option<(LocalSig, String)>> {
|
||||
let mut file = match std::fs::File::open(path) {
|
||||
Ok(file) => file,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let digest =
|
||||
crate::takeout::sha1_of(&mut file).map_err(|e| std::io::Error::other(e.message))?;
|
||||
Ok(Some((LocalSig::from(&file.metadata()?), digest)))
|
||||
}
|
||||
|
||||
/// Open a local file for upload, or `None` when it is gone.
|
||||
///
|
||||
/// A scan plans uploads from a listing, and the file can be deleted before its
|
||||
@@ -2337,6 +2547,8 @@ mod tests {
|
||||
is_dir,
|
||||
mtime: 1,
|
||||
size: 1,
|
||||
revision: None,
|
||||
sha1: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2580,6 +2792,7 @@ mod tests {
|
||||
local_mtime_ns: None,
|
||||
remote_rev: Some(mtime.to_string()),
|
||||
remote_hash: Some(size.to_string()),
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)])
|
||||
}
|
||||
@@ -2588,20 +2801,23 @@ mod tests {
|
||||
fn an_unchanged_mtime_reuses_the_recorded_size() {
|
||||
// The whole point: this file's node key never gets unlocked.
|
||||
let base = baseline_with("a.txt", 1700, 4096);
|
||||
assert_eq!(unchanged_remote_size(&base, "a.txt", 1700), Some(4096));
|
||||
assert_eq!(
|
||||
unchanged_remote_size(&base, "a.txt", 1700, None),
|
||||
Some(4096)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_moved_mtime_forces_the_size_to_be_read() {
|
||||
let base = baseline_with("a.txt", 1700, 4096);
|
||||
assert_eq!(unchanged_remote_size(&base, "a.txt", 1701), None);
|
||||
assert_eq!(unchanged_remote_size(&base, "a.txt", 1701, None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_missing_from_the_baseline_forces_the_size_to_be_read() {
|
||||
// Nothing recorded to trust — a file the sync has never seen.
|
||||
let base = baseline_with("a.txt", 1700, 4096);
|
||||
assert_eq!(unchanged_remote_size(&base, "new.txt", 1700), None);
|
||||
assert_eq!(unchanged_remote_size(&base, "new.txt", 1700, None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2618,9 +2834,10 @@ mod tests {
|
||||
local_mtime_ns: None,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)]);
|
||||
assert_eq!(unchanged_remote_size(&base, "d", 0), None);
|
||||
assert_eq!(unchanged_remote_size(&base, "d", 0, None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2633,6 +2850,7 @@ mod tests {
|
||||
local_mtime_ns: None,
|
||||
remote_rev: Some("1700".into()),
|
||||
remote_hash: Some("4096".into()),
|
||||
remote_revision_id: None,
|
||||
};
|
||||
assert_eq!(remote_sig(&e), Some((1700, 4096)));
|
||||
}
|
||||
@@ -2647,6 +2865,7 @@ mod tests {
|
||||
local_mtime_ns: None,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
remote_revision_id: None,
|
||||
};
|
||||
assert_eq!(remote_sig(&e), None);
|
||||
}
|
||||
@@ -2699,8 +2918,11 @@ mod tests {
|
||||
let op = Pending::Download {
|
||||
rel: "photo.jpg".into(),
|
||||
uid: uid(),
|
||||
mtime: 0,
|
||||
size: 10,
|
||||
seen: RemoteSeen {
|
||||
mtime: 0,
|
||||
size: 10,
|
||||
revision: None,
|
||||
},
|
||||
local: None,
|
||||
};
|
||||
assert_eq!(op.kind(), ActivityKind::Download);
|
||||
@@ -2713,13 +2935,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn conflict_path_keeps_extension_and_directory() {
|
||||
let p = conflict_path(Path::new("/home/me/docs/notes.txt"), 1700);
|
||||
let p = conflict_path_as(Path::new("/home/me/docs/notes.txt"), "laptop", 1700, 0);
|
||||
assert_eq!(
|
||||
p,
|
||||
PathBuf::from("/home/me/docs/notes (sync-conflict 1700).txt")
|
||||
PathBuf::from("/home/me/docs/notes (sync-conflict laptop 1700).txt")
|
||||
);
|
||||
let no_ext = conflict_path_as(Path::new("/home/me/README"), "laptop", 42, 2);
|
||||
assert_eq!(
|
||||
no_ext,
|
||||
PathBuf::from("/home/me/README (sync-conflict laptop 42-2)")
|
||||
);
|
||||
// The engine names its copies after this machine.
|
||||
assert_eq!(
|
||||
conflict_path(Path::new("/home/me/README"), 42),
|
||||
conflict_path_as(Path::new("/home/me/README"), crate::conflict_label(), 42, 0)
|
||||
);
|
||||
let no_ext = conflict_path(Path::new("/home/me/README"), 42);
|
||||
assert_eq!(no_ext, PathBuf::from("/home/me/README (sync-conflict 42)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2735,8 +2965,8 @@ mod tests {
|
||||
assert_eq!(std::fs::read(&existing).unwrap(), b"older conflict");
|
||||
assert_eq!(std::fs::read(&preserved).unwrap(), b"new local edit");
|
||||
assert_eq!(
|
||||
preserved.file_name().unwrap(),
|
||||
"notes (sync-conflict 1700-1).txt"
|
||||
preserved,
|
||||
conflict_path_as(&source, crate::conflict_label(), 1700, 1)
|
||||
);
|
||||
assert!(!source.exists());
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
@@ -3018,6 +3248,7 @@ mod tests {
|
||||
local_size: sig.size,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
remote_revision_id: None,
|
||||
};
|
||||
(rel.to_string(), entry)
|
||||
}
|
||||
|
||||
@@ -83,6 +83,46 @@ pub(super) struct RemoteItem {
|
||||
pub(super) is_dir: bool,
|
||||
pub(super) mtime: i64,
|
||||
pub(super) size: i64,
|
||||
/// The active revision's id. The light walk carries it for every file, so
|
||||
/// it is the signal that tells another client's new revision from a link
|
||||
/// merely re-stamped.
|
||||
pub(super) revision: Option<String>,
|
||||
/// The active revision's content SHA-1, read only for files the walk
|
||||
/// enumerated in full (no baseline, or a moved mtime). `None` is never
|
||||
/// proof of anything.
|
||||
pub(super) sha1: Option<String>,
|
||||
}
|
||||
|
||||
/// What a pass saw of a remote file: the signature its baseline records once
|
||||
/// the file is synced, and what an upload checks Drive still holds before it
|
||||
/// supersedes it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) struct RemoteSeen {
|
||||
pub(super) mtime: i64,
|
||||
pub(super) size: i64,
|
||||
pub(super) revision: Option<String>,
|
||||
}
|
||||
|
||||
impl RemoteSeen {
|
||||
/// Whether `node` is still the revision this was taken from. The revision
|
||||
/// id decides it when both sides have one; otherwise the modification time
|
||||
/// and size, which is all a baseline row from before schema 44 holds.
|
||||
pub(super) fn still_current(&self, node: &Node) -> bool {
|
||||
match (&self.revision, crate::node_revision_id(node)) {
|
||||
(Some(seen), Some(now)) => *seen == now,
|
||||
_ => (self.mtime, self.size) == remote_file_sig(node),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&RemoteItem> for RemoteSeen {
|
||||
fn from(item: &RemoteItem) -> Self {
|
||||
Self {
|
||||
mtime: item.mtime,
|
||||
size: item.size,
|
||||
revision: item.revision.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The result of a reconcile pass: what it moved, how many paths were kept as
|
||||
@@ -113,7 +153,7 @@ impl Outcome {
|
||||
Applied::Uploaded => self.uploaded += 1,
|
||||
Applied::Downloaded => self.downloaded += 1,
|
||||
Applied::Conflict => self.conflicts += 1,
|
||||
Applied::Vanished => {}
|
||||
Applied::Vanished | Applied::Adopted => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,10 +197,16 @@ pub(super) enum Pending {
|
||||
CreateDir { rel: String, parent: NodeUid },
|
||||
/// Upload a brand-new local file into `parent`.
|
||||
UploadNew { rel: String, parent: NodeUid },
|
||||
/// Upload a changed local file as a new revision of `uid`.
|
||||
UploadRevision { rel: String, uid: NodeUid },
|
||||
/// Download remote `uid` to the local path, stamping `mtime`. `size` is the
|
||||
/// remote's reported size, used as the transfer's expected total.
|
||||
/// Upload a changed local file as a new revision of `uid`, if Drive still
|
||||
/// holds the revision `seen` describes (B249).
|
||||
UploadRevision {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
seen: RemoteSeen,
|
||||
},
|
||||
/// Download remote `uid` to the local path, stamping `seen.mtime`.
|
||||
/// `seen.size` is the remote's reported size, used as the transfer's
|
||||
/// expected total.
|
||||
///
|
||||
/// `local` is what the planning walk saw at this path, or `None` if nothing
|
||||
/// was there. A download can run for minutes, so the apply step revalidates
|
||||
@@ -170,16 +216,26 @@ pub(super) enum Pending {
|
||||
Download {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
seen: RemoteSeen,
|
||||
local: Option<LocalSig>,
|
||||
},
|
||||
/// First sync of a path present on both sides at the same size, with a
|
||||
/// digest on Drive: hash the local file, and record the baseline if it holds
|
||||
/// the same bytes. A different file of the same size goes the way of
|
||||
/// [`Pending::Conflict`]: two machines syncing one folder must not read as
|
||||
/// in step while they hold different content.
|
||||
Adopt {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
seen: RemoteSeen,
|
||||
sha1: String,
|
||||
local: LocalSig,
|
||||
},
|
||||
/// Both sides changed: set the local copy aside, then download remote `uid`.
|
||||
Conflict {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
seen: RemoteSeen,
|
||||
},
|
||||
/// Both sides changed during a [`push pass`](Core::push_pass): upload the local
|
||||
/// copy into `parent` under a conflict name, leaving the remote file as it is.
|
||||
@@ -194,6 +250,7 @@ impl Pending {
|
||||
| Pending::UploadNew { rel, .. }
|
||||
| Pending::UploadRevision { rel, .. }
|
||||
| Pending::Download { rel, .. }
|
||||
| Pending::Adopt { rel, .. }
|
||||
| Pending::Conflict { rel, .. }
|
||||
| Pending::PushConflict { rel, .. } => rel,
|
||||
}
|
||||
@@ -206,7 +263,9 @@ impl Pending {
|
||||
Pending::UploadNew { .. }
|
||||
| Pending::UploadRevision { .. }
|
||||
| Pending::PushConflict { .. } => ActivityKind::Upload,
|
||||
Pending::Download { .. } | Pending::Conflict { .. } => ActivityKind::Download,
|
||||
Pending::Download { .. } | Pending::Adopt { .. } | Pending::Conflict { .. } => {
|
||||
ActivityKind::Download
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,6 +276,7 @@ impl Pending {
|
||||
Pending::UploadNew { .. } => "new file",
|
||||
Pending::UploadRevision { .. } => "new version",
|
||||
Pending::Download { .. } => "from Drive",
|
||||
Pending::Adopt { .. } => "already the same on Drive",
|
||||
Pending::Conflict { .. } => "local changes kept as a conflict copy",
|
||||
Pending::PushConflict { .. } => "local changes uploaded as a conflict copy",
|
||||
}
|
||||
|
||||
@@ -318,7 +318,10 @@ pub(super) fn remote_subtree_changed(
|
||||
.filter(|(path, item)| match baseline.get(*path) {
|
||||
None => true,
|
||||
Some(_) if item.is_dir => false,
|
||||
Some(base) => remote_sig(base) != Some((item.mtime, item.size)),
|
||||
Some(base) => {
|
||||
remote_sig(base) != Some((item.mtime, item.size))
|
||||
|| revision_moved(Some(base), item.revision.as_deref())
|
||||
}
|
||||
})
|
||||
.map(|(path, _)| path)
|
||||
.collect();
|
||||
@@ -329,13 +332,18 @@ pub(super) fn remote_subtree_changed(
|
||||
}
|
||||
|
||||
/// The size the baseline recorded for `rel`, if its remote signature's mtime
|
||||
/// still matches `mtime`.
|
||||
/// still matches `mtime` and its revision is still `revision`.
|
||||
pub(super) fn unchanged_remote_size(
|
||||
baseline: &HashMap<String, StoredSyncEntry>,
|
||||
rel: &str,
|
||||
mtime: i64,
|
||||
revision: Option<&str>,
|
||||
) -> Option<i64> {
|
||||
match baseline.get(rel).and_then(remote_sig) {
|
||||
let base = baseline.get(rel)?;
|
||||
if revision_moved(Some(base), revision) {
|
||||
return None;
|
||||
}
|
||||
match remote_sig(base) {
|
||||
Some((recorded, size)) if recorded == mtime => Some(size),
|
||||
_ => None,
|
||||
}
|
||||
@@ -350,6 +358,17 @@ pub(super) fn join_rel(prefix: &str, name: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether Drive's active revision `now` is not the one the baseline row was
|
||||
/// synced at. Only a revision id on both sides can say so; a row from before
|
||||
/// schema 44, or a node read without one, leaves it to the modification time
|
||||
/// and size.
|
||||
pub(super) fn revision_moved(base: Option<&StoredSyncEntry>, now: Option<&str>) -> bool {
|
||||
match (base.and_then(|b| b.remote_revision_id.as_deref()), now) {
|
||||
(Some(recorded), Some(now)) => recorded != now,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stored remote signature of a baseline row, if it has one.
|
||||
pub(super) fn remote_sig(e: &StoredSyncEntry) -> Option<(i64, i64)> {
|
||||
match (&e.remote_rev, &e.remote_hash) {
|
||||
@@ -380,7 +399,8 @@ pub(super) fn rel_to_path(rel: &str) -> PathBuf {
|
||||
rel.split('/').collect()
|
||||
}
|
||||
|
||||
/// The name for a conflict copy of `path`, e.g. `notes (sync-conflict 1700000000).txt`.
|
||||
/// The name for a conflict copy of `path`, e.g.
|
||||
/// `notes (sync-conflict laptop 1700000000).txt`.
|
||||
#[cfg(test)]
|
||||
pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
|
||||
conflict_path_with_suffix(path, stamp, 0)
|
||||
@@ -389,16 +409,17 @@ pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
|
||||
/// As [`conflict_path`], with a deterministic suffix used when that name already
|
||||
/// exists. This keeps conflict preservation from ever replacing an older copy.
|
||||
pub(super) fn conflict_path_with_suffix(path: &Path, stamp: i64, suffix: u32) -> PathBuf {
|
||||
conflict_path_as(path, crate::conflict_label(), stamp, suffix)
|
||||
}
|
||||
|
||||
/// [`conflict_path_with_suffix`] with the machine's label given.
|
||||
pub(super) fn conflict_path_as(path: &Path, label: &str, stamp: i64, suffix: u32) -> PathBuf {
|
||||
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("file");
|
||||
let ext = path.extension().and_then(|s| s.to_str());
|
||||
let suffix = if suffix == 0 {
|
||||
String::new()
|
||||
} else {
|
||||
format!("-{suffix}")
|
||||
};
|
||||
let tag = crate::conflict_tag(label, stamp, suffix);
|
||||
let name = match ext {
|
||||
Some(ext) => format!("{stem} (sync-conflict {stamp}{suffix}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {stamp}{suffix})"),
|
||||
Some(ext) => format!("{stem} (sync-conflict {tag}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {tag})"),
|
||||
};
|
||||
match path.parent() {
|
||||
Some(dir) => dir.join(name),
|
||||
@@ -419,11 +440,14 @@ pub(super) enum FilePlan {
|
||||
DeleteLocal,
|
||||
DeleteRemote,
|
||||
ForgetBaseline,
|
||||
/// First sync of a file that already exists on both sides with identical
|
||||
/// content: record the baseline without transferring anything. Without this,
|
||||
/// First sync of a file that exists on both sides at the same size, with a
|
||||
/// digest on Drive: record the baseline without transferring anything if
|
||||
/// the local file hashes the same, else keep a conflict copy. Without this,
|
||||
/// a folder synced for the first time cuts a `(sync-conflict)` copy of every
|
||||
/// pre-existing file, because a missing baseline makes both sides read as
|
||||
/// "changed" (see the mass first-sync conflict storm in docs/BUGS.md).
|
||||
/// "changed" (see the mass first-sync conflict storm in docs/BUGS.md). Size
|
||||
/// alone is not enough: a second machine joining the folder would read a
|
||||
/// different file of the same size as in step (B248).
|
||||
AdoptBaseline,
|
||||
}
|
||||
|
||||
@@ -442,8 +466,13 @@ pub(super) fn plan_file(
|
||||
// left the file the same length, so it was never uploaded.
|
||||
baseline.is_none_or(|base| !LocalSig::from(item).same_content(&LocalSig::from(base)))
|
||||
});
|
||||
// Another client's new revision shows in its id even when the link's
|
||||
// modification time and the size come out the same.
|
||||
let remote_changed = remote.is_some_and(|item| {
|
||||
baseline.is_none_or(|base| remote_sig(base) != Some((item.mtime, item.size)))
|
||||
baseline.is_none_or(|base| {
|
||||
remote_sig(base) != Some((item.mtime, item.size))
|
||||
|| revision_moved(Some(base), item.revision.as_deref())
|
||||
})
|
||||
});
|
||||
|
||||
match (local, remote) {
|
||||
@@ -453,10 +482,13 @@ pub(super) fn plan_file(
|
||||
(false, true) => FilePlan::Download,
|
||||
// With a baseline, both-changed is a genuine concurrent edit and must
|
||||
// be preserved as a conflict. Without one, "both changed" only means
|
||||
// this is the first sync and there was nothing to compare to: adopt
|
||||
// the baseline when the sizes already match rather than conflict-copy
|
||||
// an already-identical file. Divergent sizes stay a real conflict.
|
||||
(true, true) if baseline.is_none() && l.size == r.size => FilePlan::AdoptBaseline,
|
||||
// this is the first sync and there was nothing to compare to: when the
|
||||
// sizes match and Drive has a digest to compare against, the apply
|
||||
// step hashes the local file before adopting it. Divergent sizes, or
|
||||
// no digest, stay a real conflict: a missing digest proves nothing.
|
||||
(true, true) if baseline.is_none() && l.size == r.size && r.sha1.is_some() => {
|
||||
FilePlan::AdoptBaseline
|
||||
}
|
||||
(true, true) => FilePlan::Conflict,
|
||||
},
|
||||
(Some(_), None) if baseline.is_none() || local_changed => FilePlan::UploadNew,
|
||||
@@ -488,6 +520,8 @@ mod file_plan_tests {
|
||||
is_dir: false,
|
||||
mtime,
|
||||
size,
|
||||
revision: None,
|
||||
sha1: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -505,6 +539,7 @@ mod file_plan_tests {
|
||||
local_size,
|
||||
remote_rev: Some(remote_mtime.to_string()),
|
||||
remote_hash: Some(remote_size.to_string()),
|
||||
remote_revision_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,12 +661,28 @@ mod file_plan_tests {
|
||||
fn first_sync_of_an_identical_file_adopts_the_baseline_not_a_conflict() {
|
||||
// No baseline yet (first sync). Same size on both sides is a pre-existing
|
||||
// identical file, not a concurrent edit: adopt it rather than cut a
|
||||
// conflict copy of every file in the folder.
|
||||
// conflict copy of every file in the folder. The apply step still
|
||||
// hashes the local file before it records anything.
|
||||
let local = local(10, 40);
|
||||
let remote = RemoteItem {
|
||||
sha1: Some("da39a3ee".into()),
|
||||
..remote(30, 40)
|
||||
};
|
||||
assert_eq!(
|
||||
plan_file(Some(&local), Some(&remote), None),
|
||||
FilePlan::AdoptBaseline
|
||||
);
|
||||
}
|
||||
|
||||
/// A file on Drive without a digest cannot be compared without a download,
|
||||
/// and the same size is not the same content (B248).
|
||||
#[test]
|
||||
fn first_sync_without_a_remote_digest_is_a_conflict() {
|
||||
let local = local(10, 40);
|
||||
let remote = remote(30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&local), Some(&remote), None),
|
||||
FilePlan::AdoptBaseline
|
||||
FilePlan::Conflict
|
||||
);
|
||||
}
|
||||
|
||||
@@ -647,6 +698,64 @@ mod file_plan_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Another computer's new revision can leave the modification time and
|
||||
/// the size as they were. The revision id still tells it apart (B249).
|
||||
#[test]
|
||||
fn a_new_remote_revision_is_a_change_even_at_the_same_mtime_and_size() {
|
||||
let mut base = baseline(10, 20, 30, 40);
|
||||
base.remote_revision_id = Some("r1".into());
|
||||
let unedited = local(10, 20);
|
||||
let same = RemoteItem {
|
||||
revision: Some("r1".into()),
|
||||
..remote(30, 40)
|
||||
};
|
||||
let newer = RemoteItem {
|
||||
revision: Some("r2".into()),
|
||||
..remote(30, 40)
|
||||
};
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&same), Some(&base)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&newer), Some(&base)),
|
||||
FilePlan::Download
|
||||
);
|
||||
let edited = local(11, 20);
|
||||
assert_eq!(
|
||||
plan_file(Some(&edited), Some(&newer), Some(&base)),
|
||||
FilePlan::Conflict
|
||||
);
|
||||
|
||||
// A row from before schema 44 has no id to compare, and a node read
|
||||
// without one says nothing: both fall back to the signature.
|
||||
let old = baseline(10, 20, 30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&newer), Some(&old)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&remote(30, 40)), Some(&base)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
}
|
||||
|
||||
/// The walk reuses the recorded size only while the revision is the one
|
||||
/// the baseline holds.
|
||||
#[test]
|
||||
fn a_moved_revision_makes_the_walk_read_the_size_again() {
|
||||
let mut base = baseline(10, 20, 30, 40);
|
||||
base.remote_revision_id = Some("r1".into());
|
||||
let rows = HashMap::from([("file".to_string(), base)]);
|
||||
assert_eq!(
|
||||
unchanged_remote_size(&rows, "file", 30, Some("r1")),
|
||||
Some(40)
|
||||
);
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 30, None), Some(40));
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 30, Some("r2")), None);
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 31, Some("r1")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_remote_signature_is_treated_as_changed() {
|
||||
let local = local(10, 20);
|
||||
|
||||
@@ -625,7 +625,7 @@ pub(crate) fn sha1_of(reader: &mut (dyn Read + Send)) -> CoreResult<String> {
|
||||
Ok(hex_lower(&hasher.finalize()))
|
||||
}
|
||||
|
||||
fn hex_lower(bytes: &[u8]) -> String {
|
||||
pub(crate) fn hex_lower(bytes: &[u8]) -> String {
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
out.push_str(&format!("{byte:02x}"));
|
||||
|
||||
Reference in new issue
Block a user