mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-04 08:56:59 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
217738318e | ||
|
|
9291580fd6 | ||
|
|
812eb3d0bc | ||
|
|
31863a1da1 | ||
|
|
57a825887c | ||
|
|
1436c5f37e | ||
|
|
f0a43fb923 | ||
|
|
b47620651f | ||
|
|
2baad6720d | ||
|
|
c271a1897c | ||
|
|
d26dc8ada4 | ||
|
|
93f02672e6 | ||
|
|
307305282e | ||
|
|
a8ed6060af | ||
|
|
9dfe4c5450 | ||
|
|
da013c29d6 | ||
|
|
457a9d26f4 | ||
|
|
a68dae9422 | ||
|
|
e743cd60cf | ||
|
|
f9b1638603 | ||
|
|
6539c7314b | ||
|
|
903452a15b | ||
|
|
4e0fd3c0b1 | ||
|
|
a6f019ebe6 | ||
|
|
0f3c06f982 | ||
|
|
4f141c78ca | ||
|
|
ce60e006ac | ||
|
|
1612a9d07a | ||
|
|
378667fb37 | ||
|
|
3b81cc5240 | ||
|
|
d850265225 | ||
|
|
e32f72bfca | ||
|
|
2502af6e3c | ||
|
|
4757b957d2 | ||
|
|
8e54ed0517 | ||
|
|
9bbc994a4f | ||
|
|
ffa0fe698d | ||
|
|
25c46cfe0a | ||
|
|
573a29e596 | ||
|
|
ce2bc0ac84 | ||
|
|
b37d5e7928 | ||
|
|
18587124ac | ||
|
|
cd18029944 | ||
|
|
d6dd41407f | ||
|
|
60445d3b26 | ||
|
|
c2f93449ad | ||
|
|
4676a4e32b | ||
|
|
16fca10e6c | ||
|
|
dbc4d9a8c5 | ||
|
|
9c93860cd8 | ||
|
|
85ec91d6b3 | ||
|
|
d5d3b2da3f | ||
|
|
e5f3f612f8 | ||
|
|
3f74b95b34 | ||
|
|
7269cb5118 | ||
|
|
6d75a56b00 | ||
|
|
eb1c105bbf | ||
|
|
85e80d8aac | ||
|
|
23dc5c3723 | ||
|
|
50ae25bb88 | ||
|
|
9b2b9f24c5 | ||
|
|
ae6650f7e0 | ||
|
|
f029e652a1 | ||
|
|
9b3651e4e8 | ||
|
|
0dc476dfb7 | ||
|
|
de258cbd60 | ||
|
|
4ea18e9187 | ||
|
|
a6f36d56f3 | ||
|
|
77d0f254fc | ||
|
|
4eccdeeda6 | ||
|
|
473842eefc | ||
|
|
5e0f30cc2c | ||
|
|
b3e33a7002 |
No files matched your search
@@ -67,7 +67,13 @@ jobs:
|
|||||||
User aur
|
User aur
|
||||||
EOF
|
EOF
|
||||||
# Pin the host key to the fingerprint published on aur.archlinux.org.
|
# Pin the host key to the fingerprint published on aur.archlinux.org.
|
||||||
ssh-keyscan -t ed25519 aur.archlinux.org > "$HOME/.ssh/known_hosts"
|
# The scan sometimes returns nothing from the runner, so retry it.
|
||||||
|
for attempt in 1 2 3 4 5; do
|
||||||
|
ssh-keyscan -T 15 -t ed25519 aur.archlinux.org > "$HOME/.ssh/known_hosts" &&
|
||||||
|
[ -s "$HOME/.ssh/known_hosts" ] && break
|
||||||
|
echo "ssh-keyscan attempt $attempt returned no key; retrying"
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
ssh-keygen -lf "$HOME/.ssh/known_hosts" | grep -q 'SHA256:RFzBCUItH9LZS0cKB5UE6ceAYhBD5C8GeOBip8Z11+4'
|
ssh-keygen -lf "$HOME/.ssh/known_hosts" | grep -q 'SHA256:RFzBCUItH9LZS0cKB5UE6ceAYhBD5C8GeOBip8Z11+4'
|
||||||
chown -R builder: "$HOME/.ssh"
|
chown -R builder: "$HOME/.ssh"
|
||||||
chmod 600 "$HOME/.ssh/aur"
|
chmod 600 "$HOME/.ssh/aur"
|
||||||
|
|||||||
@@ -101,6 +101,16 @@ jobs:
|
|||||||
- name: Simulation runs
|
- name: Simulation runs
|
||||||
run: 'cargo test -p pdfs-fuse --lib --locked sim:: -- --ignored --test-threads=1'
|
run: 'cargo test -p pdfs-fuse --lib --locked sim:: -- --ignored --test-threads=1'
|
||||||
|
|
||||||
|
# A hung step writes every daemon thread's stack next to the run's
|
||||||
|
# state; without them a hang that does not replay cannot be told apart.
|
||||||
|
- name: Keep the stacks of a hung run
|
||||||
|
if: failure()
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: simulation-stacks
|
||||||
|
path: /tmp/pdfs-sim-*/stacks.txt
|
||||||
|
if-no-files-found: ignore
|
||||||
|
|
||||||
# Keeps the release-profile dependencies cached on main, where the release
|
# Keeps the release-profile dependencies cached on main, where the release
|
||||||
# workflow can restore them; a tag cannot reuse the caches of earlier tags.
|
# workflow can restore them; a tag cannot reuse the caches of earlier tags.
|
||||||
release-cache:
|
release-cache:
|
||||||
|
|||||||
Generated
+4
-4
@@ -3474,7 +3474,7 @@ checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-cli"
|
name = "pdfs-cli"
|
||||||
version = "2.10.0"
|
version = "3.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"clap",
|
"clap",
|
||||||
@@ -3489,7 +3489,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-core"
|
name = "pdfs-core"
|
||||||
version = "2.10.0"
|
version = "3.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"dbus-secret-service-keyring-store",
|
"dbus-secret-service-keyring-store",
|
||||||
@@ -3513,7 +3513,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-fuse"
|
name = "pdfs-fuse"
|
||||||
version = "2.10.0"
|
version = "3.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64 0.23.1",
|
"base64 0.23.1",
|
||||||
@@ -3535,7 +3535,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-gui"
|
name = "pdfs-gui"
|
||||||
version = "2.10.0"
|
version = "3.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-channel",
|
"async-channel",
|
||||||
|
|||||||
+1
-1
@@ -3,7 +3,7 @@ resolver = "2"
|
|||||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "2.10.0"
|
version = "3.0.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||||
|
|||||||
@@ -38,8 +38,9 @@ scriptable CLI.
|
|||||||
- **Synced folders.** Back up folders such as `~/Documents` under your computer in Proton Drive,
|
- **Synced folders.** Back up folders such as `~/Documents` under your computer in Proton Drive,
|
||||||
as a full two-way copy or online only. Gitignore-style `.pdfsignore` rules, conflict copies
|
as a full two-way copy or online only. Gitignore-style `.pdfsignore` rules, conflict copies
|
||||||
instead of overwrites, pause and bandwidth limits.
|
instead of overwrites, pause and bandwidth limits.
|
||||||
- **Safe writes.** Changes are staged on disk and queued before they are acknowledged, so they
|
- **Works at local speed.** Making, renaming and deleting files and folders returns at once, online
|
||||||
survive a lost connection, a crash or a reboot and upload when they can.
|
or offline. Changes are staged on disk and queued before they are acknowledged, so they survive
|
||||||
|
a lost connection, a crash or a reboot and reach Drive when they can.
|
||||||
|
|
||||||
**Desktop app**
|
**Desktop app**
|
||||||
|
|
||||||
@@ -50,7 +51,8 @@ scriptable CLI.
|
|||||||
- **Sharing.** Invite people as viewer, editor or admin, create public links with password and
|
- **Sharing.** Invite people as viewer, editor or admin, create public links with password and
|
||||||
expiry, accept invitations.
|
expiry, accept invitations.
|
||||||
- **Sync overview.** What is uploading, what is waiting, what failed and why, and a history of
|
- **Sync overview.** What is uploading, what is waiting, what failed and why, and a history of
|
||||||
what happened.
|
what happened. A change Drive refuses (storage full, access taken away) shows at once, on the
|
||||||
|
file too, and can be exported or discarded.
|
||||||
- **Tray icon** with status, pause and quick actions, and a **search launcher** (`pdfs-prompt`)
|
- **Tray icon** with status, pause and quick actions, and a **search launcher** (`pdfs-prompt`)
|
||||||
that searches Proton Drive and your home folder together, as its own window, through fuzzel,
|
that searches Proton Drive and your home folder together, as its own window, through fuzzel,
|
||||||
rofi or wofi, or live in `fzf`.
|
rofi or wofi, or live in `fzf`.
|
||||||
|
|||||||
+123
-1
@@ -14,7 +14,7 @@ use pdfs_core::cache::ContentCache;
|
|||||||
use pdfs_core::config::AppDirs;
|
use pdfs_core::config::AppDirs;
|
||||||
use pdfs_core::control::{
|
use pdfs_core::control::{
|
||||||
ConflictKeep, ErrorKind, MountKind, RefreshScope, Request as CtlRequest,
|
ConflictKeep, ErrorKind, MountKind, RefreshScope, Request as CtlRequest,
|
||||||
Response as CtlResponse, RestoreItem as CtlRestoreItem, ShareEntryKind, SyncPhase,
|
Response as CtlResponse, RestoreItem as CtlRestoreItem, ShareEntryKind, SyncIssue, SyncPhase,
|
||||||
pending_summary,
|
pending_summary,
|
||||||
};
|
};
|
||||||
use pdfs_core::db::Db;
|
use pdfs_core::db::Db;
|
||||||
@@ -677,6 +677,25 @@ enum SyncCmd {
|
|||||||
},
|
},
|
||||||
/// List uploads and changes not yet on Proton Drive.
|
/// List uploads and changes not yet on Proton Drive.
|
||||||
Queue,
|
Queue,
|
||||||
|
/// List queued changes Proton Drive refused or that keep failing, and what to do.
|
||||||
|
Issues,
|
||||||
|
/// Save the content of a queued upload (by id from `sync queue`) to a file.
|
||||||
|
Export {
|
||||||
|
/// Queue id.
|
||||||
|
id: i64,
|
||||||
|
/// Where to save it: a new file, or an existing folder to save it in.
|
||||||
|
dest: PathBuf,
|
||||||
|
},
|
||||||
|
/// Drop a queued change (by id from `sync queue`) and undo it here: a new file
|
||||||
|
/// or folder goes, anything else goes back to what Proton Drive has. Its
|
||||||
|
/// content is lost unless saved with `sync export` first.
|
||||||
|
Discard {
|
||||||
|
/// Queue id.
|
||||||
|
id: i64,
|
||||||
|
/// Do not ask first.
|
||||||
|
#[arg(long)]
|
||||||
|
yes: bool,
|
||||||
|
},
|
||||||
/// Retry a queued operation now (by id from `sync queue`), or every failed one.
|
/// Retry a queued operation now (by id from `sync queue`), or every failed one.
|
||||||
Retry {
|
Retry {
|
||||||
/// Queue id; omit to retry every failed operation.
|
/// Queue id; omit to retry every failed operation.
|
||||||
@@ -1028,7 +1047,17 @@ fn cmd_sync(action: SyncCmd) -> Result<()> {
|
|||||||
})?)?
|
})?)?
|
||||||
}
|
}
|
||||||
SyncCmd::Queue => return cmd_sync_queue(),
|
SyncCmd::Queue => return cmd_sync_queue(),
|
||||||
|
SyncCmd::Issues => return cmd_sync_issues(),
|
||||||
|
SyncCmd::Export { id, dest } => return cmd_sync_export(id, &dest),
|
||||||
SyncCmd::Limit { up, down } => return cmd_sync_limit(up, down),
|
SyncCmd::Limit { up, down } => return cmd_sync_limit(up, down),
|
||||||
|
SyncCmd::Discard { id, yes } => {
|
||||||
|
if !yes {
|
||||||
|
confirm(&format!(
|
||||||
|
"Discard queued change {id} and undo it here? Content not exported is lost."
|
||||||
|
))?;
|
||||||
|
}
|
||||||
|
ok_or_bail(control_request(CtlRequest::DiscardPendingOp { id })?)?
|
||||||
|
}
|
||||||
SyncCmd::Retry { id } => ok_or_bail(control_request(CtlRequest::RetryPendingOp { id })?)?,
|
SyncCmd::Retry { id } => ok_or_bail(control_request(CtlRequest::RetryPendingOp { id })?)?,
|
||||||
SyncCmd::Resume { folder: None } => {
|
SyncCmd::Resume { folder: None } => {
|
||||||
ok_or_bail(control_request(CtlRequest::SetSyncPaused {
|
ok_or_bail(control_request(CtlRequest::SetSyncPaused {
|
||||||
@@ -1106,6 +1135,9 @@ fn cmd_sync_queue() -> Result<()> {
|
|||||||
(false, _) => "due now".to_string(),
|
(false, _) => "due now".to_string(),
|
||||||
};
|
};
|
||||||
println!("[{}] {} {} ({state})", op.id, op.kind, op.path);
|
println!("[{}] {} {} ({state})", op.id, op.kind, op.path);
|
||||||
|
if let Some(issue) = op.issue {
|
||||||
|
println!(" needs attention: {}", issue_text(issue));
|
||||||
|
}
|
||||||
if let Some(error) = &op.last_error {
|
if let Some(error) = &op.last_error {
|
||||||
println!(" failed {} time(s): {error}", op.attempts);
|
println!(" failed {} time(s): {error}", op.attempts);
|
||||||
}
|
}
|
||||||
@@ -1117,6 +1149,91 @@ fn cmd_sync_queue() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `pdfs sync issues`: the queued changes that need the user, and what to do.
|
||||||
|
fn cmd_sync_issues() -> Result<()> {
|
||||||
|
let response = control_request(CtlRequest::ListPendingOps)?;
|
||||||
|
let items = match response {
|
||||||
|
CtlResponse::PendingOps { items } => items,
|
||||||
|
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||||
|
other => bail!("unexpected response: {other:?}"),
|
||||||
|
};
|
||||||
|
let issues: Vec<_> = items.into_iter().filter(|op| op.failing).collect();
|
||||||
|
if json_enabled() {
|
||||||
|
println!("{}", serde_json::to_string_pretty(&issues)?);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if issues.is_empty() {
|
||||||
|
println!("No sync issues.");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
for op in issues {
|
||||||
|
println!("[{}] {} {}", op.id, op.kind, op.path);
|
||||||
|
match op.issue {
|
||||||
|
Some(issue) => println!(" {}", issue_text(issue)),
|
||||||
|
None => println!(" keeps failing; it is retried."),
|
||||||
|
}
|
||||||
|
if let Some(error) = &op.last_error {
|
||||||
|
println!(" last error: {error}");
|
||||||
|
}
|
||||||
|
if op.exportable {
|
||||||
|
println!(
|
||||||
|
" save a copy: pdfs sync export {} <file or folder>",
|
||||||
|
op.id
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a sync issue means, and what the user can do about it.
|
||||||
|
fn issue_text(issue: SyncIssue) -> &'static str {
|
||||||
|
match issue {
|
||||||
|
SyncIssue::Quota => {
|
||||||
|
"Proton Drive storage is full. Free up space or upgrade; the change is retried."
|
||||||
|
}
|
||||||
|
SyncIssue::Access => {
|
||||||
|
"you can no longer change this on Proton Drive, for example because a share was \
|
||||||
|
revoked. Save a copy to keep the content."
|
||||||
|
}
|
||||||
|
SyncIssue::Missing => {
|
||||||
|
"the file or its folder is gone from Proton Drive. Save a copy to keep the content."
|
||||||
|
}
|
||||||
|
SyncIssue::Limit => {
|
||||||
|
"the folder holds too many items or is nested too deep. Move the file to another \
|
||||||
|
folder."
|
||||||
|
}
|
||||||
|
SyncIssue::Rejected => {
|
||||||
|
"Proton Drive rejected the change, for example the name. Rename the file."
|
||||||
|
}
|
||||||
|
SyncIssue::Unknown => "Proton Drive refused the change.",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `pdfs sync export`: save a queued upload's content. A folder as `dest`
|
||||||
|
/// gets the file under its own name.
|
||||||
|
fn cmd_sync_export(id: i64, dest: &Path) -> Result<()> {
|
||||||
|
let mut dest =
|
||||||
|
std::path::absolute(dest).with_context(|| format!("resolving {}", dest.display()))?;
|
||||||
|
if dest.is_dir() {
|
||||||
|
let items = match control_request(CtlRequest::ListPendingOps)? {
|
||||||
|
CtlResponse::PendingOps { items } => items,
|
||||||
|
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||||
|
other => bail!("unexpected response: {other:?}"),
|
||||||
|
};
|
||||||
|
let Some(op) = items.iter().find(|op| op.id == id) else {
|
||||||
|
bail!("no queued change {id}; see `pdfs sync queue`");
|
||||||
|
};
|
||||||
|
let name = Path::new(&op.path)
|
||||||
|
.file_name()
|
||||||
|
.with_context(|| format!("{} has no file name", op.path))?;
|
||||||
|
dest.push(name);
|
||||||
|
}
|
||||||
|
ok_or_bail(control_request(CtlRequest::ExportPendingOp {
|
||||||
|
id,
|
||||||
|
dest: dest.display().to_string(),
|
||||||
|
})?)
|
||||||
|
}
|
||||||
|
|
||||||
/// `pdfs sync restore`: list what this machine's device holds, confirm where
|
/// `pdfs sync restore`: list what this machine's device holds, confirm where
|
||||||
/// each folder should land, then hand the mapping to the daemon.
|
/// each folder should land, then hand the mapping to the daemon.
|
||||||
///
|
///
|
||||||
@@ -1983,6 +2100,7 @@ fn mount_once(mountpoint: Option<PathBuf>) -> Result<pdfs_fuse::MountOutcome> {
|
|||||||
sweep_mode: config.resolved_conflict_sweep(),
|
sweep_mode: config.resolved_conflict_sweep(),
|
||||||
upload_limit: config.upload_limit.unwrap_or(0),
|
upload_limit: config.upload_limit.unwrap_or(0),
|
||||||
download_limit: config.download_limit.unwrap_or(0),
|
download_limit: config.download_limit.unwrap_or(0),
|
||||||
|
local_first: config.local_first.unwrap_or(true),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1990,6 +2108,9 @@ fn mount_once(mountpoint: Option<PathBuf>) -> Result<pdfs_fuse::MountOutcome> {
|
|||||||
if let Err(e) = rt.block_on(auth::persist(&session)) {
|
if let Err(e) = rt.block_on(auth::persist(&session)) {
|
||||||
tracing::debug!(error = %e, "no new tokens to persist on shutdown");
|
tracing::debug!(error = %e, "no new tokens to persist on shutdown");
|
||||||
}
|
}
|
||||||
|
// Dropping the runtime waits for every blocking task, however long; a call
|
||||||
|
// still stuck on Drive would hold the stop until systemd kills it (B165).
|
||||||
|
rt.shutdown_timeout(std::time::Duration::from_secs(2));
|
||||||
|
|
||||||
let outcome = result.context("mount failed")?;
|
let outcome = result.context("mount failed")?;
|
||||||
Ok(outcome)
|
Ok(outcome)
|
||||||
@@ -3433,6 +3554,7 @@ mod login_wait_tests {
|
|||||||
shared_by_unverified: false,
|
shared_by_unverified: false,
|
||||||
trashed_at: 0,
|
trashed_at: 0,
|
||||||
trashed_from: None,
|
trashed_from: None,
|
||||||
|
issue: None,
|
||||||
};
|
};
|
||||||
assert_eq!(shared_by_line(&entry), None, "no invitation, no line");
|
assert_eq!(shared_by_line(&entry), None, "no invitation, no line");
|
||||||
entry.shared_by = "alice@proton.me".into();
|
entry.shared_by = "alice@proton.me".into();
|
||||||
|
|||||||
@@ -845,6 +845,9 @@ impl ContentCache {
|
|||||||
}
|
}
|
||||||
let bytes = std::fs::metadata(&tmp)?.len();
|
let bytes = std::fs::metadata(&tmp)?.len();
|
||||||
std::fs::rename(&tmp, &blob)?;
|
std::fs::rename(&tmp, &blob)?;
|
||||||
|
// Linked from the blob already cached, the temp name was a second name
|
||||||
|
// of it, and the rename did nothing (`docs/BUGS.md` B187).
|
||||||
|
let _ = std::fs::remove_file(&tmp);
|
||||||
let meta = serde_json::to_vec(&Meta { mtime, size })?;
|
let meta = serde_json::to_vec(&Meta { mtime, size })?;
|
||||||
std::fs::write(self.meta_path(uid), meta)?;
|
std::fs::write(self.meta_path(uid), meta)?;
|
||||||
self.db
|
self.db
|
||||||
@@ -2197,6 +2200,24 @@ mod tests {
|
|||||||
assert_eq!(c.read_range(&u, 200, 5, 0, 5).unwrap(), b"newer");
|
assert_eq!(c.read_range(&u, 200, 5, 0, 5).unwrap(), b"newer");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A landed create and the revision after it can adopt the same staged
|
||||||
|
/// blob. The link to the temp name is then a second name of the cached
|
||||||
|
/// blob, and a rename between two names of one file does nothing: the
|
||||||
|
/// temp name stayed until the next start swept it (`docs/BUGS.md` B187).
|
||||||
|
#[test]
|
||||||
|
fn store_file_twice_from_the_same_blob_leaves_no_temp_file() {
|
||||||
|
let (c, d) = cache();
|
||||||
|
let u = uid("a");
|
||||||
|
let src = d.path().join("staged-blob");
|
||||||
|
std::fs::write(&src, b"staged").unwrap();
|
||||||
|
|
||||||
|
c.store_file(&u, 100, 6, &src).unwrap();
|
||||||
|
c.store_file(&u, 100, 6, &src).unwrap();
|
||||||
|
|
||||||
|
assert!(!c.blob_path(&u).with_extension("tmp").exists());
|
||||||
|
assert_eq!(c.read_range(&u, 100, 6, 0, 6).unwrap(), b"staged");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reconcile_rebuilds_index_from_disk() {
|
fn reconcile_rebuilds_index_from_disk() {
|
||||||
let dir = TempDir::new();
|
let dir = TempDir::new();
|
||||||
|
|||||||
@@ -154,6 +154,13 @@ pub struct AppConfig {
|
|||||||
/// configs predating the field.
|
/// configs predating the field.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub online_map: bool,
|
pub online_map: bool,
|
||||||
|
/// Record a new folder, a new file, a rename or a delete through the mount
|
||||||
|
/// locally and send it from the queue, instead of waiting for Drive in the
|
||||||
|
/// syscall. `None` means on. `false` brings back the old path for one
|
||||||
|
/// release (docs/MILESTONE-3.0.0.md §10). Defaulted for configs predating
|
||||||
|
/// the field.
|
||||||
|
#[serde(default)]
|
||||||
|
pub local_first: Option<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The My Files page's view choices, remembered between runs.
|
/// The My Files page's view choices, remembered between runs.
|
||||||
@@ -228,6 +235,7 @@ impl Default for AppConfig {
|
|||||||
tray_hidden: false,
|
tray_hidden: false,
|
||||||
language: None,
|
language: None,
|
||||||
online_map: false,
|
online_map: false,
|
||||||
|
local_first: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -584,6 +592,7 @@ mod tests {
|
|||||||
tray_hidden: true,
|
tray_hidden: true,
|
||||||
language: Some("de".to_string()),
|
language: Some("de".to_string()),
|
||||||
online_map: true,
|
online_map: true,
|
||||||
|
local_first: Some(false),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_string(&config).unwrap();
|
let json = serde_json::to_string(&config).unwrap();
|
||||||
let decoded: AppConfig = serde_json::from_str(&json).unwrap();
|
let decoded: AppConfig = serde_json::from_str(&json).unwrap();
|
||||||
|
|||||||
@@ -477,6 +477,15 @@ pub enum Request {
|
|||||||
/// Retry a backed-off queued op now instead of waiting out its backoff;
|
/// Retry a backed-off queued op now instead of waiting out its backoff;
|
||||||
/// every failed op when `id` is `None`. Replies with [`Response::Ok`].
|
/// every failed op when `id` is `None`. Replies with [`Response::Ok`].
|
||||||
RetryPendingOp { id: Option<i64> },
|
RetryPendingOp { id: Option<i64> },
|
||||||
|
/// Copy the content of a queued upload (`id` from
|
||||||
|
/// [`Request::ListPendingOps`]) to `dest`, an absolute path that must not
|
||||||
|
/// exist yet. The op stays queued. Replies with [`Response::Ok`].
|
||||||
|
ExportPendingOp { id: i64, dest: String },
|
||||||
|
/// Drop a queued change (`id` from [`Request::ListPendingOps`]) and undo
|
||||||
|
/// it locally: a node made here goes, a node Drive has goes back to what
|
||||||
|
/// Drive has, along with every other change queued on it. Its content is
|
||||||
|
/// lost unless it was exported first. Replies with [`Response::Ok`].
|
||||||
|
DiscardPendingOp { id: i64 },
|
||||||
/// List the `(sync-conflict …)` copies under My Files and in synced
|
/// List the `(sync-conflict …)` copies under My Files and in synced
|
||||||
/// folders. Replies with
|
/// folders. Replies with
|
||||||
/// [`Response::Conflicts`].
|
/// [`Response::Conflicts`].
|
||||||
@@ -880,6 +889,59 @@ pub struct PendingOpInfo {
|
|||||||
pub parked: bool,
|
pub parked: bool,
|
||||||
/// Failed often enough to count as stuck.
|
/// Failed often enough to count as stuck.
|
||||||
pub failing: bool,
|
pub failing: bool,
|
||||||
|
/// Why Drive refused it, when it did; such an op needs attention at once.
|
||||||
|
#[serde(default)]
|
||||||
|
pub issue: Option<SyncIssue>,
|
||||||
|
/// Carries file content that [`Request::ExportPendingOp`] can save.
|
||||||
|
#[serde(default)]
|
||||||
|
pub exportable: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why Drive refused a queued change (in [`PendingOpInfo`]). The change stays
|
||||||
|
/// queued and keeps being retried; nothing is deleted.
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum SyncIssue {
|
||||||
|
/// The account's storage is full.
|
||||||
|
Quota,
|
||||||
|
/// The account may no longer change the file or its folder, for example
|
||||||
|
/// because a share was revoked.
|
||||||
|
Access,
|
||||||
|
/// The file or the folder it goes into is gone from Drive.
|
||||||
|
Missing,
|
||||||
|
/// The folder holds too many items, or is nested too deep.
|
||||||
|
Limit,
|
||||||
|
/// Drive rejected the change, for example a name it does not accept.
|
||||||
|
Rejected,
|
||||||
|
/// An issue from a newer daemon.
|
||||||
|
#[serde(other)]
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SyncIssue {
|
||||||
|
/// The name stored in the database and sent on the wire.
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
SyncIssue::Quota => "quota",
|
||||||
|
SyncIssue::Access => "access",
|
||||||
|
SyncIssue::Missing => "missing",
|
||||||
|
SyncIssue::Limit => "limit",
|
||||||
|
SyncIssue::Rejected => "rejected",
|
||||||
|
SyncIssue::Unknown => "unknown",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The issue stored as `name`.
|
||||||
|
pub fn parse(name: &str) -> Self {
|
||||||
|
match name {
|
||||||
|
"quota" => SyncIssue::Quota,
|
||||||
|
"access" => SyncIssue::Access,
|
||||||
|
"missing" => SyncIssue::Missing,
|
||||||
|
"limit" => SyncIssue::Limit,
|
||||||
|
"rejected" => SyncIssue::Rejected,
|
||||||
|
_ => SyncIssue::Unknown,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One synced local folder on this machine's device (in [`Response::SyncFolders`]).
|
/// One synced local folder on this machine's device (in [`Response::SyncFolders`]).
|
||||||
@@ -1234,6 +1296,10 @@ pub struct DirEntry {
|
|||||||
/// not known, and for anything not in the trash.
|
/// not known, and for anything not in the trash.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub trashed_from: Option<String>,
|
pub trashed_from: Option<String>,
|
||||||
|
/// Why Drive refused a change queued on the node, for a [`Request::ListDir`]
|
||||||
|
/// entry; `None` otherwise.
|
||||||
|
#[serde(default)]
|
||||||
|
pub issue: Option<SyncIssue>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One hit in a [`Request::Search`] result. Like [`DirEntry`] but carries the
|
/// One hit in a [`Request::Search`] result. Like [`DirEntry`] but carries the
|
||||||
@@ -3290,6 +3356,17 @@ mod tests {
|
|||||||
},
|
},
|
||||||
r#"{"SaveRevisionAsByUid":{"uid":"vol~link","revision_id":"rev-3","dest":"/tmp/out.bin"}}"#,
|
r#"{"SaveRevisionAsByUid":{"uid":"vol~link","revision_id":"rev-3","dest":"/tmp/out.bin"}}"#,
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
Request::ExportPendingOp {
|
||||||
|
id: 7,
|
||||||
|
dest: "/tmp/out.bin".into(),
|
||||||
|
},
|
||||||
|
r#"{"ExportPendingOp":{"id":7,"dest":"/tmp/out.bin"}}"#,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
Request::DiscardPendingOp { id: 7 },
|
||||||
|
r#"{"DiscardPendingOp":{"id":7}}"#,
|
||||||
|
),
|
||||||
];
|
];
|
||||||
for (request, wire) in cases {
|
for (request, wire) in cases {
|
||||||
assert_eq!(serde_json::to_string(&request).unwrap(), wire);
|
assert_eq!(serde_json::to_string(&request).unwrap(), wire);
|
||||||
@@ -3298,6 +3375,25 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A daemon that predates sync issues sends neither field, and a newer one
|
||||||
|
/// may send an issue this build does not know.
|
||||||
|
#[test]
|
||||||
|
fn a_queued_op_from_another_daemon_version_still_decodes() {
|
||||||
|
let old = r#"{"id":1,"kind":"create","path":"a.txt","attempts":0,"last_error":null,
|
||||||
|
"queued_at":5,"next_attempt_at":5,"parked":false,"failing":false}"#;
|
||||||
|
let op: PendingOpInfo = serde_json::from_str(old).unwrap();
|
||||||
|
assert_eq!(op.issue, None);
|
||||||
|
assert!(!op.exportable);
|
||||||
|
|
||||||
|
let newer = old.replace(r#""failing":false"#, r#""failing":true,"issue":"embargo""#);
|
||||||
|
let op: PendingOpInfo = serde_json::from_str(&newer).unwrap();
|
||||||
|
assert_eq!(op.issue, Some(SyncIssue::Unknown));
|
||||||
|
assert_eq!(
|
||||||
|
SyncIssue::parse(SyncIssue::Quota.as_str()),
|
||||||
|
SyncIssue::Quota
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// A front-end that predates the favorites filter sends no `favorites`
|
/// A front-end that predates the favorites filter sends no `favorites`
|
||||||
/// field, and must keep getting the whole timeline rather than an empty page.
|
/// field, and must keep getting the whole timeline rather than an empty page.
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -5,11 +5,13 @@
|
|||||||
//! is already on disk somewhere, so a change here would silently diverge from
|
//! is already on disk somewhere, so a change here would silently diverge from
|
||||||
//! what an existing install actually has. Add a new one instead.
|
//! what an existing install actually has. Add a new one instead.
|
||||||
|
|
||||||
|
use rusqlite::OptionalExtension;
|
||||||
|
|
||||||
use super::Db;
|
use super::Db;
|
||||||
use crate::Result;
|
use crate::Result;
|
||||||
|
|
||||||
/// Current schema version. Bump on every forward migration added below.
|
/// Current schema version. Bump on every forward migration added below.
|
||||||
pub(super) const SCHEMA_VERSION: i64 = 36;
|
pub(super) const SCHEMA_VERSION: i64 = 42;
|
||||||
|
|
||||||
impl Db {
|
impl Db {
|
||||||
pub(super) fn migrate(&self) -> Result<()> {
|
pub(super) fn migrate(&self) -> Result<()> {
|
||||||
@@ -348,6 +350,96 @@ impl Db {
|
|||||||
tx.execute_batch(MIGRATION_V36)?;
|
tx.execute_batch(MIGRATION_V36)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if current < 37 {
|
||||||
|
// Same guards as V26-V36.
|
||||||
|
let sql: Option<String> = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
if sql.is_some_and(|sql| !sql.contains("AUTOINCREMENT")) {
|
||||||
|
tx.execute_batch(MIGRATION_V37)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current < 38 {
|
||||||
|
// Same guards as V26-V37.
|
||||||
|
let has_column: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'issue'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
let has_ops: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
if has_ops && !has_column {
|
||||||
|
tx.execute_batch(MIGRATION_V38)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current < 39 {
|
||||||
|
// Same guards as V26-V38.
|
||||||
|
let has_column: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'lid'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
let has_ops: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
if has_ops && !has_column {
|
||||||
|
tx.execute_batch(MIGRATION_V39)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current < 40 {
|
||||||
|
// Same guards as V26-V39.
|
||||||
|
let has_column: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM pragma_table_info('nodes') WHERE name = 'parent_lid'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
let has_nodes: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
if has_nodes && !has_column {
|
||||||
|
tx.execute_batch(MIGRATION_V40)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current < 41 {
|
||||||
|
// Needs both tables; on a database that never had them there is
|
||||||
|
// nothing to rewrite.
|
||||||
|
let tables: i64 = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master
|
||||||
|
WHERE type = 'table' AND name IN ('nodes', 'pending_op', 'pins')",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if tables == 3 {
|
||||||
|
tx.execute_batch(MIGRATION_V41)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if current < 42 {
|
||||||
|
// Same guards as V26-V40.
|
||||||
|
let has_column: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'sent_sha1'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
let has_ops: bool = tx.query_row(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)? > 0;
|
||||||
|
if has_ops && !has_column {
|
||||||
|
tx.execute_batch(MIGRATION_V42)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||||
@@ -1112,3 +1204,129 @@ CREATE INDEX idx_nodes_parent ON nodes(parent_uid);
|
|||||||
CREATE INDEX idx_nodes_name_nocase ON nodes(name COLLATE NOCASE);
|
CREATE INDEX idx_nodes_name_nocase ON nodes(name COLLATE NOCASE);
|
||||||
CREATE INDEX idx_nodes_parent_live ON nodes(parent_uid) WHERE trashed = 0;
|
CREATE INDEX idx_nodes_parent_live ON nodes(parent_uid) WHERE trashed = 0;
|
||||||
";
|
";
|
||||||
|
|
||||||
|
/// Schema v37: a `pending_op` id is never given out twice.
|
||||||
|
///
|
||||||
|
/// A plain rowid is the largest one plus one, so an op queued right after the
|
||||||
|
/// newest one was dropped took its id. The drain retires the op it is
|
||||||
|
/// working on by id, and so deleted the new op: a trash queued over a rename
|
||||||
|
/// on the wire was lost when the rename landed. `AUTOINCREMENT` never reuses
|
||||||
|
/// an id. Ids are kept, as is every index.
|
||||||
|
const MIGRATION_V37: &str = "
|
||||||
|
CREATE TABLE pending_op_v37 (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kind TEXT NOT NULL,
|
||||||
|
uid TEXT NOT NULL,
|
||||||
|
blob_path TEXT,
|
||||||
|
meta_json TEXT,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
last_error TEXT,
|
||||||
|
next_attempt_at INTEGER NOT NULL DEFAULT 0,
|
||||||
|
parent_uid TEXT,
|
||||||
|
name TEXT,
|
||||||
|
claimed_at INTEGER NOT NULL DEFAULT 0,
|
||||||
|
access_deferred_since INTEGER NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
INSERT INTO pending_op_v37
|
||||||
|
(id, kind, uid, blob_path, meta_json, created_at, attempts, last_error, next_attempt_at,
|
||||||
|
parent_uid, name, claimed_at, access_deferred_since)
|
||||||
|
SELECT id, kind, uid, blob_path, meta_json, created_at, attempts, last_error,
|
||||||
|
next_attempt_at, parent_uid, name, claimed_at, access_deferred_since
|
||||||
|
FROM pending_op;
|
||||||
|
DROP TABLE pending_op;
|
||||||
|
ALTER TABLE pending_op_v37 RENAME TO pending_op;
|
||||||
|
CREATE INDEX pending_op_uid ON pending_op(uid);
|
||||||
|
CREATE INDEX pending_op_parent ON pending_op(parent_uid);
|
||||||
|
CREATE INDEX idx_pending_op_claimed
|
||||||
|
ON pending_op(uid) WHERE claimed_at <> 0;
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Schema v38: why Drive refused a queued op, when it did (a sync issue).
|
||||||
|
/// `NULL` for an op that has not been refused, which is every op migrated.
|
||||||
|
const MIGRATION_V38: &str = "
|
||||||
|
ALTER TABLE pending_op ADD COLUMN issue TEXT;
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Schema v39: a queued op names its node and its parent by local id too.
|
||||||
|
///
|
||||||
|
/// A create made inside a folder that had not landed named the folder by its
|
||||||
|
/// placeholder uid, and every landing had to find and rewrite those strings.
|
||||||
|
/// The local id stays the same when the folder lands. An op whose node has no
|
||||||
|
/// row keeps `NULL` and is matched by uid, as before.
|
||||||
|
const MIGRATION_V39: &str = "
|
||||||
|
ALTER TABLE pending_op ADD COLUMN lid INTEGER;
|
||||||
|
ALTER TABLE pending_op ADD COLUMN parent_lid INTEGER;
|
||||||
|
UPDATE pending_op
|
||||||
|
SET lid = (SELECT lid FROM nodes WHERE nodes.uid = pending_op.uid),
|
||||||
|
parent_lid = (SELECT lid FROM nodes WHERE nodes.uid = pending_op.parent_uid);
|
||||||
|
CREATE INDEX pending_op_lid ON pending_op(lid);
|
||||||
|
CREATE INDEX pending_op_parent_lid ON pending_op(parent_lid);
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Schema v40: a node row names its parent by local id too.
|
||||||
|
///
|
||||||
|
/// The local id stays when a folder made here lands and its row takes the uid
|
||||||
|
/// Drive gave it, so a child linked by it needs no rewrite. `NULL` while the
|
||||||
|
/// parent has no row, as `parent_uid` then names nothing either.
|
||||||
|
const MIGRATION_V40: &str = "
|
||||||
|
ALTER TABLE nodes ADD COLUMN parent_lid INTEGER;
|
||||||
|
UPDATE nodes SET parent_lid = (SELECT p.lid FROM nodes p WHERE p.uid = nodes.parent_uid);
|
||||||
|
CREATE INDEX idx_nodes_parent_lid ON nodes(parent_lid);
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Schema v41: a node made on this machine goes by `local~<lid>` until it
|
||||||
|
/// lands, a function of its row, where it used to get a `local~<ms>-<n>`
|
||||||
|
/// placeholder. Every placeholder still stored is rewritten to its row's
|
||||||
|
/// stand-in: the row itself and its `node_json`, its children, the ops on it
|
||||||
|
/// or into it, and its pins. A placeholder op with no row keeps its uid and
|
||||||
|
/// drains as before.
|
||||||
|
const MIGRATION_V41: &str = "
|
||||||
|
CREATE TEMP TABLE local_v41 AS
|
||||||
|
SELECT uid AS old, 'local~' || lid AS new, lid FROM nodes
|
||||||
|
WHERE uid LIKE 'local~%' AND uid <> 'local~' || lid;
|
||||||
|
UPDATE pending_op SET uid = (SELECT new FROM local_v41 WHERE old = pending_op.uid)
|
||||||
|
WHERE uid IN (SELECT old FROM local_v41);
|
||||||
|
UPDATE pending_op SET parent_uid = (SELECT new FROM local_v41 WHERE old = pending_op.parent_uid)
|
||||||
|
WHERE parent_uid IN (SELECT old FROM local_v41);
|
||||||
|
UPDATE pending_op
|
||||||
|
SET meta_json = json_set(meta_json, '$.uid',
|
||||||
|
(SELECT new FROM local_v41 WHERE old = json_extract(pending_op.meta_json, '$.uid')))
|
||||||
|
WHERE json_valid(meta_json)
|
||||||
|
AND json_extract(meta_json, '$.uid') IN (SELECT old FROM local_v41);
|
||||||
|
UPDATE pending_op
|
||||||
|
SET meta_json = json_set(meta_json, '$.original_parent_uid',
|
||||||
|
(SELECT new FROM local_v41
|
||||||
|
WHERE old = json_extract(pending_op.meta_json, '$.original_parent_uid')))
|
||||||
|
WHERE json_valid(meta_json)
|
||||||
|
AND json_extract(meta_json, '$.original_parent_uid') IN (SELECT old FROM local_v41);
|
||||||
|
UPDATE pins SET uid = (SELECT new FROM local_v41 WHERE old = pins.uid)
|
||||||
|
WHERE uid IN (SELECT old FROM local_v41);
|
||||||
|
UPDATE nodes
|
||||||
|
SET node_json = json_set(node_json, '$.parent_uid.link_id',
|
||||||
|
(SELECT CAST(lid AS TEXT) FROM local_v41 WHERE old = nodes.parent_uid))
|
||||||
|
WHERE parent_uid IN (SELECT old FROM local_v41)
|
||||||
|
AND json_valid(node_json)
|
||||||
|
AND json_extract(node_json, '$.parent_uid.volume_id') = 'local';
|
||||||
|
UPDATE nodes SET parent_uid = (SELECT new FROM local_v41 WHERE old = nodes.parent_uid)
|
||||||
|
WHERE parent_uid IN (SELECT old FROM local_v41);
|
||||||
|
UPDATE nodes
|
||||||
|
SET node_json = json_set(node_json, '$.uid.link_id', CAST(lid AS TEXT))
|
||||||
|
WHERE lid IN (SELECT lid FROM local_v41)
|
||||||
|
AND json_valid(node_json)
|
||||||
|
AND json_extract(node_json, '$.uid.volume_id') = 'local';
|
||||||
|
UPDATE nodes SET uid = 'local~' || lid WHERE lid IN (SELECT lid FROM local_v41);
|
||||||
|
DROP TABLE local_v41;
|
||||||
|
";
|
||||||
|
|
||||||
|
/// Schema v42: the SHA-1 of each blob a queued create sent, space-separated.
|
||||||
|
/// A create whose answer was lost may have made its file with any of them,
|
||||||
|
/// and the op that finds that file again may run after a restart. `NULL` for
|
||||||
|
/// an op that has sent nothing, which is every op migrated: what the daemon
|
||||||
|
/// that wrote it sent is not known. Likewise the parent and name each attempt
|
||||||
|
/// was sent under, as a JSON list of pairs: a rename since then moves the op
|
||||||
|
/// but not the file Drive may have made.
|
||||||
|
const MIGRATION_V42: &str = "
|
||||||
|
ALTER TABLE pending_op ADD COLUMN sent_sha1 TEXT;
|
||||||
|
ALTER TABLE pending_op ADD COLUMN sent_to TEXT;
|
||||||
|
";
|
||||||
@@ -56,9 +56,9 @@ pub use local::LocalFileHit;
|
|||||||
pub use maintenance::{DbStats, VacuumOutcome};
|
pub use maintenance::{DbStats, VacuumOutcome};
|
||||||
pub use nodes::{PublishedSharedRoot, SearchHit, StoredNode};
|
pub use nodes::{PublishedSharedRoot, SearchHit, StoredNode};
|
||||||
pub use ops::{
|
pub use ops::{
|
||||||
AttachedBlob, CreateLanding, FAILING_ATTEMPTS, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME,
|
AttachedBlob, CreateLanding, CreateRetired, FAILING_ATTEMPTS, LOCAL_VOLUME, OP_CREATE,
|
||||||
OP_REVISION, OP_TRASH, PARK_EXPIRY_MS, PARK_UNTIL, PendingCounts, PendingOp, RenameMeta,
|
OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_EXPIRY_MS, PARK_UNTIL, PendingCounts,
|
||||||
op_supersedes,
|
PendingOp, RenameMeta, local_lid, local_uid, op_supersedes,
|
||||||
};
|
};
|
||||||
pub use photos::{StoredPhoto, StoredPlace, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN, TimelineRow};
|
pub use photos::{StoredPhoto, StoredPlace, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN, TimelineRow};
|
||||||
pub use pins::PinRow;
|
pub use pins::PinRow;
|
||||||
|
|||||||
@@ -3,9 +3,11 @@
|
|||||||
|
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
use rusqlite::types::Value;
|
||||||
use rusqlite::{OptionalExtension, Transaction, params};
|
use rusqlite::{OptionalExtension, Transaction, params};
|
||||||
|
|
||||||
use super::Db;
|
use super::Db;
|
||||||
|
use super::ops::{OP_CREATE, OP_MKDIR, OP_RENAME, local_lid, local_uid, wake_ops_waiting_for_tx};
|
||||||
use crate::{Access, Result};
|
use crate::{Access, Result};
|
||||||
use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId};
|
use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId};
|
||||||
use proton_drive_rs::{Node, NodeKind};
|
use proton_drive_rs::{Node, NodeKind};
|
||||||
@@ -64,12 +66,13 @@ impl Db {
|
|||||||
/// snapshot while offline, including after a live event expired its TTL.
|
/// snapshot while offline, including after a live event expired its TTL.
|
||||||
pub fn visible_children(&self, parent: &NodeUid) -> Result<Vec<Node>> {
|
pub fn visible_children(&self, parent: &NodeUid) -> Result<Vec<Node>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let (key, parent) = parent_key(&conn, &parent.to_string())?;
|
||||||
"SELECT node_json FROM nodes
|
let mut stmt = conn.prepare(&format!(
|
||||||
WHERE parent_uid = ?1 AND trashed = 0 AND node_json IS NOT NULL
|
"SELECT {NODE_NOW} FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
ORDER BY name, uid",
|
WHERE n.{key} = ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||||
)?;
|
ORDER BY n.name, n.uid"
|
||||||
let rows = stmt.query_map([parent.to_string()], |row| row.get::<_, String>(0))?;
|
))?;
|
||||||
|
let rows = stmt.query_map([parent], |row| row.get::<_, String>(0))?;
|
||||||
let mut nodes = Vec::new();
|
let mut nodes = Vec::new();
|
||||||
for row in rows {
|
for row in rows {
|
||||||
nodes.push(serde_json::from_str(&row?)?);
|
nodes.push(serde_json::from_str(&row?)?);
|
||||||
@@ -266,10 +269,25 @@ impl Db {
|
|||||||
/// resolve its shared-tree access. Removing the op first could resurrect
|
/// resolve its shared-tree access. Removing the op first could resurrect
|
||||||
/// that row after a crash; removing the row first could strand a now
|
/// that row after a crash; removing the row first could strand a now
|
||||||
/// unauthorizable op. One transaction closes both windows.
|
/// unauthorizable op. One transaction closes both windows.
|
||||||
pub fn complete_trash_op(&self, op_id: i64, uid: &NodeUid) -> Result<()> {
|
///
|
||||||
|
/// An op that found the name still held by this trash failed with a
|
||||||
|
/// backoff, and the name is free now. It is made due at once
|
||||||
|
/// ([`Db::wake_ops_waiting_for`]).
|
||||||
|
///
|
||||||
|
/// Returns the staged blob of the create the trash withdrew, if it kept
|
||||||
|
/// one, for the caller to discard.
|
||||||
|
pub fn complete_trash_op(&self, op_id: i64, uid: &NodeUid) -> Result<Option<String>> {
|
||||||
let uid = uid.to_string();
|
let uid = uid.to_string();
|
||||||
let mut conn = self.conn.lock();
|
let mut conn = self.conn.lock();
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
|
let (name, blob): (Option<String>, Option<String>) = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT name, blob_path FROM pending_op WHERE id = ?1",
|
||||||
|
params![op_id],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
|
)
|
||||||
|
.optional()?
|
||||||
|
.unwrap_or_default();
|
||||||
let rowid: Option<i64> = tx
|
let rowid: Option<i64> = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT rowid FROM nodes WHERE uid = ?1",
|
"SELECT rowid FROM nodes WHERE uid = ?1",
|
||||||
@@ -282,18 +300,23 @@ impl Db {
|
|||||||
tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![rowid])?;
|
tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![rowid])?;
|
||||||
}
|
}
|
||||||
tx.execute("DELETE FROM pending_op WHERE id = ?1", params![op_id])?;
|
tx.execute("DELETE FROM pending_op WHERE id = ?1", params![op_id])?;
|
||||||
|
if let Some(name) = name {
|
||||||
|
wake_ops_waiting_for_tx(&tx, &name)?;
|
||||||
|
}
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(blob)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a folder node has any non-trashed children in the database.
|
/// Check if a folder node has any non-trashed children in the database.
|
||||||
pub fn has_children(&self, parent_uid: &NodeUid) -> Result<bool> {
|
pub fn has_children(&self, parent_uid: &NodeUid) -> Result<bool> {
|
||||||
let uid_str = parent_uid.to_string();
|
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let (key, parent) = parent_key(&conn, &parent_uid.to_string())?;
|
||||||
let count: i64 = conn.query_row(
|
let count: i64 = conn.query_row(
|
||||||
"SELECT COUNT(*) FROM nodes
|
&format!(
|
||||||
WHERE parent_uid = ?1 AND trashed = 0 AND node_json IS NOT NULL",
|
"SELECT COUNT(*) FROM nodes
|
||||||
params![uid_str],
|
WHERE {key} = ?1 AND trashed = 0 AND node_json IS NOT NULL"
|
||||||
|
),
|
||||||
|
params![parent],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
Ok(count > 0)
|
Ok(count > 0)
|
||||||
@@ -330,11 +353,11 @@ impl Db {
|
|||||||
pub fn path_relative_to(&self, ancestor_uid: &str, uid: &str) -> Result<Option<String>> {
|
pub fn path_relative_to(&self, ancestor_uid: &str, uid: &str) -> Result<Option<String>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
"WITH RECURSIVE anc(uid, parent_uid, name, depth) AS (
|
"WITH RECURSIVE anc(uid, parent_lid, name, depth) AS (
|
||||||
SELECT uid, parent_uid, name, 0 FROM nodes WHERE uid = ?1
|
SELECT uid, parent_lid, name, 0 FROM nodes WHERE uid = ?1
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT n.uid, n.parent_uid, n.name, anc.depth + 1
|
SELECT n.uid, n.parent_lid, n.name, anc.depth + 1
|
||||||
FROM nodes n JOIN anc ON n.uid = anc.parent_uid
|
FROM nodes n JOIN anc ON n.lid = anc.parent_lid
|
||||||
WHERE anc.depth < 1024
|
WHERE anc.depth < 1024
|
||||||
)
|
)
|
||||||
SELECT uid, name FROM anc ORDER BY depth",
|
SELECT uid, name FROM anc ORDER BY depth",
|
||||||
@@ -383,12 +406,13 @@ impl Db {
|
|||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let rows: Vec<HitRow> = if query.chars().count() < TRIGRAM_MIN {
|
let rows: Vec<HitRow> = if query.chars().count() < TRIGRAM_MIN {
|
||||||
let pat = format!("%{}%", like_escape(query));
|
let pat = format!("%{}%", like_escape(query));
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT node_json, uid, path FROM nodes
|
"SELECT {NODE_NOW}, n.uid, n.path
|
||||||
WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL
|
FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
AND (?3 IS NULL OR path IS NULL OR path LIKE ?3 ESCAPE '\\')
|
WHERE n.name LIKE ?1 ESCAPE '\\' AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||||
ORDER BY name LIMIT ?2",
|
AND (?3 IS NULL OR n.path IS NULL OR n.path LIKE ?3 ESCAPE '\\')
|
||||||
)?;
|
ORDER BY n.name LIMIT ?2"
|
||||||
|
))?;
|
||||||
collect_hits(stmt.query_map(params![pat, limit as i64, scope_pat], hit_row)?)?
|
collect_hits(stmt.query_map(params![pat, limit as i64, scope_pat], hit_row)?)?
|
||||||
} else {
|
} else {
|
||||||
// Escape double quotes and quote each term, then combine with AND so
|
// Escape double quotes and quote each term, then combine with AND so
|
||||||
@@ -398,13 +422,14 @@ impl Db {
|
|||||||
.map(|word| format!("\"{}\"", word.replace('"', "\"\"")))
|
.map(|word| format!("\"{}\"", word.replace('"', "\"\"")))
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(" AND ");
|
.join(" AND ");
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT n.node_json, n.uid, n.path
|
"SELECT {NODE_NOW}, n.uid, n.path
|
||||||
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
||||||
|
LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
WHERE f.name MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
WHERE f.name MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||||
AND (?3 IS NULL OR n.path IS NULL OR n.path LIKE ?3 ESCAPE '\\')
|
AND (?3 IS NULL OR n.path IS NULL OR n.path LIKE ?3 ESCAPE '\\')
|
||||||
ORDER BY f.rank LIMIT ?2",
|
ORDER BY f.rank LIMIT ?2"
|
||||||
)?;
|
))?;
|
||||||
collect_hits(stmt.query_map(params![phrase, limit as i64, scope_pat], hit_row)?)?
|
collect_hits(stmt.query_map(params![phrase, limit as i64, scope_pat], hit_row)?)?
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -441,20 +466,22 @@ impl Db {
|
|||||||
let lane_limit = limit.div_ceil(2);
|
let lane_limit = limit.div_ceil(2);
|
||||||
let mut rows: Vec<HitRow> = if terms.is_empty() {
|
let mut rows: Vec<HitRow> = if terms.is_empty() {
|
||||||
let pat = format!("%{}%", like_escape(query));
|
let pat = format!("%{}%", like_escape(query));
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT node_json, uid, path FROM nodes
|
"SELECT {NODE_NOW}, n.uid, n.path
|
||||||
WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL
|
FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
ORDER BY name LIMIT ?2",
|
WHERE n.name LIKE ?1 ESCAPE '\\' AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||||
)?;
|
ORDER BY n.name LIMIT ?2"
|
||||||
|
))?;
|
||||||
collect_hits(stmt.query_map(params![pat, limit as i64], hit_row)?)?
|
collect_hits(stmt.query_map(params![pat, limit as i64], hit_row)?)?
|
||||||
} else {
|
} else {
|
||||||
let expression = terms.join(" OR ");
|
let expression = terms.join(" OR ");
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT n.node_json, n.uid, n.path
|
"SELECT {NODE_NOW}, n.uid, n.path
|
||||||
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
||||||
|
LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
WHERE nodes_fts MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
WHERE nodes_fts MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||||
ORDER BY f.rank LIMIT ?2",
|
ORDER BY f.rank LIMIT ?2"
|
||||||
)?;
|
))?;
|
||||||
collect_hits(stmt.query_map(params![expression, lane_limit as i64], hit_row)?)?
|
collect_hits(stmt.query_map(params![expression, lane_limit as i64], hit_row)?)?
|
||||||
};
|
};
|
||||||
// Trigram rank does not know that a name *starts* with the query, and a
|
// Trigram rank does not know that a name *starts* with the query, and a
|
||||||
@@ -464,12 +491,13 @@ impl Db {
|
|||||||
if !terms.is_empty()
|
if !terms.is_empty()
|
||||||
&& let Some(first) = query.chars().next()
|
&& let Some(first) = query.chars().next()
|
||||||
{
|
{
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT node_json, uid, path FROM nodes
|
"SELECT {NODE_NOW}, n.uid, n.path
|
||||||
WHERE name COLLATE NOCASE LIKE ?1 ESCAPE '\\'
|
FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
AND trashed = 0 AND node_json IS NOT NULL
|
WHERE n.name COLLATE NOCASE LIKE ?1 ESCAPE '\\'
|
||||||
ORDER BY name COLLATE NOCASE LIMIT ?2",
|
AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||||
)?;
|
ORDER BY n.name COLLATE NOCASE LIMIT ?2"
|
||||||
|
))?;
|
||||||
let mut extra = Vec::new();
|
let mut extra = Vec::new();
|
||||||
for pattern in [
|
for pattern in [
|
||||||
format!("{}%", like_escape(query)),
|
format!("{}%", like_escape(query)),
|
||||||
@@ -505,18 +533,26 @@ impl Db {
|
|||||||
/// re-enumerates from the remote on next access.
|
/// re-enumerates from the remote on next access.
|
||||||
pub fn set_listed(&self, uid: &NodeUid, listed: bool) -> Result<()> {
|
pub fn set_listed(&self, uid: &NodeUid, listed: bool) -> Result<()> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
let (key, uid) = row_key(&uid.to_string());
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE nodes SET listed = ?2 WHERE uid = ?1",
|
&format!("UPDATE nodes SET listed = ?2 WHERE {key} = ?1"),
|
||||||
params![uid.to_string(), listed as i64],
|
params![uid, listed as i64],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load every persisted node for cold-start hydration of the `State` maps.
|
/// Load every persisted node for cold-start hydration of the `State` maps.
|
||||||
|
///
|
||||||
|
/// Each node names its parent as the parent's row has it now, found by
|
||||||
|
/// local id: a folder that landed has its new uid there, while the node
|
||||||
|
/// stored below it may still hold the placeholder.
|
||||||
pub fn load_all(&self) -> Result<Vec<StoredNode>> {
|
pub fn load_all(&self) -> Result<Vec<StoredNode>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt =
|
let mut stmt = conn.prepare(&format!(
|
||||||
conn.prepare("SELECT node_json, listed, lid FROM nodes WHERE node_json IS NOT NULL")?;
|
"SELECT {NODE_NOW}, n.listed, n.lid
|
||||||
|
FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
|
WHERE n.node_json IS NOT NULL"
|
||||||
|
))?;
|
||||||
let rows = stmt.query_map([], |row| {
|
let rows = stmt.query_map([], |row| {
|
||||||
let json: String = row.get(0)?;
|
let json: String = row.get(0)?;
|
||||||
let listed: i64 = row.get(1)?;
|
let listed: i64 = row.get(1)?;
|
||||||
@@ -538,7 +574,10 @@ impl Db {
|
|||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let json: Option<String> = conn
|
let json: Option<String> = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT node_json FROM nodes WHERE uid = ?1 AND node_json IS NOT NULL",
|
&format!(
|
||||||
|
"SELECT {NODE_NOW} FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
|
WHERE n.uid = ?1 AND n.node_json IS NOT NULL"
|
||||||
|
),
|
||||||
params![uid],
|
params![uid],
|
||||||
|r| r.get(0),
|
|r| r.get(0),
|
||||||
)
|
)
|
||||||
@@ -549,16 +588,24 @@ impl Db {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The local id of the node stored under `uid`, or `None` when it has no
|
/// The uid of the node with local id `lid`: `None` when it has no row, or
|
||||||
/// row. It stays the same for as long as the row exists, including when a
|
/// is not on Drive yet.
|
||||||
/// create lands and the row takes the uid Drive gave it.
|
pub fn uid_of_lid(&self, lid: i64) -> Result<Option<String>> {
|
||||||
pub fn lid_of(&self, uid: &str) -> Result<Option<i64>> {
|
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
Ok(conn
|
Ok(conn
|
||||||
.query_row("SELECT lid FROM nodes WHERE uid = ?1", params![uid], |r| {
|
.query_row("SELECT uid FROM nodes WHERE lid = ?1", params![lid], |r| {
|
||||||
r.get(0)
|
r.get(0)
|
||||||
})
|
})
|
||||||
.optional()?)
|
.optional()?
|
||||||
|
.flatten())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The local id of the node stored under `uid`, or `None` when it has no
|
||||||
|
/// row. It stays the same for as long as the row exists, including when a
|
||||||
|
/// create lands and the row takes the uid Drive gave it: a node made on
|
||||||
|
/// this machine is found by its stand-in ([`local_uid`]) before and after.
|
||||||
|
pub fn lid_of(&self, uid: &str) -> Result<Option<i64>> {
|
||||||
|
node_lid(&self.read(), uid)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The local id of each node, in order, giving a row to each that has none.
|
/// The local id of each node, in order, giving a row to each that has none.
|
||||||
@@ -574,23 +621,31 @@ impl Db {
|
|||||||
{
|
{
|
||||||
let mut known = tx.prepare_cached("SELECT lid FROM nodes WHERE uid = ?1")?;
|
let mut known = tx.prepare_cached("SELECT lid FROM nodes WHERE uid = ?1")?;
|
||||||
let mut stub = tx.prepare_cached(
|
let mut stub = tx.prepare_cached(
|
||||||
"INSERT INTO nodes (uid, parent_uid, name, is_dir, mtime, trashed)
|
"INSERT INTO nodes (uid, parent_uid, name, is_dir, mtime, trashed, parent_lid)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||||
)?;
|
)?;
|
||||||
for node in nodes {
|
for node in nodes {
|
||||||
let uid = node.uid.to_string();
|
let uid = node.uid.to_string();
|
||||||
let lid = match known.query_row(params![uid], |r| r.get(0)).optional()? {
|
let lid = match known.query_row(params![uid], |r| r.get(0)).optional()? {
|
||||||
Some(lid) => lid,
|
Some(lid) => lid,
|
||||||
None => {
|
None => {
|
||||||
|
let parent = node.parent_uid.as_ref().map(ToString::to_string);
|
||||||
|
let parent_lid = match &parent {
|
||||||
|
Some(parent) => node_lid(&tx, parent)?,
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
stub.execute(params![
|
stub.execute(params![
|
||||||
uid,
|
uid,
|
||||||
node.parent_uid.as_ref().map(ToString::to_string),
|
parent,
|
||||||
node.name,
|
node.name,
|
||||||
node.is_folder() as i64,
|
node.is_folder() as i64,
|
||||||
node.modification_time,
|
node.modification_time,
|
||||||
node.trashed as i64,
|
node.trashed as i64,
|
||||||
|
parent_lid,
|
||||||
])?;
|
])?;
|
||||||
tx.last_insert_rowid()
|
let lid = tx.last_insert_rowid();
|
||||||
|
adopt_children_tx(&tx, &uid, lid)?;
|
||||||
|
lid
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
lids.push(lid);
|
lids.push(lid);
|
||||||
@@ -600,15 +655,51 @@ impl Db {
|
|||||||
Ok(lids)
|
Ok(lids)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Give a node being made on this machine its row under `parent_uid`, and
|
||||||
|
/// return its local id. The row goes by [`local_uid`] of that id until
|
||||||
|
/// the node lands.
|
||||||
|
///
|
||||||
|
/// Like [`Db::lids_for`], only a stub: the caller upserts the node.
|
||||||
|
pub fn add_local_row(
|
||||||
|
&self,
|
||||||
|
parent_uid: &str,
|
||||||
|
name: &str,
|
||||||
|
is_dir: bool,
|
||||||
|
mtime: i64,
|
||||||
|
) -> Result<i64> {
|
||||||
|
let mut conn = self.conn.lock();
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO nodes (uid, parent_uid, name, is_dir, mtime, trashed, parent_lid)
|
||||||
|
VALUES (NULL, ?1, ?2, ?3, ?4, 0,
|
||||||
|
COALESCE(?5, (SELECT lid FROM nodes WHERE uid = ?1)))",
|
||||||
|
params![
|
||||||
|
parent_uid,
|
||||||
|
name,
|
||||||
|
is_dir as i64,
|
||||||
|
mtime,
|
||||||
|
local_lid(parent_uid)
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
let lid = tx.last_insert_rowid();
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE nodes SET uid = ?2 WHERE lid = ?1",
|
||||||
|
params![lid, local_uid(lid)],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(lid)
|
||||||
|
}
|
||||||
|
|
||||||
/// Read the persisted incremental-sync cursor (a `DriveEventId`), if any.
|
/// Read the persisted incremental-sync cursor (a `DriveEventId`), if any.
|
||||||
/// The daemon resumes from this on restart instead of reseeding to the
|
/// The daemon resumes from this on restart instead of reseeding to the
|
||||||
/// server head, so changes made while unmounted are still applied (P2).
|
/// server head, so changes made while unmounted are still applied (P2).
|
||||||
pub fn children_if_listed(&self, parent: &NodeUid) -> Result<Option<Vec<Node>>> {
|
pub fn children_if_listed(&self, parent: &NodeUid) -> Result<Option<Vec<Node>>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let (key, folder) = row_key(&parent.to_string());
|
||||||
let listed: Option<i64> = conn
|
let listed: Option<i64> = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT listed FROM nodes WHERE uid = ?1",
|
&format!("SELECT listed FROM nodes WHERE {key} = ?1"),
|
||||||
params![parent.to_string()],
|
params![folder],
|
||||||
|r| r.get(0),
|
|r| r.get(0),
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
@@ -624,11 +715,12 @@ impl Db {
|
|||||||
/// had: a listing dropped as stale may miss or keep a child.
|
/// had: a listing dropped as stale may miss or keep a child.
|
||||||
pub fn known_children(&self, parent: &NodeUid) -> Result<Vec<Node>> {
|
pub fn known_children(&self, parent: &NodeUid) -> Result<Vec<Node>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let (key, parent) = parent_key(&conn, &parent.to_string())?;
|
||||||
"SELECT node_json FROM nodes
|
let mut stmt = conn.prepare(&format!(
|
||||||
WHERE parent_uid = ?1 AND node_json IS NOT NULL AND trashed = 0",
|
"SELECT {NODE_NOW} FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
)?;
|
WHERE n.{key} = ?1 AND n.node_json IS NOT NULL AND n.trashed = 0"
|
||||||
let rows = stmt.query_map(params![parent.to_string()], |r| r.get::<_, String>(0))?;
|
))?;
|
||||||
|
let rows = stmt.query_map(params![parent], |r| r.get::<_, String>(0))?;
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for json in rows {
|
for json in rows {
|
||||||
out.push(serde_json::from_str(&json?)?);
|
out.push(serde_json::from_str(&json?)?);
|
||||||
@@ -636,6 +728,40 @@ impl Db {
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What a queued change does to `parent`'s listing, which Drive does not
|
||||||
|
/// show until it lands: the children made or moved in by a queued create,
|
||||||
|
/// mkdir or rename, as the DB holds them, and the nodes a queued rename
|
||||||
|
/// moves elsewhere.
|
||||||
|
///
|
||||||
|
/// A listing read from Drive is laid over with these, or a folder listed
|
||||||
|
/// while a create in it is queued, after a restart say, loses the file
|
||||||
|
/// until the create lands (`docs/BUGS.md` B132).
|
||||||
|
pub fn queued_children(&self, parent: &NodeUid) -> Result<(Vec<Node>, HashSet<String>)> {
|
||||||
|
let conn = self.read();
|
||||||
|
let (key, parent) = parent_key(&conn, &parent.to_string())?;
|
||||||
|
let mut stmt = conn.prepare(&format!(
|
||||||
|
"SELECT {NODE_NOW} FROM nodes n LEFT JOIN nodes p ON p.lid = n.parent_lid
|
||||||
|
WHERE n.{key} = ?1 AND n.node_json IS NOT NULL AND n.trashed = 0
|
||||||
|
AND EXISTS (SELECT 1 FROM pending_op o
|
||||||
|
WHERE (o.lid = n.lid OR o.uid = n.uid) AND o.kind IN (?2, ?3, ?4))"
|
||||||
|
))?;
|
||||||
|
let rows = stmt.query_map(params![parent, OP_CREATE, OP_MKDIR, OP_RENAME], |r| {
|
||||||
|
r.get::<_, String>(0)
|
||||||
|
})?;
|
||||||
|
let mut here = Vec::new();
|
||||||
|
for json in rows {
|
||||||
|
here.push(serde_json::from_str(&json?)?);
|
||||||
|
}
|
||||||
|
let mut stmt = conn.prepare(&format!(
|
||||||
|
"SELECT DISTINCT n.uid FROM pending_op p JOIN nodes n ON n.lid = p.lid OR n.uid = p.uid
|
||||||
|
WHERE p.kind = ?2 AND n.{key} IS NOT ?1"
|
||||||
|
))?;
|
||||||
|
let gone = stmt
|
||||||
|
.query_map(params![parent, OP_RENAME], |r| r.get::<_, String>(0))?
|
||||||
|
.collect::<std::result::Result<HashSet<_>, _>>()?;
|
||||||
|
Ok((here, gone))
|
||||||
|
}
|
||||||
|
|
||||||
// --- Content-cache LRU index (P4) -------------------------------------
|
// --- Content-cache LRU index (P4) -------------------------------------
|
||||||
//
|
//
|
||||||
// Replaces the per-eviction `read_dir` scans in `ContentCache`. Each cached
|
// Replaces the per-eviction `read_dir` scans in `ContentCache`. Each cached
|
||||||
@@ -655,8 +781,53 @@ struct PriorRow {
|
|||||||
trashed: bool,
|
trashed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Hand a drained placeholder's row, and the rows below it, to the real uid it
|
/// [`Db::lid_of`] on `conn`, which may be a transaction.
|
||||||
/// landed as.
|
pub(super) fn node_lid(conn: &rusqlite::Connection, uid: &str) -> Result<Option<i64>> {
|
||||||
|
let row = match local_lid(uid) {
|
||||||
|
Some(lid) => conn.query_row("SELECT lid FROM nodes WHERE lid = ?1", params![lid], |r| {
|
||||||
|
r.get(0)
|
||||||
|
}),
|
||||||
|
None => conn.query_row("SELECT lid FROM nodes WHERE uid = ?1", params![uid], |r| {
|
||||||
|
r.get(0)
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
Ok(row.optional()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The column and value that find `uid`'s own row. A stand-in is found by its
|
||||||
|
/// local id: a create that lands gives the row the uid Drive made before the
|
||||||
|
/// tree has it, and the tree asks by the stand-in until then (`docs/BUGS.md`
|
||||||
|
/// B177).
|
||||||
|
fn row_key(uid: &str) -> (&'static str, Value) {
|
||||||
|
match local_lid(uid) {
|
||||||
|
Some(lid) => ("lid", Value::Integer(lid)),
|
||||||
|
None => ("uid", Value::Text(uid.to_owned())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The column and value a row names `parent` by: its local id when it has a
|
||||||
|
/// row, which the children keep when it lands, or its uid when it has none,
|
||||||
|
/// as a device folder's root does.
|
||||||
|
fn parent_key(conn: &rusqlite::Connection, parent: &str) -> Result<(&'static str, Value)> {
|
||||||
|
Ok(match node_lid(conn, parent)? {
|
||||||
|
Some(lid) => ("parent_lid", Value::Integer(lid)),
|
||||||
|
None => ("parent_uid", Value::Text(parent.to_owned())),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `n.node_json` naming its parent as the parent's row `p`, joined by
|
||||||
|
/// `n.parent_lid`, has it now. A folder that lands takes its real uid in its
|
||||||
|
/// own row only, so a node stored below it still names the stand-in.
|
||||||
|
const NODE_NOW: &str = "CASE WHEN p.uid IS NULL OR p.uid = n.parent_uid THEN n.node_json
|
||||||
|
ELSE json_set(n.node_json, '$.parent_uid', json_object(
|
||||||
|
'volume_id', substr(p.uid, 1, instr(p.uid, '~') - 1),
|
||||||
|
'link_id', substr(p.uid, instr(p.uid, '~') + 1))) END";
|
||||||
|
|
||||||
|
/// Hand a drained placeholder's row to the real uid it landed as.
|
||||||
|
///
|
||||||
|
/// Nothing else is rewritten. The rows below it and the ops made inside it
|
||||||
|
/// name it by its local id, which stays; what reads them for a uid takes the
|
||||||
|
/// one the row has now ([`NODE_NOW`], `PARENT_NOW` in `ops.rs`).
|
||||||
///
|
///
|
||||||
/// Dropping the placeholder row and waiting for the server's copy of the real
|
/// Dropping the placeholder row and waiting for the server's copy of the real
|
||||||
/// node left a window with no row for either uid. A queued child drained in
|
/// node left a window with no row for either uid. A queued child drained in
|
||||||
@@ -671,10 +842,13 @@ pub(super) fn adopt_placeholder_row_tx(
|
|||||||
local: &str,
|
local: &str,
|
||||||
real: &str,
|
real: &str,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
// A folder pinned before it landed stays pinned: the pin keyed by the
|
||||||
|
// placeholder was dropped with it (`docs/BUGS.md` B148).
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"UPDATE nodes SET parent_uid = ?2 WHERE parent_uid = ?1",
|
"UPDATE OR IGNORE pins SET uid = ?2 WHERE uid = ?1",
|
||||||
params![local, real],
|
params![local, real],
|
||||||
)?;
|
)?;
|
||||||
|
tx.execute("DELETE FROM pins WHERE uid = ?1", params![local])?;
|
||||||
let row: Option<(i64, Option<String>)> = tx
|
let row: Option<(i64, Option<String>)> = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT lid, node_json FROM nodes WHERE uid = ?1",
|
"SELECT lid, node_json FROM nodes WHERE uid = ?1",
|
||||||
@@ -682,6 +856,12 @@ pub(super) fn adopt_placeholder_row_tx(
|
|||||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
|
// Ops queued before schema 39 have no `parent_lid`, nor do ops made
|
||||||
|
// under a placeholder that never had a row.
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE pending_op SET parent_uid = ?2 WHERE parent_lid IS NULL AND parent_uid = ?1",
|
||||||
|
params![local, real],
|
||||||
|
)?;
|
||||||
let Some((lid, json)) = row else {
|
let Some((lid, json)) = row else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
@@ -693,6 +873,19 @@ pub(super) fn adopt_placeholder_row_tx(
|
|||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
if let Some((known_lid, _)) = known {
|
if let Some((known_lid, _)) = known {
|
||||||
|
// Ops queued against the row Drive listed now name the landing one.
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE pending_op SET lid = ?2 WHERE lid = ?1",
|
||||||
|
params![known_lid, lid],
|
||||||
|
)?;
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE pending_op SET parent_lid = ?2 WHERE parent_lid = ?1",
|
||||||
|
params![known_lid, lid],
|
||||||
|
)?;
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE nodes SET parent_lid = ?2 WHERE parent_lid = ?1",
|
||||||
|
params![known_lid, lid],
|
||||||
|
)?;
|
||||||
tx.execute("DELETE FROM nodes WHERE lid = ?1", params![known_lid])?;
|
tx.execute("DELETE FROM nodes WHERE lid = ?1", params![known_lid])?;
|
||||||
tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![known_lid])?;
|
tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![known_lid])?;
|
||||||
}
|
}
|
||||||
@@ -709,9 +902,36 @@ pub(super) fn adopt_placeholder_row_tx(
|
|||||||
upsert_node_tx(tx, &node)
|
upsert_node_tx(tx, &node)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Link the rows that name `uid` as their parent to its new row `lid`: they
|
||||||
|
/// were stored before it, or under a row of the same uid since dropped.
|
||||||
|
fn adopt_children_tx(tx: &Transaction<'_>, uid: &str, lid: i64) -> Result<()> {
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE nodes SET parent_lid = ?2 WHERE parent_uid = ?1 AND parent_lid IS NOT ?2",
|
||||||
|
params![uid, lid],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
||||||
let json = serde_json::to_string(node)?;
|
|
||||||
let uid = node.uid.to_string();
|
let uid = node.uid.to_string();
|
||||||
|
// A node written under its stand-in after it landed is its row, which has
|
||||||
|
// the uid Drive gave it by now: written under the stand-in, it would bring
|
||||||
|
// the placeholder row back beside it.
|
||||||
|
if let Some(lid) = local_lid(&uid) {
|
||||||
|
let landed: Option<String> = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT uid FROM nodes WHERE lid = ?1 AND uid IS NOT ?2",
|
||||||
|
params![lid, uid],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
if let Some(real) = landed.as_deref().and_then(parse_node_uid) {
|
||||||
|
let mut node = node.clone();
|
||||||
|
node.uid = real;
|
||||||
|
return upsert_node_tx(tx, &node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let json = serde_json::to_string(node)?;
|
||||||
let parent_uid = node.parent_uid.as_ref().map(|u| u.to_string());
|
let parent_uid = node.parent_uid.as_ref().map(|u| u.to_string());
|
||||||
|
|
||||||
let prior: Option<PriorRow> = tx
|
let prior: Option<PriorRow> = tx
|
||||||
@@ -733,26 +953,34 @@ fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
|||||||
// where resolving it from scratch is a recursive walk to the root. A node
|
// where resolving it from scratch is a recursive walk to the root. A node
|
||||||
// whose parent is not cached (a device folder's root lives in `device`, never
|
// whose parent is not cached (a device folder's root lives in `device`, never
|
||||||
// in `nodes`) starts a path of its own, which is what the walk did too.
|
// in `nodes`) starts a path of its own, which is what the walk did too.
|
||||||
let path = match &parent_uid {
|
let (parent_lid, path) = match &parent_uid {
|
||||||
None => String::new(),
|
None => (None, String::new()),
|
||||||
Some(parent) => {
|
Some(parent) => {
|
||||||
let parent_path: Option<Option<String>> = tx
|
let parent_row: Option<(i64, Option<String>)> = match node_lid(tx, parent)? {
|
||||||
.query_row(
|
Some(lid) => tx
|
||||||
"SELECT path FROM nodes WHERE uid = ?1",
|
.query_row(
|
||||||
params![parent],
|
"SELECT lid, path FROM nodes WHERE lid = ?1",
|
||||||
|row| row.get(0),
|
params![lid],
|
||||||
)
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
.optional()?;
|
)
|
||||||
join_path(parent_path.flatten().as_deref().unwrap_or(""), &node.name)
|
.optional()?,
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
let (lid, parent_path) = parent_row.unzip();
|
||||||
|
(
|
||||||
|
lid,
|
||||||
|
join_path(parent_path.flatten().as_deref().unwrap_or(""), &node.name),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"INSERT INTO nodes
|
"INSERT INTO nodes
|
||||||
(uid, parent_uid, name, is_dir, size, mtime, trashed, node_json, path)
|
(uid, parent_uid, name, is_dir, size, mtime, trashed, node_json, path, parent_lid)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||||
ON CONFLICT(uid) DO UPDATE SET
|
ON CONFLICT(uid) DO UPDATE SET
|
||||||
parent_uid = excluded.parent_uid,
|
parent_uid = excluded.parent_uid,
|
||||||
|
parent_lid = excluded.parent_lid,
|
||||||
name = excluded.name,
|
name = excluded.name,
|
||||||
is_dir = excluded.is_dir,
|
is_dir = excluded.is_dir,
|
||||||
size = excluded.size,
|
size = excluded.size,
|
||||||
@@ -770,6 +998,7 @@ fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
|||||||
node.trashed as i64,
|
node.trashed as i64,
|
||||||
json,
|
json,
|
||||||
path,
|
path,
|
||||||
|
parent_lid,
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
let rowid: i64 = tx.query_row(
|
let rowid: i64 = tx.query_row(
|
||||||
@@ -777,6 +1006,9 @@ fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
|||||||
params![uid],
|
params![uid],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
|
if prior.is_none() {
|
||||||
|
adopt_children_tx(tx, &uid, rowid)?;
|
||||||
|
}
|
||||||
|
|
||||||
// FTS5 has no UPSERT, so the node's own row is always replaced.
|
// FTS5 has no UPSERT, so the node's own row is always replaced.
|
||||||
let indexable = node_is_indexable_tx(tx, &uid)?;
|
let indexable = node_is_indexable_tx(tx, &uid)?;
|
||||||
@@ -799,7 +1031,7 @@ fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
|||||||
|| prior.trashed != node.trashed
|
|| prior.trashed != node.trashed
|
||||||
});
|
});
|
||||||
if node.is_folder() && subtree_moved {
|
if node.is_folder() && subtree_moved {
|
||||||
reindex_subtree_tx(tx, &uid, &path, indexable)?;
|
reindex_subtree_tx(tx, rowid, &path, indexable)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -820,22 +1052,22 @@ fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
|||||||
/// this deep.
|
/// this deep.
|
||||||
fn reindex_subtree_tx(
|
fn reindex_subtree_tx(
|
||||||
tx: &Transaction<'_>,
|
tx: &Transaction<'_>,
|
||||||
folder: &str,
|
folder: i64,
|
||||||
folder_path: &str,
|
folder_path: &str,
|
||||||
folder_indexable: bool,
|
folder_indexable: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let descendants: Vec<(i64, String, bool)> = {
|
let descendants: Vec<(i64, String, bool)> = {
|
||||||
let mut stmt = tx.prepare(
|
let mut stmt = tx.prepare(
|
||||||
"WITH RECURSIVE sub(rowid, uid, name, path, blocked, depth) AS (
|
"WITH RECURSIVE sub(rowid, name, path, blocked, depth) AS (
|
||||||
SELECT n.rowid, n.uid, n.name,
|
SELECT n.rowid, n.name,
|
||||||
CASE WHEN ?2 = '' THEN n.name ELSE ?2 || '/' || n.name END,
|
CASE WHEN ?2 = '' THEN n.name ELSE ?2 || '/' || n.name END,
|
||||||
n.trashed, 0
|
n.trashed, 0
|
||||||
FROM nodes n WHERE n.parent_uid = ?1
|
FROM nodes n WHERE n.parent_lid = ?1
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT n.rowid, n.uid, n.name,
|
SELECT n.rowid, n.name,
|
||||||
sub.path || '/' || n.name,
|
sub.path || '/' || n.name,
|
||||||
MAX(sub.blocked, n.trashed), sub.depth + 1
|
MAX(sub.blocked, n.trashed), sub.depth + 1
|
||||||
FROM nodes n JOIN sub ON n.parent_uid = sub.uid
|
FROM nodes n JOIN sub ON n.parent_lid = sub.rowid
|
||||||
WHERE sub.depth < 256
|
WHERE sub.depth < 256
|
||||||
)
|
)
|
||||||
SELECT rowid, path, blocked FROM sub",
|
SELECT rowid, path, blocked FROM sub",
|
||||||
@@ -876,7 +1108,8 @@ fn reindex_subtree_tx(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn direct_child_uids_tx(tx: &Transaction<'_>, parent: &str) -> Result<Vec<String>> {
|
fn direct_child_uids_tx(tx: &Transaction<'_>, parent: &str) -> Result<Vec<String>> {
|
||||||
let mut stmt = tx.prepare("SELECT uid FROM nodes WHERE parent_uid = ?1")?;
|
let (key, parent) = parent_key(tx, parent)?;
|
||||||
|
let mut stmt = tx.prepare(&format!("SELECT uid FROM nodes WHERE {key} = ?1"))?;
|
||||||
let rows = stmt.query_map([parent], |row| row.get::<_, String>(0))?;
|
let rows = stmt.query_map([parent], |row| row.get::<_, String>(0))?;
|
||||||
let mut uids = Vec::new();
|
let mut uids = Vec::new();
|
||||||
for row in rows {
|
for row in rows {
|
||||||
@@ -901,14 +1134,14 @@ fn direct_child_uids_tx(tx: &Transaction<'_>, parent: &str) -> Result<Vec<String
|
|||||||
/// ends at one whose parent is a row we have already visited.
|
/// ends at one whose parent is a row we have already visited.
|
||||||
fn node_is_indexable_tx(tx: &Transaction<'_>, uid: &str) -> Result<bool> {
|
fn node_is_indexable_tx(tx: &Transaction<'_>, uid: &str) -> Result<bool> {
|
||||||
let indexable: i64 = tx.query_row(
|
let indexable: i64 = tx.query_row(
|
||||||
"WITH RECURSIVE ancestors(uid, parent_uid, trashed, depth, path) AS (
|
"WITH RECURSIVE ancestors(lid, parent_lid, trashed, depth, path) AS (
|
||||||
SELECT uid, parent_uid, trashed, 0, char(31) || uid || char(31)
|
SELECT lid, parent_lid, trashed, 0, ',' || lid || ','
|
||||||
FROM nodes WHERE uid = ?1
|
FROM nodes WHERE uid = ?1
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT n.uid, n.parent_uid, n.trashed, a.depth + 1,
|
SELECT n.lid, n.parent_lid, n.trashed, a.depth + 1,
|
||||||
a.path || n.uid || char(31)
|
a.path || n.lid || ','
|
||||||
FROM ancestors a JOIN nodes n ON n.uid = a.parent_uid
|
FROM ancestors a JOIN nodes n ON n.lid = a.parent_lid
|
||||||
WHERE instr(a.path, char(31) || n.uid || char(31)) = 0
|
WHERE instr(a.path, ',' || n.lid || ',') = 0
|
||||||
)
|
)
|
||||||
SELECT CASE
|
SELECT CASE
|
||||||
WHEN COUNT(*) = 0 THEN 0
|
WHEN COUNT(*) = 0 THEN 0
|
||||||
@@ -916,8 +1149,7 @@ fn node_is_indexable_tx(tx: &Transaction<'_>, uid: &str) -> Result<bool> {
|
|||||||
WHEN EXISTS (
|
WHEN EXISTS (
|
||||||
SELECT 1 FROM ancestors a
|
SELECT 1 FROM ancestors a
|
||||||
WHERE a.depth = (SELECT MAX(depth) FROM ancestors)
|
WHERE a.depth = (SELECT MAX(depth) FROM ancestors)
|
||||||
AND a.parent_uid IS NOT NULL
|
AND EXISTS (SELECT 1 FROM nodes p WHERE p.lid = a.parent_lid)
|
||||||
AND EXISTS (SELECT 1 FROM nodes p WHERE p.uid = a.parent_uid)
|
|
||||||
) THEN 0
|
) THEN 0
|
||||||
ELSE 1
|
ELSE 1
|
||||||
END
|
END
|
||||||
@@ -935,7 +1167,7 @@ fn tombstone_subtree_tx(tx: &Transaction<'_>, root_uid: &str) -> Result<()> {
|
|||||||
SELECT rowid, uid, node_json FROM nodes WHERE uid = ?1
|
SELECT rowid, uid, node_json FROM nodes WHERE uid = ?1
|
||||||
UNION
|
UNION
|
||||||
SELECT n.rowid, n.uid, n.node_json FROM nodes n
|
SELECT n.rowid, n.uid, n.node_json FROM nodes n
|
||||||
JOIN subtree s ON n.parent_uid = s.uid
|
JOIN subtree s ON n.parent_lid = s.rowid
|
||||||
)
|
)
|
||||||
SELECT rowid, uid, node_json FROM subtree",
|
SELECT rowid, uid, node_json FROM subtree",
|
||||||
)?;
|
)?;
|
||||||
|
|||||||
+599
-81
@@ -2,11 +2,13 @@
|
|||||||
//! the remote has not yet seen. Rebuilt into memory on mount and drained in row
|
//! the remote has not yet seen. Rebuilt into memory on mount and drained in row
|
||||||
//! order, so a child never drains before the parent that gives it a real uid.
|
//! order, so a child never drains before the parent that gives it a real uid.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use rusqlite::{OptionalExtension, params};
|
use rusqlite::{OptionalExtension, params};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use super::Db;
|
use super::Db;
|
||||||
use super::nodes::adopt_placeholder_row_tx;
|
use super::nodes::{adopt_placeholder_row_tx, node_lid};
|
||||||
use crate::Result;
|
use crate::Result;
|
||||||
|
|
||||||
/// The `kind` of a [`PendingOp`] that uploads a staged file as a new revision.
|
/// The `kind` of a [`PendingOp`] that uploads a staged file as a new revision.
|
||||||
@@ -92,52 +94,193 @@ pub fn op_supersedes(kind: &str) -> bool {
|
|||||||
/// Other ops are taken from the node tree only for revisions. A queued rename
|
/// Other ops are taken from the node tree only for revisions. A queued rename
|
||||||
/// that moved a node *into* the folder has to land, or the node would stay
|
/// that moved a node *into* the folder has to land, or the node would stay
|
||||||
/// where it was on the server instead of going to the trash with the folder.
|
/// where it was on the server instead of going to the trash with the folder.
|
||||||
|
///
|
||||||
|
/// A create already on the wire cannot be stopped, so its row is not deleted
|
||||||
|
/// but turned into a trash of the node it is making, still claimed by the
|
||||||
|
/// upload. [`Db::finish_create`] points it at the landed node. Until then it
|
||||||
|
/// holds the name, as any queued trash does ([`Db::has_pending_trash_named`]):
|
||||||
|
/// a rename over the file sent first found the name taken and landed under a
|
||||||
|
/// conflict name (`docs/BUGS.md` B129).
|
||||||
|
///
|
||||||
|
/// So does a create that was sent before and failed. Its answer may have been
|
||||||
|
/// lost after Drive made the node, which would then keep the deleted file's
|
||||||
|
/// name and bytes (`docs/BUGS.md` B151). Such a trash keeps the create's
|
||||||
|
/// parent and blob, which the drain looks for the node by, and which it
|
||||||
|
/// discards once the trash is done ([`Db::complete_trash_op`]).
|
||||||
|
///
|
||||||
|
/// Sent is where it was sent to ([`Db::note_create_target`]), not only its
|
||||||
|
/// attempts: a write after the failure clears those for a prompt retry, and
|
||||||
|
/// the delete then dropped the create whole (`docs/BUGS.md` B191).
|
||||||
fn drop_doomed_ops(tx: &rusqlite::Transaction<'_>, uid: &str) -> Result<Vec<String>> {
|
fn drop_doomed_ops(tx: &rusqlite::Transaction<'_>, uid: &str) -> Result<Vec<String>> {
|
||||||
const DOOMED: &str = "
|
const DOOMED: &str = "
|
||||||
WITH RECURSIVE
|
WITH RECURSIVE
|
||||||
below(uid) AS (
|
below(uid, lid) AS (
|
||||||
SELECT ?1
|
SELECT ?1, ?2
|
||||||
UNION
|
UNION
|
||||||
SELECT n.uid FROM nodes n JOIN below b ON n.parent_uid = b.uid
|
SELECT n.uid, n.lid FROM nodes n JOIN below b
|
||||||
|
ON n.parent_lid = b.lid OR (n.parent_lid IS NULL AND n.parent_uid = b.uid)
|
||||||
),
|
),
|
||||||
queued(uid) AS (
|
queued(uid, lid) AS (
|
||||||
SELECT ?1
|
SELECT ?1, ?2
|
||||||
UNION
|
UNION
|
||||||
SELECT p.uid FROM pending_op p
|
SELECT p.uid, p.lid FROM pending_op p
|
||||||
WHERE p.kind IN ('create', 'mkdir')
|
WHERE p.kind IN ('create', 'mkdir')
|
||||||
AND p.parent_uid IN (SELECT uid FROM below)
|
AND (p.parent_lid IN (SELECT lid FROM below)
|
||||||
|
OR p.parent_uid IN (SELECT uid FROM below))
|
||||||
UNION
|
UNION
|
||||||
SELECT p.uid FROM pending_op p JOIN queued q ON p.parent_uid = q.uid
|
SELECT p.uid, p.lid FROM pending_op p JOIN queued q
|
||||||
|
ON p.parent_lid = q.lid OR p.parent_uid = q.uid
|
||||||
),
|
),
|
||||||
doomed(id) AS (
|
doomed(id) AS (
|
||||||
SELECT id FROM pending_op
|
SELECT id FROM pending_op
|
||||||
WHERE uid IN (SELECT uid FROM queued)
|
WHERE uid IN (SELECT uid FROM queued)
|
||||||
OR (kind = 'revision' AND uid IN (SELECT uid FROM below))
|
OR lid IN (SELECT lid FROM queued)
|
||||||
|
OR (kind = 'revision'
|
||||||
|
AND (uid IN (SELECT uid FROM below) OR lid IN (SELECT lid FROM below)))
|
||||||
)";
|
)";
|
||||||
|
const SENT: &str = "kind IN ('create', 'mkdir')
|
||||||
|
AND (claimed_at <> 0 OR attempts > 0 OR sent_to IS NOT NULL)";
|
||||||
|
let lid = node_lid(tx, uid)?;
|
||||||
let blobs: Vec<String> = {
|
let blobs: Vec<String> = {
|
||||||
let mut stmt = tx.prepare(&format!(
|
let mut stmt = tx.prepare(&format!(
|
||||||
"{DOOMED}
|
"{DOOMED}
|
||||||
SELECT blob_path FROM pending_op
|
SELECT blob_path FROM pending_op
|
||||||
WHERE id IN (SELECT id FROM doomed) AND blob_path IS NOT NULL"
|
WHERE id IN (SELECT id FROM doomed) AND blob_path IS NOT NULL
|
||||||
|
AND NOT ({SENT})"
|
||||||
))?;
|
))?;
|
||||||
let rows = stmt.query_map(params![uid], |r| r.get::<_, String>(0))?;
|
let rows = stmt.query_map(params![uid, lid], |r| r.get::<_, String>(0))?;
|
||||||
rows.collect::<std::result::Result<Vec<_>, _>>()?
|
rows.collect::<std::result::Result<Vec<_>, _>>()?
|
||||||
};
|
};
|
||||||
tx.execute(
|
tx.execute(
|
||||||
&format!("{DOOMED} DELETE FROM pending_op WHERE id IN (SELECT id FROM doomed)"),
|
&format!(
|
||||||
params![uid],
|
"{DOOMED}
|
||||||
|
UPDATE pending_op
|
||||||
|
SET kind = 'trash', issue = NULL,
|
||||||
|
attempts = 0, last_error = NULL, next_attempt_at = 0,
|
||||||
|
parent_uid = CASE WHEN {PARENT_NOW} LIKE '{LOCAL_VOLUME}~%' THEN NULL
|
||||||
|
ELSE {PARENT_NOW} END,
|
||||||
|
parent_lid = CASE WHEN {PARENT_NOW} LIKE '{LOCAL_VOLUME}~%' THEN NULL
|
||||||
|
ELSE parent_lid END
|
||||||
|
WHERE id IN (SELECT id FROM doomed) AND {SENT}"
|
||||||
|
),
|
||||||
|
params![uid, lid],
|
||||||
|
)?;
|
||||||
|
tx.execute(
|
||||||
|
&format!(
|
||||||
|
"{DOOMED}
|
||||||
|
DELETE FROM pending_op
|
||||||
|
WHERE id IN (SELECT id FROM doomed)
|
||||||
|
AND NOT (kind = 'trash' AND uid LIKE '{LOCAL_VOLUME}~%')"
|
||||||
|
),
|
||||||
|
params![uid, lid],
|
||||||
)?;
|
)?;
|
||||||
Ok(blobs)
|
Ok(blobs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `pending_op` rows, `?1` being [`OP_RENAME`], that move a node away from
|
||||||
|
/// the name `?2`.
|
||||||
|
const MOVES_FROM: &str = "kind = ?1
|
||||||
|
AND json_extract(meta_json, '$.original_name') = ?2
|
||||||
|
AND (name <> ?2 OR parent_uid <> json_extract(meta_json, '$.original_parent_uid'))";
|
||||||
|
|
||||||
|
/// [`Db::wake_ops_waiting_for`] within a transaction.
|
||||||
|
pub(super) fn wake_ops_waiting_for_tx(tx: &rusqlite::Transaction<'_>, name: &str) -> Result<()> {
|
||||||
|
tx.execute(
|
||||||
|
&format!(
|
||||||
|
"UPDATE pending_op SET next_attempt_at = 0
|
||||||
|
WHERE name = ?2 AND kind IN (?3, ?4, ?1) AND attempts > 0
|
||||||
|
AND next_attempt_at < ?5
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM pending_op WHERE kind = ?6 AND name = ?2)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM pending_op WHERE {MOVES_FROM})"
|
||||||
|
),
|
||||||
|
params![OP_RENAME, name, OP_CREATE, OP_MKDIR, PARK_UNTIL, OP_TRASH],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Point queued op `id` at its node and its parent by local id, from the uids
|
||||||
|
/// it names (schema 39), a stand-in ([`local_uid`]) included. A uid with no
|
||||||
|
/// row leaves `NULL`, and the op is matched by uid, as before.
|
||||||
|
fn fill_op_lids(conn: &rusqlite::Connection, id: i64) -> Result<()> {
|
||||||
|
let (uid, parent): (String, Option<String>) = conn.query_row(
|
||||||
|
"SELECT uid, parent_uid FROM pending_op WHERE id = ?1",
|
||||||
|
params![id],
|
||||||
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||||
|
)?;
|
||||||
|
let parent_lid = match parent {
|
||||||
|
Some(parent) => node_lid(conn, &parent)?,
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE pending_op SET lid = ?2, parent_lid = ?3 WHERE id = ?1",
|
||||||
|
params![id, node_lid(conn, &uid)?, parent_lid],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ops on the node `?1` names, `?L` being its local id
|
||||||
|
/// ([`Db::lid_of`]): queued under that uid, or under the node's other one, its
|
||||||
|
/// stand-in before it landed or its real uid after.
|
||||||
|
const ON_NODE: &str = "(uid = ?1 OR lid = ?L)";
|
||||||
|
|
||||||
|
/// [`ON_NODE`] with `?L` numbered `n`.
|
||||||
|
fn on_node(n: usize) -> String {
|
||||||
|
ON_NODE.replace("?L", &format!("?{n}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete op `id` if it is a rename with a newer rename of the same node queued
|
||||||
|
/// behind it, which says where the node goes now. Returns whether it did.
|
||||||
|
fn drop_replaced_rename(conn: &rusqlite::Connection, id: i64) -> Result<bool> {
|
||||||
|
let dropped = conn.execute(
|
||||||
|
"DELETE FROM pending_op
|
||||||
|
WHERE id = ?1 AND kind = ?2
|
||||||
|
AND EXISTS (SELECT 1 FROM pending_op AS newer
|
||||||
|
WHERE (newer.uid = pending_op.uid OR newer.lid = pending_op.lid)
|
||||||
|
AND newer.kind = ?2 AND newer.id > ?1)",
|
||||||
|
params![id, OP_RENAME],
|
||||||
|
)?;
|
||||||
|
Ok(dropped > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The uid a queued op's node has now: the one its row holds, which a landing
|
||||||
|
/// create has already moved to its real uid, or the one the op was queued with
|
||||||
|
/// when the node has no row.
|
||||||
|
const UID_NOW: &str = "COALESCE((SELECT uid FROM nodes WHERE lid = pending_op.lid), uid)";
|
||||||
|
|
||||||
|
/// The uid a queued op's parent has now: the one its row holds, which a
|
||||||
|
/// landing folder has already moved to its real uid, or the one the op was
|
||||||
|
/// queued with when the parent has no row.
|
||||||
|
const PARENT_NOW: &str =
|
||||||
|
"COALESCE((SELECT uid FROM nodes WHERE lid = pending_op.parent_lid), parent_uid)";
|
||||||
|
|
||||||
/// The volume id given to a node that exists only on this machine, so far. A
|
/// The volume id given to a node that exists only on this machine, so far. A
|
||||||
/// real [`NodeUid`] is `{volume}~{link}`, so a placeholder is `local~<uuid>` and
|
/// real [`NodeUid`] is `{volume}~{link}`, so a placeholder is `local~<lid>` and
|
||||||
/// round-trips through the same `Display`/parse path as any other uid.
|
/// round-trips through the same `Display`/parse path as any other uid.
|
||||||
///
|
///
|
||||||
/// Nothing bearing this volume may be handed to the API — it would 404. The
|
/// Nothing bearing this volume may be handed to the API — it would 404. The
|
||||||
/// drain replaces it with the uid the server assigns.
|
/// drain replaces it with the uid the server assigns.
|
||||||
pub const LOCAL_VOLUME: &str = "local";
|
pub const LOCAL_VOLUME: &str = "local";
|
||||||
|
|
||||||
|
/// The uid a node made on this machine goes by until Drive gives it one: its
|
||||||
|
/// row's local id under [`LOCAL_VOLUME`].
|
||||||
|
///
|
||||||
|
/// A function of the row, so it never changes: whatever still holds it once
|
||||||
|
/// the node has landed finds the node, and its real uid, through
|
||||||
|
/// [`local_lid`].
|
||||||
|
pub fn local_uid(lid: i64) -> String {
|
||||||
|
format!("{LOCAL_VOLUME}~{lid}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The local id a [`local_uid`] stands for, or `None` for any other uid,
|
||||||
|
/// including a placeholder minted before schema 41.
|
||||||
|
pub fn local_lid(uid: &str) -> Option<i64> {
|
||||||
|
let digits = uid.strip_prefix(LOCAL_VOLUME)?.strip_prefix('~')?;
|
||||||
|
if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
digits.parse().ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// A `next_attempt_at` far enough in the future that [`Db::next_due_op`] never
|
/// A `next_attempt_at` far enough in the future that [`Db::next_due_op`] never
|
||||||
/// selects the op (ms since epoch, ≈ year 2223). Used to *park* a queued create
|
/// selects the op (ms since epoch, ≈ year 2223). Used to *park* a queued create
|
||||||
/// for a transient file — a browser's in-flight `*.crdownload`/`*.part`, an
|
/// for a transient file — a browser's in-flight `*.crdownload`/`*.part`, an
|
||||||
@@ -214,7 +357,8 @@ pub struct PendingCounts {
|
|||||||
/// never drain on their own, so counting them as "waiting to upload" would
|
/// never drain on their own, so counting them as "waiting to upload" would
|
||||||
/// have the front-end promise progress that is not coming.
|
/// have the front-end promise progress that is not coming.
|
||||||
pub parked: i64,
|
pub parked: i64,
|
||||||
/// Ops that have failed at least [`FAILING_ATTEMPTS`] times. A permanently
|
/// Ops that have failed at least [`FAILING_ATTEMPTS`] times, or that Drive
|
||||||
|
/// refused ([`Db::set_op_issue`]). A permanently
|
||||||
/// failing op never wedges the queue — the backoff sees to that — which is
|
/// failing op never wedges the queue — the backoff sees to that — which is
|
||||||
/// exactly why it needs surfacing: it retries forever, invisibly.
|
/// exactly why it needs surfacing: it retries forever, invisibly.
|
||||||
pub failing: i64,
|
pub failing: i64,
|
||||||
@@ -242,6 +386,20 @@ pub struct CreateLanding<'a> {
|
|||||||
pub landed: (&'a str, &'a str),
|
pub landed: (&'a str, &'a str),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What became of a queued create when it landed, from [`Db::finish_create`].
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum CreateRetired {
|
||||||
|
/// Its row is gone and nothing of it is left queued.
|
||||||
|
Landed,
|
||||||
|
/// A newer write attached to it on the wire and is now queued as a
|
||||||
|
/// revision of the real node: its blob and its rewritten sidecar.
|
||||||
|
Newer { blob: String, meta: String },
|
||||||
|
/// Its row had been dropped while it was on the wire: the node was deleted,
|
||||||
|
/// or a rename replaced it. The node Drive just made is unwanted, and a
|
||||||
|
/// trash of it is queued under this id, claimed by the caller.
|
||||||
|
Withdrawn { trash: i64 },
|
||||||
|
}
|
||||||
|
|
||||||
struct CreateRow {
|
struct CreateRow {
|
||||||
blob: Option<String>,
|
blob: Option<String>,
|
||||||
meta: Option<String>,
|
meta: Option<String>,
|
||||||
@@ -278,14 +436,64 @@ impl Db {
|
|||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the trash of a create a delete withdrew after it was sent is
|
||||||
|
/// queued for `name` in `parent_uid` ([`drop_doomed_ops`]).
|
||||||
|
///
|
||||||
|
/// The drain looks for the node that create may have made by its folder
|
||||||
|
/// and name, so a create of that name there waits for the trash: sent
|
||||||
|
/// first, it made a node the trash took for the withdrawn one.
|
||||||
|
pub fn has_withdrawn_create(&self, parent_uid: &str, name: &str) -> Result<bool> {
|
||||||
|
let conn = self.read();
|
||||||
|
conn.query_row(
|
||||||
|
&format!(
|
||||||
|
"SELECT EXISTS(
|
||||||
|
SELECT 1 FROM pending_op
|
||||||
|
WHERE kind = ?1 AND {PARENT_NOW} = ?2 AND name = ?3
|
||||||
|
AND uid LIKE '{LOCAL_VOLUME}~%'
|
||||||
|
)"
|
||||||
|
),
|
||||||
|
params![OP_TRASH, parent_uid, name],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a queued rename moves a node away from `name`.
|
||||||
|
///
|
||||||
|
/// Like a queued trash ([`Db::has_pending_trash_named`]) it frees the name
|
||||||
|
/// once it lands, so an op that wants the name can wait for it. A create
|
||||||
|
/// renamed while it was on the wire queues one (`Db::finish_create`).
|
||||||
|
pub fn has_pending_move_from(&self, name: &str) -> Result<bool> {
|
||||||
|
let conn = self.read();
|
||||||
|
conn.query_row(
|
||||||
|
&format!("SELECT EXISTS(SELECT 1 FROM pending_op WHERE {MOVES_FROM})"),
|
||||||
|
params![OP_RENAME, name],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make the ops that failed because `name` was held due at once, now that
|
||||||
|
/// what held it let go. Nothing changes while another queued trash or
|
||||||
|
/// rename still holds it, or for a parked op.
|
||||||
|
pub fn wake_ops_waiting_for(&self, name: &str) -> Result<()> {
|
||||||
|
let mut conn = self.conn.lock();
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
wake_ops_waiting_for_tx(&tx, name)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether a specific desired-state operation is still queued for a node.
|
/// Whether a specific desired-state operation is still queued for a node.
|
||||||
pub fn has_pending_op(&self, uid: &str, kind: &str) -> Result<bool> {
|
pub fn has_pending_op(&self, uid: &str, kind: &str) -> Result<bool> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
"SELECT EXISTS(
|
&format!(
|
||||||
SELECT 1 FROM pending_op WHERE uid = ?1 AND kind = ?2
|
"SELECT EXISTS(SELECT 1 FROM pending_op WHERE {} AND kind = ?2)",
|
||||||
)",
|
on_node(3)
|
||||||
params![uid, kind],
|
),
|
||||||
|
params![uid, kind, lid],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)
|
)
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
@@ -294,9 +502,13 @@ impl Db {
|
|||||||
/// Read an existing operation's metadata before a superseding enqueue.
|
/// Read an existing operation's metadata before a superseding enqueue.
|
||||||
pub fn pending_op_meta(&self, uid: &str, kind: &str) -> Result<Option<String>> {
|
pub fn pending_op_meta(&self, uid: &str, kind: &str) -> Result<Option<String>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
"SELECT meta_json FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
&format!(
|
||||||
params![uid, kind],
|
"SELECT meta_json FROM pending_op WHERE {} AND kind = ?2",
|
||||||
|
on_node(3)
|
||||||
|
),
|
||||||
|
params![uid, kind, lid],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)
|
)
|
||||||
.optional()
|
.optional()
|
||||||
@@ -304,21 +516,32 @@ impl Db {
|
|||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Queue `op`, returning its id and the staged blob of the op it replaced.
|
||||||
|
///
|
||||||
|
/// A rename a drain worker holds is not replaced but queued behind: the
|
||||||
|
/// claim keeps the newer one from being sent until it retires, so the two
|
||||||
|
/// land in the order they were made (`docs/BUGS.md` B141). Should it fail,
|
||||||
|
/// [`Db::record_op_failure`] or [`Db::release_op_claim`] drops it.
|
||||||
pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option<String>)> {
|
pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option<String>)> {
|
||||||
let mut conn = self.conn.lock();
|
let mut conn = self.conn.lock();
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
let superseded: Option<String> = if op_supersedes(&op.kind) {
|
let superseded: Option<String> = if op_supersedes(&op.kind) {
|
||||||
|
let lid = node_lid(&tx, &op.uid)?;
|
||||||
|
let same = format!(
|
||||||
|
"{} AND kind = ?2 AND (kind <> ?3 OR claimed_at = 0)",
|
||||||
|
on_node(4)
|
||||||
|
);
|
||||||
let blob: Option<String> = tx
|
let blob: Option<String> = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
&format!("SELECT blob_path FROM pending_op WHERE {same}"),
|
||||||
params![op.uid, op.kind],
|
params![op.uid, op.kind, OP_RENAME, lid],
|
||||||
|r| r.get(0),
|
|r| r.get(0),
|
||||||
)
|
)
|
||||||
.optional()?
|
.optional()?
|
||||||
.flatten();
|
.flatten();
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"DELETE FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
&format!("DELETE FROM pending_op WHERE {same}"),
|
||||||
params![op.uid, op.kind],
|
params![op.uid, op.kind, OP_RENAME, lid],
|
||||||
)?;
|
)?;
|
||||||
blob
|
blob
|
||||||
} else {
|
} else {
|
||||||
@@ -340,6 +563,7 @@ impl Db {
|
|||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
let id = tx.last_insert_rowid();
|
let id = tx.last_insert_rowid();
|
||||||
|
fill_op_lids(&tx, id)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok((id, superseded))
|
Ok((id, superseded))
|
||||||
}
|
}
|
||||||
@@ -366,6 +590,7 @@ impl Db {
|
|||||||
params![OP_TRASH, uid, name, created_at],
|
params![OP_TRASH, uid, name, created_at],
|
||||||
)?;
|
)?;
|
||||||
let id = tx.last_insert_rowid();
|
let id = tx.last_insert_rowid();
|
||||||
|
fill_op_lids(&tx, id)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok((id, blobs))
|
Ok((id, blobs))
|
||||||
}
|
}
|
||||||
@@ -386,10 +611,14 @@ impl Db {
|
|||||||
meta_json: &str,
|
meta_json: &str,
|
||||||
) -> Result<Option<AttachedBlob>> {
|
) -> Result<Option<AttachedBlob>> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
let existing: Option<(i64, Option<String>)> = conn
|
let existing: Option<(i64, Option<String>)> = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT id, blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
&format!(
|
||||||
params![uid, OP_CREATE],
|
"SELECT id, blob_path FROM pending_op WHERE {} AND kind = ?2",
|
||||||
|
on_node(3)
|
||||||
|
),
|
||||||
|
params![uid, OP_CREATE, lid],
|
||||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
@@ -429,21 +658,30 @@ impl Db {
|
|||||||
/// row names another parent or name than `landing.sent`, a rename of `real`
|
/// row names another parent or name than `landing.sent`, a rename of `real`
|
||||||
/// to that target is queued in the same transaction, or the move would be
|
/// to that target is queued in the same transaction, or the move would be
|
||||||
/// lost with the row.
|
/// lost with the row.
|
||||||
|
///
|
||||||
|
/// A delete while the create is on the wire cannot stop the upload. It
|
||||||
|
/// turns the row into a trash ([`Db::delete_ops_for_uid`]), which is
|
||||||
|
/// pointed at `real` here, still claimed by the caller, so the node is not
|
||||||
|
/// left on Drive holding the name (`docs/BUGS.md` B129). A row gone
|
||||||
|
/// altogether is queued as such a trash. The placeholder row is left to
|
||||||
|
/// the delete that withdrew it.
|
||||||
pub fn finish_create(
|
pub fn finish_create(
|
||||||
&self,
|
&self,
|
||||||
id: i64,
|
id: i64,
|
||||||
uploaded: Option<&str>,
|
uploaded: Option<&str>,
|
||||||
landing: &CreateLanding<'_>,
|
landing: &CreateLanding<'_>,
|
||||||
rewrite: impl FnOnce(&str) -> Option<String>,
|
rewrite: impl FnOnce(&str) -> Option<String>,
|
||||||
) -> Result<Option<(String, String)>> {
|
) -> Result<CreateRetired> {
|
||||||
let (local, real) = (landing.local, landing.real);
|
let (local, real) = (landing.local, landing.real);
|
||||||
let mut conn = self.conn.lock();
|
let mut conn = self.conn.lock();
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
let row: Option<CreateRow> = tx
|
let row: Option<CreateRow> = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT blob_path, meta_json, parent_uid, name, created_at
|
&format!(
|
||||||
FROM pending_op WHERE id = ?1",
|
"SELECT blob_path, meta_json, {PARENT_NOW}, name, created_at
|
||||||
params![id],
|
FROM pending_op WHERE id = ?1 AND kind <> ?2"
|
||||||
|
),
|
||||||
|
params![id, OP_TRASH],
|
||||||
|r| {
|
|r| {
|
||||||
Ok(CreateRow {
|
Ok(CreateRow {
|
||||||
blob: r.get(0)?,
|
blob: r.get(0)?,
|
||||||
@@ -455,6 +693,29 @@ impl Db {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
|
let withdrawn = tx.execute(
|
||||||
|
"UPDATE pending_op SET uid = ?2, name = ?3 WHERE id = ?1 AND kind = ?4",
|
||||||
|
params![id, real, landing.landed.1, OP_TRASH],
|
||||||
|
)?;
|
||||||
|
if withdrawn > 0 {
|
||||||
|
tx.commit()?;
|
||||||
|
return Ok(CreateRetired::Withdrawn { trash: id });
|
||||||
|
}
|
||||||
|
if row.is_none() {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map_or(1, |d| d.as_millis() as i64);
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO pending_op
|
||||||
|
(kind, uid, parent_uid, name, created_at, next_attempt_at, claimed_at)
|
||||||
|
VALUES (?1, ?2, NULL, ?3, ?4, 0, ?4)",
|
||||||
|
params![OP_TRASH, real, landing.landed.1, now],
|
||||||
|
)?;
|
||||||
|
let trash = tx.last_insert_rowid();
|
||||||
|
fill_op_lids(&tx, trash)?;
|
||||||
|
tx.commit()?;
|
||||||
|
return Ok(CreateRetired::Withdrawn { trash });
|
||||||
|
}
|
||||||
let newer = match &row {
|
let newer = match &row {
|
||||||
Some(CreateRow {
|
Some(CreateRow {
|
||||||
blob: Some(blob),
|
blob: Some(blob),
|
||||||
@@ -467,7 +728,8 @@ impl Db {
|
|||||||
Some((_, meta)) => {
|
Some((_, meta)) => {
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"UPDATE pending_op
|
"UPDATE pending_op
|
||||||
SET kind = ?2, uid = ?3, parent_uid = NULL, name = NULL, meta_json = ?4,
|
SET kind = ?2, uid = ?3, parent_uid = NULL, parent_lid = NULL, name = NULL,
|
||||||
|
meta_json = ?4,
|
||||||
attempts = 0, last_error = NULL, next_attempt_at = 0
|
attempts = 0, last_error = NULL, next_attempt_at = 0
|
||||||
WHERE id = ?1",
|
WHERE id = ?1",
|
||||||
params![id, OP_REVISION, real, meta],
|
params![id, OP_REVISION, real, meta],
|
||||||
@@ -490,22 +752,105 @@ impl Db {
|
|||||||
original_name: Some(landing.landed.1.to_string()),
|
original_name: Some(landing.landed.1.to_string()),
|
||||||
})?;
|
})?;
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"DELETE FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
&format!("DELETE FROM pending_op WHERE {} AND kind = ?2", on_node(3)),
|
||||||
params![real, OP_RENAME],
|
params![real, OP_RENAME, node_lid(&tx, local)?],
|
||||||
)?;
|
)?;
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"INSERT INTO pending_op (kind, uid, parent_uid, name, meta_json, created_at)
|
"INSERT INTO pending_op (kind, uid, parent_uid, name, meta_json, created_at)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||||
params![OP_RENAME, real, parent, name, meta, created_at],
|
params![OP_RENAME, real, parent, name, meta, created_at],
|
||||||
)?;
|
)?;
|
||||||
|
fill_op_lids(&tx, tx.last_insert_rowid())?;
|
||||||
}
|
}
|
||||||
tx.execute(
|
|
||||||
"UPDATE pending_op SET parent_uid = ?2 WHERE parent_uid = ?1",
|
|
||||||
params![local, real],
|
|
||||||
)?;
|
|
||||||
adopt_placeholder_row_tx(&tx, local, real)?;
|
adopt_placeholder_row_tx(&tx, local, real)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(newer)
|
Ok(match newer {
|
||||||
|
Some((blob, meta)) => CreateRetired::Newer { blob, meta },
|
||||||
|
None => CreateRetired::Landed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record that queued create `id` is sending a blob with SHA-1 `sha`.
|
||||||
|
///
|
||||||
|
/// Drive may make the file and lose its answer. The op that next finds
|
||||||
|
/// the file, the create's retry or the trash a delete made of it, takes
|
||||||
|
/// it for its own when it holds any bytes sent so. Kept on the row, so a
|
||||||
|
/// newer blob, the turn into a trash and a restart all keep it
|
||||||
|
/// (`docs/BUGS.md` B156, B183).
|
||||||
|
pub fn note_create_sent(&self, id: i64, sha: &str) -> Result<()> {
|
||||||
|
self.conn.lock().execute(
|
||||||
|
"UPDATE pending_op
|
||||||
|
SET sent_sha1 = CASE
|
||||||
|
WHEN sent_sha1 IS NULL THEN ?2
|
||||||
|
WHEN instr(' ' || sent_sha1 || ' ', ' ' || ?2 || ' ') > 0 THEN sent_sha1
|
||||||
|
ELSE sent_sha1 || ' ' || ?2 END
|
||||||
|
WHERE id = ?1",
|
||||||
|
params![id, sha],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record that queued create `id` is being sent to `parent` under `name`.
|
||||||
|
///
|
||||||
|
/// A rename while the op waits rewrites its target, but the file a lost
|
||||||
|
/// answer left stays where it was sent. The retry and the trash look
|
||||||
|
/// there too (`docs/BUGS.md` B184).
|
||||||
|
pub fn note_create_target(&self, id: i64, parent: &str, name: &str) -> Result<()> {
|
||||||
|
let mut conn = self.conn.lock();
|
||||||
|
let tx = conn.transaction()?;
|
||||||
|
let sent: Option<String> = tx
|
||||||
|
.query_row("SELECT sent_to FROM pending_op WHERE id = ?1", [id], |r| {
|
||||||
|
r.get(0)
|
||||||
|
})
|
||||||
|
.optional()?
|
||||||
|
.flatten();
|
||||||
|
let mut targets: Vec<(String, String)> = sent
|
||||||
|
.and_then(|json| serde_json::from_str(&json).ok())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let target = (parent.to_string(), name.to_string());
|
||||||
|
if !targets.contains(&target) {
|
||||||
|
targets.push(target);
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE pending_op SET sent_to = ?2 WHERE id = ?1",
|
||||||
|
params![id, serde_json::to_string(&targets)?],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The parent and name queued op `id` was sent under as a create, in the
|
||||||
|
/// order it was ([`Db::note_create_target`]).
|
||||||
|
pub fn create_targets(&self, id: i64) -> Result<Vec<(String, String)>> {
|
||||||
|
let sent: Option<String> = self
|
||||||
|
.conn
|
||||||
|
.lock()
|
||||||
|
.query_row("SELECT sent_to FROM pending_op WHERE id = ?1", [id], |r| {
|
||||||
|
r.get(0)
|
||||||
|
})
|
||||||
|
.optional()?
|
||||||
|
.flatten();
|
||||||
|
Ok(sent
|
||||||
|
.and_then(|json| serde_json::from_str(&json).ok())
|
||||||
|
.unwrap_or_default())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The SHA-1 of each blob queued op `id` sent as a create
|
||||||
|
/// ([`Db::note_create_sent`]).
|
||||||
|
pub fn create_sent(&self, id: i64) -> Result<Vec<String>> {
|
||||||
|
let sent: Option<String> = self
|
||||||
|
.conn
|
||||||
|
.lock()
|
||||||
|
.query_row(
|
||||||
|
"SELECT sent_sha1 FROM pending_op WHERE id = ?1",
|
||||||
|
[id],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.optional()?
|
||||||
|
.flatten();
|
||||||
|
Ok(sent
|
||||||
|
.map(|sent| sent.split(' ').map(str::to_string).collect())
|
||||||
|
.unwrap_or_default())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Replace the sidecar of a queued op, for a baseline that has moved under
|
/// Replace the sidecar of a queued op, for a baseline that has moved under
|
||||||
@@ -524,9 +869,13 @@ impl Db {
|
|||||||
/// queued, which is the ordinary case.
|
/// queued, which is the ordinary case.
|
||||||
pub fn update_op_meta(&self, uid: &str, kind: &str, meta_json: &str) -> Result<bool> {
|
pub fn update_op_meta(&self, uid: &str, kind: &str, meta_json: &str) -> Result<bool> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
let n = conn.execute(
|
let n = conn.execute(
|
||||||
"UPDATE pending_op SET meta_json = ?3 WHERE uid = ?1 AND kind = ?2",
|
&format!(
|
||||||
params![uid, kind, meta_json],
|
"UPDATE pending_op SET meta_json = ?3 WHERE {} AND kind = ?2",
|
||||||
|
on_node(4)
|
||||||
|
),
|
||||||
|
params![uid, kind, meta_json, lid],
|
||||||
)?;
|
)?;
|
||||||
Ok(n > 0)
|
Ok(n > 0)
|
||||||
}
|
}
|
||||||
@@ -539,12 +888,25 @@ impl Db {
|
|||||||
/// *is* the rename. Returns false when the create has already drained, in
|
/// *is* the rename. Returns false when the create has already drained, in
|
||||||
/// which case the node has a real uid and the caller must rename it there
|
/// which case the node has a real uid and the caller must rename it there
|
||||||
/// instead (offline.md Phase 3b).
|
/// instead (offline.md Phase 3b).
|
||||||
|
///
|
||||||
|
/// A create backing off is made due: what it failed on may have been the
|
||||||
|
/// name it no longer wants, such as one held by a queued change. A parked
|
||||||
|
/// create stays parked.
|
||||||
pub fn rewrite_op_target(&self, uid: &str, parent_uid: &str, name: &str) -> Result<bool> {
|
pub fn rewrite_op_target(&self, uid: &str, parent_uid: &str, name: &str) -> Result<bool> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
|
let parent_lid = node_lid(&conn, parent_uid)?;
|
||||||
let n = conn.execute(
|
let n = conn.execute(
|
||||||
"UPDATE pending_op SET parent_uid = ?2, name = ?3
|
&format!(
|
||||||
WHERE uid = ?1 AND kind IN (?4, ?5)",
|
"UPDATE pending_op
|
||||||
params![uid, parent_uid, name, OP_CREATE, OP_MKDIR],
|
SET parent_uid = ?2, name = ?3, parent_lid = ?7,
|
||||||
|
next_attempt_at = CASE WHEN next_attempt_at < ?6 THEN 0 ELSE next_attempt_at END
|
||||||
|
WHERE {} AND kind IN (?4, ?5)",
|
||||||
|
on_node(8)
|
||||||
|
),
|
||||||
|
params![
|
||||||
|
uid, parent_uid, name, OP_CREATE, OP_MKDIR, PARK_UNTIL, parent_lid, lid
|
||||||
|
],
|
||||||
)?;
|
)?;
|
||||||
Ok(n > 0)
|
Ok(n > 0)
|
||||||
}
|
}
|
||||||
@@ -560,10 +922,19 @@ impl Db {
|
|||||||
/// already is.
|
/// already is.
|
||||||
pub fn set_create_hold(&self, uid: &str, held: bool) -> Result<bool> {
|
pub fn set_create_hold(&self, uid: &str, held: bool) -> Result<bool> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
let n = conn.execute(
|
let n = conn.execute(
|
||||||
"UPDATE pending_op SET next_attempt_at = ?2
|
&format!(
|
||||||
WHERE uid = ?1 AND kind IN (?3, ?4)",
|
"UPDATE pending_op SET next_attempt_at = ?2 WHERE {} AND kind IN (?3, ?4)",
|
||||||
params![uid, if held { PARK_UNTIL } else { 0 }, OP_CREATE, OP_MKDIR],
|
on_node(5)
|
||||||
|
),
|
||||||
|
params![
|
||||||
|
uid,
|
||||||
|
if held { PARK_UNTIL } else { 0 },
|
||||||
|
OP_CREATE,
|
||||||
|
OP_MKDIR,
|
||||||
|
lid
|
||||||
|
],
|
||||||
)?;
|
)?;
|
||||||
Ok(n > 0)
|
Ok(n > 0)
|
||||||
}
|
}
|
||||||
@@ -571,9 +942,13 @@ impl Db {
|
|||||||
/// Check if a queued create or mkdir op exists for `uid`.
|
/// Check if a queued create or mkdir op exists for `uid`.
|
||||||
pub fn has_create_op(&self, uid: &str) -> Result<bool> {
|
pub fn has_create_op(&self, uid: &str) -> Result<bool> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
let count: i64 = conn.query_row(
|
let count: i64 = conn.query_row(
|
||||||
"SELECT COUNT(*) FROM pending_op WHERE uid = ?1 AND kind IN (?2, ?3)",
|
&format!(
|
||||||
params![uid, OP_CREATE, OP_MKDIR],
|
"SELECT COUNT(*) FROM pending_op WHERE {} AND kind IN (?2, ?3)",
|
||||||
|
on_node(4)
|
||||||
|
),
|
||||||
|
params![uid, OP_CREATE, OP_MKDIR, lid],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
Ok(count > 0)
|
Ok(count > 0)
|
||||||
@@ -587,9 +962,10 @@ impl Db {
|
|||||||
/// blob that holds them (`docs/BUGS.md` B71).
|
/// blob that holds them (`docs/BUGS.md` B71).
|
||||||
pub fn has_any_op(&self, uid: &str) -> Result<bool> {
|
pub fn has_any_op(&self, uid: &str) -> Result<bool> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let lid = node_lid(&conn, uid)?;
|
||||||
let count: i64 = conn.query_row(
|
let count: i64 = conn.query_row(
|
||||||
"SELECT COUNT(*) FROM pending_op WHERE uid = ?1",
|
&format!("SELECT COUNT(*) FROM pending_op WHERE {}", on_node(2)),
|
||||||
params![uid],
|
params![uid, lid],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
Ok(count > 0)
|
Ok(count > 0)
|
||||||
@@ -620,17 +996,47 @@ impl Db {
|
|||||||
Ok(blobs)
|
Ok(blobs)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rewrite every queued op that points at a placeholder parent, once that
|
/// Drop the ops queued on the node `uid` names, and nothing else, returning
|
||||||
/// parent has drained and has a real uid. Also moves the node rows whose
|
/// the staged blobs they held. `None`, dropping nothing, while one of them
|
||||||
/// parent column still names the placeholder, so listings keep resolving,
|
/// is on the wire: it may land after the drop.
|
||||||
/// and readdresses the placeholder's own row.
|
///
|
||||||
pub fn remap_local_uid(&self, local: &str, real: &str) -> Result<()> {
|
/// For a change the user discards on a node Drive has. Unlike
|
||||||
|
/// [`Db::delete_ops_for_uid`], what is queued below the node stays: the
|
||||||
|
/// files made in a folder whose rename is discarded still go in it.
|
||||||
|
pub fn discard_ops_on_node(&self, uid: &str) -> Result<Option<Vec<String>>> {
|
||||||
let mut conn = self.conn.lock();
|
let mut conn = self.conn.lock();
|
||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
tx.execute(
|
let lid = node_lid(&tx, uid)?;
|
||||||
"UPDATE pending_op SET parent_uid = ?2 WHERE parent_uid = ?1",
|
let on = on_node(2);
|
||||||
params![local, real],
|
let sending: bool = tx.query_row(
|
||||||
|
&format!("SELECT EXISTS (SELECT 1 FROM pending_op WHERE {on} AND claimed_at <> 0)"),
|
||||||
|
params![uid, lid],
|
||||||
|
|r| r.get(0),
|
||||||
)?;
|
)?;
|
||||||
|
if sending {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let blobs = {
|
||||||
|
let mut stmt = tx.prepare(&format!(
|
||||||
|
"SELECT blob_path FROM pending_op WHERE {on} AND blob_path IS NOT NULL"
|
||||||
|
))?;
|
||||||
|
let rows = stmt.query_map(params![uid, lid], |r| r.get::<_, String>(0))?;
|
||||||
|
rows.collect::<std::result::Result<Vec<_>, _>>()?
|
||||||
|
};
|
||||||
|
tx.execute(
|
||||||
|
&format!("DELETE FROM pending_op WHERE {on}"),
|
||||||
|
params![uid, lid],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(Some(blobs))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What [`Db::finish_create`] does to the node rows and pins when a create
|
||||||
|
/// lands, without the op ([`adopt_placeholder_row_tx`]).
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn land_placeholder_row(&self, local: &str, real: &str) -> Result<()> {
|
||||||
|
let mut conn = self.conn.lock();
|
||||||
|
let tx = conn.transaction()?;
|
||||||
adopt_placeholder_row_tx(&tx, local, real)?;
|
adopt_placeholder_row_tx(&tx, local, real)?;
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -651,9 +1057,10 @@ impl Db {
|
|||||||
/// materialized and rejected in Rust:
|
/// materialized and rejected in Rust:
|
||||||
///
|
///
|
||||||
/// * `next_attempt_at <= now` — the backoff.
|
/// * `next_attempt_at <= now` — the backoff.
|
||||||
/// * a `local~` parent is skipped. A node created inside a folder that was
|
/// * a parent that has not landed is skipped, read through `parent_lid`
|
||||||
/// itself created offline cannot be sent anywhere until that folder is
|
/// from the parent's row ([`PARENT_NOW`]). A node created inside a folder
|
||||||
/// real. Ops replay in queue order so the parent normally drains first;
|
/// that was itself created offline cannot be sent anywhere until that
|
||||||
|
/// folder is real. Ops replay in queue order so the parent normally drains first;
|
||||||
/// this matters when the parent is backing off, where the child must wait
|
/// this matters when the parent is backing off, where the child must wait
|
||||||
/// rather than burn its own retries.
|
/// rather than burn its own retries.
|
||||||
///
|
///
|
||||||
@@ -664,11 +1071,11 @@ impl Db {
|
|||||||
pub fn next_due_op(&self, now: i64) -> Result<Option<PendingOp>> {
|
pub fn next_due_op(&self, now: i64) -> Result<Option<PendingOp>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare_cached(&format!(
|
let mut stmt = conn.prepare_cached(&format!(
|
||||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
"SELECT id, kind, {UID_NOW}, {PARENT_NOW}, name, blob_path, meta_json, created_at,
|
||||||
attempts, last_error, next_attempt_at
|
attempts, last_error, next_attempt_at
|
||||||
FROM pending_op
|
FROM pending_op
|
||||||
WHERE next_attempt_at <= ?1
|
WHERE next_attempt_at <= ?1
|
||||||
AND (parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~')
|
AND (parent_uid IS NULL OR substr({PARENT_NOW}, 1, {n}) <> '{v}~')
|
||||||
ORDER BY id LIMIT 1",
|
ORDER BY id LIMIT 1",
|
||||||
v = LOCAL_VOLUME,
|
v = LOCAL_VOLUME,
|
||||||
n = LOCAL_VOLUME.len() + 1,
|
n = LOCAL_VOLUME.len() + 1,
|
||||||
@@ -718,13 +1125,15 @@ impl Db {
|
|||||||
let tx = conn.transaction()?;
|
let tx = conn.transaction()?;
|
||||||
let op = {
|
let op = {
|
||||||
let mut stmt = tx.prepare_cached(&format!(
|
let mut stmt = tx.prepare_cached(&format!(
|
||||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
"SELECT id, kind, {UID_NOW}, {PARENT_NOW}, name, blob_path, meta_json, created_at,
|
||||||
attempts, last_error, next_attempt_at
|
attempts, last_error, next_attempt_at
|
||||||
FROM pending_op
|
FROM pending_op
|
||||||
WHERE next_attempt_at <= ?1
|
WHERE next_attempt_at <= ?1
|
||||||
AND claimed_at = 0
|
AND claimed_at = 0
|
||||||
AND (parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~')
|
AND (parent_uid IS NULL OR substr({PARENT_NOW}, 1, {n}) <> '{v}~')
|
||||||
AND uid NOT IN (SELECT uid FROM pending_op WHERE claimed_at <> 0)
|
AND uid NOT IN (SELECT uid FROM pending_op WHERE claimed_at <> 0)
|
||||||
|
AND (lid IS NULL OR lid NOT IN (
|
||||||
|
SELECT lid FROM pending_op WHERE claimed_at <> 0 AND lid IS NOT NULL))
|
||||||
ORDER BY id LIMIT 1",
|
ORDER BY id LIMIT 1",
|
||||||
v = LOCAL_VOLUME,
|
v = LOCAL_VOLUME,
|
||||||
n = LOCAL_VOLUME.len() + 1,
|
n = LOCAL_VOLUME.len() + 1,
|
||||||
@@ -763,8 +1172,17 @@ impl Db {
|
|||||||
/// `UPDATE` matches nothing), and a handler that returns without retiring
|
/// `UPDATE` matches nothing), and a handler that returns without retiring
|
||||||
/// *and* without failing would otherwise leave the row claimed by a worker
|
/// *and* without failing would otherwise leave the row claimed by a worker
|
||||||
/// that has moved on — invisible to every worker, forever.
|
/// that has moved on — invisible to every worker, forever.
|
||||||
|
///
|
||||||
|
/// A rename still here with a newer one queued behind it goes instead, as
|
||||||
|
/// in [`Db::record_op_failure`]. The newer one may have been queued after
|
||||||
|
/// the failure was recorded: kept, the older one sat out its backoff while
|
||||||
|
/// the newer one landed, and then put the old name back (`docs/BUGS.md`
|
||||||
|
/// B158).
|
||||||
pub fn release_op_claim(&self, id: i64) -> Result<()> {
|
pub fn release_op_claim(&self, id: i64) -> Result<()> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
if drop_replaced_rename(&conn, id)? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE pending_op SET claimed_at = 0 WHERE id = ?1",
|
"UPDATE pending_op SET claimed_at = 0 WHERE id = ?1",
|
||||||
params![id],
|
params![id],
|
||||||
@@ -801,7 +1219,7 @@ impl Db {
|
|||||||
&format!(
|
&format!(
|
||||||
"SELECT MIN(next_attempt_at) FROM pending_op \
|
"SELECT MIN(next_attempt_at) FROM pending_op \
|
||||||
WHERE claimed_at = 0 \
|
WHERE claimed_at = 0 \
|
||||||
AND (parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~')",
|
AND (parent_uid IS NULL OR substr({PARENT_NOW}, 1, {n}) <> '{v}~')",
|
||||||
v = LOCAL_VOLUME,
|
v = LOCAL_VOLUME,
|
||||||
n = LOCAL_VOLUME.len() + 1,
|
n = LOCAL_VOLUME.len() + 1,
|
||||||
),
|
),
|
||||||
@@ -817,11 +1235,11 @@ impl Db {
|
|||||||
/// drain wants [`next_due_op`](Self::next_due_op) instead.
|
/// drain wants [`next_due_op`](Self::next_due_op) instead.
|
||||||
pub fn pending_ops(&self) -> Result<Vec<PendingOp>> {
|
pub fn pending_ops(&self) -> Result<Vec<PendingOp>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
"SELECT id, kind, {UID_NOW}, {PARENT_NOW}, name, blob_path, meta_json, created_at,
|
||||||
attempts, last_error, next_attempt_at
|
attempts, last_error, next_attempt_at
|
||||||
FROM pending_op ORDER BY id",
|
FROM pending_op ORDER BY id"
|
||||||
)?;
|
))?;
|
||||||
let rows = stmt
|
let rows = stmt
|
||||||
.query_map([], |r| {
|
.query_map([], |r| {
|
||||||
Ok(PendingOp {
|
Ok(PendingOp {
|
||||||
@@ -849,13 +1267,13 @@ impl Db {
|
|||||||
/// the one query that does.
|
/// the one query that does.
|
||||||
pub fn parked_create_ops(&self) -> Result<Vec<PendingOp>> {
|
pub fn parked_create_ops(&self) -> Result<Vec<PendingOp>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
"SELECT id, kind, {UID_NOW}, {PARENT_NOW}, name, blob_path, meta_json, created_at,
|
||||||
attempts, last_error, next_attempt_at
|
attempts, last_error, next_attempt_at
|
||||||
FROM pending_op
|
FROM pending_op
|
||||||
WHERE next_attempt_at >= ?1 AND kind IN (?2, ?3)
|
WHERE next_attempt_at >= ?1 AND kind IN (?2, ?3)
|
||||||
ORDER BY created_at",
|
ORDER BY created_at"
|
||||||
)?;
|
))?;
|
||||||
let rows = stmt
|
let rows = stmt
|
||||||
.query_map(params![PARK_UNTIL, OP_CREATE, OP_MKDIR], |r| {
|
.query_map(params![PARK_UNTIL, OP_CREATE, OP_MKDIR], |r| {
|
||||||
Ok(PendingOp {
|
Ok(PendingOp {
|
||||||
@@ -900,15 +1318,15 @@ impl Db {
|
|||||||
|r| r.get(0),
|
|r| r.get(0),
|
||||||
)?;
|
)?;
|
||||||
let failing = conn.query_row(
|
let failing = conn.query_row(
|
||||||
"SELECT COUNT(*) FROM pending_op WHERE attempts >= ?1",
|
"SELECT COUNT(*) FROM pending_op WHERE attempts >= ?1 OR issue IS NOT NULL",
|
||||||
params![FAILING_ATTEMPTS],
|
params![FAILING_ATTEMPTS],
|
||||||
|r| r.get(0),
|
|r| r.get(0),
|
||||||
)?;
|
)?;
|
||||||
let last_error = conn
|
let last_error = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT last_error FROM pending_op
|
"SELECT last_error FROM pending_op
|
||||||
WHERE attempts >= ?1 AND last_error IS NOT NULL
|
WHERE (attempts >= ?1 OR issue IS NOT NULL) AND last_error IS NOT NULL
|
||||||
ORDER BY attempts DESC LIMIT 1",
|
ORDER BY issue IS NULL, attempts DESC LIMIT 1",
|
||||||
params![FAILING_ATTEMPTS],
|
params![FAILING_ATTEMPTS],
|
||||||
|r| r.get::<_, Option<String>>(0),
|
|r| r.get::<_, Option<String>>(0),
|
||||||
)
|
)
|
||||||
@@ -978,18 +1396,118 @@ impl Db {
|
|||||||
/// Clears any access-deferral window: whatever the op was waiting for has
|
/// Clears any access-deferral window: whatever the op was waiting for has
|
||||||
/// now been reported, so the next uncleared deferral starts its own window
|
/// now been reported, so the next uncleared deferral starts its own window
|
||||||
/// rather than escalating again on the following recheck.
|
/// rather than escalating again on the following recheck.
|
||||||
|
///
|
||||||
|
/// A rename with a newer one queued behind it ([`Db::enqueue_op`]) is
|
||||||
|
/// dropped instead: retried, it could land after the newer one and undo it.
|
||||||
pub fn record_op_failure(&self, id: i64, error: &str, next_attempt_at: i64) -> Result<()> {
|
pub fn record_op_failure(&self, id: i64, error: &str, next_attempt_at: i64) -> Result<()> {
|
||||||
|
self.record_failure(id, None, error, next_attempt_at)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`Db::record_op_failure`] for an attempt at op `id` as a `kind` op. An
|
||||||
|
/// op that has become another kind while the attempt ran is left as it
|
||||||
|
/// is: a create deleted on the wire is a trash now, and the create's
|
||||||
|
/// failure would hold the trash back for the create's backoff
|
||||||
|
/// (`docs/BUGS.md` B171).
|
||||||
|
pub fn record_attempt_failure(
|
||||||
|
&self,
|
||||||
|
id: i64,
|
||||||
|
kind: &str,
|
||||||
|
error: &str,
|
||||||
|
next_attempt_at: i64,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.record_failure(id, Some(kind), error, next_attempt_at)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_failure(
|
||||||
|
&self,
|
||||||
|
id: i64,
|
||||||
|
kind: Option<&str>,
|
||||||
|
error: &str,
|
||||||
|
next_attempt_at: i64,
|
||||||
|
) -> Result<()> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
|
if drop_replaced_rename(&conn, id)? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE pending_op
|
"UPDATE pending_op
|
||||||
SET attempts = attempts + 1, last_error = ?2, next_attempt_at = ?3,
|
SET attempts = attempts + 1, last_error = ?2, next_attempt_at = ?3,
|
||||||
access_deferred_since = 0
|
access_deferred_since = 0
|
||||||
WHERE id = ?1",
|
WHERE id = ?1 AND (?4 IS NULL OR kind = ?4)",
|
||||||
params![id, error, next_attempt_at],
|
params![id, error, next_attempt_at, kind],
|
||||||
)?;
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Record why Drive refused op `id`, or that the latest failure was not a
|
||||||
|
/// refusal (`None`). An op with an issue counts as failing at once.
|
||||||
|
pub fn set_op_issue(&self, id: i64, issue: Option<&str>) -> Result<()> {
|
||||||
|
self.conn.lock().execute(
|
||||||
|
"UPDATE pending_op SET issue = ?2 WHERE id = ?1",
|
||||||
|
params![id, issue],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The issue of every op Drive refused, by op id.
|
||||||
|
pub fn op_issues(&self) -> Result<HashMap<i64, String>> {
|
||||||
|
let conn = self.read();
|
||||||
|
let mut stmt = conn.prepare("SELECT id, issue FROM pending_op WHERE issue IS NOT NULL")?;
|
||||||
|
let rows = stmt
|
||||||
|
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?
|
||||||
|
.collect::<rusqlite::Result<_>>()?;
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The issue each node with a refused change has, by local id: what the
|
||||||
|
/// file browser shows on the node itself.
|
||||||
|
pub fn node_issues(&self) -> Result<HashMap<i64, String>> {
|
||||||
|
let conn = self.read();
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT COALESCE(o.lid, n.lid), o.issue FROM pending_op o
|
||||||
|
LEFT JOIN nodes n ON o.lid IS NULL AND n.uid = o.uid
|
||||||
|
WHERE o.issue IS NOT NULL",
|
||||||
|
)?;
|
||||||
|
let rows = stmt
|
||||||
|
.query_map([], |r| Ok((r.get::<_, Option<i64>>(0)?, r.get(1)?)))?
|
||||||
|
.filter_map(|row| match row {
|
||||||
|
Ok((Some(lid), issue)) => Some(Ok((lid, issue))),
|
||||||
|
Ok((None, _)) => None,
|
||||||
|
Err(e) => Some(Err(e)),
|
||||||
|
})
|
||||||
|
.collect::<rusqlite::Result<_>>()?;
|
||||||
|
Ok(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make a create due again if it was renamed or moved
|
||||||
|
/// ([`Db::rewrite_op_target`]) while the attempt that just failed was on
|
||||||
|
/// the wire, `parent_uid` and `name` being what that attempt sent. What it
|
||||||
|
/// failed on may have been the target it no longer has. Likewise if a
|
||||||
|
/// write replaced its blob (`blob_path` being the one the attempt read):
|
||||||
|
/// the old one is discarded as the new one attaches, and an attempt that
|
||||||
|
/// had not opened it yet failed on a file that is gone. Returns whether it
|
||||||
|
/// was.
|
||||||
|
pub fn retry_if_retargeted(
|
||||||
|
&self,
|
||||||
|
id: i64,
|
||||||
|
parent_uid: Option<&str>,
|
||||||
|
name: Option<&str>,
|
||||||
|
blob_path: Option<&str>,
|
||||||
|
) -> Result<bool> {
|
||||||
|
let conn = self.conn.lock();
|
||||||
|
let n = conn.execute(
|
||||||
|
&format!(
|
||||||
|
"UPDATE pending_op SET next_attempt_at = 0
|
||||||
|
WHERE id = ?1 AND kind IN (?4, ?5) AND next_attempt_at < ?6
|
||||||
|
AND ({PARENT_NOW} IS NOT ?2 OR name IS NOT ?3 OR blob_path IS NOT ?7)"
|
||||||
|
),
|
||||||
|
params![
|
||||||
|
id, parent_uid, name, OP_CREATE, OP_MKDIR, PARK_UNTIL, blob_path
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(n > 0)
|
||||||
|
}
|
||||||
|
|
||||||
/// Defer a due operation without recording a failed remote attempt.
|
/// Defer a due operation without recording a failed remote attempt.
|
||||||
///
|
///
|
||||||
/// Access checks happen before the drain calls Proton, so a locally denied
|
/// Access checks happen before the drain calls Proton, so a locally denied
|
||||||
|
|||||||
@@ -64,11 +64,11 @@ impl Db {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Whether `uid` is pinned — directly, or because a strict ancestor folder
|
/// Whether `uid` is pinned — directly, or because a strict ancestor folder
|
||||||
/// is pinned recursively. The ancestor check walks `parent_uid` to the root
|
/// is pinned recursively. The ancestor check walks `parent_lid` to the root
|
||||||
/// via a CTE; a direct pin is honoured even when the node has no `nodes` row
|
/// via a CTE; a direct pin is honoured even when the node has no `nodes` row
|
||||||
/// yet (e.g. a CLI that never hydrates the node cache).
|
/// yet (e.g. a CLI that never hydrates the node cache).
|
||||||
///
|
///
|
||||||
/// The walk is depth-capped like [`super::utils::path_of`]: a `parent_uid`
|
/// The walk is depth-capped like [`super::utils::path_of`]: a parent
|
||||||
/// cycle is corrupt data the API can hand us, and `UNION ALL` over one never
|
/// cycle is corrupt data the API can hand us, and `UNION ALL` over one never
|
||||||
/// terminates — while holding the daemon's only SQLite connection, on a path
|
/// terminates — while holding the daemon's only SQLite connection, on a path
|
||||||
/// that runs per cached read.
|
/// that runs per cached read.
|
||||||
@@ -85,11 +85,11 @@ impl Db {
|
|||||||
let anc: Option<i64> = conn
|
let anc: Option<i64> = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
&format!(
|
&format!(
|
||||||
"WITH RECURSIVE anc(uid, parent_uid, depth) AS (
|
"WITH RECURSIVE anc(uid, parent_lid, depth) AS (
|
||||||
SELECT uid, parent_uid, 0 FROM nodes WHERE uid = ?1
|
SELECT uid, parent_lid, 0 FROM nodes WHERE uid = ?1
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT n.uid, n.parent_uid, anc.depth + 1
|
SELECT n.uid, n.parent_lid, anc.depth + 1
|
||||||
FROM nodes n JOIN anc ON n.uid = anc.parent_uid
|
FROM nodes n JOIN anc ON n.lid = anc.parent_lid
|
||||||
WHERE anc.depth < {MAX_PATH_DEPTH}
|
WHERE anc.depth < {MAX_PATH_DEPTH}
|
||||||
)
|
)
|
||||||
SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid
|
SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid
|
||||||
@@ -109,10 +109,11 @@ impl Db {
|
|||||||
pub fn pinned_uids(&self) -> Result<Vec<String>> {
|
pub fn pinned_uids(&self) -> Result<Vec<String>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
"WITH RECURSIVE sub(uid) AS (
|
"WITH RECURSIVE sub(uid, lid) AS (
|
||||||
SELECT uid FROM pins WHERE recursive = 1
|
SELECT p.uid, n.lid FROM pins p LEFT JOIN nodes n ON n.uid = p.uid
|
||||||
|
WHERE p.recursive = 1
|
||||||
UNION
|
UNION
|
||||||
SELECT n.uid FROM nodes n JOIN sub ON n.parent_uid = sub.uid
|
SELECT n.uid, n.lid FROM nodes n JOIN sub ON n.parent_lid = sub.lid
|
||||||
)
|
)
|
||||||
SELECT uid FROM sub
|
SELECT uid FROM sub
|
||||||
UNION
|
UNION
|
||||||
@@ -126,15 +127,16 @@ impl Db {
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every descendant uid of `folder` (all depths), via a `parent_uid` CTE.
|
/// Every descendant uid of `folder` (all depths), via a `parent_lid` CTE.
|
||||||
/// Used to evict a recursively-pinned subtree's cached blobs on unpin.
|
/// Used to evict a recursively-pinned subtree's cached blobs on unpin.
|
||||||
pub fn descendants(&self, folder: &str) -> Result<Vec<String>> {
|
pub fn descendants(&self, folder: &str) -> Result<Vec<String>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
"WITH RECURSIVE sub(uid) AS (
|
"WITH RECURSIVE sub(uid, lid) AS (
|
||||||
SELECT uid FROM nodes WHERE parent_uid = ?1
|
SELECT c.uid, c.lid FROM nodes c JOIN nodes f ON c.parent_lid = f.lid
|
||||||
|
WHERE f.uid = ?1
|
||||||
UNION
|
UNION
|
||||||
SELECT n.uid FROM nodes n JOIN sub ON n.parent_uid = sub.uid
|
SELECT n.uid, n.lid FROM nodes n JOIN sub ON n.parent_lid = sub.lid
|
||||||
)
|
)
|
||||||
SELECT uid FROM sub",
|
SELECT uid FROM sub",
|
||||||
)?;
|
)?;
|
||||||
|
|||||||
@@ -95,11 +95,27 @@ impl Db {
|
|||||||
/// Resolve a persisted node's effective access through its nearest recorded
|
/// Resolve a persisted node's effective access through its nearest recorded
|
||||||
/// share root. `None` means the node itself is not persisted, which callers
|
/// share root. `None` means the node itself is not persisted, which callers
|
||||||
/// must not treat as owned: stale handles need to fail closed.
|
/// must not treat as owned: stale handles need to fail closed.
|
||||||
|
///
|
||||||
|
/// A stand-in ([`local_uid`](super::local_uid)) finds its row by local id,
|
||||||
|
/// so it still does once its create has landed and the row has the uid
|
||||||
|
/// Drive gave it. The tree learns that uid a moment later, and a rename in
|
||||||
|
/// between was refused (`docs/BUGS.md` B174).
|
||||||
pub fn effective_node_access(&self, uid: &NodeUid) -> Result<Option<Access>> {
|
pub fn effective_node_access(&self, uid: &NodeUid) -> Result<Option<Access>> {
|
||||||
let conn = self.read();
|
let conn = self.read();
|
||||||
|
let key = uid.to_string();
|
||||||
|
let key = match super::local_lid(&key) {
|
||||||
|
Some(lid) => conn
|
||||||
|
.query_row("SELECT uid FROM nodes WHERE lid = ?1", [lid], |row| {
|
||||||
|
row.get::<_, Option<String>>(0)
|
||||||
|
})
|
||||||
|
.optional()?
|
||||||
|
.flatten()
|
||||||
|
.unwrap_or(key),
|
||||||
|
None => key,
|
||||||
|
};
|
||||||
let exists: bool = conn.query_row(
|
let exists: bool = conn.query_row(
|
||||||
"SELECT EXISTS(SELECT 1 FROM nodes WHERE uid = ?1 AND node_json IS NOT NULL)",
|
"SELECT EXISTS(SELECT 1 FROM nodes WHERE uid = ?1 AND node_json IS NOT NULL)",
|
||||||
[uid.to_string()],
|
[&key],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
if !exists {
|
if !exists {
|
||||||
@@ -107,25 +123,26 @@ impl Db {
|
|||||||
}
|
}
|
||||||
let value = conn
|
let value = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"WITH RECURSIVE ancestors(uid, parent_uid, depth, path) AS (
|
"WITH RECURSIVE ancestors(uid, parent_uid, parent_lid, depth, path) AS (
|
||||||
SELECT uid, parent_uid, 0, char(31) || uid || char(31)
|
SELECT uid, parent_uid, parent_lid, 0, char(31) || uid || char(31)
|
||||||
FROM nodes WHERE uid = ?1
|
FROM nodes WHERE uid = ?1
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT a.parent_uid, n.parent_uid, a.depth + 1,
|
SELECT COALESCE(n.uid, a.parent_uid), n.parent_uid, n.parent_lid,
|
||||||
a.path || a.parent_uid || char(31)
|
a.depth + 1,
|
||||||
|
a.path || COALESCE(n.uid, a.parent_uid) || char(31)
|
||||||
FROM ancestors a
|
FROM ancestors a
|
||||||
LEFT JOIN nodes n ON n.uid = a.parent_uid
|
LEFT JOIN nodes n ON n.lid = a.parent_lid
|
||||||
WHERE a.parent_uid IS NOT NULL
|
WHERE a.parent_uid IS NOT NULL
|
||||||
AND instr(
|
AND instr(
|
||||||
a.path,
|
a.path,
|
||||||
char(31) || a.parent_uid || char(31)
|
char(31) || COALESCE(n.uid, a.parent_uid) || char(31)
|
||||||
) = 0
|
) = 0
|
||||||
)
|
)
|
||||||
SELECT sa.access
|
SELECT sa.access
|
||||||
FROM ancestors a JOIN share_access sa ON sa.root_uid = a.uid
|
FROM ancestors a JOIN share_access sa ON sa.root_uid = a.uid
|
||||||
ORDER BY a.depth ASC
|
ORDER BY a.depth ASC
|
||||||
LIMIT 1",
|
LIMIT 1",
|
||||||
[uid.to_string()],
|
[&key],
|
||||||
|row| row.get::<_, String>(0),
|
|row| row.get::<_, String>(0),
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
|
|||||||
+1419
-14
File diff suppressed because it is too large.
Load diff
@@ -72,18 +72,18 @@ pub(super) fn path_of(conn: &Connection, uid: &str) -> Result<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve a path by walking `parent_uid` to the root via a recursive CTE.
|
/// Resolve a path by walking `parent_lid` to the root via a recursive CTE.
|
||||||
///
|
///
|
||||||
/// A cycle in `parent_uid` is corrupt data the API can still hand us, and
|
/// A cycle in the parents is corrupt data the API can still hand us, and
|
||||||
/// `UNION ALL` over one never terminates — the walk is therefore depth-capped
|
/// `UNION ALL` over one never terminates — the walk is therefore depth-capped
|
||||||
/// and returns the truncated path rather than hanging the caller.
|
/// and returns the truncated path rather than hanging the caller.
|
||||||
pub(super) fn walk_path_of(conn: &Connection, uid: &str) -> Result<String> {
|
pub(super) fn walk_path_of(conn: &Connection, uid: &str) -> Result<String> {
|
||||||
let mut stmt = conn.prepare(&format!(
|
let mut stmt = conn.prepare(&format!(
|
||||||
"WITH RECURSIVE anc(uid, parent_uid, name, depth) AS (
|
"WITH RECURSIVE anc(parent_lid, parent_uid, name, depth) AS (
|
||||||
SELECT uid, parent_uid, name, 0 FROM nodes WHERE uid = ?1
|
SELECT parent_lid, parent_uid, name, 0 FROM nodes WHERE uid = ?1
|
||||||
UNION ALL
|
UNION ALL
|
||||||
SELECT n.uid, n.parent_uid, n.name, anc.depth + 1
|
SELECT n.parent_lid, n.parent_uid, n.name, anc.depth + 1
|
||||||
FROM nodes n JOIN anc ON n.uid = anc.parent_uid
|
FROM nodes n JOIN anc ON n.lid = anc.parent_lid
|
||||||
WHERE anc.depth < {MAX_PATH_DEPTH}
|
WHERE anc.depth < {MAX_PATH_DEPTH}
|
||||||
)
|
)
|
||||||
SELECT name, parent_uid FROM anc ORDER BY depth DESC"
|
SELECT name, parent_uid FROM anc ORDER BY depth DESC"
|
||||||
|
|||||||
@@ -377,6 +377,7 @@ fn touched_topics(request: &CtlRequest) -> &'static [Topic] {
|
|||||||
| R::RestoreSyncFolders { .. }
|
| R::RestoreSyncFolders { .. }
|
||||||
| R::RestoreDeviceFolders { .. } => &[Topic::Locations, Topic::Devices],
|
| R::RestoreDeviceFolders { .. } => &[Topic::Locations, Topic::Devices],
|
||||||
R::RetryPendingOp { .. } => &[Topic::Queue],
|
R::RetryPendingOp { .. } => &[Topic::Queue],
|
||||||
|
R::DiscardPendingOp { .. } => &[Topic::Queue, Topic::Files],
|
||||||
R::ResolveConflict { .. } => &[Topic::Files, Topic::Trash, Topic::Conflicts],
|
R::ResolveConflict { .. } => &[Topic::Files, Topic::Trash, Topic::Conflicts],
|
||||||
R::ShareNode { .. }
|
R::ShareNode { .. }
|
||||||
| R::ShareNodeByUid { .. }
|
| R::ShareNodeByUid { .. }
|
||||||
@@ -1404,6 +1405,20 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
|||||||
Ok(items) => CtlResponse::PendingOps { items },
|
Ok(items) => CtlResponse::PendingOps { items },
|
||||||
Err(e) => CtlResponse::error(e),
|
Err(e) => CtlResponse::error(e),
|
||||||
},
|
},
|
||||||
|
Ok(CtlRequest::ExportPendingOp { id, dest }) => {
|
||||||
|
match core.export_pending_op(id, Path::new(&dest)) {
|
||||||
|
Ok(written) => CtlResponse::Ok {
|
||||||
|
message: format!("wrote {} to {dest}", human_bytes(written)),
|
||||||
|
},
|
||||||
|
Err(e) => CtlResponse::error(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(CtlRequest::DiscardPendingOp { id }) => match core.discard_pending_op(id) {
|
||||||
|
Ok(path) => CtlResponse::Ok {
|
||||||
|
message: format!("discarded the queued change to {path}"),
|
||||||
|
},
|
||||||
|
Err(e) => CtlResponse::error(e),
|
||||||
|
},
|
||||||
Ok(CtlRequest::RetryPendingOp { id }) => match core.retry_pending_ops(id) {
|
Ok(CtlRequest::RetryPendingOp { id }) => match core.retry_pending_ops(id) {
|
||||||
Ok(0) if id.is_some() => CtlResponse::Ok {
|
Ok(0) if id.is_some() => CtlResponse::Ok {
|
||||||
message: "nothing to retry: the operation is already due or waiting for a rename"
|
message: "nothing to retry: the operation is already due or waiting for a rename"
|
||||||
|
|||||||
+1182
-205
File diff suppressed because it is too large.
Load diff
@@ -153,6 +153,13 @@ pub(crate) trait DriveApi: Send + Sync {
|
|||||||
|
|
||||||
/// The account's storage, across every Proton product.
|
/// The account's storage, across every Proton product.
|
||||||
async fn quota(&self) -> Result<Quota>;
|
async fn quota(&self) -> Result<Quota>;
|
||||||
|
|
||||||
|
/// Called by the drain once it has let go of the blob of an upload to
|
||||||
|
/// `uid` that landed, or retired the create that made `uid`, before the
|
||||||
|
/// tree has the node, so a simulation can hold it there. Blocks the drain
|
||||||
|
/// worker it is called on.
|
||||||
|
#[cfg(test)]
|
||||||
|
fn landing(&self, _uid: &NodeUid) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The generic signatures of the client, over the object-safe ones above.
|
/// The generic signatures of the client, over the object-safe ones above.
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
//! through one place (the status, transfers, sync progress) and publishes them
|
//! through one place (the status, transfers, sync progress) and publishes them
|
||||||
//! when they differ.
|
//! when they differ.
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
use std::collections::{BTreeSet, HashMap};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -153,13 +153,15 @@ pub(crate) fn transfers_line(items: Vec<TransferItem>, jobs: Vec<JobItem>) -> Op
|
|||||||
|
|
||||||
/// Watch what changes without a single place to publish it from, and publish it
|
/// Watch what changes without a single place to publish it from, and publish it
|
||||||
/// when it differs from the last look: the status (queue counts, cache use,
|
/// when it differs from the last look: the status (queue counts, cache use,
|
||||||
/// online, pause), the transfers snapshot, and the synced folders with their
|
/// online, pause), the transfers snapshot, the synced folders with their
|
||||||
/// live progress. Looks only while someone is subscribed.
|
/// live progress, and which nodes have a sync issue. Looks only while someone
|
||||||
|
/// is subscribed.
|
||||||
pub(crate) fn run_event_sampler(core: Core, username: String, mountpoint: std::path::PathBuf) {
|
pub(crate) fn run_event_sampler(core: Core, username: String, mountpoint: std::path::PathBuf) {
|
||||||
let mut transfers: Option<Arc<str>> = None;
|
let mut transfers: Option<Arc<str>> = None;
|
||||||
let mut status: Option<String> = None;
|
let mut status: Option<String> = None;
|
||||||
let mut queue: Option<String> = None;
|
let mut queue: Option<String> = None;
|
||||||
let mut locations: Option<String> = None;
|
let mut locations: Option<String> = None;
|
||||||
|
let mut issues: Option<HashMap<i64, String>> = None;
|
||||||
let mut tick = 0u32;
|
let mut tick = 0u32;
|
||||||
while core.shutdown.sleep(SAMPLE_INTERVAL) {
|
while core.shutdown.sleep(SAMPLE_INTERVAL) {
|
||||||
if !core.events.has_subscribers() {
|
if !core.events.has_subscribers() {
|
||||||
@@ -169,6 +171,7 @@ pub(crate) fn run_event_sampler(core: Core, username: String, mountpoint: std::p
|
|||||||
status = None;
|
status = None;
|
||||||
queue = None;
|
queue = None;
|
||||||
locations = None;
|
locations = None;
|
||||||
|
issues = None;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let line = transfers_line(core.transfers.snapshot(), core.jobs_snapshot());
|
let line = transfers_line(core.transfers.snapshot(), core.jobs_snapshot());
|
||||||
@@ -196,6 +199,15 @@ pub(crate) fn run_event_sampler(core: Core, username: String, mountpoint: std::p
|
|||||||
}
|
}
|
||||||
queue = Some(queue_key);
|
queue = Some(queue_key);
|
||||||
|
|
||||||
|
// An issue shows on its node in the file browser: raised, cleared,
|
||||||
|
// landed or discarded, the listing is out of date.
|
||||||
|
if let Ok(now) = core.db.node_issues() {
|
||||||
|
if issues.as_ref().is_some_and(|last| *last != now) {
|
||||||
|
core.events.publish(&[Topic::Files]);
|
||||||
|
}
|
||||||
|
issues = Some(now);
|
||||||
|
}
|
||||||
|
|
||||||
if let Ok(now) = core.list_locations() {
|
if let Ok(now) = core.list_locations() {
|
||||||
let key = serde_json::to_string(&now).unwrap_or_default();
|
let key = serde_json::to_string(&now).unwrap_or_default();
|
||||||
if locations.as_ref().is_some_and(|last| *last != key) {
|
if locations.as_ref().is_some_and(|last| *last != key) {
|
||||||
|
|||||||
@@ -1217,7 +1217,7 @@ impl ProtonFs {
|
|||||||
/// open.
|
/// open.
|
||||||
fn serve_open(&self, ino: INodeNo, flags: OpenFlags, reply: ReplyOpen) {
|
fn serve_open(&self, ino: INodeNo, flags: OpenFlags, reply: ReplyOpen) {
|
||||||
let write_requested = flags.acc_mode() != OpenAccMode::O_RDONLY;
|
let write_requested = flags.acc_mode() != OpenAccMode::O_RDONLY;
|
||||||
let (uid, base_mtime, base_size, provisional, base_revision_id) = {
|
let (uid, mut base_mtime, mut base_size, mut provisional, mut base_revision_id) = {
|
||||||
let mut st = self.core.state();
|
let mut st = self.core.state();
|
||||||
match st.entries.get_mut(&ino.0) {
|
match st.entries.get_mut(&ino.0) {
|
||||||
Some(e) if e.node.is_file() => {
|
Some(e) if e.node.is_file() => {
|
||||||
@@ -1283,22 +1283,24 @@ impl ProtonFs {
|
|||||||
// `reply` is consumed by whichever arm answers, so the loop yields an
|
// `reply` is consumed by whichever arm answers, so the loop yields an
|
||||||
// outcome and the answer is sent once, after it.
|
// outcome and the answer is sent once, after it.
|
||||||
let outcome = 'install: {
|
let outcome = 'install: {
|
||||||
// The handle starts at the base's size. A provisional one is the
|
|
||||||
// ciphertext size, so the file would grow by the encryption overhead
|
|
||||||
// and reading the base would fail (bugs.md B100). A queued revision
|
|
||||||
// stamps a real size, so this only goes to the network for a file
|
|
||||||
// the listing has not sized yet.
|
|
||||||
let base_size = match provisional && self.core.pending_blob(&uid).is_none() {
|
|
||||||
false => base_size,
|
|
||||||
true => match self.core.resolve_provisional_size(ino.0, &uid) {
|
|
||||||
Some(size) => size,
|
|
||||||
None => {
|
|
||||||
error!(%uid, "could not settle the size of the base for a write open");
|
|
||||||
break 'install Err(Errno::EIO);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
for attempt in 0..OPEN_BASE_ATTEMPTS {
|
for attempt in 0..OPEN_BASE_ATTEMPTS {
|
||||||
|
// The handle starts at the base's size. A provisional one is the
|
||||||
|
// ciphertext size, so the file would grow by the encryption
|
||||||
|
// overhead and reading the base would fail (bugs.md B100). A
|
||||||
|
// queued revision stamps a real size, so this only goes to the
|
||||||
|
// network for a file the listing has not sized yet.
|
||||||
|
if provisional && self.core.pending_blob(&uid).is_none() {
|
||||||
|
match self.core.resolve_provisional_size(ino.0, &uid) {
|
||||||
|
Some(size) => {
|
||||||
|
base_size = size;
|
||||||
|
provisional = false;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
error!(%uid, "could not settle the size of the base for a write open");
|
||||||
|
break 'install Err(Errno::EIO);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
let pending_base = self.core.pending.lock().get(&uid).cloned();
|
let pending_base = self.core.pending.lock().get(&uid).cloned();
|
||||||
if pending_base.as_ref().is_some_and(|p| !p.meta.complete) {
|
if pending_base.as_ref().is_some_and(|p| !p.meta.complete) {
|
||||||
error!(%uid, "refusing write over incomplete queued revision");
|
error!(%uid, "refusing write over incomplete queued revision");
|
||||||
@@ -1332,17 +1334,49 @@ impl ProtonFs {
|
|||||||
// a lock-tight comparison — against the multi-gigabyte copy
|
// a lock-tight comparison — against the multi-gigabyte copy
|
||||||
// above, which is the window that actually loses data.
|
// above, which is the window that actually loses data.
|
||||||
let current = self.core.pending_blob(&uid);
|
let current = self.core.pending_blob(&uid);
|
||||||
|
let landed = self.core.landed.lock().get(&uid).cloned();
|
||||||
let mut st = self.core.state();
|
let mut st = self.core.state();
|
||||||
// Only a handle this open is about to *create* can carry a stale
|
// Only a handle this open is about to *create* can carry a stale
|
||||||
// base; if one already exists, that handle is authoritative and
|
// base; if one already exists, that handle is authoritative and
|
||||||
// the scratch prepared above is discarded either way.
|
// the scratch prepared above is discarded either way.
|
||||||
let creating = !st.active_writes.contains_key(&ino.0);
|
let creating = !st.active_writes.contains_key(&ino.0);
|
||||||
if creating && current != pending_base.map(|p| p.path) {
|
if creating && current.as_ref() != pending_base.as_ref().map(|p| &p.path) {
|
||||||
drop(st);
|
drop(st);
|
||||||
let _ = std::fs::remove_file(&path);
|
let _ = std::fs::remove_file(&path);
|
||||||
warn!(%uid, attempt, "queued revision changed while opening for write; retrying");
|
warn!(%uid, attempt, "queued revision changed while opening for write; retrying");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// With no queued revision, the base's bytes come from the cache,
|
||||||
|
// keyed by the base's mtime and size. The ones sampled above can
|
||||||
|
// name a revision this daemon has since replaced: an upload that
|
||||||
|
// lands lets go of its blob before the tree has its node, and
|
||||||
|
// the cache holds its bytes under the node only. So the base is
|
||||||
|
// the landing node while there is one, and the tree's otherwise
|
||||||
|
// (docs/BUGS.md B170).
|
||||||
|
if creating && pending_base.is_none() {
|
||||||
|
let now = landed.or_else(|| st.entries.get(&ino.0).map(|e| e.node.clone()));
|
||||||
|
if let Some(node) = now
|
||||||
|
&& (node.modification_time != base_mtime
|
||||||
|
|| node_revision_id(&node) != base_revision_id)
|
||||||
|
{
|
||||||
|
base_mtime = node.modification_time;
|
||||||
|
base_size = node_size(&node);
|
||||||
|
base_revision_id = node_revision_id(&node);
|
||||||
|
provisional = matches!(
|
||||||
|
&node.kind,
|
||||||
|
NodeKind::File {
|
||||||
|
claimed_size: None,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
);
|
||||||
|
debug!(%uid, base_mtime, base_size, "write open took the base the file has now");
|
||||||
|
if provisional {
|
||||||
|
drop(st);
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
let fh = st.next_fh;
|
let fh = st.next_fh;
|
||||||
st.next_fh += 1;
|
st.next_fh += 1;
|
||||||
let aw = st.active_writes.entry(ino.0).or_insert_with(|| {
|
let aw = st.active_writes.entry(ino.0).or_insert_with(|| {
|
||||||
@@ -1514,7 +1548,7 @@ impl ProtonFs {
|
|||||||
//
|
//
|
||||||
// A create that set out online and lost the network on the way lands in
|
// A create that set out online and lost the network on the way lands in
|
||||||
// the queue too, so a Wi-Fi drop costs the caller a wait, never an EIO.
|
// the queue too, so a Wi-Fi drop costs the caller a wait, never an EIO.
|
||||||
let minted = if !transient && self.core.is_online() && !is_local_uid(&parent_uid) {
|
let minted = if !transient && self.core.sends_inline() && !is_local_uid(&parent_uid) {
|
||||||
if let Err(error) = self.core.require_uid_writable(&parent_uid) {
|
if let Err(error) = self.core.require_uid_writable(&parent_uid) {
|
||||||
reply.error(error);
|
reply.error(error);
|
||||||
return;
|
return;
|
||||||
@@ -1620,7 +1654,7 @@ impl ProtonFs {
|
|||||||
// queued — the folder becomes a placeholder that the drain turns into a
|
// queued — the folder becomes a placeholder that the drain turns into a
|
||||||
// real one (offline.md Phase 3b).
|
// real one (offline.md Phase 3b).
|
||||||
// Losing the network on the way queues it the same way.
|
// Losing the network on the way queues it the same way.
|
||||||
let minted = if self.core.is_online() && !is_local_uid(&parent_uid) {
|
let minted = if self.core.sends_inline() && !is_local_uid(&parent_uid) {
|
||||||
if let Err(error) = self.core.require_uid_writable(&parent_uid) {
|
if let Err(error) = self.core.require_uid_writable(&parent_uid) {
|
||||||
reply.error(error);
|
reply.error(error);
|
||||||
return;
|
return;
|
||||||
@@ -1770,10 +1804,13 @@ impl ProtonFs {
|
|||||||
self.core.cache.clear_scratch_durable(&h.path);
|
self.core.cache.clear_scratch_durable(&h.path);
|
||||||
reply.ok();
|
reply.ok();
|
||||||
}
|
}
|
||||||
(Some(h), false) => match self.core.queue_revision(&h) {
|
(Some(mut h), false) => {
|
||||||
Ok(()) => reply.ok(),
|
self.core.follow_landed_create(&mut h);
|
||||||
Err(e) => reply.error(e),
|
match self.core.queue_revision(&mut h) {
|
||||||
},
|
Ok(()) => reply.ok(),
|
||||||
|
Err(e) => reply.error(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
(None, _) => reply.ok(),
|
(None, _) => reply.ok(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1796,7 +1833,7 @@ impl ProtonFs {
|
|||||||
reply.error(error);
|
reply.error(error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let (ino, uid) = match self.core.lookup_child(parent, name) {
|
let (ino, mut uid) = match self.core.lookup_child(parent, name) {
|
||||||
Ok(x) => x,
|
Ok(x) => x,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
reply.error(e);
|
reply.error(e);
|
||||||
@@ -1909,7 +1946,7 @@ impl ProtonFs {
|
|||||||
reply.error(error);
|
reply.error(error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if let Err(e) = self.core.remove_replaced(victim_uid, newname) {
|
if let Err(e) = self.core.remove_replaced(*victim_ino, victim_uid, newname) {
|
||||||
reply.error(e);
|
reply.error(e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1921,6 +1958,8 @@ impl ProtonFs {
|
|||||||
// to rename: the queued op *is* the node, so rewriting its target is the
|
// to rename: the queued op *is* the node, so rewriting its target is the
|
||||||
// whole rename. Nothing reaches the API, which is why this works offline
|
// whole rename. Nothing reaches the API, which is why this works offline
|
||||||
// (offline.md Phase 3b).
|
// (offline.md Phase 3b).
|
||||||
|
// A create that lands meanwhile leaves no op to rewrite; the node is
|
||||||
|
// then renamed by the uid Drive gave it (docs/BUGS.md B142).
|
||||||
if is_local_uid(&uid) {
|
if is_local_uid(&uid) {
|
||||||
match self.core.db.rewrite_op_target(
|
match self.core.db.rewrite_op_target(
|
||||||
&uid.to_string(),
|
&uid.to_string(),
|
||||||
@@ -1931,6 +1970,8 @@ impl ProtonFs {
|
|||||||
self.core
|
self.core
|
||||||
.state()
|
.state()
|
||||||
.rename_in_place(ino, newparent, &new_parent_uid, newname);
|
.rename_in_place(ino, newparent, &new_parent_uid, newname);
|
||||||
|
// A create that failed on its old name is due again.
|
||||||
|
self.core.wake_drain();
|
||||||
// Finalize: a transient scratch file (its create parked, bytes
|
// Finalize: a transient scratch file (its create parked, bytes
|
||||||
// held back) renamed to a finished name is the moment the
|
// held back) renamed to a finished name is the moment the
|
||||||
// completed file is meant to reach Drive. Un-park the create so
|
// completed file is meant to reach Drive. Un-park the create so
|
||||||
@@ -1949,23 +1990,29 @@ impl ProtonFs {
|
|||||||
}
|
}
|
||||||
debug!(%uid, newname, "renamed a node whose create is still queued");
|
debug!(%uid, newname, "renamed a node whose create is still queued");
|
||||||
reply.ok();
|
reply.ok();
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
// The create drained underneath us, so the node has a real uid
|
Ok(false) => match self.core.landed_row_uid(ino) {
|
||||||
// now and this handle's is stale. A retry resolves it.
|
Some(real) => {
|
||||||
Ok(false) => {
|
debug!(local = %uid, %real, newname, "create landed under a rename");
|
||||||
warn!(%uid, name, newname, "queued create vanished under a rename");
|
uid = real;
|
||||||
self.core
|
}
|
||||||
.restore_replaced(victim.as_ref(), &new_parent_uid, newname);
|
None => {
|
||||||
reply.error(Errno::EBUSY);
|
warn!(%uid, name, newname, "queued create vanished under a rename");
|
||||||
}
|
self.core
|
||||||
|
.restore_replaced(victim.as_ref(), &new_parent_uid, newname);
|
||||||
|
reply.error(Errno::EBUSY);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
},
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(%uid, error = %e, "rewriting a queued create's target failed");
|
error!(%uid, error = %e, "rewriting a queued create's target failed");
|
||||||
self.core
|
self.core
|
||||||
.restore_replaced(victim.as_ref(), &new_parent_uid, newname);
|
.restore_replaced(victim.as_ref(), &new_parent_uid, newname);
|
||||||
reply.error(Errno::EIO);
|
reply.error(Errno::EIO);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
// Offline, or into a folder that does not exist remotely yet: queue the
|
// Offline, or into a folder that does not exist remotely yet: queue the
|
||||||
// rename rather than 404 or fail. Online moves into a real parent still
|
// rename rather than 404 or fail. Online moves into a real parent still
|
||||||
@@ -2020,7 +2067,7 @@ impl ProtonFs {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
if rename_needs_queue(
|
if rename_needs_queue(
|
||||||
self.core.is_online(),
|
self.core.sends_inline(),
|
||||||
is_local_uid(&new_parent_uid),
|
is_local_uid(&new_parent_uid),
|
||||||
newparent != parent,
|
newparent != parent,
|
||||||
newname != name,
|
newname != name,
|
||||||
@@ -2093,7 +2140,7 @@ impl ProtonFs {
|
|||||||
reply.error(error);
|
reply.error(error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let (ino, uid) = match self.core.lookup_child(parent, name) {
|
let (ino, mut uid) = match self.core.lookup_child(parent, name) {
|
||||||
Ok(x) => x,
|
Ok(x) => x,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
reply.error(e);
|
reply.error(e);
|
||||||
@@ -2121,9 +2168,20 @@ impl ProtonFs {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.core.state().forget_or_unlink(&uid);
|
self.core.state().forget_or_unlink(&uid);
|
||||||
debug!(%uid, name, "deleted a node that had not been created remotely yet");
|
// A create that landed before the ops were dropped left none: Drive
|
||||||
reply.ok();
|
// has the node, and the tree had not taken its uid yet. It is
|
||||||
return;
|
// trashed there like any other (docs/BUGS.md B173).
|
||||||
|
match self.core.landed_row_uid(ino) {
|
||||||
|
Some(real) => {
|
||||||
|
debug!(local = %uid, %real, name, "create landed under a delete");
|
||||||
|
uid = real;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
debug!(%uid, name, "deleted a node that had not been created remotely yet");
|
||||||
|
reply.ok();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Offline: queue it. Trashing is the one mutation a user expects to work
|
// Offline: queue it. Trashing is the one mutation a user expects to work
|
||||||
// regardless — the file is gone from their point of view the moment the
|
// regardless — the file is gone from their point of view the moment the
|
||||||
@@ -2133,7 +2191,7 @@ impl ProtonFs {
|
|||||||
Ok(()) => reply.ok(),
|
Ok(()) => reply.ok(),
|
||||||
Err(e) => reply.error(e),
|
Err(e) => reply.error(e),
|
||||||
};
|
};
|
||||||
if !self.core.is_online() {
|
if !self.core.sends_inline() {
|
||||||
queue_it(reply);
|
queue_it(reply);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
+733
-131
File diff suppressed because it is too large.
Load diff
@@ -86,13 +86,32 @@ pub(crate) fn no_answer(after: Duration) -> ProtonError {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The slowest upload rate an upload's deadline allows for, past the
|
||||||
|
/// metadata budget it starts with. Far below any link that works, so only a
|
||||||
|
/// transfer that has stopped moving runs into it.
|
||||||
|
const SLOWEST_UPLOAD: u64 = 32 * 1024;
|
||||||
|
|
||||||
|
/// How long an upload of `len` bytes may take before the link is treated as
|
||||||
|
/// down: the budget of a metadata call, plus the bytes at [`SLOWEST_UPLOAD`].
|
||||||
|
pub(crate) fn upload_deadline(len: u64) -> Duration {
|
||||||
|
INTERACTIVE_CALL_TIMEOUT + Duration::from_secs(len / SLOWEST_UPLOAD)
|
||||||
|
}
|
||||||
|
|
||||||
/// Run a remote call, giving up after [`INTERACTIVE_CALL_TIMEOUT`].
|
/// Run a remote call, giving up after [`INTERACTIVE_CALL_TIMEOUT`].
|
||||||
pub(crate) async fn bounded<T>(
|
pub(crate) async fn bounded<T>(
|
||||||
call: impl Future<Output = Result<T, ProtonError>>,
|
call: impl Future<Output = Result<T, ProtonError>>,
|
||||||
) -> Result<T, ProtonError> {
|
) -> Result<T, ProtonError> {
|
||||||
match tokio::time::timeout(INTERACTIVE_CALL_TIMEOUT, call).await {
|
bounded_by(INTERACTIVE_CALL_TIMEOUT, call).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run a remote call, giving up after `limit`.
|
||||||
|
pub(crate) async fn bounded_by<T>(
|
||||||
|
limit: Duration,
|
||||||
|
call: impl Future<Output = Result<T, ProtonError>>,
|
||||||
|
) -> Result<T, ProtonError> {
|
||||||
|
match tokio::time::timeout(limit, call).await {
|
||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(_) => Err(no_answer(INTERACTIVE_CALL_TIMEOUT)),
|
Err(_) => Err(no_answer(limit)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,6 +132,15 @@ impl Core {
|
|||||||
self.online.load(Ordering::Relaxed)
|
self.online.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a namespace change through the mount goes to Drive inside the
|
||||||
|
/// syscall. Only with [`AppConfig::local_first`] off, and only online;
|
||||||
|
/// otherwise it is queued and the drain sends it.
|
||||||
|
///
|
||||||
|
/// [`AppConfig::local_first`]: pdfs_core::config::AppConfig::local_first
|
||||||
|
pub(crate) fn sends_inline(&self) -> bool {
|
||||||
|
!self.local_first && self.is_online()
|
||||||
|
}
|
||||||
|
|
||||||
/// Run a remote call a caller is waiting on, giving up after
|
/// Run a remote call a caller is waiting on, giving up after
|
||||||
/// [`INTERACTIVE_CALL_TIMEOUT`]. The timer is built inside the runtime: a
|
/// [`INTERACTIVE_CALL_TIMEOUT`]. The timer is built inside the runtime: a
|
||||||
/// FUSE worker has no reactor of its own.
|
/// FUSE worker has no reactor of its own.
|
||||||
@@ -123,6 +151,16 @@ impl Core {
|
|||||||
self.rt.block_on(bounded(call))
|
self.rt.block_on(bounded(call))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Run a remote call, giving up after `limit`: an upload, whose budget
|
||||||
|
/// grows with its size ([`upload_deadline`]).
|
||||||
|
pub(crate) fn block_on_within<T>(
|
||||||
|
&self,
|
||||||
|
limit: Duration,
|
||||||
|
call: impl Future<Output = Result<T, ProtonError>>,
|
||||||
|
) -> Result<T, ProtonError> {
|
||||||
|
self.rt.block_on(bounded_by(limit, call))
|
||||||
|
}
|
||||||
|
|
||||||
/// Report a failed remote call. Returns whether it failed for want of a
|
/// Report a failed remote call. Returns whether it failed for want of a
|
||||||
/// network, in which case the mount is now offline and the caller should
|
/// network, in which case the mount is now offline and the caller should
|
||||||
/// take its queued path instead of failing.
|
/// take its queued path instead of failing.
|
||||||
@@ -301,6 +339,26 @@ mod tests {
|
|||||||
assert!(!is_network_error(&ProtonError::invalid_operation("bug")));
|
assert!(!is_network_error(&ProtonError::invalid_operation("bug")));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_upload_gets_longer_the_bigger_it_is() {
|
||||||
|
assert_eq!(upload_deadline(0), INTERACTIVE_CALL_TIMEOUT);
|
||||||
|
assert_eq!(
|
||||||
|
upload_deadline(1 << 30),
|
||||||
|
INTERACTIVE_CALL_TIMEOUT + Duration::from_secs(32_768)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_call_past_its_deadline_reads_as_a_lost_link() {
|
||||||
|
let rt = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_time()
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
let limit = Duration::from_millis(20);
|
||||||
|
let result: Result<(), _> = rt.block_on(bounded_by(limit, std::future::pending()));
|
||||||
|
assert!(result.is_err_and(|e| is_network_error(&e)));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_young_outage_is_probed_often_and_an_old_one_rarely() {
|
fn a_young_outage_is_probed_often_and_an_old_one_rarely() {
|
||||||
let young = Duration::from_secs(60);
|
let young = Duration::from_secs(60);
|
||||||
|
|||||||
@@ -244,6 +244,9 @@ pub struct MountOptions {
|
|||||||
/// Upload and download caps in bytes per second (`0` = no cap).
|
/// Upload and download caps in bytes per second (`0` = no cap).
|
||||||
pub upload_limit: u64,
|
pub upload_limit: u64,
|
||||||
pub download_limit: u64,
|
pub download_limit: u64,
|
||||||
|
/// Namespace changes through the mount are queued, not sent in the
|
||||||
|
/// syscall. See [`pdfs_core::config::AppConfig::local_first`].
|
||||||
|
pub local_first: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Spawn one FUSE session rooted at an arbitrary remote node.
|
/// Spawn one FUSE session rooted at an arbitrary remote node.
|
||||||
@@ -385,6 +388,7 @@ pub(crate) fn mount_with(
|
|||||||
sweep_mode,
|
sweep_mode,
|
||||||
upload_limit,
|
upload_limit,
|
||||||
download_limit,
|
download_limit,
|
||||||
|
local_first,
|
||||||
} = options;
|
} = options;
|
||||||
// Start the uptime clock here rather than at the first `pdfs diagnostics`,
|
// Start the uptime clock here rather than at the first `pdfs diagnostics`,
|
||||||
// so the age it reports is the daemon's own.
|
// so the age it reports is the daemon's own.
|
||||||
@@ -429,6 +433,10 @@ pub(crate) fn mount_with(
|
|||||||
sync_paused_until: Arc::new(AtomicI64::new(paused_until)),
|
sync_paused_until: Arc::new(AtomicI64::new(paused_until)),
|
||||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||||
hidden: Arc::new(Mutex::new(HashSet::new())),
|
hidden: Arc::new(Mutex::new(HashSet::new())),
|
||||||
|
creating: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
sealed_unread: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
landed: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
unadopted: Arc::new(Mutex::new(HashMap::new())),
|
||||||
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
||||||
shutdown: Arc::new(crate::shutdown::Shutdown::default()),
|
shutdown: Arc::new(crate::shutdown::Shutdown::default()),
|
||||||
upload_times: Arc::new(Mutex::new(HashMap::new())),
|
upload_times: Arc::new(Mutex::new(HashMap::new())),
|
||||||
@@ -442,6 +450,7 @@ pub(crate) fn mount_with(
|
|||||||
trash_progress: Arc::new(tokio::sync::Notify::new()),
|
trash_progress: Arc::new(tokio::sync::Notify::new()),
|
||||||
conflict_notified: Arc::new(Mutex::new(HashSet::new())),
|
conflict_notified: Arc::new(Mutex::new(HashSet::new())),
|
||||||
sweep_mode,
|
sweep_mode,
|
||||||
|
local_first,
|
||||||
self_changes: Arc::new(Mutex::new(HashMap::new())),
|
self_changes: Arc::new(Mutex::new(HashMap::new())),
|
||||||
thumb_gen: Arc::new(Mutex::new(HashSet::new())),
|
thumb_gen: Arc::new(Mutex::new(HashSet::new())),
|
||||||
thumb_gen_budget: Arc::new(tokio::sync::Semaphore::new(
|
thumb_gen_budget: Arc::new(tokio::sync::Semaphore::new(
|
||||||
@@ -670,7 +679,9 @@ pub(crate) fn mount_with(
|
|||||||
// The photos volume has its own event stream; without it the gallery only
|
// The photos volume has its own event stream; without it the gallery only
|
||||||
// learns about a phone-side deletion when the timeline goes stale.
|
// learns about a phone-side deletion when the timeline goes stale.
|
||||||
rt.spawn(run_photos_event_sync(core.clone()));
|
rt.spawn(run_photos_event_sync(core.clone()));
|
||||||
if online {
|
// Not in a simulation: `client` there reaches no API, and the thread, still
|
||||||
|
// waiting on it after the stop, held the database a test opens next.
|
||||||
|
if online && matches!(host, Host::Daemon) {
|
||||||
let repair_core = core.clone();
|
let repair_core = core.clone();
|
||||||
let root_uid = core.primary_root_uid.clone();
|
let root_uid = core.primary_root_uid.clone();
|
||||||
if let Err(error) = std::thread::Builder::new()
|
if let Err(error) = std::thread::Builder::new()
|
||||||
@@ -739,6 +750,7 @@ pub(crate) fn mount_with(
|
|||||||
warn!(id, error = %e, "unmount on-demand folder failed");
|
warn!(id, error = %e, "unmount on-demand folder failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
core.queue_unreleased_writes();
|
||||||
|
|
||||||
// Last, because the workers may legitimately be finishing an upload, and
|
// Last, because the workers may legitimately be finishing an upload, and
|
||||||
// nothing they can still do is harmful once the mounts are down.
|
// nothing they can still do is harmful once the mounts are down.
|
||||||
@@ -775,9 +787,10 @@ fn signal_stop(rt: &tokio::runtime::Handle) -> std::sync::mpsc::Receiver<()> {
|
|||||||
sig_rx
|
sig_rx
|
||||||
}
|
}
|
||||||
|
|
||||||
/// How long teardown waits for the FUSE worker pool before proceeding without
|
/// How long teardown waits for the background workers and then the FUSE worker
|
||||||
/// it. Long enough for an in-flight block to finish, short enough that
|
/// pool before proceeding without them. Long enough for an in-flight block to
|
||||||
/// `systemctl --user stop` still returns inside `TimeoutStopSec`.
|
/// finish, short enough that `systemctl --user stop` still returns inside
|
||||||
|
/// `TimeoutStopSec`.
|
||||||
const WORKER_JOIN_DEADLINE: Duration = Duration::from_secs(10);
|
const WORKER_JOIN_DEADLINE: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
/// Signal every background loop, wake the ones that are blocked, and join them.
|
/// Signal every background loop, wake the ones that are blocked, and join them.
|
||||||
@@ -792,7 +805,8 @@ const WORKER_JOIN_DEADLINE: Duration = Duration::from_secs(10);
|
|||||||
/// inside `accept`, which only returns when a connection arrives — so it gets
|
/// inside `accept`, which only returns when a connection arrives — so it gets
|
||||||
/// one, from us, after the flag is already set. A join that takes a while is
|
/// one, from us, after the flag is already set. A join that takes a while is
|
||||||
/// expected and correct: a worker part-way through an upload finishes it rather
|
/// expected and correct: a worker part-way through an upload finishes it rather
|
||||||
/// than abandoning the user's bytes mid-flight.
|
/// than abandoning the user's bytes mid-flight. Up to [`WORKER_JOIN_DEADLINE`]:
|
||||||
|
/// past it, systemd kills the daemon anyway, and a queued op is sent again.
|
||||||
fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
||||||
// Say so before the joins: a teardown that takes a while must not be read as
|
// Say so before the joins: a teardown that takes a while must not be read as
|
||||||
// a missed watchdog ping and turned into a kill.
|
// a missed watchdog ping and turned into a kill.
|
||||||
@@ -806,25 +820,71 @@ fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::Jo
|
|||||||
// unlink cannot put it back to sleep.
|
// unlink cannot put it back to sleep.
|
||||||
let _ = std::os::unix::net::UnixStream::connect(control_socket);
|
let _ = std::os::unix::net::UnixStream::connect(control_socket);
|
||||||
let _ = std::fs::remove_file(control_socket);
|
let _ = std::fs::remove_file(control_socket);
|
||||||
for worker in workers {
|
let deadline = Instant::now() + WORKER_JOIN_DEADLINE;
|
||||||
if worker.join().is_err() {
|
let running = join_until(workers, deadline);
|
||||||
warn!("a background worker panicked before shutdown");
|
if !running.is_empty() {
|
||||||
}
|
warn!(
|
||||||
|
running = running.join(","),
|
||||||
|
"background workers did not finish in time; shutting down around them"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
// The FUSE worker pool last, and *before* the caller drops the tokio
|
// The FUSE worker pool last, and *before* the caller drops the tokio
|
||||||
// runtime: a worker still inside a job when the runtime goes away panics on
|
// runtime: a worker still inside a job when the runtime goes away panics on
|
||||||
// its next timer with "A Tokio 1.x context was found, but it is being
|
// its next timer with "A Tokio 1.x context was found, but it is being
|
||||||
// shutdown" (seen 2026-09-18 09:58:35). Bounded, so a worker wedged on the
|
// shutdown" (seen 2026-09-18 09:58:35). Bounded, so a worker wedged on the
|
||||||
// network cannot hold the stop the user asked for.
|
// network cannot hold the stop the user asked for.
|
||||||
core.workers.stop_and_join(WORKER_JOIN_DEADLINE);
|
core.workers
|
||||||
|
.stop_and_join(deadline.saturating_duration_since(Instant::now()));
|
||||||
debug!("background workers stopped");
|
debug!("background workers stopped");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Join `workers` until `deadline`, and name the ones still running then.
|
||||||
|
///
|
||||||
|
/// A stop used to wait for each one however long it took. The sync engine's
|
||||||
|
/// mirror pass waits on Drive with no deadline, and a stop that waited for it
|
||||||
|
/// was killed by systemd (`docs/BUGS.md` B165).
|
||||||
|
fn join_until(workers: Vec<std::thread::JoinHandle<()>>, deadline: Instant) -> Vec<String> {
|
||||||
|
let mut running = Vec::new();
|
||||||
|
for worker in workers {
|
||||||
|
while !worker.is_finished() && Instant::now() < deadline {
|
||||||
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
if !worker.is_finished() {
|
||||||
|
running.push(worker.thread().name().unwrap_or("unnamed").to_string());
|
||||||
|
} else if worker.join().is_err() {
|
||||||
|
warn!("a background worker panicked before shutdown");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
running
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use super::{SecondaryInsertRejection, SecondaryMountRegistry, teardown_session};
|
use super::{SecondaryInsertRejection, SecondaryMountRegistry, join_until, teardown_session};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_stop_names_the_workers_still_running_at_its_deadline() {
|
||||||
|
let (release, wait) = std::sync::mpsc::channel::<()>();
|
||||||
|
let done = std::thread::Builder::new()
|
||||||
|
.name("done".into())
|
||||||
|
.spawn(|| {})
|
||||||
|
.unwrap();
|
||||||
|
let stuck = std::thread::Builder::new()
|
||||||
|
.name("stuck".into())
|
||||||
|
.spawn(move || {
|
||||||
|
let _ = wait.recv();
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let running = join_until(
|
||||||
|
vec![done, stuck],
|
||||||
|
Instant::now() + Duration::from_millis(100),
|
||||||
|
);
|
||||||
|
assert_eq!(running, ["stuck"]);
|
||||||
|
drop(release);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn session_is_unroutable_before_teardown_starts() {
|
fn session_is_unroutable_before_teardown_starts() {
|
||||||
|
|||||||
@@ -1,12 +1,16 @@
|
|||||||
//! The user's view of the pending-op queue: what has not reached the remote
|
//! The user's view of the pending-op queue: what has not reached the remote
|
||||||
//! yet, why, and a way to stop waiting out a backoff.
|
//! yet, why, and a way to stop waiting out a backoff.
|
||||||
//!
|
//!
|
||||||
//! Read-only apart from "retry now", which only moves a row's next-attempt time.
|
//! Read-only apart from "retry now", which only moves a row's next-attempt time,
|
||||||
//! Nothing here drops a queued op: its staged blob is the only copy of the
|
//! "export", which copies a staged blob out, and "discard". Only a discard the
|
||||||
//! user's write (see `docs/RECOVERY.md`).
|
//! user asks for drops a queued op: its staged blob is the only copy of the
|
||||||
|
//! user's write (see `docs/RECOVERY.md`), so it is the user's call, made after
|
||||||
|
//! an export or for a change with no content.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use pdfs_core::control::PendingOpInfo;
|
use std::io::Write;
|
||||||
|
|
||||||
|
use pdfs_core::control::{PendingOpInfo, SyncIssue};
|
||||||
use pdfs_core::db::{FAILING_ATTEMPTS, PARK_UNTIL};
|
use pdfs_core::db::{FAILING_ATTEMPTS, PARK_UNTIL};
|
||||||
|
|
||||||
impl Core {
|
impl Core {
|
||||||
@@ -16,10 +20,15 @@ impl Core {
|
|||||||
.db
|
.db
|
||||||
.pending_ops()
|
.pending_ops()
|
||||||
.map_err(|e| CoreError::internal(format!("listing the queue: {e}")))?;
|
.map_err(|e| CoreError::internal(format!("listing the queue: {e}")))?;
|
||||||
|
let issues = self
|
||||||
|
.db
|
||||||
|
.op_issues()
|
||||||
|
.map_err(|e| CoreError::internal(format!("listing sync issues: {e}")))?;
|
||||||
Ok(ops
|
Ok(ops
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|op| {
|
.map(|op| {
|
||||||
let parked = op.next_attempt_at >= PARK_UNTIL;
|
let parked = op.next_attempt_at >= PARK_UNTIL;
|
||||||
|
let exportable = exportable(&op).is_some();
|
||||||
let path = self
|
let path = self
|
||||||
.db
|
.db
|
||||||
.node_path(&op.uid)
|
.node_path(&op.uid)
|
||||||
@@ -37,7 +46,9 @@ impl Core {
|
|||||||
queued_at: op.created_at / 1000,
|
queued_at: op.created_at / 1000,
|
||||||
next_attempt_at: (!parked).then_some(op.next_attempt_at / 1000),
|
next_attempt_at: (!parked).then_some(op.next_attempt_at / 1000),
|
||||||
parked,
|
parked,
|
||||||
failing: op.attempts >= FAILING_ATTEMPTS,
|
failing: op.attempts >= FAILING_ATTEMPTS || issues.contains_key(&op.id),
|
||||||
|
issue: issues.get(&op.id).map(|issue| SyncIssue::parse(issue)),
|
||||||
|
exportable,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
@@ -57,4 +68,235 @@ impl Core {
|
|||||||
}
|
}
|
||||||
Ok(moved)
|
Ok(moved)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Copy the content of queued op `id` to `dest`, which must not exist yet.
|
||||||
|
/// The op stays queued. Returns the bytes written.
|
||||||
|
pub(crate) fn export_pending_op(&self, id: i64, dest: &Path) -> CoreResult<u64> {
|
||||||
|
if !dest.is_absolute() {
|
||||||
|
return Err(CoreError::invalid(format!(
|
||||||
|
"destination must be an absolute path: {}",
|
||||||
|
dest.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let ops = self
|
||||||
|
.db
|
||||||
|
.pending_ops()
|
||||||
|
.map_err(|e| CoreError::internal(format!("listing the queue: {e}")))?;
|
||||||
|
let op = ops
|
||||||
|
.iter()
|
||||||
|
.find(|op| op.id == id)
|
||||||
|
.ok_or_else(|| CoreError::not_found(format!("no queued change {id}")))?;
|
||||||
|
let blob = exportable(op).ok_or_else(|| {
|
||||||
|
CoreError::invalid(format!("queued change {id} carries no file content"))
|
||||||
|
})?;
|
||||||
|
let written = copy_to_new_file(blob, dest)?;
|
||||||
|
info!(id, dest = %dest.display(), written, "exported a queued change's content");
|
||||||
|
Ok(written)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop queued op `id` and undo the local change it stood for. Returns the
|
||||||
|
/// node's path.
|
||||||
|
///
|
||||||
|
/// A node only made here goes, as if it had been deleted before it was
|
||||||
|
/// sent. A node Drive has goes back to what Drive has: every change queued
|
||||||
|
/// on it goes, since each was made on top of the one before. What is queued
|
||||||
|
/// below it stays.
|
||||||
|
pub(crate) fn discard_pending_op(&self, id: i64) -> CoreResult<String> {
|
||||||
|
let ops = self
|
||||||
|
.db
|
||||||
|
.pending_ops()
|
||||||
|
.map_err(|e| CoreError::internal(format!("listing the queue: {e}")))?;
|
||||||
|
let op = ops
|
||||||
|
.into_iter()
|
||||||
|
.find(|op| op.id == id)
|
||||||
|
.ok_or_else(|| CoreError::not_found(format!("no queued change {id}")))?;
|
||||||
|
let uid = parse_node_uid(&op.uid)
|
||||||
|
.ok_or_else(|| CoreError::internal(format!("queued change {id} names {}", op.uid)))?;
|
||||||
|
let path = self
|
||||||
|
.db
|
||||||
|
.node_path(&op.uid)
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
.filter(|path| !path.is_empty())
|
||||||
|
.or(op.name)
|
||||||
|
.unwrap_or_else(|| op.uid.clone());
|
||||||
|
if self.is_open_anywhere(&uid) {
|
||||||
|
return Err(CoreError::conflict(format!(
|
||||||
|
"{path} is open; close it first"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if is_local_uid(&uid) {
|
||||||
|
self.discard_local_node(&uid, &path)?;
|
||||||
|
} else {
|
||||||
|
self.discard_remote_change(&uid, &path)?;
|
||||||
|
}
|
||||||
|
info!(id, %uid, path, "discarded a queued change");
|
||||||
|
Ok(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Forget a node made here, and the ops that would have sent it.
|
||||||
|
fn discard_local_node(&self, uid: &NodeUid, path: &str) -> CoreResult<()> {
|
||||||
|
// Its children's ops go with its own, and their content with them.
|
||||||
|
if self
|
||||||
|
.db
|
||||||
|
.has_children(uid)
|
||||||
|
.map_err(|e| CoreError::internal(format!("listing {path}: {e}")))?
|
||||||
|
{
|
||||||
|
return Err(CoreError::conflict(format!(
|
||||||
|
"{path} is not empty; discard what is in it first"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
self.discard_queued_ops(uid)
|
||||||
|
.map_err(|_| CoreError::internal(format!("dropping the changes to {path}")))?;
|
||||||
|
self.for_each_mount(|st, notify| {
|
||||||
|
background::forget_and_notify(st, notify, uid);
|
||||||
|
});
|
||||||
|
self.cache.evict(uid);
|
||||||
|
self.evict_reader(uid);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Put a node Drive has back as Drive has it.
|
||||||
|
fn discard_remote_change(&self, uid: &NodeUid, path: &str) -> CoreResult<()> {
|
||||||
|
// Asked first: the change is only dropped once what replaces it is known.
|
||||||
|
let node = self
|
||||||
|
.fetch_node_remote(uid)
|
||||||
|
.map_err(|e| CoreError::from_api(&e, "reading the node from Drive"))?
|
||||||
|
.filter(|node| !node.trashed);
|
||||||
|
self.cancel_upload(uid);
|
||||||
|
let blobs = self
|
||||||
|
.db
|
||||||
|
.discard_ops_on_node(&uid.to_string())
|
||||||
|
.map_err(|e| CoreError::internal(format!("dropping the changes to {path}: {e}")))?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
CoreError::conflict(format!("{path} is being sent right now; try again"))
|
||||||
|
})?;
|
||||||
|
self.release_dropped_ops(uid, blobs);
|
||||||
|
self.hidden.lock().remove(uid);
|
||||||
|
match node {
|
||||||
|
Some(node) => self.for_each_mount(|st, notify| {
|
||||||
|
put_back(st, notify, node);
|
||||||
|
}),
|
||||||
|
None => self.for_each_mount(|st, notify| {
|
||||||
|
background::forget_and_notify(st, notify, uid);
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
self.cache.evict(uid);
|
||||||
|
self.evict_reader(uid);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Show `node` as Drive has it, where Drive has it, under the inode it has.
|
||||||
|
fn put_back(st: &mut State, notify: &mut NotifyBatch, node: Node) {
|
||||||
|
let parent = node.parent_uid.as_ref().and_then(|parent| {
|
||||||
|
st.by_uid
|
||||||
|
.get(parent)
|
||||||
|
.copied()
|
||||||
|
.map(|ino| (ino, parent.clone()))
|
||||||
|
});
|
||||||
|
let here = st.by_uid.get(&node.uid).and_then(|ino| {
|
||||||
|
st.entries
|
||||||
|
.get(ino)
|
||||||
|
.map(|entry| (*ino, entry.parent, entry.node.name.clone()))
|
||||||
|
});
|
||||||
|
match (here, parent) {
|
||||||
|
(Some((ino, from, name)), Some((to, to_uid))) => {
|
||||||
|
st.relocate(ino, from, to, &to_uid, &node.name);
|
||||||
|
st.write_through(node.clone());
|
||||||
|
st.intern_mem(to, node);
|
||||||
|
notify.inval_entry(from, name);
|
||||||
|
notify.inval_inode(ino);
|
||||||
|
}
|
||||||
|
(here, parent) => {
|
||||||
|
if let Some((_, from, name)) = here {
|
||||||
|
st.forget_mem(&node.uid);
|
||||||
|
st.invalidate_listing(from);
|
||||||
|
notify.inval_entry(from, name);
|
||||||
|
}
|
||||||
|
if let Some((to, _)) = parent {
|
||||||
|
st.invalidate_listing(to);
|
||||||
|
notify.inval_entry(to, node.name.clone());
|
||||||
|
}
|
||||||
|
st.write_through(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy `blob` to `dest`, which must not exist: never replace something the
|
||||||
|
/// user already has.
|
||||||
|
fn copy_to_new_file(blob: &Path, dest: &Path) -> CoreResult<u64> {
|
||||||
|
let mut source = File::open(blob)
|
||||||
|
.map_err(|e| CoreError::internal(format!("open {}: {e}", blob.display())))?;
|
||||||
|
let mut out = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(dest)
|
||||||
|
.map_err(|e| match e.kind() {
|
||||||
|
std::io::ErrorKind::AlreadyExists => {
|
||||||
|
CoreError::invalid(format!("{} already exists", dest.display()))
|
||||||
|
}
|
||||||
|
_ => CoreError::internal(format!("create {}: {e}", dest.display())),
|
||||||
|
})?;
|
||||||
|
std::io::copy(&mut source, &mut out)
|
||||||
|
.and_then(|written| out.flush().and(out.sync_all()).map(|()| written))
|
||||||
|
.map_err(|e| CoreError::internal(format!("write {}: {e}", dest.display())))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The staged blob of an op that carries file content.
|
||||||
|
fn exportable(op: &PendingOp) -> Option<&Path> {
|
||||||
|
matches!(op.kind.as_str(), OP_CREATE | OP_REVISION)
|
||||||
|
.then_some(op.blob_path.as_deref())
|
||||||
|
.flatten()
|
||||||
|
.map(Path::new)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn op(kind: &str, blob: Option<&str>) -> PendingOp {
|
||||||
|
PendingOp {
|
||||||
|
id: 1,
|
||||||
|
kind: kind.to_string(),
|
||||||
|
uid: "vol~a".to_string(),
|
||||||
|
parent_uid: None,
|
||||||
|
name: None,
|
||||||
|
blob_path: blob.map(str::to_string),
|
||||||
|
meta_json: None,
|
||||||
|
created_at: 0,
|
||||||
|
attempts: 0,
|
||||||
|
last_error: None,
|
||||||
|
next_attempt_at: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_a_change_that_carries_file_content_can_be_exported() {
|
||||||
|
assert!(exportable(&op(OP_CREATE, Some("/staging/a"))).is_some());
|
||||||
|
assert!(exportable(&op(OP_REVISION, Some("/staging/a"))).is_some());
|
||||||
|
assert!(exportable(&op(OP_CREATE, None)).is_none());
|
||||||
|
assert!(exportable(&op(OP_RENAME, None)).is_none());
|
||||||
|
assert!(exportable(&op(OP_TRASH, None)).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_export_copies_the_bytes_and_never_replaces_a_file() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("pdfs-export-{}", std::process::id()));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let blob = dir.join("blob");
|
||||||
|
std::fs::write(&blob, b"queued bytes").unwrap();
|
||||||
|
|
||||||
|
let dest = dir.join("copy.txt");
|
||||||
|
assert_eq!(copy_to_new_file(&blob, &dest).unwrap(), 12);
|
||||||
|
assert_eq!(std::fs::read(&dest).unwrap(), b"queued bytes");
|
||||||
|
|
||||||
|
std::fs::write(&dest, b"the user's own").unwrap();
|
||||||
|
assert!(copy_to_new_file(&blob, &dest).is_err());
|
||||||
|
assert_eq!(std::fs::read(&dest).unwrap(), b"the user's own");
|
||||||
|
// The staged blob is the only copy of the change: an export leaves it.
|
||||||
|
assert_eq!(std::fs::read(&blob).unwrap(), b"queued bytes");
|
||||||
|
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -499,9 +499,14 @@ impl Core {
|
|||||||
}
|
}
|
||||||
// A node created offline and never written has no blob and no remote: it
|
// A node created offline and never written has no blob and no remote: it
|
||||||
// is an empty file, and asking the API about a `local~` uid would only
|
// is an empty file, and asking the API about a `local~` uid would only
|
||||||
// earn a 404 (offline.md Phase 3b).
|
// earn a 404 (offline.md Phase 3b). Unless its create landed since the
|
||||||
|
// read looked: the landing moved the queued write to the uid Drive
|
||||||
|
// gave it, and the row says which (`docs/BUGS.md` B179).
|
||||||
if is_local_uid(uid) {
|
if is_local_uid(uid) {
|
||||||
return Ok(Vec::new());
|
return match self.landed_stand_in_uid(uid) {
|
||||||
|
Some(real) => self.read_range(&real, mtime, fsize, offset, len, cache_blocks),
|
||||||
|
None => Ok(Vec::new()),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
self.read_range_remote(uid, mtime, fsize, offset, len, cache_blocks)
|
self.read_range_remote(uid, mtime, fsize, offset, len, cache_blocks)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -75,6 +75,7 @@ fn shared_entry_of(core: &Core, n: Node) -> DirEntry {
|
|||||||
shared_by_unverified,
|
shared_by_unverified,
|
||||||
trashed_at: 0,
|
trashed_at: 0,
|
||||||
trashed_from: None,
|
trashed_from: None,
|
||||||
|
issue: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -33,7 +33,13 @@
|
|||||||
//!
|
//!
|
||||||
//! A test can also hold the next read of one node on its way to Drive
|
//! A test can also hold the next read of one node on its way to Drive
|
||||||
//! ([`FakeClient::hold_next_read`]), to make a change while the daemon waits
|
//! ([`FakeClient::hold_next_read`]), to make a change while the daemon waits
|
||||||
//! for the answer.
|
//! for the answer, or on its way back ([`FakeClient::hold_answer_to_next_read`]),
|
||||||
|
//! so the daemon gets an answer a change made meanwhile has overtaken. It can
|
||||||
|
//! also have a new file's next reads answered as Drive lists it before its
|
||||||
|
//! create is committed ([`FakeClient::answer_reads_unrevised`]; B163), and
|
||||||
|
//! hold a folder's listing on its way back with one of its files listed that
|
||||||
|
//! way ([`FakeClient::hold_answer_to_next_listing`]), or hold a new file's
|
||||||
|
//! create on its way to Drive ([`FakeClient::hold_next_create`]).
|
||||||
|
|
||||||
use std::collections::{BTreeMap, HashMap};
|
use std::collections::{BTreeMap, HashMap};
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -330,6 +336,18 @@ impl FakeDrive {
|
|||||||
online: AtomicBool::new(true),
|
online: AtomicBool::new(true),
|
||||||
requests: AtomicU64::new(0),
|
requests: AtomicU64::new(0),
|
||||||
hold: Mutex::new(None),
|
hold: Mutex::new(None),
|
||||||
|
hold_answer: Mutex::new(None),
|
||||||
|
hold_trash: Mutex::new(None),
|
||||||
|
hold_landing: Mutex::new(None),
|
||||||
|
drop_after_create: AtomicBool::new(false),
|
||||||
|
lose_create_reply: AtomicBool::new(false),
|
||||||
|
hold_create_reply: Mutex::new(None),
|
||||||
|
hold_create: Mutex::new(None),
|
||||||
|
lose_revision_reply: AtomicBool::new(false),
|
||||||
|
lose_revision_read_back: AtomicBool::new(false),
|
||||||
|
lose_read: Mutex::new(None),
|
||||||
|
unrevised_reads: Mutex::new(None),
|
||||||
|
hold_listing: Mutex::new(None),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -390,6 +408,23 @@ impl FakeDrive {
|
|||||||
tree
|
tree
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every name a node has had in the root, trashed or not, as far back as
|
||||||
|
/// Drive keeps its versions.
|
||||||
|
pub(crate) fn names_ever_in_root(&self) -> Vec<String> {
|
||||||
|
let server = self.server.lock();
|
||||||
|
let mut names: Vec<String> = server
|
||||||
|
.nodes
|
||||||
|
.values()
|
||||||
|
.flatten()
|
||||||
|
.filter_map(|(_, remote)| remote.as_ref())
|
||||||
|
.filter(|remote| remote.parent.as_ref() == Some(&server.root))
|
||||||
|
.map(|remote| remote.name.clone())
|
||||||
|
.collect();
|
||||||
|
names.sort();
|
||||||
|
names.dedup();
|
||||||
|
names
|
||||||
|
}
|
||||||
|
|
||||||
/// The uid at `path` below the root, as Drive has it now.
|
/// The uid at `path` below the root, as Drive has it now.
|
||||||
pub(crate) fn lookup(&self, path: &str) -> Option<NodeUid> {
|
pub(crate) fn lookup(&self, path: &str) -> Option<NodeUid> {
|
||||||
let server = self.server.lock();
|
let server = self.server.lock();
|
||||||
@@ -577,6 +612,32 @@ struct ClientInner {
|
|||||||
requests: AtomicU64,
|
requests: AtomicU64,
|
||||||
/// The read [`FakeClient::hold_next_read`] is waiting for.
|
/// The read [`FakeClient::hold_next_read`] is waiting for.
|
||||||
hold: Mutex<Option<Arc<HoldState>>>,
|
hold: Mutex<Option<Arc<HoldState>>>,
|
||||||
|
/// The read [`FakeClient::hold_answer_to_next_read`] is waiting for.
|
||||||
|
hold_answer: Mutex<Option<Arc<HoldState>>>,
|
||||||
|
/// The trash [`FakeClient::hold_next_trash`] is waiting for.
|
||||||
|
hold_trash: Mutex<Option<Arc<HoldState>>>,
|
||||||
|
/// The landing [`FakeClient::hold_next_landing`] is waiting for.
|
||||||
|
hold_landing: Mutex<Option<Arc<HoldState>>>,
|
||||||
|
/// Set by [`FakeClient::drop_link_after_next_create`].
|
||||||
|
drop_after_create: AtomicBool,
|
||||||
|
/// Set by [`FakeClient::lose_reply_to_next_create`].
|
||||||
|
lose_create_reply: AtomicBool,
|
||||||
|
/// The create [`FakeClient::hold_reply_to_next_create`] is waiting for.
|
||||||
|
hold_create_reply: Mutex<Option<Arc<HoldState>>>,
|
||||||
|
/// The create [`FakeClient::hold_next_create`] is waiting for.
|
||||||
|
hold_create: Mutex<Option<Arc<HoldState>>>,
|
||||||
|
/// Set by [`FakeClient::lose_reply_to_next_revision`].
|
||||||
|
lose_revision_reply: AtomicBool,
|
||||||
|
/// Set by [`FakeClient::lose_read_back_of_next_revision`].
|
||||||
|
lose_revision_read_back: AtomicBool,
|
||||||
|
/// The file whose next read goes unanswered, once its revision is made.
|
||||||
|
lose_read: Mutex<Option<NodeUid>>,
|
||||||
|
/// Set by [`FakeClient::answer_reads_unrevised`]: the file and how many
|
||||||
|
/// more of its reads are answered without an active revision.
|
||||||
|
unrevised_reads: Mutex<Option<(NodeUid, usize)>>,
|
||||||
|
/// The listing [`FakeClient::hold_answer_to_next_listing`] is waiting
|
||||||
|
/// for, and the file it lists without an active revision.
|
||||||
|
hold_listing: Mutex<Option<(Arc<HoldState>, NodeUid)>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct HoldState {
|
struct HoldState {
|
||||||
@@ -585,8 +646,37 @@ struct HoldState {
|
|||||||
released: AtomicBool,
|
released: AtomicBool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A read held on its way to Drive, from [`FakeClient::hold_next_read`]. It
|
/// Wait while `state` holds a call.
|
||||||
/// goes on when this is dropped.
|
/// `node` as Drive lists a file whose create it has not committed yet: no
|
||||||
|
/// active revision, no size.
|
||||||
|
fn unrevise(node: &mut Node) {
|
||||||
|
if let NodeKind::File {
|
||||||
|
total_size_on_storage,
|
||||||
|
active_revision_state,
|
||||||
|
active_revision_id,
|
||||||
|
claimed_size,
|
||||||
|
content_sha1,
|
||||||
|
..
|
||||||
|
} = &mut node.kind
|
||||||
|
{
|
||||||
|
*total_size_on_storage = 0;
|
||||||
|
*active_revision_state = None;
|
||||||
|
*active_revision_id = None;
|
||||||
|
*claimed_size = None;
|
||||||
|
*content_sha1 = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_until_released(state: &HoldState) {
|
||||||
|
state.reached.store(true, Ordering::SeqCst);
|
||||||
|
while !state.released.load(Ordering::SeqCst) {
|
||||||
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A call held on its way to or from Drive, from [`FakeClient::hold_next_read`]
|
||||||
|
/// or [`FakeClient::hold_reply_to_next_create`]. It goes on when this is
|
||||||
|
/// dropped.
|
||||||
pub(crate) struct Held(Arc<HoldState>);
|
pub(crate) struct Held(Arc<HoldState>);
|
||||||
|
|
||||||
impl Held {
|
impl Held {
|
||||||
@@ -616,6 +706,18 @@ impl FakeClient {
|
|||||||
*self.0.faults.lock() = faults;
|
*self.0.faults.lock() = faults;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Take the link down as soon as the next file create has been answered,
|
||||||
|
/// so whatever the daemon asks next goes unanswered.
|
||||||
|
pub(crate) fn drop_link_after_next_create(&self) {
|
||||||
|
self.0.drop_after_create.store(true, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make the next file create on Drive and lose its answer, as a link that
|
||||||
|
/// drops it on the way back does.
|
||||||
|
pub(crate) fn lose_reply_to_next_create(&self) {
|
||||||
|
self.0.lose_create_reply.store(true, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
/// Hold the next read of `uid` alone, before it reaches Drive, until the
|
/// Hold the next read of `uid` alone, before it reaches Drive, until the
|
||||||
/// answer is dropped. Only one read is held at a time.
|
/// answer is dropped. Only one read is held at a time.
|
||||||
pub(crate) fn hold_next_read(&self, uid: &NodeUid) -> Held {
|
pub(crate) fn hold_next_read(&self, uid: &NodeUid) -> Held {
|
||||||
@@ -628,20 +730,115 @@ impl FakeClient {
|
|||||||
Held(state)
|
Held(state)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Wait here if `uids` is the read a [`Held`] is for.
|
/// Answer the next read of `uid` alone with what Drive holds now, then
|
||||||
async fn wait_if_held(&self, uids: &[NodeUid]) {
|
/// hold the answer until it is dropped, as a slow link does. Only one
|
||||||
|
/// answer is held at a time.
|
||||||
|
pub(crate) fn hold_answer_to_next_read(&self, uid: &NodeUid) -> Held {
|
||||||
|
let state = Arc::new(HoldState {
|
||||||
|
uid: uid.clone(),
|
||||||
|
reached: AtomicBool::new(false),
|
||||||
|
released: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
*self.0.hold_answer.lock() = Some(state.clone());
|
||||||
|
Held(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hold the next trash of `uid` alone, before it reaches Drive, until the
|
||||||
|
/// answer is dropped.
|
||||||
|
pub(crate) fn hold_next_trash(&self, uid: &NodeUid) -> Held {
|
||||||
|
let state = Arc::new(HoldState {
|
||||||
|
uid: uid.clone(),
|
||||||
|
reached: AtomicBool::new(false),
|
||||||
|
released: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
*self.0.hold_trash.lock() = Some(state.clone());
|
||||||
|
Held(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hold the drain once the next upload to `uid` has landed and its blob
|
||||||
|
/// is let go of, before the tree has the node (`DriveApi::landing`), until
|
||||||
|
/// the answer is dropped.
|
||||||
|
pub(crate) fn hold_next_landing(&self, uid: &NodeUid) -> Held {
|
||||||
|
let state = Arc::new(HoldState {
|
||||||
|
uid: uid.clone(),
|
||||||
|
reached: AtomicBool::new(false),
|
||||||
|
released: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
*self.0.hold_landing.lock() = Some(state.clone());
|
||||||
|
Held(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make the next file create on Drive, then hold its answer until the
|
||||||
|
/// answer is dropped, as a slow link does.
|
||||||
|
pub(crate) fn hold_reply_to_next_create(&self) -> Held {
|
||||||
|
let state = Arc::new(HoldState {
|
||||||
|
uid: self.0.drive.root(),
|
||||||
|
reached: AtomicBool::new(false),
|
||||||
|
released: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
*self.0.hold_create_reply.lock() = Some(state.clone());
|
||||||
|
Held(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hold the next file create before it reaches Drive, until the answer
|
||||||
|
/// is dropped.
|
||||||
|
pub(crate) fn hold_next_create(&self) -> Held {
|
||||||
|
let state = Arc::new(HoldState {
|
||||||
|
uid: self.0.drive.root(),
|
||||||
|
reached: AtomicBool::new(false),
|
||||||
|
released: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
*self.0.hold_create.lock() = Some(state.clone());
|
||||||
|
Held(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make the next revision of a file on Drive and lose its answer.
|
||||||
|
pub(crate) fn lose_reply_to_next_revision(&self) {
|
||||||
|
self.0.lose_revision_reply.store(true, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make the next revision of a file on Drive and answer it, then lose the
|
||||||
|
/// answer to the next read of that file alone, as the read-back after an
|
||||||
|
/// upload meets on a link that drops it.
|
||||||
|
pub(crate) fn lose_read_back_of_next_revision(&self) {
|
||||||
|
self.0.lose_revision_read_back.store(true, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Answer the next `reads` reads of file `uid` alone that find it as Drive
|
||||||
|
/// lists a file whose create it has not committed yet: no active revision,
|
||||||
|
/// no size.
|
||||||
|
pub(crate) fn answer_reads_unrevised(&self, uid: &NodeUid, reads: usize) {
|
||||||
|
*self.0.unrevised_reads.lock() = Some((uid.clone(), reads));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Answer the next listing of `folder`'s nodes with what Drive holds now,
|
||||||
|
/// `unrevised` in it as Drive lists a file whose create it has not
|
||||||
|
/// committed yet, then hold the answer until it is dropped.
|
||||||
|
pub(crate) fn hold_answer_to_next_listing(
|
||||||
|
&self,
|
||||||
|
folder: &NodeUid,
|
||||||
|
unrevised: &NodeUid,
|
||||||
|
) -> Held {
|
||||||
|
let state = Arc::new(HoldState {
|
||||||
|
uid: folder.clone(),
|
||||||
|
reached: AtomicBool::new(false),
|
||||||
|
released: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
*self.0.hold_listing.lock() = Some((state.clone(), unrevised.clone()));
|
||||||
|
Held(state)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait here if `uids` is the read a [`Held`] in `slot` is for.
|
||||||
|
async fn wait_if_held(&self, slot: &Mutex<Option<Arc<HoldState>>>, uids: &[NodeUid]) {
|
||||||
let state = {
|
let state = {
|
||||||
let mut hold = self.0.hold.lock();
|
let mut hold = slot.lock();
|
||||||
match hold.as_ref() {
|
match hold.as_ref() {
|
||||||
Some(state) if uids == std::slice::from_ref(&state.uid) => hold.take(),
|
Some(state) if uids == std::slice::from_ref(&state.uid) => hold.take(),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let Some(state) = state else { return };
|
let Some(state) = state else { return };
|
||||||
state.reached.store(true, Ordering::SeqCst);
|
wait_until_released(&state).await;
|
||||||
while !state.released.load(Ordering::SeqCst) {
|
|
||||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Requests made through this client so far.
|
/// Requests made through this client so far.
|
||||||
@@ -868,7 +1065,17 @@ impl DriveApi for FakeClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn enumerate_nodes(&self, uids: &[NodeUid]) -> Result<Vec<Node>> {
|
async fn enumerate_nodes(&self, uids: &[NodeUid]) -> Result<Vec<Node>> {
|
||||||
self.wait_if_held(uids).await;
|
self.wait_if_held(&self.0.hold, uids).await;
|
||||||
|
{
|
||||||
|
let mut lose = self.0.lose_read.lock();
|
||||||
|
if lose
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|uid| uids == std::slice::from_ref(uid))
|
||||||
|
{
|
||||||
|
*lose = None;
|
||||||
|
return Err(no_answer());
|
||||||
|
}
|
||||||
|
}
|
||||||
let lag = self.lag();
|
let lag = self.lag();
|
||||||
let nodes: Vec<Node> = self
|
let nodes: Vec<Node> = self
|
||||||
.request(false, |server| {
|
.request(false, |server| {
|
||||||
@@ -879,20 +1086,49 @@ impl DriveApi for FakeClient {
|
|||||||
.collect())
|
.collect())
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
self.wait_if_held(&self.0.hold_answer, uids).await;
|
||||||
|
let mut nodes = nodes;
|
||||||
|
{
|
||||||
|
let mut unrevised = self.0.unrevised_reads.lock();
|
||||||
|
if let Some((uid, left)) = unrevised.as_mut()
|
||||||
|
&& uids == std::slice::from_ref(uid)
|
||||||
|
&& !nodes.is_empty()
|
||||||
|
{
|
||||||
|
*left -= 1;
|
||||||
|
if *left == 0 {
|
||||||
|
*unrevised = None;
|
||||||
|
}
|
||||||
|
nodes.iter_mut().for_each(unrevise);
|
||||||
|
}
|
||||||
|
}
|
||||||
self.cache(&nodes);
|
self.cache(&nodes);
|
||||||
Ok(nodes)
|
Ok(nodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn enumerate_nodes_light(&self, uids: &[NodeUid]) -> Result<Vec<Node>> {
|
async fn enumerate_nodes_light(&self, uids: &[NodeUid]) -> Result<Vec<Node>> {
|
||||||
let lag = self.lag();
|
let lag = self.lag();
|
||||||
self.request(false, |server| {
|
let mut nodes: Vec<Node> = self
|
||||||
let now = Instant::now();
|
.request(false, |server| {
|
||||||
Ok(uids
|
let now = Instant::now();
|
||||||
|
Ok(uids
|
||||||
|
.iter()
|
||||||
|
.filter_map(|uid| Some(to_node(uid, &server.visible(uid, lag, now)?, true)))
|
||||||
|
.collect())
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
let held = self.0.hold_listing.lock().take_if(|(state, _)| {
|
||||||
|
nodes
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|uid| Some(to_node(uid, &server.visible(uid, lag, now)?, true)))
|
.any(|node| node.parent_uid.as_ref() == Some(&state.uid))
|
||||||
.collect())
|
});
|
||||||
})
|
if let Some((state, unrevised)) = held {
|
||||||
.await
|
nodes
|
||||||
|
.iter_mut()
|
||||||
|
.filter(|node| node.uid == unrevised)
|
||||||
|
.for_each(unrevise);
|
||||||
|
wait_until_released(&state).await;
|
||||||
|
}
|
||||||
|
Ok(nodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn enumerate_folder_children_node_uids(
|
async fn enumerate_folder_children_node_uids(
|
||||||
@@ -1038,16 +1274,34 @@ impl DriveApi for FakeClient {
|
|||||||
_aead: bool,
|
_aead: bool,
|
||||||
) -> Result<NodeUid> {
|
) -> Result<NodeUid> {
|
||||||
let content = read_all(reader)?;
|
let content = read_all(reader)?;
|
||||||
self.request(true, |server| {
|
let held = self.0.hold_create.lock().take();
|
||||||
server.new_file(
|
if let Some(state) = held {
|
||||||
parent_uid,
|
wait_until_released(&state).await;
|
||||||
name,
|
}
|
||||||
media_type,
|
let made = self
|
||||||
content,
|
.request(true, |server| {
|
||||||
last_modification_time,
|
server.new_file(
|
||||||
)
|
parent_uid,
|
||||||
})
|
name,
|
||||||
.await
|
media_type,
|
||||||
|
content,
|
||||||
|
last_modification_time,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
if made.is_ok() && self.0.drop_after_create.swap(false, Ordering::SeqCst) {
|
||||||
|
self.set_online(false);
|
||||||
|
}
|
||||||
|
let held = self.0.hold_create_reply.lock().take();
|
||||||
|
if made.is_ok()
|
||||||
|
&& let Some(state) = held
|
||||||
|
{
|
||||||
|
wait_until_released(&state).await;
|
||||||
|
}
|
||||||
|
if made.is_ok() && self.0.lose_create_reply.swap(false, Ordering::SeqCst) {
|
||||||
|
return Err(no_answer());
|
||||||
|
}
|
||||||
|
made
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn upload_file_replacing_draft_from_dyn(
|
async fn upload_file_replacing_draft_from_dyn(
|
||||||
@@ -1103,7 +1357,14 @@ impl DriveApi for FakeClient {
|
|||||||
server.put(file_uid, Some(remote));
|
server.put(file_uid, Some(remote));
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await
|
.await?;
|
||||||
|
if self.0.lose_revision_reply.swap(false, Ordering::SeqCst) {
|
||||||
|
return Err(no_answer());
|
||||||
|
}
|
||||||
|
if self.0.lose_revision_read_back.swap(false, Ordering::SeqCst) {
|
||||||
|
*self.0.lose_read.lock() = Some(file_uid.clone());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn open_revision(&self, uid: &NodeUid) -> Result<Arc<dyn RevisionRead>> {
|
async fn open_revision(&self, uid: &NodeUid) -> Result<Arc<dyn RevisionRead>> {
|
||||||
@@ -1204,6 +1465,7 @@ impl DriveApi for FakeClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn trash_nodes(&self, uids: &[NodeUid]) -> Result<NodeOutcomes> {
|
async fn trash_nodes(&self, uids: &[NodeUid]) -> Result<NodeOutcomes> {
|
||||||
|
self.wait_if_held(&self.0.hold_trash, uids).await;
|
||||||
self.batch(uids, trash_one).await
|
self.batch(uids, trash_one).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1219,6 +1481,21 @@ impl DriveApi for FakeClient {
|
|||||||
outcome_stream(self.batch(uids, delete_one))
|
outcome_stream(self.batch(uids, delete_one))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn landing(&self, uid: &NodeUid) {
|
||||||
|
let state = {
|
||||||
|
let mut hold = self.0.hold_landing.lock();
|
||||||
|
match hold.as_ref() {
|
||||||
|
Some(state) if state.uid == *uid => hold.take(),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let Some(state) = state else { return };
|
||||||
|
state.reached.store(true, Ordering::SeqCst);
|
||||||
|
while !state.released.load(Ordering::SeqCst) {
|
||||||
|
std::thread::sleep(Duration::from_millis(5));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn quota(&self) -> Result<Quota> {
|
async fn quota(&self) -> Result<Quota> {
|
||||||
self.request(false, |server| {
|
self.request(false, |server| {
|
||||||
Ok(Quota {
|
Ok(Quota {
|
||||||
@@ -1554,6 +1831,29 @@ mod tests {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_held_answer_is_what_drive_held_when_read() {
|
||||||
|
rt().block_on(async {
|
||||||
|
let drive = FakeDrive::new();
|
||||||
|
let client = drive.client(1, Faults::none());
|
||||||
|
let root = drive.root();
|
||||||
|
let file = client.upload_file(&root, "a", "", b"1").await.unwrap();
|
||||||
|
let held = client.hold_answer_to_next_read(&file);
|
||||||
|
let read = tokio::spawn({
|
||||||
|
let client = client.clone();
|
||||||
|
let file = file.clone();
|
||||||
|
async move { client.enumerate_nodes(std::slice::from_ref(&file)).await }
|
||||||
|
});
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
assert!(held.reached());
|
||||||
|
assert!(!read.is_finished());
|
||||||
|
drive.device().trash("a");
|
||||||
|
drop(held);
|
||||||
|
let nodes = read.await.unwrap().unwrap();
|
||||||
|
assert!(!nodes[0].trashed);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_rename_lags_only_by_the_update_lag() {
|
fn a_rename_lags_only_by_the_update_lag() {
|
||||||
rt().block_on(async {
|
rt().block_on(async {
|
||||||
|
|||||||
+48
-402
@@ -22,14 +22,9 @@
|
|||||||
//! one wrote down, but in-flight work is let finish rather than cut. A kill
|
//! one wrote down, but in-flight work is let finish rather than cut. A kill
|
||||||
//! at an arbitrary point needs the daemon in a process of its own.
|
//! at an arbitrary point needs the daemon in a process of its own.
|
||||||
//!
|
//!
|
||||||
//! A call that answers otherwise than the model says fails the run, unless it
|
//! Any of these fails the seed. No open bug is tolerated: the matchers that
|
||||||
//! is one of the [`KNOWN`] open bugs. Those are counted instead and the model
|
//! counted the ones still open instead (B125 to B134) went once a run on
|
||||||
//! takes the call back, or the seed ends there when the bug leaves the mount
|
//! 812eb3d passed every profile without them.
|
||||||
//! where the model cannot follow, so the runs stay a regression net until
|
|
||||||
//! they are fixed. A file gone, or a conflict copy the drain made of one of
|
|
||||||
//! the client's own, is counted too when a known bug explains it
|
|
||||||
//! (`Run::known_damage`), and so is a hang the stacks show is one of the
|
|
||||||
//! [`KNOWN_HANGS`]. `PDFS_SIM_KNOWN=fail` fails on all of them.
|
|
||||||
//!
|
//!
|
||||||
//! The runs are ignored by default because they mount FUSE; run them with
|
//! The runs are ignored by default because they mount FUSE; run them with
|
||||||
//! `cargo test -p pdfs-fuse --lib sim::run -- --ignored --test-threads=1`.
|
//! `cargo test -p pdfs-fuse --lib sim::run -- --ignored --test-threads=1`.
|
||||||
@@ -42,105 +37,11 @@ use std::io::{self, Read, Seek, SeekFrom, Write};
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
|
||||||
|
|
||||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
|
||||||
|
|
||||||
use super::daemon::{Daemon, scratch, take_logged, wait_until};
|
use super::daemon::{Daemon, scratch, take_logged, wait_until};
|
||||||
use super::fake_drive::{Entry, FakeDrive, Faults};
|
use super::fake_drive::{Entry, FakeDrive, Faults};
|
||||||
use super::model::{FsOp, Model, Tree, data, join, name_of, parent_of};
|
use super::model::{FsOp, Model, Tree, data, join};
|
||||||
use super::rng::Rng;
|
use super::rng::Rng;
|
||||||
use super::watchdog::{Hung, Watchdog};
|
use super::watchdog::Watchdog;
|
||||||
|
|
||||||
/// An open bug in `docs/BUGS.md` the runs tolerate.
|
|
||||||
struct Known {
|
|
||||||
bug: &'static str,
|
|
||||||
/// Whether a call that answered `errno` against the model's word, with
|
|
||||||
/// lines `logged` at `INFO` and above meanwhile and `before` it, is this
|
|
||||||
/// bug. `model` is from before the call.
|
|
||||||
is: fn(model: &Model, op: &FsOp, errno: i32, logged: &[String], before: &[String]) -> bool,
|
|
||||||
/// Whether it also left the mount where the model cannot follow, so the
|
|
||||||
/// seed ends there.
|
|
||||||
ends_seed: fn(model: &Model, op: &FsOp) -> bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
const KNOWN: &[Known] = &[
|
|
||||||
// A replacing rename trashes what it replaces first, so Drive refuses the
|
|
||||||
// name only when it holds a node the mount has forgotten: the create of
|
|
||||||
// the replaced file landed after its op was dropped.
|
|
||||||
Known {
|
|
||||||
bug: "B129",
|
|
||||||
is: |model, op, errno, logged, _| {
|
|
||||||
let FsOp::Rename { to, .. } = op else {
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
errno == libc::EIO
|
|
||||||
&& model.tree().contains_key(to)
|
|
||||||
&& logged
|
|
||||||
.iter()
|
|
||||||
.any(|line| line.starts_with("rename failed") && line.contains("AlreadyExists"))
|
|
||||||
},
|
|
||||||
// The model made the rename, and Drive holds a node the mount does
|
|
||||||
// not know.
|
|
||||||
ends_seed: |_, _| true,
|
|
||||||
},
|
|
||||||
// A folder made offline is listed on Drive once its listing has been
|
|
||||||
// invalidated, and Drive does not know it, or does not list it yet. Only
|
|
||||||
// the client writes its folder, so a folder of its own is never really
|
|
||||||
// gone.
|
|
||||||
Known {
|
|
||||||
bug: "B125",
|
|
||||||
is: |_, _, errno, logged, _| {
|
|
||||||
matches!(errno, libc::EIO | libc::ENOENT)
|
|
||||||
&& logged.iter().any(|line| {
|
|
||||||
line.starts_with("enumerate folder children failed")
|
|
||||||
&& line.contains("DoesNotExist")
|
|
||||||
})
|
|
||||||
},
|
|
||||||
// The folder keeps failing to list, and what is in it can come back
|
|
||||||
// as gone, so the mount no longer follows the model.
|
|
||||||
ends_seed: |_, _| true,
|
|
||||||
},
|
|
||||||
// A restart forgets the listings served offline, so the first lookup
|
|
||||||
// lists a folder on Drive, which does not hold what is still queued for
|
|
||||||
// it.
|
|
||||||
Known {
|
|
||||||
bug: "B132",
|
|
||||||
is: |_, _, errno, _, before| {
|
|
||||||
errno == libc::ENOENT
|
|
||||||
&& before
|
|
||||||
.iter()
|
|
||||||
.any(|line| line.starts_with("lost the connection to Proton"))
|
|
||||||
&& before
|
|
||||||
.iter()
|
|
||||||
.any(|line| line.starts_with("restored pending ops"))
|
|
||||||
},
|
|
||||||
// The listing stays without it until something relists the folder.
|
|
||||||
ends_seed: |_, _| true,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
/// An open bug in `docs/BUGS.md` that hangs a call, which the runs tolerate.
|
|
||||||
/// The seed ends there, since the watchdog aborted the mounts.
|
|
||||||
struct KnownHang {
|
|
||||||
bug: &'static str,
|
|
||||||
/// Whether the stacks the watchdog kept show this bug.
|
|
||||||
is: fn(stacks: &str) -> bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
const KNOWN_HANGS: &[KnownHang] = &[
|
|
||||||
// A runtime worker invalidating a file's pages waits for the page lock
|
|
||||||
// of a read, whose fetch waits for a timer that no worker drives.
|
|
||||||
KnownHang {
|
|
||||||
bug: "B130",
|
|
||||||
is: |stacks| {
|
|
||||||
stacks.split("\nthread ").any(|thread| {
|
|
||||||
thread.contains("\"tokio-rt-worker\", state D")
|
|
||||||
&& thread.contains("NotifyBatch>::flush")
|
|
||||||
})
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
/// What a run looks like.
|
/// What a run looks like.
|
||||||
pub(crate) struct Profile {
|
pub(crate) struct Profile {
|
||||||
@@ -178,16 +79,6 @@ const LOG_TAIL: usize = 40;
|
|||||||
/// for the last run and the mount.
|
/// for the last run and the mount.
|
||||||
const STALL: Duration = Duration::from_secs(120);
|
const STALL: Duration = Duration::from_secs(120);
|
||||||
|
|
||||||
/// What a known bug left on Drive in place of a file, and how the model
|
|
||||||
/// takes it in.
|
|
||||||
struct KnownDamage {
|
|
||||||
bug: &'static str,
|
|
||||||
path: String,
|
|
||||||
/// The conflict copy it made, if it made one.
|
|
||||||
copy: Option<String>,
|
|
||||||
change: FsOp,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct Client {
|
struct Client {
|
||||||
daemon: Option<Daemon>,
|
daemon: Option<Daemon>,
|
||||||
model: Model,
|
model: Model,
|
||||||
@@ -213,25 +104,14 @@ struct Run {
|
|||||||
clients: Vec<Client>,
|
clients: Vec<Client>,
|
||||||
log: Vec<String>,
|
log: Vec<String>,
|
||||||
started: Instant,
|
started: Instant,
|
||||||
/// How often each [`KNOWN`] bug turned up.
|
|
||||||
known: BTreeMap<&'static str, usize>,
|
|
||||||
/// The known bug that ended the seed early, if one did.
|
|
||||||
ended_by: Option<&'static str>,
|
|
||||||
/// What the daemons logged at `INFO` and above so far.
|
/// What the daemons logged at `INFO` and above so far.
|
||||||
logged: Vec<String>,
|
logged: Vec<String>,
|
||||||
/// Files deleted while the drain held a write to them.
|
|
||||||
held: Vec<NodeUid>,
|
|
||||||
/// Those of them made again before it went on.
|
|
||||||
remade: Vec<NodeUid>,
|
|
||||||
watchdog: Watchdog,
|
watchdog: Watchdog,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run `profile` on `seed`. It answers how often each known bug turned up,
|
/// Run `profile` on `seed`. It answers a report of what went wrong, if
|
||||||
/// or a report of what went wrong.
|
/// anything did.
|
||||||
pub(crate) fn run(
|
pub(crate) fn run(seed: u64, profile: &'static Profile) -> Result<(), String> {
|
||||||
seed: u64,
|
|
||||||
profile: &'static Profile,
|
|
||||||
) -> Result<BTreeMap<&'static str, usize>, String> {
|
|
||||||
let dir = scratch(&format!("{}-{seed}", profile.name));
|
let dir = scratch(&format!("{}-{seed}", profile.name));
|
||||||
let mut rng = Rng::new(seed);
|
let mut rng = Rng::new(seed);
|
||||||
let drive = FakeDrive::new();
|
let drive = FakeDrive::new();
|
||||||
@@ -242,18 +122,14 @@ pub(crate) fn run(
|
|||||||
clients: Vec::new(),
|
clients: Vec::new(),
|
||||||
log: Vec::new(),
|
log: Vec::new(),
|
||||||
started: Instant::now(),
|
started: Instant::now(),
|
||||||
known: BTreeMap::new(),
|
|
||||||
ended_by: None,
|
|
||||||
logged: Vec::new(),
|
logged: Vec::new(),
|
||||||
held: Vec::new(),
|
|
||||||
remade: Vec::new(),
|
|
||||||
watchdog: Watchdog::new(&dir),
|
watchdog: Watchdog::new(&dir),
|
||||||
};
|
};
|
||||||
let outcome = run.execute(&dir, &mut rng);
|
let outcome = run.execute(&dir, &mut rng);
|
||||||
run.watchdog.stop();
|
run.watchdog.stop();
|
||||||
// A hang is what failed, whatever the aborted call answered.
|
// A hang is what failed, whatever the aborted call answered.
|
||||||
let outcome = match run.watchdog.fired() {
|
let outcome = match run.watchdog.fired() {
|
||||||
Some(hung) => run.known_hang(&hung),
|
Some(hung) => Err(format!("liveness: {}", hung.what)),
|
||||||
None => outcome,
|
None => outcome,
|
||||||
};
|
};
|
||||||
for client in &mut run.clients {
|
for client in &mut run.clients {
|
||||||
@@ -264,27 +140,13 @@ pub(crate) fn run(
|
|||||||
match outcome {
|
match outcome {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
Ok(run.known)
|
Ok(())
|
||||||
}
|
}
|
||||||
Err(failure) => Err(run.report(&failure, &dir)),
|
Err(failure) => Err(run.report(&failure, &dir)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Run {
|
impl Run {
|
||||||
/// Count `hung` if it is one of the [`KNOWN_HANGS`], or fail on it.
|
|
||||||
fn known_hang(&mut self, hung: &Hung) -> Result<(), String> {
|
|
||||||
let known = KNOWN_HANGS.iter().find(|known| (known.is)(&hung.stacks));
|
|
||||||
match known {
|
|
||||||
Some(known) if tolerate_known() => {
|
|
||||||
*self.known.entry(known.bug).or_default() += 1;
|
|
||||||
self.note(format!("known bug {}; the seed ends here", known.bug));
|
|
||||||
self.ended_by = Some(known.bug);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
_ => Err(format!("liveness: {}", hung.what)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn execute(&mut self, dir: &Path, rng: &mut Rng) -> Result<(), String> {
|
fn execute(&mut self, dir: &Path, rng: &mut Rng) -> Result<(), String> {
|
||||||
for index in 0..self.profile.clients {
|
for index in 0..self.profile.clients {
|
||||||
let client = self.drive.client(rng.next_u64(), (self.profile.faults)());
|
let client = self.drive.client(rng.next_u64(), (self.profile.faults)());
|
||||||
@@ -302,9 +164,6 @@ impl Run {
|
|||||||
for step in 0..self.profile.steps {
|
for step in 0..self.profile.steps {
|
||||||
let _watching = self.watchdog.arm(self.step_limit(), format!("step {step}"));
|
let _watching = self.watchdog.arm(self.step_limit(), format!("step {step}"));
|
||||||
self.step(rng)?;
|
self.step(rng)?;
|
||||||
if self.ended_by.is_some() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
let _watching = self.watchdog.arm(
|
let _watching = self.watchdog.arm(
|
||||||
self.profile.drain_budget + STALL,
|
self.profile.drain_budget + STALL,
|
||||||
@@ -363,7 +222,11 @@ impl Run {
|
|||||||
fn settle(&mut self, budget: Duration) -> Result<(), String> {
|
fn settle(&mut self, budget: Duration) -> Result<(), String> {
|
||||||
self.note("settle".into());
|
self.note("settle".into());
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
if !wait_until(budget, || self.landed()) {
|
// What the drain logs meanwhile is what explains a known bug's damage.
|
||||||
|
if !wait_until(budget, || {
|
||||||
|
self.logged.extend(take_logged());
|
||||||
|
self.landed()
|
||||||
|
}) {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"liveness: the queues did not drain within {budget:?}\n{}",
|
"liveness: the queues did not drain within {budget:?}\n{}",
|
||||||
self.difference()
|
self.difference()
|
||||||
@@ -409,18 +272,11 @@ impl Run {
|
|||||||
data: bytes,
|
data: bytes,
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
if self.ended_by.is_some() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if !wait_until(PICKUP_BUDGET, || held.reached()) {
|
if !wait_until(PICKUP_BUDGET, || held.reached()) {
|
||||||
self.note(format!("client {index}: nothing read {path} back"));
|
self.note(format!("client {index}: nothing read {path} back"));
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
self.syscall(index, FsOp::Unlink { path: path.clone() })?;
|
self.syscall(index, FsOp::Unlink { path: path.clone() })?;
|
||||||
if self.ended_by.is_some() {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
self.held.push(uid.clone());
|
|
||||||
if rng.chance(0.5) {
|
if rng.chance(0.5) {
|
||||||
self.syscall(
|
self.syscall(
|
||||||
index,
|
index,
|
||||||
@@ -429,7 +285,6 @@ impl Run {
|
|||||||
data: data(rng),
|
data: data(rng),
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
self.remade.push(uid);
|
|
||||||
}
|
}
|
||||||
let faults = (self.profile.faults)();
|
let faults = (self.profile.faults)();
|
||||||
let latency = Duration::from_millis(faults.latency_ms.1);
|
let latency = Duration::from_millis(faults.latency_ms.1);
|
||||||
@@ -444,11 +299,9 @@ impl Run {
|
|||||||
fn syscall(&mut self, index: usize, op: FsOp) -> Result<(), String> {
|
fn syscall(&mut self, index: usize, op: FsOp) -> Result<(), String> {
|
||||||
let client = &mut self.clients[index];
|
let client = &mut self.clients[index];
|
||||||
let root = client.root(index);
|
let root = client.root(index);
|
||||||
let model = client.model.clone();
|
|
||||||
let before = client.model.content(target(&op)).cloned();
|
let before = client.model.content(target(&op)).cloned();
|
||||||
let expected = client.model.apply(&op);
|
let expected = client.model.apply(&op);
|
||||||
self.logged.extend(take_logged());
|
self.logged.extend(take_logged());
|
||||||
let logged_before = self.logged.len();
|
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let actual = perform(&root, &op, before.as_deref());
|
let actual = perform(&root, &op, before.as_deref());
|
||||||
let took = start.elapsed();
|
let took = start.elapsed();
|
||||||
@@ -458,25 +311,7 @@ impl Run {
|
|||||||
outcome(&actual)
|
outcome(&actual)
|
||||||
));
|
));
|
||||||
let actual = actual?;
|
let actual = actual?;
|
||||||
let logged = take_logged();
|
self.logged.extend(take_logged());
|
||||||
self.logged.extend(logged.iter().cloned());
|
|
||||||
if actual != expected
|
|
||||||
&& let Err(errno) = actual
|
|
||||||
&& tolerate_known()
|
|
||||||
&& let Some(known) = KNOWN.iter().find(|known| {
|
|
||||||
(known.is)(&model, &op, errno, &logged, &self.logged[..logged_before])
|
|
||||||
})
|
|
||||||
{
|
|
||||||
*self.known.entry(known.bug).or_default() += 1;
|
|
||||||
if (known.ends_seed)(&model, &op) {
|
|
||||||
self.note(format!("known bug {}; the seed ends here", known.bug));
|
|
||||||
self.ended_by = Some(known.bug);
|
|
||||||
} else {
|
|
||||||
self.note(format!("known bug {}; taken back", known.bug));
|
|
||||||
}
|
|
||||||
self.clients[index].model = model;
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if actual != expected {
|
if actual != expected {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"posix: client {index}: {} answered {}, expected {}",
|
"posix: client {index}: {} answered {}, expected {}",
|
||||||
@@ -513,61 +348,15 @@ impl Run {
|
|||||||
}
|
}
|
||||||
// Each client's folder is its own, so a file of it gone from Drive was
|
// Each client's folder is its own, so a file of it gone from Drive was
|
||||||
// deleted by the client.
|
// deleted by the client.
|
||||||
let own_deletes: Vec<String> = self
|
if let Some(line) = self.logged.iter().find(|line| is_own_delete_kept(line)) {
|
||||||
.logged
|
return Err(format!(
|
||||||
.iter()
|
"no false conflicts: the drain took a client's own delete for another device's: {line}"
|
||||||
.filter(|line| is_own_delete_kept(line))
|
));
|
||||||
.cloned()
|
|
||||||
.collect();
|
|
||||||
for line in own_deletes {
|
|
||||||
let bug = if self
|
|
||||||
.remade
|
|
||||||
.iter()
|
|
||||||
.any(|uid| line.contains(&format!(" uid={uid} ")))
|
|
||||||
{
|
|
||||||
// Made again while the drain held its write: the new file's
|
|
||||||
// write took the freed op id.
|
|
||||||
"B133"
|
|
||||||
} else if !self
|
|
||||||
.held
|
|
||||||
.iter()
|
|
||||||
.any(|uid| line.contains(&format!(" uid={uid} ")))
|
|
||||||
&& !line.contains(" name=\"recovered-")
|
|
||||||
{
|
|
||||||
// The tree still placed the file, so the drain looked before
|
|
||||||
// the unlink had dropped its write.
|
|
||||||
"B134"
|
|
||||||
} else {
|
|
||||||
return Err(format!(
|
|
||||||
"no false conflicts: the drain took a client's own delete for another device's: {line}"
|
|
||||||
));
|
|
||||||
};
|
|
||||||
if !tolerate_known() {
|
|
||||||
return Err(format!(
|
|
||||||
"no false conflicts: the drain took a client's own delete for another device's: {line}"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
*self.known.entry(bug).or_default() += 1;
|
|
||||||
self.note(format!("known bug {bug}: {line}"));
|
|
||||||
}
|
}
|
||||||
let drive = self.drive_tree();
|
let drive = self.drive_tree();
|
||||||
let mut explained = Vec::new();
|
|
||||||
for index in 0..self.clients.len() {
|
|
||||||
for damage in self.known_damage(index, &drive) {
|
|
||||||
*self.known.entry(damage.bug).or_default() += 1;
|
|
||||||
self.note(format!(
|
|
||||||
"known bug {}: client {index}'s {} landed as {:?}",
|
|
||||||
damage.bug, damage.path, damage.copy
|
|
||||||
));
|
|
||||||
let _ = self.clients[index].model.apply(&damage.change);
|
|
||||||
if let Some(copy) = damage.copy {
|
|
||||||
explained.push(join(&owned_folder(index), ©));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let conflicts: Vec<String> = drive
|
let conflicts: Vec<String> = drive
|
||||||
.into_keys()
|
.into_keys()
|
||||||
.filter(|path| is_conflict_copy(path) && !explained.contains(path))
|
.filter(|path| is_conflict_copy(path))
|
||||||
.collect();
|
.collect();
|
||||||
if !conflicts.is_empty() {
|
if !conflicts.is_empty() {
|
||||||
return Err(format!("no false conflicts: {conflicts:?}"));
|
return Err(format!("no false conflicts: {conflicts:?}"));
|
||||||
@@ -589,155 +378,18 @@ impl Run {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Whether every client's queue is empty and Drive holds what its model
|
/// Whether every client's queue is empty and Drive holds what its model
|
||||||
/// says, but for what known bugs explain.
|
/// says.
|
||||||
fn landed(&self) -> bool {
|
fn landed(&self) -> bool {
|
||||||
let drive = self.drive_tree();
|
let drive = self.drive_tree();
|
||||||
self.clients.iter().enumerate().all(|(index, client)| {
|
self.clients.iter().enumerate().all(|(index, client)| {
|
||||||
let mut model = client.model.clone();
|
|
||||||
for damage in self.known_damage(index, &drive) {
|
|
||||||
let _ = model.apply(&damage.change);
|
|
||||||
}
|
|
||||||
client
|
client
|
||||||
.daemon()
|
.daemon()
|
||||||
.pending()
|
.pending()
|
||||||
.is_ok_and(|items| items.is_empty())
|
.is_ok_and(|items| items.is_empty())
|
||||||
&& subtree(&drive, &owned_folder(index)) == *model.tree()
|
&& subtree(&drive, &owned_folder(index)) == *client.model.tree()
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The files of client `index` that a known bug left otherwise on Drive
|
|
||||||
/// than the model says. A conflict copy takes Drive holding it with the
|
|
||||||
/// model's bytes, and the drain saying it made that copy for that name:
|
|
||||||
///
|
|
||||||
/// - B126: the file's queued create ran into its name while a queued
|
|
||||||
/// rename was about to free it, so it is there only as the copy.
|
|
||||||
/// - B127: the file's queued create landed but its answer was lost, so
|
|
||||||
/// the retry found it and made the copy next to it.
|
|
||||||
///
|
|
||||||
/// A lost file takes the drain saying it adopted a node for that name
|
|
||||||
/// whose queued trash then landed:
|
|
||||||
///
|
|
||||||
/// - B128: the file's queued create ran into the name of the file it
|
|
||||||
/// replaced and took that over, trash and all.
|
|
||||||
///
|
|
||||||
/// A conflict copy of a file the client deleted takes the drain saying it
|
|
||||||
/// kept that file's write as the copy, because Drive had the file in the
|
|
||||||
/// trash while the tree still placed it:
|
|
||||||
///
|
|
||||||
/// - B134: the unlink trashed the file on Drive before it dropped the
|
|
||||||
/// write.
|
|
||||||
fn known_damage(&self, index: usize, drive: &Tree) -> Vec<KnownDamage> {
|
|
||||||
if !tolerate_known() {
|
|
||||||
return Vec::new();
|
|
||||||
}
|
|
||||||
let drive = subtree(drive, &owned_folder(index));
|
|
||||||
let model = self.clients[index].model.tree();
|
|
||||||
let made_copy = |path: &str, copy: &str| {
|
|
||||||
let wanted = format!("wanted={:?}", name_of(path));
|
|
||||||
let name = format!("name={:?}", name_of(copy));
|
|
||||||
self.logged.iter().any(|line| {
|
|
||||||
line.starts_with("name is taken remotely; creating under a conflict name")
|
|
||||||
&& line.contains(&wanted)
|
|
||||||
&& line.contains(&name)
|
|
||||||
})
|
|
||||||
};
|
|
||||||
let adopted_trashed = |path: &str| {
|
|
||||||
let wanted = format!("wanted={:?}", name_of(path));
|
|
||||||
self.logged.iter().any(|line| {
|
|
||||||
line.starts_with("the name is held by our own unanswered create; adopting it")
|
|
||||||
&& line.contains(&wanted)
|
|
||||||
&& line.split(' ').any(|field| {
|
|
||||||
field.strip_prefix("twin=").is_some_and(|twin| {
|
|
||||||
let uid = format!("uid={twin}");
|
|
||||||
self.logged.iter().any(|line| {
|
|
||||||
line.starts_with("pending trash landed")
|
|
||||||
&& line.split(' ').any(|field| field == uid)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
};
|
|
||||||
let kept_deleted = |copy: &str| {
|
|
||||||
let alt = format!(" alt={:?}", name_of(copy));
|
|
||||||
self.logged.iter().any(|line| {
|
|
||||||
line.starts_with("queued write landed as a conflict copy")
|
|
||||||
&& line.ends_with(&alt)
|
|
||||||
&& line.split(' ').any(|field| {
|
|
||||||
field.starts_with("uid=")
|
|
||||||
&& self.logged.iter().any(|kept| {
|
|
||||||
is_own_delete_kept(kept)
|
|
||||||
&& !kept.contains(" name=\"recovered-")
|
|
||||||
&& kept.split(' ').any(|other| other == field)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
};
|
|
||||||
let mut damage = Vec::new();
|
|
||||||
for (copy, content) in &drive {
|
|
||||||
let Some(content) = content else { continue };
|
|
||||||
if !is_conflict_copy(copy) || model.contains_key(copy) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if !kept_deleted(copy) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
damage.push(KnownDamage {
|
|
||||||
bug: "B134",
|
|
||||||
path: copy.clone(),
|
|
||||||
copy: Some(copy.clone()),
|
|
||||||
change: FsOp::Create {
|
|
||||||
path: copy.clone(),
|
|
||||||
data: content.clone(),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
for (path, content) in model {
|
|
||||||
let Some(content) = content else { continue };
|
|
||||||
let copy = drive.iter().find(|(copy, other)| {
|
|
||||||
is_conflict_copy(copy)
|
|
||||||
&& !model.contains_key(*copy)
|
|
||||||
&& parent_of(copy) == parent_of(path)
|
|
||||||
&& other.as_ref() == Some(content)
|
|
||||||
&& made_copy(path, copy)
|
|
||||||
});
|
|
||||||
let Some((copy, _)) = copy else {
|
|
||||||
if !drive.contains_key(path) && adopted_trashed(path) {
|
|
||||||
damage.push(KnownDamage {
|
|
||||||
bug: "B128",
|
|
||||||
path: path.clone(),
|
|
||||||
copy: None,
|
|
||||||
change: FsOp::Unlink { path: path.clone() },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let (bug, change) = match drive.get(path) {
|
|
||||||
None => (
|
|
||||||
"B126",
|
|
||||||
FsOp::Rename {
|
|
||||||
from: path.clone(),
|
|
||||||
to: copy.clone(),
|
|
||||||
},
|
|
||||||
),
|
|
||||||
Some(Some(landed)) if landed == content => (
|
|
||||||
"B127",
|
|
||||||
FsOp::Create {
|
|
||||||
path: copy.clone(),
|
|
||||||
data: content.clone(),
|
|
||||||
},
|
|
||||||
),
|
|
||||||
Some(_) => continue,
|
|
||||||
};
|
|
||||||
damage.push(KnownDamage {
|
|
||||||
bug,
|
|
||||||
path: path.clone(),
|
|
||||||
copy: Some(copy.clone()),
|
|
||||||
change,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
damage
|
|
||||||
}
|
|
||||||
|
|
||||||
fn difference(&self) -> String {
|
fn difference(&self) -> String {
|
||||||
let drive = self.drive_tree();
|
let drive = self.drive_tree();
|
||||||
let mut out = String::new();
|
let mut out = String::new();
|
||||||
@@ -890,11 +542,6 @@ fn subtree(tree: &Tree, folder: &str) -> Tree {
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether a known bug is counted rather than failing the run.
|
|
||||||
fn tolerate_known() -> bool {
|
|
||||||
std::env::var("PDFS_SIM_KNOWN").as_deref() != Ok("fail")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn owned_folder(index: usize) -> String {
|
fn owned_folder(index: usize) -> String {
|
||||||
format!("c{index}")
|
format!("c{index}")
|
||||||
}
|
}
|
||||||
@@ -970,21 +617,10 @@ pub(crate) fn run_seeds(profile: &'static Profile, default_seeds: u64) {
|
|||||||
(1..=count).collect()
|
(1..=count).collect()
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let mut failures = Vec::new();
|
let failures: Vec<String> = seeds
|
||||||
let mut known = BTreeMap::<&str, usize>::new();
|
.into_iter()
|
||||||
for seed in seeds {
|
.filter_map(|seed| run(seed, profile).err())
|
||||||
match run(seed, profile) {
|
.collect();
|
||||||
Ok(hits) => {
|
|
||||||
for (bug, count) in hits {
|
|
||||||
*known.entry(bug).or_default() += count;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(failure) => failures.push(failure),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !known.is_empty() {
|
|
||||||
eprintln!("{}: known bugs turned up: {known:?}", profile.name);
|
|
||||||
}
|
|
||||||
assert!(failures.is_empty(), "\n{}", failures.join("\n\n"));
|
assert!(failures.is_empty(), "\n{}", failures.join("\n\n"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1044,6 +680,21 @@ mod tests {
|
|||||||
drain_budget: Duration::from_secs(120),
|
drain_budget: Duration::from_secs(120),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// Wi-Fi's latency, lag and lost replies without the outages: the queue
|
||||||
|
/// has to drain within a settle, not merely by the end.
|
||||||
|
static ONE_CLIENT_WIFI: Profile = Profile {
|
||||||
|
name: "one-client-wifi",
|
||||||
|
clients: 1,
|
||||||
|
steps: 80,
|
||||||
|
faults: Faults::wifi,
|
||||||
|
link_flaps: false,
|
||||||
|
restarts: false,
|
||||||
|
deletes_mid_upload: false,
|
||||||
|
settle_budget: Some(Duration::from_secs(30)),
|
||||||
|
syscall_budget: Duration::from_secs(10),
|
||||||
|
drain_budget: Duration::from_secs(120),
|
||||||
|
};
|
||||||
|
|
||||||
static THREE_CLIENTS: Profile = Profile {
|
static THREE_CLIENTS: Profile = Profile {
|
||||||
name: "three-clients",
|
name: "three-clients",
|
||||||
clients: 3,
|
clients: 3,
|
||||||
@@ -1081,6 +732,12 @@ mod tests {
|
|||||||
run_seeds(&ONE_CLIENT_FLAKY, 4);
|
run_seeds(&ONE_CLIENT_FLAKY, 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse --lib sim::run -- --ignored`"]
|
||||||
|
fn one_client_on_a_slow_link() {
|
||||||
|
run_seeds(&ONE_CLIENT_WIFI, 4);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse --lib sim::run -- --ignored`"]
|
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse --lib sim::run -- --ignored`"]
|
||||||
fn three_clients_one_writer_each() {
|
fn three_clients_one_writer_each() {
|
||||||
@@ -1106,15 +763,4 @@ mod tests {
|
|||||||
&kept.replace("the file was trashed remotely", "revision changed")
|
&kept.replace("the file was trashed remotely", "revision changed")
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn an_invalidation_stuck_on_a_page_lock_is_b130() {
|
|
||||||
let b130 = |stacks: &str| KNOWN_HANGS.iter().any(|known| (known.is)(stacks));
|
|
||||||
let stuck = "thread 1 \"fuser-0\", state S, waiting in fuse_dev_do_read\n\
|
|
||||||
\nthread 2 \"tokio-rt-worker\", state D, waiting in __folio_lock\n\
|
|
||||||
\x20 11: <pdfs_fuse::NotifyBatch>::flush\n";
|
|
||||||
assert!(b130(stuck));
|
|
||||||
assert!(!b130(&stuck.replace("state D", "state S")));
|
|
||||||
assert!(!b130(&stuck.replace("NotifyBatch", "Other")));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -212,6 +212,12 @@ pub(crate) fn lid_ino(lid: i64) -> u64 {
|
|||||||
lid.unsigned_abs() + crate::ROOT_INO
|
lid.unsigned_abs() + crate::ROOT_INO
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The local id of the node at inode `ino`, the inverse of [`lid_ino`]:
|
||||||
|
/// `None` for the root and a fallback inode.
|
||||||
|
pub(crate) fn ino_lid(ino: u64) -> Option<i64> {
|
||||||
|
(ino > crate::ROOT_INO && ino < FALLBACK_INO).then(|| (ino - crate::ROOT_INO) as i64)
|
||||||
|
}
|
||||||
|
|
||||||
/// Mutable inode bookkeeping, guarded by a mutex because fuser drives the
|
/// Mutable inode bookkeeping, guarded by a mutex because fuser drives the
|
||||||
/// `Filesystem` trait through `&self`.
|
/// `Filesystem` trait through `&self`.
|
||||||
///
|
///
|
||||||
@@ -878,6 +884,11 @@ impl State {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write `node` through as it is, in order with the tree's other changes.
|
||||||
|
pub(crate) fn write_through(&mut self, node: Node) {
|
||||||
|
self.outbox.push(DbWrite::Upsert(vec![node]));
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn intern(&mut self, parent: u64, mut node: Node) -> u64 {
|
pub(crate) fn intern(&mut self, parent: u64, mut node: Node) -> u64 {
|
||||||
self.keep_revision_mtime(&mut node);
|
self.keep_revision_mtime(&mut node);
|
||||||
self.outbox.push(DbWrite::Upsert(vec![node.clone()]));
|
self.outbox.push(DbWrite::Upsert(vec![node.clone()]));
|
||||||
@@ -1325,6 +1336,13 @@ mod tests {
|
|||||||
use proton_drive_rs::ShareMembership;
|
use proton_drive_rs::ShareMembership;
|
||||||
use proton_drive_rs::proton_sdk::ids::{LinkId, ShareId, ShareMembershipId, VolumeId};
|
use proton_drive_rs::proton_sdk::ids::{LinkId, ShareId, ShareMembershipId, VolumeId};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_inode_gives_back_the_lid_it_was_made_from() {
|
||||||
|
assert_eq!(ino_lid(lid_ino(7)), Some(7));
|
||||||
|
assert_eq!(ino_lid(crate::ROOT_INO), None);
|
||||||
|
assert_eq!(ino_lid(FALLBACK_INO + 3), None);
|
||||||
|
}
|
||||||
|
|
||||||
fn uid(link: &str) -> NodeUid {
|
fn uid(link: &str) -> NodeUid {
|
||||||
NodeUid::new(VolumeId::from("vol"), LinkId::from(link))
|
NodeUid::new(VolumeId::from("vol"), LinkId::from(link))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,6 +102,9 @@ impl Core {
|
|||||||
|
|
||||||
/// One reconcile pass over a mirror folder, updating its `state` column.
|
/// One reconcile pass over a mirror folder, updating its `state` column.
|
||||||
fn reconcile_pass(&self, folder: &StoredSyncFolder) {
|
fn reconcile_pass(&self, folder: &StoredSyncFolder) {
|
||||||
|
if self.shutdown.is_stopping() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
// Hold the folder's lock for the whole pass so a mode switch can't evict the
|
// Hold the folder's lock for the whole pass so a mode switch can't evict the
|
||||||
// local tree (and mount FUSE over it) while we walk and upload it.
|
// local tree (and mount FUSE over it) while we walk and upload it.
|
||||||
let lock = self.sync_lock(folder.id);
|
let lock = self.sync_lock(folder.id);
|
||||||
@@ -148,8 +151,6 @@ impl Core {
|
|||||||
self.push_pass(folder.id, &local_root, &remote_root, &rules)
|
self.push_pass(folder.id, &local_root, &remote_root, &rules)
|
||||||
} else {
|
} else {
|
||||||
match self.do_reconcile(folder.id, &local_root, &remote_root, &rules) {
|
match self.do_reconcile(folder.id, &local_root, &remote_root, &rules) {
|
||||||
Ok(outcome) => Ok(outcome),
|
|
||||||
Err(PassAbort::Failed(e)) => Err(e),
|
|
||||||
// The user asked for on-demand while this pass was running. Rather
|
// The user asked for on-demand while this pass was running. Rather
|
||||||
// than make them wait out a walk-and-download whose results are
|
// than make them wait out a walk-and-download whose results are
|
||||||
// about to be deleted, drop it here and do the only part that still
|
// about to be deleted, drop it here and do the only part that still
|
||||||
@@ -164,6 +165,7 @@ impl Core {
|
|||||||
self.progress_begin(folder.id);
|
self.progress_begin(folder.id);
|
||||||
self.push_pass(folder.id, &local_root, &remote_root, &rules)
|
self.push_pass(folder.id, &local_root, &remote_root, &rules)
|
||||||
}
|
}
|
||||||
|
result => result,
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
self.progress_end(folder.id);
|
self.progress_end(folder.id);
|
||||||
@@ -216,7 +218,15 @@ impl Core {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
// A push pass is never interrupted, so only the stop gets here: the
|
||||||
|
// folder keeps the state it had, which the next start's pass updates.
|
||||||
|
Err(PassAbort::Stopped | PassAbort::Interrupted) => {
|
||||||
|
info!(id = folder.id, "sync: pass cut short by the stop");
|
||||||
|
let _ = self
|
||||||
|
.db
|
||||||
|
.sync_folder_set_state(folder.id, ¤t.state, current.last_sync);
|
||||||
|
}
|
||||||
|
Err(PassAbort::Failed(e)) => {
|
||||||
warn!(id = folder.id, error = %e, "sync: reconcile failed");
|
warn!(id = folder.id, error = %e, "sync: reconcile failed");
|
||||||
let _ = self
|
let _ = self
|
||||||
.db
|
.db
|
||||||
@@ -294,7 +304,7 @@ impl Core {
|
|||||||
local_root: &Path,
|
local_root: &Path,
|
||||||
remote_root: &NodeUid,
|
remote_root: &NodeUid,
|
||||||
rules: &IgnoreRules,
|
rules: &IgnoreRules,
|
||||||
) -> Result<Outcome, String> {
|
) -> Result<Outcome, PassAbort> {
|
||||||
let baseline = self
|
let baseline = self
|
||||||
.db
|
.db
|
||||||
.sync_entries(folder_id)
|
.sync_entries(folder_id)
|
||||||
@@ -400,7 +410,7 @@ impl Core {
|
|||||||
&mut remote_dirs,
|
&mut remote_dirs,
|
||||||
std::mem::take(&mut batch),
|
std::mem::take(&mut batch),
|
||||||
&mut outcome,
|
&mut outcome,
|
||||||
);
|
)?;
|
||||||
batch_depth = depth;
|
batch_depth = depth;
|
||||||
}
|
}
|
||||||
let item = &local[rel];
|
let item = &local[rel];
|
||||||
@@ -498,7 +508,7 @@ impl Core {
|
|||||||
&mut remote_dirs,
|
&mut remote_dirs,
|
||||||
std::mem::take(&mut batch),
|
std::mem::take(&mut batch),
|
||||||
&mut outcome,
|
&mut outcome,
|
||||||
);
|
)?;
|
||||||
|
|
||||||
// Paths the baseline knows that are no longer on disk: the user deleted them
|
// Paths the baseline knows that are no longer on disk: the user deleted them
|
||||||
// locally, and the deletion has to reach Drive before the mount starts
|
// locally, and the deletion has to reach Drive before the mount starts
|
||||||
@@ -625,7 +635,7 @@ impl Core {
|
|||||||
&mut remote_dirs,
|
&mut remote_dirs,
|
||||||
std::mem::take(&mut batch),
|
std::mem::take(&mut batch),
|
||||||
&mut outcome,
|
&mut outcome,
|
||||||
);
|
)?;
|
||||||
batch_depth = depth;
|
batch_depth = depth;
|
||||||
// A depth boundary is the pass's natural checkpoint — everything
|
// A depth boundary is the pass's natural checkpoint — everything
|
||||||
// queued has been applied, so stopping here leaves nothing half done.
|
// queued has been applied, so stopping here leaves nothing half done.
|
||||||
@@ -813,7 +823,7 @@ impl Core {
|
|||||||
&mut remote_dirs,
|
&mut remote_dirs,
|
||||||
std::mem::take(&mut batch),
|
std::mem::take(&mut batch),
|
||||||
&mut outcome,
|
&mut outcome,
|
||||||
);
|
)?;
|
||||||
|
|
||||||
// Deferred folder deletions, deepest first.
|
// Deferred folder deletions, deepest first.
|
||||||
delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count()));
|
delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count()));
|
||||||
@@ -941,6 +951,9 @@ impl Core {
|
|||||||
// is safe precisely because the walk has applied nothing — a *partial*
|
// is safe precisely because the walk has applied nothing — a *partial*
|
||||||
// remote map, on the other hand, would be read as "the remote deleted
|
// remote map, on the other hand, would be read as "the remote deleted
|
||||||
// everything we haven't reached yet", so it must never reach the diff.
|
// everything we haven't reached yet", so it must never reach the diff.
|
||||||
|
if self.shutdown.is_stopping() {
|
||||||
|
return Err(PassAbort::Stopped);
|
||||||
|
}
|
||||||
if self.ondemand_queued(folder_id) {
|
if self.ondemand_queued(folder_id) {
|
||||||
return Err(PassAbort::Interrupted);
|
return Err(PassAbort::Interrupted);
|
||||||
}
|
}
|
||||||
@@ -1051,6 +1064,9 @@ impl Core {
|
|||||||
/// `remote_dirs` (so deeper paths resolve their parent), and what each op did
|
/// `remote_dirs` (so deeper paths resolve their parent), and what each op did
|
||||||
/// is tallied into `outcome`. Driven by `block_on` from the (non-runtime) sync
|
/// is tallied into `outcome`. Driven by `block_on` from the (non-runtime) sync
|
||||||
/// engine thread, spawning the tasks onto the shared runtime.
|
/// engine thread, spawning the tasks onto the shared runtime.
|
||||||
|
///
|
||||||
|
/// Fails with [`PassAbort::Stopped`] once the daemon is stopping, after the
|
||||||
|
/// ops already sending have finished.
|
||||||
fn flush_batch(
|
fn flush_batch(
|
||||||
&self,
|
&self,
|
||||||
folder_id: i64,
|
folder_id: i64,
|
||||||
@@ -1058,9 +1074,9 @@ impl Core {
|
|||||||
remote_dirs: &mut HashMap<String, NodeUid>,
|
remote_dirs: &mut HashMap<String, NodeUid>,
|
||||||
batch: Vec<Pending>,
|
batch: Vec<Pending>,
|
||||||
outcome: &mut Outcome,
|
outcome: &mut Outcome,
|
||||||
) {
|
) -> Result<(), PassAbort> {
|
||||||
if batch.is_empty() {
|
if batch.is_empty() {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
self.progress_queued(folder_id, batch.len());
|
self.progress_queued(folder_id, batch.len());
|
||||||
let core = self.clone();
|
let core = self.clone();
|
||||||
@@ -1074,14 +1090,22 @@ impl Core {
|
|||||||
let root = root.clone();
|
let root = root.clone();
|
||||||
set.spawn(async move {
|
set.spawn(async move {
|
||||||
let _permit = sem.acquire_owned().await.expect("semaphore not closed");
|
let _permit = sem.acquire_owned().await.expect("semaphore not closed");
|
||||||
core.apply_pending(folder_id, &root, op).await
|
// An op still waiting for its turn when the stop comes is
|
||||||
|
// not started; the ones already sending finish.
|
||||||
|
if core.shutdown.is_stopping() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(core.apply_pending(folder_id, &root, op).await)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
let mut join_errors = 0usize;
|
let mut join_errors = 0usize;
|
||||||
while let Some(joined) = set.join_next().await {
|
while let Some(joined) = set.join_next().await {
|
||||||
match joined {
|
match joined {
|
||||||
Ok(result) => out.push(result),
|
Ok(Some(result)) => out.push(result),
|
||||||
|
Ok(None) => {}
|
||||||
|
// The runtime ends with the daemon and cancels what is left.
|
||||||
|
Err(e) if e.is_cancelled() => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
join_errors += 1;
|
join_errors += 1;
|
||||||
warn!(error = %e, "sync: task panicked");
|
warn!(error = %e, "sync: task panicked");
|
||||||
@@ -1106,6 +1130,13 @@ impl Core {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The pass goes no further: the next depth's paths may need a folder
|
||||||
|
// this batch did not get to make, and went on to fail one by one
|
||||||
|
// (`docs/BUGS.md` B169).
|
||||||
|
if self.shutdown.is_stopping() {
|
||||||
|
return Err(PassAbort::Stopped);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Apply one [`Pending`] op (async, so it can run concurrently in a batch),
|
/// Apply one [`Pending`] op (async, so it can run concurrently in a batch),
|
||||||
|
|||||||
@@ -227,6 +227,10 @@ pub(super) enum PassAbort {
|
|||||||
Interrupted,
|
Interrupted,
|
||||||
/// The pass could not establish its diff (a walk or the baseline load failed).
|
/// The pass could not establish its diff (a walk or the baseline load failed).
|
||||||
Failed(String),
|
Failed(String),
|
||||||
|
/// The daemon is stopping. What the pass sent before the stop has landed or
|
||||||
|
/// will be cancelled with the runtime; nothing more is sent, and the next
|
||||||
|
/// start's pass picks up from the baseline.
|
||||||
|
Stopped,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl From<String> for PassAbort {
|
impl From<String> for PassAbort {
|
||||||
|
|||||||
@@ -530,7 +530,7 @@ fn open_archive(
|
|||||||
|
|
||||||
/// The lowercase-hex SHA-1 of everything `reader` yields — the digest Proton's
|
/// The lowercase-hex SHA-1 of everything `reader` yields — the digest Proton's
|
||||||
/// photo duplicate detection is keyed on.
|
/// photo duplicate detection is keyed on.
|
||||||
fn sha1_of(reader: &mut (dyn Read + Send)) -> CoreResult<String> {
|
pub(crate) fn sha1_of(reader: &mut (dyn Read + Send)) -> CoreResult<String> {
|
||||||
let mut hasher = Sha1::new();
|
let mut hasher = Sha1::new();
|
||||||
let mut buffer = vec![0u8; 256 * 1024];
|
let mut buffer = vec![0u8; 256 * 1024];
|
||||||
loop {
|
loop {
|
||||||
|
|||||||
@@ -46,6 +46,7 @@
|
|||||||
.badge-pinned { color: var(--success-color); }
|
.badge-pinned { color: var(--success-color); }
|
||||||
.badge-cached { color: var(--success-color); }
|
.badge-cached { color: var(--success-color); }
|
||||||
.badge-cloud { color: color-mix(in srgb, currentColor 55%, transparent); }
|
.badge-cloud { color: color-mix(in srgb, currentColor 55%, transparent); }
|
||||||
|
.badge-issue { color: var(--error-color); }
|
||||||
.browser-statusbar { background-color: alpha(currentColor, 0.025); }
|
.browser-statusbar { background-color: alpha(currentColor, 0.025); }
|
||||||
scale.browser-status-meter trough { min-width: 104px; min-height: 6px; }
|
scale.browser-status-meter trough { min-width: 104px; min-height: 6px; }
|
||||||
.bulk-bar { padding: 6px 8px; border-radius: 12px; background: color-mix(in srgb, var(--accent-bg-color) 12%, transparent); }
|
.bulk-bar { padding: 6px 8px; border-radius: 12px; background: color-mix(in srgb, var(--accent-bg-color) 12%, transparent); }
|
||||||
|
|||||||
@@ -75,8 +75,8 @@ use pdfs_core::control::{
|
|||||||
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, MappingProgress, PendingOpInfo,
|
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, MappingProgress, PendingOpInfo,
|
||||||
PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope, Request, Response,
|
PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope, Request, Response,
|
||||||
RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry, ShareEntryKind, SharedItem,
|
RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry, ShareEntryKind, SharedItem,
|
||||||
SyncFolderInfo, SyncPhase, SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection,
|
SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress, ThumbnailBuildStatus, Topic,
|
||||||
TransferItem, send,
|
TransferDirection, TransferItem, send,
|
||||||
};
|
};
|
||||||
|
|
||||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||||
|
|||||||
@@ -341,6 +341,59 @@ fn detail_label(entry: &ActivityEntry) -> Option<String> {
|
|||||||
&[("name", alt)],
|
&[("name", alt)],
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if let Some((reason, copy)) = detail.split_once("; local changes uploaded as ") {
|
||||||
|
let args = [("name", copy)];
|
||||||
|
return Some(match edit_conflict(reason) {
|
||||||
|
// Translators: {name} is the copy the changes made on this
|
||||||
|
// computer were uploaded as.
|
||||||
|
EditConflict::Gone => gettext_f(
|
||||||
|
"It no longer exists on Proton Drive, so your changes were uploaded as {name}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
// Translators: {name} is the copy the changes made on this
|
||||||
|
// computer were uploaded as.
|
||||||
|
EditConflict::Trashed => gettext_f(
|
||||||
|
"It was moved to the Trash on Proton Drive, so your changes were uploaded as {name}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
// Translators: {name} is the copy the changes made on this
|
||||||
|
// computer were uploaded as.
|
||||||
|
EditConflict::Changed => gettext_f(
|
||||||
|
"It was changed on Proton Drive, so your changes were uploaded as {name}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
// Translators: {name} is the copy the changes made on this
|
||||||
|
// computer were uploaded as.
|
||||||
|
EditConflict::Other => gettext_f("Your changes were uploaded as {name}", &args),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some((reason, path)) = detail.split_once("; local changes kept at ") {
|
||||||
|
let path = tilde_path(path);
|
||||||
|
let args = [("path", path.as_str())];
|
||||||
|
return Some(match edit_conflict(reason) {
|
||||||
|
// Translators: {path} is the file on this computer that holds the
|
||||||
|
// changes which could not be uploaded.
|
||||||
|
EditConflict::Gone => gettext_f(
|
||||||
|
"It no longer exists on Proton Drive, so your changes were kept in {path}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
// Translators: {path} is the file on this computer that holds the
|
||||||
|
// changes which could not be uploaded.
|
||||||
|
EditConflict::Trashed => gettext_f(
|
||||||
|
"It was moved to the Trash on Proton Drive, so your changes were kept in {path}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
// Translators: {path} is the file on this computer that holds the
|
||||||
|
// changes which could not be uploaded.
|
||||||
|
EditConflict::Changed => gettext_f(
|
||||||
|
"It was changed on Proton Drive, so your changes were kept in {path}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
// Translators: {path} is the file on this computer that holds the
|
||||||
|
// changes which could not be uploaded.
|
||||||
|
EditConflict::Other => gettext_f("Your changes were kept in {path}", &args),
|
||||||
|
});
|
||||||
|
}
|
||||||
if let Some(base) = detail.strip_prefix("differs from ") {
|
if let Some(base) = detail.strip_prefix("differs from ") {
|
||||||
// Translators: {name} is the file the conflicting copy was made from.
|
// Translators: {name} is the file the conflicting copy was made from.
|
||||||
return Some(gettext_f("Differs from {name}", &[("name", base)]));
|
return Some(gettext_f("Differs from {name}", &[("name", base)]));
|
||||||
@@ -388,6 +441,26 @@ fn detail_label(entry: &ActivityEntry) -> Option<String> {
|
|||||||
Some(capitalize(detail))
|
Some(capitalize(detail))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Why a queued edit could not land on its file, from the reason the daemon
|
||||||
|
/// gives before "; local changes uploaded as" or "; local changes kept at".
|
||||||
|
enum EditConflict {
|
||||||
|
Gone,
|
||||||
|
Trashed,
|
||||||
|
Changed,
|
||||||
|
Other,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn edit_conflict(reason: &str) -> EditConflict {
|
||||||
|
match reason {
|
||||||
|
"the file no longer exists remotely" => EditConflict::Gone,
|
||||||
|
"the file was trashed remotely" => EditConflict::Trashed,
|
||||||
|
r if r.starts_with("the remote revision changed under the queued write") => {
|
||||||
|
EditConflict::Changed
|
||||||
|
}
|
||||||
|
_ => EditConflict::Other,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// A sync pass summary such as "2 uploaded, 1 folder(s) created", as a
|
/// A sync pass summary such as "2 uploaded, 1 folder(s) created", as a
|
||||||
/// list of counts. `None` when a part is not one the daemon writes.
|
/// list of counts. `None` when a part is not one the daemon writes.
|
||||||
fn sync_summary(detail: &str) -> Option<String> {
|
fn sync_summary(detail: &str) -> Option<String> {
|
||||||
@@ -596,6 +669,38 @@ mod tests {
|
|||||||
assert_eq!(d.details, vec!["1 person can view"]);
|
assert_eq!(d.details, vec!["1 person can view"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_edit_kept_as_a_copy_says_why_in_words() {
|
||||||
|
let d = describe(&entry(
|
||||||
|
ActivityKind::Upload,
|
||||||
|
"notes.txt",
|
||||||
|
"the remote revision changed under the queued write \
|
||||||
|
(based on revision r1, remote now at r2); \
|
||||||
|
local changes uploaded as notes (sync-conflict 1700000000).txt",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
d.details,
|
||||||
|
vec![
|
||||||
|
"It was changed on Proton Drive, so your changes were uploaded as \
|
||||||
|
notes (sync-conflict 1700000000).txt"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
let d = describe(&entry(
|
||||||
|
ActivityKind::Upload,
|
||||||
|
"notes.txt",
|
||||||
|
"the file was trashed remotely; local changes kept at /var/staging/7",
|
||||||
|
false,
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
d.details,
|
||||||
|
vec![
|
||||||
|
"It was moved to the Trash on Proton Drive, so your changes were kept in \
|
||||||
|
/var/staging/7"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn unknown_details_show_as_written() {
|
fn unknown_details_show_as_written() {
|
||||||
let d = describe(&entry(
|
let d = describe(&entry(
|
||||||
|
|||||||
@@ -2495,9 +2495,13 @@ pub(crate) fn start_upload(ui: &Rc<Ui>, sources: Vec<String>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The sync-state badge for an entry: `(icon, css-class)`, or `None` for folders
|
/// The sync-state badge for an entry: `(icon, css-class)`, or `None` for folders
|
||||||
/// (which carry no per-file cache state). Pinned (kept offline) ranks above merely
|
/// (which carry no per-file cache state). A sync issue ranks first, on folders
|
||||||
|
/// too: the change waits for the user. Pinned (kept offline) ranks above merely
|
||||||
/// cached (downloaded, evictable); everything else is online-only.
|
/// cached (downloaded, evictable); everything else is online-only.
|
||||||
pub(crate) fn badge_for(entry: &DirEntry) -> Option<(&'static str, &'static str)> {
|
pub(crate) fn badge_for(entry: &DirEntry) -> Option<(&'static str, &'static str)> {
|
||||||
|
if entry.issue.is_some() {
|
||||||
|
return Some(("dialog-warning-symbolic", "badge-issue"));
|
||||||
|
}
|
||||||
if entry.is_dir {
|
if entry.is_dir {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -2513,9 +2517,11 @@ pub(crate) fn badge_for(entry: &DirEntry) -> Option<(&'static str, &'static str)
|
|||||||
/// Paint `badge` to reflect the entry's sync state (see [`badge_for`]). Clears any
|
/// Paint `badge` to reflect the entry's sync state (see [`badge_for`]). Clears any
|
||||||
/// prior colour class first, since list factories recycle cells.
|
/// prior colour class first, since list factories recycle cells.
|
||||||
pub(crate) fn apply_badge(badge: >k4::Image, entry: &DirEntry) {
|
pub(crate) fn apply_badge(badge: >k4::Image, entry: &DirEntry) {
|
||||||
for class in ["badge-pinned", "badge-cached", "badge-cloud"] {
|
for class in ["badge-pinned", "badge-cached", "badge-cloud", "badge-issue"] {
|
||||||
badge.remove_css_class(class);
|
badge.remove_css_class(class);
|
||||||
}
|
}
|
||||||
|
let issue = entry.issue.map(super::locations::issue_text);
|
||||||
|
badge.set_tooltip_text(issue.as_deref());
|
||||||
match badge_for(entry) {
|
match badge_for(entry) {
|
||||||
Some((icon, class)) => {
|
Some((icon, class)) => {
|
||||||
badge.set_icon_name(Some(icon));
|
badge.set_icon_name(Some(icon));
|
||||||
@@ -2915,6 +2921,7 @@ pub(crate) fn repaint_search(ui: &Rc<Ui>, hits: &[SearchHit]) {
|
|||||||
shared_by_unverified: false,
|
shared_by_unverified: false,
|
||||||
trashed_at: 0,
|
trashed_at: 0,
|
||||||
trashed_from: None,
|
trashed_from: None,
|
||||||
|
issue: None,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
sort_entries(&mut entries, ui.browser.view.get());
|
sort_entries(&mut entries, ui.browser.view.get());
|
||||||
@@ -2952,11 +2959,11 @@ fn listing_summary(files: usize, folders: usize, file_bytes: u64) -> String {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
MoveTarget, hit_location, listing_summary, move_target_allowed,
|
MoveTarget, badge_for, hit_location, listing_summary, move_target_allowed,
|
||||||
show_thumbnail_build_progress, sort_entries,
|
show_thumbnail_build_progress, sort_entries,
|
||||||
};
|
};
|
||||||
use pdfs_core::config::{FileSort, FilesView};
|
use pdfs_core::config::{FileSort, FilesView};
|
||||||
use pdfs_core::control::{DirEntry, ThumbnailBuildStatus};
|
use pdfs_core::control::{DirEntry, SyncIssue, ThumbnailBuildStatus};
|
||||||
|
|
||||||
fn entry(name: &str, is_dir: bool, size: u64, modified: i64) -> DirEntry {
|
fn entry(name: &str, is_dir: bool, size: u64, modified: i64) -> DirEntry {
|
||||||
DirEntry {
|
DirEntry {
|
||||||
@@ -2974,6 +2981,7 @@ mod tests {
|
|||||||
shared_by_unverified: false,
|
shared_by_unverified: false,
|
||||||
trashed_at: 0,
|
trashed_at: 0,
|
||||||
trashed_from: None,
|
trashed_from: None,
|
||||||
|
issue: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2981,6 +2989,20 @@ mod tests {
|
|||||||
entries.iter().map(|e| e.name.as_str()).collect()
|
entries.iter().map(|e| e.name.as_str()).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_sync_issue_shows_on_the_node_before_its_cache_state() {
|
||||||
|
let mut file = entry("a.txt", false, 1, 0);
|
||||||
|
file.pinned = true;
|
||||||
|
assert_eq!(badge_for(&file).unwrap().1, "badge-pinned");
|
||||||
|
file.issue = Some(SyncIssue::Quota);
|
||||||
|
assert_eq!(badge_for(&file).unwrap().1, "badge-issue");
|
||||||
|
|
||||||
|
let mut folder = entry("d", true, 0, 0);
|
||||||
|
assert_eq!(badge_for(&folder), None);
|
||||||
|
folder.issue = Some(SyncIssue::Missing);
|
||||||
|
assert_eq!(badge_for(&folder).unwrap().1, "badge-issue");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_search_hit_location_is_its_parent_folder() {
|
fn a_search_hit_location_is_its_parent_folder() {
|
||||||
assert_eq!(hit_location("Work/Old/a.txt"), "Work/Old");
|
assert_eq!(hit_location("Work/Old/a.txt"), "Work/Old");
|
||||||
|
|||||||
@@ -70,8 +70,9 @@ pub(crate) struct LocationRow {
|
|||||||
progress: gtk4::ProgressBar,
|
progress: gtk4::ProgressBar,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What a queue row shows that can change: id, attempts, next attempt, parked.
|
/// What a queue row shows that can change: id, attempts, next attempt, parked,
|
||||||
pub(crate) type QueueKey = (i64, i64, Option<i64>, bool);
|
/// issue.
|
||||||
|
pub(crate) type QueueKey = (i64, i64, Option<i64>, bool, Option<SyncIssue>);
|
||||||
|
|
||||||
/// The "Waiting to Upload" list: the daemon's pending-op queue.
|
/// The "Waiting to Upload" list: the daemon's pending-op queue.
|
||||||
pub(crate) struct QueueState {
|
pub(crate) struct QueueState {
|
||||||
@@ -591,7 +592,7 @@ fn repaint_queue(ui: &Rc<Ui>, items: &[PendingOpInfo]) {
|
|||||||
items.sort_by_key(|op| (!op.failing, op.attempts == 0, op.id));
|
items.sort_by_key(|op| (!op.failing, op.attempts == 0, op.id));
|
||||||
let key: Vec<QueueKey> = items
|
let key: Vec<QueueKey> = items
|
||||||
.iter()
|
.iter()
|
||||||
.map(|op| (op.id, op.attempts, op.next_attempt_at, op.parked))
|
.map(|op| (op.id, op.attempts, op.next_attempt_at, op.parked, op.issue))
|
||||||
.collect();
|
.collect();
|
||||||
if *queue.painted.borrow() == key {
|
if *queue.painted.borrow() == key {
|
||||||
return;
|
return;
|
||||||
@@ -648,6 +649,10 @@ fn queue_row(ui: &Rc<Ui>, op: &PendingOpInfo, now: i64) -> adw::ActionRow {
|
|||||||
"{action} · {state}",
|
"{action} · {state}",
|
||||||
&[("action", &action), ("state", &state)],
|
&[("action", &action), ("state", &state)],
|
||||||
);
|
);
|
||||||
|
if let Some(issue) = op.issue {
|
||||||
|
subtitle.push('\n');
|
||||||
|
subtitle.push_str(&issue_text(issue));
|
||||||
|
}
|
||||||
if let Some(error) = &op.last_error {
|
if let Some(error) = &op.last_error {
|
||||||
subtitle.push('\n');
|
subtitle.push('\n');
|
||||||
// Translators: {n} is how many attempts failed, {error} the last error message.
|
// Translators: {n} is how many attempts failed, {error} the last error message.
|
||||||
@@ -662,7 +667,7 @@ fn queue_row(ui: &Rc<Ui>, op: &PendingOpInfo, now: i64) -> adw::ActionRow {
|
|||||||
.title(glib::markup_escape_text(&op.path).as_str())
|
.title(glib::markup_escape_text(&op.path).as_str())
|
||||||
.subtitle(glib::markup_escape_text(&subtitle).as_str())
|
.subtitle(glib::markup_escape_text(&subtitle).as_str())
|
||||||
.title_lines(1)
|
.title_lines(1)
|
||||||
.subtitle_lines(3)
|
.subtitle_lines(if op.issue.is_some() { 5 } else { 3 })
|
||||||
.tooltip_text(&op.path)
|
.tooltip_text(&op.path)
|
||||||
.build();
|
.build();
|
||||||
let image = gtk4::Image::from_icon_name(if op.failing {
|
let image = gtk4::Image::from_icon_name(if op.failing {
|
||||||
@@ -687,9 +692,157 @@ fn queue_row(ui: &Rc<Ui>, op: &PendingOpInfo, now: i64) -> adw::ActionRow {
|
|||||||
retry.connect_clicked(move |_| retry_queued(&ui, Some(id)));
|
retry.connect_clicked(move |_| retry_queued(&ui, Some(id)));
|
||||||
row.add_suffix(&retry);
|
row.add_suffix(&retry);
|
||||||
}
|
}
|
||||||
|
if op.failing && op.exportable {
|
||||||
|
let export = gtk4::Button::builder()
|
||||||
|
.label(gettext("Export"))
|
||||||
|
.tooltip_text(gettext("Save a copy of this file's content elsewhere"))
|
||||||
|
.valign(gtk4::Align::Center)
|
||||||
|
.build();
|
||||||
|
export.add_css_class("flat");
|
||||||
|
let ui = ui.clone();
|
||||||
|
let id = op.id;
|
||||||
|
let name = Path::new(&op.path)
|
||||||
|
.file_name()
|
||||||
|
.map(|name| name.to_string_lossy().into_owned())
|
||||||
|
.unwrap_or_default();
|
||||||
|
export.connect_clicked(move |_| prompt_export_queued(&ui, id, &name));
|
||||||
|
row.add_suffix(&export);
|
||||||
|
}
|
||||||
|
if op.failing {
|
||||||
|
let discard = gtk4::Button::builder()
|
||||||
|
.label(gettext("Discard"))
|
||||||
|
.tooltip_text(gettext("Undo this change here instead of sending it"))
|
||||||
|
.valign(gtk4::Align::Center)
|
||||||
|
.build();
|
||||||
|
discard.add_css_class("flat");
|
||||||
|
let ui = ui.clone();
|
||||||
|
let id = op.id;
|
||||||
|
let path = op.path.clone();
|
||||||
|
let exportable = op.exportable;
|
||||||
|
discard.connect_clicked(move |_| prompt_discard_queued(&ui, id, &path, exportable));
|
||||||
|
row.add_suffix(&discard);
|
||||||
|
}
|
||||||
row
|
row
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What a sync issue means and what the user can do about it. The change stays
|
||||||
|
/// queued either way.
|
||||||
|
pub(crate) fn issue_text(issue: SyncIssue) -> String {
|
||||||
|
match issue {
|
||||||
|
SyncIssue::Quota => gettext(
|
||||||
|
"Your Proton Drive storage is full. Free up space or upgrade your plan, and this change uploads on its own.",
|
||||||
|
),
|
||||||
|
SyncIssue::Access => gettext(
|
||||||
|
"You can no longer change this on Proton Drive. Ask the owner for edit access, or export the file to keep your version.",
|
||||||
|
),
|
||||||
|
SyncIssue::Missing => gettext(
|
||||||
|
"The folder this change belongs in is gone from Proton Drive. Export the file to keep your version.",
|
||||||
|
),
|
||||||
|
SyncIssue::Limit => gettext(
|
||||||
|
"Proton Drive's limit on items in a folder or on folder depth is reached. Make room in the folder, then retry.",
|
||||||
|
),
|
||||||
|
SyncIssue::Rejected | SyncIssue::Unknown => {
|
||||||
|
gettext("Proton Drive refused this change. It stays queued and is tried again.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask where to save a copy of a queued file's content, then have the daemon
|
||||||
|
/// write it there. The change itself stays queued.
|
||||||
|
fn prompt_export_queued(ui: &Rc<Ui>, id: i64, name: &str) {
|
||||||
|
let window = ui_window(ui);
|
||||||
|
let dialog = gtk4::FileDialog::builder()
|
||||||
|
.title(gettext("Export File"))
|
||||||
|
.initial_name(name)
|
||||||
|
.build();
|
||||||
|
let ui = ui.clone();
|
||||||
|
dialog.save(window.as_ref(), gio::Cancellable::NONE, move |res| {
|
||||||
|
let Ok(file) = res else { return };
|
||||||
|
let Some(dest) = file.path().and_then(|p| p.to_str().map(str::to_string)) else {
|
||||||
|
toast_error(
|
||||||
|
&ui,
|
||||||
|
&gettext("Couldn't export the file"),
|
||||||
|
&gettext("That location isn't a local file."),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let rx = spawn_request(
|
||||||
|
ui.dirs.control_socket(),
|
||||||
|
Request::ExportPendingOp { id, dest },
|
||||||
|
);
|
||||||
|
let ui = ui.clone();
|
||||||
|
glib::spawn_future_local(async move {
|
||||||
|
match rx.recv().await {
|
||||||
|
Ok(Ok(Response::Ok { .. })) => toast(&ui, &gettext("File exported")),
|
||||||
|
Ok(Ok(Response::Error { message, kind })) => {
|
||||||
|
toast_failure(&ui, &gettext("Couldn't export the file"), &message, kind)
|
||||||
|
}
|
||||||
|
_ => toast_error(
|
||||||
|
&ui,
|
||||||
|
&gettext("Couldn't export the file"),
|
||||||
|
&gettext("The Proton Drive service didn't respond."),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Confirm, then have the daemon drop a queued change and undo it here.
|
||||||
|
fn prompt_discard_queued(ui: &Rc<Ui>, id: i64, path: &str, exportable: bool) {
|
||||||
|
let body = if exportable {
|
||||||
|
// Translators: {path} is the file whose queued change would be dropped.
|
||||||
|
gettext_f(
|
||||||
|
"Your changes to {path} are undone here and never reach Proton Drive. Export the file first to keep your version.",
|
||||||
|
&[("path", path)],
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
// Translators: {path} is the file or folder whose queued change would be dropped.
|
||||||
|
gettext_f(
|
||||||
|
"The change to {path} is undone here and never reaches Proton Drive.",
|
||||||
|
&[("path", path)],
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let dialog = adw::AlertDialog::builder()
|
||||||
|
.heading(gettext("Discard Change?"))
|
||||||
|
.body(body)
|
||||||
|
.build();
|
||||||
|
dialog.add_response("cancel", &gettext("Cancel"));
|
||||||
|
dialog.add_response("discard", &gettext("Discard"));
|
||||||
|
dialog.set_response_appearance("discard", adw::ResponseAppearance::Destructive);
|
||||||
|
dialog.set_default_response(Some("cancel"));
|
||||||
|
dialog.set_close_response("cancel");
|
||||||
|
let window = ui_window(ui);
|
||||||
|
let ui = ui.clone();
|
||||||
|
dialog.connect_response(None, move |_, resp| {
|
||||||
|
if resp == "discard" {
|
||||||
|
discard_queued(&ui, id);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
dialog.present(window.as_ref());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn discard_queued(ui: &Rc<Ui>, id: i64) {
|
||||||
|
let rx = spawn_request(ui.dirs.control_socket(), Request::DiscardPendingOp { id });
|
||||||
|
let ui = ui.clone();
|
||||||
|
glib::spawn_future_local(async move {
|
||||||
|
match rx.recv().await {
|
||||||
|
Ok(Ok(Response::Ok { .. })) => {
|
||||||
|
toast(&ui, &gettext("Change discarded"));
|
||||||
|
ui.locations.queue.painted.borrow_mut().clear();
|
||||||
|
refresh_queue(&ui);
|
||||||
|
}
|
||||||
|
Ok(Ok(Response::Error { message, kind })) => {
|
||||||
|
toast_failure(&ui, &gettext("Couldn't discard the change"), &message, kind)
|
||||||
|
}
|
||||||
|
_ => toast_error(
|
||||||
|
&ui,
|
||||||
|
&gettext("Couldn't discard the change"),
|
||||||
|
&gettext("The Proton Drive service didn't respond."),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/// Ask the daemon to stop waiting out one op's backoff, or every failed op's.
|
/// Ask the daemon to stop waiting out one op's backoff, or every failed op's.
|
||||||
fn retry_queued(ui: &Rc<Ui>, id: Option<i64>) {
|
fn retry_queued(ui: &Rc<Ui>, id: Option<i64>) {
|
||||||
let rx = spawn_request(ui.dirs.control_socket(), Request::RetryPendingOp { id });
|
let rx = spawn_request(ui.dirs.control_socket(), Request::RetryPendingOp { id });
|
||||||
|
|||||||
@@ -618,6 +618,7 @@ pub(crate) fn shared_item_as_entry(item: &SharedItem) -> DirEntry {
|
|||||||
shared_by_unverified: false,
|
shared_by_unverified: false,
|
||||||
trashed_at: 0,
|
trashed_at: 0,
|
||||||
trashed_from: None,
|
trashed_from: None,
|
||||||
|
issue: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -533,6 +533,7 @@ mod tests {
|
|||||||
shared_by_unverified: false,
|
shared_by_unverified: false,
|
||||||
trashed_at,
|
trashed_at,
|
||||||
trashed_from: from.map(str::to_string),
|
trashed_from: from.map(str::to_string),
|
||||||
|
issue: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -189,7 +189,7 @@ Because Proton Drive does not support partial byte writes, modified files must b
|
|||||||
- Durably publishes the scratch data and authored-range sidecar into `staging` under a `{uid}-{millis}-{counter}` name. Temporary data and metadata are synced before atomic rename, and their directory is synced before the source is removed.
|
- Durably publishes the scratch data and authored-range sidecar into `staging` under a `{uid}-{millis}-{counter}` name. Temporary data and metadata are synced before atomic rename, and their directory is synced before the source is removed.
|
||||||
- Transactionally queues or supersedes a pending database operation (`PendingOp`), so insertion failure cannot erase the previously acknowledged upload. For `OP_REVISION` ops, execution is debounced to give rapid follow-up writes (e.g. `aria2c` preallocation followed by writing) time to supersede the staged blob before network transmission. The debounce is adaptive (`Core::revision_debounce`): it starts at `DRAIN_REVISION_DEBOUNCE = 2s` and, once a node has been uploaded, widens toward how long that upload actually took, bounded by `DRAIN_REVISION_DEBOUNCE_MAX = 60s`. A file saved faster than it can be sent therefore supersedes in the queue rather than mid-upload.
|
- Transactionally queues or supersedes a pending database operation (`PendingOp`), so insertion failure cannot erase the previously acknowledged upload. For `OP_REVISION` ops, execution is debounced to give rapid follow-up writes (e.g. `aria2c` preallocation followed by writing) time to supersede the staged blob before network transmission. The debounce is adaptive (`Core::revision_debounce`): it starts at `DRAIN_REVISION_DEBOUNCE = 2s` and, once a node has been uploaded, widens toward how long that upload actually took, bounded by `DRAIN_REVISION_DEBOUNCE_MAX = 60s`. A file saved faster than it can be sent therefore supersedes in the queue rather than mid-upload.
|
||||||
- Signals `Core::cancel_upload` for the node before touching the queue. A drain worker may already be reading the blob this write supersedes; the flag is read by the upload's `CountingReader`, which refuses the SDK's next block. `queue_trash` and `discard_queued_ops` do the same, because they unlink the blob outright.
|
- Signals `Core::cancel_upload` for the node before touching the queue. A drain worker may already be reading the blob this write supersedes; the flag is read by the upload's `CountingReader`, which refuses the SDK's next block. `queue_trash` and `discard_queued_ops` do the same, because they unlink the blob outright.
|
||||||
3. **Async Drain Threads (`run_pending_drain`):** `DRAIN_WORKERS = 3` background workers share the operations queue. They pick work up through `Db::claim_next_due_op`, which marks the row `claimed_at` in the same transaction that selects it and excludes any op whose uid another worker already holds — ordering only has to hold *per node*, and that exclusion is what guarantees it. Worker 0 additionally runs the queue's idle chores (LRU touch flush, `recover_fsynced_writes`). A claim is process-local state: the single-writer `flock` means one found at open belongs to a crashed run, and `Db::clear_op_claims` drops the lot so those ops are not invisible forever. Each worker handles revision uploads, resolves conflicts, and cleans up staging files. Upon landing a revision upload (`refresh_after_upload`), it rebaselines both still-queued ops (`rebaseline_pending`) and open write handles targeting the same node to prevent false self-conflict copies on subsequent writes.
|
3. **Async Drain Threads (`run_pending_drain`):** `DRAIN_WORKERS = 16` background workers share the operations queue; one that lands an op wakes the idle ones, since it may have made others claimable. They pick work up through `Db::claim_next_due_op`, which marks the row `claimed_at` in the same transaction that selects it and excludes any op whose uid another worker already holds — ordering only has to hold *per node*, and that exclusion is what guarantees it. Worker 0 additionally runs the queue's idle chores (LRU touch flush, `recover_fsynced_writes`). A claim is process-local state: the single-writer `flock` means one found at open belongs to a crashed run, and `Db::clear_op_claims` drops the lot so those ops are not invisible forever. Each worker handles revision uploads, resolves conflicts, and cleans up staging files. Upon landing a revision upload (`refresh_after_upload`), it rebaselines both still-queued ops (`rebaseline_pending`) and open write handles targeting the same node to prevent false self-conflict copies on subsequent writes.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
sequenceDiagram
|
sequenceDiagram
|
||||||
@@ -337,7 +337,7 @@ The background daemon relies on the following thread topology:
|
|||||||
2. **FUSE Workers Pool (11 threads, two lanes):** Bounded thread pool handling network operations. Split into 3 threads reserved for metadata (`lookup`, `readdir`) and 8 general threads that serve transfers (block reads) and fall back to metadata when no transfer is waiting. Reserved threads never accept a transfer — that is what keeps a directory listing from queuing behind saturated downloads (audit A6).
|
2. **FUSE Workers Pool (11 threads, two lanes):** Bounded thread pool handling network operations. Split into 3 threads reserved for metadata (`lookup`, `readdir`) and 8 general threads that serve transfers (block reads) and fall back to metadata when no transfer is waiting. Reserved threads never accept a transfer — that is what keeps a directory listing from queuing behind saturated downloads (audit A6).
|
||||||
3. **Control Threads (`pdfs-control`):** Listen on the Unix socket and admit at most 64 concurrent, timeout-bound handlers. Long-lived subscriptions hold one handler each, up to 8 (see §6).
|
3. **Control Threads (`pdfs-control`):** Listen on the Unix socket and admit at most 64 concurrent, timeout-bound handlers. Long-lived subscriptions hold one handler each, up to 8 (see §6).
|
||||||
4. **Sync Engine Threads (`pdfs-sync`, `pdfs-sync-poll`):** Serialize sync passes. Wake on debounced inotify changes, the periodic poll, remote events, or a user request.
|
4. **Sync Engine Threads (`pdfs-sync`, `pdfs-sync-poll`):** Serialize sync passes. Wake on debounced inotify changes, the periodic poll, remote events, or a user request.
|
||||||
5. **Drain Workers (`pdfs-drain-0` … `pdfs-drain-2`):** `DRAIN_WORKERS = 3` threads share the queue of staged writes and changes (`PendingOp`), with per-node ordering (§4), and retry failures with exponential backoff. Worker 0 also runs the queue's idle chores.
|
5. **Drain Workers (`pdfs-drain-0` … `pdfs-drain-15`):** `DRAIN_WORKERS = 16` threads share the queue of staged writes and changes (`PendingOp`), with per-node ordering (§4), and retry failures with exponential backoff. Worker 0 also runs the queue's idle chores.
|
||||||
6. **Conflict Sweep Thread (`pdfs-conflict-sweep`, optional):** Reconciles leftover `(sync-conflict …)` copies — 30 s warmup, then one pass every 5 minutes. A copy proven identical to its live sibling (equal size **and** equal `content_sha1`) is redundant; anything it cannot prove identical is surfaced to the activity feed and left alone. Governed by `AppConfig.conflict_sweep` / `PDFS_CONFLICT_SWEEP` (`SweepMode`): **report-only by default**, `off` skips the thread entirely, and only `enforce` lets it trash. Because it deletes, the enforcing path re-verifies revision id, size, digest, name, parent, queued ops and open handles immediately before acting rather than trusting its own pass-start snapshot. See `docs/BUGS.md` B69 and B71.
|
6. **Conflict Sweep Thread (`pdfs-conflict-sweep`, optional):** Reconciles leftover `(sync-conflict …)` copies — 30 s warmup, then one pass every 5 minutes. A copy proven identical to its live sibling (equal size **and** equal `content_sha1`) is redundant; anything it cannot prove identical is surfaced to the activity feed and left alone. Governed by `AppConfig.conflict_sweep` / `PDFS_CONFLICT_SWEEP` (`SweepMode`): **report-only by default**, `off` skips the thread entirely, and only `enforce` lets it trash. Because it deletes, the enforcing path re-verifies revision id, size, digest, name, parent, queued ops and open handles immediately before acting rather than trusting its own pass-start snapshot. See `docs/BUGS.md` B69 and B71.
|
||||||
7. **Remote Event Tasks (Tokio):** `run_event_sync` follows Proton's event stream for the Drive volume and applies each change to every mounted inode space, invalidating kernel entries as it goes. The cursor is persisted after every batch, so changes made while the daemon was stopped are applied on the next start. `run_photos_event_sync` follows the photos volume separately; it only drops gallery rows or the timeline's freshness stamp, since photos are not part of the mount.
|
7. **Remote Event Tasks (Tokio):** `run_event_sync` follows Proton's event stream for the Drive volume and applies each change to every mounted inode space, invalidating kernel entries as it goes. The cursor is persisted after every batch, so changes made while the daemon was stopped are applied on the next start. `run_photos_event_sync` follows the photos volume separately; it only drops gallery rows or the timeline's freshness stamp, since photos are not part of the mount.
|
||||||
8. **Event Sampler (`pdfs-events`):** Publishes status, transfer and sync-progress changes to subscribed front-ends (§6).
|
8. **Event Sampler (`pdfs-events`):** Publishes status, transfer and sync-progress changes to subscribed front-ends (§6).
|
||||||
|
|||||||
+1372
-32
File diff suppressed because it is too large.
Load diff
@@ -11,6 +11,161 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [3.0.0] — 2026-10-05
|
||||||
|
|
||||||
|
Schema: **42**. The queue table is rebuilt so an id is never given out twice; queued changes
|
||||||
|
are kept. Queued changes get a column for the reason Drive refused them, and two for the
|
||||||
|
content a new file was sent with and where, and name their file and folder by local id as well
|
||||||
|
as by Drive's id. Stored files and folders name their parent
|
||||||
|
folder by local id too. A file or folder made on this machine that Drive does not have yet goes
|
||||||
|
by its local id, in the queue, the stored tree and pins alike.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- **Sync issues.** A change Drive refuses because the storage is full, access was taken away,
|
||||||
|
its folder is gone, a folder limit is reached or Drive rejected it shows up at once as needing
|
||||||
|
attention, with what it means and what to do, instead of after six failed attempts. The
|
||||||
|
change stays queued and is still retried.
|
||||||
|
- **Export a queued file.** The Sync page's Export button and `pdfs sync export <id> <dest>`
|
||||||
|
save a copy of a queued upload's content. The upload stays queued.
|
||||||
|
- **`pdfs sync issues`** lists the changes that need attention; `pdfs sync queue` shows the
|
||||||
|
issue next to each change.
|
||||||
|
- **Discard a queued change.** The Sync page's Discard button and `pdfs sync discard <id>` drop
|
||||||
|
a change Drive will not take and undo it here: a new file or folder goes, anything else goes
|
||||||
|
back to what Drive has. Export a file first to keep its content.
|
||||||
|
- **Sync issues show on the file.** A file or folder with a change Drive refused carries a
|
||||||
|
warning badge in the file browser, and its tooltip says what it means and what to do.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- **Making, renaming and deleting through the mount no longer waits for Drive.** A new folder,
|
||||||
|
a new file, a rename or a delete is recorded locally and the call returns at once; the queue
|
||||||
|
sends it, as it does offline. `"local_first": false` in `config.json` brings back the old way
|
||||||
|
for this release.
|
||||||
|
- **The queue sends up to 16 changes at once instead of 3.** A thousand new files in a hundred
|
||||||
|
folders reach Drive about three times as fast on a LAN, and four times as fast on Wi-Fi.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- **A file made under the name of a file whose upload is still going out keeps its name.** It
|
||||||
|
no longer lands as a conflict copy, and the mount no longer shows that copy's name (B154).
|
||||||
|
- **A change to a file no longer lands as a conflict copy of itself when the link loses Drive's
|
||||||
|
answer to its upload** (B155).
|
||||||
|
- **A file written again while its upload was being retried no longer lands as a conflict copy
|
||||||
|
of itself when the link lost Drive's answer to the first try** (B156).
|
||||||
|
- **Writing to a file twice in a row no longer fails with "Input/output error" when the link
|
||||||
|
lost Drive's answer to reading back the file's last upload** (B157).
|
||||||
|
- **A file renamed twice in a row, while the first name was still taken, keeps the second name**
|
||||||
|
(B158).
|
||||||
|
- **Opening a file right after the daemon restarts no longer fails with "Resource temporarily
|
||||||
|
unavailable"** (B159).
|
||||||
|
- **A new folder, or a new empty file, reaches Drive at once**, not up to half a minute later
|
||||||
|
(B160, B192).
|
||||||
|
- **A file moved while it was still being uploaded keeps its new name in the mount** on a slow
|
||||||
|
link (B161).
|
||||||
|
- **A folder removed and made again at once keeps what goes into it.** It used to be created
|
||||||
|
inside the removed folder and was trashed along with it (B162).
|
||||||
|
- **A new file no longer reads as empty just after it is uploaded.** Drive can list it a moment
|
||||||
|
before its contents are committed, and the mount took that answer (B163).
|
||||||
|
- **A folder no longer lists a new file twice while it is uploaded.** A listing taken as the
|
||||||
|
upload landed showed it both as Drive had it and as the queue did (B164).
|
||||||
|
- **Stopping the daemon no longer hangs until systemd kills it.** A stop waits up to 10 s for
|
||||||
|
work in progress, then exits and logs what was still running (B165).
|
||||||
|
- **A folder no longer drops or empties a new file while others are uploaded.** A file whose
|
||||||
|
upload landed while the folder was being listed was missing from that listing, or listed as
|
||||||
|
empty (B166).
|
||||||
|
- **The mount comes up faster on a large Drive.** Rebuilding the folder listings from the cache
|
||||||
|
took close to 2 s with 37,000 files at every start (B167).
|
||||||
|
- **Files deleted with their folder while uploading stay deleted.** The mount no longer tries
|
||||||
|
to upload them to the top of your Drive instead (B168).
|
||||||
|
- **Stopping the daemon during a folder sync no longer waits for the sync.** The uploads in
|
||||||
|
progress finish, and the rest waits for the next start (B169).
|
||||||
|
- **Writing to a file again just as its upload finishes no longer fails.** A write right after
|
||||||
|
it failed with "Input/output error" (B170).
|
||||||
|
- **A file deleted while its upload was retrying is removed from Drive right away.** It no
|
||||||
|
longer waits for the upload's next retry (B171).
|
||||||
|
- **A file deleted just as it reached Drive is removed from Drive too.** It no longer stayed
|
||||||
|
there, and the next file with that name no longer became a conflict copy (B173).
|
||||||
|
- **Renaming a file just as it reaches Drive no longer fails with "Permission denied"**
|
||||||
|
(B174).
|
||||||
|
- **A file replaced by a rename just as it reached Drive is removed from Drive too.** It no
|
||||||
|
longer stayed there, and the next rename of its name no longer moved it instead (B175).
|
||||||
|
- **A write closed just as its file reached Drive is no longer lost** (B176).
|
||||||
|
- **Opening a folder just as it reaches Drive no longer fails with "Input/output error"**
|
||||||
|
(B177).
|
||||||
|
- **Deleting a folder while sync is paused no longer leaves a queued change behind** for a
|
||||||
|
file in it that was never uploaded (B178).
|
||||||
|
- **A file read just as it reaches Drive no longer reads empty** (B179).
|
||||||
|
- **Stopping the daemon no longer takes 10 seconds more than it needs** when an idle upload
|
||||||
|
worker missed the stop (B180).
|
||||||
|
- **A folder made while offline or with sync paused no longer fails with "Input/output
|
||||||
|
error"** once it is refreshed or the connection comes back (B125).
|
||||||
|
- **The daemon no longer hangs when a file being read changes on Drive** (B130).
|
||||||
|
- **The Activity page says in your language why an edit was kept as a conflicting copy**,
|
||||||
|
instead of the daemon's English (B181).
|
||||||
|
- **A file written after a restart no longer lands as a conflict copy of itself** when the
|
||||||
|
connection dropped just as it first reached Drive (B182).
|
||||||
|
- **A file deleted after a restart is removed from Drive too** when the connection lost Drive's
|
||||||
|
answer to its upload and the file was written again meanwhile. It no longer stayed there and
|
||||||
|
turned the next file under its name into a conflict copy (B183).
|
||||||
|
- **A file renamed while the connection lost Drive's answer to its upload lands once.** It no
|
||||||
|
longer stayed on Drive under its old name too, or, when deleted, under its old name alone
|
||||||
|
(B184).
|
||||||
|
- **A new file written again just as it starts uploading no longer waits out a retry** before
|
||||||
|
it reaches Drive (B185).
|
||||||
|
- **A file deleted while open can still be read and written until it is closed**, offline too.
|
||||||
|
A write into it no longer failed with "Input/output error" (B186).
|
||||||
|
- **An uploaded file no longer leaves a copy of itself in the cache** until the next start
|
||||||
|
(B187).
|
||||||
|
- **Changes that failed during a short outage go out soon after the connection is back**, not
|
||||||
|
after a retry delay of up to five minutes (B189).
|
||||||
|
- **A file saved just before the daemon stops reaches Drive** with what was written. It could
|
||||||
|
keep its old content, or stay empty (B190).
|
||||||
|
- **A file renamed just as its upload finished keeps its new name.** The mount no longer showed
|
||||||
|
the old name, and a second rename no longer failed with "No such file or directory" (B152,
|
||||||
|
B172).
|
||||||
|
- **A change that waits for a name to be freed no longer waits seconds after it is.** A rename or
|
||||||
|
upload over a file being deleted goes out as soon as the delete lands (B153).
|
||||||
|
- **A file deleted right after it was made no longer stays on Drive when the link lost Drive's
|
||||||
|
answer.** The node Drive made is trashed (B151).
|
||||||
|
- **A new file no longer lands twice when the link loses Drive's answer to its upload.** The
|
||||||
|
file Drive made is taken as it is, instead of a second upload as a conflict copy (B127).
|
||||||
|
- **A change to a file deleted elsewhere, together with its folder, no longer retries forever.**
|
||||||
|
The conflict copy that keeps it now lands in the location's root (B150).
|
||||||
|
- **`pdfs rename` and `pdfs move` right after a change through the mount are no longer undone.**
|
||||||
|
They are queued behind it, and they work on a file that has not reached Drive yet (B149).
|
||||||
|
- **A folder kept offline before it reached Drive stays kept offline.** Its pin was lost when the
|
||||||
|
folder landed (B148).
|
||||||
|
- **A queued change no longer holds up the queue forever on a link that stops answering.**
|
||||||
|
Sending a new folder, a delete or a file waits at most 20 seconds for Drive, plus time for
|
||||||
|
the file's size, and then the queue treats the link as down and tries again.
|
||||||
|
- **A file deleted while its first upload is on the wire no longer stays on Drive.** The node
|
||||||
|
the upload made is trashed as soon as it lands, so its name is free again (B129).
|
||||||
|
- **A delete queued while a rename is sent is no longer lost.** The rename landing could remove
|
||||||
|
the delete from the queue, leaving the file on Drive (B137).
|
||||||
|
- **A file made offline over one deleted offline is no longer lost.** Its upload could take
|
||||||
|
over the deleted file, and the delete then took the new bytes along with it (B128).
|
||||||
|
- **A file deleted while its upload is on the wire no longer comes back as a `recovered-…`
|
||||||
|
copy** when another file is made right after (B133).
|
||||||
|
- **A file made where another one is leaving no longer lands as a conflict copy.** Its upload
|
||||||
|
waits for the rename or trash that frees the name (B126, B138).
|
||||||
|
- **A file closed just as its first upload lands keeps its last write.** The write was refused
|
||||||
|
and left in staging (B139).
|
||||||
|
- **Files made in a new folder are still there after a restart.** A folder that landed as the
|
||||||
|
daemon stopped, or one listed from Drive while files in it were queued, came back empty
|
||||||
|
(B132, B140).
|
||||||
|
- **A file renamed twice in quick succession ends up under the second name.** The first rename
|
||||||
|
could land after the second and undo it (B141).
|
||||||
|
- **Renaming a new file just as its upload lands no longer fails with "Device or resource
|
||||||
|
busy".** The rename goes to the file Drive made (B142).
|
||||||
|
- **A file whose upload landed just as the connection dropped no longer gets a conflict copy at
|
||||||
|
its next save.** The daemon reads the file back once the connection returns (B143).
|
||||||
|
- **A rename whose target name was just freed no longer lands as a conflict copy.** The rename
|
||||||
|
is sent once more under the name it asked for (B144).
|
||||||
|
- **Saving a download under its final name no longer fails with "Device or resource busy".**
|
||||||
|
A file made a moment before was taken for deleted and dropped from the queue (B145).
|
||||||
|
- **A file renamed just as its upload lands keeps its new name.** The tree could take back the
|
||||||
|
name Drive still had (B146).
|
||||||
|
- **A new file made under a name another file just left no longer takes that file's place on
|
||||||
|
Drive.** Both could land on one node, and one file's content was lost (B147).
|
||||||
|
|
||||||
## [2.10.0] — 2026-10-03
|
## [2.10.0] — 2026-10-03
|
||||||
|
|
||||||
Schema: **36**. Every node row gets a local id that never changes; existing rows keep their
|
Schema: **36**. Every node row gets a local id that never changes; existing rows keep their
|
||||||
@@ -1444,6 +1599,8 @@ First stable release: FUSE files-on-demand mount, sync daemon under `proton-driv
|
|||||||
- The outstanding FUSE defects tracked in `docs/BUGS.md`, plus a truncate defect, validated
|
- The outstanding FUSE defects tracked in `docs/BUGS.md`, plus a truncate defect, validated
|
||||||
by a new POSIX compliance suite for the filesystem.
|
by a new POSIX compliance suite for the filesystem.
|
||||||
|
|
||||||
|
[3.0.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v3.0.0
|
||||||
|
|
||||||
[2.10.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v2.10.0
|
[2.10.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v2.10.0
|
||||||
[2.9.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v2.9.0
|
[2.9.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v2.9.0
|
||||||
[2.8.2]: https://github.com/narrrl/proton-drive-linux/releases/tag/v2.8.2
|
[2.8.2]: https://github.com/narrrl/proton-drive-linux/releases/tag/v2.8.2
|
||||||
|
|||||||
@@ -82,6 +82,9 @@ commands are for scripts and for cases where the mount is not available.
|
|||||||
| `pdfs sync resume [--folder ID]` | Resume after a pause. |
|
| `pdfs sync resume [--folder ID]` | Resume after a pause. |
|
||||||
| `pdfs sync limit [--up RATE] [--down RATE]` | Cap bandwidth, for example `500K` or `2M` per second. `0` removes a cap. Without options, prints the current caps. |
|
| `pdfs sync limit [--up RATE] [--down RATE]` | Cap bandwidth, for example `500K` or `2M` per second. `0` removes a cap. Without options, prints the current caps. |
|
||||||
| `pdfs sync queue` | List changes that have not reached Proton Drive yet. |
|
| `pdfs sync queue` | List changes that have not reached Proton Drive yet. |
|
||||||
|
| `pdfs sync issues` | List queued changes Proton Drive refused or that keep failing, and what to do. |
|
||||||
|
| `pdfs sync export ID DEST` | Save a copy of a queued upload's content. The upload stays queued. |
|
||||||
|
| `pdfs sync discard ID [--yes]` | Drop a queued change and undo it here: a new file or folder goes, anything else goes back to what Proton Drive has. Content not exported first is lost. |
|
||||||
| `pdfs sync retry [ID]` | Retry one queued operation, or every failed one. |
|
| `pdfs sync retry [ID]` | Retry one queued operation, or every failed one. |
|
||||||
| `pdfs sync restore [--device UID] [--yes]` | Re-attach this computer's backed-up folders to local folders and download them. `--device` restores another computer's folders. See [RECOVERY.md](RECOVERY.md). |
|
| `pdfs sync restore [--device UID] [--yes]` | Re-attach this computer's backed-up folders to local folders and download them. `--device` restores another computer's folders. See [RECOVERY.md](RECOVERY.md). |
|
||||||
|
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ fix it.
|
|||||||
| `ignore_patterns` | list | see below | — | Global ignore rules for synced folders |
|
| `ignore_patterns` | list | see below | — | Global ignore rules for synced folders |
|
||||||
| `device_uid` | string | none | Computers → Continue This Backup Here… | The computer this machine continues. Unset means "match by hostname" |
|
| `device_uid` | string | none | Computers → Continue This Backup Here… | The computer this machine continues. Unset means "match by hostname" |
|
||||||
| `conflict_sweep` | `off` / `report` / `enforce` | `report` | — | See [`conflict_sweep`](#conflict_sweep) |
|
| `conflict_sweep` | `off` / `report` / `enforce` | `report` | — | See [`conflict_sweep`](#conflict_sweep) |
|
||||||
|
| `local_first` | bool | `true` | — | See [`local_first`](#local_first) |
|
||||||
| `language` | gettext code | system | General → Appearance | Language of the app, tray and prompt, such as `de` or `pt_BR` |
|
| `language` | gettext code | system | General → Appearance | Language of the app, tray and prompt, such as `de` or `pt_BR` |
|
||||||
| `proton_theme` | bool | follow system | General → Appearance | Proton's colours instead of the system theme |
|
| `proton_theme` | bool | follow system | General → Appearance | Proton's colours instead of the system theme |
|
||||||
| `online_map` | bool | `false` | General → Appearance | Street map for Places from OpenFreeMap |
|
| `online_map` | bool | `false` | General → Appearance | Street map for Places from OpenFreeMap |
|
||||||
@@ -116,6 +117,15 @@ were copied from.
|
|||||||
|
|
||||||
`PDFS_CONFLICT_SWEEP` overrides the setting for one run of the service.
|
`PDFS_CONFLICT_SWEEP` overrides the setting for one run of the service.
|
||||||
|
|
||||||
|
### `local_first`
|
||||||
|
|
||||||
|
A new folder, a new file, a rename or a delete made through the mount is recorded locally and the
|
||||||
|
call returns at once. The change goes to Drive from the queue, the same way it does offline, and
|
||||||
|
`pdfs status` counts it until it lands.
|
||||||
|
|
||||||
|
Set it to `false` to have these calls wait for Drive again, as before 3.0.0. The switch is there
|
||||||
|
for one release, in case the queued path misbehaves for you; please report it if it does.
|
||||||
|
|
||||||
### `open_with`
|
### `open_with`
|
||||||
|
|
||||||
Decides how files picked in the search launcher or the app are opened. Rules are tried in order;
|
Decides how files picked in the search launcher or the app are opened. Rules are tried in order;
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
# 3.0.0: what is done and what is left
|
||||||
|
|
||||||
|
Tracks `docs/MILESTONE-3.0.0.md`. Update it with every commit that moves a phase.
|
||||||
|
|
||||||
|
## Done
|
||||||
|
|
||||||
|
- **Phase 0**: fake Drive, simulation runs, CI job (2.9.0).
|
||||||
|
- **Phase 1**: queued-op preconditions and own sealed revisions in the database (2.9.0).
|
||||||
|
- **Phase 2**: local ids, inode = local id, one tree for every mount (2.10.0).
|
||||||
|
- **Phase 3, first part**: mkdir, create, rename, unlink and rmdir on the mount are recorded
|
||||||
|
locally and sent from the queue; `"local_first": false` brings back the old path (5e0f30c).
|
||||||
|
- Fixes from the simulation and account runs since then: B125, B127, B130 and B141 to B193.
|
||||||
|
- **Sync issues, first cut** (§6.2): refusals sorted into issues, shown at once in the app,
|
||||||
|
tray and CLI; Export button and `pdfs sync export`; `pdfs sync issues`; Discard button and
|
||||||
|
`pdfs sync discard`; a warning badge on the node in the file browser.
|
||||||
|
|
||||||
|
## Phase 3: left for 3.0.0
|
||||||
|
|
||||||
|
1. **Sync issues (§6.2)**: first cut done.
|
||||||
|
- [x] The drain sorts a refusal from Drive into an issue kind (quota, access, missing,
|
||||||
|
limit, rejected) and stores it on the queued op (schema 38).
|
||||||
|
- [x] An op with an issue counts as "needs attention" at once, not after six failures.
|
||||||
|
- [x] The queue listing carries the issue kind and whether its content can be exported.
|
||||||
|
- [x] Export a queued file's content (control request, `pdfs sync export`).
|
||||||
|
- [x] `pdfs sync issues`, and the issue shown in `pdfs sync queue`.
|
||||||
|
- [x] The app's Sync page shows each issue with what the user can do; Export and Retry
|
||||||
|
buttons; translated in every catalog.
|
||||||
|
- [x] A name collision lands under a conflict name (create, rename, move, re-homed create)
|
||||||
|
and goes in the activity log, as does a queued edit kept as a conflict copy. The app
|
||||||
|
words each one in the user's language (the last one since B181).
|
||||||
|
- [x] Discard an issue on purpose (after export, or for a change with no content), and
|
||||||
|
undo the local change it stood for: a node made here goes, a node Drive has goes back
|
||||||
|
to what Drive has. Control request, `pdfs sync discard`, Discard button with a
|
||||||
|
confirmation; translated in every catalog.
|
||||||
|
- [x] Show the issue on the node itself: a folder listing carries each node's issue, and
|
||||||
|
the file browser shows a warning badge with what it means. The listing is published
|
||||||
|
again when the set of issues changes.
|
||||||
|
2. **Planner and executor (§5.4, §5.5)**: ops keyed by local id; remove `mint_local_uid`,
|
||||||
|
`finish_create` and `adopt_real_uid`; a landing create sets `remote_uid` on its row.
|
||||||
|
- [x] Queued ops carry `lid` and `parent_lid` (schema 39). The claim reads the parent's uid
|
||||||
|
through `parent_lid`, and a landing folder rewrites the ops inside it by local id.
|
||||||
|
- [x] Node rows link to their parent by lid (`nodes.parent_lid`, schema 40), and a restart
|
||||||
|
places each node under its parent's row as it is now.
|
||||||
|
- [x] Queries over children, subtrees and ancestors by `parent_lid`, and no rewrite of the
|
||||||
|
children's `parent_uid` when a folder lands (R3), nor of the ops made inside it (R2).
|
||||||
|
- [x] No more placeholders: a node made here goes by its local id until it lands. In four
|
||||||
|
parts:
|
||||||
|
- [x] 6a: a node made here goes by `local~<lid>`, a function of its row, instead of a
|
||||||
|
minted `local~<ms>-<n>`; schema 41 rewrites the old placeholders (nodes, ops, pins).
|
||||||
|
- [x] 6b: an op is found by its node's local id under either uid, the claim keeps one op
|
||||||
|
per local id on the wire, and the drain sends the uid the row has now;
|
||||||
|
`remap_local_uid` is gone.
|
||||||
|
- [x] 6c: children, subtrees, ancestors and ops' parents by lid, so a landing create
|
||||||
|
rewrites no other row or op; what is read for a uid names the parent as its row has it
|
||||||
|
now. The row keeps its stand-in in `nodes.uid` rather than `NULL`: it is as stable,
|
||||||
|
and `NULL` would have changed some 40 node queries for no change in behaviour. Pins
|
||||||
|
stay keyed by uid and move with the row in the landing transaction (B148).
|
||||||
|
- [x] 6d: a node written under its stand-in after it landed updates its landed row
|
||||||
|
instead of bringing the placeholder row back, so a write that finds its create
|
||||||
|
landed takes the uid from the row and no longer polls the tree for up to 2 s
|
||||||
|
(`landed_uid` is gone). In memory, `pending`, the uploads and the tree stay keyed by
|
||||||
|
the uid Drive knows: ops are matched by lid in the database (6b), `hidden` and
|
||||||
|
`creating` only ever hold remote uids, and `landed_row_uid` is the one place at the
|
||||||
|
mount's edge where a stand-in a handle still holds becomes the real uid.
|
||||||
|
- [x] An executor that runs ops in dependency order, many at once: 16 drain workers instead
|
||||||
|
of 3, and a worker that lands an op wakes the idle ones, which slept through the ops it
|
||||||
|
made claimable. The claim already keeps the order: a create waits for its parent's
|
||||||
|
uid, a node's ops go one at a time. Ops stay blocking calls on worker threads: each
|
||||||
|
one spends its time waiting on Drive, so tasks would buy nothing threads do not.
|
||||||
|
3. **Speed (§7)**: run independent ops in parallel, batch trashes and moves; measure 1,000 files
|
||||||
|
and 100 folders on Wi-Fi against LAN (target: within 10 %).
|
||||||
|
- [x] The measurement: `a_thousand_files_drain_as_fast_on_wifi_as_on_lan` in `sim/daemon.rs`,
|
||||||
|
run with `PDFS_SIM_MEASURE=1`. It prints the times and asserts nothing yet.
|
||||||
|
- [x] Baseline on eb1c105, three drain threads: the syscalls take 3.7 s on LAN and 4.3 s on
|
||||||
|
Wi-Fi; the drain takes 138 s on LAN and 938 s on Wi-Fi, 6.8 times as long.
|
||||||
|
- [x] Sixteen workers: the syscalls take 5.2 s on LAN and 5.0 s on Wi-Fi, which meets the
|
||||||
|
target; the drain takes 47 s on LAN and 247 s on Wi-Fi. Most of a create on Wi-Fi is
|
||||||
|
its read-back waiting for Drive to list the new node (250 ms, then 1 s), so batching
|
||||||
|
(step 8) helps trashes and moves, not creates.
|
||||||
|
- [x] Batched trashes and moves: left for after 3.0.0 (see below). The target is met
|
||||||
|
without them, the drain waits on the creates' read-back, and splitting one answer into
|
||||||
|
many results is where the B15x to B17x races lived.
|
||||||
|
4. **Done-when checks**:
|
||||||
|
- [ ] Simulation passes on the Wi-Fi profile: `one_client_on_a_slow_link` failed in CI on
|
||||||
|
seeds 2 and 3. Two bugs (B127, B151) and a harness gap: a settle did not read the log, so
|
||||||
|
it could not explain a known bug's damage. All three fixed; not yet confirmed in CI.
|
||||||
|
Replays then found two races on the LAN profile too (B152, B153), both fixed. CI on
|
||||||
|
7269cb5 failed LAN seed 9, a regression from the B153 fix (B154), and Wi-Fi seed 2,
|
||||||
|
a write whose answer was lost (B155). Both fixed. One slow-link run of seed 2 failed
|
||||||
|
once with a rename answering `ENOENT` as a folder move landed. Not reproduced since;
|
||||||
|
it may be B152. Replays of slow-link seed 4 failed two of six times on a file written
|
||||||
|
again after its create lost its answer (B156, fixed). With sixteen drain workers
|
||||||
|
(16fca10), seed 4 failed every replay on a write after an upload that was not read
|
||||||
|
back (B157, fixed). Seed 3 then failed four of ten runs on two renames of one file
|
||||||
|
landing out of order (B158, fixed). Flaky-link seed 4 failed one of ten runs on a
|
||||||
|
lookup that answered `EAGAIN` while the root was listed again after a restart (B159,
|
||||||
|
fixed). Slow-link seed 4 also had folders reach Drive 30 s late, after the
|
||||||
|
drain's idle poll (B160, fixed). Slow-link seed 3 failed on a file moved while its
|
||||||
|
create was on the wire, which the mount showed under its old name (B161, fixed).
|
||||||
|
Seed 4 failed on a folder removed and made again, whose create adopted the removed
|
||||||
|
folder (B162, fixed). CI on 903452a passed every profile. CI on f9b1638 failed
|
||||||
|
slow-link seed 2: a step that unlinks an open file, appends and reads it back hung
|
||||||
|
for 150 s. The B130 matcher did not fire, and CI kept no stacks; one replay passed.
|
||||||
|
It may be a kernel notice sent from a runtime worker by another path than B130's,
|
||||||
|
which a68dae9 (not yet pushed then) also takes off the runtime. CI now keeps a hung
|
||||||
|
run's stacks.
|
||||||
|
- [x] Known-bug list in `sim/run.rs` is empty: the run on 812eb3d (`PDFS_SIM_KNOWN=fail`,
|
||||||
|
twelve seeds) passed every profile, so the matchers are gone. That confirms B125 to
|
||||||
|
B133; B134 stays open with `local_first` off only, which the runs do not mount. The confirmation run (`PDFS_SIM_KNOWN=fail`)
|
||||||
|
on da013c2 failed `one_client_on_a_flaky_link` seeds 6 and 11 on two new bugs, both
|
||||||
|
after restarts: a write over a create never read back (B182) and a trash that forgot
|
||||||
|
what its create sent (B183). Replays with those fixed found a third: a create renamed
|
||||||
|
after it lost its answer (B184). All three fixed; no known-bug matcher fired. It was
|
||||||
|
stopped before `three_clients`. The rerun on 3073052 found a create that waited out
|
||||||
|
a retry after a write replaced its blob (B185, fixed), and a file deleted offline
|
||||||
|
while open that lost the bytes its handle read (B186, fixed; older than 3.0.0).
|
||||||
|
The run on 2baad67 found that a restart before such a file's trash went out queued
|
||||||
|
its kept bytes again, for good (B188, fixed), and changes that sat out their retry
|
||||||
|
after an outage only the drain had met (B189, fixed; older than 3.0.0), and a file
|
||||||
|
saved just before a stop whose release the stop dropped (B190, fixed; older too). The run
|
||||||
|
on 1436c5f found a file deleted after its failed create was written again left on Drive
|
||||||
|
(B191, fixed). Its test for B193 found a new empty file waiting out the idle poll (B192,
|
||||||
|
fixed), and seed 6 a deleted file's trash taking a file of the user's renamed to the name
|
||||||
|
its create was sent under (B193, fixed).
|
||||||
|
- [ ] A clean account run. The one on a6f36d5 (2026-10-04, LAN: 427 passed, 12 failed)
|
||||||
|
failed B113 and B121 on every on-demand mount, two mirror-to-mirror move cases, and
|
||||||
|
the journal check. B121 was a bug (B149, fixed); B113's case still expected creates
|
||||||
|
to reach Drive at once and now accepts queued ones. The move cases were the harness:
|
||||||
|
it retried every source after one was busy, and expected a refusal where a mirror to
|
||||||
|
mirror move carries the local copy along. The journal showed a conflict copy retrying
|
||||||
|
into a deleted folder (B150, fixed). Not confirmed yet. The run on 2.10.0
|
||||||
|
(2026-10-04: 260 passed, 3 failed) failed a new file that read back empty (B163,
|
||||||
|
fixed), a 128-entry folder that listed 129 (B164, fixed), and its cleanup, which ran
|
||||||
|
while the daemon was restarting: the stop waited on a busy thread until systemd killed
|
||||||
|
it (B165, bounded). The next run on the working tree (2026-10-04: 259 passed, 4
|
||||||
|
failed) found a wide folder dropping files whose create landed while it was listed
|
||||||
|
(B166, fixed), or listing one empty: the new file that read back empty again
|
||||||
|
(B166 too). Its cleanup found no control socket because it ran 1.5 s after the
|
||||||
|
last restart, before the new daemon had bound it: `pdfs status` reports that as not
|
||||||
|
running, which the run's queue wait takes for an empty queue (the script's side).
|
||||||
|
Looking into it found the start spending 1.7 s rebuilding listings (B167, fixed).
|
||||||
|
Its journal check caught a file removed with its folder while uploading being
|
||||||
|
sent again, to the root (B168, fixed), and a stop that let a mirror pass go on
|
||||||
|
until the runtime ended under it (B169, fixed). CI on 9bbc994 failed
|
||||||
|
`one_client_on_a_good_link`: a write opened as an upload landed took the revision
|
||||||
|
it replaced as its base, and the next write failed with `EIO` (B170, fixed). The
|
||||||
|
suite's seed 9 found a deleted file's trash waiting out its create's backoff (B171,
|
||||||
|
fixed). Run three at a time on 2502af6, seed 8 failed two of nine runs: a file renamed
|
||||||
|
while its upload was read back showed its old name again (B172, fixed). The loop
|
||||||
|
on d850265 failed seed 12 once in nine runs: a file deleted just as its create
|
||||||
|
landed stayed on Drive (B173, fixed). The suite on 378667f failed seed 6: a rename
|
||||||
|
in the same gap was refused with `EACCES` (B174, fixed). The loop on 378667f failed
|
||||||
|
seed 8: a file replaced by a rename in that gap stayed on Drive (B175, fixed), and
|
||||||
|
seed 12: a write closed as its file's create landed was dropped (B176, fixed).
|
||||||
|
The account run on 378667f: a folder listed as its create landed asked Drive by
|
||||||
|
its stand-in and answered `EIO` (B177, fixed). A folder trashed while sync was
|
||||||
|
paused kept a create below it as a trash: a paused drain worker had claimed it
|
||||||
|
(B178, fixed). A file read as its create landed read empty (B179, fixed).
|
||||||
|
The suite on 0f3c06f: a stop gave up on a drain worker that slept through its
|
||||||
|
wake-up (B180, fixed). The account run on 903452a (2026-10-04, Wi-Fi: 262 passed,
|
||||||
|
1 failed) failed only its cleanup, which found no control socket: after the mirror
|
||||||
|
target's last restart the queue wait took the missing daemon for an empty queue
|
||||||
|
(the script's side, fixed in 6539c73).
|
||||||
|
5. **Release**: 2.10.0 is tagged; the 2.9.0 tag still waits for the user's go-ahead.
|
||||||
|
|
||||||
|
## After 3.0.0
|
||||||
|
|
||||||
|
- Remove the old online path in `filesystem.rs` and the `local_first` switch (the release after
|
||||||
|
3.0.0).
|
||||||
|
- **Phase 4**: one applier for remote changes.
|
||||||
|
- **Phase 5**: mirror folders on the same planner and executor.
|
||||||
|
- **Batched trashes and moves** (plan step 8). Not needed for the §7 target: the syscalls take
|
||||||
|
5.2 s on LAN and 5.0 s on Wi-Fi. The drain's time goes to the creates' read-back (250 ms,
|
||||||
|
then 1 s each), which batching does not shorten. A batch call answers for many ops at once,
|
||||||
|
and splitting that answer back per op is the step the B15x to B17x races came from. First
|
||||||
|
measure `rm -r` of 1,000 files on Wi-Fi. If it is slow, drop the queued trashes of a trashed
|
||||||
|
folder's children before batching anything: trashing the folder takes them with it.
|
||||||
|
- **Any time**: `Size` and block geometry as types; a deadline on the remote calls outside
|
||||||
|
the drain (the drain's have one: `link::upload_deadline`, `Core::block_on_within`).
|
||||||
|
- Open questions in §12 of the milestone.
|
||||||
|
- `PLAN.md`: the acceptance script notices network drops and reports `interrupted` instead of
|
||||||
|
TIMEOUT.
|
||||||
+28
-1
@@ -1,15 +1,42 @@
|
|||||||
# Roadmap
|
# Roadmap
|
||||||
|
|
||||||
Planned work and open verification, as of 2.5.0. Individual defects and their status live in
|
Planned work and open verification, as of 3.0.0. Individual defects and their status live in
|
||||||
[BUGS.md](BUGS.md), which is the authoritative ledger; this page groups what is left by theme.
|
[BUGS.md](BUGS.md), which is the authoritative ledger; this page groups what is left by theme.
|
||||||
Nothing here is a promise of a date.
|
Nothing here is a promise of a date.
|
||||||
|
|
||||||
|
- [Sync engine](#sync-engine)
|
||||||
- [Data safety and recovery](#data-safety-and-recovery)
|
- [Data safety and recovery](#data-safety-and-recovery)
|
||||||
- [Features](#features)
|
- [Features](#features)
|
||||||
- [Performance and correctness](#performance-and-correctness)
|
- [Performance and correctness](#performance-and-correctness)
|
||||||
- [Verification still owed](#verification-still-owed)
|
- [Verification still owed](#verification-still-owed)
|
||||||
- [Release assurance](#release-assurance)
|
- [Release assurance](#release-assurance)
|
||||||
|
|
||||||
|
## Sync engine
|
||||||
|
|
||||||
|
3.0.0 records changes made through the mount locally and sends them from the queue
|
||||||
|
([MILESTONE-3.0.0.md](MILESTONE-3.0.0.md)). What comes after it:
|
||||||
|
|
||||||
|
| Item | State | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| Remove the online path | Not started | The old way of sending a change before the syscall returns, and the `local_first` switch that brings it back, go in the release after 3.0.0 |
|
||||||
|
| One applier for remote changes | Not started | Phase 4 of the milestone: a remote tree, and remote changes applied through the same planner as local ones |
|
||||||
|
| Synced folders on the queue | Not started | Phase 5: synced folders sent by the same planner and executor as the mount |
|
||||||
|
| Batch trashes and moves | Deferred | See below |
|
||||||
|
|
||||||
|
**Batching trashes and moves.** `trash_nodes` and `move_nodes_streaming` take up to 150 nodes per
|
||||||
|
call, but the queue still sends one node per call. It was left out of 3.0.0 on purpose:
|
||||||
|
|
||||||
|
- The speed target of the milestone (§7) is met without it. Making 1,000 files in 100 folders
|
||||||
|
takes 5.2 s on a LAN and 5.0 s on Wi-Fi.
|
||||||
|
- What the queue spends its time on is the creates. Each one waits for Drive to list the new node
|
||||||
|
before it counts as landed (250 ms, then 1 s), and batching does not help with that.
|
||||||
|
- Sending many ops in one call means splitting one answer back into many results. Races in
|
||||||
|
exactly that step were the largest class of bugs found for 3.0.0 (B15x to B17x).
|
||||||
|
|
||||||
|
Before building it, measure `rm -r` of a folder with 1,000 files on Wi-Fi. If that is slow, the
|
||||||
|
cheaper fix is to drop the queued trashes of the files in a folder when the folder itself is
|
||||||
|
trashed, since trashing the folder takes its contents with it.
|
||||||
|
|
||||||
## Data safety and recovery
|
## Data safety and recovery
|
||||||
|
|
||||||
| Item | State | Notes |
|
| Item | State | Notes |
|
||||||
|
|||||||
@@ -128,7 +128,13 @@ pdfs sync retry 42 # one operation
|
|||||||
```
|
```
|
||||||
|
|
||||||
The app shows the same list on **Sync → Overview**. An operation that keeps failing shows its
|
The app shows the same list on **Sync → Overview**. An operation that keeps failing shows its
|
||||||
error there.
|
error there. A change Proton Drive will not take can be dropped instead, which undoes it here;
|
||||||
|
save a file's content first if you want to keep it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pdfs sync export 42 ~/Desktop/
|
||||||
|
pdfs sync discard 42
|
||||||
|
```
|
||||||
|
|
||||||
**A file in a synced folder does not upload** — check that:
|
**A file in a synced folder does not upload** — check that:
|
||||||
|
|
||||||
|
|||||||
@@ -167,6 +167,10 @@ The version you do not keep goes to the Trash, so a wrong choice can be undone.
|
|||||||
- **Limit bandwidth** under **Preferences → General → Network**, or with
|
- **Limit bandwidth** under **Preferences → General → Network**, or with
|
||||||
`pdfs sync limit --up 2M --down 10M`. `0` removes a limit.
|
`pdfs sync limit --up 2M --down 10M`. `0` removes a limit.
|
||||||
- **Retry** a failed upload from **Sync → Overview**, or with `pdfs sync retry`.
|
- **Retry** a failed upload from **Sync → Overview**, or with `pdfs sync retry`.
|
||||||
|
- **Discard** a change Proton Drive refused from the same list, or with `pdfs sync discard`: it is
|
||||||
|
undone here and never sent. Export a file first to keep your version.
|
||||||
|
- A file or folder with a change Proton Drive refused carries a warning badge in **Files**; hover
|
||||||
|
it to see why.
|
||||||
|
|
||||||
## Photos
|
## Photos
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@
|
|||||||
# Build + install: cd packaging && makepkg -fi
|
# Build + install: cd packaging && makepkg -fi
|
||||||
|
|
||||||
pkgname=proton-drive-linux
|
pkgname=proton-drive-linux
|
||||||
pkgver=2.10.0
|
pkgver=3.0.0
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client for Linux: files-on-demand FUSE mount, CLI, GTK4 app + tray"
|
pkgdesc="Unofficial Proton Drive client for Linux: files-on-demand FUSE mount, CLI, GTK4 app + tray"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
pkgbase = proton-drive-for-linux-bin
|
pkgbase = proton-drive-for-linux-bin
|
||||||
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)
|
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)
|
||||||
pkgver = 2.8.2
|
pkgver = 2.10.0
|
||||||
pkgrel = 2
|
pkgrel = 1
|
||||||
url = https://proton-drive.narl.io
|
url = https://proton-drive.narl.io
|
||||||
arch = x86_64
|
arch = x86_64
|
||||||
license = MIT
|
license = MIT
|
||||||
@@ -21,21 +21,21 @@ pkgbase = proton-drive-for-linux-bin
|
|||||||
optdepends = xdg-utils: "Open folder" action in the tray menu
|
optdepends = xdg-utils: "Open folder" action in the tray menu
|
||||||
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
||||||
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
||||||
provides = proton-drive-for-linux=2.8.2
|
provides = proton-drive-for-linux=2.10.0
|
||||||
provides = proton-drive-linux=2.8.2
|
provides = proton-drive-linux=2.10.0
|
||||||
provides = pdfs
|
provides = pdfs
|
||||||
conflicts = proton-drive-linux
|
conflicts = proton-drive-linux
|
||||||
conflicts = proton-drive-for-linux
|
conflicts = proton-drive-for-linux
|
||||||
conflicts = proton-drive-for-linux-git
|
conflicts = proton-drive-for-linux-git
|
||||||
options = !strip
|
options = !strip
|
||||||
options = !debug
|
options = !debug
|
||||||
source = proton-drive-linux-2.8.2-x86_64.tar.gz::https://github.com/narrrl/proton-drive-linux/releases/download/v2.8.2/proton-drive-linux-2.8.2-x86_64.tar.gz
|
source = proton-drive-linux-2.10.0-x86_64.tar.gz::https://github.com/narrrl/proton-drive-linux/releases/download/v2.10.0/proton-drive-linux-2.10.0-x86_64.tar.gz
|
||||||
source = io.narl.proton-drive-linux-2.8.2.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.8.2/packaging/io.narl.proton-drive-linux.desktop
|
source = io.narl.proton-drive-linux-2.10.0.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.10.0/packaging/io.narl.proton-drive-linux.desktop
|
||||||
source = io.narl.proton-drive-linux-tray-2.8.2.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.8.2/packaging/io.narl.proton-drive-linux-tray.desktop
|
source = io.narl.proton-drive-linux-tray-2.10.0.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.10.0/packaging/io.narl.proton-drive-linux-tray.desktop
|
||||||
source = io.narl.proton-drive-linux-2.8.2.svg::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.8.2/packaging/io.narl.proton-drive-linux.svg
|
source = io.narl.proton-drive-linux-2.10.0.svg::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.10.0/packaging/io.narl.proton-drive-linux.svg
|
||||||
source = proton-drive-2.8.2.service::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.8.2/packaging/proton-drive.service
|
source = proton-drive-2.10.0.service::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.10.0/packaging/proton-drive.service
|
||||||
source = LICENSE-2.8.2::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.8.2/LICENSE
|
source = LICENSE-2.10.0::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v2.10.0/LICENSE
|
||||||
sha256sums = c0af4ff57dc6f587b1b8431f3180b00d45211389f3dc675569af45f64e89cb36
|
sha256sums = b653dbd7d5939c504c55a91e14f1cd3e38aabc94d8bbd6344d2f79b3cd670313
|
||||||
sha256sums = d190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123
|
sha256sums = d190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123
|
||||||
sha256sums = 1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0
|
sha256sums = 1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0
|
||||||
sha256sums = e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828
|
sha256sums = e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
# Maintainer: Nils Pukropp <contact@narl.io>
|
# Maintainer: Nils Pukropp <contact@narl.io>
|
||||||
|
|
||||||
pkgname=proton-drive-for-linux-bin
|
pkgname=proton-drive-for-linux-bin
|
||||||
pkgver=2.8.2
|
pkgver=2.10.0
|
||||||
pkgrel=2
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)"
|
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
url="https://proton-drive.narl.io"
|
url="https://proton-drive.narl.io"
|
||||||
@@ -34,7 +34,7 @@ source=("proton-drive-linux-$pkgver-x86_64.tar.gz::$_repo/releases/download/$_ta
|
|||||||
"io.narl.proton-drive-linux-$pkgver.svg::$_raw/packaging/io.narl.proton-drive-linux.svg"
|
"io.narl.proton-drive-linux-$pkgver.svg::$_raw/packaging/io.narl.proton-drive-linux.svg"
|
||||||
"proton-drive-$pkgver.service::$_raw/packaging/proton-drive.service"
|
"proton-drive-$pkgver.service::$_raw/packaging/proton-drive.service"
|
||||||
"LICENSE-$pkgver::$_raw/LICENSE")
|
"LICENSE-$pkgver::$_raw/LICENSE")
|
||||||
sha256sums=('c0af4ff57dc6f587b1b8431f3180b00d45211389f3dc675569af45f64e89cb36'
|
sha256sums=('b653dbd7d5939c504c55a91e14f1cd3e38aabc94d8bbd6344d2f79b3cd670313'
|
||||||
'd190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123'
|
'd190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123'
|
||||||
'1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0'
|
'1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0'
|
||||||
'e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828'
|
'e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828'
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
pkgbase = proton-drive-for-linux-git
|
pkgbase = proton-drive-for-linux-git
|
||||||
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)
|
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)
|
||||||
pkgver = 2.8.2.r0.g50483bf
|
pkgver = 2.10.0.r6.ga6f36d5
|
||||||
pkgrel = 1
|
pkgrel = 1
|
||||||
url = https://proton-drive.narl.io
|
url = https://proton-drive.narl.io
|
||||||
arch = x86_64
|
arch = x86_64
|
||||||
@@ -24,8 +24,8 @@ pkgbase = proton-drive-for-linux-git
|
|||||||
optdepends = xdg-utils: "Open folder" action in the tray menu
|
optdepends = xdg-utils: "Open folder" action in the tray menu
|
||||||
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
||||||
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
||||||
provides = proton-drive-for-linux=2.8.2.r0.g50483bf
|
provides = proton-drive-for-linux=2.10.0.r6.ga6f36d5
|
||||||
provides = proton-drive-linux=2.8.2.r0.g50483bf
|
provides = proton-drive-linux=2.10.0.r6.ga6f36d5
|
||||||
provides = pdfs
|
provides = pdfs
|
||||||
conflicts = proton-drive-linux
|
conflicts = proton-drive-linux
|
||||||
conflicts = proton-drive-for-linux
|
conflicts = proton-drive-for-linux
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Maintainer: Nils Pukropp <contact@narl.io>
|
# Maintainer: Nils Pukropp <contact@narl.io>
|
||||||
|
|
||||||
pkgname=proton-drive-for-linux-git
|
pkgname=proton-drive-for-linux-git
|
||||||
pkgver=2.8.2.r0.g50483bf
|
pkgver=2.10.0.r6.ga6f36d5
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)"
|
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
pkgbase = proton-drive-for-linux
|
pkgbase = proton-drive-for-linux
|
||||||
pkgdesc = Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray
|
pkgdesc = Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray
|
||||||
pkgver = 2.8.2
|
pkgver = 2.10.0
|
||||||
pkgrel = 1
|
pkgrel = 1
|
||||||
url = https://proton-drive.narl.io
|
url = https://proton-drive.narl.io
|
||||||
arch = x86_64
|
arch = x86_64
|
||||||
@@ -23,13 +23,13 @@ pkgbase = proton-drive-for-linux
|
|||||||
optdepends = xdg-utils: "Open folder" action in the tray menu
|
optdepends = xdg-utils: "Open folder" action in the tray menu
|
||||||
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
||||||
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
||||||
provides = proton-drive-linux=2.8.2
|
provides = proton-drive-linux=2.10.0
|
||||||
provides = pdfs
|
provides = pdfs
|
||||||
conflicts = proton-drive-linux
|
conflicts = proton-drive-linux
|
||||||
conflicts = proton-drive-for-linux-bin
|
conflicts = proton-drive-for-linux-bin
|
||||||
conflicts = proton-drive-for-linux-git
|
conflicts = proton-drive-for-linux-git
|
||||||
options = !lto
|
options = !lto
|
||||||
source = proton-drive-for-linux-2.8.2.tar.gz::https://github.com/narrrl/proton-drive-linux/archive/refs/tags/v2.8.2.tar.gz
|
source = proton-drive-for-linux-2.10.0.tar.gz::https://github.com/narrrl/proton-drive-linux/archive/refs/tags/v2.10.0.tar.gz
|
||||||
sha256sums = 7f2df1d2748c3d8daaa4c05d242fcd293cb1983572e44bb61a35188dbe5f342c
|
sha256sums = 7aa5fcb76dca13262bd678c6f280795ab331a20d5549c4aa7d2c3d4eec6fcd8e
|
||||||
|
|
||||||
pkgname = proton-drive-for-linux
|
pkgname = proton-drive-for-linux
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# Maintainer: Nils Pukropp <contact@narl.io>
|
# Maintainer: Nils Pukropp <contact@narl.io>
|
||||||
|
|
||||||
pkgname=proton-drive-for-linux
|
pkgname=proton-drive-for-linux
|
||||||
pkgver=2.8.2
|
pkgver=2.10.0
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray"
|
pkgdesc="Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
@@ -24,7 +24,7 @@ conflicts=('proton-drive-linux' 'proton-drive-for-linux-bin' 'proton-drive-for-l
|
|||||||
# LTO has broken GTK/Rust links for this workspace; the release profile keeps it off.
|
# LTO has broken GTK/Rust links for this workspace; the release profile keeps it off.
|
||||||
options=('!lto')
|
options=('!lto')
|
||||||
source=("$pkgname-$pkgver.tar.gz::$_repo/archive/refs/tags/v$pkgver.tar.gz")
|
source=("$pkgname-$pkgver.tar.gz::$_repo/archive/refs/tags/v$pkgver.tar.gz")
|
||||||
sha256sums=('7f2df1d2748c3d8daaa4c05d242fcd293cb1983572e44bb61a35188dbe5f342c')
|
sha256sums=('7aa5fcb76dca13262bd678c6f280795ab331a20d5549c4aa7d2c3d4eec6fcd8e')
|
||||||
|
|
||||||
_srcname="proton-drive-linux-$pkgver"
|
_srcname="proton-drive-linux-$pkgver"
|
||||||
|
|
||||||
|
|||||||
+320
-165
File diff suppressed because it is too large.
Load diff
+304
-166
File diff suppressed because it is too large.
Load diff
+327
-165
File diff suppressed because it is too large.
Load diff
+53
-14
@@ -1658,6 +1658,10 @@ def test_regression_b113_move_before_upload(ctx: Context) -> None:
|
|||||||
to an empty original. Pausing sync holds the bytes back while the files
|
to an empty original. Pausing sync holds the bytes back while the files
|
||||||
move, which is the order the bug needs. The names must not be transient
|
move, which is the order the bug needs. The names must not be transient
|
||||||
ones like `*.part`: those mint nothing and queue their create instead (B70).
|
ones like `*.part`: those mint nothing and queue their create instead (B70).
|
||||||
|
|
||||||
|
Since changes through the mount are queued (local-first), the create is
|
||||||
|
queued as well and the moves rewrite it before anything reaches Drive. The
|
||||||
|
files must land where they were moved, without conflict copies, either way.
|
||||||
"""
|
"""
|
||||||
daemon = ctx.require_daemon()
|
daemon = ctx.require_daemon()
|
||||||
if not is_fuse(ctx.root):
|
if not is_fuse(ctx.root):
|
||||||
@@ -1685,9 +1689,8 @@ def test_regression_b113_move_before_upload(ctx: Context) -> None:
|
|||||||
os.rename(root / "both.bin", moved / "both moved.bin")
|
os.rename(root / "both.bin", moved / "both moved.bin")
|
||||||
held = [item for item in daemon.queue() if not preexisting(item)]
|
held = [item for item in daemon.queue() if not preexisting(item)]
|
||||||
check(bool(held), "nothing was queued while sync was paused, so the moves raced nothing")
|
check(bool(held), "nothing was queued while sync was paused, so the moves raced nothing")
|
||||||
creates = [item["path"] for item in held if item["kind"] == "create"]
|
|
||||||
check(not creates, f"creates were queued, not minted on Drive, so B113 was not replayed: {creates}")
|
|
||||||
ctx.note("held_ops", len(held))
|
ctx.note("held_ops", len(held))
|
||||||
|
ctx.note("held_creates", sum(item["kind"] == "create" for item in held))
|
||||||
finally:
|
finally:
|
||||||
daemon.command("sync", "resume")
|
daemon.command("sync", "resume")
|
||||||
daemon.wait_for_queue()
|
daemon.wait_for_queue()
|
||||||
@@ -3297,8 +3300,15 @@ def remember_preexisting_ops(queue: list[dict]) -> None:
|
|||||||
print(f"NOTE: {plural(len(new), 'queued op')} predate this target, for {shown}; queue waits ignore them")
|
print(f"NOTE: {plural(len(new), 'queued op')} predate this target, for {shown}; queue waits ignore them")
|
||||||
|
|
||||||
|
|
||||||
def queue_settled(mount: dict, queue) -> bool:
|
def queue_settled(mount: dict | None, queue) -> bool:
|
||||||
"""Whether everything this run queued has drained. `queue` lists the ops."""
|
"""Whether everything this run queued has drained. `queue` lists the ops.
|
||||||
|
|
||||||
|
`mount` is `pdfs --json status`'s "mount", None while no daemon answers:
|
||||||
|
just after a restart, before the new one listens. That is not a drained
|
||||||
|
queue, only an unread one.
|
||||||
|
"""
|
||||||
|
if mount is None:
|
||||||
|
return False
|
||||||
if mount.get("pending_uploads", 0) == 0 and mount.get("pending_changes", 0) == 0:
|
if mount.get("pending_uploads", 0) == 0 and mount.get("pending_changes", 0) == 0:
|
||||||
return True
|
return True
|
||||||
if not PREEXISTING_OPS:
|
if not PREEXISTING_OPS:
|
||||||
@@ -3353,10 +3363,10 @@ class Daemon:
|
|||||||
deadline = time.monotonic() + self.timeout
|
deadline = time.monotonic() + self.timeout
|
||||||
last = None
|
last = None
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
last = self.status().get("mount") or {}
|
last = self.status().get("mount")
|
||||||
if queue_settled(last, self.queue):
|
if queue_settled(last, self.queue):
|
||||||
return
|
return
|
||||||
if last.get("parked_uploads"):
|
if last and last.get("parked_uploads"):
|
||||||
# A transient name's create waits for a rename, not for the
|
# A transient name's create waits for a rename, not for the
|
||||||
# drain, so waiting out the timeout would only hide which file.
|
# drain, so waiting out the timeout would only hide which file.
|
||||||
parked = [
|
parked = [
|
||||||
@@ -3846,7 +3856,7 @@ class ManagedSyncPair:
|
|||||||
last = None
|
last = None
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
value = json.loads(self.command("status", json_output=True))
|
value = json.loads(self.command("status", json_output=True))
|
||||||
last = value.get("mount") or {}
|
last = value.get("mount")
|
||||||
if queue_settled(
|
if queue_settled(
|
||||||
last,
|
last,
|
||||||
lambda: json.loads(self.command("sync", "queue", json_output=True))["items"],
|
lambda: json.loads(self.command("sync", "queue", json_output=True))["items"],
|
||||||
@@ -4161,12 +4171,28 @@ class MoveContext:
|
|||||||
self.pair = first.pair
|
self.pair = first.pair
|
||||||
|
|
||||||
def move(self, *paths: Path) -> subprocess.CompletedProcess:
|
def move(self, *paths: Path) -> subprocess.CompletedProcess:
|
||||||
"""`pdfs move`, bounded: a wedged mount must not hang the harness."""
|
"""`pdfs move`, bounded: a wedged mount must not hang the harness.
|
||||||
command = [self.pair.pdfs, "move", *(str(path) for path in paths)]
|
|
||||||
result = subprocess.run(
|
A mirror's sync pass holds the folder, and the daemon refuses a move
|
||||||
command, text=True, capture_output=True, timeout=self.pair.timeout
|
that waited 5 s for it. A periodic pass over a folder earlier runs
|
||||||
)
|
filled can take longer, so the move is tried again until it ends.
|
||||||
detail = result.stderr.strip() or result.stdout.strip()
|
With several sources, the ones an attempt already moved are gone, and
|
||||||
|
only the rest are tried again."""
|
||||||
|
*sources, dest = paths
|
||||||
|
deadline = time.monotonic() + self.pair.timeout
|
||||||
|
while True:
|
||||||
|
command = [self.pair.pdfs, "move", *(str(path) for path in (*sources, dest))]
|
||||||
|
result = subprocess.run(
|
||||||
|
command, text=True, capture_output=True, timeout=self.pair.timeout
|
||||||
|
)
|
||||||
|
detail = result.stderr.strip() or result.stdout.strip()
|
||||||
|
if not (result.returncode and "busy syncing" in detail) or time.monotonic() > deadline:
|
||||||
|
break
|
||||||
|
left = [source for source in sources if os.path.lexists(source)]
|
||||||
|
if not left:
|
||||||
|
break
|
||||||
|
sources = left
|
||||||
|
time.sleep(2)
|
||||||
if result.returncode and "different Proton Drive volumes" in detail:
|
if result.returncode and "different Proton Drive volumes" in detail:
|
||||||
raise Skip(f"the locations are on different volumes: {detail}")
|
raise Skip(f"the locations are on different volumes: {detail}")
|
||||||
return result
|
return result
|
||||||
@@ -4362,6 +4388,19 @@ def test_move_refuses_a_mirror_copy_drive_lacks(mc: MoveContext) -> None:
|
|||||||
# copy after a move would lose it.
|
# copy after a move would lose it.
|
||||||
os.symlink("data.txt", source.root / name / "link")
|
os.symlink("data.txt", source.root / name / "link")
|
||||||
source.settle()
|
source.settle()
|
||||||
|
if dest.mode == "mirror" and os.stat(source.folder).st_dev == os.stat(dest.folder).st_dev:
|
||||||
|
# Between two mirrors on one filesystem the local copy is renamed
|
||||||
|
# along, so the move loses nothing and goes ahead.
|
||||||
|
mc.move_ok(source.root / name, dest.root)
|
||||||
|
check((dest.root / name / "link").is_symlink(), "the symlink did not move along")
|
||||||
|
dest.settle()
|
||||||
|
check_bytes(
|
||||||
|
read(dest.root / name / "data.txt"),
|
||||||
|
files[Path(name) / "data.txt"],
|
||||||
|
f"{dest.name}: data.txt after the move",
|
||||||
|
)
|
||||||
|
check(not (source.root / name).exists(), f"{source.name}: {name} is still there")
|
||||||
|
return
|
||||||
detail = mc.move_refused(source.root / name, dest.root)
|
detail = mc.move_refused(source.root / name, dest.root)
|
||||||
check("not fully synced" in detail, f"unexpected refusal: {detail}")
|
check("not fully synced" in detail, f"unexpected refusal: {detail}")
|
||||||
check((source.root / name / "link").is_symlink(), "the symlink was lost by a refused move")
|
check((source.root / name / "link").is_symlink(), "the symlink was lost by a refused move")
|
||||||
@@ -4405,7 +4444,7 @@ MOVE_CASES = [
|
|||||||
Case("copy a tree out of My files", test_copy_tree_out_of_my_files, (LIVE,)),
|
Case("copy a tree out of My files", test_copy_tree_out_of_my_files, (LIVE,)),
|
||||||
Case("move refuses a name taken at the destination", test_move_refuses_a_name_taken_at_the_destination, (LIVE,)),
|
Case("move refuses a name taken at the destination", test_move_refuses_a_name_taken_at_the_destination, (LIVE,)),
|
||||||
Case("move refuses a folder into itself", test_move_refuses_a_folder_into_itself, (LIVE,)),
|
Case("move refuses a folder into itself", test_move_refuses_a_folder_into_itself, (LIVE,)),
|
||||||
Case("move refuses a mirror copy Drive lacks", test_move_refuses_a_mirror_copy_drive_lacks, (LIVE,)),
|
Case("move never loses a mirror copy Drive lacks", test_move_refuses_a_mirror_copy_drive_lacks, (LIVE,)),
|
||||||
Case("move refuses a file still being written", test_move_refuses_a_file_still_being_written, (LIVE,)),
|
Case("move refuses a file still being written", test_move_refuses_a_file_still_being_written, (LIVE,)),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user