mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-06 01:12:10 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bd9481902 | ||
|
|
b2d5870834 | ||
|
|
078774f6a1 | ||
|
|
326a9b4e92 | ||
|
|
78dc72a34d | ||
|
|
438b2b463e | ||
|
|
4cd49168c4 | ||
|
|
481c3fe0c7 | ||
|
|
098da2e97a | ||
|
|
d03d3eac3f | ||
|
|
433e206203 | ||
|
|
35418eea3f | ||
|
|
e48c4bb4b2 | ||
|
|
7ad9231858 | ||
|
|
e396c6b4d6 | ||
|
|
b37ab78451 | ||
|
|
6e3d2c335e |
No files matched your search
Generated
+66
-8
@@ -2200,6 +2200,25 @@ dependencies = [
|
||||
"system-deps",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http",
|
||||
"indexmap",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.7.1"
|
||||
@@ -2367,6 +2386,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
@@ -3518,7 +3538,7 @@ checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-cli"
|
||||
version = "1.5.0"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap 4.6.4",
|
||||
@@ -3533,7 +3553,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-core"
|
||||
version = "1.5.0"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"directories",
|
||||
@@ -3546,15 +3566,17 @@ dependencies = [
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-fuse"
|
||||
version = "1.5.0"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"fuser",
|
||||
"futures",
|
||||
@@ -3565,13 +3587,15 @@ dependencies = [
|
||||
"pdfs-core",
|
||||
"proton-drive-rs",
|
||||
"serde_json",
|
||||
"sha1 0.10.7",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-gui"
|
||||
version = "1.5.0"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-channel",
|
||||
@@ -3832,9 +3856,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-drive-rs"
|
||||
version = "0.5.1"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3abae5e90e6889b120208159eb2c60b332612e5b708fdf3c14287307ce01715d"
|
||||
checksum = "58322795538d66137d2eb071590375c5287e69059c429e2309dc37368ab5b588"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
@@ -3859,9 +3883,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-sdk"
|
||||
version = "0.5.1"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fa729310c1a66e12bb41b9f5f036c86b9857e76536556052203ffecdb0b44c3"
|
||||
checksum = "d77482a3160d14926c65537c64bc95cf27d609597a524c97d97d97d1777e9603"
|
||||
dependencies = [
|
||||
"aes-gcm 0.11.0",
|
||||
"async-trait",
|
||||
@@ -4252,6 +4276,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
@@ -5110,6 +5135,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"libc",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
@@ -5291,6 +5317,12 @@ dependencies = [
|
||||
"cipher 0.4.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
@@ -6232,6 +6264,20 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"typed-path",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.6"
|
||||
@@ -6244,6 +6290,18 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.1"
|
||||
|
||||
+9
-3
@@ -3,15 +3,15 @@ resolver = "2"
|
||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.5.0"
|
||||
version = "1.9.0"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||
rust-version = "1.96"
|
||||
|
||||
[workspace.dependencies]
|
||||
proton-sdk = { version = "0.5.1" }
|
||||
proton-drive-rs = { version = "0.5.1" }
|
||||
proton-sdk = { version = "0.6.1" }
|
||||
proton-drive-rs = { version = "0.6.1" }
|
||||
|
||||
# Internal crates.
|
||||
pdfs-core = { path = "crates/pdfs-core" }
|
||||
@@ -32,6 +32,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
directories = "5"
|
||||
ignore = "0.4"
|
||||
sha2 = "0.10"
|
||||
# Google Takeout exports are zip sets; the importer reads their central
|
||||
# directories and streams entries out without extracting to disk first.
|
||||
zip = { version = "8", default-features = false, features = ["deflate"] }
|
||||
# Proton's photo duplicate detection is keyed on the plaintext SHA-1, so an
|
||||
# import has to speak SHA-1 whatever its age.
|
||||
sha1 = "0.10"
|
||||
# Thumbnail generation for photos the server has none for. Decoders only for the
|
||||
# formats a camera or phone actually produces — the rest of the crate's codecs
|
||||
# would be dead weight in the daemon.
|
||||
|
||||
+207
-2
@@ -158,6 +158,28 @@ enum Command {
|
||||
},
|
||||
/// List the photo albums (including ones shared with this account).
|
||||
Albums,
|
||||
/// Import a Google Photos Takeout export into Proton Photos.
|
||||
///
|
||||
/// Pass every zip of the export at once — Google splits one export across
|
||||
/// numbered parts, and a photo's metadata often sits in a different part
|
||||
/// than the photo. Albums in the export are recreated as Proton albums, and
|
||||
/// photos already on the account (same name and same content) are skipped,
|
||||
/// so an interrupted import can simply be run again.
|
||||
ImportGooglePhotos {
|
||||
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
||||
#[arg(required = true)]
|
||||
archives: Vec<PathBuf>,
|
||||
/// Report what would be imported without uploading anything.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
/// Stay attached and print progress until the import finishes.
|
||||
#[arg(long)]
|
||||
wait: bool,
|
||||
},
|
||||
/// Show how the running (or last) Google Photos import is doing.
|
||||
ImportStatus,
|
||||
/// Stop the running Google Photos import.
|
||||
CancelImport,
|
||||
/// List the photos in one album (newest capture first).
|
||||
Album {
|
||||
/// Album node uid in `volume~link` form (from `pdfs albums`).
|
||||
@@ -589,6 +611,13 @@ fn main() -> Result<()> {
|
||||
} => cmd_photos(limit, offset, favorites),
|
||||
Command::Favorite { uid, remove } => cmd_favorite(uid, remove),
|
||||
Command::Albums => cmd_albums(),
|
||||
Command::ImportGooglePhotos {
|
||||
archives,
|
||||
dry_run,
|
||||
wait,
|
||||
} => cmd_import_google_photos(archives, dry_run, wait),
|
||||
Command::ImportStatus => cmd_import_status(),
|
||||
Command::CancelImport => cmd_cancel_import(),
|
||||
Command::Album { uid, limit, offset } => cmd_album(uid, limit, offset),
|
||||
Command::OpenPhoto { uid } => cmd_open_photo(uid),
|
||||
Command::Search { query, limit } => cmd_search(query, limit),
|
||||
@@ -1336,6 +1365,11 @@ fn cmd_status() -> Result<()> {
|
||||
online,
|
||||
pending_uploads,
|
||||
pending_changes,
|
||||
parked_uploads,
|
||||
failing_ops,
|
||||
failing_error,
|
||||
staged_bytes,
|
||||
staged_oldest_secs,
|
||||
..
|
||||
}) => {
|
||||
let state = if online { "" } else { ", offline" };
|
||||
@@ -1344,6 +1378,25 @@ fn cmd_status() -> Result<()> {
|
||||
None => String::new(),
|
||||
};
|
||||
println!("Mounted at {mountpoint} ({pinned} pinned{state}{queued})");
|
||||
// Everything below is a queue that is not making progress on its
|
||||
// own. Printed only when it exists, because in the ordinary case
|
||||
// there is nothing to say.
|
||||
if staged_bytes > 0 {
|
||||
println!(
|
||||
"Staged {} not yet uploaded (oldest {})",
|
||||
human_bytes(staged_bytes),
|
||||
human_age(staged_oldest_secs),
|
||||
);
|
||||
}
|
||||
if parked_uploads > 0 {
|
||||
println!("Parked {parked_uploads} upload(s) waiting for a final filename");
|
||||
}
|
||||
if failing_ops > 0 {
|
||||
println!(
|
||||
"Stuck {failing_ops} queued operation(s) keep failing: {}",
|
||||
failing_error.as_deref().unwrap_or("no error recorded"),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(other) => println!("Mount: unexpected response {other:?}"),
|
||||
Err(_) => println!("Mount: not running."),
|
||||
@@ -1687,6 +1740,104 @@ fn cmd_albums() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it gets
|
||||
// absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
|
||||
match control_request(CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
})? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
if !wait {
|
||||
println!("Run `pdfs import-status` to follow it.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Poll rather than stream: the control protocol is request/response, and an
|
||||
// import runs for hours — a held-open socket would be the only long-lived
|
||||
// connection the daemon has.
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus {
|
||||
running: true,
|
||||
summary: _,
|
||||
} => continue,
|
||||
CtlResponse::ImportStatus { summary, .. } => {
|
||||
print_import_summary(summary.as_ref());
|
||||
return Ok(());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_import_status() -> Result<()> {
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus { running, summary } => {
|
||||
if running {
|
||||
println!("Import running. `pdfs transfers` shows its progress.");
|
||||
}
|
||||
print_import_summary(summary.as_ref());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_cancel_import() -> Result<()> {
|
||||
match control_request(CtlRequest::CancelImport)? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
let Some(summary) = summary else {
|
||||
println!("No import has run yet.");
|
||||
return;
|
||||
};
|
||||
println!("Photos in export : {}", summary.found);
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
println!("Already on Proton: {}", summary.duplicates);
|
||||
if summary.skipped_trashed > 0 {
|
||||
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
||||
}
|
||||
if summary.albums_created > 0 || summary.album_links > 0 {
|
||||
println!(
|
||||
"Albums : {} created, {} photos filed",
|
||||
summary.albums_created, summary.album_links
|
||||
);
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
println!("Failed : {}", summary.failed);
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
println!("Uploaded bytes : {}", summary.bytes);
|
||||
}
|
||||
if summary.cancelled {
|
||||
println!("Import was cancelled before it finished.");
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
||||
match control_request(CtlRequest::AlbumPhotos { uid, offset, limit })? {
|
||||
CtlResponse::Photos { items, .. } if items.is_empty() => println!("No photos."),
|
||||
@@ -1886,6 +2037,18 @@ fn path_arg(path: &Path) -> Result<String> {
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8"))
|
||||
}
|
||||
|
||||
/// Format an age in seconds for a report line, coarsely — the point of the
|
||||
/// number is whether it is minutes or weeks.
|
||||
fn human_age(secs: u64) -> String {
|
||||
const UNITS: [(&str, u64); 4] = [("d", 86_400), ("h", 3_600), ("m", 60), ("s", 1)];
|
||||
for (unit, scale) in UNITS {
|
||||
if secs >= scale {
|
||||
return format!("{}{unit}", secs / scale);
|
||||
}
|
||||
}
|
||||
"just now".to_string()
|
||||
}
|
||||
|
||||
/// Format a byte count for a report line.
|
||||
fn human_bytes(bytes: u64) -> String {
|
||||
const UNITS: [(&str, f64); 4] = [
|
||||
@@ -1925,6 +2088,32 @@ fn cmd_cache(action: CacheCmd) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask for a deep integrity check and wait for it to finish.
|
||||
///
|
||||
/// The daemon answers the request immediately and runs the check on a thread —
|
||||
/// it holds the single database connection for as long as it takes, which would
|
||||
/// otherwise stall the mount — so a caller that needs the verdict polls for it.
|
||||
/// `pdfs doctor` is that caller: it is a diagnostic the user is watching, so
|
||||
/// waiting is right, and the bound is generous enough for a large database
|
||||
/// without hanging forever if the daemon never finishes.
|
||||
fn await_integrity_check() -> Result<CtlResponse> {
|
||||
const POLL: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
const GIVE_UP_AFTER: std::time::Duration = std::time::Duration::from_secs(120);
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
let response = control_request(CtlRequest::CacheInspect { deep: true })?;
|
||||
match &response {
|
||||
CtlResponse::CacheReport {
|
||||
integrity_running: true,
|
||||
integrity_checked: false,
|
||||
..
|
||||
} if started.elapsed() < GIVE_UP_AFTER => std::thread::sleep(POLL),
|
||||
_ => return Ok(response),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_cache_inspect(deep: bool) -> Result<()> {
|
||||
let response = control_request(CtlRequest::CacheInspect { deep })?;
|
||||
if emit_json(&response)? {
|
||||
@@ -1940,6 +2129,7 @@ fn cmd_cache_inspect(deep: bool) -> Result<()> {
|
||||
cache_budget,
|
||||
integrity_problems,
|
||||
integrity_checked,
|
||||
integrity_running,
|
||||
} => {
|
||||
println!(
|
||||
"Database schema v{schema_version}, {}",
|
||||
@@ -1972,6 +2162,11 @@ fn cmd_cache_inspect(deep: bool) -> Result<()> {
|
||||
println!(" {problem}");
|
||||
}
|
||||
}
|
||||
} else if integrity_running {
|
||||
// The daemon runs the check on a thread rather than holding its
|
||||
// only database connection for the length of it, so the answer
|
||||
// belongs to a later request.
|
||||
println!("\nIntegrity checking in the background — run this again for the result");
|
||||
} else {
|
||||
println!("\nIntegrity not checked (pass --deep)");
|
||||
}
|
||||
@@ -2209,14 +2404,24 @@ fn cmd_diagnose() -> Result<()> {
|
||||
format!("{pending} waiting to upload")
|
||||
},
|
||||
);
|
||||
match control_request(CtlRequest::CacheInspect { deep: true }) {
|
||||
match await_integrity_check() {
|
||||
Ok(CtlResponse::CacheReport {
|
||||
integrity_problems, ..
|
||||
integrity_problems,
|
||||
integrity_checked: true,
|
||||
..
|
||||
}) if integrity_problems.is_empty() => report.finding(
|
||||
DiagnoseLevel::Ok,
|
||||
" database integrity",
|
||||
"SQLite integrity_check passed",
|
||||
),
|
||||
Ok(CtlResponse::CacheReport {
|
||||
integrity_checked: false,
|
||||
..
|
||||
}) => report.finding(
|
||||
DiagnoseLevel::Warn,
|
||||
" database integrity",
|
||||
"still checking — run `pdfs cache inspect --deep` for the result",
|
||||
),
|
||||
Ok(CtlResponse::CacheReport {
|
||||
integrity_problems, ..
|
||||
}) => report.finding(
|
||||
|
||||
@@ -18,6 +18,8 @@ tracing.workspace = true
|
||||
directories.workspace = true
|
||||
ignore.workspace = true
|
||||
sha2.workspace = true
|
||||
sha1.workspace = true
|
||||
zip.workspace = true
|
||||
rusqlite.workspace = true
|
||||
keyring.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
+1225
-98
File diff suppressed because it is too large.
Load diff
@@ -103,6 +103,33 @@ pub enum Request {
|
||||
/// re-encrypted for our timeline, which the SDK does not implement yet; the
|
||||
/// daemon reports that as an error rather than silently doing nothing.
|
||||
SetPhotoFavorite { uid: String, favorite: bool },
|
||||
/// Import Google Photos **Takeout** archives into Proton Photos.
|
||||
///
|
||||
/// `archives` are absolute paths to the export's `.zip` parts on the
|
||||
/// daemon's own filesystem — the whole set at once, because a photo and its
|
||||
/// metadata sidecar routinely land in different parts. Photos already on the
|
||||
/// account (matched by name *and* content digest) are skipped, so a library
|
||||
/// partly synced by Proton Photos elsewhere does not import twice, and an
|
||||
/// interrupted import resumes by simply being run again.
|
||||
///
|
||||
/// With `dry_run` the daemon scans and reports what it *would* do without
|
||||
/// uploading anything. Otherwise it acks immediately with
|
||||
/// [`Response::Ok`] and works in the background — an export is hours of
|
||||
/// upload, far past any socket timeout. Progress shows up in
|
||||
/// [`Request::GetQueueStatus`] as a job, and the final counts in
|
||||
/// [`Request::ImportStatus`].
|
||||
ImportTakeout {
|
||||
archives: Vec<String>,
|
||||
#[serde(default)]
|
||||
dry_run: bool,
|
||||
},
|
||||
/// How the running (or last finished) Takeout import is doing. Replies with
|
||||
/// [`Response::ImportStatus`].
|
||||
ImportStatus,
|
||||
/// Ask the running Takeout import to stop. It finishes the photo on the wire
|
||||
/// and files what it has uploaded into its albums, then reports
|
||||
/// `cancelled`. Replies with [`Response::Ok`].
|
||||
CancelImport,
|
||||
/// Upload the photo at `source_path` under the given name and media type.
|
||||
///
|
||||
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
||||
@@ -1052,6 +1079,35 @@ pub struct PhotoMonth {
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
/// How a Google Photos Takeout import is going, or how the last one ended
|
||||
/// ([`Response::ImportStatus`]).
|
||||
///
|
||||
/// The counts are cumulative for the run, so the same shape serves progress and
|
||||
/// the final report. `found` counts *distinct* photos: Google stores a separate
|
||||
/// copy of a photo in each album folder it appears in, and those fold into one
|
||||
/// upload that joins several albums.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ImportSummary {
|
||||
/// Distinct photos the export holds.
|
||||
pub found: usize,
|
||||
/// Photos uploaded by this run.
|
||||
pub uploaded: usize,
|
||||
/// Photos already on the account, matched by name *and* content digest.
|
||||
pub duplicates: usize,
|
||||
/// Photos that could not be read or uploaded.
|
||||
pub failed: usize,
|
||||
/// Photos in the export's trash bucket, which are never imported.
|
||||
pub skipped_trashed: usize,
|
||||
/// Albums created by this run.
|
||||
pub albums_created: usize,
|
||||
/// Album memberships added by this run.
|
||||
pub album_links: usize,
|
||||
/// Bytes uploaded.
|
||||
pub bytes: u64,
|
||||
/// True when the run stopped early because it was cancelled.
|
||||
pub cancelled: bool,
|
||||
}
|
||||
|
||||
/// One album in a [`Response::Albums`] listing.
|
||||
///
|
||||
/// An album is a folder node on the photos volume carrying album properties, so
|
||||
@@ -1193,6 +1249,28 @@ pub enum Response {
|
||||
/// calling a queued `mkdir` an upload is a lie.
|
||||
#[serde(default)]
|
||||
pending_changes: u64,
|
||||
/// Queued writes held back because the file still wears a transient
|
||||
/// name (a browser's `*.crdownload`, an editor's `*.swp`). They are not
|
||||
/// waiting on the network and will not drain until the finalising
|
||||
/// rename, so they are counted apart from `pending_uploads`.
|
||||
#[serde(default)]
|
||||
parked_uploads: u64,
|
||||
/// Queued ops that have failed enough times to be considered stuck.
|
||||
/// They keep retrying forever — correctly, since their staged bytes are
|
||||
/// the only copy — which is why they have to be visible.
|
||||
#[serde(default)]
|
||||
failing_ops: u64,
|
||||
/// The most recent error from a stuck op, when there is one.
|
||||
#[serde(default)]
|
||||
failing_error: Option<String>,
|
||||
/// Bytes held in `staging/`: writes accepted from the kernel whose only
|
||||
/// copy is on local disk. Never evictable, never counted against the
|
||||
/// cache budget — hence reported separately.
|
||||
#[serde(default)]
|
||||
staged_bytes: u64,
|
||||
/// Age of the oldest staged write, in seconds.
|
||||
#[serde(default)]
|
||||
staged_oldest_secs: u64,
|
||||
},
|
||||
/// A human-readable success message.
|
||||
Ok { message: String },
|
||||
@@ -1217,6 +1295,12 @@ pub enum Response {
|
||||
/// Whether the integrity check actually ran, so an empty problem list
|
||||
/// is not mistaken for a clean bill of health it never gave.
|
||||
integrity_checked: bool,
|
||||
/// Whether a deep check is running right now. It reads every page of
|
||||
/// the database under the daemon's only connection, so it is answered
|
||||
/// asynchronously: this says "ask again shortly", which is a different
|
||||
/// thing to say than either of the two above.
|
||||
#[serde(default)]
|
||||
integrity_running: bool,
|
||||
},
|
||||
/// The pin registry.
|
||||
Pins { pins: Vec<Pin> },
|
||||
@@ -1242,6 +1326,14 @@ pub enum Response {
|
||||
available: bool,
|
||||
items: Vec<AlbumInfo>,
|
||||
},
|
||||
/// How the Takeout import is doing (reply to [`Request::ImportStatus`]).
|
||||
/// `running` is false once it has finished, when `summary` is the final
|
||||
/// report; `summary` is `None` when no import has run this session.
|
||||
ImportStatus {
|
||||
running: bool,
|
||||
#[serde(default)]
|
||||
summary: Option<ImportSummary>,
|
||||
},
|
||||
/// The months the timeline spans (reply to [`Request::PhotoMonths`]),
|
||||
/// newest first.
|
||||
PhotoMonths { months: Vec<PhotoMonth> },
|
||||
|
||||
@@ -33,7 +33,7 @@ impl Db {
|
||||
/// whose stored `kind` no longer parses (written by an older build) are
|
||||
/// skipped rather than failing the whole read.
|
||||
pub fn activity_list(&self, limit: usize) -> Result<Vec<ActivityEntry>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT time, kind, target, detail, ok FROM activity
|
||||
ORDER BY id DESC LIMIT ?1",
|
||||
|
||||
@@ -76,7 +76,7 @@ impl Db {
|
||||
|
||||
/// The persisted album listing, in stored order.
|
||||
pub fn albums_list(&self) -> Result<Vec<StoredAlbum>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid, name, photo_count, cover_uid, last_activity, shared
|
||||
FROM albums ORDER BY seq",
|
||||
@@ -98,7 +98,7 @@ impl Db {
|
||||
|
||||
/// Number of albums in the persisted listing.
|
||||
pub fn albums_count(&self) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let n: i64 = conn.query_row("SELECT COUNT(*) FROM albums", [], |r| r.get(0))?;
|
||||
Ok(n.max(0) as usize)
|
||||
}
|
||||
@@ -164,7 +164,7 @@ impl Db {
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind FROM album_photos
|
||||
WHERE album_uid = ?1 ORDER BY seq LIMIT ?2 OFFSET ?3",
|
||||
@@ -190,7 +190,7 @@ impl Db {
|
||||
|
||||
/// How many photos of `album_uid` are persisted.
|
||||
pub fn album_photos_count(&self, album_uid: &str) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let n: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM album_photos WHERE album_uid = ?1",
|
||||
[album_uid],
|
||||
@@ -207,7 +207,7 @@ impl Db {
|
||||
if uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let placeholders = vec!["?"; uids.len()].join(",");
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind FROM album_photos
|
||||
|
||||
@@ -42,6 +42,30 @@ impl Db {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bump `last_accessed` for many keys in one transaction.
|
||||
///
|
||||
/// The read path touches a key per 128 KiB the kernel asks for — 32 times
|
||||
/// per 4 MiB block — and each of those was its own write transaction under
|
||||
/// the single connection mutex, contending with every `lookup` and
|
||||
/// `readdir`. Callers accumulate touches in memory and flush them here.
|
||||
pub fn cache_accessed_batch(&self, touches: &[(String, i64)]) -> Result<()> {
|
||||
if touches.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
{
|
||||
let mut stmt = tx.prepare_cached(
|
||||
"UPDATE cache_entries SET last_accessed = ?2 WHERE cache_key = ?1",
|
||||
)?;
|
||||
for (key, at) in touches {
|
||||
stmt.execute(params![key, at])?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop a single cache-entry row (one evicted blob or block).
|
||||
pub fn cache_remove(&self, key: &str) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
@@ -75,7 +99,7 @@ impl Db {
|
||||
/// because `enforce_block_budget` runs on every cached block — once per
|
||||
/// 4 MiB of every cold read — while holding the shared connection.
|
||||
pub fn cache_total_bytes(&self, kind: &str) -> Result<u64> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let total: i64 = conn.query_row(
|
||||
"SELECT COALESCE(SUM(size_bytes), 0) FROM cache_entries WHERE kind = ?1",
|
||||
params![kind],
|
||||
@@ -95,7 +119,7 @@ impl Db {
|
||||
kind: &str,
|
||||
limit: usize,
|
||||
) -> Result<Vec<(String, u64)>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare_cached(
|
||||
"SELECT cache_key, size_bytes FROM cache_entries
|
||||
WHERE kind = ?1 ORDER BY last_accessed ASC LIMIT ?2",
|
||||
@@ -115,7 +139,7 @@ impl Db {
|
||||
/// (`clear_unpinned`); incremental eviction wants
|
||||
/// [`cache_eviction_candidates`](Self::cache_eviction_candidates).
|
||||
pub fn cache_entries_by_kind(&self, kind: &str) -> Result<Vec<(String, u64)>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT cache_key, size_bytes FROM cache_entries
|
||||
WHERE kind = ?1 ORDER BY last_accessed ASC",
|
||||
|
||||
@@ -15,7 +15,7 @@ pub struct StoredDevice {
|
||||
|
||||
impl Db {
|
||||
pub fn device_get(&self) -> Result<Option<StoredDevice>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT uid, share_id, root_uid, name, created FROM device LIMIT 1",
|
||||
[],
|
||||
|
||||
@@ -41,32 +41,43 @@ impl Db {
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// Write one batch of walked entries under scan generation `generation`. The
|
||||
/// FTS index is *not* touched here — it is rebuilt once in
|
||||
/// [`local_finish_scan`](Self::local_finish_scan).
|
||||
/// Write one batch of walked entries under scan generation `generation`,
|
||||
/// keeping the FTS index in step with them.
|
||||
///
|
||||
/// The index is maintained incrementally rather than rebuilt at the end of
|
||||
/// the scan, because a rebuild reads and re-tokenises every row in the table
|
||||
/// — trigrams over a few hundred thousand paths — inside one transaction on
|
||||
/// the single shared connection, which stalls the whole mount for as long as
|
||||
/// it takes. Incremental maintenance is possible here for a reason specific
|
||||
/// to this table: `local_fts` covers `name` and `path`, `path` is the
|
||||
/// table's key, and `name` is derived from it, so an existing row's indexed
|
||||
/// content can never change. Only insertions and deletions touch the index;
|
||||
/// a re-scan that finds the same files does no index work at all.
|
||||
///
|
||||
/// Hence the update-then-insert shape below rather than an upsert: the
|
||||
/// distinction between "already indexed" and "new row" is exactly what
|
||||
/// decides whether the index needs a write.
|
||||
pub fn local_upsert_batch(&self, generation: i64, entries: &[LocalEntry]) -> Result<()> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
{
|
||||
let mut stmt = tx.prepare_cached(
|
||||
"INSERT INTO local_files (path, name, is_dir, size, mtime, scan_gen)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
is_dir = excluded.is_dir,
|
||||
size = excluded.size,
|
||||
mtime = excluded.mtime,
|
||||
scan_gen = excluded.scan_gen",
|
||||
let mut update = tx.prepare_cached(
|
||||
"UPDATE local_files
|
||||
SET name = ?2, is_dir = ?3, size = ?4, mtime = ?5, scan_gen = ?6
|
||||
WHERE path = ?1",
|
||||
)?;
|
||||
let mut insert = tx.prepare_cached(
|
||||
"INSERT INTO local_files (path, name, is_dir, size, mtime, scan_gen)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
)?;
|
||||
let mut index =
|
||||
tx.prepare_cached("INSERT INTO local_fts (rowid, name, path) VALUES (?1, ?2, ?3)")?;
|
||||
for e in entries {
|
||||
stmt.execute(params![
|
||||
e.path,
|
||||
e.name,
|
||||
e.is_dir as i64,
|
||||
e.size,
|
||||
e.mtime,
|
||||
generation
|
||||
])?;
|
||||
let row = params![e.path, e.name, e.is_dir as i64, e.size, e.mtime, generation];
|
||||
if update.execute(row)? == 0 {
|
||||
insert.execute(row)?;
|
||||
index.execute(params![tx.last_insert_rowid(), e.name, e.path])?;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
@@ -74,16 +85,25 @@ impl Db {
|
||||
}
|
||||
|
||||
/// Close scan generation `generation`: drop every row an older scan wrote
|
||||
/// (those paths are gone from disk), rebuild the FTS index over what remains,
|
||||
/// and stamp the completion time. Returns the number of indexed entries.
|
||||
/// (those paths are gone from disk), drop their FTS entries, and stamp the
|
||||
/// completion time. Returns the number of indexed entries.
|
||||
///
|
||||
/// The FTS deletes name the old values explicitly, which is how an
|
||||
/// external-content FTS5 table is told to retract a row — see
|
||||
/// [`local_upsert_batch`](Self::local_upsert_batch) for why this is done
|
||||
/// per-row instead of by rebuilding the index.
|
||||
pub fn local_finish_scan(&self, generation: i64, finished_at: i64) -> Result<i64> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
tx.execute(
|
||||
"INSERT INTO local_fts (local_fts, rowid, name, path)
|
||||
SELECT 'delete', id, name, path FROM local_files WHERE scan_gen != ?1",
|
||||
params![generation],
|
||||
)?;
|
||||
tx.execute(
|
||||
"DELETE FROM local_files WHERE scan_gen != ?1",
|
||||
params![generation],
|
||||
)?;
|
||||
tx.execute_batch("INSERT INTO local_fts(local_fts) VALUES('rebuild');")?;
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('local_indexed_at', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -98,7 +118,7 @@ impl Db {
|
||||
/// has never been built. The daemon uses this to decide whether a fresh mount
|
||||
/// needs an immediate rescan or can serve the existing index.
|
||||
pub fn local_indexed_at(&self) -> Result<Option<i64>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT value FROM sync_state WHERE key = 'local_indexed_at'",
|
||||
@@ -118,7 +138,7 @@ impl Db {
|
||||
if query.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt;
|
||||
let rows = if query.chars().count() < TRIGRAM_MIN {
|
||||
let pat = format!("%{}%", like_escape(query));
|
||||
@@ -157,7 +177,7 @@ impl Db {
|
||||
if query.is_empty() || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let terms = candidate_trigrams(query);
|
||||
let lane_limit = limit.div_ceil(2);
|
||||
let mut rows: Vec<LocalFileHit> = if terms.is_empty() {
|
||||
@@ -182,16 +202,36 @@ impl Db {
|
||||
if !terms.is_empty()
|
||||
&& let Some(first) = query.chars().next()
|
||||
{
|
||||
let prefix = format!("{}%", like_escape(&first.to_string()));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT path, name, is_dir, size, mtime FROM local_files
|
||||
WHERE name COLLATE NOCASE LIKE ?1 ESCAPE '\\'
|
||||
ORDER BY name COLLATE NOCASE LIMIT ?2",
|
||||
)?;
|
||||
let fallback = stmt
|
||||
.query_map(params![prefix, lane_limit as i64], local_hit)?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
for hit in fallback {
|
||||
// Trigram rank says nothing about how well a name *starts* with the
|
||||
// query, so on a home directory full of dependency trees the pool
|
||||
// fills with vendored files before the obvious answer is reached
|
||||
// (measured: 73% of the 500 best-ranked candidates for "test" came
|
||||
// from one Go module cache). Give whole-query prefix matches their
|
||||
// own lane, newest first, and keep the single-character lane —
|
||||
// which recovers a typo that destroyed every trigram — behind it.
|
||||
let mut lanes = vec![(
|
||||
format!("{}%", like_escape(query)),
|
||||
"ORDER BY mtime DESC LIMIT ?2",
|
||||
)];
|
||||
lanes.push((
|
||||
format!("{}%", like_escape(&first.to_string())),
|
||||
"ORDER BY name COLLATE NOCASE LIMIT ?2",
|
||||
));
|
||||
let mut extra = Vec::new();
|
||||
for (pattern, order) in lanes {
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT path, name, is_dir, size, mtime FROM local_files
|
||||
WHERE name COLLATE NOCASE LIKE ?1 ESCAPE '\\' {order}"
|
||||
))?;
|
||||
extra.extend(
|
||||
stmt.query_map(params![pattern, lane_limit as i64], local_hit)?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?,
|
||||
);
|
||||
}
|
||||
// Prefix hits go in front: `rows` is truncated to `limit` below, and
|
||||
// the lane exists precisely because trigram rank was burying them.
|
||||
let trigram = std::mem::take(&mut rows);
|
||||
for hit in extra.into_iter().chain(trigram) {
|
||||
if !rows.iter().any(|existing| existing.path == hit.path) {
|
||||
rows.push(hit);
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
/// Current schema version. Bump on every forward migration added below.
|
||||
pub(super) const SCHEMA_VERSION: i64 = 20;
|
||||
pub(super) const SCHEMA_VERSION: i64 = 28;
|
||||
|
||||
impl Db {
|
||||
pub(super) fn migrate(&self) -> Result<()> {
|
||||
@@ -101,6 +101,123 @@ impl Db {
|
||||
if current < 20 {
|
||||
tx.execute_batch(MIGRATION_V20)?;
|
||||
}
|
||||
if current < 21 {
|
||||
// A V17-and-older fixture may carry only the tables its own test
|
||||
// cares about; there is nothing to reindex without `nodes`.
|
||||
let has_nodes: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_nodes {
|
||||
tx.execute_batch(MIGRATION_V21)?;
|
||||
}
|
||||
}
|
||||
if current < 22 {
|
||||
// Older fixtures (and a database that never grew a table because
|
||||
// its feature was never used) may be missing any of these, and an
|
||||
// index on an absent table is a hard error rather than a no-op.
|
||||
for (table, sql) in MIGRATION_V22 {
|
||||
let present: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
|
||||
[table],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if present {
|
||||
tx.execute_batch(sql)?;
|
||||
}
|
||||
}
|
||||
// `activity_time` indexes `time DESC` while both activity queries
|
||||
// order by `id`: it has only ever cost writes.
|
||||
tx.execute_batch("DROP INDEX IF EXISTS activity_time;")?;
|
||||
}
|
||||
if current < 23 {
|
||||
// One last full rebuild of the local index, because from here on the
|
||||
// scan maintains it incrementally (`local_upsert_batch`) and so can
|
||||
// no longer repair a `local_fts` that drifted from `local_files`.
|
||||
// Whatever it costs, it costs once, at startup, before the mount
|
||||
// serves anything.
|
||||
let present: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'local_fts'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if present {
|
||||
tx.execute_batch("INSERT INTO local_fts(local_fts) VALUES('rebuild');")?;
|
||||
}
|
||||
}
|
||||
if current < 24 {
|
||||
tx.execute_batch(MIGRATION_V24)?;
|
||||
}
|
||||
if current < 25 {
|
||||
// Same guard as V21: an old fixture may not have `nodes` at all,
|
||||
// and there is no path to materialise without it.
|
||||
let has_nodes: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_nodes {
|
||||
tx.execute_batch(MIGRATION_V25)?;
|
||||
}
|
||||
}
|
||||
if current < 26 {
|
||||
// Same guard as V22: a fixture that never exercised the queue may
|
||||
// not have `pending_op`, and altering an absent table is an error
|
||||
// rather than a no-op.
|
||||
// A fixture built from the current schema and then rewound to an
|
||||
// older `schema_version` already has the column, and `ADD COLUMN`
|
||||
// has no `IF NOT EXISTS`.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'claimed_at'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_ops: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_ops && !has_column {
|
||||
tx.execute_batch(MIGRATION_V26)?;
|
||||
}
|
||||
}
|
||||
if current < 27 {
|
||||
// Same guards as V26, for the same two reasons: the table may not
|
||||
// exist in an old fixture, and a fixture rewound from the current
|
||||
// schema already has the column.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('pending_op') \
|
||||
WHERE name = 'access_deferred_since'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_ops: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_ops && !has_column {
|
||||
tx.execute_batch(MIGRATION_V27)?;
|
||||
}
|
||||
}
|
||||
if current < 28 {
|
||||
// Same guards as V26/V27, for the same two reasons.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('sync_entry') \
|
||||
WHERE name = 'local_mtime_ns'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_entries: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_entry'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_entries && !has_column {
|
||||
tx.execute_batch(MIGRATION_V28)?;
|
||||
}
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -527,3 +644,205 @@ const MIGRATION_V20: &str = "
|
||||
ALTER TABLE photos ADD COLUMN favorite INTEGER NOT NULL DEFAULT 0;
|
||||
CREATE INDEX IF NOT EXISTS idx_photos_favorite ON photos(favorite, seq);
|
||||
";
|
||||
|
||||
/// V21: rebuild the node search index after fixing what counts as indexable.
|
||||
///
|
||||
/// The v16 backfill rooted its path walk at any node whose parent is null *or*
|
||||
/// not cached, but the write path's `node_is_indexable_tx` demanded a
|
||||
/// null-parent ancestor. Only the My Files root has one — a device folder's
|
||||
/// root is stored in `device`/`sync_folder`, never in `nodes` — so every node
|
||||
/// under a device folder was dropped from `nodes_fts` the first time it was
|
||||
/// upserted after the backfill, and was unfindable from the prompt (2,705 of
|
||||
/// 9,252 nodes on the account this was diagnosed against).
|
||||
///
|
||||
/// The rule now matches the backfill, so this replays it: same recursive walk
|
||||
/// as v16, over the current `nodes`.
|
||||
/// Two details the v16 backfill did not carry, both cheap here: a node under a
|
||||
/// trashed folder stays out (the write path has always excluded it, and
|
||||
/// `tombstone_subtree_tx` only marks a subtree it was told about), and the walk
|
||||
/// is depth-capped so a parent cycle in a corrupted database cannot spin the
|
||||
/// migration forever.
|
||||
const MIGRATION_V21: &str = "
|
||||
DROP TABLE IF EXISTS nodes_fts;
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
name, path, tokenize='trigram'
|
||||
);
|
||||
INSERT INTO nodes_fts (rowid, name, path)
|
||||
WITH RECURSIVE paths(rowid, uid, path, trashed, depth) AS (
|
||||
-- Only a true root (no parent at all) contributes nothing to the path: its
|
||||
-- name is the mount. A row whose parent is merely uncached is an ordinary
|
||||
-- folder and keeps its name, which is what `path_of` does at runtime.
|
||||
SELECT n.rowid, n.uid,
|
||||
CASE WHEN n.parent_uid IS NULL THEN '' ELSE n.name END, n.trashed, 0
|
||||
FROM nodes n
|
||||
WHERE n.parent_uid IS NULL
|
||||
OR NOT EXISTS (SELECT 1 FROM nodes p WHERE p.uid = n.parent_uid)
|
||||
UNION ALL
|
||||
SELECT n.rowid, n.uid,
|
||||
CASE WHEN paths.path = '' THEN n.name ELSE paths.path || '/' || n.name END,
|
||||
MAX(paths.trashed, n.trashed), paths.depth + 1
|
||||
FROM nodes n JOIN paths ON n.parent_uid = paths.uid
|
||||
WHERE paths.depth < 256
|
||||
)
|
||||
SELECT n.rowid, n.name, paths.path
|
||||
FROM nodes n JOIN paths ON paths.rowid = n.rowid
|
||||
WHERE paths.trashed = 0;
|
||||
";
|
||||
|
||||
/// V22: indexes for the lookups that were scanning, each paired with the table
|
||||
/// it needs to exist for.
|
||||
///
|
||||
/// Each backs a query on a hot path that had no index to use:
|
||||
///
|
||||
/// - `sync_entry(remote_uid)` — the reconcile resolves a remote uid to its
|
||||
/// local entry once per node, per pass.
|
||||
/// - `local_files(scan_gen)` — the sweep that retires the previous generation
|
||||
/// after a local index scan.
|
||||
/// - `nodes(parent_uid) WHERE trashed = 0` — every listing, with the partial
|
||||
/// index keeping it to the live rows a listing actually wants.
|
||||
///
|
||||
/// Deliberately *not* here: an index on `pending_op(next_attempt_at, id)`.
|
||||
/// `next_due_op` orders by `id`, which is the rowid, so it already stops at the
|
||||
/// first due row; offering the planner a `next_attempt_at` index makes it scan
|
||||
/// that index and then sort the matches by `id` instead —
|
||||
/// `next_due_op_does_not_scale_with_queue_length` measures the regression.
|
||||
const MIGRATION_V22: [(&str, &str); 3] = [
|
||||
(
|
||||
"sync_entry",
|
||||
"CREATE INDEX IF NOT EXISTS idx_sync_entry_remote_uid ON sync_entry(remote_uid);",
|
||||
),
|
||||
(
|
||||
"local_files",
|
||||
"CREATE INDEX IF NOT EXISTS idx_local_files_scan_gen ON local_files(scan_gen);",
|
||||
),
|
||||
(
|
||||
"nodes",
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_parent_live ON nodes(parent_uid) WHERE trashed = 0;",
|
||||
),
|
||||
];
|
||||
|
||||
/// V24: the revisions this daemon sealed itself, so restarting does not reopen
|
||||
/// the conflict hole B70 layer B closed.
|
||||
///
|
||||
/// `Core::own_sealed_revs` held these in memory: a queued write whose baseline
|
||||
/// names an earlier revision is a conflict *unless* the remote sits at a
|
||||
/// revision we sealed, in which case it is one writer stalling and resuming and
|
||||
/// must supersede rather than fork a `(sync-conflict)` copy. Losing the map at
|
||||
/// shutdown meant the first drain after a restart could not tell the two apart.
|
||||
///
|
||||
/// `sealed_at` is what bounds the table: entries older than the window in which
|
||||
/// a stall→resume is plausible are pruned on write.
|
||||
const MIGRATION_V24: &str = "
|
||||
CREATE TABLE IF NOT EXISTS own_sealed_rev (
|
||||
uid TEXT PRIMARY KEY,
|
||||
revision_id TEXT NOT NULL,
|
||||
sealed_at INTEGER NOT NULL
|
||||
);
|
||||
";
|
||||
|
||||
/// V25: `nodes.path` — a node's mountpoint-relative path, stored rather than
|
||||
/// walked.
|
||||
///
|
||||
/// `path_of` was a recursive CTE per node, and search ran it once per candidate
|
||||
/// (up to a thousand per keystroke) on top of the one the write path already ran
|
||||
/// per descendant of every folder upsert. The value only changes when a node's
|
||||
/// name or parent changes, which is exactly when the write path is already
|
||||
/// walking the subtree — so it belongs in a column.
|
||||
///
|
||||
/// The backfill is the same walk V21 used for `nodes_fts`, minus the trashed
|
||||
/// filter: a trashed node stays out of the index but still has a path. Roots get
|
||||
/// `''` (their name is the mount), and a node whose parent is merely uncached
|
||||
/// keeps its own name — both matching what `path_of` returned at runtime.
|
||||
///
|
||||
/// Materialised into a temp table rather than written by a correlated subquery,
|
||||
/// so the recursive walk runs once instead of once per row.
|
||||
const MIGRATION_V25: &str = "
|
||||
ALTER TABLE nodes ADD COLUMN path TEXT;
|
||||
|
||||
CREATE TEMP TABLE node_paths (uid TEXT PRIMARY KEY, path TEXT NOT NULL);
|
||||
|
||||
INSERT INTO node_paths (uid, path)
|
||||
WITH RECURSIVE paths(uid, path, depth) AS (
|
||||
SELECT n.uid, CASE WHEN n.parent_uid IS NULL THEN '' ELSE n.name END, 0
|
||||
FROM nodes n
|
||||
WHERE n.parent_uid IS NULL
|
||||
OR NOT EXISTS (SELECT 1 FROM nodes p WHERE p.uid = n.parent_uid)
|
||||
UNION ALL
|
||||
SELECT n.uid,
|
||||
CASE WHEN paths.path = '' THEN n.name ELSE paths.path || '/' || n.name END,
|
||||
paths.depth + 1
|
||||
FROM nodes n JOIN paths ON n.parent_uid = paths.uid
|
||||
WHERE paths.depth < 256
|
||||
)
|
||||
SELECT uid, path FROM paths;
|
||||
|
||||
UPDATE nodes
|
||||
SET path = (SELECT p.path FROM node_paths p WHERE p.uid = nodes.uid)
|
||||
WHERE EXISTS (SELECT 1 FROM node_paths p WHERE p.uid = nodes.uid);
|
||||
|
||||
DROP TABLE node_paths;
|
||||
";
|
||||
|
||||
/// V26: `pending_op.claimed_at` — the drain's claim column.
|
||||
///
|
||||
/// The drain was one thread taking one op at a time, so a 10 GiB upload held
|
||||
/// every queued rename and small write behind it for as long as it ran. Several
|
||||
/// workers can share the queue only if a claim is visible to the others: this
|
||||
/// column is what makes "already being drained by someone" a fact about the row
|
||||
/// rather than about one thread's local variable.
|
||||
///
|
||||
/// `0` is unclaimed; anything else is the ms at which a worker took it. A
|
||||
/// claim is process-local state that outlives nothing — the single-writer lock
|
||||
/// means a non-zero value found at open belongs to a crashed run, and
|
||||
/// [`Db::clear_op_claims`](super::Db::clear_op_claims) drops the lot.
|
||||
///
|
||||
/// The index is partial so it costs nothing on the ordinary row: only the
|
||||
/// handful of claimed rows are in it, which is exactly the set the claim query
|
||||
/// has to subtract to keep two workers off one node.
|
||||
const MIGRATION_V26: &str = "
|
||||
ALTER TABLE pending_op ADD COLUMN claimed_at INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pending_op_claimed
|
||||
ON pending_op(uid) WHERE claimed_at <> 0;
|
||||
";
|
||||
|
||||
/// Schema v27: when an op was first deferred by the local access check.
|
||||
///
|
||||
/// An access deferral consumes no attempt and records no error — the op has not
|
||||
/// failed remotely, it has not been sent at all. That is right for a permission
|
||||
/// change the user is about to undo, and wrong for a condition that never
|
||||
/// clears: the row is re-deferred every
|
||||
/// [`DRAIN_ACCESS_RECHECK`](../../../pdfs_fuse/drain/constant.DRAIN_ACCESS_RECHECK.html)
|
||||
/// forever while `attempts`, `last_error`, `failing` and the logs all stay
|
||||
/// empty, so a queue that can never drain is indistinguishable from a busy one.
|
||||
/// Found in the field as 28 revision ops holding 18 GiB of accepted writes for
|
||||
/// 30 days, none of them visible anywhere.
|
||||
///
|
||||
/// `0` means "not currently access-deferred"; anything else is the ms at which
|
||||
/// the first consecutive deferral happened, which is what
|
||||
/// [`Db::defer_op_for_access`](super::Db::defer_op_for_access) measures the
|
||||
/// deferral against.
|
||||
const MIGRATION_V27: &str = "
|
||||
ALTER TABLE pending_op ADD COLUMN access_deferred_since INTEGER NOT NULL DEFAULT 0;
|
||||
";
|
||||
|
||||
/// Schema v28: sub-second precision for a mirror folder's local baseline.
|
||||
///
|
||||
/// A reconcile pass decides whether the local side of a path changed by
|
||||
/// comparing `(local_mtime, local_size)` against the baseline. Both sides of
|
||||
/// that comparison were whole seconds, so an edit that landed in the same second
|
||||
/// as the last sync and left the file the same length — a fixed-size record
|
||||
/// rewritten, an image re-exported, a database page updated in place — read as
|
||||
/// *unchanged* and was never uploaded. The remote copy silently stayed at the
|
||||
/// older content until something else about the file moved (bugs.md B25).
|
||||
///
|
||||
/// Nullable rather than a converted `local_mtime * 1_000_000_000`: the
|
||||
/// nanosecond part of an existing baseline is genuinely unknown, and inventing
|
||||
/// zero for it would make every already-synced file compare as changed and
|
||||
/// re-upload an entire mirror on first start. `NULL` means "this row predates
|
||||
/// the column", which the comparison reads as "fall back to whole seconds" —
|
||||
/// exactly the behaviour that row was written under. Each row gains a real value
|
||||
/// the next time its path is synced.
|
||||
const MIGRATION_V28: &str = "
|
||||
ALTER TABLE sync_entry ADD COLUMN local_mtime_ns INTEGER;
|
||||
";
|
||||
@@ -2,9 +2,17 @@
|
||||
//! inode bookkeeping, full-text search, content-cache LRU tracking, and pins.
|
||||
//!
|
||||
//! Only the daemon (`pdfs-fuse`) opens this for writes; the GUI and CLI reach
|
||||
//! the same data through the control socket. The connection is wrapped in a
|
||||
//! `Mutex` because the FUSE callbacks are synchronous and already serialize
|
||||
//! behind the `State` lock, so a connection pool would be overkill.
|
||||
//! the same data through the control socket. There is one write connection
|
||||
//! behind a `Mutex` — SQLite allows exactly one writer, and the single-instance
|
||||
//! `flock` makes sure it is ours — and a small pool of read-only connections
|
||||
//! behind [`Db::read`], which every `SELECT`-only method uses.
|
||||
//!
|
||||
//! The pool is what makes WAL worth having. The original justification for a
|
||||
//! single connection was that "FUSE callbacks already serialize behind the
|
||||
//! `State` lock"; that stopped being true when reads moved onto an 11-thread
|
||||
//! worker pool, and a lookup then queued behind whatever listing happened to be
|
||||
//! committing. Under WAL a reader never blocks the writer and vice versa, so the
|
||||
//! only thing left serialising them was this process's own mutex.
|
||||
//!
|
||||
//! This module is the P0 foundation: it opens the database, enables WAL, and
|
||||
//! applies the forward-only schema migrations. Write-through of nodes, the
|
||||
@@ -18,8 +26,10 @@
|
||||
use parking_lot::Mutex;
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::Result;
|
||||
use std::path::Path;
|
||||
use crate::{Error, Result};
|
||||
use std::fs::File;
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
mod activity;
|
||||
mod albums;
|
||||
@@ -50,6 +60,7 @@ pub use ops::{
|
||||
PendingCounts, PendingOp, RenameMeta, op_supersedes,
|
||||
};
|
||||
pub use photos::{StoredPhoto, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN, TimelineRow};
|
||||
pub use pins::PinRow;
|
||||
pub use sync::{StoredSyncEntry, StoredSyncFolder};
|
||||
pub use trash::StoredTrash;
|
||||
|
||||
@@ -65,18 +76,208 @@ const WAL_SIZE_LIMIT: i64 = 64 * 1024 * 1024;
|
||||
/// genuinely stuck holder surfaces as an error rather than an apparent hang.
|
||||
const BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Page cache, in KiB (applied negated, which is how SQLite reads a size in KiB
|
||||
/// rather than in pages). 32 MiB against a database that is typically tens of
|
||||
/// MiB: large enough to hold the working set of a listing-heavy session, small
|
||||
/// enough to be unremarkable in a desktop daemon's RSS.
|
||||
const CACHE_SIZE_KIB: i64 = 32 * 1024;
|
||||
|
||||
/// Ceiling on the memory-mapped window over the database file. 256 MiB covers
|
||||
/// any realistic index while keeping the mapping bounded.
|
||||
const MMAP_SIZE: i64 = 256 * 1024 * 1024;
|
||||
|
||||
/// WAL pages between automatic checkpoints (default 1000, i.e. ~4 MiB). Halved
|
||||
/// so the synchronous checkpoint a committing thread inherits stays short.
|
||||
const WAL_AUTOCHECKPOINT_PAGES: i64 = 500;
|
||||
|
||||
/// Handle to the unified metadata database.
|
||||
///
|
||||
/// Cheap to wrap in an `Arc`; clone the `Arc`, not this. All access goes through
|
||||
/// the inner `Mutex<Connection>`.
|
||||
pub struct Db {
|
||||
conn: Mutex<Connection>,
|
||||
/// Read-only connections for `SELECT`-only methods. `None` for an in-memory
|
||||
/// database, where a second connection would open a different, empty one —
|
||||
/// those fall back to the write connection.
|
||||
readers: Option<ReadPool>,
|
||||
/// The single-writer lock, held open for as long as this handle lives.
|
||||
///
|
||||
/// Never read; dropping the `File` is what releases the `flock`, so it has
|
||||
/// to be owned here rather than by `open`'s stack frame. `None` for
|
||||
/// in-memory databases, which nothing else can reach.
|
||||
_single_writer: Option<File>,
|
||||
}
|
||||
|
||||
/// Idle read-only connections, opened on demand and kept for reuse.
|
||||
///
|
||||
/// Not a bounded pool: the number in flight is bounded by the number of threads
|
||||
/// that can be inside a read at once (the FUSE worker lanes plus the control
|
||||
/// handlers), and blocking one of those on a permit would reintroduce exactly
|
||||
/// the queueing this exists to remove. Only the *idle* set is capped, so a burst
|
||||
/// does not leave a connection per thread parked forever.
|
||||
struct ReadPool {
|
||||
path: PathBuf,
|
||||
idle: Mutex<Vec<Connection>>,
|
||||
}
|
||||
|
||||
/// How many idle read connections to keep. Enough that the steady state never
|
||||
/// reopens, small enough to be unremarkable.
|
||||
const MAX_IDLE_READERS: usize = 4;
|
||||
|
||||
impl ReadPool {
|
||||
fn take(&self) -> Option<Connection> {
|
||||
if let Some(conn) = self.idle.lock().pop() {
|
||||
return Some(conn);
|
||||
}
|
||||
match open_read_only(&self.path) {
|
||||
Ok(conn) => Some(conn),
|
||||
Err(error) => {
|
||||
// Falling back to the write connection is correct, just slower,
|
||||
// so this is a warning and not a failed read.
|
||||
tracing::warn!(%error, "could not open a read-only connection");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn put(&self, conn: Connection) {
|
||||
let mut idle = self.idle.lock();
|
||||
if idle.len() < MAX_IDLE_READERS {
|
||||
idle.push(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A connection to read through: one borrowed from [`ReadPool`], or the write
|
||||
/// connection when there is no pool (in-memory) or opening one failed.
|
||||
///
|
||||
/// Derefs to `Connection`, so a method converts by swapping `self.conn.lock()`
|
||||
/// for `self.read()` and nothing else.
|
||||
pub(super) struct Reader<'a> {
|
||||
pooled: Option<Connection>,
|
||||
writer: Option<parking_lot::MutexGuard<'a, Connection>>,
|
||||
pool: Option<&'a ReadPool>,
|
||||
}
|
||||
|
||||
impl std::ops::Deref for Reader<'_> {
|
||||
type Target = Connection;
|
||||
|
||||
fn deref(&self) -> &Connection {
|
||||
self.pooled
|
||||
.as_ref()
|
||||
.or(self.writer.as_deref())
|
||||
.expect("a Reader always holds one of the two")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Reader<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let (Some(conn), Some(pool)) = (self.pooled.take(), self.pool) {
|
||||
pool.put(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Open a second connection to the same file for reads only.
|
||||
///
|
||||
/// `SQLITE_OPEN_READ_ONLY` plus `query_only` is belt and braces: the flag stops
|
||||
/// a routing mistake at the file, the pragma stops it at the statement. The rest
|
||||
/// mirror [`Db::open_configured`] — a read connection has its own page cache,
|
||||
/// its own temp store and its own busy timeout, and gets nothing from the
|
||||
/// writer's.
|
||||
///
|
||||
/// `journal_mode` is deliberately absent: it is a property of the database file,
|
||||
/// which the writer already set to WAL, and a read-only connection cannot change
|
||||
/// it anyway.
|
||||
fn open_read_only(path: &Path) -> Result<Connection> {
|
||||
use rusqlite::OpenFlags;
|
||||
let conn = Connection::open_with_flags(
|
||||
path,
|
||||
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
|
||||
)?;
|
||||
conn.busy_timeout(BUSY_TIMEOUT)?;
|
||||
conn.pragma_update(None, "cache_size", -CACHE_SIZE_KIB)?;
|
||||
conn.pragma_update(None, "temp_store", "MEMORY")?;
|
||||
conn.pragma_update(None, "mmap_size", MMAP_SIZE)?;
|
||||
conn.pragma_update(None, "query_only", true)?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Open (creating if absent) the database at `path`, enable WAL, and bring
|
||||
/// the schema up to [`SCHEMA_VERSION`].
|
||||
///
|
||||
/// Takes the single-instance lock first (see
|
||||
/// [`acquire_single_writer_lock`]), and rebuilds from empty if what is on
|
||||
/// disk turns out not to be a readable database (see [`is_corrupt`]).
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
let single_writer = acquire_single_writer_lock(path)?;
|
||||
let conn = match Self::open_configured(path) {
|
||||
Ok(conn) => conn,
|
||||
Err(e) if is_corrupt(&e) => {
|
||||
// Every byte in here is derived from the account or from the
|
||||
// content cache directory, so the recovery is to start over —
|
||||
// loudly, and keeping the damaged file for a post-mortem rather
|
||||
// than deleting evidence.
|
||||
tracing::error!(
|
||||
db = %path.display(),
|
||||
error = %e,
|
||||
"cache database is corrupt — moving it aside and rebuilding \
|
||||
(metadata will be re-fetched; no user data is stored here)"
|
||||
);
|
||||
quarantine_corrupt_db(path)?;
|
||||
Self::open_configured(path)?
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let db = Self {
|
||||
conn: Mutex::new(conn),
|
||||
// No pool for `:memory:`. A second connection to that name is a
|
||||
// second, empty database rather than another view of this one, so
|
||||
// every routed read would find a schema that does not exist.
|
||||
readers: (!is_in_memory(path)).then(|| ReadPool {
|
||||
path: path.to_path_buf(),
|
||||
idle: Mutex::new(Vec::new()),
|
||||
}),
|
||||
_single_writer: single_writer,
|
||||
};
|
||||
// A drain claim names a worker thread in the process that took it, and
|
||||
// the single-writer lock above says that process is not running. Any
|
||||
// claim still on disk is therefore a crashed run's, and leaving it would
|
||||
// hide those ops from every worker for good.
|
||||
match db.clear_op_claims() {
|
||||
Ok(0) => {}
|
||||
Ok(n) => tracing::info!(ops = n, "released drain claims left by a previous run"),
|
||||
Err(e) => tracing::warn!(error = %e, "clearing stale drain claims failed"),
|
||||
}
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
/// A connection for a `SELECT`-only method.
|
||||
///
|
||||
/// Reads see the last committed state, which is the same thing they saw
|
||||
/// through the write connection: every method here commits before it
|
||||
/// returns, and nothing holds a transaction open across calls.
|
||||
pub(super) fn read(&self) -> Reader<'_> {
|
||||
if let Some(pool) = &self.readers
|
||||
&& let Some(conn) = pool.take()
|
||||
{
|
||||
return Reader {
|
||||
pooled: Some(conn),
|
||||
writer: None,
|
||||
pool: Some(pool),
|
||||
};
|
||||
}
|
||||
Reader {
|
||||
pooled: None,
|
||||
writer: Some(self.conn.lock()),
|
||||
pool: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Open the file, apply every PRAGMA, and migrate. Split out of
|
||||
/// [`open`](Self::open) so the corruption path can retry it verbatim.
|
||||
fn open_configured(path: &Path) -> Result<Connection> {
|
||||
let conn = Connection::open(path)?;
|
||||
// WAL: readers never block the single writer. NORMAL sync is the
|
||||
// standard durability/throughput tradeoff for WAL.
|
||||
@@ -101,12 +302,39 @@ impl Db {
|
||||
// (Bare SQLite does default to 0, which is where the belief that this
|
||||
// was unset came from. That default is not what we get.)
|
||||
conn.busy_timeout(BUSY_TIMEOUT)?;
|
||||
// Page cache. The default is 2 MiB, against a schema whose hot table
|
||||
// stores a JSON blob per node — a listing of a few thousand files
|
||||
// evicts itself while it is being read.
|
||||
conn.pragma_update(None, "cache_size", -CACHE_SIZE_KIB)?;
|
||||
// Sorts and recursive CTEs materialise their intermediate results.
|
||||
// Every `ORDER BY … COLLATE NOCASE` listing and every ancestor walk
|
||||
// spills those to a file in `/tmp` by default; they are small and
|
||||
// short-lived, so memory is both faster and less surprising.
|
||||
conn.pragma_update(None, "temp_store", "MEMORY")?;
|
||||
// Read the database through the page cache of the OS rather than
|
||||
// `pread` per page. Bounded rather than unlimited so the daemon's RSS
|
||||
// does not track the database size.
|
||||
conn.pragma_update(None, "mmap_size", MMAP_SIZE)?;
|
||||
// Checkpointing is synchronous in whichever thread happens to commit
|
||||
// the transaction that crosses the threshold — often a FUSE callback.
|
||||
// A smaller autocheckpoint makes that debt smaller and more frequent
|
||||
// rather than rare and multi-second.
|
||||
conn.pragma_update(None, "wal_autocheckpoint", WAL_AUTOCHECKPOINT_PAGES)?;
|
||||
|
||||
let db = Self {
|
||||
conn: Mutex::new(conn),
|
||||
readers: None,
|
||||
_single_writer: None,
|
||||
};
|
||||
db.migrate()?;
|
||||
Ok(db)
|
||||
Ok(db.into_conn())
|
||||
}
|
||||
|
||||
/// Unwrap the connection back out of a temporary handle. Only
|
||||
/// [`open_configured`](Self::open_configured) uses it, to run `migrate`
|
||||
/// (which is written against `&self`) before the real handle exists.
|
||||
fn into_conn(self) -> Connection {
|
||||
self.conn.into_inner()
|
||||
}
|
||||
|
||||
/// Open an in-memory database. For tests.
|
||||
@@ -119,6 +347,8 @@ impl Db {
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
let db = Self {
|
||||
conn: Mutex::new(conn),
|
||||
readers: None,
|
||||
_single_writer: None,
|
||||
};
|
||||
db.migrate()?;
|
||||
Ok(db)
|
||||
@@ -132,5 +362,102 @@ impl Db {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `path` names the in-memory database rather than a file. Both
|
||||
/// spellings SQLite accepts are checked, because neither has a lock file or a
|
||||
/// corruption story.
|
||||
fn is_in_memory(path: &Path) -> bool {
|
||||
let p = path.as_os_str();
|
||||
p == ":memory:" || p.is_empty()
|
||||
}
|
||||
|
||||
/// Take the single-instance write lock for the database at `path`.
|
||||
///
|
||||
/// The daemon is the only writer by design — the front-ends go through the
|
||||
/// control socket — but nothing stopped a hand-run `pdfs mount` from starting
|
||||
/// alongside the systemd unit, at which point two processes own the same inode
|
||||
/// space, the same content cache, and the same drain queue. SQLite's own
|
||||
/// locking makes each *statement* safe and does nothing about that.
|
||||
///
|
||||
/// `flock` on a sibling `.lock` file rather than on the database itself:
|
||||
/// SQLite uses POSIX record locks, so the two never interact, and a lock file
|
||||
/// left behind carries no state — the lock lives in the kernel and dies with
|
||||
/// the process, however it exits.
|
||||
fn acquire_single_writer_lock(path: &Path) -> Result<Option<File>> {
|
||||
if is_in_memory(path) {
|
||||
return Ok(None);
|
||||
}
|
||||
let lock_path = lock_path_for(path);
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.truncate(false)
|
||||
.write(true)
|
||||
.open(&lock_path)?;
|
||||
// SAFETY: `file` owns the descriptor and outlives the call.
|
||||
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if rc != 0 {
|
||||
let e = std::io::Error::last_os_error();
|
||||
if e.raw_os_error() == Some(libc::EWOULDBLOCK) {
|
||||
return Err(Error::Other(format!(
|
||||
"another Proton Drive daemon is already using {} — \
|
||||
stop it first (`systemctl --user stop proton-drive.service`)",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
return Err(Error::Io(e));
|
||||
}
|
||||
Ok(Some(file))
|
||||
}
|
||||
|
||||
fn lock_path_for(path: &Path) -> PathBuf {
|
||||
let mut name = path.as_os_str().to_owned();
|
||||
name.push(".lock");
|
||||
PathBuf::from(name)
|
||||
}
|
||||
|
||||
/// Whether `e` says the file is not a usable database, as opposed to any other
|
||||
/// failure. These are the only two conditions rebuilding fixes; a disk error, a
|
||||
/// permission problem or a newer-than-supported schema must all still fail.
|
||||
fn is_corrupt(e: &Error) -> bool {
|
||||
matches!(
|
||||
e,
|
||||
Error::Db(rusqlite::Error::SqliteFailure(inner, _))
|
||||
if inner.code == rusqlite::ErrorCode::DatabaseCorrupt
|
||||
|| inner.code == rusqlite::ErrorCode::NotADatabase
|
||||
)
|
||||
}
|
||||
|
||||
/// Move a damaged database (and its WAL sidecars) out of the way so the next
|
||||
/// open starts from empty.
|
||||
///
|
||||
/// Renamed rather than deleted: this is a cache, but it is also the only
|
||||
/// evidence of what went wrong, and it costs one file the user can remove.
|
||||
fn quarantine_corrupt_db(path: &Path) -> Result<()> {
|
||||
let stamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
for suffix in ["", "-wal", "-shm"] {
|
||||
let mut from = path.as_os_str().to_owned();
|
||||
from.push(suffix);
|
||||
let from = PathBuf::from(from);
|
||||
if !from.exists() {
|
||||
continue;
|
||||
}
|
||||
let mut to = path.as_os_str().to_owned();
|
||||
to.push(format!(".corrupt-{stamp}{suffix}"));
|
||||
let to = PathBuf::from(to);
|
||||
// A failure to rename the sidecars is survivable — SQLite discards a
|
||||
// WAL whose database header no longer matches — but failing to move the
|
||||
// database itself is not, because the retry would hit the same file.
|
||||
match std::fs::rename(&from, &to) {
|
||||
Ok(()) => tracing::warn!(from = %from.display(), to = %to.display(), "quarantined"),
|
||||
Err(e) if suffix.is_empty() => return Err(Error::Io(e)),
|
||||
Err(e) => tracing::warn!(file = %from.display(), error = %e, "could not quarantine"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -74,7 +74,7 @@ impl Db {
|
||||
|
||||
/// Every local location, with device fields joined from `sync_folder`.
|
||||
pub fn mount_list(&self) -> Result<Vec<MountSpec>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT
|
||||
m.id,
|
||||
|
||||
@@ -10,7 +10,9 @@ use crate::{Access, Result};
|
||||
use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId};
|
||||
use proton_drive_rs::{Node, NodeKind};
|
||||
|
||||
use super::utils::{TRIGRAM_MIN, collect_pairs, like_escape, pair, path_of};
|
||||
use super::utils::{
|
||||
HitRow, TRIGRAM_MIN, collect_hits, hit_row, join_path, like_escape, path_of, walk_path_of,
|
||||
};
|
||||
|
||||
pub struct StoredNode {
|
||||
pub node: Node,
|
||||
@@ -59,7 +61,7 @@ impl Db {
|
||||
/// flag. Synthetic shared listings use this to serve the last completed
|
||||
/// snapshot while offline, including after a live event expired its TTL.
|
||||
pub fn visible_children(&self, parent: &NodeUid) -> Result<Vec<Node>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_json FROM nodes
|
||||
WHERE parent_uid = ?1 AND trashed = 0 AND node_json IS NOT NULL
|
||||
@@ -285,7 +287,7 @@ impl Db {
|
||||
/// Check if a folder node has any non-trashed children in the database.
|
||||
pub fn has_children(&self, parent_uid: &NodeUid) -> Result<bool> {
|
||||
let uid_str = parent_uid.to_string();
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM nodes WHERE parent_uid = ?1 AND trashed = 0",
|
||||
params![uid_str],
|
||||
@@ -294,6 +296,25 @@ impl Db {
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
/// A node's mountpoint-relative path, or `None` when it is not cached.
|
||||
///
|
||||
/// One indexed read of the stored `path`. Callers that need to relate many
|
||||
/// nodes to a handful of roots — search decorating each hit with its
|
||||
/// mountpoint — resolve the roots once through this and do the rest as
|
||||
/// string arithmetic, instead of a `path_relative_to` walk per pair.
|
||||
pub fn node_path(&self, uid: &str) -> Result<Option<String>> {
|
||||
let conn = self.read();
|
||||
let exists: Option<i64> = conn
|
||||
.query_row("SELECT 1 FROM nodes WHERE uid = ?1", params![uid], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.optional()?;
|
||||
if exists.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(path_of(&conn, uid)?))
|
||||
}
|
||||
|
||||
/// Resolve `uid` relative to an ancestor node.
|
||||
///
|
||||
/// This is used when a remote subtree has its own local sync mount: the
|
||||
@@ -302,7 +323,7 @@ impl Db {
|
||||
/// chain keeps callers from accidentally joining a Drive-wide path onto the
|
||||
/// wrong mountpoint. The ancestor itself resolves to the empty path.
|
||||
pub fn path_relative_to(&self, ancestor_uid: &str, uid: &str) -> Result<Option<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"WITH RECURSIVE anc(uid, parent_uid, name, depth) AS (
|
||||
SELECT uid, parent_uid, name, 0 FROM nodes WHERE uid = ?1
|
||||
@@ -338,15 +359,15 @@ impl Db {
|
||||
if query.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let rows: Vec<(String, String)> = if query.chars().count() < TRIGRAM_MIN {
|
||||
let conn = self.read();
|
||||
let rows: Vec<HitRow> = if query.chars().count() < TRIGRAM_MIN {
|
||||
let pat = format!("%{}%", like_escape(query));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_json, uid FROM nodes
|
||||
"SELECT node_json, uid, path FROM nodes
|
||||
WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL
|
||||
ORDER BY name LIMIT ?2",
|
||||
)?;
|
||||
collect_pairs(stmt.query_map(params![pat, limit as i64], pair)?)?
|
||||
collect_hits(stmt.query_map(params![pat, limit as i64], hit_row)?)?
|
||||
} else {
|
||||
// Escape double quotes and quote each term, then combine with AND so
|
||||
// all terms must match but can appear in any order or position.
|
||||
@@ -356,18 +377,21 @@ impl Db {
|
||||
.collect::<Vec<_>>()
|
||||
.join(" AND ");
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT n.node_json, n.uid
|
||||
"SELECT n.node_json, n.uid, n.path
|
||||
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
||||
WHERE f.name MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||
ORDER BY f.rank LIMIT ?2",
|
||||
)?;
|
||||
collect_pairs(stmt.query_map(params![phrase, limit as i64], pair)?)?
|
||||
collect_hits(stmt.query_map(params![phrase, limit as i64], hit_row)?)?
|
||||
};
|
||||
|
||||
let mut hits = Vec::with_capacity(rows.len());
|
||||
for (json, uid) in rows {
|
||||
for (json, uid, path) in rows {
|
||||
let node: Node = serde_json::from_str(&json)?;
|
||||
let path = path_of(&conn, &uid)?;
|
||||
let path = match path {
|
||||
Some(path) => path,
|
||||
None => walk_path_of(&conn, &uid)?,
|
||||
};
|
||||
hits.push(SearchHit { node, path });
|
||||
}
|
||||
Ok(hits)
|
||||
@@ -382,54 +406,65 @@ impl Db {
|
||||
if query.is_empty() || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let terms = candidate_trigrams(query);
|
||||
let lane_limit = limit.div_ceil(2);
|
||||
let mut rows: Vec<(String, String)> = if terms.is_empty() {
|
||||
let mut rows: Vec<HitRow> = if terms.is_empty() {
|
||||
let pat = format!("%{}%", like_escape(query));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_json, uid FROM nodes
|
||||
"SELECT node_json, uid, path FROM nodes
|
||||
WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL
|
||||
ORDER BY name LIMIT ?2",
|
||||
)?;
|
||||
collect_pairs(stmt.query_map(params![pat, limit as i64], pair)?)?
|
||||
collect_hits(stmt.query_map(params![pat, limit as i64], hit_row)?)?
|
||||
} else {
|
||||
let expression = terms.join(" OR ");
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT n.node_json, n.uid
|
||||
"SELECT n.node_json, n.uid, n.path
|
||||
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
||||
WHERE nodes_fts MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||
ORDER BY f.rank LIMIT ?2",
|
||||
)?;
|
||||
collect_pairs(stmt.query_map(params![expression, lane_limit as i64], pair)?)?
|
||||
collect_hits(stmt.query_map(params![expression, lane_limit as i64], hit_row)?)?
|
||||
};
|
||||
// A short substitution can destroy every trigram (`vedio` vs `video`).
|
||||
// Feed the scorer an additional indexed same-initial lane so it can
|
||||
// recover those candidates without scanning the full node table.
|
||||
// Trigram rank does not know that a name *starts* with the query, and a
|
||||
// short substitution can destroy every trigram outright (`vedio` vs
|
||||
// `video`). Two indexed lanes cover both without scanning the node
|
||||
// table: whole-query prefix first, then same-initial.
|
||||
if !terms.is_empty()
|
||||
&& let Some(first) = query.chars().next()
|
||||
{
|
||||
let prefix = format!("{}%", like_escape(&first.to_string()));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_json, uid FROM nodes
|
||||
"SELECT node_json, uid, path FROM nodes
|
||||
WHERE name COLLATE NOCASE LIKE ?1 ESCAPE '\\'
|
||||
AND trashed = 0 AND node_json IS NOT NULL
|
||||
ORDER BY name COLLATE NOCASE LIMIT ?2",
|
||||
)?;
|
||||
let fallback =
|
||||
collect_pairs(stmt.query_map(params![prefix, lane_limit as i64], pair)?)?;
|
||||
for row in fallback {
|
||||
if !rows.iter().any(|(_, uid)| uid == &row.1) {
|
||||
let mut extra = Vec::new();
|
||||
for pattern in [
|
||||
format!("{}%", like_escape(query)),
|
||||
format!("{}%", like_escape(&first.to_string())),
|
||||
] {
|
||||
extra.extend(collect_hits(
|
||||
stmt.query_map(params![pattern, lane_limit as i64], hit_row)?,
|
||||
)?);
|
||||
}
|
||||
let trigram = std::mem::take(&mut rows);
|
||||
for row in extra.into_iter().chain(trigram) {
|
||||
if !rows.iter().any(|(_, uid, _)| uid == &row.1) {
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
rows.truncate(limit);
|
||||
}
|
||||
rows.into_iter()
|
||||
.map(|(json, uid)| {
|
||||
.map(|(json, uid, path)| {
|
||||
Ok(SearchHit {
|
||||
node: serde_json::from_str(&json)?,
|
||||
path: path_of(&conn, &uid)?,
|
||||
path: match path {
|
||||
Some(path) => path,
|
||||
None => walk_path_of(&conn, &uid)?,
|
||||
},
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -449,7 +484,7 @@ impl Db {
|
||||
|
||||
/// Load every persisted node for cold-start hydration of the `State` maps.
|
||||
pub fn load_all(&self) -> Result<Vec<StoredNode>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT node_json, listed FROM nodes WHERE node_json IS NOT NULL")?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
@@ -470,7 +505,7 @@ impl Db {
|
||||
/// when the API is unreachable, so the mount can still serve the cached tree
|
||||
/// (offline.md Phase 1).
|
||||
pub fn node_by_uid(&self, uid: &str) -> Result<Option<Node>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let json: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT node_json FROM nodes WHERE uid = ?1 AND node_json IS NOT NULL",
|
||||
@@ -488,7 +523,7 @@ impl Db {
|
||||
/// The daemon resumes from this on restart instead of reseeding to the
|
||||
/// server head, so changes made while unmounted are still applied (P2).
|
||||
pub fn children_if_listed(&self, parent: &NodeUid) -> Result<Option<Vec<Node>>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let listed: Option<i64> = conn
|
||||
.query_row(
|
||||
"SELECT listed FROM nodes WHERE uid = ?1",
|
||||
@@ -521,13 +556,57 @@ impl Db {
|
||||
// in sync on every store/read/evict, so it is authoritative for eviction.
|
||||
}
|
||||
|
||||
/// What a node's row looked like before this upsert, for the fields that decide
|
||||
/// whether the subtree below it has to be touched at all.
|
||||
struct PriorRow {
|
||||
parent_uid: Option<String>,
|
||||
name: String,
|
||||
path: Option<String>,
|
||||
trashed: bool,
|
||||
}
|
||||
|
||||
fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
||||
let json = serde_json::to_string(node)?;
|
||||
let uid = node.uid.to_string();
|
||||
let parent_uid = node.parent_uid.as_ref().map(|u| u.to_string());
|
||||
|
||||
let prior: Option<PriorRow> = tx
|
||||
.query_row(
|
||||
"SELECT parent_uid, name, path, trashed FROM nodes WHERE uid = ?1",
|
||||
params![uid],
|
||||
|row| {
|
||||
Ok(PriorRow {
|
||||
parent_uid: row.get(0)?,
|
||||
name: row.get(1)?,
|
||||
path: row.get(2)?,
|
||||
trashed: row.get::<_, i64>(3)? != 0,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()?;
|
||||
|
||||
// A node's path is its parent's path plus its own name — one indexed lookup,
|
||||
// where resolving it from scratch is a recursive walk to the root. A node
|
||||
// whose parent is not cached (a device folder's root lives in `device`, never
|
||||
// in `nodes`) starts a path of its own, which is what the walk did too.
|
||||
let path = match &parent_uid {
|
||||
None => String::new(),
|
||||
Some(parent) => {
|
||||
let parent_path: Option<Option<String>> = tx
|
||||
.query_row(
|
||||
"SELECT path FROM nodes WHERE uid = ?1",
|
||||
params![parent],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
join_path(parent_path.flatten().as_deref().unwrap_or(""), &node.name)
|
||||
}
|
||||
};
|
||||
|
||||
tx.execute(
|
||||
"INSERT INTO nodes
|
||||
(uid, parent_uid, name, is_dir, size, mtime, trashed, node_json)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
(uid, parent_uid, name, is_dir, size, mtime, trashed, node_json, path)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
||||
ON CONFLICT(uid) DO UPDATE SET
|
||||
parent_uid = excluded.parent_uid,
|
||||
name = excluded.name,
|
||||
@@ -535,59 +614,116 @@ fn upsert_node_tx(tx: &Transaction<'_>, node: &Node) -> Result<()> {
|
||||
size = excluded.size,
|
||||
mtime = excluded.mtime,
|
||||
trashed = excluded.trashed,
|
||||
node_json = excluded.node_json",
|
||||
node_json = excluded.node_json,
|
||||
path = excluded.path",
|
||||
params![
|
||||
uid,
|
||||
node.parent_uid.as_ref().map(|u| u.to_string()),
|
||||
parent_uid,
|
||||
node.name,
|
||||
node.is_folder() as i64,
|
||||
node_size(node),
|
||||
node.modification_time,
|
||||
node.trashed as i64,
|
||||
json,
|
||||
path,
|
||||
],
|
||||
)?;
|
||||
// FTS5 has no UPSERT. Folder moves also change every descendant path, so
|
||||
// refresh the whole affected subtree while keeping rowid deletes indexed.
|
||||
let rowid: i64 = tx.query_row(
|
||||
"SELECT rowid FROM nodes WHERE uid = ?1",
|
||||
params![uid],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
let mut affected = vec![rowid];
|
||||
if node.is_folder() {
|
||||
let mut stmt = tx.prepare(
|
||||
"WITH RECURSIVE descendants(rowid, uid) AS (
|
||||
SELECT rowid, uid FROM nodes WHERE parent_uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.rowid, n.uid FROM nodes n
|
||||
JOIN descendants d ON n.parent_uid = d.uid
|
||||
) SELECT rowid FROM descendants",
|
||||
)?;
|
||||
for descendant in stmt.query_map(params![uid], |row| row.get(0))? {
|
||||
affected.push(descendant?);
|
||||
}
|
||||
}
|
||||
for affected_rowid in affected {
|
||||
|
||||
// FTS5 has no UPSERT, so the node's own row is always replaced.
|
||||
let indexable = node_is_indexable_tx(tx, &uid)?;
|
||||
tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![rowid])?;
|
||||
if indexable {
|
||||
tx.execute(
|
||||
"DELETE FROM nodes_fts WHERE rowid = ?1",
|
||||
params![affected_rowid],
|
||||
"INSERT INTO nodes_fts (rowid, name, path) VALUES (?1, ?2, ?3)",
|
||||
params![rowid, node.name, path],
|
||||
)?;
|
||||
let indexed: Option<(String, String)> = tx
|
||||
.query_row(
|
||||
"SELECT name, uid FROM nodes WHERE rowid = ?1",
|
||||
params![affected_rowid],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some((name, indexed_uid)) = indexed
|
||||
&& node_is_indexable_tx(tx, &indexed_uid)?
|
||||
{
|
||||
let path = path_of(tx, &indexed_uid)?;
|
||||
tx.execute(
|
||||
"INSERT INTO nodes_fts (rowid, name, path) VALUES (?1, ?2, ?3)",
|
||||
params![affected_rowid, name, path],
|
||||
)?;
|
||||
}
|
||||
|
||||
// Everything below only matters when the subtree's paths or reachability
|
||||
// actually moved. Re-listing a folder upserts it unchanged, and paying for a
|
||||
// full descendant walk on each of those was most of what a large tree spent
|
||||
// its time on.
|
||||
let subtree_moved = prior.is_none_or(|prior| {
|
||||
prior.parent_uid != parent_uid
|
||||
|| prior.name != node.name
|
||||
|| prior.path.as_deref() != Some(path.as_str())
|
||||
|| prior.trashed != node.trashed
|
||||
});
|
||||
if node.is_folder() && subtree_moved {
|
||||
reindex_subtree_tx(tx, &uid, &path, indexable)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rewrite every descendant's stored path and search-index row after `folder`
|
||||
/// moved, was renamed, or changed reachability.
|
||||
///
|
||||
/// One recursive walk carries both the new path and whether anything on the way
|
||||
/// down is trashed, so each descendant costs two trivial statements instead of a
|
||||
/// `path_of` walk plus an ancestor walk of its own. `folder_indexable` is the
|
||||
/// answer `node_is_indexable_tx` gave for the folder itself: what is above the
|
||||
/// subtree is the same for every node in it.
|
||||
///
|
||||
/// The depth cap is a liveness guard, not a policy: `UNION ALL` over a parent
|
||||
/// cycle (a's parent is b, b's parent is a — corrupt data, but the API can hand
|
||||
/// it to us) never terminates, and this runs inside the write transaction that
|
||||
/// holds the daemon's only SQLite connection. Real Drive trees are nowhere near
|
||||
/// this deep.
|
||||
fn reindex_subtree_tx(
|
||||
tx: &Transaction<'_>,
|
||||
folder: &str,
|
||||
folder_path: &str,
|
||||
folder_indexable: bool,
|
||||
) -> Result<()> {
|
||||
let descendants: Vec<(i64, String, bool)> = {
|
||||
let mut stmt = tx.prepare(
|
||||
"WITH RECURSIVE sub(rowid, uid, name, path, blocked, depth) AS (
|
||||
SELECT n.rowid, n.uid, n.name,
|
||||
CASE WHEN ?2 = '' THEN n.name ELSE ?2 || '/' || n.name END,
|
||||
n.trashed, 0
|
||||
FROM nodes n WHERE n.parent_uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.rowid, n.uid, n.name,
|
||||
sub.path || '/' || n.name,
|
||||
MAX(sub.blocked, n.trashed), sub.depth + 1
|
||||
FROM nodes n JOIN sub ON n.parent_uid = sub.uid
|
||||
WHERE sub.depth < 256
|
||||
)
|
||||
SELECT rowid, path, blocked FROM sub",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![folder, folder_path], |row| {
|
||||
Ok((
|
||||
row.get::<_, i64>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, i64>(2)? != 0,
|
||||
))
|
||||
})?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows {
|
||||
out.push(row?);
|
||||
}
|
||||
out
|
||||
};
|
||||
if descendants.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut set_path = tx.prepare("UPDATE nodes SET path = ?2 WHERE rowid = ?1")?;
|
||||
let mut unindex = tx.prepare("DELETE FROM nodes_fts WHERE rowid = ?1")?;
|
||||
let mut index = tx.prepare(
|
||||
"INSERT INTO nodes_fts (rowid, name, path)
|
||||
SELECT rowid, name, ?2 FROM nodes WHERE rowid = ?1",
|
||||
)?;
|
||||
for (rowid, path, blocked) in descendants {
|
||||
set_path.execute(params![rowid, path])?;
|
||||
unindex.execute(params![rowid])?;
|
||||
if folder_indexable && !blocked {
|
||||
index.execute(params![rowid, path])?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -603,13 +739,27 @@ fn direct_child_uids_tx(tx: &Transaction<'_>, parent: &str) -> Result<Vec<String
|
||||
Ok(uids)
|
||||
}
|
||||
|
||||
/// Whether a node belongs in the search index: it exists, and nothing on the
|
||||
/// way up to the top of its cached chain is trashed.
|
||||
///
|
||||
/// The chain does *not* have to reach a `parent_uid IS NULL` row. Only the My
|
||||
/// Files root is stored that way; a device folder's root lives in `device` /
|
||||
/// `sync_folder` and never gets a `nodes` row, so requiring a null-parent
|
||||
/// ancestor silently excluded every device-folder subtree from search — 29% of
|
||||
/// the nodes on the account this was found on. A subtree whose topmost cached
|
||||
/// row has an uncached parent is ordinary, reachable data.
|
||||
///
|
||||
/// What is still excluded is a *cycle*: the walk's own guard stops it, but
|
||||
/// [`path_of`] has no such guard, so an indexed cycle would hang a search. A
|
||||
/// clean chain ends at a row whose parent is null or simply not cached; a cycle
|
||||
/// ends at one whose parent is a row we have already visited.
|
||||
fn node_is_indexable_tx(tx: &Transaction<'_>, uid: &str) -> Result<bool> {
|
||||
let indexable: i64 = tx.query_row(
|
||||
"WITH RECURSIVE ancestors(uid, parent_uid, trashed, path) AS (
|
||||
SELECT uid, parent_uid, trashed, char(31) || uid || char(31)
|
||||
"WITH RECURSIVE ancestors(uid, parent_uid, trashed, depth, path) AS (
|
||||
SELECT uid, parent_uid, trashed, 0, char(31) || uid || char(31)
|
||||
FROM nodes WHERE uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.uid, n.parent_uid, n.trashed,
|
||||
SELECT n.uid, n.parent_uid, n.trashed, a.depth + 1,
|
||||
a.path || n.uid || char(31)
|
||||
FROM ancestors a JOIN nodes n ON n.uid = a.parent_uid
|
||||
WHERE instr(a.path, char(31) || n.uid || char(31)) = 0
|
||||
@@ -617,7 +767,12 @@ fn node_is_indexable_tx(tx: &Transaction<'_>, uid: &str) -> Result<bool> {
|
||||
SELECT CASE
|
||||
WHEN COUNT(*) = 0 THEN 0
|
||||
WHEN MAX(trashed) != 0 THEN 0
|
||||
WHEN MAX(CASE WHEN parent_uid IS NULL THEN 1 ELSE 0 END) = 0 THEN 0
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM ancestors a
|
||||
WHERE a.depth = (SELECT MAX(depth) FROM ancestors)
|
||||
AND a.parent_uid IS NOT NULL
|
||||
AND EXISTS (SELECT 1 FROM nodes p WHERE p.uid = a.parent_uid)
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END
|
||||
FROM ancestors",
|
||||
|
||||
+286
-12
@@ -119,15 +119,32 @@ pub struct PendingOp {
|
||||
}
|
||||
|
||||
/// How much work the queue owes the server, by whether it carries bytes.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PendingCounts {
|
||||
/// Queued `create`/`revision` ops: files whose content is not on the remote.
|
||||
pub uploads: i64,
|
||||
/// Queued `mkdir`/`rename`/`trash` ops: metadata the remote has not been
|
||||
/// told about.
|
||||
pub changes: i64,
|
||||
/// Ops parked at [`PARK_UNTIL`]: a transient file's bytes, held back until
|
||||
/// the rename that gives it its finished name. They are queued but will
|
||||
/// never drain on their own, so counting them as "waiting to upload" would
|
||||
/// have the front-end promise progress that is not coming.
|
||||
pub parked: i64,
|
||||
/// Ops that have failed at least [`FAILING_ATTEMPTS`] times. A permanently
|
||||
/// failing op never wedges the queue — the backoff sees to that — which is
|
||||
/// exactly why it needs surfacing: it retries forever, invisibly.
|
||||
pub failing: i64,
|
||||
/// The most recent error text from a failing op, for the front-end to show
|
||||
/// instead of the bare count.
|
||||
pub last_error: Option<String>,
|
||||
}
|
||||
|
||||
/// Failed attempts after which an op is reported as failing rather than merely
|
||||
/// retrying. Six is where the backoff has reached its ceiling, so an op past it
|
||||
/// is retrying at the slowest rate it ever will and has been for minutes.
|
||||
pub const FAILING_ATTEMPTS: i64 = 6;
|
||||
|
||||
/// The outcome of folding freshly written bytes into a queued create.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AttachedBlob {
|
||||
@@ -140,7 +157,7 @@ pub struct AttachedBlob {
|
||||
impl Db {
|
||||
/// Whether a specific desired-state operation is still queued for a node.
|
||||
pub fn has_pending_op(&self, uid: &str, kind: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT EXISTS(
|
||||
SELECT 1 FROM pending_op WHERE uid = ?1 AND kind = ?2
|
||||
@@ -153,7 +170,7 @@ impl Db {
|
||||
|
||||
/// Read an existing operation's metadata before a superseding enqueue.
|
||||
pub fn pending_op_meta(&self, uid: &str, kind: &str) -> Result<Option<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT meta_json FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
||||
params![uid, kind],
|
||||
@@ -352,7 +369,7 @@ impl Db {
|
||||
|
||||
/// Check if a queued create or mkdir op exists for `uid`.
|
||||
pub fn has_create_op(&self, uid: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM pending_op WHERE uid = ?1 AND kind IN (?2, ?3)",
|
||||
params![uid, OP_CREATE, OP_MKDIR],
|
||||
@@ -368,7 +385,7 @@ impl Db {
|
||||
/// bytes are still owed an upload, and dropping it would discard the staged
|
||||
/// blob that holds them (`docs/BUGS.md` B71).
|
||||
pub fn has_any_op(&self, uid: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let count: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM pending_op WHERE uid = ?1",
|
||||
params![uid],
|
||||
@@ -464,7 +481,7 @@ impl Db {
|
||||
/// sort, and no separate index to maintain. The common case (something is
|
||||
/// due) exits on the first row.
|
||||
pub fn next_due_op(&self, now: i64) -> Result<Option<PendingOp>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare_cached(&format!(
|
||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
||||
attempts, last_error, next_attempt_at
|
||||
@@ -495,17 +512,115 @@ impl Db {
|
||||
Ok(op)
|
||||
}
|
||||
|
||||
/// Take the oldest due op for this worker, marking it claimed so no other
|
||||
/// drain worker takes it, or `None` when there is nothing to do.
|
||||
///
|
||||
/// The drain is several workers over one queue, which is only safe because
|
||||
/// the claim is a column: a row is offered to exactly one worker, and the
|
||||
/// worker that took it is the only one that may retire, defer or fail it.
|
||||
///
|
||||
/// Two exclusions on top of [`next_due_op`](Self::next_due_op)'s:
|
||||
///
|
||||
/// * `claimed_at = 0` — not already in flight somewhere else.
|
||||
/// * no *other* claimed op shares this `uid`. Ordering only has to hold per
|
||||
/// node, but there it has to hold absolutely: a queued rename and a queued
|
||||
/// revision of one file must land in the order they were made, and two
|
||||
/// workers on one node would also race over its staged blob and its
|
||||
/// pending entry. `ORDER BY id` plus this exclusion gives per-uid serial
|
||||
/// order for free — the older op is claimed first and the newer one is
|
||||
/// invisible until it retires.
|
||||
///
|
||||
/// Select and mark are one transaction, so two workers cannot both see the
|
||||
/// row as unclaimed.
|
||||
pub fn claim_next_due_op(&self, now: i64) -> Result<Option<PendingOp>> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let op = {
|
||||
let mut stmt = tx.prepare_cached(&format!(
|
||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
||||
attempts, last_error, next_attempt_at
|
||||
FROM pending_op
|
||||
WHERE next_attempt_at <= ?1
|
||||
AND claimed_at = 0
|
||||
AND (parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~')
|
||||
AND uid NOT IN (SELECT uid FROM pending_op WHERE claimed_at <> 0)
|
||||
ORDER BY id LIMIT 1",
|
||||
v = LOCAL_VOLUME,
|
||||
n = LOCAL_VOLUME.len() + 1,
|
||||
))?;
|
||||
stmt.query_row(params![now], |r| {
|
||||
Ok(PendingOp {
|
||||
id: r.get(0)?,
|
||||
kind: r.get(1)?,
|
||||
uid: r.get(2)?,
|
||||
parent_uid: r.get(3)?,
|
||||
name: r.get(4)?,
|
||||
blob_path: r.get(5)?,
|
||||
meta_json: r.get(6)?,
|
||||
created_at: r.get(7)?,
|
||||
attempts: r.get(8)?,
|
||||
last_error: r.get(9)?,
|
||||
next_attempt_at: r.get(10)?,
|
||||
})
|
||||
})
|
||||
.optional()?
|
||||
};
|
||||
if let Some(op) = op.as_ref() {
|
||||
tx.execute(
|
||||
"UPDATE pending_op SET claimed_at = ?2 WHERE id = ?1",
|
||||
params![op.id, now.max(1)],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(op)
|
||||
}
|
||||
|
||||
/// Give a claimed op back to the queue, whatever became of the attempt.
|
||||
///
|
||||
/// Called on every path out of a drain attempt rather than only the failing
|
||||
/// ones: a handler that retires its own row leaves nothing to release (the
|
||||
/// `UPDATE` matches nothing), and a handler that returns without retiring
|
||||
/// *and* without failing would otherwise leave the row claimed by a worker
|
||||
/// that has moved on — invisible to every worker, forever.
|
||||
pub fn release_op_claim(&self, id: i64) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"UPDATE pending_op SET claimed_at = 0 WHERE id = ?1",
|
||||
params![id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop every claim in the table.
|
||||
///
|
||||
/// A claim describes a worker in *this* process, so one surviving on disk
|
||||
/// means the previous run died holding it. The single-writer lock
|
||||
/// ([`Db::open`]) is what makes that inference safe: nobody else can be
|
||||
/// draining this queue while we are opening it.
|
||||
pub fn clear_op_claims(&self) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
Ok(conn.execute(
|
||||
"UPDATE pending_op SET claimed_at = 0 WHERE claimed_at <> 0",
|
||||
[],
|
||||
)?)
|
||||
}
|
||||
|
||||
/// The earliest `next_attempt_at` among all queued ops, or `None` if the
|
||||
/// queue is empty. Used by the drain loop to sleep exactly until the next
|
||||
/// debounced or backed-off op becomes eligible rather than waiting the full
|
||||
/// idle-poll interval.
|
||||
///
|
||||
/// Claimed ops are excluded: another worker is already on them, so they are
|
||||
/// not work this one is waiting for, and counting them would have an idle
|
||||
/// worker spin on a row it cannot have.
|
||||
pub fn earliest_due_at(&self) -> Result<Option<i64>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let ts: Option<i64> = conn
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT MIN(next_attempt_at) FROM pending_op \
|
||||
WHERE parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~'",
|
||||
WHERE claimed_at = 0 \
|
||||
AND (parent_uid IS NULL OR substr(parent_uid, 1, {n}) <> '{v}~')",
|
||||
v = LOCAL_VOLUME,
|
||||
n = LOCAL_VOLUME.len() + 1,
|
||||
),
|
||||
@@ -520,7 +635,7 @@ impl Db {
|
||||
/// Every queued op, oldest first. For status read-outs and the CLI — the
|
||||
/// drain wants [`next_due_op`](Self::next_due_op) instead.
|
||||
pub fn pending_ops(&self) -> Result<Vec<PendingOp>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
||||
attempts, last_error, next_attempt_at
|
||||
@@ -553,7 +668,7 @@ impl Db {
|
||||
/// mount owes the server, but it carries no bytes and reporting it as an
|
||||
/// upload is simply untrue.
|
||||
pub fn pending_op_counts(&self) -> Result<PendingCounts> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let uploads = conn.query_row(
|
||||
"SELECT COUNT(*) FROM pending_op WHERE kind IN (?1, ?2)",
|
||||
params![OP_REVISION, OP_CREATE],
|
||||
@@ -564,7 +679,33 @@ impl Db {
|
||||
params![OP_REVISION, OP_CREATE],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(PendingCounts { uploads, changes })
|
||||
let parked = conn.query_row(
|
||||
"SELECT COUNT(*) FROM pending_op WHERE next_attempt_at >= ?1",
|
||||
params![PARK_UNTIL],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let failing = conn.query_row(
|
||||
"SELECT COUNT(*) FROM pending_op WHERE attempts >= ?1",
|
||||
params![FAILING_ATTEMPTS],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
let last_error = conn
|
||||
.query_row(
|
||||
"SELECT last_error FROM pending_op
|
||||
WHERE attempts >= ?1 AND last_error IS NOT NULL
|
||||
ORDER BY attempts DESC LIMIT 1",
|
||||
params![FAILING_ATTEMPTS],
|
||||
|r| r.get::<_, Option<String>>(0),
|
||||
)
|
||||
.optional()?
|
||||
.flatten();
|
||||
Ok(PendingCounts {
|
||||
uploads,
|
||||
changes,
|
||||
parked,
|
||||
failing,
|
||||
last_error,
|
||||
})
|
||||
}
|
||||
|
||||
/// Nothing bearing this volume may be handed to the API — it would 404. The
|
||||
@@ -580,11 +721,16 @@ impl Db {
|
||||
|
||||
/// Record a failed attempt and when to next try. Leaves the row in place —
|
||||
/// the staged bytes are still the only copy of the user's write.
|
||||
///
|
||||
/// Clears any access-deferral window: whatever the op was waiting for has
|
||||
/// now been reported, so the next uncleared deferral starts its own window
|
||||
/// rather than escalating again on the following recheck.
|
||||
pub fn record_op_failure(&self, id: i64, error: &str, next_attempt_at: i64) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"UPDATE pending_op
|
||||
SET attempts = attempts + 1, last_error = ?2, next_attempt_at = ?3
|
||||
SET attempts = attempts + 1, last_error = ?2, next_attempt_at = ?3,
|
||||
access_deferred_since = 0
|
||||
WHERE id = ?1",
|
||||
params![id, error, next_attempt_at],
|
||||
)?;
|
||||
@@ -609,4 +755,132 @@ impl Db {
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Defer an op the local access check refused, and report how long it has
|
||||
/// been refused for.
|
||||
///
|
||||
/// Identical to [`defer_op_without_attempt`](Self::defer_op_without_attempt)
|
||||
/// on the retry clock, plus the one thing that was missing: the first
|
||||
/// deferral of a run stamps `access_deferred_since`, and every deferral
|
||||
/// returns that stamp. A caller can therefore tell a deferral that is one
|
||||
/// recheck old from one that has been repeating unchanged for days, which is
|
||||
/// the difference between a permission the user is mid-way through changing
|
||||
/// and a queue that will never drain.
|
||||
///
|
||||
/// Returns the ms at which the current run of deferrals began — `now` on the
|
||||
/// first one.
|
||||
pub fn defer_op_for_access(&self, id: i64, now: i64, next_attempt_at: i64) -> Result<i64> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"UPDATE pending_op
|
||||
SET next_attempt_at = CASE
|
||||
WHEN next_attempt_at >= ?4 THEN next_attempt_at
|
||||
ELSE ?3
|
||||
END,
|
||||
access_deferred_since = CASE
|
||||
WHEN access_deferred_since = 0 THEN ?2
|
||||
ELSE access_deferred_since
|
||||
END
|
||||
WHERE id = ?1",
|
||||
params![id, now, next_attempt_at, PARK_UNTIL],
|
||||
)?;
|
||||
let since = conn
|
||||
.query_row(
|
||||
"SELECT access_deferred_since FROM pending_op WHERE id = ?1",
|
||||
params![id],
|
||||
|r| r.get::<_, i64>(0),
|
||||
)
|
||||
.optional()?;
|
||||
// A row that drained out from under this call leaves nothing to measure;
|
||||
// reporting `now` says "this run just started", which is the reading that
|
||||
// never escalates something that no longer exists.
|
||||
Ok(since.filter(|since| *since != 0).unwrap_or(now))
|
||||
}
|
||||
|
||||
/// Forget an op's access-deferral window, after an attempt the access check
|
||||
/// let through. The op is no longer blocked, so a later deferral is the
|
||||
/// start of a new run rather than the continuation of an old one.
|
||||
pub fn clear_op_access_deferral(&self, id: i64) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"UPDATE pending_op SET access_deferred_since = 0
|
||||
WHERE id = ?1 AND access_deferred_since <> 0",
|
||||
params![id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Every staged blob the queue still refers to.
|
||||
///
|
||||
/// The startup staging reconcile subtracts this from what is actually in
|
||||
/// `staging/`: the difference is bytes the kernel was told were written and
|
||||
/// that nothing is going to upload.
|
||||
pub fn op_blob_paths(&self) -> Result<std::collections::HashSet<String>> {
|
||||
let conn = self.read();
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT blob_path FROM pending_op WHERE blob_path IS NOT NULL")?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
|
||||
let mut out = std::collections::HashSet::new();
|
||||
for row in rows {
|
||||
out.insert(row?);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Record that this daemon sealed `revision_id` on `uid`, and prune entries
|
||||
/// past [`OWN_SEALED_TTL_MS`].
|
||||
///
|
||||
/// One row per node, overwritten as the node advances. See
|
||||
/// [`own_sealed_rev`](Self::own_sealed_rev) for what reads it.
|
||||
pub fn set_own_sealed_rev(&self, uid: &str, revision_id: &str, now_ms: i64) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO own_sealed_rev (uid, revision_id, sealed_at)
|
||||
VALUES (?1, ?2, ?3)
|
||||
ON CONFLICT(uid) DO UPDATE SET
|
||||
revision_id = excluded.revision_id,
|
||||
sealed_at = excluded.sealed_at",
|
||||
params![uid, revision_id, now_ms],
|
||||
)?;
|
||||
conn.execute(
|
||||
"DELETE FROM own_sealed_rev WHERE sealed_at < ?1",
|
||||
params![now_ms - OWN_SEALED_TTL_MS],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The last revision of `uid` this daemon sealed itself, if it is still
|
||||
/// within [`OWN_SEALED_TTL_MS`].
|
||||
///
|
||||
/// The drain asks before treating a moved-on remote as a conflict: a
|
||||
/// revision we sealed means one writer stalled and resumed, which
|
||||
/// supersedes rather than forking a `(sync-conflict)` copy (docs/BUGS.md
|
||||
/// B70 layer B).
|
||||
pub fn own_sealed_rev(&self, uid: &str, now_ms: i64) -> Result<Option<String>> {
|
||||
let conn = self.read();
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT revision_id FROM own_sealed_rev
|
||||
WHERE uid = ?1 AND sealed_at >= ?2",
|
||||
params![uid, now_ms - OWN_SEALED_TTL_MS],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
/// Forget what we sealed on `uid` — it has been trashed, so there is no
|
||||
/// revision left to chain onto.
|
||||
pub fn clear_own_sealed_rev(&self, uid: &str) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute("DELETE FROM own_sealed_rev WHERE uid = ?1", params![uid])?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a revision this daemon sealed stays recognisable as its own.
|
||||
///
|
||||
/// Bounds a table that would otherwise grow with every write for the life of
|
||||
/// the install. A week is far longer than any stall→resume — a browser download
|
||||
/// that pauses and continues, an editor that reopens its handle — and short
|
||||
/// enough that the table tracks recent work rather than accumulating history.
|
||||
pub const OWN_SEALED_TTL_MS: i64 = 7 * 24 * 60 * 60 * 1000;
|
||||
@@ -115,7 +115,7 @@ impl Db {
|
||||
range: Option<(i64, i64)>,
|
||||
favorites: bool,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
// Built up so any combination of the optional filters is one indexed
|
||||
// query rather than a statement per case.
|
||||
let mut sql = String::from(
|
||||
@@ -174,7 +174,7 @@ impl Db {
|
||||
&self,
|
||||
kind: Option<crate::control::PhotoKind>,
|
||||
) -> Result<Vec<(i32, i32, usize)>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let (filter, binds): (&str, Vec<i64>) = match kind {
|
||||
Some(k) => (" WHERE kind = ?1", vec![k.as_i64()]),
|
||||
None => ("", Vec::new()),
|
||||
@@ -201,7 +201,7 @@ impl Db {
|
||||
/// Per-tab counts for the Photos page subtitle: `(photos, videos, raw)`.
|
||||
pub fn photos_counts(&self) -> Result<(usize, usize, usize)> {
|
||||
use crate::control::PhotoKind;
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare("SELECT kind, COUNT(*) FROM photos GROUP BY kind")?;
|
||||
let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?)))?;
|
||||
let (mut photos, mut videos, mut raw) = (0usize, 0usize, 0usize);
|
||||
@@ -223,7 +223,7 @@ impl Db {
|
||||
if uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let placeholders = vec!["?"; uids.len()].join(",");
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind, favorite FROM photos
|
||||
@@ -273,7 +273,7 @@ impl Db {
|
||||
|
||||
/// Number of photos in the persisted timeline.
|
||||
pub fn photos_count(&self) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let n: i64 = conn.query_row("SELECT COUNT(*) FROM photos", [], |r| r.get(0))?;
|
||||
Ok(n.max(0) as usize)
|
||||
}
|
||||
|
||||
@@ -4,8 +4,21 @@
|
||||
use rusqlite::{OptionalExtension, params};
|
||||
|
||||
use super::Db;
|
||||
use super::utils::MAX_PATH_DEPTH;
|
||||
use crate::Result;
|
||||
|
||||
/// One row of [`Db::pin_list`]. `recursive` is pin *policy* — whether the whole
|
||||
/// subtree is kept — while `is_dir` is the node's kind as the node cache knows
|
||||
/// it, or `None` for a pin whose node is not cached. Reading the former as the
|
||||
/// latter is what made a non-recursively pinned folder render as a file.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PinRow {
|
||||
pub uid: String,
|
||||
pub path: String,
|
||||
pub recursive: bool,
|
||||
pub is_dir: Option<bool>,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
pub fn pin_add(&self, uid: &str, path: &str, recursive: bool) -> Result<()> {
|
||||
let conn = self.conn.lock();
|
||||
@@ -25,16 +38,23 @@ impl Db {
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Every directly-pinned entry `(uid, path, recursive)`, ordered by uid.
|
||||
pub fn pin_list(&self) -> Result<Vec<(String, String, bool)>> {
|
||||
let conn = self.conn.lock();
|
||||
let mut stmt = conn.prepare("SELECT uid, path, recursive FROM pins ORDER BY uid")?;
|
||||
/// Every directly-pinned entry, ordered by uid.
|
||||
pub fn pin_list(&self) -> Result<Vec<PinRow>> {
|
||||
let conn = self.read();
|
||||
// `recursive` is pin policy; the node's kind comes from `nodes`, and is
|
||||
// NULL for a pin whose node was never cached.
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT p.uid, p.path, p.recursive, n.is_dir
|
||||
FROM pins p LEFT JOIN nodes n ON n.uid = p.uid
|
||||
ORDER BY p.uid",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, String>(1)?,
|
||||
r.get::<_, i64>(2)? != 0,
|
||||
))
|
||||
Ok(PinRow {
|
||||
uid: r.get(0)?,
|
||||
path: r.get(1)?,
|
||||
recursive: r.get::<_, i64>(2)? != 0,
|
||||
is_dir: r.get::<_, Option<i64>>(3)?.map(|is_dir| is_dir != 0),
|
||||
})
|
||||
})?;
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
@@ -47,8 +67,13 @@ impl Db {
|
||||
/// is pinned recursively. The ancestor check walks `parent_uid` to the root
|
||||
/// via a CTE; a direct pin is honoured even when the node has no `nodes` row
|
||||
/// yet (e.g. a CLI that never hydrates the node cache).
|
||||
///
|
||||
/// The walk is depth-capped like [`super::utils::path_of`]: a `parent_uid`
|
||||
/// cycle is corrupt data the API can hand us, and `UNION ALL` over one never
|
||||
/// terminates — while holding the daemon's only SQLite connection, on a path
|
||||
/// that runs per cached read.
|
||||
pub fn is_pinned(&self, uid: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let direct: Option<i64> = conn
|
||||
.query_row("SELECT 1 FROM pins WHERE uid = ?1", params![uid], |r| {
|
||||
r.get(0)
|
||||
@@ -59,14 +84,18 @@ impl Db {
|
||||
}
|
||||
let anc: Option<i64> = conn
|
||||
.query_row(
|
||||
"WITH RECURSIVE anc(uid, parent_uid) AS (
|
||||
SELECT uid, parent_uid FROM nodes WHERE uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.uid, n.parent_uid FROM nodes n JOIN anc ON n.uid = anc.parent_uid
|
||||
)
|
||||
SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid
|
||||
WHERE a.uid != ?1 AND p.recursive = 1
|
||||
LIMIT 1",
|
||||
&format!(
|
||||
"WITH RECURSIVE anc(uid, parent_uid, depth) AS (
|
||||
SELECT uid, parent_uid, 0 FROM nodes WHERE uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.uid, n.parent_uid, anc.depth + 1
|
||||
FROM nodes n JOIN anc ON n.uid = anc.parent_uid
|
||||
WHERE anc.depth < {MAX_PATH_DEPTH}
|
||||
)
|
||||
SELECT 1 FROM anc a JOIN pins p ON p.uid = a.uid
|
||||
WHERE a.uid != ?1 AND p.recursive = 1
|
||||
LIMIT 1"
|
||||
),
|
||||
params![uid],
|
||||
|r| r.get(0),
|
||||
)
|
||||
@@ -78,7 +107,7 @@ impl Db {
|
||||
/// every descendant of a recursively-pinned folder. Used to build the
|
||||
/// eviction-exempt set (the budget enforcer hashes these into cache keys).
|
||||
pub fn pinned_uids(&self) -> Result<Vec<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"WITH RECURSIVE sub(uid) AS (
|
||||
SELECT uid FROM pins WHERE recursive = 1
|
||||
@@ -100,7 +129,7 @@ impl Db {
|
||||
/// Every descendant uid of `folder` (all depths), via a `parent_uid` CTE.
|
||||
/// Used to evict a recursively-pinned subtree's cached blobs on unpin.
|
||||
pub fn descendants(&self, folder: &str) -> Result<Vec<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"WITH RECURSIVE sub(uid) AS (
|
||||
SELECT uid FROM nodes WHERE parent_uid = ?1
|
||||
|
||||
@@ -16,7 +16,7 @@ impl Db {
|
||||
/// remains the restart/offline authority and every explicit role change is
|
||||
/// written through before this cache is updated.
|
||||
pub fn all_share_access(&self) -> Result<HashMap<NodeUid, Access>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare("SELECT root_uid, access FROM share_access")?;
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
@@ -42,7 +42,7 @@ impl Db {
|
||||
|
||||
/// Read the effective access recorded for a shared-tree root.
|
||||
pub fn share_access(&self, uid: &NodeUid) -> Result<Option<Access>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let value = conn
|
||||
.query_row(
|
||||
"SELECT access FROM share_access WHERE root_uid = ?1",
|
||||
@@ -96,7 +96,7 @@ impl Db {
|
||||
/// share root. `None` means the node itself is not persisted, which callers
|
||||
/// must not treat as owned: stale handles need to fail closed.
|
||||
pub fn effective_node_access(&self, uid: &NodeUid) -> Result<Option<Access>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let exists: bool = conn.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM nodes WHERE uid = ?1)",
|
||||
[uid.to_string()],
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::Result;
|
||||
|
||||
impl Db {
|
||||
pub fn get_event_cursor(&self) -> Result<Option<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let v = conn
|
||||
.query_row(
|
||||
"SELECT value FROM sync_state WHERE key = 'event_cursor'",
|
||||
@@ -32,7 +32,7 @@ impl Db {
|
||||
|
||||
/// Read a `sync_state` value as a string.
|
||||
pub fn state_str(&self, key: &str) -> Result<Option<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let v = conn
|
||||
.query_row(
|
||||
"SELECT value FROM sync_state WHERE key = ?1",
|
||||
@@ -66,7 +66,7 @@ impl Db {
|
||||
/// dropping it would leave a file that exists nowhere but this machine with
|
||||
/// nothing left to create it. (Writes to a node that is itself still queued
|
||||
pub fn state_i64(&self, key: &str) -> Result<Option<i64>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let v: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT value FROM sync_state WHERE key = ?1",
|
||||
|
||||
@@ -35,6 +35,14 @@ pub struct StoredSyncEntry {
|
||||
pub remote_uid: Option<String>,
|
||||
pub local_mtime: i64,
|
||||
pub local_size: i64,
|
||||
/// The same modification time to nanosecond precision, when it is known.
|
||||
///
|
||||
/// `local_mtime` alone cannot tell an edit that landed inside the same
|
||||
/// second as the last sync — and left the file the same length — from no
|
||||
/// edit at all, so those never uploaded (bugs.md B25). `None` on a row
|
||||
/// written before schema 28, which the comparison reads as "compare whole
|
||||
/// seconds", so upgrading does not make an entire mirror look changed.
|
||||
pub local_mtime_ns: Option<i64>,
|
||||
pub remote_rev: Option<String>,
|
||||
pub remote_hash: Option<String>,
|
||||
}
|
||||
@@ -57,7 +65,7 @@ impl Db {
|
||||
|
||||
/// Every synced folder, oldest first.
|
||||
pub fn sync_folder_list(&self) -> Result<Vec<StoredSyncFolder>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync
|
||||
FROM sync_folder ORDER BY id",
|
||||
@@ -83,7 +91,7 @@ impl Db {
|
||||
|
||||
/// Look up one synced folder by id.
|
||||
pub fn sync_folder_get(&self, id: i64) -> Result<Option<StoredSyncFolder>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync
|
||||
FROM sync_folder WHERE id = ?1",
|
||||
@@ -166,9 +174,10 @@ impl Db {
|
||||
|
||||
/// The whole per-file sync baseline for a folder, keyed by relative path.
|
||||
pub fn sync_entries(&self, folder_id: i64) -> Result<HashMap<String, StoredSyncEntry>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash,
|
||||
local_mtime_ns
|
||||
FROM sync_entry WHERE folder_id = ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![folder_id], |r| {
|
||||
@@ -179,6 +188,7 @@ impl Db {
|
||||
local_size: r.get(3)?,
|
||||
remote_rev: r.get(4)?,
|
||||
remote_hash: r.get(5)?,
|
||||
local_mtime_ns: r.get(6)?,
|
||||
})
|
||||
})?;
|
||||
let mut map = HashMap::new();
|
||||
@@ -196,7 +206,7 @@ impl Db {
|
||||
/// `sync_folder` and descendants come from the last successful
|
||||
/// `sync_entry` baseline.
|
||||
pub fn mirror_contains_uid(&self, uid: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT EXISTS(
|
||||
SELECT 1
|
||||
@@ -216,14 +226,16 @@ impl Db {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO sync_entry
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev,
|
||||
remote_hash, local_mtime_ns)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
ON CONFLICT(folder_id, rel_path) DO UPDATE SET
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash",
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash,
|
||||
local_mtime_ns = excluded.local_mtime_ns",
|
||||
params![
|
||||
folder_id,
|
||||
e.rel_path,
|
||||
@@ -232,6 +244,7 @@ impl Db {
|
||||
e.local_size,
|
||||
e.remote_rev,
|
||||
e.remote_hash,
|
||||
e.local_mtime_ns,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
|
||||
@@ -867,6 +867,67 @@ fn shared_root_pinned_name_survives_database_reopen() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
/// Two writers on one database is the failure mode the single-instance lock
|
||||
/// exists to prevent — a hand-run `pdfs mount` next to the systemd unit gives
|
||||
/// two daemons the same inode space, content cache, and drain queue. The second
|
||||
/// open has to fail, and it has to stop failing once the first handle is gone.
|
||||
#[test]
|
||||
fn a_second_writer_is_refused_while_the_first_holds_the_database() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-single-writer-{}-{}.db",
|
||||
std::process::id(),
|
||||
now_test_id()
|
||||
));
|
||||
let first = Db::open(&path).unwrap();
|
||||
let second = Db::open(&path);
|
||||
assert!(second.is_err(), "a second writer was allowed in");
|
||||
|
||||
drop(first);
|
||||
// The lock lives in the kernel and dies with the descriptor, so nothing has
|
||||
// to be cleaned up for the next daemon to start.
|
||||
Db::open(&path).expect("reopen after the first handle is dropped");
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(path.with_extension("db.lock"));
|
||||
}
|
||||
|
||||
/// `cache.db` is derived state — every row can be re-fetched or re-scanned — so
|
||||
/// a file that is not a database at all must not stop the daemon starting. It
|
||||
/// is moved aside (not deleted: it is the only evidence) and rebuilt.
|
||||
#[test]
|
||||
fn a_corrupt_database_is_moved_aside_and_rebuilt() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-corrupt-{}-{}.db",
|
||||
std::process::id(),
|
||||
now_test_id()
|
||||
));
|
||||
std::fs::write(&path, b"this is not an SQLite database").unwrap();
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.set_state_str("shared_with_me_name", "rebuilt").unwrap();
|
||||
assert_eq!(
|
||||
db.state_str("shared_with_me_name").unwrap().as_deref(),
|
||||
Some("rebuilt")
|
||||
);
|
||||
drop(db);
|
||||
|
||||
let dir = path.parent().unwrap();
|
||||
let stem = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let quarantined: Vec<_> = std::fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.file_name().to_string_lossy().to_string())
|
||||
.filter(|name| name.starts_with(&format!("{stem}.corrupt-")))
|
||||
.collect();
|
||||
assert_eq!(quarantined.len(), 1, "the damaged file was not kept");
|
||||
|
||||
for name in quarantined {
|
||||
let _ = std::fs::remove_file(dir.join(name));
|
||||
}
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(path.with_extension("db.lock"));
|
||||
}
|
||||
|
||||
fn local(path: &str, name: &str, is_dir: bool) -> LocalEntry {
|
||||
LocalEntry {
|
||||
path: path.into(),
|
||||
@@ -911,6 +972,25 @@ fn local_scan_indexes_then_prunes_stale_paths() {
|
||||
assert_eq!(db.search_local("notes", 10).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
/// The index is maintained incrementally rather than rebuilt at the end of a
|
||||
/// scan, so the case that used to be covered for free — a file the scan sees
|
||||
/// again — needs its own test: a re-scan must neither lose the entry nor
|
||||
/// duplicate it.
|
||||
#[test]
|
||||
fn rescanning_an_unchanged_file_keeps_exactly_one_index_entry() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
for (generation, mtime) in [(1_000, 5), (2_000, 9)] {
|
||||
let g = db.local_begin_scan().unwrap();
|
||||
let mut e = local("/home/u/docs/report.pdf", "report.pdf", false);
|
||||
e.mtime = mtime;
|
||||
db.local_upsert_batch(g, &[e]).unwrap();
|
||||
assert_eq!(db.local_finish_scan(g, generation).unwrap(), 1);
|
||||
let hits = db.search_local("report", 10).unwrap();
|
||||
assert_eq!(hits.len(), 1, "generation {generation}");
|
||||
assert_eq!(hits[0].mtime, mtime);
|
||||
}
|
||||
}
|
||||
|
||||
/// Queries below the trigram minimum still match, via the `LIKE` fallback.
|
||||
#[test]
|
||||
fn local_search_short_query_like_fallback() {
|
||||
@@ -989,6 +1069,62 @@ fn search_excludes_trashed_and_respects_limit() {
|
||||
assert_eq!(db.search("alph", 1).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
/// A device folder's root is stored in `device`/`sync_folder`, never in
|
||||
/// `nodes`, so its children's `parent_uid` points at a row that does not exist.
|
||||
/// Requiring the ancestor walk to reach a `parent_uid IS NULL` row therefore
|
||||
/// excluded every device-folder subtree from search: 2,705 of 9,252 nodes on
|
||||
/// the account this was found on, including everything under `~/Downloads`.
|
||||
#[test]
|
||||
fn search_finds_nodes_below_an_uncached_parent() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
// "device-root" is deliberately never upserted.
|
||||
db.upsert_node(&folder("downloads", Some("device-root"), "Downloads"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("t", "downloads", "tickets.pdf", 1))
|
||||
.unwrap();
|
||||
|
||||
let hits = db.search("tickets", 10).unwrap();
|
||||
assert_eq!(hits.len(), 1, "a device folder's files must be searchable");
|
||||
assert_eq!(hits[0].path, "Downloads/tickets.pdf");
|
||||
}
|
||||
|
||||
/// The relaxed rule must not also start indexing trash: a node whose *parent*
|
||||
/// is trashed stays out even though the walk now terminates happily.
|
||||
#[test]
|
||||
fn search_excludes_a_node_under_a_trashed_ancestor() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
let mut folder_gone = folder("boxed", Some("root"), "Boxed");
|
||||
folder_gone.trashed = true;
|
||||
db.upsert_node(&folder_gone).unwrap();
|
||||
db.upsert_node(&file("f1", "boxed", "buried.txt", 1))
|
||||
.unwrap();
|
||||
|
||||
assert!(db.search("buried", 10).unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// A parent cycle is corrupt data the API can still hand us. It must stay out
|
||||
/// of the index, and — because the short-query `LIKE` lane reads `nodes`
|
||||
/// directly and never consults the index — resolving one's path must terminate
|
||||
/// rather than spin inside the daemon's only SQLite connection.
|
||||
#[test]
|
||||
fn a_parent_cycle_is_unindexed_and_never_hangs_a_search() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("a", Some("b"), "Alphaville"))
|
||||
.unwrap();
|
||||
db.upsert_node(&folder("b", Some("a"), "Betamax")).unwrap();
|
||||
|
||||
// Index lane: the cycle is not indexed, so it cannot be found.
|
||||
assert!(db.search("alphaville", 10).unwrap().is_empty());
|
||||
assert!(db.search("betamax", 10).unwrap().is_empty());
|
||||
|
||||
// LIKE lane (below the trigram minimum): the rows are reachable, and what
|
||||
// matters is that resolving their paths returns at all.
|
||||
let hits = db.search("al", 10).unwrap();
|
||||
assert_eq!(hits.len(), 1);
|
||||
assert!(hits[0].path.ends_with("Alphaville"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_short_query_like_fallback() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -1373,6 +1509,7 @@ fn migration_v17_adds_share_access_to_a_v16_fixture() {
|
||||
DROP TABLE albums;
|
||||
DROP INDEX idx_photos_favorite;
|
||||
ALTER TABLE photos DROP COLUMN favorite;
|
||||
ALTER TABLE nodes DROP COLUMN path;
|
||||
UPDATE sync_state SET value = '16' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
@@ -1758,6 +1895,123 @@ fn failed_superseding_insert_keeps_the_old_pending_op() {
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||
}
|
||||
|
||||
/// B53's remaining half. The insert-failure regression above proves the *row*
|
||||
/// rolls back; this proves the two things that make the rollback usable — the
|
||||
/// old blob is not reported as superseded (so the caller does not delete the
|
||||
/// bytes the surviving row still points at), and the rollback survives a
|
||||
/// restart rather than living only in the connection that failed.
|
||||
#[test]
|
||||
fn a_rolled_back_supersede_keeps_its_blob_and_survives_reopen() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-db-b53-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("queue.sqlite");
|
||||
let op = |blob: &str| PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("durable").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some(blob.to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch(
|
||||
"CREATE TRIGGER reject_replacement BEFORE INSERT ON pending_op
|
||||
WHEN NEW.blob_path = '/staging/replacement'
|
||||
BEGIN SELECT RAISE(ABORT, 'injected insert failure'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// The failure must not hand back the old blob path. That value is the
|
||||
// caller's instruction to delete those bytes, and the row that owns
|
||||
// them is still queued.
|
||||
assert!(db.enqueue_op(&op("/staging/replacement")).is_err());
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER reject_replacement")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1, "old or new — never neither");
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||
|
||||
// And once the injected failure is gone, superseding works and *does*
|
||||
// report the blob it retired.
|
||||
let (_, superseded) = db.enqueue_op(&op("/staging/replacement")).unwrap();
|
||||
assert_eq!(superseded.as_deref(), Some("/staging/original"));
|
||||
assert_eq!(db.pending_ops().unwrap().len(), 1);
|
||||
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
/// A full disk, as SQLite reports it. `max_page_count` makes the database
|
||||
/// refuse to grow, which is the same `SQLITE_FULL` an out-of-space filesystem
|
||||
/// produces — and the queue must come out of it with the acknowledged work
|
||||
/// still in it (B53).
|
||||
#[test]
|
||||
fn a_full_database_rolls_the_supersede_back_rather_than_erasing_it() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let op = |blob: &str| PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("full").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some(blob.to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||
|
||||
// Cap the file at its current size, so any page the insert needs fails.
|
||||
let pages: i64 = db
|
||||
.with_conn(|conn| Ok(conn.query_row("PRAGMA page_count", [], |r| r.get(0))?))
|
||||
.unwrap();
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch(&format!("PRAGMA max_page_count = {pages}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// A payload far larger than one page, so growth is unavoidable.
|
||||
let mut fat = op("/staging/replacement");
|
||||
fat.meta_json = Some("x".repeat(512 * 1024));
|
||||
let result = db.enqueue_op(&fat);
|
||||
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch("PRAGMA max_page_count = 1073741823")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(result.is_err(), "a database that cannot grow must refuse");
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(
|
||||
ops[0].blob_path.as_deref(),
|
||||
Some("/staging/original"),
|
||||
"the acknowledged upload is still queued and still owns its bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_trash_replacement_returns_owned_blobs_after_commit() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -2271,6 +2525,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -2287,6 +2542,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -2472,14 +2728,41 @@ fn pin_add_list_remove_roundtrip() {
|
||||
db.pin_add("vol~d", "docs", true).unwrap();
|
||||
let list = db.pin_list().unwrap();
|
||||
assert_eq!(list.len(), 2);
|
||||
assert_eq!(list[0], ("vol~a".into(), "docs/a.txt".into(), false));
|
||||
assert_eq!(list[1], ("vol~d".into(), "docs".into(), true));
|
||||
// No `nodes` row for either pin, so the kind is unknown rather than guessed
|
||||
// from the recursive flag.
|
||||
assert_eq!(
|
||||
list[0],
|
||||
PinRow {
|
||||
uid: "vol~a".into(),
|
||||
path: "docs/a.txt".into(),
|
||||
recursive: false,
|
||||
is_dir: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
list[1],
|
||||
PinRow {
|
||||
uid: "vol~d".into(),
|
||||
path: "docs".into(),
|
||||
recursive: true,
|
||||
is_dir: None,
|
||||
}
|
||||
);
|
||||
|
||||
// Once the node is cached, the pin reports the real kind — a folder pinned
|
||||
// non-recursively must not look like a file.
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("d", Some("root"), "docs")).unwrap();
|
||||
db.pin_add("vol~d", "docs", false).unwrap();
|
||||
let list = db.pin_list().unwrap();
|
||||
assert!(!list[1].recursive, "pin policy stays non-recursive");
|
||||
assert_eq!(list[1].is_dir, Some(true), "but the node is still a folder");
|
||||
|
||||
// Re-pin refreshes path/flag, not a duplicate row.
|
||||
db.pin_add("vol~a", "moved/a.txt", false).unwrap();
|
||||
let list = db.pin_list().unwrap();
|
||||
assert_eq!(list.len(), 2);
|
||||
assert_eq!(list[0].1, "moved/a.txt");
|
||||
assert_eq!(list[0].path, "moved/a.txt");
|
||||
|
||||
assert!(db.pin_remove("vol~a").unwrap());
|
||||
assert!(!db.pin_remove("vol~a").unwrap());
|
||||
@@ -2545,6 +2828,26 @@ fn non_recursive_folder_pin_does_not_cover_children() {
|
||||
assert!(!db.is_pinned(&du("rep")).unwrap());
|
||||
}
|
||||
|
||||
/// A `parent_uid` cycle is corrupt data the API can hand us, and `is_pinned`'s
|
||||
/// ancestor walk is `UNION ALL` — uncapped it never terminates, while holding
|
||||
/// the daemon's only SQLite connection, on a path that runs per cached read.
|
||||
/// The answer here matters less than the fact that there is one.
|
||||
#[test]
|
||||
fn is_pinned_terminates_on_a_parent_cycle() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let du = |l: &str| uid(l).to_string();
|
||||
// a -> b -> a, with nothing pinned above either.
|
||||
db.upsert_node(&folder("a", Some("b"), "A")).unwrap();
|
||||
db.upsert_node(&folder("b", Some("a"), "B")).unwrap();
|
||||
db.upsert_node(&file("leaf", "a", "leaf.txt", 1)).unwrap();
|
||||
|
||||
assert!(!db.is_pinned(&du("leaf")).unwrap());
|
||||
|
||||
// And a pin *inside* the cycle is still found rather than walked past.
|
||||
db.pin_add(&du("a"), "A", true).unwrap();
|
||||
assert!(db.is_pinned(&du("leaf")).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_objects_exist() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -3099,3 +3402,502 @@ fn clear_state_prefix_takes_only_the_prefixed_keys() {
|
||||
"the listing's own stamp is not a per-album one"
|
||||
);
|
||||
}
|
||||
|
||||
// --- `nodes.path`, and the subtree work it replaces (DB3) -------------------
|
||||
|
||||
/// Read the stored path column directly: the point of these tests is that the
|
||||
/// column itself is right, not that `path_of` can still fall back to a walk.
|
||||
fn stored_path(db: &Db, link: &str) -> Option<String> {
|
||||
use rusqlite::OptionalExtension;
|
||||
db.with_conn(|conn| {
|
||||
Ok(conn
|
||||
.query_row(
|
||||
"SELECT path FROM nodes WHERE uid = ?1",
|
||||
[uid(link).to_string()],
|
||||
|row| row.get::<_, Option<String>>(0),
|
||||
)
|
||||
.optional()?
|
||||
.flatten())
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_node_stores_its_path_when_it_is_written() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("docs", Some("root"), "Documents"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("f1", "docs", "report.pdf", 1))
|
||||
.unwrap();
|
||||
|
||||
// The root's name is the mount, so it contributes nothing.
|
||||
assert_eq!(stored_path(&db, "root").as_deref(), Some(""));
|
||||
assert_eq!(stored_path(&db, "docs").as_deref(), Some("Documents"));
|
||||
assert_eq!(
|
||||
stored_path(&db, "f1").as_deref(),
|
||||
Some("Documents/report.pdf")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_node_below_an_uncached_parent_starts_its_own_path() {
|
||||
// A device folder's root lives in `device`/`sync_folder` and never gets a
|
||||
// `nodes` row, so its children are the top of the cached chain.
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("orphan", Some("never-cached"), "Work"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("f1", "orphan", "notes.txt", 1))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(stored_path(&db, "orphan").as_deref(), Some("Work"));
|
||||
assert_eq!(stored_path(&db, "f1").as_deref(), Some("Work/notes.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renaming_a_folder_rewrites_every_descendant_path() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("a", Some("root"), "Before"))
|
||||
.unwrap();
|
||||
db.upsert_node(&folder("b", Some("a"), "Inner")).unwrap();
|
||||
db.upsert_node(&file("f1", "b", "deep.txt", 1)).unwrap();
|
||||
assert_eq!(
|
||||
stored_path(&db, "f1").as_deref(),
|
||||
Some("Before/Inner/deep.txt")
|
||||
);
|
||||
|
||||
db.upsert_node(&folder("a", Some("root"), "After")).unwrap();
|
||||
|
||||
assert_eq!(stored_path(&db, "a").as_deref(), Some("After"));
|
||||
assert_eq!(stored_path(&db, "b").as_deref(), Some("After/Inner"));
|
||||
assert_eq!(
|
||||
stored_path(&db, "f1").as_deref(),
|
||||
Some("After/Inner/deep.txt")
|
||||
);
|
||||
let hit = &db.search("deep.txt", 10).unwrap()[0];
|
||||
assert_eq!(
|
||||
hit.path, "After/Inner/deep.txt",
|
||||
"the search index must carry the rewritten path too"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn moving_a_folder_rewrites_every_descendant_path() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("old", Some("root"), "Old")).unwrap();
|
||||
db.upsert_node(&folder("new", Some("root"), "New")).unwrap();
|
||||
db.upsert_node(&folder("moved", Some("old"), "Moved"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("f1", "moved", "deep.txt", 1)).unwrap();
|
||||
|
||||
db.upsert_node(&folder("moved", Some("new"), "Moved"))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(stored_path(&db, "moved").as_deref(), Some("New/Moved"));
|
||||
assert_eq!(
|
||||
stored_path(&db, "f1").as_deref(),
|
||||
Some("New/Moved/deep.txt")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trashing_a_folder_drops_its_subtree_from_the_index() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("a", Some("root"), "Holder"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("f1", "a", "hidden.txt", 1)).unwrap();
|
||||
assert_eq!(db.search("hidden", 10).unwrap().len(), 1);
|
||||
|
||||
let mut trashed = folder("a", Some("root"), "Holder");
|
||||
trashed.trashed = true;
|
||||
db.upsert_node(&trashed).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
db.search("hidden", 10).unwrap().len(),
|
||||
0,
|
||||
"a file under a trashed folder is not reachable and must not be findable"
|
||||
);
|
||||
|
||||
// …and comes back when the folder does.
|
||||
db.upsert_node(&folder("a", Some("root"), "Holder"))
|
||||
.unwrap();
|
||||
assert_eq!(db.search("hidden", 10).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
/// The re-listing case, which is most of what the write path actually does: a
|
||||
/// folder upserted with the same name and parent must not touch its subtree.
|
||||
///
|
||||
/// Written as a ratio between two subtree sizes, so it fails on the scaling
|
||||
/// regression rather than on a slow machine. Before this, every re-listing paid
|
||||
/// a `path_of` walk plus an ancestor walk per descendant.
|
||||
#[test]
|
||||
fn re_listing_a_folder_does_not_scale_with_its_subtree() {
|
||||
fn elapsed_for(children: usize) -> std::time::Duration {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("big", Some("root"), "Big")).unwrap();
|
||||
let kids: Vec<Node> = (0..children)
|
||||
.map(|i| file(&format!("k{i}"), "big", &format!("child-{i}.txt"), 1))
|
||||
.collect();
|
||||
db.upsert_nodes(&kids).unwrap();
|
||||
|
||||
let unchanged = folder("big", Some("root"), "Big");
|
||||
let start = std::time::Instant::now();
|
||||
for _ in 0..20 {
|
||||
db.upsert_node(&unchanged).unwrap();
|
||||
}
|
||||
start.elapsed()
|
||||
}
|
||||
|
||||
let small = elapsed_for(100);
|
||||
let large = elapsed_for(2_000);
|
||||
assert!(
|
||||
large < small * 8,
|
||||
"re-listing a folder scaled with its subtree: {small:?} for 100 children, \
|
||||
{large:?} for 2000 (20x the descendants)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_v25_backfills_paths_for_an_existing_tree() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v24-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("docs", Some("root"), "Documents"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("f1", "docs", "report.pdf", 1))
|
||||
.unwrap();
|
||||
db.upsert_node(&folder("orphan", Some("never-cached"), "Work"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("f2", "orphan", "notes.txt", 1))
|
||||
.unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V24 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE nodes DROP COLUMN path;
|
||||
UPDATE sync_state SET value = '24' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
assert_eq!(stored_path(&db, "root").as_deref(), Some(""));
|
||||
assert_eq!(stored_path(&db, "docs").as_deref(), Some("Documents"));
|
||||
assert_eq!(
|
||||
stored_path(&db, "f1").as_deref(),
|
||||
Some("Documents/report.pdf"),
|
||||
"an existing tree must come out of the migration with the same paths \
|
||||
the walk used to produce"
|
||||
);
|
||||
// A subtree whose top has an uncached parent is ordinary data, not an error.
|
||||
assert_eq!(stored_path(&db, "orphan").as_deref(), Some("Work"));
|
||||
assert_eq!(stored_path(&db, "f2").as_deref(), Some("Work/notes.txt"));
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// A queue carried over from V26 keeps its rows, and every one of them starts
|
||||
/// out not access-deferred — a migration must not invent a deferral window that
|
||||
/// would report an ordinary queued upload as blocked.
|
||||
#[test]
|
||||
fn migration_v27_adds_the_access_deferral_column_to_a_v26_queue() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v26-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.enqueue_op(&crate::db::PendingOp {
|
||||
id: 0,
|
||||
kind: crate::db::OP_REVISION.to_string(),
|
||||
uid: "v~l".to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some("/tmp/blob".to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V26 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE pending_op DROP COLUMN access_deferred_since;
|
||||
UPDATE sync_state SET value = '26' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1, "the migration keeps the queued write");
|
||||
let now = 5_000i64;
|
||||
assert_eq!(
|
||||
db.defer_op_for_access(ops[0].id, now, now + 5_000).unwrap(),
|
||||
now,
|
||||
"a carried-over op is not already deferred"
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// A mirror baseline carried over from V27 keeps its rows, and each one reads
|
||||
/// back with no nanosecond time. That `None` is the whole point: it makes the
|
||||
/// comparison fall back to whole seconds for rows written before the column
|
||||
/// existed, instead of asserting a sub-second time of zero and re-uploading
|
||||
/// every already-synced file in the folder (bugs.md B25).
|
||||
#[test]
|
||||
fn migration_v28_adds_sub_second_local_times_without_disturbing_a_v27_baseline() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v27-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
let folder = db
|
||||
.sync_folder_add("/home/me/Mirror", "vol~root", "share")
|
||||
.unwrap();
|
||||
db.sync_entry_upsert(
|
||||
folder,
|
||||
&StoredSyncEntry {
|
||||
rel_path: "notes.txt".into(),
|
||||
remote_uid: Some("vol~notes".into()),
|
||||
local_mtime: 1_700_000_000,
|
||||
local_mtime_ns: Some(1_700_000_000_123_456_789),
|
||||
local_size: 42,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("42".into()),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V27 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE sync_entry DROP COLUMN local_mtime_ns;
|
||||
UPDATE sync_state SET value = '27' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let entries = db.sync_entries(1).unwrap();
|
||||
let entry = entries.get("notes.txt").expect("the baseline row survives");
|
||||
assert_eq!(entry.local_mtime, 1_700_000_000);
|
||||
assert_eq!(entry.local_size, 42);
|
||||
assert_eq!(
|
||||
entry.local_mtime_ns, None,
|
||||
"a carried-over row must not claim a sub-second time it never recorded"
|
||||
);
|
||||
|
||||
// And the column is live: a refreshed row keeps the precision it is given.
|
||||
db.sync_entry_upsert(
|
||||
1,
|
||||
&StoredSyncEntry {
|
||||
local_mtime_ns: Some(1_700_000_001_000_000_500),
|
||||
..entry.clone()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.sync_entries(1).unwrap()["notes.txt"].local_mtime_ns,
|
||||
Some(1_700_000_001_000_000_500)
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
// --- read-only connection pool (DB2) ---------------------------------------
|
||||
|
||||
/// A read must not queue behind whatever happens to be writing.
|
||||
///
|
||||
/// The write connection is held for the whole measurement — which is what a
|
||||
/// large listing's commit does to it — while a reader asks for a search. Before
|
||||
/// the pool this was the same mutex, so the read waited for the writer;
|
||||
/// `busy_timeout` never entered into it, because the contention was this
|
||||
/// process's own lock rather than SQLite's.
|
||||
#[test]
|
||||
fn a_read_does_not_wait_for_the_write_connection() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-readpool-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let db = std::sync::Arc::new(Db::open(&path).unwrap());
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&file("f1", "root", "findme.txt", 1))
|
||||
.unwrap();
|
||||
|
||||
let held = std::time::Duration::from_millis(500);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let writer = {
|
||||
let db = db.clone();
|
||||
std::thread::spawn(move || {
|
||||
db.with_conn(|_conn| {
|
||||
tx.send(()).unwrap();
|
||||
std::thread::sleep(held);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
})
|
||||
};
|
||||
rx.recv().unwrap();
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let hits = db.search("findme", 10).unwrap();
|
||||
let waited = start.elapsed();
|
||||
writer.join().unwrap();
|
||||
|
||||
assert_eq!(hits.len(), 1, "the read must still see committed data");
|
||||
assert!(
|
||||
waited < held / 2,
|
||||
"the read waited {waited:?} for a write connection held {held:?}"
|
||||
);
|
||||
|
||||
drop(db);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// §5: the drain is several workers over one queue, and the claim column is the
|
||||
/// only thing keeping them off each other's work. Two properties matter — no
|
||||
/// row is ever handed to two workers, and no *node* is ever worked by two
|
||||
/// workers even though its ops are separate rows.
|
||||
#[test]
|
||||
fn a_claimed_op_is_invisible_to_every_other_worker() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let root = uid("root").to_string();
|
||||
for i in 0..3 {
|
||||
db.enqueue_op(&bulk_op(i, &root)).unwrap();
|
||||
}
|
||||
|
||||
let first = db.claim_next_due_op(10).unwrap().expect("an op is due");
|
||||
let second = db
|
||||
.claim_next_due_op(10)
|
||||
.unwrap()
|
||||
.expect("another op is due");
|
||||
let third = db
|
||||
.claim_next_due_op(10)
|
||||
.unwrap()
|
||||
.expect("a third op is due");
|
||||
assert!(
|
||||
db.claim_next_due_op(10).unwrap().is_none(),
|
||||
"a fourth claim must find nothing left: every queued op is taken"
|
||||
);
|
||||
|
||||
let mut ids = [first.id, second.id, third.id];
|
||||
ids.sort_unstable();
|
||||
assert_eq!(
|
||||
ids,
|
||||
[1, 2, 3],
|
||||
"each worker must get a distinct row, oldest first"
|
||||
);
|
||||
|
||||
// Releasing puts one back, and it is the one that comes out next.
|
||||
db.release_op_claim(second.id).unwrap();
|
||||
let again = db.claim_next_due_op(10).unwrap().expect("the released op");
|
||||
assert_eq!(again.id, second.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_workers_never_share_a_node() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let root = uid("root").to_string();
|
||||
// Two ops of different kinds against one node — a queued rename and a
|
||||
// queued revision, which is an ordinary `mv` plus a save.
|
||||
let mut revision = bulk_op(0, &root);
|
||||
revision.parent_uid = None;
|
||||
db.enqueue_op(&revision).unwrap();
|
||||
let rename = PendingOp {
|
||||
kind: OP_RENAME.to_string(),
|
||||
parent_uid: Some(root.clone()),
|
||||
blob_path: None,
|
||||
meta_json: None,
|
||||
..bulk_op(0, &root)
|
||||
};
|
||||
db.enqueue_op(&rename).unwrap();
|
||||
|
||||
let claimed = db.claim_next_due_op(10).unwrap().expect("an op is due");
|
||||
assert_eq!(claimed.kind, OP_REVISION, "the older op goes first");
|
||||
assert!(
|
||||
db.claim_next_due_op(10).unwrap().is_none(),
|
||||
"the second op targets the same node, so it must wait for the first: \
|
||||
two workers on one uid would race over its staged blob and land its \
|
||||
changes out of order"
|
||||
);
|
||||
|
||||
db.release_op_claim(claimed.id).unwrap();
|
||||
db.delete_op(claimed.id).unwrap();
|
||||
assert_eq!(
|
||||
db.claim_next_due_op(10).unwrap().map(|o| o.kind),
|
||||
Some(OP_RENAME.to_string()),
|
||||
"once the revision retires, the rename is claimable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claims_do_not_survive_the_process_that_took_them() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let root = uid("root").to_string();
|
||||
for i in 0..2 {
|
||||
db.enqueue_op(&bulk_op(i, &root)).unwrap();
|
||||
}
|
||||
db.claim_next_due_op(10).unwrap().unwrap();
|
||||
db.claim_next_due_op(10).unwrap().unwrap();
|
||||
assert!(db.claim_next_due_op(10).unwrap().is_none());
|
||||
|
||||
// What `Db::open` does after the single-writer lock says the previous run
|
||||
// is gone. Without it those ops are invisible to every worker forever.
|
||||
assert_eq!(db.clear_op_claims().unwrap(), 2);
|
||||
assert!(
|
||||
db.claim_next_due_op(10).unwrap().is_some(),
|
||||
"a crashed run's claims must not park its queue"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_idle_worker_does_not_wait_on_an_op_another_worker_has() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let root = uid("root").to_string();
|
||||
db.enqueue_op(&bulk_op(0, &root)).unwrap();
|
||||
|
||||
assert_eq!(db.earliest_due_at().unwrap(), Some(0));
|
||||
db.claim_next_due_op(10).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
db.earliest_due_at().unwrap(),
|
||||
None,
|
||||
"an op somebody else is draining is not work this worker is waiting for"
|
||||
);
|
||||
}
|
||||
@@ -39,7 +39,7 @@ impl Db {
|
||||
/// The persisted trash listing, folders first then by name — the order the
|
||||
/// Trash page shows it in.
|
||||
pub fn trash_list(&self) -> Result<Vec<StoredTrash>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid, name, is_dir, size, mtime FROM trash
|
||||
ORDER BY is_dir DESC, name COLLATE NOCASE",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Small query helpers shared across the table modules.
|
||||
|
||||
use rusqlite::{Connection, params};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::Result;
|
||||
|
||||
@@ -8,14 +8,19 @@ use crate::Result;
|
||||
/// grams), so short queries fall back to a `LIKE` scan over `nodes.name`.
|
||||
pub(super) const TRIGRAM_MIN: usize = 3;
|
||||
|
||||
pub(super) fn pair(row: &rusqlite::Row) -> rusqlite::Result<(String, String)> {
|
||||
Ok((row.get(0)?, row.get(1)?))
|
||||
/// One raw search row: the node's JSON, its uid, and its stored path — `None`
|
||||
/// only for a row written before the `path` column existed, which
|
||||
/// [`path_of`] repairs by walking.
|
||||
pub(super) type HitRow = (String, String, Option<String>);
|
||||
|
||||
pub(super) fn hit_row(row: &rusqlite::Row) -> rusqlite::Result<HitRow> {
|
||||
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
|
||||
}
|
||||
|
||||
/// Drain a `query_map` of [`pair`] rows into a `Vec`, propagating row errors.
|
||||
pub(super) fn collect_pairs(
|
||||
rows: impl Iterator<Item = rusqlite::Result<(String, String)>>,
|
||||
) -> Result<Vec<(String, String)>> {
|
||||
/// Drain a `query_map` of [`hit_row`] rows into a `Vec`, propagating row errors.
|
||||
pub(super) fn collect_hits(
|
||||
rows: impl Iterator<Item = rusqlite::Result<HitRow>>,
|
||||
) -> Result<Vec<HitRow>> {
|
||||
let mut out = Vec::new();
|
||||
for r in rows {
|
||||
out.push(r?);
|
||||
@@ -31,19 +36,58 @@ pub(super) fn like_escape(s: &str) -> String {
|
||||
.replace('_', "\\_")
|
||||
}
|
||||
|
||||
/// Resolve a node's mountpoint-relative path by walking `parent_uid` to the
|
||||
/// root via a recursive CTE. The root (the node with no parent) is excluded, so
|
||||
/// a top-level file `report.pdf` yields `"report.pdf"`, not `"My Files/report.pdf"`.
|
||||
/// How deep the parent walk may go before it gives up. Real Drive trees are
|
||||
/// nowhere near this; a parent *cycle* is unbounded, and this query is on the
|
||||
/// path of every search result, including the `LIKE` lane that reads `nodes`
|
||||
/// directly and so never passed through the index's cycle check.
|
||||
pub(super) const MAX_PATH_DEPTH: usize = 256;
|
||||
|
||||
/// Append `name` to a parent's stored path. An empty parent path is the root,
|
||||
/// whose name is the mount and so contributes nothing.
|
||||
pub(super) fn join_path(parent_path: &str, name: &str) -> String {
|
||||
if parent_path.is_empty() {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("{parent_path}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a node's mountpoint-relative path. The root (the node with no
|
||||
/// parent) is excluded, so a top-level file `report.pdf` yields
|
||||
/// `"report.pdf"`, not `"My Files/report.pdf"`.
|
||||
///
|
||||
/// Reads the stored `nodes.path`, which the write path maintains (see
|
||||
/// `upsert_node_tx`). Falls back to [`walk_path_of`] for a row that predates the
|
||||
/// column or was never written through — a search serving a keystroke used to
|
||||
/// run that walk once per candidate, which is the reason the column exists.
|
||||
pub(super) fn path_of(conn: &Connection, uid: &str) -> Result<String> {
|
||||
let mut stmt = conn.prepare(
|
||||
let stored: Option<Option<String>> = conn
|
||||
.query_row("SELECT path FROM nodes WHERE uid = ?1", params![uid], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.optional()?;
|
||||
match stored.flatten() {
|
||||
Some(path) => Ok(path),
|
||||
None => walk_path_of(conn, uid),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a path by walking `parent_uid` to the root via a recursive CTE.
|
||||
///
|
||||
/// A cycle in `parent_uid` is corrupt data the API can still hand us, and
|
||||
/// `UNION ALL` over one never terminates — the walk is therefore depth-capped
|
||||
/// and returns the truncated path rather than hanging the caller.
|
||||
pub(super) fn walk_path_of(conn: &Connection, uid: &str) -> Result<String> {
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"WITH RECURSIVE anc(uid, parent_uid, name, depth) AS (
|
||||
SELECT uid, parent_uid, name, 0 FROM nodes WHERE uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.uid, n.parent_uid, n.name, anc.depth + 1
|
||||
FROM nodes n JOIN anc ON n.uid = anc.parent_uid
|
||||
WHERE anc.depth < {MAX_PATH_DEPTH}
|
||||
)
|
||||
SELECT name, parent_uid FROM anc ORDER BY depth DESC",
|
||||
)?;
|
||||
SELECT name, parent_uid FROM anc ORDER BY depth DESC"
|
||||
))?;
|
||||
let rows = stmt.query_map(params![uid], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, Option<String>>(1)?))
|
||||
})?;
|
||||
|
||||
@@ -45,6 +45,27 @@ pub enum Error {
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// Whether this is the local disk being full (`ENOSPC`), as opposed to any
|
||||
/// other I/O failure.
|
||||
///
|
||||
/// Worth distinguishing because it is the one storage failure the user can
|
||||
/// act on, and because the caller's response differs: a full disk is met by
|
||||
/// evicting cached blobs and answering `ENOSPC`, where `EIO` would read as a
|
||||
/// broken file. `std::io::ErrorKind::StorageFull` is still unstable, so this
|
||||
/// compares the raw code.
|
||||
pub fn is_disk_full(&self) -> bool {
|
||||
match self {
|
||||
Error::Io(e) => e.raw_os_error() == Some(libc::ENOSPC),
|
||||
// SQLite reports it as its own code rather than surfacing the errno.
|
||||
Error::Db(rusqlite::Error::SqliteFailure(e, _)) => {
|
||||
e.code == rusqlite::ErrorCode::DiskFull
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
/// A failure on its way to a front-end: prose for the user, plus the
|
||||
|
||||
@@ -20,6 +20,7 @@ pub mod search;
|
||||
pub mod service;
|
||||
pub mod shell;
|
||||
pub mod syncignore;
|
||||
pub mod takeout;
|
||||
|
||||
pub use access::{Access, access_for, perm_bits};
|
||||
pub use error::{CoreError, CoreResult, Error, Result};
|
||||
|
||||
@@ -26,10 +26,20 @@ const SKIP_DIRS: &[&str] = &[
|
||||
"venv",
|
||||
".venv",
|
||||
"vendor",
|
||||
"dist",
|
||||
"dist-newstyle",
|
||||
"build",
|
||||
"Trash",
|
||||
];
|
||||
|
||||
/// Directory *paths* skipped when they end with one of these, for caches whose
|
||||
/// own name is too generic to blacklist (`mod`). The Go module cache was 63,399
|
||||
/// of the 110,187 entries indexed on the machine this was measured on — 58% of
|
||||
/// the index, crowding real files out of the bounded candidate pool the fuzzy
|
||||
/// scorer draws from. The other big dependency caches (`.cargo`, `.rustup`,
|
||||
/// `.gradle`, `.m2`) are already skipped by the hidden-file filter.
|
||||
const SKIP_PATH_SUFFIXES: &[&str] = &["go/pkg/mod"];
|
||||
|
||||
/// Hard cap on indexed entries, so a pathological home directory cannot grow the
|
||||
/// database without bound. Reaching it stops the walk early.
|
||||
const MAX_ENTRIES: usize = 400_000;
|
||||
@@ -90,7 +100,10 @@ pub fn scan(
|
||||
if excludes.iter().any(|e| path.starts_with(e)) {
|
||||
return false;
|
||||
}
|
||||
!matches!(entry.file_name().to_str(), Some(name) if SKIP_DIRS.contains(&name))
|
||||
if matches!(entry.file_name().to_str(), Some(name) if SKIP_DIRS.contains(&name)) {
|
||||
return false;
|
||||
}
|
||||
!is_skipped_path(path)
|
||||
});
|
||||
|
||||
let state = parking_lot::Mutex::new(SinkState {
|
||||
@@ -150,6 +163,17 @@ impl<F: FnMut(Vec<LocalEntry>)> SinkState<F> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `path` is (or is inside) one of the [`SKIP_PATH_SUFFIXES`] caches.
|
||||
/// Matched on the path's own components so a directory merely *named* `mod`
|
||||
/// keeps being indexed.
|
||||
fn is_skipped_path(path: &Path) -> bool {
|
||||
let path = path.to_string_lossy();
|
||||
SKIP_PATH_SUFFIXES.iter().any(|suffix| {
|
||||
path.strip_suffix(suffix)
|
||||
.is_some_and(|head| head.is_empty() || head.ends_with('/'))
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert a walker entry into a [`LocalEntry`], reusing the metadata the walker
|
||||
/// already fetched. Entries with non-UTF-8 paths are dropped: the index (and the
|
||||
/// JSON wire format) is UTF-8 only.
|
||||
@@ -221,4 +245,27 @@ mod tests {
|
||||
|
||||
std::fs::remove_dir_all(&tmp).unwrap();
|
||||
}
|
||||
|
||||
/// The Go module cache is the single largest source of index noise on a
|
||||
/// developer's machine (58% of the entries on the one this was measured
|
||||
/// on), and it cannot be pruned by directory name: `mod` is far too
|
||||
/// common a name to blacklist outright.
|
||||
#[test]
|
||||
fn the_go_module_cache_is_pruned_but_a_plain_mod_dir_is_not() {
|
||||
let tmp = std::env::temp_dir().join(format!("pdfs-scan-gomod-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
std::fs::create_dir_all(tmp.join("go/pkg/mod/example.com/lib")).unwrap();
|
||||
std::fs::create_dir_all(tmp.join("kernel/mod")).unwrap();
|
||||
std::fs::write(tmp.join("go/pkg/mod/example.com/lib/vendored.go"), b"x").unwrap();
|
||||
std::fs::write(tmp.join("kernel/mod/mine.conf"), b"x").unwrap();
|
||||
|
||||
let mut got = Vec::new();
|
||||
scan(std::slice::from_ref(&tmp), &[], |batch| got.extend(batch));
|
||||
|
||||
let names: Vec<&str> = got.iter().map(|e| e.name.as_str()).collect();
|
||||
assert!(!names.contains(&"vendored.go"), "{names:?}");
|
||||
assert!(names.contains(&"mine.conf"), "{names:?}");
|
||||
|
||||
std::fs::remove_dir_all(&tmp).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,887 @@
|
||||
//! Reading a Google Photos **Takeout** export: what is in the archives, which
|
||||
//! album each photo belongs to, and when it was taken.
|
||||
//!
|
||||
//! This module is pure structure — it opens the archives' central directories
|
||||
//! and reads the small JSON sidecars, never the media bytes. The import itself
|
||||
//! (hashing, duplicate detection, upload, album creation) lives in the daemon;
|
||||
//! everything here is offline-testable and has no Proton dependency.
|
||||
//!
|
||||
//! # What a Takeout export looks like
|
||||
//!
|
||||
//! Google splits an export into numbered zips (`takeout-20260817T...-001.zip`,
|
||||
//! `-002.zip`, …) that are *independent* archives — no zip64 spanning — but a
|
||||
//! photo and its JSON sidecar can land in different parts, and one album's
|
||||
//! folder can be spread over several. So the scan treats the whole set as one
|
||||
//! namespace keyed by the path inside the export:
|
||||
//!
|
||||
//! ```text
|
||||
//! Takeout/Google Photos/Iceland 2019/IMG_0042.jpg
|
||||
//! Takeout/Google Photos/Iceland 2019/IMG_0042.jpg.supplemental-metadata.json
|
||||
//! Takeout/Google Photos/Iceland 2019/metadata.json <- album title
|
||||
//! Takeout/Google Photos/Photos from 2019/IMG_0043.jpg <- no album
|
||||
//! Takeout/Google Photos/Trash/IMG_0044.jpg <- skipped
|
||||
//! ```
|
||||
//!
|
||||
//! The service folder (`Google Photos`) is localized, as are the year folders
|
||||
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
||||
//! service name: the first component is dropped when it is the export root, the
|
||||
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
|
||||
//! **album** when it carries an album metadata sidecar naming it; anything else
|
||||
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
||||
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
||||
//! unavoidable, because a trashed photo is not marked as such anywhere else.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{CoreError, CoreResult};
|
||||
|
||||
/// Folder names Google uses for the trash bucket, lowercased. A Takeout export
|
||||
/// only contains one when the user asked for it, and its photos are deleted —
|
||||
/// importing them would resurrect them, so they are skipped.
|
||||
///
|
||||
/// Localized, and necessarily incomplete: a locale not listed here imports its
|
||||
/// trash as ordinary timeline photos. Extend as reports come in rather than
|
||||
/// guessing at transliterations.
|
||||
pub const TRASH_FOLDERS: &[&str] = &[
|
||||
"trash",
|
||||
"bin",
|
||||
"papierkorb",
|
||||
"corbeille",
|
||||
"papelera",
|
||||
"papelera de reciclaje",
|
||||
"cestino",
|
||||
"lixeira",
|
||||
"prullenbak",
|
||||
"kosz",
|
||||
"koš",
|
||||
"skräpkorgen",
|
||||
"papperskorg",
|
||||
"papirkurv",
|
||||
"roskakori",
|
||||
"kuka",
|
||||
"回收站",
|
||||
"回收筒",
|
||||
"ゴミ箱",
|
||||
"휴지통",
|
||||
"корзина",
|
||||
"кошик",
|
||||
"çöp kutusu",
|
||||
"σκουπίδια",
|
||||
];
|
||||
|
||||
/// Sidecar file names that describe the *folder* (its album title) rather than
|
||||
/// one photo, lowercased. Google has renamed this file more than once.
|
||||
const ALBUM_METADATA_NAMES: &[&str] = &[
|
||||
"metadata.json",
|
||||
"album-metadata.json",
|
||||
"print-subscriptions.json",
|
||||
"shared_album_comments.json",
|
||||
"user-generated-memory-titles.json",
|
||||
];
|
||||
|
||||
/// The suffix Google appends to a photo's sidecar, lowercased. Truncation means
|
||||
/// only a *prefix* of it may survive (see [`strip_sidecar_suffix`]).
|
||||
const SUPPLEMENTAL: &str = ".supplemental-metadata";
|
||||
|
||||
/// File extensions imported as photos or videos. Anything else in the export —
|
||||
/// `archive_browser.html`, `.csv` activity logs, `print-subscriptions.json` —
|
||||
/// is not media and is ignored.
|
||||
const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
|
||||
("jpg", "image/jpeg"),
|
||||
("jpeg", "image/jpeg"),
|
||||
("png", "image/png"),
|
||||
("gif", "image/gif"),
|
||||
("webp", "image/webp"),
|
||||
("bmp", "image/bmp"),
|
||||
("tif", "image/tiff"),
|
||||
("tiff", "image/tiff"),
|
||||
("heic", "image/heic"),
|
||||
("heif", "image/heif"),
|
||||
("avif", "image/avif"),
|
||||
("dng", "image/x-adobe-dng"),
|
||||
("cr2", "image/x-canon-cr2"),
|
||||
("cr3", "image/x-canon-cr3"),
|
||||
("nef", "image/x-nikon-nef"),
|
||||
("arw", "image/x-sony-arw"),
|
||||
("orf", "image/x-olympus-orf"),
|
||||
("rw2", "image/x-panasonic-rw2"),
|
||||
("raf", "image/x-fuji-raf"),
|
||||
("mp4", "video/mp4"),
|
||||
("m4v", "video/x-m4v"),
|
||||
("mov", "video/quicktime"),
|
||||
("avi", "video/x-msvideo"),
|
||||
("mkv", "video/x-matroska"),
|
||||
("webm", "video/webm"),
|
||||
("3gp", "video/3gpp"),
|
||||
("mts", "video/mp2t"),
|
||||
("m2ts", "video/mp2t"),
|
||||
("mpg", "video/mpeg"),
|
||||
("mpeg", "video/mpeg"),
|
||||
];
|
||||
|
||||
/// The media type for `name`'s extension, or `None` when it is not media this
|
||||
/// import handles.
|
||||
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
||||
let ext = Path::new(name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())?
|
||||
.to_ascii_lowercase();
|
||||
MEDIA_EXTENSIONS
|
||||
.iter()
|
||||
.find(|(candidate, _)| *candidate == ext)
|
||||
.map(|(_, media_type)| *media_type)
|
||||
}
|
||||
|
||||
/// Where a photo sits in the export, which decides whether it joins an album.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Bucket {
|
||||
/// A named album folder — the photo joins that album *and* the timeline.
|
||||
Album(String),
|
||||
/// A year folder (`Photos from 2019`) or the export root: timeline only.
|
||||
Timeline,
|
||||
/// The archive bucket: timeline only, like Google shows it.
|
||||
Archived,
|
||||
}
|
||||
|
||||
/// One media file found in the export.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutPhoto {
|
||||
/// Index into the archive list the scan was given.
|
||||
pub archive: usize,
|
||||
/// The entry's full path inside that archive — how the importer reopens it.
|
||||
pub entry: String,
|
||||
/// The name to upload under: the sidecar's `title` when it has one (Google
|
||||
/// mangles long names on disk but keeps the original there), else the file
|
||||
/// name on disk.
|
||||
pub name: String,
|
||||
/// Uncompressed size in bytes, from the zip's central directory.
|
||||
pub size: u64,
|
||||
/// Media type from the extension.
|
||||
pub media_type: String,
|
||||
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
|
||||
/// `creationTime`). `None` when the photo has no sidecar — the upload then
|
||||
/// defaults it to now, as the SDK does.
|
||||
pub capture_time: Option<i64>,
|
||||
/// The sidecar's `favorited` flag.
|
||||
pub favorite: bool,
|
||||
/// Which bucket the file was found in.
|
||||
pub bucket: Bucket,
|
||||
}
|
||||
|
||||
/// Everything one scan found, ready to be turned into an import plan.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutScan {
|
||||
/// Every importable media file, in archive then entry order.
|
||||
pub photos: Vec<TakeoutPhoto>,
|
||||
/// Album names in the order first seen, so the importer creates them in a
|
||||
/// stable order.
|
||||
pub albums: Vec<String>,
|
||||
/// Media files skipped because they sit in the trash bucket.
|
||||
pub skipped_trashed: usize,
|
||||
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
||||
/// activity `.csv`s, `.mp` motion-photo parts).
|
||||
pub skipped_other: usize,
|
||||
}
|
||||
|
||||
impl TakeoutScan {
|
||||
/// Total bytes of media to be read, before duplicate detection.
|
||||
pub fn total_bytes(&self) -> u64 {
|
||||
self.photos.iter().map(|p| p.size).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
||||
/// file also carries geo data, people tags and view counts.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct PhotoSidecar {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(rename = "photoTakenTime", default)]
|
||||
photo_taken_time: Option<SidecarTime>,
|
||||
#[serde(rename = "creationTime", default)]
|
||||
creation_time: Option<SidecarTime>,
|
||||
#[serde(default)]
|
||||
favorited: bool,
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, as a *string*.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarTime {
|
||||
#[serde(default)]
|
||||
timestamp: Option<String>,
|
||||
}
|
||||
|
||||
impl SidecarTime {
|
||||
fn epoch_seconds(&self) -> Option<i64> {
|
||||
self.timestamp.as_ref()?.parse().ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// A folder-level sidecar. `title` is the album name as the user typed it,
|
||||
/// which is what the folder name is derived (and sometimes truncated) from.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct AlbumSidecar {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
}
|
||||
|
||||
/// One entry as the scan sees it, independent of where it came from.
|
||||
struct RawEntry {
|
||||
archive: usize,
|
||||
entry: String,
|
||||
/// `<folder>/<file>` with the export root and service folder removed.
|
||||
relative: String,
|
||||
size: u64,
|
||||
}
|
||||
|
||||
impl RawEntry {
|
||||
/// The folder component of `relative`, or `""` at the service root.
|
||||
fn folder(&self) -> &str {
|
||||
match self.relative.rfind('/') {
|
||||
Some(index) => &self.relative[..index],
|
||||
None => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// The file name component of `relative`.
|
||||
fn file_name(&self) -> &str {
|
||||
match self.relative.rfind('/') {
|
||||
Some(index) => &self.relative[index + 1..],
|
||||
None => &self.relative,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan a set of Takeout zip archives into one plan.
|
||||
///
|
||||
/// The archives are treated as one export: an album folder split across parts
|
||||
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
||||
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
||||
/// this stays fast on a 200 GB export.
|
||||
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<RawEntry> = Vec::new();
|
||||
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
|
||||
for (index, path) in archives.iter().enumerate() {
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
|
||||
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
|
||||
})?;
|
||||
|
||||
for position in 0..zip.len() {
|
||||
let mut entry = zip.by_index(position).map_err(|e| {
|
||||
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
|
||||
})?;
|
||||
if entry.is_dir() {
|
||||
continue;
|
||||
}
|
||||
// `enclosed_name` refuses paths that escape the archive root, which
|
||||
// is exactly the check we want before using one as a key.
|
||||
let Some(name) = entry.enclosed_name() else {
|
||||
continue;
|
||||
};
|
||||
let name = name.to_string_lossy().replace('\\', "/");
|
||||
let Some(relative) = strip_export_prefix(&name) else {
|
||||
continue;
|
||||
};
|
||||
let size = entry.size();
|
||||
|
||||
// Sidecars are read now, while the entry is open: they are a few
|
||||
// hundred bytes each and reopening the archive per photo later would
|
||||
// be a second full pass.
|
||||
if relative.to_ascii_lowercase().ends_with(".json") {
|
||||
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.insert(relative.to_string(), body);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push(RawEntry {
|
||||
archive: index,
|
||||
entry: name.clone(),
|
||||
relative: relative.to_string(),
|
||||
size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(assemble(entries, &sidecars))
|
||||
}
|
||||
|
||||
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
||||
/// inside the export. Split out from [`scan_archives`] so the matching rules can
|
||||
/// be tested without building zip fixtures.
|
||||
fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> TakeoutScan {
|
||||
// Album titles per folder, from the folder-level sidecars.
|
||||
let mut album_titles: HashMap<String, String> = HashMap::new();
|
||||
for (path, body) in sidecars {
|
||||
let (folder, file) = split_folder(path);
|
||||
if !ALBUM_METADATA_NAMES.contains(&file.to_ascii_lowercase().as_str()) {
|
||||
continue;
|
||||
}
|
||||
if let Ok(album) = serde_json::from_slice::<AlbumSidecar>(body)
|
||||
&& let Some(title) = album.title.filter(|t| !t.trim().is_empty())
|
||||
{
|
||||
album_titles.insert(folder.to_string(), title);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-photo sidecars, indexed by folder so a truncated name only ever
|
||||
// matches within its own folder.
|
||||
let mut per_folder_sidecars: HashMap<String, BTreeMap<String, &Vec<u8>>> = HashMap::new();
|
||||
for (path, body) in sidecars {
|
||||
let (folder, file) = split_folder(path);
|
||||
if ALBUM_METADATA_NAMES.contains(&file.to_ascii_lowercase().as_str()) {
|
||||
continue;
|
||||
}
|
||||
per_folder_sidecars
|
||||
.entry(folder.to_string())
|
||||
.or_default()
|
||||
.insert(file.to_string(), body);
|
||||
}
|
||||
|
||||
let mut scan = TakeoutScan::default();
|
||||
let mut seen_albums: HashMap<String, ()> = HashMap::new();
|
||||
|
||||
for raw in &entries {
|
||||
let folder = raw.folder().to_string();
|
||||
let file_name = raw.file_name().to_string();
|
||||
|
||||
if is_trash_folder(&folder) {
|
||||
if media_type_of(&file_name).is_some() {
|
||||
scan.skipped_trashed += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let Some(media_type) = media_type_of(&file_name) else {
|
||||
scan.skipped_other += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
let sidecar = per_folder_sidecars
|
||||
.get(&folder)
|
||||
.and_then(|in_folder| find_sidecar(&file_name, in_folder))
|
||||
.and_then(|body| serde_json::from_slice::<PhotoSidecar>(body).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
let bucket = match album_titles.get(&folder) {
|
||||
Some(title) => Bucket::Album(title.clone()),
|
||||
None if is_archive_folder(&folder) => Bucket::Archived,
|
||||
None => Bucket::Timeline,
|
||||
};
|
||||
if let Bucket::Album(title) = &bucket
|
||||
&& seen_albums.insert(title.clone(), ()).is_none()
|
||||
{
|
||||
scan.albums.push(title.clone());
|
||||
}
|
||||
|
||||
// The sidecar's `title` is the name Google had before it mangled the
|
||||
// on-disk one (truncation, `(1)` suffixes, character replacement), so it
|
||||
// is the better name to upload under — but only when it still looks like
|
||||
// the same file, i.e. the extension agrees.
|
||||
let name = sidecar
|
||||
.title
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|title| !title.is_empty() && same_extension(title, &file_name))
|
||||
.unwrap_or(&file_name)
|
||||
.to_string();
|
||||
|
||||
scan.photos.push(TakeoutPhoto {
|
||||
archive: raw.archive,
|
||||
entry: raw.entry.clone(),
|
||||
name,
|
||||
size: raw.size,
|
||||
media_type: media_type.to_string(),
|
||||
capture_time: sidecar
|
||||
.photo_taken_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
.or_else(|| {
|
||||
sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
}),
|
||||
favorite: sidecar.favorited,
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
|
||||
scan
|
||||
}
|
||||
|
||||
/// Drop the export root (`Takeout/`) and the localized service folder from a
|
||||
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
|
||||
///
|
||||
/// `None` for a path with no service folder at all (the export's own
|
||||
/// `archive_browser.html`), which carries nothing to import.
|
||||
fn strip_export_prefix(path: &str) -> Option<&str> {
|
||||
let mut rest = path;
|
||||
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
|
||||
// the user may not have it — so the root is dropped only when a service
|
||||
// folder follows it, which the component count decides.
|
||||
let components = rest.split('/').count();
|
||||
if components >= 3 {
|
||||
rest = rest.split_once('/')?.1;
|
||||
}
|
||||
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
|
||||
let (_service, tail) = rest.split_once('/')?;
|
||||
Some(tail)
|
||||
}
|
||||
|
||||
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
||||
fn split_folder(path: &str) -> (&str, &str) {
|
||||
match path.rfind('/') {
|
||||
Some(index) => (&path[..index], &path[index + 1..]),
|
||||
None => ("", path),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_trash_folder(folder: &str) -> bool {
|
||||
let name = folder.rsplit('/').next().unwrap_or(folder).to_lowercase();
|
||||
TRASH_FOLDERS.contains(&name.as_str())
|
||||
}
|
||||
|
||||
/// Google's archive bucket, which is not an album but is not the plain timeline
|
||||
/// either. Only the English name is matched; a missed locale simply imports as
|
||||
/// [`Bucket::Timeline`], which is the same upload either way.
|
||||
fn is_archive_folder(folder: &str) -> bool {
|
||||
let name = folder.rsplit('/').next().unwrap_or(folder).to_lowercase();
|
||||
name == "archive"
|
||||
}
|
||||
|
||||
/// Whether two file names end in the same extension, case-insensitively.
|
||||
fn same_extension(a: &str, b: &str) -> bool {
|
||||
let ext = |name: &str| {
|
||||
Path::new(name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(str::to_ascii_lowercase)
|
||||
};
|
||||
ext(a) == ext(b)
|
||||
}
|
||||
|
||||
/// Strip a sidecar's `.json` and any `.supplemental-metadata` suffix, including
|
||||
/// the *truncated* forms Google produces.
|
||||
///
|
||||
/// Google caps the sidecar's file name at 51 characters, so a long photo name
|
||||
/// leaves only a prefix of the suffix behind — `IMG_1234.jpg.supplemental-me`,
|
||||
/// `…-metad`, or nothing at all. Any non-empty prefix of the suffix is stripped;
|
||||
/// a bare `.json` sidecar loses only that.
|
||||
fn strip_sidecar_suffix(sidecar: &str) -> String {
|
||||
let stem = sidecar
|
||||
.strip_suffix(".json")
|
||||
.or_else(|| sidecar.strip_suffix(".JSON"))
|
||||
.unwrap_or(sidecar);
|
||||
// Longest match first so `.supplemental-metadata` wins over `.s`.
|
||||
for length in (2..=SUPPLEMENTAL.len()).rev() {
|
||||
let candidate = &SUPPLEMENTAL[..length];
|
||||
if let Some(head) = stem.strip_suffix(candidate) {
|
||||
return head.to_string();
|
||||
}
|
||||
}
|
||||
stem.to_string()
|
||||
}
|
||||
|
||||
/// Move a trailing `(N)` from after the extension to before it: Google writes
|
||||
/// the sidecar of `IMG_1234(1).jpg` as `IMG_1234.jpg(1).json`.
|
||||
fn unswap_duplicate_marker(stem: &str) -> Option<String> {
|
||||
let marker_start = stem.rfind('(')?;
|
||||
let marker = &stem[marker_start..];
|
||||
if !marker.ends_with(')')
|
||||
|| !marker[1..marker.len() - 1]
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_digit())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let head = &stem[..marker_start];
|
||||
let dot = head.rfind('.')?;
|
||||
Some(format!("{}{}{}", &head[..dot], marker, &head[dot..]))
|
||||
}
|
||||
|
||||
/// Find the sidecar belonging to `file_name` among the JSON files of its folder.
|
||||
///
|
||||
/// Tried in order of confidence: the exact name, the `(N)`-swapped name, then a
|
||||
/// prefix match for the case where truncation cut into the photo name itself.
|
||||
fn find_sidecar<'a>(
|
||||
file_name: &str,
|
||||
in_folder: &BTreeMap<String, &'a Vec<u8>>,
|
||||
) -> Option<&'a Vec<u8>> {
|
||||
let mut prefix_match: Option<&'a Vec<u8>> = None;
|
||||
let mut prefix_len = 0usize;
|
||||
|
||||
for (sidecar_name, body) in in_folder {
|
||||
let stem = strip_sidecar_suffix(sidecar_name);
|
||||
if stem == file_name {
|
||||
return Some(body);
|
||||
}
|
||||
if unswap_duplicate_marker(&stem).as_deref() == Some(file_name) {
|
||||
return Some(body);
|
||||
}
|
||||
// Truncated: the stem is a prefix of the photo's name. Keep the longest
|
||||
// such prefix — a folder holding `IMG_1.jpg` and `IMG_12.jpg` must not
|
||||
// let the shorter stem claim the longer photo.
|
||||
if !stem.is_empty() && file_name.starts_with(&stem) && stem.len() > prefix_len {
|
||||
prefix_len = stem.len();
|
||||
prefix_match = Some(body);
|
||||
}
|
||||
}
|
||||
prefix_match
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn entry(archive: usize, path: &str, size: u64) -> RawEntry {
|
||||
RawEntry {
|
||||
archive,
|
||||
entry: path.to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().to_string(),
|
||||
size,
|
||||
}
|
||||
}
|
||||
|
||||
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
||||
let relative = strip_export_prefix(path).unwrap().to_string();
|
||||
map.insert(relative, body.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_export_root_and_localized_service_folder() {
|
||||
assert_eq!(
|
||||
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
||||
Some("Island 2019/IMG_1.jpg")
|
||||
);
|
||||
// Already unwrapped by the user: no `Takeout/` root left.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
||||
Some("IMG_1.jpg")
|
||||
);
|
||||
// Nothing below a service folder is nothing to import.
|
||||
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn album_folder_becomes_an_album_year_folder_does_not() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland 2019/metadata.json",
|
||||
r#"{"title":"Iceland 2019"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_2.jpg", 20),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.albums, vec!["Iceland 2019"]);
|
||||
assert_eq!(scan.photos[0].bucket, Bucket::Album("Iceland 2019".into()));
|
||||
assert_eq!(scan.photos[1].bucket, Bucket::Timeline);
|
||||
assert_eq!(scan.total_bytes(), 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trashed_photos_are_skipped_and_counted() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Papierkorb/IMG_1.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Bin/IMG_2.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_3.jpg", 10),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(scan.skipped_trashed, 2);
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_3.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_media_entries_are_ignored() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/notes.txt", 10),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.skipped_other, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_supplies_capture_time_and_favourite() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.supplemental-metadata.json",
|
||||
r#"{"title":"IMG_1.jpg","photoTakenTime":{"timestamp":"1560000000"},"favorited":true}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert!(scan.photos[0].favorite);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creation_time_is_the_fallback_capture_time() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_supplemental_suffixes_still_match() {
|
||||
// Google caps the sidecar name at 51 chars, cutting into the suffix.
|
||||
assert_eq!(
|
||||
strip_sidecar_suffix("IMG_1.jpg.supplemental-me.json"),
|
||||
"IMG_1.jpg"
|
||||
);
|
||||
assert_eq!(
|
||||
strip_sidecar_suffix("IMG_1.jpg.supplemental-metadata.json"),
|
||||
"IMG_1.jpg"
|
||||
);
|
||||
assert_eq!(strip_sidecar_suffix("IMG_1.jpg.json"), "IMG_1.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_marker_after_the_extension_is_moved_back() {
|
||||
assert_eq!(
|
||||
unswap_duplicate_marker("IMG_1.jpg(1)").as_deref(),
|
||||
Some("IMG_1(1).jpg")
|
||||
);
|
||||
assert_eq!(unswap_duplicate_marker("IMG_1.jpg"), None);
|
||||
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg(1).json",
|
||||
r#"{"photoTakenTime":{"timestamp":"1600000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1(1).jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_600_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longest_prefix_wins_when_the_photo_name_itself_was_truncated() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.json",
|
||||
r#"{"photoTakenTime":{"timestamp":"1"}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_12.json",
|
||||
r#"{"photoTakenTime":{"timestamp":"2"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_12345.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_title_replaces_a_mangled_on_disk_name() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/a_very_long_name_that_google_.jpg.json",
|
||||
r#"{"title":"a_very_long_name_that_google_truncated.jpg"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/a_very_long_name_that_google_.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
scan.photos[0].name,
|
||||
"a_very_long_name_that_google_truncated.jpg"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_with_a_different_extension_is_not_trusted() {
|
||||
// Motion photos name their `.MP` part after the `.jpg`; taking the title
|
||||
// verbatim would upload the video under an image name.
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.mp4.json",
|
||||
r#"{"title":"IMG_1.jpg"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.mp4",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.mp4");
|
||||
assert_eq!(scan.photos[0].media_type, "video/mp4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_album_split_across_archives_stays_one_album() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland 2019/metadata.json",
|
||||
r#"{"title":"Iceland 2019"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", 10),
|
||||
entry(1, "Takeout/Google Photos/Iceland 2019/IMG_2.jpg", 10),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.albums, vec!["Iceland 2019"]);
|
||||
assert_eq!(scan.photos.len(), 2);
|
||||
assert_eq!(scan.photos[1].archive, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_bucket_imports_to_the_timeline() {
|
||||
let scan = assemble(
|
||||
vec![entry(0, "Takeout/Google Photos/Archive/IMG_1.jpg", 10)],
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].bucket, Bucket::Archived);
|
||||
assert!(scan.albums.is_empty());
|
||||
}
|
||||
|
||||
/// Write a zip holding `files` and return its path. No `tempfile`
|
||||
/// dev-dependency: the daemon crates deliberately carry none, and one
|
||||
/// uniquely named file under the temp dir is all this needs.
|
||||
fn write_zip(tag: &str, files: &[(&str, &[u8])]) -> PathBuf {
|
||||
use std::io::Write;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-takeout-test-{tag}-{}.zip",
|
||||
std::process::id()
|
||||
));
|
||||
let file = std::fs::File::create(&path).unwrap();
|
||||
let mut zip = zip::ZipWriter::new(file);
|
||||
let options: zip::write::FileOptions<'_, ()> =
|
||||
zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
for (name, body) in files {
|
||||
zip.start_file(*name, options).unwrap();
|
||||
zip.write_all(body).unwrap();
|
||||
}
|
||||
zip.finish().unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scans_a_two_part_export_where_the_sidecar_is_in_the_other_part() {
|
||||
// The case that makes the whole set one scan: part 1 has the photo,
|
||||
// part 2 has its metadata and the album title.
|
||||
let first = write_zip(
|
||||
"part1",
|
||||
&[(
|
||||
"Takeout/Google Photos/Iceland 2019/IMG_1.jpg",
|
||||
b"the photo bytes",
|
||||
)],
|
||||
);
|
||||
let second = write_zip(
|
||||
"part2",
|
||||
&[
|
||||
(
|
||||
"Takeout/Google Photos/Iceland 2019/metadata.json",
|
||||
br#"{"title":"Iceland 2019"}"#.as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Photos/Iceland 2019/IMG_1.jpg.supplemental-metadata.json",
|
||||
br#"{"title":"IMG_1.jpg","photoTakenTime":{"timestamp":"1560000000"}}"#
|
||||
.as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Photos/Trash/IMG_9.jpg",
|
||||
b"deleted".as_slice(),
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
let scan = scan_archives(&[first.clone(), second.clone()]).unwrap();
|
||||
let _ = std::fs::remove_file(&first);
|
||||
let _ = std::fs::remove_file(&second);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
|
||||
assert_eq!(scan.photos[0].archive, 0, "the photo is in part 1");
|
||||
assert_eq!(scan.photos[0].size, "the photo bytes".len() as u64);
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert_eq!(scan.photos[0].bucket, Bucket::Album("Iceland 2019".into()));
|
||||
assert_eq!(scan.albums, vec!["Iceland 2019"]);
|
||||
assert_eq!(scan.skipped_trashed, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_a_zip_is_a_readable_error() {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
||||
std::fs::write(&path, b"not a zip at all").unwrap();
|
||||
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
assert!(
|
||||
error.message.contains("not a readable zip"),
|
||||
"unexpected message: {}",
|
||||
error.message
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ description = "FUSE on-demand filesystem for Proton Drive"
|
||||
|
||||
[dependencies]
|
||||
pdfs-core.workspace = true
|
||||
zip.workspace = true
|
||||
sha1.workspace = true
|
||||
proton-drive-rs.workspace = true
|
||||
futures.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Remote-event invalidation and local-index background workers.
|
||||
|
||||
use super::*;
|
||||
use crate::shutdown::Shutdown;
|
||||
|
||||
/// Drop `uid` from one mount's tree and tell that mount's kernel session the
|
||||
/// entry is gone. Shared by the trash and delete paths, which differ only in
|
||||
@@ -8,35 +9,20 @@ use super::*;
|
||||
///
|
||||
/// Returns whether this mount actually held the node, so the caller can evict
|
||||
/// its content blob once if *any* mount did.
|
||||
fn forget_and_notify(st: &mut State, notifier: Option<&Notifier>, uid: &NodeUid) -> bool {
|
||||
fn forget_and_notify(st: &mut State, notify: &mut NotifyBatch, uid: &NodeUid) -> bool {
|
||||
// Capture the inode before `forget` clears the uid mapping.
|
||||
let child = st.by_uid.get(uid).copied();
|
||||
let Some((parent, name)) = st.forget(uid) else {
|
||||
return false;
|
||||
};
|
||||
if let Some(notifier) = notifier {
|
||||
match child {
|
||||
Some(child) => {
|
||||
let _ = notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
||||
}
|
||||
None => {
|
||||
let _ = notifier.inval_entry(INodeNo(parent), OsStr::new(&name));
|
||||
}
|
||||
}
|
||||
match child {
|
||||
Some(child) => notify.delete(parent, child, name),
|
||||
None => notify.inval_entry(parent, name),
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn invalidate_access_changes(notifier: Option<&Notifier>, changed: &[u64]) {
|
||||
let Some(notifier) = notifier else {
|
||||
return;
|
||||
};
|
||||
for &ino in changed {
|
||||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
fn hide_foreign_deleted_and_notify(st: &mut State, notifier: Option<&Notifier>, uid: &NodeUid) {
|
||||
fn hide_foreign_deleted_and_notify(st: &mut State, notify: &mut NotifyBatch, uid: &NodeUid) {
|
||||
let child = st.by_uid.get(uid).copied();
|
||||
let dentry = child.and_then(|ino| {
|
||||
st.entries
|
||||
@@ -45,9 +31,9 @@ fn hide_foreign_deleted_and_notify(st: &mut State, notifier: Option<&Notifier>,
|
||||
});
|
||||
let changed = st.downgrade_shared_subtree(uid);
|
||||
st.hide_foreign_subtree(uid);
|
||||
invalidate_access_changes(notifier, &changed);
|
||||
if let (Some(notifier), Some((parent, name)), Some(child)) = (notifier, dentry, child) {
|
||||
let _ = notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
||||
notify.extend_inodes(&changed);
|
||||
if let (Some((parent, name)), Some(child)) = (dentry, child) {
|
||||
notify.delete(parent, child, name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,18 +118,18 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
&& foreign_delete
|
||||
{
|
||||
apply_foreign_delete(&core.db, node_uid, |uid| {
|
||||
core.for_each_mount(|st, notifier| {
|
||||
hide_foreign_deleted_and_notify(st, notifier, uid);
|
||||
core.for_each_mount(|st, notify| {
|
||||
hide_foreign_deleted_and_notify(st, notify, uid);
|
||||
});
|
||||
})
|
||||
} else {
|
||||
apply_access_downgrade(&core.db, event, |root| {
|
||||
core.for_each_mount(|st, notifier| {
|
||||
core.for_each_mount(|st, notify| {
|
||||
let changed = match root {
|
||||
Some(uid) => st.downgrade_shared_subtree(uid),
|
||||
None => st.downgrade_known_shared_access(),
|
||||
};
|
||||
invalidate_access_changes(notifier, &changed);
|
||||
notify.extend_inodes(&changed);
|
||||
});
|
||||
})
|
||||
};
|
||||
@@ -173,19 +159,17 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
debug!(uid = %node_uid, "ignoring remote event for a node with a queued write");
|
||||
}
|
||||
let mut had_node = false;
|
||||
core.for_each_mount(|st, notifier| {
|
||||
core.for_each_mount(|st, notify| {
|
||||
if *is_trashed {
|
||||
// Trashing makes a node vanish from its parent listing.
|
||||
had_node |= forget_and_notify(st, notifier, node_uid);
|
||||
had_node |= forget_and_notify(st, notify, node_uid);
|
||||
} else if !ours && let Some(&ino) = st.by_uid.get(node_uid) {
|
||||
// Known node changed: drop its cached attrs/data (and
|
||||
// listing if it is a directory) so the next access
|
||||
// re-fetches. Its content blob may now be stale too.
|
||||
had_node = true;
|
||||
st.invalidate_listing(ino);
|
||||
if let Some(notifier) = notifier {
|
||||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||||
}
|
||||
notify.inval_inode(ino);
|
||||
}
|
||||
});
|
||||
// Once, not once per mount: the content cache is per-daemon.
|
||||
@@ -209,8 +193,8 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
DriveEvent::NodeDeleted { node_uid, .. } => {
|
||||
core.cache.evict(node_uid);
|
||||
if !foreign_delete {
|
||||
core.for_each_mount(|st, notifier| {
|
||||
forget_and_notify(st, notifier, node_uid);
|
||||
core.for_each_mount(|st, notify| {
|
||||
forget_and_notify(st, notify, node_uid);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -223,13 +207,11 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
| DriveEvent::SharedWithMeUpdated { .. } => {
|
||||
warn!("event access continuity lost; downgrading shared trees and resyncing lazily");
|
||||
apply_result = apply_result.and(core.db.clear_state(SHARED_WITH_ME_SYNCED_MS));
|
||||
core.for_each_mount(|st, notifier| {
|
||||
core.for_each_mount(|st, notify| {
|
||||
let dirs: Vec<u64> = st.children.keys().copied().collect();
|
||||
for &ino in &dirs {
|
||||
st.invalidate_listing(ino);
|
||||
if let Some(notifier) = notifier {
|
||||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||||
}
|
||||
notify.inval_inode(ino);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -303,18 +285,16 @@ fn flush_dirty_parents(core: &Core, dirty: &DirtyParents) {
|
||||
if dirty.foreign.is_empty() && dirty.ours.is_empty() {
|
||||
return;
|
||||
}
|
||||
core.for_each_mount(|st, notifier| {
|
||||
let drop_listing = |st: &mut State, parent_uid: &NodeUid| {
|
||||
let Some(&parent) = st.by_uid.get(parent_uid) else {
|
||||
return;
|
||||
};
|
||||
st.invalidate_listing(parent);
|
||||
if let Some(notifier) = notifier {
|
||||
let _ = notifier.inval_inode(INodeNo(parent), 0, 0);
|
||||
}
|
||||
fn drop_listing(st: &mut State, notify: &mut NotifyBatch, parent_uid: &NodeUid) {
|
||||
let Some(&parent) = st.by_uid.get(parent_uid) else {
|
||||
return;
|
||||
};
|
||||
st.invalidate_listing(parent);
|
||||
notify.inval_inode(parent);
|
||||
}
|
||||
core.for_each_mount(|st, notify| {
|
||||
for parent_uid in &dirty.foreign {
|
||||
drop_listing(st, parent_uid);
|
||||
drop_listing(st, notify, parent_uid);
|
||||
}
|
||||
for (parent_uid, children) in &dirty.ours {
|
||||
if dirty.foreign.contains(parent_uid) {
|
||||
@@ -328,7 +308,7 @@ fn flush_dirty_parents(core: &Core, dirty: &DirtyParents) {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
drop_listing(st, parent_uid);
|
||||
drop_listing(st, notify, parent_uid);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -458,6 +438,7 @@ pub(super) fn run_local_index(
|
||||
indexing: Arc<AtomicBool>,
|
||||
transfers: Arc<TransferRegistry>,
|
||||
mountpoint: PathBuf,
|
||||
shutdown: Arc<Shutdown>,
|
||||
) {
|
||||
loop {
|
||||
let age = db.local_indexed_at().ok().flatten();
|
||||
@@ -466,7 +447,11 @@ pub(super) fn run_local_index(
|
||||
if stale {
|
||||
scan_local_once(&db, &indexing, &transfers, &mountpoint);
|
||||
}
|
||||
std::thread::sleep(LOCAL_INDEX_CHECK);
|
||||
// Interruptible, so teardown joins this thread instead of leaving it to
|
||||
// keep rewriting the index for a mount that is gone (bugs.md B44).
|
||||
if !shutdown.sleep(LOCAL_INDEX_CHECK) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+140
-37
@@ -29,7 +29,23 @@ use super::{Core, count_noun, human_bytes, human_duration, parse_uid};
|
||||
|
||||
const MAX_CONTROL_REQUEST_BYTES: u64 = 1024 * 1024;
|
||||
const CONTROL_READ_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// A client that connects, sends a request and then never reads its reply used
|
||||
/// to park a handler thread forever in `write_all`; 64 of those exhaust
|
||||
/// [`MAX_CONTROL_HANDLERS`] permanently. Generous enough for the largest
|
||||
/// responses (photo bytes) over a local socket, finite so a wedged peer can't
|
||||
/// hold a permit.
|
||||
const CONTROL_WRITE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_CONTROL_HANDLERS: usize = 64;
|
||||
/// Ceiling on any client-supplied row `limit`. Every in-tree caller asks for far
|
||||
/// less (200 for activity, 60 for a photo page), but the value goes straight
|
||||
/// into `SQL LIMIT ?` under the daemon's single connection — an unclamped one
|
||||
/// materialises the whole account, and then serialises it into one JSON line.
|
||||
const MAX_ROW_LIMIT: usize = 1000;
|
||||
|
||||
/// Clamp a client-supplied row limit to [`MAX_ROW_LIMIT`].
|
||||
fn clamp_limit(limit: usize) -> usize {
|
||||
limit.min(MAX_ROW_LIMIT)
|
||||
}
|
||||
static ACTIVE_CONTROL_HANDLERS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
struct ControlHandlerPermit;
|
||||
@@ -88,6 +104,31 @@ fn read_request_line(reader: &mut impl BufRead) -> std::io::Result<Option<String
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
|
||||
/// Turn a CLI-supplied path into the mount that owns it and the path relative
|
||||
/// to *that* mount's root.
|
||||
///
|
||||
/// [`rel_to_mount`] is the older, primary-only reading, and it is what every
|
||||
/// mutating request still uses. It rejects anything outside the primary
|
||||
/// mountpoint, which leaves a secondary on-demand mount — a synced folder
|
||||
/// switched out of mirror mode — with no way to be named at all: `pdfs ls` and
|
||||
/// `pdfs refresh` both answered "is not under the mountpoint", so a folder
|
||||
/// serving a stale listing had no manual escape hatch (`docs/BUGS.md` B86).
|
||||
///
|
||||
/// A relative path keeps the old meaning, relative to the primary mount, since
|
||||
/// there is nothing in it to choose a mount by.
|
||||
fn route_to_mount(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Core, PathBuf)> {
|
||||
let p = Path::new(path);
|
||||
if !p.is_absolute() {
|
||||
return Ok((core.clone(), p.to_path_buf()));
|
||||
}
|
||||
if let Some(routed) = core.rooted_at(p) {
|
||||
return Ok(routed);
|
||||
}
|
||||
// Fall back to the primary reading so the error text stays the familiar one
|
||||
// when the path is simply outside everything this daemon has mounted.
|
||||
rel_to_mount(mountpoint, path).map(|rel| (core.clone(), rel))
|
||||
}
|
||||
|
||||
/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path
|
||||
/// must live under `mountpoint`; a relative path is taken as already relative to
|
||||
/// the mount root.
|
||||
@@ -109,6 +150,10 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
warn!(error = %e, "control: could not set request timeout");
|
||||
return;
|
||||
}
|
||||
if let Err(e) = stream.set_write_timeout(Some(CONTROL_WRITE_TIMEOUT)) {
|
||||
warn!(error = %e, "control: could not set response timeout");
|
||||
return;
|
||||
}
|
||||
let mut reader = BufReader::new(match stream.try_clone() {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
@@ -128,7 +173,13 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Ok(CtlRequest::Status) => {
|
||||
let pins = core.cache.list_pins();
|
||||
let queued = core.db.pending_op_counts().unwrap_or_default();
|
||||
let staging = core.cache.staging_usage();
|
||||
CtlResponse::Status {
|
||||
parked_uploads: queued.parked.max(0) as u64,
|
||||
failing_ops: queued.failing.max(0) as u64,
|
||||
failing_error: queued.last_error.clone(),
|
||||
staged_bytes: staging.bytes,
|
||||
staged_oldest_secs: staging.oldest_secs,
|
||||
username: username.to_string(),
|
||||
mountpoint: mountpoint.display().to_string(),
|
||||
pinned: pins.len(),
|
||||
@@ -167,8 +218,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Ok(CtlRequest::ListPins) => CtlResponse::Pins {
|
||||
pins: core.cache.list_pins(),
|
||||
},
|
||||
Ok(CtlRequest::ListDir { path }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => match core.list_dir(&rel) {
|
||||
Ok(CtlRequest::ListDir { path }) => match route_to_mount(core, mountpoint, &path) {
|
||||
Ok((core, rel)) => match core.list_dir(&rel) {
|
||||
Ok(entries) => CtlResponse::Entries { entries },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
@@ -176,9 +227,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Ok(CtlRequest::Refresh { scope }) => {
|
||||
let result = match &scope {
|
||||
RefreshScope::Dir { path } => {
|
||||
rel_to_mount(mountpoint, path).and_then(|rel| core.refresh_dir(&rel))
|
||||
}
|
||||
RefreshScope::Dir { path } => route_to_mount(core, mountpoint, path)
|
||||
.and_then(|(core, rel)| core.refresh_dir(&rel)),
|
||||
RefreshScope::Trash => {
|
||||
core.invalidate_trash();
|
||||
Ok(())
|
||||
@@ -204,7 +254,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
kind,
|
||||
range,
|
||||
favorites,
|
||||
}) => match core.photos_timeline(offset, limit, kind, range, favorites) {
|
||||
}) => match core.photos_timeline(offset, clamp_limit(limit), kind, range, favorites) {
|
||||
Ok(Some(items)) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
@@ -245,7 +295,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// An album page is a timeline page as far as the front-end is
|
||||
// concerned, so it comes back in the same reply — with `counts`
|
||||
// unset, which describe the whole timeline rather than this album.
|
||||
Some(album) => match core.album_photos(&album, offset, limit) {
|
||||
Some(album) => match core.album_photos(&album, offset, clamp_limit(limit)) {
|
||||
Ok(items) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
@@ -279,6 +329,51 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
None => CtlResponse::error(CoreError::invalid(format!("bad uid: {uid}"))),
|
||||
},
|
||||
// The scan alone walks every archive's central directory, so even the
|
||||
// dry run goes to a background thread and is read back through
|
||||
// `ImportStatus` rather than answered inline.
|
||||
Ok(CtlRequest::ImportTakeout { archives, dry_run }) => {
|
||||
if archives.is_empty() {
|
||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||
} else {
|
||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)))
|
||||
} else {
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ImportStatus) => {
|
||||
let (running, summary) = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus { running, summary }
|
||||
}
|
||||
Ok(CtlRequest::CancelImport) => {
|
||||
if crate::takeout::cancel_import() {
|
||||
CtlResponse::Ok {
|
||||
message: "stopping the import".to_string(),
|
||||
}
|
||||
} else {
|
||||
CtlResponse::error(CoreError::invalid("no import is running"))
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::UploadPhoto {
|
||||
name,
|
||||
media_type,
|
||||
@@ -346,22 +441,24 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::Search { query, limit }) => match core.search(&query, limit) {
|
||||
Ok(CtlRequest::Search { query, limit }) => match core.search(&query, clamp_limit(limit)) {
|
||||
Ok(hits) => CtlResponse::SearchResults { hits },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::SearchLocal { query, limit }) => match core.search_local(&query, limit) {
|
||||
Ok(hits) => CtlResponse::LocalResults {
|
||||
hits,
|
||||
indexing: core.indexing.load(Ordering::Relaxed),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::SearchLocal { query, limit }) => {
|
||||
match core.search_local(&query, clamp_limit(limit)) {
|
||||
Ok(hits) => CtlResponse::LocalResults {
|
||||
hits,
|
||||
indexing: core.indexing.load(Ordering::Relaxed),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::SearchV2 {
|
||||
query,
|
||||
limit,
|
||||
filters,
|
||||
}) => match core.search_v2(&query, limit, &filters) {
|
||||
}) => match core.search_v2(&query, clamp_limit(limit), &filters) {
|
||||
Ok((drive_hits, local_hits)) => CtlResponse::SearchResultsV2 {
|
||||
drive_hits,
|
||||
local_hits,
|
||||
@@ -540,19 +637,20 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
}
|
||||
Ok(CtlRequest::CacheInspect { deep }) => match core.db.stats() {
|
||||
Ok(stats) => {
|
||||
// The deep check reads every page, so it runs only on request;
|
||||
// `integrity_checked` tells the caller which answer it is
|
||||
// holding, since "no problems found" and "never looked" are
|
||||
// both an empty list.
|
||||
let (integrity_problems, integrity_checked) = if deep {
|
||||
match core.db.integrity_check() {
|
||||
Ok(problems) => (problems, true),
|
||||
Err(e) => (vec![format!("integrity check failed: {e:?}")], true),
|
||||
}
|
||||
// The deep check reads every page — tens of seconds on a real
|
||||
// database, all of it holding the connection every FUSE handler
|
||||
// needs — so it runs on a thread and this request returns
|
||||
// whatever the last completed check found. `integrity_checked`
|
||||
// tells the caller which answer it is holding, since "no
|
||||
// problems found" and "never looked" are both an empty list;
|
||||
// `integrity_running` says a fresh answer is on its way.
|
||||
let (integrity_problems, integrity_checked, integrity_running) = if deep {
|
||||
core.start_or_read_integrity_check()
|
||||
} else {
|
||||
(Vec::new(), false)
|
||||
(Vec::new(), false, false)
|
||||
};
|
||||
CtlResponse::CacheReport {
|
||||
integrity_running,
|
||||
schema_version: stats.schema_version,
|
||||
db_bytes: stats.total_bytes(),
|
||||
db_reclaimable_bytes: stats.reclaimable_bytes(),
|
||||
@@ -565,17 +663,16 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
}
|
||||
Err(e) => CtlResponse::error(CoreError::internal(format!("read db stats: {e:?}"))),
|
||||
},
|
||||
Ok(CtlRequest::CacheVacuum) => match core.db.vacuum() {
|
||||
Ok(outcome) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"vacuumed: {:.1} MiB freed ({:.1} MiB → {:.1} MiB), {} WAL frame(s) checkpointed",
|
||||
outcome.freed_bytes() as f64 / 1_048_576.0,
|
||||
outcome.before_bytes as f64 / 1_048_576.0,
|
||||
outcome.after_bytes as f64 / 1_048_576.0,
|
||||
outcome.wal_frames_checkpointed,
|
||||
),
|
||||
// Ack and vacuum on a thread. `VACUUM` rewrites the whole database
|
||||
// under the connection mutex — minutes on a large one — and the
|
||||
// request's own socket timeout is 10 s, so running it inline both froze
|
||||
// the mount and lost its own answer. Progress shows up as a job in
|
||||
// `GetQueueStatus`; the outcome goes to the log.
|
||||
Ok(CtlRequest::CacheVacuum) => match core.start_vacuum() {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: "compacting the cache database in the background".into(),
|
||||
},
|
||||
Err(e) => CtlResponse::error(CoreError::internal(format!("vacuum: {e:?}"))),
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::GetQueueStatus) => CtlResponse::Transfers {
|
||||
items: core.transfers.snapshot(),
|
||||
@@ -1023,7 +1120,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::ListActivity { limit }) => CtlResponse::Activity {
|
||||
items: core.list_activity(limit),
|
||||
items: core.list_activity(clamp_limit(limit)),
|
||||
},
|
||||
// The request did not parse: the caller sent something malformed.
|
||||
Err(e) => CtlResponse::error(CoreError::invalid(format!("bad request: {e}"))),
|
||||
@@ -1056,6 +1153,12 @@ pub(crate) fn run_control_socket(
|
||||
) {
|
||||
info!("control socket listening");
|
||||
for conn in listener.incoming() {
|
||||
// Checked on every accept, including the deliberate poke teardown sends
|
||||
// to wake this thread out of a blocking accept (bugs.md B44).
|
||||
if core.shutdown.is_stopping() {
|
||||
info!("control socket closing");
|
||||
return;
|
||||
}
|
||||
match conn {
|
||||
Ok(stream) => {
|
||||
let Some(permit) = ControlHandlerPermit::acquire() else {
|
||||
|
||||
@@ -536,7 +536,14 @@ impl Core {
|
||||
// folder the daemon no longer tracks, and nothing would ever unmount it.
|
||||
// Tear it down first — including before trashing the remote tree it
|
||||
// serves, which would otherwise leave it answering for deleted nodes.
|
||||
if let Some(mount) = self.mounts.lock().remove(&id)
|
||||
// Taken in its own statement so the registry lock is released before the
|
||||
// teardown: an edition-2024 `if let` chain holds the guard for the whole
|
||||
// body, and `teardown` unmounts and joins the session thread — which can
|
||||
// be inside a multi-second transfer. The shutdown path takes the same
|
||||
// lock, so holding it here risks systemd's stop timeout landing mid-
|
||||
// unmount and stranding the endpoint.
|
||||
let taken = self.mounts.lock().remove(&id);
|
||||
if let Some(mount) = taken
|
||||
&& let Err(error) = commit_after_teardown(mount.teardown(), || ())
|
||||
{
|
||||
let _ = self.db.sync_folder_set_state(id, "error", now_secs());
|
||||
@@ -584,19 +591,8 @@ impl Core {
|
||||
pub(crate) fn fork_state(&self) -> Core {
|
||||
let mut fork = self.clone();
|
||||
fork.primary = false;
|
||||
let share_access = self.state.lock().share_access.clone();
|
||||
fork.state = Arc::new(Mutex::new(State {
|
||||
entries: HashMap::new(),
|
||||
by_uid: HashMap::new(),
|
||||
children: HashMap::new(),
|
||||
next_ino: 2,
|
||||
active_writes: HashMap::new(),
|
||||
handles: HashMap::new(),
|
||||
next_fh: 1,
|
||||
access_changes: std::collections::HashSet::new(),
|
||||
share_access,
|
||||
db: self.db.clone(),
|
||||
}));
|
||||
let share_access = self.state().share_access.clone();
|
||||
fork.state = Arc::new(Mutex::new(State::new(self.db.clone(), share_access, 2)));
|
||||
// A fresh inode space needs a fresh notification channel: this fork's
|
||||
// session is the only one that knows these inodes, so it must be the one
|
||||
// notified about them. Filled in by `spawn_ondemand_mount`.
|
||||
@@ -866,8 +862,11 @@ impl Core {
|
||||
Ok(format!("{} is now on-demand", local.display()))
|
||||
}
|
||||
ApplicableMountMode::Mirror => {
|
||||
// ondemand→mirror: tear down the secondary mount first.
|
||||
if let Some(mount) = self.mounts.lock().remove(&id)
|
||||
// ondemand→mirror: tear down the secondary mount first. Taken
|
||||
// before the teardown so the registry lock is not held across
|
||||
// the unmount and thread join (see `remove_sync_folder`).
|
||||
let taken = self.mounts.lock().remove(&id);
|
||||
if let Some(mount) = taken
|
||||
&& let Err(error) = commit_after_teardown(mount.teardown(), || ())
|
||||
{
|
||||
let _ = self.db.sync_folder_set_state(id, "error", now_secs());
|
||||
|
||||
+687
-67
@@ -20,10 +20,14 @@
|
||||
//! are kept as a conflict copy rather than discarded — see
|
||||
//! [`Core::revision_conflict`].
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use parking_lot::Mutex;
|
||||
|
||||
use pdfs_core::batch;
|
||||
use pdfs_core::cache::{Baseline, StagedWrite};
|
||||
@@ -36,17 +40,132 @@ use tracing::{debug, error, info, warn};
|
||||
use super::state::Intervals;
|
||||
use super::transfers::CountingReader;
|
||||
use super::{
|
||||
Core, DRAIN_BACKOFF_MAX, DRAIN_BACKOFF_MIN, DRAIN_IDLE_POLL, ROOT_INO, conflict_name,
|
||||
Core, DRAIN_BACKOFF_MAX, DRAIN_BACKOFF_MIN, DRAIN_IDLE_POLL, DRAIN_REVISION_DEBOUNCE,
|
||||
DRAIN_REVISION_DEBOUNCE_MAX, ROOT_INO, UPLOAD_TIME_MEMORY, WriteAuthority, conflict_name,
|
||||
is_already_exists, is_gone, is_local_uid_str, media_type_for, node_revision_id, node_size,
|
||||
now_millis, now_secs, parse_node_uid,
|
||||
};
|
||||
|
||||
const DRAIN_ACCESS_RECHECK: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
/// How long an op may stay access-deferred before the deferral is reported as a
|
||||
/// failure instead of retried in silence.
|
||||
///
|
||||
/// An access deferral costs no attempt and records no error, which is right for
|
||||
/// the case it was written for: a share downgraded while the queue was moving,
|
||||
/// undone a moment later. It is wrong for anything that does not clear. Until
|
||||
/// this window existed such an op was re-deferred every
|
||||
/// [`DRAIN_ACCESS_RECHECK`] indefinitely with `attempts` at zero, `last_error`
|
||||
/// null and only a `debug!` line to show for it, so `pdfs status` counted it
|
||||
/// among ordinary queued uploads and nothing ever said otherwise.
|
||||
///
|
||||
/// The condition that produced that in practice — a node absent from the local
|
||||
/// tree, which [`Db::effective_node_access`] used to report indistinguishably
|
||||
/// from a revoked permission — is now answered directly by
|
||||
/// [`Core::resolve_unknown_authority`]. This window remains the backstop for
|
||||
/// every deferral that does not resolve, including a refetch that keeps not
|
||||
/// helping.
|
||||
///
|
||||
/// Five minutes is far longer than a permission change takes to settle and far
|
||||
/// shorter than a user's patience with bytes that are not moving. Past it each
|
||||
/// recheck records a failure, so the op enters the ordinary backoff and, at
|
||||
/// [`FAILING_ATTEMPTS`], is counted and named by `Response::Status`.
|
||||
///
|
||||
/// [`Db::effective_node_access`]: pdfs_core::db::Db::effective_node_access
|
||||
/// [`FAILING_ATTEMPTS`]: pdfs_core::db::FAILING_ATTEMPTS
|
||||
const DRAIN_ACCESS_DEFER_LIMIT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// A registered cancellation flag for one in-flight upload, deregistered when
|
||||
/// dropped. See [`Core::begin_cancellable_upload`].
|
||||
struct UploadCancel {
|
||||
registry: Arc<Mutex<HashMap<NodeUid, Arc<AtomicBool>>>>,
|
||||
uid: NodeUid,
|
||||
flag: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl UploadCancel {
|
||||
/// Whether this upload was told to stop. Distinguishes an upload we
|
||||
/// abandoned on purpose from one the network broke.
|
||||
fn cancelled(&self) -> bool {
|
||||
self.flag.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn flag(&self) -> Arc<AtomicBool> {
|
||||
self.flag.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UploadCancel {
|
||||
fn drop(&mut self) {
|
||||
let mut registry = self.registry.lock();
|
||||
// By identity: a supersede racing this upload's last bytes may already
|
||||
// have registered the next one under the same uid.
|
||||
if registry
|
||||
.get(&self.uid)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, &self.flag))
|
||||
{
|
||||
registry.remove(&self.uid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The debounce arithmetic of [`Core::revision_debounce`], without the map:
|
||||
/// the last measured upload time for this node, bounded on both sides.
|
||||
///
|
||||
/// A node nobody has uploaded yet gets the fixed grace period, which is the
|
||||
/// same answer the fixed debounce always gave.
|
||||
fn adaptive_debounce(measured: Option<Duration>) -> Duration {
|
||||
measured
|
||||
.unwrap_or(DRAIN_REVISION_DEBOUNCE)
|
||||
.clamp(DRAIN_REVISION_DEBOUNCE, DRAIN_REVISION_DEBOUNCE_MAX)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum DrainDisposition {
|
||||
Applied,
|
||||
AccessDeferred,
|
||||
/// An authority this op needs is missing from the local tree, so no access
|
||||
/// answer exists for it yet. Carries the uid to ask the remote about.
|
||||
AuthorityUnknown(NodeUid),
|
||||
}
|
||||
|
||||
/// Why an op went back on the clock without consuming an attempt. Only affects
|
||||
/// what the user is told — every variant is deferred and reported identically.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum AccessDeferral {
|
||||
/// An authority the tree knows about refuses the write.
|
||||
NotWritable,
|
||||
/// An authority missing from the tree was just re-read from the remote and
|
||||
/// re-interned; the access check gets another look on the next pass.
|
||||
AuthorityRefetched,
|
||||
/// An authority missing from the tree could not be asked about.
|
||||
RemoteUnreachable,
|
||||
}
|
||||
|
||||
impl AccessDeferral {
|
||||
fn log_reason(self) -> &'static str {
|
||||
match self {
|
||||
Self::NotWritable => "the node is not writable",
|
||||
Self::AuthorityRefetched => "the node was missing locally and has been re-read",
|
||||
Self::RemoteUnreachable => "the node is missing locally and the remote is unreachable",
|
||||
}
|
||||
}
|
||||
|
||||
/// Phrased to complete "<reason> for 300s." in a recorded `last_error`, so
|
||||
/// the text the user sees names the actual condition rather than the
|
||||
/// permission question it used to be flattened into.
|
||||
fn failure_reason(self) -> &'static str {
|
||||
match self {
|
||||
Self::NotWritable => "the share has not been writable",
|
||||
Self::AuthorityRefetched => {
|
||||
"the node has been missing from the local tree, and re-reading it from the remote \
|
||||
has not restored it"
|
||||
}
|
||||
Self::RemoteUnreachable => {
|
||||
"the node has been missing from the local tree and the remote has been unreachable"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pending_op_authorities(op: &PendingOp) -> Result<Vec<NodeUid>, Box<dyn std::error::Error>> {
|
||||
@@ -89,7 +208,7 @@ fn pending_op_authorities(op: &PendingOp) -> Result<Vec<NodeUid>, Box<dyn std::e
|
||||
|
||||
fn run_authorized_drain(
|
||||
op: &PendingOp,
|
||||
mut require: impl FnMut(&NodeUid) -> Result<(), fuser::Errno>,
|
||||
mut authority_of: impl FnMut(&NodeUid) -> WriteAuthority,
|
||||
apply: impl FnOnce() -> Result<(), Box<dyn std::error::Error>>,
|
||||
) -> Result<DrainDisposition, Box<dyn std::error::Error>> {
|
||||
// This check is the admission/linearization point. A downgrade that lands
|
||||
@@ -97,15 +216,14 @@ fn run_authorized_drain(
|
||||
// attempt. Every later queued operation rechecks. No permission lock is
|
||||
// held across remote latency.
|
||||
for authority in pending_op_authorities(op)? {
|
||||
match require(&authority) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.code() == libc::EACCES => {
|
||||
return Ok(DrainDisposition::AccessDeferred);
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(
|
||||
format!("drain access check failed with errno {}", error.code()).into(),
|
||||
);
|
||||
match authority_of(&authority) {
|
||||
WriteAuthority::Writable => {}
|
||||
WriteAuthority::Denied => return Ok(DrainDisposition::AccessDeferred),
|
||||
// Nothing in the tree can answer for this node. Deferring on it
|
||||
// waits for a permission change that has no reason to come, so the
|
||||
// caller asks the remote instead (B83).
|
||||
WriteAuthority::Unknown => {
|
||||
return Ok(DrainDisposition::AuthorityUnknown(authority));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -127,44 +245,85 @@ impl Core {
|
||||
/// that is due — one file wedged against a folder that no longer exists must
|
||||
/// not hold up an unrelated upload behind it. The worker only blocks once
|
||||
/// nothing is due at all.
|
||||
pub(crate) fn run_pending_drain(&self) {
|
||||
///
|
||||
/// Several of these run at once ([`DRAIN_WORKERS`]), sharing the queue
|
||||
/// through [`Db::claim_next_due_op`]: the alternative was that a 10 GiB
|
||||
/// upload held every queued rename, trash and small write behind it for as
|
||||
/// long as it took. Ordering only has to hold *per node*, and the claim
|
||||
/// query is what guarantees it — no two workers are ever on one uid.
|
||||
///
|
||||
/// `primary` marks the one worker that also runs the queue's idle chores.
|
||||
/// They are cheap, but they are not per-op work and doing them in every
|
||||
/// worker would just multiply them.
|
||||
///
|
||||
/// [`DRAIN_WORKERS`]: super::DRAIN_WORKERS
|
||||
/// [`Db::claim_next_due_op`]: pdfs_core::db::Db::claim_next_due_op
|
||||
pub(crate) fn run_pending_drain(&self, primary: bool) {
|
||||
loop {
|
||||
// Between ops, never inside one: an op that has started is either
|
||||
// retired or released, so stopping here cannot leave the queue with
|
||||
// a row claimed by a worker that no longer exists (bugs.md B44).
|
||||
if self.shutdown.is_stopping() {
|
||||
debug!(primary, "drain worker stopping");
|
||||
return;
|
||||
}
|
||||
let now = now_millis();
|
||||
// One row, chosen by the database. Reading the whole queue to pick
|
||||
// one op made a long queue quadratic to drain, and held the shared
|
||||
// connection — and so every FUSE metadata call — for the duration.
|
||||
let due = self.db.next_due_op(now).unwrap_or_default();
|
||||
|
||||
let Some(op) = due.filter(|_| self.online.load(Ordering::Relaxed)) else {
|
||||
self.recover_fsynced_writes();
|
||||
// A debounced or backed-off op may be waiting: sleep only
|
||||
// until it becomes due rather than the full idle-poll.
|
||||
self.wait_for_drain_work_or_due();
|
||||
continue;
|
||||
// One row, chosen and *claimed* by the database. Reading the whole
|
||||
// queue to pick one op made a long queue quadratic to drain, and
|
||||
// held the shared connection — and so every FUSE metadata call —
|
||||
// for the duration.
|
||||
let due = match self.db.claim_next_due_op(now) {
|
||||
Ok(due) => due,
|
||||
Err(error) => {
|
||||
error!(%error, "claiming the next pending operation failed");
|
||||
self.wait_for_drain_work();
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let online = self.online.load(Ordering::Relaxed);
|
||||
let op = match due {
|
||||
Some(op) if online => op,
|
||||
idle => {
|
||||
// Claimed but offline: hand it straight back, or it stays
|
||||
// invisible to every worker until this one happens to be
|
||||
// the next to notice the reconnect.
|
||||
if let Some(op) = idle {
|
||||
self.release_claim(&op);
|
||||
}
|
||||
if primary {
|
||||
// Nothing to upload is exactly when the connection is
|
||||
// free, so this is where the read path's buffered LRU
|
||||
// touches get written (`ContentCache::flush_touches`).
|
||||
// Cheap and a no-op when empty.
|
||||
self.cache.flush_touches();
|
||||
self.recover_fsynced_writes();
|
||||
}
|
||||
// A debounced or backed-off op may be waiting: sleep only
|
||||
// until it becomes due rather than the full idle-poll.
|
||||
self.wait_for_drain_work_or_due();
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let outcome = run_authorized_drain(
|
||||
&op,
|
||||
|uid| self.require_uid_writable(uid),
|
||||
|uid| self.uid_write_authority(uid),
|
||||
|| self.drain_op(&op),
|
||||
);
|
||||
if matches!(outcome, Ok(DrainDisposition::AccessDeferred)) {
|
||||
let next_attempt_at = now_millis() + DRAIN_ACCESS_RECHECK.as_millis() as i64;
|
||||
if let Err(error) = self.db.defer_op_without_attempt(op.id, next_attempt_at) {
|
||||
error!(
|
||||
uid = %op.uid,
|
||||
%error,
|
||||
"deferring access-blocked pending operation failed"
|
||||
);
|
||||
self.wait_for_drain_work();
|
||||
} else {
|
||||
debug!(
|
||||
uid = %op.uid,
|
||||
next_attempt_at,
|
||||
"pending operation deferred until access is writable"
|
||||
);
|
||||
}
|
||||
self.defer_for_access(&op, AccessDeferral::NotWritable);
|
||||
continue;
|
||||
}
|
||||
if let Ok(DrainDisposition::AuthorityUnknown(ref authority)) = outcome {
|
||||
self.resolve_unknown_authority(&op, authority);
|
||||
continue;
|
||||
}
|
||||
// The access check let this attempt through, so any earlier run of
|
||||
// deferrals is over and must not be counted against the next one.
|
||||
if matches!(outcome, Ok(DrainDisposition::Applied))
|
||||
&& let Err(error) = self.db.clear_op_access_deferral(op.id)
|
||||
{
|
||||
debug!(uid = %op.uid, %error, "clearing an access-deferral window failed");
|
||||
}
|
||||
if let Err(e) = outcome {
|
||||
let attempts = op.attempts + 1;
|
||||
let err_str = e.to_string();
|
||||
@@ -184,9 +343,234 @@ impl Core {
|
||||
now_millis() + backoff.as_millis() as i64,
|
||||
) {
|
||||
error!(uid = %op.uid, error = %e, "recording a drain failure failed");
|
||||
self.release_claim(&op);
|
||||
self.wait_for_drain_work();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Unconditionally, on every path: a handler that retired its own row
|
||||
// leaves nothing to release, and one that returned without retiring
|
||||
// must not leave the row claimed by a worker that has moved on.
|
||||
self.release_claim(&op);
|
||||
}
|
||||
}
|
||||
|
||||
/// Put an op the access check refused back on the clock, and decide whether
|
||||
/// its refusal is still news.
|
||||
///
|
||||
/// Inside [`DRAIN_ACCESS_DEFER_LIMIT`] a deferral is what it always was: no
|
||||
/// attempt consumed, no error recorded, retried in five seconds. Past it the
|
||||
/// deferral is reported through [`Db::record_op_failure`] like any other
|
||||
/// stuck op, which is what puts it in `attempts`, in `last_error`, and — at
|
||||
/// [`FAILING_ATTEMPTS`] — in the `failing_ops` count and error text that
|
||||
/// `pdfs status` and the GUI show. The row and its staged blob stay exactly
|
||||
/// where they are either way; this changes only whether the user can see
|
||||
/// them.
|
||||
///
|
||||
/// [`Db::record_op_failure`]: pdfs_core::db::Db::record_op_failure
|
||||
/// [`FAILING_ATTEMPTS`]: pdfs_core::db::FAILING_ATTEMPTS
|
||||
fn defer_for_access(&self, op: &PendingOp, reason: AccessDeferral) {
|
||||
let now = now_millis();
|
||||
let next_attempt_at = now + DRAIN_ACCESS_RECHECK.as_millis() as i64;
|
||||
let since = match self.db.defer_op_for_access(op.id, now, next_attempt_at) {
|
||||
Ok(since) => since,
|
||||
Err(error) => {
|
||||
error!(
|
||||
uid = %op.uid,
|
||||
%error,
|
||||
"deferring access-blocked pending operation failed"
|
||||
);
|
||||
self.release_claim(op);
|
||||
self.wait_for_drain_work();
|
||||
return;
|
||||
}
|
||||
};
|
||||
let blocked = Duration::from_millis(now.saturating_sub(since).max(0) as u64);
|
||||
if since == now {
|
||||
// The first deferral of a run, at `warn!`: a write the mount accepted
|
||||
// is not going anywhere for now, and that is worth a line even when
|
||||
// the next recheck clears it.
|
||||
warn!(
|
||||
uid = %op.uid,
|
||||
kind = %op.kind,
|
||||
reason = reason.log_reason(),
|
||||
"pending operation deferred"
|
||||
);
|
||||
}
|
||||
if blocked >= DRAIN_ACCESS_DEFER_LIMIT {
|
||||
let attempts = op.attempts + 1;
|
||||
let backoff = DRAIN_BACKOFF_MIN
|
||||
.saturating_mul(1u32 << attempts.min(6))
|
||||
.min(DRAIN_BACKOFF_MAX);
|
||||
let error = format!(
|
||||
"{} for {}s. The staged bytes are kept.",
|
||||
reason.failure_reason(),
|
||||
blocked.as_secs()
|
||||
);
|
||||
warn!(uid = %op.uid, attempts, blocked_secs = blocked.as_secs(),
|
||||
"pending operation has been access-blocked past the limit; reporting it");
|
||||
if let Err(error) =
|
||||
self.db
|
||||
.record_op_failure(op.id, &error, now_millis() + backoff.as_millis() as i64)
|
||||
{
|
||||
error!(uid = %op.uid, %error, "recording an access-deferral failure failed");
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
uid = %op.uid,
|
||||
next_attempt_at,
|
||||
blocked_secs = blocked.as_secs(),
|
||||
reason = reason.log_reason(),
|
||||
"pending operation deferred until access is writable"
|
||||
);
|
||||
}
|
||||
self.release_claim(op);
|
||||
}
|
||||
|
||||
/// Answer an authority the local tree has no row for by asking the remote,
|
||||
/// rather than deferring on a permission question nobody asked (B83).
|
||||
///
|
||||
/// Three outcomes, and the point of the whole exercise is that they are
|
||||
/// three:
|
||||
///
|
||||
/// * the node is still there — the tree simply lost it (a pruned subtree, a
|
||||
/// mount whose registration went away). Re-intern it and the next pass has
|
||||
/// a real access answer to work with.
|
||||
/// * the node is gone remotely. That is a permanent, *known* condition, so
|
||||
/// it is recorded as a failure immediately instead of waiting out
|
||||
/// [`DRAIN_ACCESS_DEFER_LIMIT`] to say so. The row and its staged blob are
|
||||
/// kept, as on every other failure path — the bytes are still the user's,
|
||||
/// and `pdfs recover` is how they get them back.
|
||||
/// * we could not ask. Ordinary deferral; the network is the network.
|
||||
fn resolve_unknown_authority(&self, op: &PendingOp, authority: &NodeUid) {
|
||||
match self.fetch_node_remote(authority) {
|
||||
Ok(Some(node)) => {
|
||||
if let Err(error) = self.db.upsert_node(&node) {
|
||||
error!(uid = %op.uid, %authority, %error,
|
||||
"re-interning a queued operation's missing authority failed");
|
||||
} else {
|
||||
info!(uid = %op.uid, %authority,
|
||||
"re-interned an authority missing from the local tree; retrying");
|
||||
}
|
||||
// Deferred, not applied: the re-read has to go through the same
|
||||
// access check as everything else, on the next pass. The window
|
||||
// keeps running, so a refetch that never actually helps still
|
||||
// gets reported instead of looping in silence.
|
||||
self.defer_for_access(op, AccessDeferral::AuthorityRefetched);
|
||||
}
|
||||
Ok(None) => {
|
||||
let attempts = op.attempts + 1;
|
||||
let backoff = DRAIN_BACKOFF_MIN
|
||||
.saturating_mul(1u32 << attempts.min(6))
|
||||
.min(DRAIN_BACKOFF_MAX);
|
||||
let error = format!(
|
||||
"{authority} no longer exists remotely, so this operation has nothing to \
|
||||
apply to. The staged bytes are kept."
|
||||
);
|
||||
warn!(uid = %op.uid, %authority, attempts,
|
||||
"a queued operation's authority is gone remotely; reporting it");
|
||||
if let Err(error) = self.db.record_op_failure(
|
||||
op.id,
|
||||
&error,
|
||||
now_millis() + backoff.as_millis() as i64,
|
||||
) {
|
||||
error!(uid = %op.uid, %error, "recording a missing-authority failure failed");
|
||||
}
|
||||
self.release_claim(op);
|
||||
}
|
||||
Err(error) => {
|
||||
debug!(uid = %op.uid, %authority, %error,
|
||||
"could not ask the remote about a missing authority");
|
||||
self.defer_for_access(op, AccessDeferral::RemoteUnreachable);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand a claimed op back to the queue, logging rather than propagating —
|
||||
/// there is no caller left to propagate to, and the drain must not stop.
|
||||
fn release_claim(&self, op: &PendingOp) {
|
||||
if let Err(error) = self.db.release_op_claim(op.id) {
|
||||
error!(uid = %op.uid, %error, "releasing a drain claim failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a freshly queued revision of `uid` should wait before a worker
|
||||
/// may pick it up.
|
||||
///
|
||||
/// [`DRAIN_REVISION_DEBOUNCE`] exists for a tool that preallocates a file
|
||||
/// and then writes it, where two seconds is the right scale. It is the wrong
|
||||
/// scale for a large file on a slow link: an editor saving a 200 MB project
|
||||
/// every ten seconds queues a write, the drain starts sending it, the next
|
||||
/// save supersedes it mid-flight, and the file never finishes uploading
|
||||
/// while the user keeps working. Waiting roughly as long as the last upload
|
||||
/// of *this* node took moves that supersede into the queue, where replacing
|
||||
/// a row costs nothing.
|
||||
///
|
||||
/// Never shorter than the fixed debounce and never longer than
|
||||
/// [`DRAIN_REVISION_DEBOUNCE_MAX`]: a node whose upload is genuinely slow
|
||||
/// still has to reach Drive, and a single pathological stall must not park
|
||||
/// it.
|
||||
///
|
||||
/// [`DRAIN_REVISION_DEBOUNCE`]: super::DRAIN_REVISION_DEBOUNCE
|
||||
/// [`DRAIN_REVISION_DEBOUNCE_MAX`]: super::DRAIN_REVISION_DEBOUNCE_MAX
|
||||
pub(crate) fn revision_debounce(&self, uid: &NodeUid) -> Duration {
|
||||
let measured = self
|
||||
.upload_times
|
||||
.lock()
|
||||
.get(&uid.to_string())
|
||||
.map(|&(ms, _)| Duration::from_millis(ms));
|
||||
adaptive_debounce(measured)
|
||||
}
|
||||
|
||||
/// Remember how long an upload of `uid` took, for the next write's
|
||||
/// debounce.
|
||||
///
|
||||
/// Only a *completed* upload is a measurement. A cancelled or failed one
|
||||
/// says how long it ran before it stopped, which is not how long the file
|
||||
/// takes to send, and feeding that back would let one network fault widen
|
||||
/// the debounce for good.
|
||||
fn record_upload_time(&self, uid: &NodeUid, took: Duration) {
|
||||
let mut times = self.upload_times.lock();
|
||||
let now = now_millis();
|
||||
if times.len() >= UPLOAD_TIME_MEMORY
|
||||
&& !times.contains_key(&uid.to_string())
|
||||
&& let Some(oldest) = times
|
||||
.iter()
|
||||
.min_by_key(|(_, (_, at))| *at)
|
||||
.map(|(k, _)| k.clone())
|
||||
{
|
||||
times.remove(&oldest);
|
||||
}
|
||||
times.insert(uid.to_string(), (took.as_millis() as u64, now));
|
||||
}
|
||||
|
||||
/// Register a cancellation flag for an upload of `uid` that is about to
|
||||
/// start, returning a guard that deregisters it.
|
||||
///
|
||||
/// Deregistration is by identity rather than by key: a supersede that
|
||||
/// arrives just as this upload finishes may already have registered the
|
||||
/// *next* upload's flag under the same uid, and removing that one would
|
||||
/// leave the newer upload uncancellable.
|
||||
fn begin_cancellable_upload(&self, uid: &NodeUid) -> UploadCancel {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
self.upload_cancel.lock().insert(uid.clone(), flag.clone());
|
||||
UploadCancel {
|
||||
registry: self.upload_cancel.clone(),
|
||||
uid: uid.clone(),
|
||||
flag,
|
||||
}
|
||||
}
|
||||
|
||||
/// Tell an upload of `uid` that is on the wire to stop, if there is one.
|
||||
///
|
||||
/// Called when a newer revision of the file is queued. The upload does not
|
||||
/// stop instantly — it stops when the SDK next asks its reader for bytes,
|
||||
/// which for a 4 MiB block is soon enough to matter and late enough to be
|
||||
/// safe: the staged blob it is reading stays open until it lets go.
|
||||
pub(crate) fn cancel_upload(&self, uid: &NodeUid) {
|
||||
if let Some(flag) = self.upload_cancel.lock().get(uid) {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
debug!(%uid, "cancelling a superseded upload");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,6 +735,9 @@ impl Core {
|
||||
}
|
||||
self.db.complete_trash_op(op.id, &uid)?;
|
||||
self.own_sealed_revs.lock().remove(&uid);
|
||||
if let Err(e) = self.db.clear_own_sealed_rev(&uid.to_string()) {
|
||||
debug!(%uid, error = %e, "clearing the sealed-revision record failed");
|
||||
}
|
||||
self.invalidate_trash();
|
||||
self.log_activity(ActivityKind::Trash, &name, "trashed", true);
|
||||
info!(%uid, name, "pending trash landed");
|
||||
@@ -625,7 +1012,16 @@ impl Core {
|
||||
// instead of forking — so the finished bytes win the name rather than
|
||||
// being exiled to a `(sync-conflict)` copy.
|
||||
let remote_rev = node_revision_id(&node);
|
||||
let own_rev = self.own_sealed_revs.lock().get(uid).cloned();
|
||||
// In memory first, then the table it is written through to: the map is
|
||||
// this process's cache of it, and after a restart only the table has
|
||||
// the answer (which is the whole reason it is persisted).
|
||||
let own_rev = match self.own_sealed_revs.lock().get(uid).cloned() {
|
||||
Some(rev) => Some(rev),
|
||||
None => self
|
||||
.db
|
||||
.own_sealed_rev(&uid.to_string(), now_millis())
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
if is_own_self_supersede(meta.complete, remote_rev.as_deref(), own_rev.as_deref()) {
|
||||
debug!(%uid, ?remote_rev,
|
||||
"queued write chains onto our own sealed revision; not a conflict");
|
||||
@@ -709,8 +1105,9 @@ impl Core {
|
||||
self.db.delete_op(op.id)?;
|
||||
// Dropping the pending entry hands the node back to the remote's truth:
|
||||
// reads stop coming from the staged blob, and the event sync stops
|
||||
// skipping it as "ahead of the server" (offline.md Phase 3a).
|
||||
self.pending.lock().remove(uid);
|
||||
// skipping it as "ahead of the server" (offline.md Phase 3a). Only if it
|
||||
// is still *this* write's entry — see [`Core::release_pending`].
|
||||
self.release_pending(uid, blob);
|
||||
self.cache.discard_staged(blob);
|
||||
self.cache.evict(uid);
|
||||
self.evict_reader(uid);
|
||||
@@ -748,7 +1145,7 @@ impl Core {
|
||||
error!(%uid, name, reason, staged = %blob.display(),
|
||||
"cannot place a conflicted write; bytes kept in staging");
|
||||
self.db.delete_op(op.id)?;
|
||||
self.pending.lock().remove(uid);
|
||||
self.release_pending(uid, blob);
|
||||
self.cache.evict(uid);
|
||||
self.evict_reader(uid);
|
||||
self.log_activity(
|
||||
@@ -760,6 +1157,24 @@ impl Core {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop `uid`'s pending entry, but only while it still names `blob`.
|
||||
///
|
||||
/// A supersede can replace the entry while the op that owned it is in
|
||||
/// flight, so an unconditional `remove` retires a *newer* write's staged
|
||||
/// blob: reads fall back to a stale remote, the next `remote_baseline` forks
|
||||
/// a spurious `(sync-conflict)` copy of bytes that were never in conflict,
|
||||
/// and the incomplete-write interlock — which keys off the pending entry —
|
||||
/// is disarmed, letting a gap-fill read from a revision that no longer
|
||||
/// describes the file.
|
||||
pub(crate) fn release_pending(&self, uid: &NodeUid, blob: &Path) {
|
||||
let mut pending = self.pending.lock();
|
||||
if let Some(current) = pending.get(uid)
|
||||
&& current.path == blob
|
||||
{
|
||||
pending.remove(uid);
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a folder a queued op targets has stopped being a place a node can
|
||||
/// go. A failure to ask is not an answer: only a definite "trashed" or "not
|
||||
/// there" counts, so a network fault leaves the op to retry normally.
|
||||
@@ -826,7 +1241,7 @@ impl Core {
|
||||
/// root yet, which is not where the drain runs; the drain must not panic
|
||||
/// over it regardless, since that would stop the queue for good.
|
||||
pub(crate) fn root_uid(&self) -> Option<NodeUid> {
|
||||
let st = self.state.lock();
|
||||
let st = self.state();
|
||||
st.entries.get(&ROOT_INO).map(|e| e.uid.clone())
|
||||
}
|
||||
|
||||
@@ -872,28 +1287,43 @@ impl Core {
|
||||
let guard =
|
||||
self.transfers
|
||||
.begin(&name, meta.uid.clone(), TransferDirection::Upload, meta.len);
|
||||
let reader = CountingReader::new(File::open(&blob)?, &guard);
|
||||
self.rt.block_on(self.client.upload_new_revision_from(
|
||||
// Registered before the first byte moves, so a write that lands while
|
||||
// this is on the wire can stop it rather than paying for a revision the
|
||||
// op it queued will immediately replace.
|
||||
let cancel = self.begin_cancellable_upload(&uid);
|
||||
let reader = CountingReader::new(File::open(&blob)?, &guard).with_cancel(cancel.flag());
|
||||
let started = Instant::now();
|
||||
let sent = self.rt.block_on(self.client.upload_new_revision_from(
|
||||
&uid,
|
||||
reader,
|
||||
meta.len as i64,
|
||||
Vec::new(),
|
||||
None,
|
||||
))?;
|
||||
));
|
||||
let took = started.elapsed();
|
||||
if let Err(e) = sent {
|
||||
if cancel.cancelled() {
|
||||
// Not a failure, and not this op's row to retire: the write that
|
||||
// cancelled us deleted it and took ownership of the staged blob
|
||||
// when it superseded this revision. Leaving the blob alone here
|
||||
// is the whole point — it is either already unlinked or it
|
||||
// belongs to the newer op.
|
||||
info!(%uid, "upload abandoned; a newer write superseded it");
|
||||
return Ok(());
|
||||
}
|
||||
return Err(e.into());
|
||||
}
|
||||
drop(cancel);
|
||||
drop(guard);
|
||||
// Only a completed upload times how long this file takes to send, which
|
||||
// is what the next write's debounce is scaled from.
|
||||
self.record_upload_time(&uid, took);
|
||||
|
||||
// Retire the op before dropping the blob: a crash between the two leaves
|
||||
// an orphaned file (harmless), whereas the reverse would leave a queued
|
||||
// op pointing at nothing.
|
||||
self.db.delete_op(op.id)?;
|
||||
{
|
||||
let mut p = self.pending.lock();
|
||||
if let Some(op_in_mem) = p.get(&uid)
|
||||
&& op_in_mem.path == blob
|
||||
{
|
||||
p.remove(&uid);
|
||||
}
|
||||
}
|
||||
self.release_pending(&uid, &blob);
|
||||
|
||||
// The staged blob now matches the sealed revision, so a pinned file keeps
|
||||
// it as its cached content rather than re-downloading what we just sent.
|
||||
@@ -969,7 +1399,16 @@ impl Core {
|
||||
// change and does not fork (B70 layer B; consulted in
|
||||
// `revision_conflict`).
|
||||
if let Some(rev) = sealed_rev.clone() {
|
||||
self.own_sealed_revs.lock().insert(uid.clone(), rev);
|
||||
self.own_sealed_revs.lock().insert(uid.clone(), rev.clone());
|
||||
// Write through: losing this at shutdown reopened the fork
|
||||
// window B70 layer B closed, for the first drain after a
|
||||
// restart.
|
||||
if let Err(e) = self
|
||||
.db
|
||||
.set_own_sealed_rev(&uid.to_string(), &rev, now_millis())
|
||||
{
|
||||
warn!(%uid, error = %e, "persisting our sealed revision failed");
|
||||
}
|
||||
}
|
||||
self.for_each_state(|st| {
|
||||
for aw in st.active_writes.values_mut() {
|
||||
@@ -1190,12 +1629,10 @@ mod tests {
|
||||
let applied = Cell::new(false);
|
||||
let disposition = run_authorized_drain(
|
||||
&op,
|
||||
|uid| {
|
||||
db.effective_node_access(uid)
|
||||
.unwrap()
|
||||
.filter(|access| access.writable())
|
||||
.map(|_| ())
|
||||
.ok_or(fuser::Errno::EACCES)
|
||||
|uid| match db.effective_node_access(uid).unwrap() {
|
||||
Some(access) if access.writable() => WriteAuthority::Writable,
|
||||
Some(_) => WriteAuthority::Denied,
|
||||
None => WriteAuthority::Unknown,
|
||||
},
|
||||
|| {
|
||||
applied.set(true);
|
||||
@@ -1235,7 +1672,7 @@ mod tests {
|
||||
|
||||
let disposition = run_authorized_drain(
|
||||
&op,
|
||||
|_| Err(fuser::Errno::EACCES),
|
||||
|_| WriteAuthority::Denied,
|
||||
|| {
|
||||
applied.set(true);
|
||||
Ok(())
|
||||
@@ -1256,13 +1693,157 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
/// A deferral that never clears has to stop being silent. Found in the field
|
||||
/// as revision ops whose node had left the local tree: `require_uid_access`
|
||||
/// reads an unknown node as `EACCES`, so they were re-deferred every five
|
||||
/// seconds for thirty days with `attempts` at zero, `last_error` null and
|
||||
/// `failing_ops` at zero — 18 GiB of accepted writes that `pdfs status`
|
||||
/// reported as ordinary queued uploads.
|
||||
#[test]
|
||||
fn an_access_deferral_that_outlives_the_limit_is_reported_as_failing() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static N: AtomicU64 = AtomicU64::new(0);
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"pdfs-drain-defer-limit-{}-{}",
|
||||
std::process::id(),
|
||||
N.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let db = Db::open(&dir.join("cache.db")).unwrap();
|
||||
db.enqueue_op(&pending(OP_REVISION)).unwrap();
|
||||
let op = db.pending_ops().unwrap().remove(0);
|
||||
|
||||
// Two deferrals a recheck apart: the window opens on the first and the
|
||||
// second is measured against it, not against itself.
|
||||
let now = 1_000_000i64;
|
||||
let since = db.defer_op_for_access(op.id, now, now + 5_000).unwrap();
|
||||
assert_eq!(since, now, "the first deferral opens the window");
|
||||
let later = now + DRAIN_ACCESS_RECHECK.as_millis() as i64;
|
||||
assert_eq!(
|
||||
db.defer_op_for_access(op.id, later, later + 5_000).unwrap(),
|
||||
now,
|
||||
"a later deferral keeps the original stamp"
|
||||
);
|
||||
let queued = db.pending_ops().unwrap();
|
||||
assert_eq!(queued[0].attempts, 0, "deferrals consume no attempt");
|
||||
assert!(queued[0].last_error.is_none());
|
||||
|
||||
// Past the limit the same deferral is reported like any other stuck op.
|
||||
let past = now + DRAIN_ACCESS_DEFER_LIMIT.as_millis() as i64;
|
||||
assert_eq!(
|
||||
db.defer_op_for_access(op.id, past, past + 5_000).unwrap(),
|
||||
now
|
||||
);
|
||||
db.record_op_failure(op.id, "not writable locally", past + 10_000)
|
||||
.unwrap();
|
||||
let failed = db.pending_ops().unwrap();
|
||||
assert_eq!(failed.len(), 1, "the row and its staged bytes stay queued");
|
||||
assert_eq!(failed[0].attempts, 1);
|
||||
assert_eq!(
|
||||
failed[0].last_error.as_deref(),
|
||||
Some("not writable locally")
|
||||
);
|
||||
|
||||
// Reporting closes the window, so the next deferral starts a fresh one
|
||||
// rather than escalating on every recheck from here on.
|
||||
let after = past + 20_000;
|
||||
assert_eq!(
|
||||
db.defer_op_for_access(op.id, after, after + 5_000).unwrap(),
|
||||
after,
|
||||
"recording the failure re-armed the window"
|
||||
);
|
||||
drop(db);
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
/// An op that gets through the access check is no longer blocked, so a
|
||||
/// deferral after it must not inherit the earlier run's age and escalate
|
||||
/// immediately.
|
||||
#[test]
|
||||
fn clearing_the_window_makes_the_next_deferral_a_fresh_run() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static N: AtomicU64 = AtomicU64::new(0);
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"pdfs-drain-defer-clear-{}-{}",
|
||||
std::process::id(),
|
||||
N.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let db = Db::open(&dir.join("cache.db")).unwrap();
|
||||
db.enqueue_op(&pending(OP_REVISION)).unwrap();
|
||||
let op = db.pending_ops().unwrap().remove(0);
|
||||
|
||||
let now = 2_000_000i64;
|
||||
assert_eq!(
|
||||
db.defer_op_for_access(op.id, now, now + 5_000).unwrap(),
|
||||
now
|
||||
);
|
||||
db.clear_op_access_deferral(op.id).unwrap();
|
||||
let later = now + DRAIN_ACCESS_DEFER_LIMIT.as_millis() as i64;
|
||||
assert_eq!(
|
||||
db.defer_op_for_access(op.id, later, later + 5_000).unwrap(),
|
||||
later,
|
||||
"the window reopens at the new deferral, not the old one"
|
||||
);
|
||||
drop(db);
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_authority_the_tree_does_not_have_is_reported_as_unknown_not_deferred() {
|
||||
// The B83 distinction, at the point it is made: a uid with no row must
|
||||
// not come back as `AccessDeferred`, because deferring waits on a
|
||||
// permission change that nothing is going to make. The caller answers
|
||||
// an `AuthorityUnknown` by asking the remote instead.
|
||||
let op = pending(OP_REVISION);
|
||||
let applied = Cell::new(false);
|
||||
let disposition = run_authorized_drain(
|
||||
&op,
|
||||
|_| WriteAuthority::Unknown,
|
||||
|| {
|
||||
applied.set(true);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
disposition,
|
||||
DrainDisposition::AuthorityUnknown(parse_node_uid(&op.uid).unwrap())
|
||||
);
|
||||
assert!(!applied.get(), "an unresolved authority never applies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_parent_authority_names_the_parent_not_the_node() {
|
||||
// A create's authority is its parent, so that is the uid the remote has
|
||||
// to be asked about — reporting the node instead would refetch the
|
||||
// thing that does not exist yet.
|
||||
let mut op = pending(OP_CREATE);
|
||||
let parent = folder_node("vanished-parent", None);
|
||||
op.parent_uid = Some(parent.uid.to_string());
|
||||
let disposition =
|
||||
run_authorized_drain(&op, |_| WriteAuthority::Unknown, || Ok(())).unwrap();
|
||||
assert_eq!(
|
||||
disposition,
|
||||
DrainDisposition::AuthorityUnknown(parent.uid.clone())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_and_editor_drain_authorities_apply_normally() {
|
||||
for access in [Access::Owner, Access::Editor] {
|
||||
let applied = Cell::new(false);
|
||||
let disposition = run_authorized_drain(
|
||||
&pending(OP_REVISION),
|
||||
|_| access.writable().then_some(()).ok_or(fuser::Errno::EACCES),
|
||||
|_| {
|
||||
if access.writable() {
|
||||
WriteAuthority::Writable
|
||||
} else {
|
||||
WriteAuthority::Denied
|
||||
}
|
||||
},
|
||||
|| {
|
||||
applied.set(true);
|
||||
Ok(())
|
||||
@@ -1376,3 +1957,42 @@ mod tests {
|
||||
assert!(!is_own_self_supersede(true, None, None));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod debounce_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_node_never_uploaded_keeps_the_fixed_grace_period() {
|
||||
assert_eq!(adaptive_debounce(None), DRAIN_REVISION_DEBOUNCE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fast_upload_does_not_shorten_the_grace_period() {
|
||||
// The 2 seconds are not about upload time at all — they are the window
|
||||
// in which a preallocate-then-write tool supersedes its own first
|
||||
// close. A file that sends in 50 ms must still get them.
|
||||
assert_eq!(
|
||||
adaptive_debounce(Some(Duration::from_millis(50))),
|
||||
DRAIN_REVISION_DEBOUNCE
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slow_upload_widens_the_grace_period_to_match() {
|
||||
// The case the fixed debounce got wrong: saves arriving faster than the
|
||||
// upload completes, each superseding one already on the wire.
|
||||
assert_eq!(
|
||||
adaptive_debounce(Some(Duration::from_secs(25))),
|
||||
Duration::from_secs(25)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pathological_upload_cannot_park_a_node() {
|
||||
assert_eq!(
|
||||
adaptive_debounce(Some(Duration::from_secs(6000))),
|
||||
DRAIN_REVISION_DEBOUNCE_MAX
|
||||
);
|
||||
}
|
||||
}
|
||||
+517
-254
File diff suppressed because it is too large.
Load diff
+1570
-270
File diff suppressed because it is too large.
Load diff
+109
-40
@@ -255,6 +255,14 @@ pub(super) fn spawn_session(
|
||||
root: Node,
|
||||
) -> std::io::Result<BackgroundSession> {
|
||||
let mut config = Config::default();
|
||||
// `n_threads` stays at fuser's default of 1 (with `clone_fd` off) on
|
||||
// purpose, for now. A concurrent dispatch loop is worth having, but
|
||||
// `release` currently relies on being serialised against the `open`+`read`
|
||||
// an application issues right after `close(2)` — see the comment on
|
||||
// `Filesystem::release`. Raising it before that ordering is made explicit
|
||||
// (a per-node staging-in-flight barrier) reintroduces the acceptance
|
||||
// suite's "concurrent file 1 mismatch". Kernel-side caps that do *not*
|
||||
// depend on that ordering are negotiated in `ProtonFs::init`.
|
||||
config.mount_options = vec![
|
||||
MountOption::FSName("protondrive".to_string()),
|
||||
MountOption::Subtype("protondrive".to_string()),
|
||||
@@ -349,23 +357,17 @@ pub fn mount(
|
||||
let core = Core {
|
||||
client: client.clone(),
|
||||
rt: rt.clone(),
|
||||
maintenance: Arc::new(Mutex::new(Default::default())),
|
||||
primary_root_uid: root.uid.clone(),
|
||||
primary: true,
|
||||
state: Arc::new(Mutex::new(State {
|
||||
entries: HashMap::new(),
|
||||
by_uid: HashMap::new(),
|
||||
children: HashMap::new(),
|
||||
next_ino: 2,
|
||||
active_writes: HashMap::new(),
|
||||
handles: HashMap::new(),
|
||||
next_fh: 1,
|
||||
access_changes: HashSet::new(),
|
||||
share_access,
|
||||
db: db.clone(),
|
||||
})),
|
||||
state: Arc::new(Mutex::new(State::new(db.clone(), share_access, 2))),
|
||||
cache: Arc::new(cache),
|
||||
readers: Arc::new(Mutex::new(HashMap::new())),
|
||||
stream_ring: Arc::new(Mutex::new(StreamRing::default())),
|
||||
block_ring: Arc::new(Mutex::new(BlockRing::default())),
|
||||
block_flight: Arc::new(BlockFlight::default()),
|
||||
content_repairs: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
|
||||
prefetch: Arc::new(Prefetch::default()),
|
||||
prefetch_budget: Arc::new(tokio::sync::Semaphore::new(PREFETCH_BUDGET)),
|
||||
workers: Arc::new(Workers::new(FUSE_WORKERS)?),
|
||||
db,
|
||||
shared_publication: Arc::new(Mutex::new(())),
|
||||
@@ -375,6 +377,9 @@ pub fn mount(
|
||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||
hidden: Arc::new(Mutex::new(HashSet::new())),
|
||||
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
||||
shutdown: Arc::new(crate::shutdown::Shutdown::default()),
|
||||
upload_times: Arc::new(Mutex::new(HashMap::new())),
|
||||
upload_cancel: Arc::new(Mutex::new(HashMap::new())),
|
||||
timeline_refreshing: Arc::new(AtomicBool::new(false)),
|
||||
albums_refreshing: Arc::new(AtomicBool::new(false)),
|
||||
trash_refreshing: Arc::new(AtomicBool::new(false)),
|
||||
@@ -385,7 +390,9 @@ pub fn mount(
|
||||
self_changes: Arc::new(Mutex::new(HashMap::new())),
|
||||
thumb_gen: Arc::new(Mutex::new(HashSet::new())),
|
||||
no_thumbnail: Arc::new(Mutex::new(HashMap::new())),
|
||||
quota: Arc::new(Mutex::new(None)),
|
||||
size_upgrades: Arc::new(Mutex::new(HashMap::new())),
|
||||
size_waiters: Arc::new(Default::default()),
|
||||
notifier: Arc::new(OnceLock::new()),
|
||||
session_live: Arc::new(AtomicBool::new(false)),
|
||||
transfers: TransferRegistry::new(),
|
||||
@@ -407,16 +414,32 @@ pub fn mount(
|
||||
// aside at open. After `hydrate_pending`, so a recovered partial write can
|
||||
// see an already-queued write to the same node.
|
||||
core.recover_fsynced_writes();
|
||||
{
|
||||
// And finally the staged blobs neither of those accounts for: bytes a
|
||||
// failed release left behind with nothing pointing at them.
|
||||
core.reconcile_staging();
|
||||
// Several drain workers over one queue. One thread meant a single large
|
||||
// upload held every queued rename, trash and small write behind it for as
|
||||
// long as it ran; the claim column (`Db::claim_next_due_op`) keeps the
|
||||
// workers off each other's nodes, which is the only ordering the queue
|
||||
// actually needs. Worker 0 additionally runs the queue's idle chores.
|
||||
//
|
||||
// Every long-lived thread started from here on is kept in `workers` and
|
||||
// joined at the end of this function. Leaving them running turned an
|
||||
// in-process remount into a second generation of drain workers, sync engines
|
||||
// and sweeps all operating on a mount that no longer exists (bugs.md B44).
|
||||
let mut workers: Vec<std::thread::JoinHandle<()>> = Vec::new();
|
||||
for worker in 0..DRAIN_WORKERS {
|
||||
let core = core.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-drain".into())
|
||||
.spawn(move || core.run_pending_drain())?;
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name(format!("pdfs-drain-{worker}"))
|
||||
.spawn(move || core.run_pending_drain(worker == 0))?,
|
||||
);
|
||||
}
|
||||
|
||||
// Start the folder-sync engine. It watches every mirror folder, polls the
|
||||
// remotes, and reconciles on its own thread — never in front of a FUSE call.
|
||||
sync::spawn(core.clone(), sync_rx);
|
||||
workers.extend(sync::spawn(core.clone(), sync_rx));
|
||||
|
||||
// Reconcile leftover `(sync-conflict …)` copies: drop the ones that turned
|
||||
// out identical to the live file, surface the ones that genuinely diverge.
|
||||
@@ -428,18 +451,22 @@ pub fn mount(
|
||||
} else {
|
||||
info!(mode = ?sweep_mode, "conflict sweep enabled");
|
||||
let core = core.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-conflict-sweep".into())
|
||||
.spawn(move || core.run_conflict_sweep_loop())?;
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-conflict-sweep".into())
|
||||
.spawn(move || core.run_conflict_sweep_loop())?,
|
||||
);
|
||||
}
|
||||
|
||||
// Mounted from the cache: watch for the network coming back so the mount can
|
||||
// stop being read-only-ish without the user restarting the daemon.
|
||||
if !online {
|
||||
let core = core.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-online-probe".into())
|
||||
.spawn(move || core.run_online_probe())?;
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-online-probe".into())
|
||||
.spawn(move || core.run_online_probe())?,
|
||||
);
|
||||
}
|
||||
|
||||
// Keep the launcher prompt's "This computer" index fresh. Its own thread:
|
||||
@@ -449,9 +476,12 @@ pub fn mount(
|
||||
let indexing = core.indexing.clone();
|
||||
let transfers = core.transfers.clone();
|
||||
let mountpoint = mountpoint.to_path_buf();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-localindex".into())
|
||||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint))?;
|
||||
let shutdown = core.shutdown.clone();
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-localindex".into())
|
||||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint, shutdown))?,
|
||||
);
|
||||
}
|
||||
|
||||
// Bind the control socket before the FUSE session takes over the thread. A
|
||||
@@ -488,23 +518,29 @@ pub fn mount(
|
||||
let control_core = core.clone();
|
||||
let username = username.clone();
|
||||
let mountpoint = mountpoint.to_path_buf();
|
||||
if let Err(error) = std::thread::Builder::new()
|
||||
match std::thread::Builder::new()
|
||||
.name("pdfs-control".into())
|
||||
.spawn(move || run_control_socket(control_core, username, mountpoint, listener))
|
||||
{
|
||||
let secondaries = core.mounts.lock().close_and_drain();
|
||||
for (id, mount) in secondaries {
|
||||
if let Err(unmount_error) = mount.teardown() {
|
||||
warn!(id, error = %unmount_error, "secondary teardown after control startup failure failed");
|
||||
Ok(handle) => workers.push(handle),
|
||||
Err(error) => {
|
||||
// A failure here abandons the mount, so the background threads
|
||||
// started above have to be told to stop with it.
|
||||
stop_workers(&core, control_socket, workers);
|
||||
let secondaries = core.mounts.lock().close_and_drain();
|
||||
for (id, mount) in secondaries {
|
||||
if let Err(unmount_error) = mount.teardown() {
|
||||
warn!(id, error = %unmount_error, "secondary teardown after control startup failure failed");
|
||||
}
|
||||
}
|
||||
if let Err(unmount_error) = teardown_session(&core.session_live, || {
|
||||
umount_session_unblocked(bg, main_conn)
|
||||
}) {
|
||||
warn!(error = %unmount_error, "unmount after control startup failure failed");
|
||||
}
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
return Err(error);
|
||||
}
|
||||
if let Err(unmount_error) = teardown_session(&core.session_live, || {
|
||||
umount_session_unblocked(bg, main_conn)
|
||||
}) {
|
||||
warn!(error = %unmount_error, "unmount after control startup failure failed");
|
||||
}
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
// Re-establish on-demand mounts only after the primary session is live.
|
||||
@@ -613,10 +649,43 @@ pub fn mount(
|
||||
}
|
||||
}
|
||||
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
// Last, because the workers may legitimately be finishing an upload, and
|
||||
// nothing they can still do is harmful once the mounts are down.
|
||||
stop_workers(&core, control_socket, workers);
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Signal every background loop, wake the ones that are blocked, and join them.
|
||||
///
|
||||
/// The mount used to return with its drain workers, sync engine, sweep, online
|
||||
/// probe, indexer and control listener all still running on `Core` clones. The
|
||||
/// process usually exited straight afterwards and hid it; an in-process remount
|
||||
/// did not, and started a second full set on top of the first (bugs.md B44).
|
||||
///
|
||||
/// Two of the waits need waking rather than merely flagging: the drain workers
|
||||
/// block on their own condvar (`wake_drain`), and the control listener blocks
|
||||
/// inside `accept`, which only returns when a connection arrives — so it gets
|
||||
/// one, from us, after the flag is already set. A join that takes a while is
|
||||
/// expected and correct: a worker part-way through an upload finishes it rather
|
||||
/// than abandoning the user's bytes mid-flight.
|
||||
fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
||||
core.shutdown.stop();
|
||||
core.wake_drain();
|
||||
let _ = core.sync_tx.send(sync::SyncMsg::Stop);
|
||||
// Connect while the socket still exists — that is the whole point of the
|
||||
// poke — then unlink it so nothing else can arrive behind us. The listener
|
||||
// re-checks the flag after every accept, so even a connection that races the
|
||||
// unlink cannot put it back to sleep.
|
||||
let _ = std::os::unix::net::UnixStream::connect(control_socket);
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
for worker in workers {
|
||||
if worker.join().is_err() {
|
||||
warn!("a background worker panicked before shutdown");
|
||||
}
|
||||
}
|
||||
debug!("background workers stopped");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
@@ -149,10 +149,10 @@ impl Core {
|
||||
.pin_list()
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.into_iter()
|
||||
.map(|(uid, path, recursive)| ProfilePin {
|
||||
uid,
|
||||
path,
|
||||
recursive,
|
||||
.map(|row| ProfilePin {
|
||||
uid: row.uid,
|
||||
path: row.path,
|
||||
recursive: row.recursive,
|
||||
})
|
||||
.collect();
|
||||
let cfg = pdfs_core::config::AppDirs::new()
|
||||
|
||||
+797
-107
File diff suppressed because it is too large.
Load diff
@@ -12,6 +12,9 @@ use pdfs_core::control::{
|
||||
BookmarkInfo, DirEntry, InvitationInfo, PublicLinkInfo, ShareEntry, ShareEntryKind, SharedItem,
|
||||
};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use tracing::debug;
|
||||
|
||||
use crate::QUOTA_TTL;
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use proton_drive_rs::{MemberRole, NodeKind};
|
||||
|
||||
@@ -533,9 +536,33 @@ impl Core {
|
||||
.rt
|
||||
.block_on(self.client.quota())
|
||||
.map_err(|e| CoreError::from_api(&e, "account quota"))?;
|
||||
*self.quota.lock() = Some((std::time::Instant::now(), q.max_space, q.used_space));
|
||||
Ok((q.max_space, q.used_space))
|
||||
}
|
||||
|
||||
/// The account quota as `statfs(2)` needs it: cached for [`QUOTA_TTL`],
|
||||
/// refetched after that, and `None` when there is no usable answer at all —
|
||||
/// offline with nothing cached.
|
||||
///
|
||||
/// Never blocks on the network for a *fresh* value when a recent one exists,
|
||||
/// because the callers are `df` and every "is there room?" preflight, and a
|
||||
/// stale byte count is worth more to them than a stalled syscall. Offline,
|
||||
/// the last known figures keep being served rather than degrading to zero.
|
||||
pub(crate) fn account_quota_cached(&self) -> Option<(i64, i64)> {
|
||||
if let Some((at, max_space, used_space)) = *self.quota.lock()
|
||||
&& at.elapsed() < QUOTA_TTL
|
||||
{
|
||||
return Some((max_space, used_space));
|
||||
}
|
||||
match self.account_quota() {
|
||||
Ok(q) => Some(q),
|
||||
Err(error) => {
|
||||
debug!(%error, "account quota refresh failed; serving the last known figures");
|
||||
self.quota.lock().map(|(_, max, used)| (max, used))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- shared by me -----------------------------------------------------
|
||||
|
||||
/// List the nodes I have shared with others, each with a summary of its share
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
//! One shutdown signal shared by every long-lived worker in the daemon.
|
||||
//!
|
||||
//! The mount used to end by unmounting and returning, leaving the drain
|
||||
//! workers, the sync engine, the conflict sweep, the online probe and the local
|
||||
//! indexer running on `Core` clones. In a process that exits immediately that is
|
||||
//! invisible; in an in-process remount — which is how the daemon recovers from a
|
||||
//! mount going away underneath it — it leaks a whole generation of threads that
|
||||
//! keep writing to the database and calling the API on behalf of a mount that no
|
||||
//! longer exists (bugs.md B44).
|
||||
//!
|
||||
//! The primitive is deliberately small: a flag plus a condvar. Every loop that
|
||||
//! would otherwise call `thread::sleep` calls [`Shutdown::sleep`] instead, which
|
||||
//! returns early — and reports `false` — the moment [`Shutdown::stop`] is
|
||||
//! called, so teardown does not have to wait out a five-minute sweep interval to
|
||||
//! join a thread.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use parking_lot::{Condvar, Mutex};
|
||||
|
||||
/// The daemon's stop signal. Set once, never cleared: a `Core` that has been
|
||||
/// told to stop belongs to a mount that is going away.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Shutdown {
|
||||
stopping: Mutex<bool>,
|
||||
changed: Condvar,
|
||||
}
|
||||
|
||||
impl Shutdown {
|
||||
/// Whether teardown has begun. Loops check this where they would otherwise
|
||||
/// start another unit of work.
|
||||
pub(crate) fn is_stopping(&self) -> bool {
|
||||
*self.stopping.lock()
|
||||
}
|
||||
|
||||
/// Begin teardown and wake everything waiting in [`sleep`](Self::sleep).
|
||||
/// Idempotent, because more than one path can reach teardown (a signal, the
|
||||
/// kernel mount ending, a failed startup).
|
||||
pub(crate) fn stop(&self) {
|
||||
let mut stopping = self.stopping.lock();
|
||||
*stopping = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
|
||||
/// Wait up to `duration`, or until teardown starts.
|
||||
///
|
||||
/// Returns `false` if the daemon is stopping — including when it was already
|
||||
/// stopping on entry, so `while shutdown.sleep(interval)` is a complete and
|
||||
/// correct loop condition and no separate check is needed at the top.
|
||||
pub(crate) fn sleep(&self, duration: Duration) -> bool {
|
||||
let mut stopping = self.stopping.lock();
|
||||
if !*stopping {
|
||||
self.changed.wait_for(&mut stopping, duration);
|
||||
}
|
||||
!*stopping
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
#[test]
|
||||
fn sleeping_returns_early_once_stopped() {
|
||||
let shutdown = Arc::new(Shutdown::default());
|
||||
assert!(!shutdown.is_stopping());
|
||||
|
||||
let waker = shutdown.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
waker.stop();
|
||||
});
|
||||
|
||||
let started = Instant::now();
|
||||
// An hour's worth of interval, cut short by the stop.
|
||||
assert!(!shutdown.sleep(Duration::from_secs(3600)));
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(5),
|
||||
"stop must interrupt the wait, not wait it out"
|
||||
);
|
||||
assert!(shutdown.is_stopping());
|
||||
// Already stopping: the next wait does not sleep at all.
|
||||
let started = Instant::now();
|
||||
assert!(!shutdown.sleep(Duration::from_secs(3600)));
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleeping_runs_to_the_timeout_while_running() {
|
||||
let shutdown = Shutdown::default();
|
||||
assert!(shutdown.sleep(Duration::from_millis(10)));
|
||||
assert!(!shutdown.is_stopping());
|
||||
}
|
||||
}
|
||||
+267
-85
@@ -141,6 +141,11 @@ pub(crate) struct PendingRevision {
|
||||
pub(crate) meta: StagedWrite,
|
||||
}
|
||||
|
||||
/// One directory's entries as `readdir` reports them: `(ino, is_dir, name)` per
|
||||
/// entry, in reply order. The kernel's file-type enum is mapped in at reply
|
||||
/// time so this module stays free of `fuser`.
|
||||
pub(crate) type DirListing = Vec<(u64, bool, String)>;
|
||||
|
||||
/// Mutable inode bookkeeping, guarded by a mutex because fuser drives the
|
||||
/// `Filesystem` trait through `&self`.
|
||||
pub(crate) struct State {
|
||||
@@ -158,6 +163,16 @@ pub(crate) struct State {
|
||||
/// have no entry here.
|
||||
pub(crate) handles: HashMap<u64, u64>,
|
||||
pub(crate) next_fh: u64,
|
||||
/// One frozen listing per open directory handle, taken at `opendir`.
|
||||
///
|
||||
/// `readdir` resumes from a byte offset into the listing it is walking, so
|
||||
/// that listing has to stay the same across the calls of one `getdents`
|
||||
/// sequence. Rebuilding it live meant a create or a trash landing between
|
||||
/// two pages shifted every later entry, and the caller silently skipped or
|
||||
/// repeated whatever crossed the page boundary. Each entry is
|
||||
/// `(ino, is_dir, name)` — the kernel type is mapped at reply time so this
|
||||
/// module stays free of `fuser` (bugs.md B43).
|
||||
pub(crate) dir_snapshots: HashMap<u64, Arc<DirListing>>,
|
||||
/// Resident inode attributes whose effective access changed during an
|
||||
/// intern/root refresh. Core drains this set and notifies the matching
|
||||
/// kernel only after releasing the State lock.
|
||||
@@ -166,13 +181,164 @@ pub(crate) struct State {
|
||||
/// on every explicit role/tombstone change. Descendant inheritance reads
|
||||
/// this map instead of issuing one SQLite query per interned inode.
|
||||
pub(crate) share_access: HashMap<NodeUid, Access>,
|
||||
/// Unified SQLite metadata cache. Every map mutation below writes through to
|
||||
/// it inside the `State` lock so the DB stays the authoritative copy across
|
||||
/// restarts (see plan.md P1).
|
||||
/// Unified SQLite metadata cache. Read from here (`has_children`) but never
|
||||
/// written: mutations go to [`State::outbox`] and are applied after the lock
|
||||
/// is released. See [`StateGuard`].
|
||||
pub(crate) db: Arc<Db>,
|
||||
/// Write-throughs this mutation owes SQLite, applied by [`StateGuard`] once
|
||||
/// the inode lock is released.
|
||||
///
|
||||
/// The DB stays the authoritative copy across restarts, so every map
|
||||
/// mutation still writes through — it just no longer does so with the
|
||||
/// whole-mount lock held. A single listing's write-through is one
|
||||
/// transaction over up to a few thousand rows, and every other FUSE callback
|
||||
/// used to wait behind it for no reason: the maps were already updated
|
||||
/// before the first row was written.
|
||||
outbox: Vec<DbWrite>,
|
||||
}
|
||||
|
||||
/// A write-through queued by a `State` mutation, applied after the lock drops.
|
||||
///
|
||||
/// Deliberately a small closed set rather than a boxed closure: these run
|
||||
/// outside the lock, and what they can touch should be readable here.
|
||||
pub(crate) enum DbWrite {
|
||||
/// One node or a whole listing. Both go through `upsert_nodes`, which is a
|
||||
/// single transaction either way, so there is nothing to gain by splitting
|
||||
/// them — and a `Node` is large enough that a variant holding one inline
|
||||
/// would set the size of every other.
|
||||
Upsert(Vec<Node>),
|
||||
Delete(NodeUid),
|
||||
SetListed(NodeUid, bool),
|
||||
SetShareAccess(NodeUid, Access),
|
||||
}
|
||||
|
||||
impl DbWrite {
|
||||
fn apply(self, db: &Db) {
|
||||
let outcome = match &self {
|
||||
Self::Upsert(nodes) => db.upsert_nodes(nodes),
|
||||
Self::Delete(uid) => db.delete_node(uid),
|
||||
Self::SetListed(uid, listed) => db.set_listed(uid, *listed),
|
||||
Self::SetShareAccess(uid, access) => db.set_share_access(uid, *access),
|
||||
};
|
||||
if let Err(error) = outcome {
|
||||
let what = match &self {
|
||||
Self::Upsert(_) => "upsert_nodes",
|
||||
Self::Delete(_) => "delete_node",
|
||||
Self::SetListed(..) => "set_listed",
|
||||
Self::SetShareAccess(..) => "set_share_access",
|
||||
};
|
||||
warn!(%error, write = what, "db write-through failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The inode lock, held for the duration of a mutation, plus the write-throughs
|
||||
/// that mutation owed the database.
|
||||
///
|
||||
/// Dropping it releases the lock *first* and only then writes, which is the
|
||||
/// whole point: interning a five-thousand-child listing commits one transaction
|
||||
/// — several fsyncs' worth — and nothing about that needs the maps frozen. The
|
||||
/// write still happens synchronously on this thread before the caller proceeds,
|
||||
/// so a read that follows a mutation still sees it.
|
||||
pub(crate) struct StateGuard<'a> {
|
||||
guard: Option<parking_lot::MutexGuard<'a, State>>,
|
||||
db: &'a Arc<Db>,
|
||||
}
|
||||
|
||||
/// Take a mount's inode lock through a [`StateGuard`], for the loops that walk
|
||||
/// every live mount rather than this `Core`'s own.
|
||||
pub(crate) fn lock_state<'a>(
|
||||
state: &'a parking_lot::Mutex<State>,
|
||||
db: &'a Arc<Db>,
|
||||
) -> StateGuard<'a> {
|
||||
StateGuard::new(state.lock(), db)
|
||||
}
|
||||
|
||||
impl<'a> StateGuard<'a> {
|
||||
pub(crate) fn new(guard: parking_lot::MutexGuard<'a, State>, db: &'a Arc<Db>) -> Self {
|
||||
Self {
|
||||
guard: Some(guard),
|
||||
db,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for StateGuard<'_> {
|
||||
type Target = State;
|
||||
|
||||
fn deref(&self) -> &State {
|
||||
self.guard.as_ref().expect("held until drop")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::DerefMut for StateGuard<'_> {
|
||||
fn deref_mut(&mut self) -> &mut State {
|
||||
self.guard.as_mut().expect("held until drop")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StateGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
let Some(mut guard) = self.guard.take() else {
|
||||
return;
|
||||
};
|
||||
let writes = std::mem::take(&mut guard.outbox);
|
||||
// The lock goes before the writes do. That ordering is the feature.
|
||||
drop(guard);
|
||||
for write in writes {
|
||||
write.apply(self.db);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl State {
|
||||
/// A fresh, empty inode space over `db`.
|
||||
///
|
||||
/// `next_ino` differs by caller: a real mount hands out [`crate::ROOT_INO`]
|
||||
/// to its root and starts allocating above it, while a bare state under test
|
||||
/// has no root and starts at 1.
|
||||
/// Apply the write-throughs this state owes SQLite, here and now.
|
||||
///
|
||||
/// Production code never calls this: [`StateGuard`] releases the inode lock
|
||||
/// first and then applies them, which is the whole point. A test driving a
|
||||
/// bare `State` has no guard, so it calls this where the daemon would have
|
||||
/// dropped one.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn flush_outbox(&mut self) {
|
||||
let db = self.db.clone();
|
||||
for write in std::mem::take(&mut self.outbox) {
|
||||
write.apply(&db);
|
||||
}
|
||||
}
|
||||
|
||||
/// The database, with everything this state owes it already written.
|
||||
///
|
||||
/// What a test means by `st.db` is almost always "the database as of now",
|
||||
/// which in the daemon is the state after the guard dropped. Handing back an
|
||||
/// owned `Arc` rather than a borrow keeps it usable inside an `assert_eq!`
|
||||
/// that also touches `st`.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn flushed_db(&mut self) -> Arc<Db> {
|
||||
self.flush_outbox();
|
||||
self.db.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn new(db: Arc<Db>, share_access: HashMap<NodeUid, Access>, next_ino: u64) -> Self {
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
by_uid: HashMap::new(),
|
||||
children: HashMap::new(),
|
||||
next_ino,
|
||||
active_writes: HashMap::new(),
|
||||
handles: HashMap::new(),
|
||||
next_fh: 1,
|
||||
dir_snapshots: HashMap::new(),
|
||||
access_changes: HashSet::new(),
|
||||
share_access,
|
||||
db,
|
||||
outbox: Vec::new(),
|
||||
}
|
||||
}
|
||||
/// Whether `uid` names an owned node currently reachable from this mount's
|
||||
/// root. Resident entries can outlive their dentries for open-handle and
|
||||
/// access-revocation semantics, so `by_uid` membership alone is not proof
|
||||
@@ -278,11 +444,9 @@ impl State {
|
||||
|| stored.unwrap_or(Access::Viewer),
|
||||
|membership| access_for(membership.role_exact(), parent_access, true),
|
||||
);
|
||||
if let Err(error) = self.db.set_share_access(&node.uid, access) {
|
||||
warn!(uid = %node.uid, ?access, %error, "db set_share_access failed");
|
||||
} else {
|
||||
self.share_access.insert(node.uid.clone(), access);
|
||||
}
|
||||
self.outbox
|
||||
.push(DbWrite::SetShareAccess(node.uid.clone(), access));
|
||||
self.share_access.insert(node.uid.clone(), access);
|
||||
return access;
|
||||
}
|
||||
parent_access
|
||||
@@ -313,11 +477,9 @@ impl State {
|
||||
None => (None, Access::Unknown),
|
||||
};
|
||||
let access = access_for(role, parent_access, true);
|
||||
if let Err(error) = self.db.set_share_access(uid, access) {
|
||||
warn!(%uid, ?access, %error, "db set observed share access failed");
|
||||
} else {
|
||||
self.share_access.insert(uid.clone(), access);
|
||||
}
|
||||
self.outbox
|
||||
.push(DbWrite::SetShareAccess(uid.clone(), access));
|
||||
self.share_access.insert(uid.clone(), access);
|
||||
let Some(root) = root else {
|
||||
return Vec::new();
|
||||
};
|
||||
@@ -602,18 +764,14 @@ impl State {
|
||||
})
|
||||
.collect();
|
||||
for (uid, access) in discovered {
|
||||
if let Err(error) = self.db.set_share_access(&uid, access) {
|
||||
warn!(%uid, ?access, %error, "db set hydrated share access failed");
|
||||
} else {
|
||||
self.share_access.insert(uid, access);
|
||||
}
|
||||
self.outbox
|
||||
.push(DbWrite::SetShareAccess(uid.clone(), access));
|
||||
self.share_access.insert(uid, access);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn intern(&mut self, parent: u64, node: Node) -> u64 {
|
||||
if let Err(e) = self.db.upsert_node(&node) {
|
||||
warn!(uid = %node.uid, error = %e, "db upsert_node failed");
|
||||
}
|
||||
self.outbox.push(DbWrite::Upsert(vec![node.clone()]));
|
||||
self.intern_mem(parent, node)
|
||||
}
|
||||
|
||||
@@ -685,9 +843,7 @@ impl State {
|
||||
/// This is what keeps `ls` on a large folder quick: one commit for the
|
||||
/// listing, rather than one autocommit — and one fsync — per child.
|
||||
pub(crate) fn intern_batch(&mut self, parent: u64, nodes: Vec<Node>) -> Vec<u64> {
|
||||
if let Err(e) = self.db.upsert_nodes(&nodes) {
|
||||
warn!(error = %e, "db upsert_nodes failed");
|
||||
}
|
||||
self.outbox.push(DbWrite::Upsert(nodes.clone()));
|
||||
nodes
|
||||
.into_iter()
|
||||
.map(|node| self.intern_mem(parent, node))
|
||||
@@ -769,9 +925,7 @@ impl State {
|
||||
/// `(parent_ino, name)` when the node was known, so the caller can notify
|
||||
/// the kernel.
|
||||
pub(crate) fn forget(&mut self, uid: &NodeUid) -> Option<(u64, String)> {
|
||||
if let Err(e) = self.db.delete_node(uid) {
|
||||
warn!(%uid, error = %e, "db delete_node failed");
|
||||
}
|
||||
self.outbox.push(DbWrite::Delete(uid.clone()));
|
||||
self.forget_mem(uid)
|
||||
}
|
||||
|
||||
@@ -834,9 +988,7 @@ impl State {
|
||||
kids.push(ino);
|
||||
}
|
||||
}
|
||||
if let Err(e) = self.db.upsert_node(&node) {
|
||||
warn!(uid = %node.uid, error = %e, "db upsert_node failed for a queued rename");
|
||||
}
|
||||
self.outbox.push(DbWrite::Upsert(vec![node]));
|
||||
self.recompute_descendant_access(ino);
|
||||
}
|
||||
|
||||
@@ -851,9 +1003,7 @@ impl State {
|
||||
self.children.remove(&ino);
|
||||
if let Some(e) = self.entries.get(&ino) {
|
||||
let uid = e.uid.clone();
|
||||
if let Err(err) = self.db.set_listed(&uid, false) {
|
||||
warn!(%uid, error = %err, "db set_listed(false) failed");
|
||||
}
|
||||
self.outbox.push(DbWrite::SetListed(uid, false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -920,14 +1070,15 @@ impl State {
|
||||
/// file is as long as the caller was told it is. Without it a restart serves
|
||||
/// the stale size and the file reads as truncated — or empty — while its
|
||||
/// bytes sit safely in staging (offline.md Phase 3).
|
||||
pub(crate) fn record_pending_write(&mut self, ino: u64, size: u64, mtime: i64) {
|
||||
/// Returns the node to persist rather than queueing it in the outbox, and is
|
||||
/// the one mutation here whose write-through the caller must do itself:
|
||||
/// failure has to be reported, not logged. Dropping the row silently means
|
||||
/// the daemon acknowledged a write and then serves the old size for it, so
|
||||
/// the caller has to be able to turn it into a failed `release`.
|
||||
pub(crate) fn record_pending_write(&mut self, ino: u64, size: u64, mtime: i64) -> Option<Node> {
|
||||
self.set_size(ino, size);
|
||||
self.touch_mtime(ino, mtime);
|
||||
if let Some(e) = self.entries.get(&ino)
|
||||
&& let Err(err) = self.db.upsert_node(&e.node)
|
||||
{
|
||||
warn!(uid = %e.uid, error = %err, "db upsert_node failed for a queued write");
|
||||
}
|
||||
self.entries.get(&ino).map(|e| e.node.clone())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1008,18 +1159,7 @@ mod tests {
|
||||
let dir = TempDir::new();
|
||||
let db = Db::open(&dir.0.join("cache.db")).unwrap();
|
||||
let share_access = db.all_share_access().unwrap();
|
||||
let st = State {
|
||||
entries: HashMap::new(),
|
||||
by_uid: HashMap::new(),
|
||||
children: HashMap::new(),
|
||||
next_ino: 1,
|
||||
active_writes: HashMap::new(),
|
||||
handles: HashMap::new(),
|
||||
next_fh: 1,
|
||||
access_changes: HashSet::new(),
|
||||
share_access,
|
||||
db: Arc::new(db),
|
||||
};
|
||||
let st = State::new(Arc::new(db), share_access, 1);
|
||||
(st, dir)
|
||||
}
|
||||
|
||||
@@ -1032,8 +1172,8 @@ mod tests {
|
||||
let f = st.intern(src, node("f", "src", "f.txt", false));
|
||||
st.children.insert(src, vec![f]);
|
||||
st.children.insert(dst, vec![]);
|
||||
st.db.set_listed(&uid("src"), true).unwrap();
|
||||
st.db.set_listed(&uid("dst"), true).unwrap();
|
||||
st.flushed_db().set_listed(&uid("src"), true).unwrap();
|
||||
st.flushed_db().set_listed(&uid("dst"), true).unwrap();
|
||||
(st, dir, src, dst)
|
||||
}
|
||||
|
||||
@@ -1047,11 +1187,17 @@ mod tests {
|
||||
st.relocate(f, src, dst, &uid("dst"), "f.txt");
|
||||
|
||||
assert!(
|
||||
st.db.children_if_listed(&uid("dst")).unwrap().is_none(),
|
||||
st.flushed_db()
|
||||
.children_if_listed(&uid("dst"))
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"destination re-enumerates: it gained a child whose row was just deleted"
|
||||
);
|
||||
assert!(
|
||||
st.db.children_if_listed(&uid("src")).unwrap().is_none(),
|
||||
st.flushed_db()
|
||||
.children_if_listed(&uid("src"))
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"source re-enumerates: its listing predates the move, and a stale \
|
||||
`listed` flag would hide every later remote change under it"
|
||||
);
|
||||
@@ -1074,12 +1220,41 @@ mod tests {
|
||||
let (mut st, _dir, src, _dst) = two_folders();
|
||||
let f = st.by_uid[&uid("f")];
|
||||
st.relocate(f, src, src, &uid("src"), "renamed.txt");
|
||||
assert!(st.db.children_if_listed(&uid("src")).unwrap().is_none());
|
||||
assert!(
|
||||
st.flushed_db()
|
||||
.children_if_listed(&uid("src"))
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(!st.children.contains_key(&src));
|
||||
assert_eq!(st.by_uid.get(&uid("f")), Some(&f), "the inode is stable");
|
||||
assert_eq!(st.entries[&f].node.name, "renamed.txt");
|
||||
}
|
||||
|
||||
/// DB1: a mutation's write-through must not happen with the inode lock
|
||||
/// held. Interning a listing commits a transaction over every child, and
|
||||
/// every other FUSE callback used to wait behind that for no reason — the
|
||||
/// maps it needs were updated before the first row was written.
|
||||
#[test]
|
||||
fn a_mutation_writes_through_only_after_the_lock_is_released() {
|
||||
let (st, _dir) = state();
|
||||
let db = st.db.clone();
|
||||
let root = uid("root").to_string();
|
||||
let state = parking_lot::Mutex::new(st);
|
||||
{
|
||||
let mut guard = lock_state(&state, &db);
|
||||
guard.intern(0, node("root", "none", "My Files", true));
|
||||
assert!(
|
||||
db.node_by_uid(&root).unwrap().is_none(),
|
||||
"the row must not be written while the lock is held"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
db.node_by_uid(&root).unwrap().is_some(),
|
||||
"…and must be written by the time the guard is gone"
|
||||
);
|
||||
}
|
||||
|
||||
/// The failure mode the fix exists for, stated directly: forgetting the node
|
||||
/// prunes it from the resident listing but leaves the DB claiming the source
|
||||
/// is fully enumerated. This pins *why* `relocate` cannot just call `forget`.
|
||||
@@ -1087,7 +1262,7 @@ mod tests {
|
||||
fn forget_alone_leaves_the_source_claiming_a_complete_listing() {
|
||||
let (mut st, _dir, _src, _dst) = two_folders();
|
||||
st.forget(&uid("f"));
|
||||
let listed = st.db.children_if_listed(&uid("src")).unwrap();
|
||||
let listed = st.flushed_db().children_if_listed(&uid("src")).unwrap();
|
||||
assert!(
|
||||
listed.is_some(),
|
||||
"forget does not clear the flag — which is exactly why relocate must"
|
||||
@@ -1136,7 +1311,7 @@ mod tests {
|
||||
assert!(!st.has_children(folder), "cleared memory children");
|
||||
|
||||
// DB children
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.upsert_node(&node("file2_uid", "dir_uid", "file2.txt", false))
|
||||
.unwrap();
|
||||
assert!(
|
||||
@@ -1162,7 +1337,7 @@ mod tests {
|
||||
assert_eq!(st.entries[&shared_ino].access, Access::Editor);
|
||||
assert_eq!(st.entries[&child].access, Access::Editor);
|
||||
assert_eq!(
|
||||
st.db.share_access(&uid("shared")).unwrap(),
|
||||
st.flushed_db().share_access(&uid("shared")).unwrap(),
|
||||
Some(Access::Editor)
|
||||
);
|
||||
}
|
||||
@@ -1174,8 +1349,8 @@ mod tests {
|
||||
let virtual_uid = NodeUid::new(VolumeId::from("virtual"), LinkId::from("sharedwithme"));
|
||||
let mut virtual_root = node("placeholder", "root", "Shared with me", true);
|
||||
virtual_root.uid = virtual_uid.clone();
|
||||
st.db.upsert_node(&virtual_root).unwrap();
|
||||
st.db
|
||||
st.flushed_db().upsert_node(&virtual_root).unwrap();
|
||||
st.flushed_db()
|
||||
.set_share_access(&virtual_uid, Access::Viewer)
|
||||
.unwrap();
|
||||
let virtual_ino = st.intern_published_share_root(owned_root, virtual_root, Access::Viewer);
|
||||
@@ -1183,15 +1358,17 @@ mod tests {
|
||||
let mut shared = node("shared", "placeholder", "Retained", true);
|
||||
shared.parent_uid = Some(virtual_uid.clone());
|
||||
shared.membership = Some(membership(6));
|
||||
st.db.upsert_node(&shared).unwrap();
|
||||
st.db.set_share_access(&shared.uid, Access::Editor).unwrap();
|
||||
st.flushed_db().upsert_node(&shared).unwrap();
|
||||
st.flushed_db()
|
||||
.set_share_access(&shared.uid, Access::Editor)
|
||||
.unwrap();
|
||||
let shared_ino =
|
||||
st.intern_published_share_root(virtual_ino, shared.clone(), Access::Editor);
|
||||
let child = st.intern(
|
||||
shared_ino,
|
||||
node("shared-child", "shared", "pending.txt", false),
|
||||
);
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.enqueue_op(&pdfs_core::db::PendingOp {
|
||||
id: 0,
|
||||
kind: pdfs_core::db::OP_RENAME.to_string(),
|
||||
@@ -1209,20 +1386,20 @@ mod tests {
|
||||
assert_eq!(st.entries[&shared_ino].access, Access::Editor);
|
||||
assert_eq!(st.entries[&child].access, Access::Editor);
|
||||
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.publish_shared_roots(&virtual_uid, std::slice::from_ref(&shared.uid), &[])
|
||||
.unwrap();
|
||||
let snapshot = st.db.visible_children(&virtual_uid).unwrap();
|
||||
let snapshot = st.flushed_db().visible_children(&virtual_uid).unwrap();
|
||||
assert_eq!(snapshot.len(), 1);
|
||||
st.intern_published_share_root(virtual_ino, snapshot[0].clone(), Access::Viewer);
|
||||
|
||||
assert_eq!(
|
||||
st.db.share_access(&shared.uid).unwrap(),
|
||||
st.flushed_db().share_access(&shared.uid).unwrap(),
|
||||
Some(Access::Viewer)
|
||||
);
|
||||
assert_eq!(st.entries[&shared_ino].access, Access::Viewer);
|
||||
assert_eq!(st.entries[&child].access, Access::Viewer);
|
||||
assert_eq!(st.db.pending_ops().unwrap().len(), 1);
|
||||
assert_eq!(st.flushed_db().pending_ops().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1233,12 +1410,12 @@ mod tests {
|
||||
let child_node = node("offline-child", "offline-share", "nested folder", true);
|
||||
let leaf_node = node("offline-leaf", "offline-child", "child.txt", false);
|
||||
for node in [&root_node, &shared_node, &child_node, &leaf_node] {
|
||||
st.db.upsert_node(node).unwrap();
|
||||
st.flushed_db().upsert_node(node).unwrap();
|
||||
}
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.set_share_access(&uid("offline-share"), Access::Viewer)
|
||||
.unwrap();
|
||||
st.share_access = st.db.all_share_access().unwrap();
|
||||
st.share_access = st.flushed_db().all_share_access().unwrap();
|
||||
|
||||
// Reconstruct in deliberately non-topological order, matching the
|
||||
// database hydration path that first allocates every inode and only
|
||||
@@ -1296,7 +1473,7 @@ mod tests {
|
||||
/// `ORPHAN_INO`. `nodes[0]` becomes the mount root.
|
||||
fn hydrate_from(st: &mut State, nodes: &[Node]) -> Vec<u64> {
|
||||
for node in nodes {
|
||||
st.db.upsert_node(node).unwrap();
|
||||
st.flushed_db().upsert_node(node).unwrap();
|
||||
}
|
||||
let inos: Vec<u64> = nodes
|
||||
.iter()
|
||||
@@ -1394,10 +1571,10 @@ mod tests {
|
||||
#[test]
|
||||
fn persisted_authority_outranks_the_own_volume_fail_open() {
|
||||
let (mut st, _dir) = state();
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.set_share_access(&uid("device-root"), Access::Viewer)
|
||||
.unwrap();
|
||||
st.share_access = st.db.all_share_access().unwrap();
|
||||
st.share_access = st.flushed_db().all_share_access().unwrap();
|
||||
let nodes = [
|
||||
node("root", "none", "My Files", true),
|
||||
node("documents", "device-root", "Documents", true),
|
||||
@@ -1431,7 +1608,7 @@ mod tests {
|
||||
assert_eq!(st.require_writable(ino).unwrap_err().code(), libc::EACCES);
|
||||
}
|
||||
assert_eq!(
|
||||
st.db.share_access(&uid("shared")).unwrap(),
|
||||
st.flushed_db().share_access(&uid("shared")).unwrap(),
|
||||
Some(Access::Viewer)
|
||||
);
|
||||
}
|
||||
@@ -1441,8 +1618,10 @@ mod tests {
|
||||
for persisted in [Access::Viewer, Access::Editor] {
|
||||
let (mut st, _dir) = state();
|
||||
let owned_root = st.intern(0, node("root", "none", "My Files", true));
|
||||
st.db.set_share_access(&uid("shared"), persisted).unwrap();
|
||||
st.share_access = st.db.all_share_access().unwrap();
|
||||
st.flushed_db()
|
||||
.set_share_access(&uid("shared"), persisted)
|
||||
.unwrap();
|
||||
st.share_access = st.flushed_db().all_share_access().unwrap();
|
||||
let shared = st.intern(
|
||||
owned_root,
|
||||
node("shared", "root", "Shared without membership", true),
|
||||
@@ -1454,7 +1633,10 @@ mod tests {
|
||||
|
||||
assert_eq!(st.entries[&shared].access, persisted);
|
||||
assert_eq!(st.entries[&child].access, persisted);
|
||||
assert_eq!(st.db.share_access(&uid("shared")).unwrap(), Some(persisted));
|
||||
assert_eq!(
|
||||
st.flushed_db().share_access(&uid("shared")).unwrap(),
|
||||
Some(persisted)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1462,7 +1644,7 @@ mod tests {
|
||||
fn explicit_missing_share_role_fails_closed_and_marks_kernel_attrs() {
|
||||
let (mut st, _dir) = state();
|
||||
let owned_root = st.intern(0, node("root", "none", "My Files", true));
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.set_share_access(&uid("shared"), Access::Editor)
|
||||
.unwrap();
|
||||
let shared = st.intern(owned_root, node("shared", "root", "Shared", true));
|
||||
@@ -1474,7 +1656,7 @@ mod tests {
|
||||
assert_eq!(st.entries[&shared].access, Access::Viewer);
|
||||
assert_eq!(st.entries[&child].access, Access::Viewer);
|
||||
assert_eq!(
|
||||
st.db.share_access(&uid("shared")).unwrap(),
|
||||
st.flushed_db().share_access(&uid("shared")).unwrap(),
|
||||
Some(Access::Viewer)
|
||||
);
|
||||
assert!(changed.contains(&shared));
|
||||
@@ -1491,7 +1673,7 @@ mod tests {
|
||||
let virtual_uid = NodeUid::new(VolumeId::from("virtual"), LinkId::from("sharedwithme"));
|
||||
let mut virtual_node = node("sharedwithme", "root", "Shared with me", true);
|
||||
virtual_node.uid = virtual_uid.clone();
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.set_share_access(&virtual_uid, Access::Viewer)
|
||||
.unwrap();
|
||||
st.share_access.insert(virtual_uid.clone(), Access::Viewer);
|
||||
@@ -1558,7 +1740,7 @@ mod tests {
|
||||
let child = st.intern(shared, node("child", "shared", "folder", true));
|
||||
let leaf = st.intern(child, node("leaf", "child", "leaf.txt", false));
|
||||
|
||||
st.db
|
||||
st.flushed_db()
|
||||
.set_share_access(&uid("shared"), Access::Viewer)
|
||||
.unwrap();
|
||||
let changed = st.downgrade_shared_subtree(&uid("shared"));
|
||||
@@ -1572,7 +1754,7 @@ mod tests {
|
||||
assert_eq!(st.entries[&leaf].access, Access::Viewer);
|
||||
assert_eq!(st.require_writable(leaf).unwrap_err().code(), libc::EACCES);
|
||||
assert_eq!(
|
||||
st.db.effective_node_access(&uid("leaf")).unwrap(),
|
||||
st.flushed_db().effective_node_access(&uid("leaf")).unwrap(),
|
||||
Some(Access::Viewer)
|
||||
);
|
||||
|
||||
@@ -1628,7 +1810,7 @@ mod tests {
|
||||
let shared = st.intern(owned_root, shared_node);
|
||||
let shared_child = st.intern(shared, node("child", "shared", "child.txt", false));
|
||||
|
||||
st.db.downgrade_all_share_access().unwrap();
|
||||
st.flushed_db().downgrade_all_share_access().unwrap();
|
||||
let changed = st.downgrade_known_shared_access();
|
||||
|
||||
assert!(changed.contains(&shared));
|
||||
@@ -1652,7 +1834,7 @@ mod tests {
|
||||
let parent_link = st.entries[&parent].uid.link_id.to_string();
|
||||
parent = st.intern(parent, node(&link, &parent_link, &link, true));
|
||||
}
|
||||
st.db.downgrade_all_share_access().unwrap();
|
||||
st.flushed_db().downgrade_all_share_access().unwrap();
|
||||
|
||||
let changed = st.downgrade_known_shared_access();
|
||||
|
||||
|
||||
@@ -67,10 +67,16 @@ impl Core {
|
||||
/// Sweep for stale conflict copies forever, on its own thread. See the module
|
||||
/// docs for the identical/divergent policy.
|
||||
pub(crate) fn run_conflict_sweep_loop(&self) {
|
||||
std::thread::sleep(CONFLICT_SWEEP_WARMUP);
|
||||
// Both waits are interruptible, so teardown does not have to sit out a
|
||||
// five-minute interval to join this thread (bugs.md B44).
|
||||
if !self.shutdown.sleep(CONFLICT_SWEEP_WARMUP) {
|
||||
return;
|
||||
}
|
||||
loop {
|
||||
self.sweep_conflicts_once();
|
||||
std::thread::sleep(CONFLICT_SWEEP_INTERVAL);
|
||||
if !self.shutdown.sleep(CONFLICT_SWEEP_INTERVAL) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,9 +144,9 @@ impl Core {
|
||||
/// enumerated every node up front and has been doing network I/O since. Before
|
||||
/// touching anything remote this re-reads the node and refuses to act unless
|
||||
/// it is *still* the same revision, still untrashed, still unwritten and still
|
||||
/// unopened. Without that, a user edit landing inside the window is destroyed
|
||||
/// — `discard_queued_ops` below drops the queued write *and its staged blob*,
|
||||
/// which may hold the only copy of those bytes (`docs/BUGS.md` B71).
|
||||
/// unopened (`docs/BUGS.md` B71). Nothing on this path discards a queued write
|
||||
/// or its staged blob: a write that arrives after that check is left for the
|
||||
/// drain to resolve against the now-trashed node.
|
||||
fn remove_conflict_copy(&self, snapshot: &proton_drive_rs::Node, base: &str) {
|
||||
let uid = &snapshot.uid;
|
||||
let name = snapshot.name.as_str();
|
||||
@@ -166,10 +172,13 @@ impl Core {
|
||||
self.for_each_state(|st| {
|
||||
st.forget_or_unlink(uid);
|
||||
});
|
||||
if let Err(error) = self.discard_queued_ops(uid) {
|
||||
warn!(%uid, ?error, "conflict sweep: queued-op cleanup failed");
|
||||
return;
|
||||
}
|
||||
// Deliberately *not* `discard_queued_ops`: `duplicate_still_removable`
|
||||
// proved the queue empty before the trash above, but that proof is stale
|
||||
// the moment `trash_nodes` goes to the network. A `close(2)` landing in
|
||||
// that window stages bytes, and discarding here would unlink the staged
|
||||
// blob holding the only copy of them. Leaving the op queued is the
|
||||
// non-destructive outcome: the drain re-reads the node, sees it trashed,
|
||||
// and `revision_conflict` routes it to `keep_as_conflict_copy`.
|
||||
self.cache.evict(uid);
|
||||
self.evict_reader(uid);
|
||||
if let Err(e) = self.db.delete_node(uid) {
|
||||
@@ -194,8 +203,8 @@ impl Core {
|
||||
let uid = &snapshot.uid;
|
||||
let name = snapshot.name.as_str();
|
||||
|
||||
// A queued op means bytes are still owed an upload. `discard_queued_ops`
|
||||
// would throw those away along with the staged blob holding them.
|
||||
// A queued op means bytes are still owed an upload; trashing the node
|
||||
// under them turns an ordinary upload into a conflict fork.
|
||||
match self.db.has_any_op(&uid.to_string()) {
|
||||
Ok(false) => {}
|
||||
Ok(true) => {
|
||||
|
||||
+585
-77
@@ -54,7 +54,7 @@ use engine::{classify, settle_with};
|
||||
#[cfg(test)]
|
||||
use local_scan::is_write_mode;
|
||||
use local_scan::open_for_write_set;
|
||||
use model::{LocalItem, Outcome, PassAbort, Pending, RemoteItem};
|
||||
use model::{LocalItem, LocalSig, Outcome, PassAbort, Pending, RemoteItem};
|
||||
pub(crate) use planner::base_name;
|
||||
#[cfg(test)]
|
||||
use planner::conflict_path;
|
||||
@@ -525,6 +525,9 @@ impl Core {
|
||||
continue;
|
||||
}
|
||||
trashed.push(rel.clone());
|
||||
if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -687,6 +690,7 @@ impl Core {
|
||||
rel_path: rel.clone(),
|
||||
remote_uid: Some(remote.uid.to_string()),
|
||||
local_mtime: local.mtime,
|
||||
local_mtime_ns: local.mtime_ns,
|
||||
local_size: local.size,
|
||||
remote_rev: Some(remote.mtime.to_string()),
|
||||
remote_hash: Some(remote.size.to_string()),
|
||||
@@ -717,6 +721,9 @@ impl Core {
|
||||
uid: remote.uid.clone(),
|
||||
mtime: remote.mtime,
|
||||
size: remote.size,
|
||||
// What the walk saw here, so the apply step can tell a
|
||||
// stale plan from a live one before it overwrites.
|
||||
local: l.map(LocalSig::from),
|
||||
});
|
||||
}
|
||||
FilePlan::Conflict => {
|
||||
@@ -739,7 +746,9 @@ impl Core {
|
||||
}
|
||||
},
|
||||
FilePlan::DeleteLocal => {
|
||||
if let Err(e) = std::fs::remove_file(local_root.join(rel_to_path(rel))) {
|
||||
if let Err(e) =
|
||||
removed_locally(std::fs::remove_file(local_root.join(rel_to_path(rel))))
|
||||
{
|
||||
warn!(rel, error = %e, "sync: remove local file failed");
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -771,6 +780,9 @@ impl Core {
|
||||
outcome.errors += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -797,7 +809,9 @@ impl Core {
|
||||
// Deferred folder deletions, deepest first.
|
||||
delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count()));
|
||||
for rel in delete_local_dirs {
|
||||
if let Err(e) = std::fs::remove_dir_all(local_root.join(rel_to_path(&rel))) {
|
||||
if let Err(e) =
|
||||
removed_locally(std::fs::remove_dir_all(local_root.join(rel_to_path(&rel))))
|
||||
{
|
||||
warn!(rel, error = %e, "sync: remove local folder failed");
|
||||
self.log_activity(ActivityKind::Trash, base_name(&rel), e.to_string(), false);
|
||||
outcome.errors += 1;
|
||||
@@ -824,6 +838,9 @@ impl Core {
|
||||
outcome.errors += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = parent_uid_of(&rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, &rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -849,68 +866,9 @@ impl Core {
|
||||
writing: &HashSet<PathBuf>,
|
||||
out: &mut HashMap<String, LocalItem>,
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||
format!(
|
||||
"local path {} escaped sync root {}: {e}",
|
||||
path.display(),
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let rel = stripped.to_str().ok_or_else(|| {
|
||||
format!(
|
||||
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
// Ignore our own in-flight download temp files.
|
||||
if rel.contains(".pdfs-tmp-") {
|
||||
continue;
|
||||
}
|
||||
// Ignored paths are dropped here as well as from the classification
|
||||
// union — not for correctness (the union filter is what makes this
|
||||
// safe) but so an ignored `node_modules/` is never descended into or
|
||||
// counted against scan progress.
|
||||
if rules.is_ignored(rel, meta.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
if meta.is_dir() {
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
open_for_write: false,
|
||||
},
|
||||
);
|
||||
self.progress_scanned(folder_id, base_name(rel));
|
||||
self.walk_local(folder_id, root, &path, rules, writing, out)?;
|
||||
} else if meta.is_file() {
|
||||
// Check whether any process holds this file open for writing.
|
||||
// The canonical path is used because /proc/*/fd links resolve
|
||||
// to canonical targets.
|
||||
let ofw = std::fs::canonicalize(&path)
|
||||
.map(|canon| writing.contains(&canon))
|
||||
.unwrap_or(false);
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: false,
|
||||
mtime: system_mtime(&meta),
|
||||
size: meta.len() as i64,
|
||||
open_for_write: ofw,
|
||||
},
|
||||
);
|
||||
self.progress_scanned(folder_id, base_name(rel));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
walk_local_tree(root, dir, rules, writing, out, &mut |rel| {
|
||||
self.progress_scanned(folder_id, base_name(rel))
|
||||
})
|
||||
}
|
||||
|
||||
/// Map every remote folder under `folder` to its uid by rel path, ignoring
|
||||
@@ -1069,6 +1027,7 @@ impl Core {
|
||||
rel_path: rel.to_string(),
|
||||
remote_uid: Some(uid.to_string()),
|
||||
local_mtime: 0,
|
||||
local_mtime_ns: None,
|
||||
local_size: 0,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
@@ -1175,6 +1134,9 @@ impl Core {
|
||||
.create_folder(parent, base_name(rel), Some(now_secs()))
|
||||
.await
|
||||
.map_err(|e| format!("create remote folder {rel}: {e}"))?;
|
||||
// The parent has gained a child that no cached listing of it
|
||||
// knows about (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
self.baseline_dir(folder_id, rel, &uid)?;
|
||||
Ok(Applied::Dir(rel.clone(), uid))
|
||||
}
|
||||
@@ -1192,8 +1154,9 @@ impl Core {
|
||||
uid,
|
||||
mtime,
|
||||
size,
|
||||
local,
|
||||
} => {
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size)
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size, *local)
|
||||
.await?;
|
||||
Ok(Applied::Downloaded)
|
||||
}
|
||||
@@ -1211,7 +1174,11 @@ impl Core {
|
||||
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
||||
info!(rel, "sync: kept local changes as a conflict copy");
|
||||
}
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size)
|
||||
// The local copy has just been moved aside, so the state this
|
||||
// download expects to find at the path is "nothing" — anything
|
||||
// that appears while it runs is a *new* racing write, and gets
|
||||
// its own conflict copy rather than being overwritten (B39).
|
||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size, None)
|
||||
.await?;
|
||||
Ok(Applied::Conflict)
|
||||
}
|
||||
@@ -1250,6 +1217,9 @@ impl Core {
|
||||
let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?;
|
||||
let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?;
|
||||
let mtime = system_mtime(&meta);
|
||||
// Taken from the descriptor being streamed, so it names the content that
|
||||
// actually goes up rather than whatever is at the path when we finish.
|
||||
let streamed = LocalSig::from(&meta);
|
||||
// Count the bytes as they are read, so this upload shows up in
|
||||
// `GetQueueStatus` next to a manual one. The uid isn't known until the
|
||||
// draft is sealed, so the transfer registers without one.
|
||||
@@ -1276,7 +1246,13 @@ impl Core {
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("upload {rel}: {e}"))?;
|
||||
self.record_file_baseline(folder_id, rel, &path, &uid).await
|
||||
// The parent has gained a child that no cached listing of it knows
|
||||
// about. Without this, a folder later switched to on-demand serves a
|
||||
// snapshot that predates this upload (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), &uid)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Upload a changed local file as a new revision of an existing remote node.
|
||||
@@ -1291,6 +1267,7 @@ impl Core {
|
||||
let file = std::fs::File::open(&path).map_err(|e| format!("open {rel}: {e}"))?;
|
||||
let meta = file.metadata().map_err(|e| format!("stat {rel}: {e}"))?;
|
||||
let mtime = system_mtime(&meta);
|
||||
let streamed = LocalSig::from(&meta);
|
||||
let reader = OwnedCountingReader::new(
|
||||
file,
|
||||
self.transfers.begin(
|
||||
@@ -1304,11 +1281,20 @@ impl Core {
|
||||
.upload_new_revision_from(uid, reader, meta.len() as i64, Vec::new(), Some(mtime))
|
||||
.await
|
||||
.map_err(|e| format!("upload revision {rel}: {e}"))?;
|
||||
self.record_file_baseline(folder_id, rel, &path, uid).await
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), uid)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Download a remote file to its local path (atomically via a temp file),
|
||||
/// stamp the local mtime to match the remote, then record baseline.
|
||||
///
|
||||
/// `expected_local` is the identity planning saw at the destination. The
|
||||
/// rename below is the destructive step, and the download in between can run
|
||||
/// for minutes, so the destination is re-examined right before it: a local
|
||||
/// file that no longer matches the plan is set aside as a conflict copy
|
||||
/// instead of being replaced (bugs.md B39).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn download_file(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
@@ -1317,6 +1303,7 @@ impl Core {
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
expected_local: Option<LocalSig>,
|
||||
) -> Result<(), String> {
|
||||
let path = local_root.join(rel_to_path(rel));
|
||||
if let Some(parent) = path.parent() {
|
||||
@@ -1351,6 +1338,13 @@ impl Core {
|
||||
out.flush().map_err(|e| format!("flush tmp {rel}: {e}"))?;
|
||||
out.sync_all().map_err(|e| format!("sync tmp {rel}: {e}"))?;
|
||||
}
|
||||
// The last moment at which the destination can still be saved. Anything
|
||||
// other than the state planning classified means a writer got there
|
||||
// first, and its bytes are the ones nothing else has a copy of.
|
||||
if let Err(e) = keep_racing_local_edit(&path, expected_local) {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(format!("set aside racing local edit for {rel}: {e}"));
|
||||
}
|
||||
std::fs::rename(&tmp, &path).map_err(|e| format!("place {rel}: {e}"))?;
|
||||
// Match local mtime to the remote's so neither side looks "changed" next pass.
|
||||
let f = std::fs::File::options()
|
||||
@@ -1366,23 +1360,36 @@ impl Core {
|
||||
.and_then(|dir| dir.sync_all())
|
||||
.map_err(|e| format!("sync directory for {rel}: {e}"))?;
|
||||
}
|
||||
self.record_file_baseline(folder_id, rel, &path, uid).await
|
||||
// Here the file on disk *is* the transferred content: it was published
|
||||
// by a rename and stamped just above, so its stat is the identity.
|
||||
self.record_file_baseline(folder_id, rel, local_sig(&path), uid)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Refresh a file's baseline from ground truth: the local stat and the
|
||||
/// remote node's reported signature. Called after every upload/download so
|
||||
/// the very next reconcile sees the path as unchanged.
|
||||
/// Refresh a file's baseline from ground truth: the local identity these
|
||||
/// bytes came from and the remote node's reported signature. Called after
|
||||
/// every upload/download so the very next reconcile sees the path as
|
||||
/// unchanged.
|
||||
///
|
||||
/// `local` is the identity the *transferred content* corresponds to, which
|
||||
/// is not always the file's current stat. An upload streams the live file,
|
||||
/// so it passes the identity it read the first byte at: if a writer changed
|
||||
/// the file mid-stream, the remote now holds a torn revision, and recording
|
||||
/// the pre-upload identity is what makes the next pass see the path as
|
||||
/// locally changed and upload a whole, clean revision over it. Recording the
|
||||
/// post-upload stat instead declared the torn revision synchronized and left
|
||||
/// it as the file's content on Drive (bugs.md B40).
|
||||
async fn record_file_baseline(
|
||||
&self,
|
||||
folder_id: i64,
|
||||
rel: &str,
|
||||
local_path: &Path,
|
||||
local: Option<LocalSig>,
|
||||
uid: &NodeUid,
|
||||
) -> Result<(), String> {
|
||||
let (lmtime, lsize) = match std::fs::metadata(local_path) {
|
||||
Ok(m) => (system_mtime(&m), m.len() as i64),
|
||||
Err(e) => return Err(format!("stat {rel}: {e}")),
|
||||
let Some(local) = local else {
|
||||
return Err(format!("stat {rel}: no local identity to baseline"));
|
||||
};
|
||||
let (lmtime, lsize) = (local.mtime, local.size);
|
||||
// Re-fetch the node so the stored remote signature is exactly what a walk
|
||||
// will report next time.
|
||||
let (rmtime, rsize) = match self.client.enumerate_nodes(std::slice::from_ref(uid)).await {
|
||||
@@ -1402,6 +1409,7 @@ impl Core {
|
||||
rel_path: rel.to_string(),
|
||||
remote_uid: Some(uid.to_string()),
|
||||
local_mtime: lmtime,
|
||||
local_mtime_ns: local.mtime_ns,
|
||||
local_size: lsize,
|
||||
remote_rev: Some(rmtime.to_string()),
|
||||
remote_hash: Some(rsize.to_string()),
|
||||
@@ -1411,6 +1419,177 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Say so when a file changed under an upload that was streaming it.
|
||||
///
|
||||
/// Purely diagnostic — the correction is already in place, because the baseline
|
||||
/// records the identity the upload *read* rather than the one on disk now, so
|
||||
/// the next pass classifies the path as locally changed and replaces the torn
|
||||
/// revision. Without the log there is nothing in the journal to explain the
|
||||
/// extra revision (bugs.md B40).
|
||||
fn warn_if_torn(rel: &str, path: &Path, streamed: LocalSig) {
|
||||
let now = local_sig(path);
|
||||
if !now.is_some_and(|now| now.same_content(&streamed)) {
|
||||
warn!(
|
||||
rel,
|
||||
?streamed,
|
||||
?now,
|
||||
"sync: file changed while uploading; the remote revision may be torn \
|
||||
and will be replaced on the next pass"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The identity a reconcile pass uses to recognise a local file it has already
|
||||
/// looked at: `None` when nothing is at the path.
|
||||
fn local_sig(path: &Path) -> Option<LocalSig> {
|
||||
std::fs::metadata(path).ok().map(|m| LocalSig {
|
||||
mtime: system_mtime(&m),
|
||||
mtime_ns: system_mtime_ns(&m),
|
||||
size: m.len() as i64,
|
||||
})
|
||||
}
|
||||
|
||||
/// Guard the destination of a download: if what is at `path` is no longer what
|
||||
/// planning classified, move it aside as a conflict copy before the caller
|
||||
/// renames over it.
|
||||
///
|
||||
/// A reconcile pass decides what to do from a walk that finished before the
|
||||
/// transfer started, and a download of a large file can run for minutes. Without
|
||||
/// this, a local edit completed inside that window was replaced by the remote
|
||||
/// version with no copy of it kept anywhere — the one shape of loss a sync
|
||||
/// engine must not have (bugs.md B39). Keeping the copy makes it re-upload as a
|
||||
/// new file on the next pass, which is the same resolution a both-sides-changed
|
||||
/// conflict already gets.
|
||||
///
|
||||
/// A path that is empty when it was expected to be is not a conflict: the
|
||||
/// download is about to recreate it, and there is nothing to preserve.
|
||||
fn keep_racing_local_edit(path: &Path, expected: Option<LocalSig>) -> std::io::Result<()> {
|
||||
let actual = local_sig(path);
|
||||
let unchanged = match (actual, expected) {
|
||||
(Some(a), Some(e)) => a.same_content(&e),
|
||||
// Nothing there: the download is about to create it, and there is
|
||||
// nothing to preserve either way.
|
||||
(None, _) => true,
|
||||
(Some(_), None) => false,
|
||||
};
|
||||
if unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
warn!(
|
||||
path = %path.display(),
|
||||
?expected,
|
||||
?actual,
|
||||
"sync: local file changed while downloading; keeping it as a conflict copy"
|
||||
);
|
||||
preserve_conflict_copy(path, now_secs()).map(|_| ())
|
||||
}
|
||||
|
||||
/// Read a local removal as the sync engine has to read it: the *state* it was
|
||||
/// asking for is "this path is gone", so a path that was already gone is a
|
||||
/// success, not a failure.
|
||||
///
|
||||
/// It matters because the baseline is only dropped on success. Treating
|
||||
/// `NotFound` as an error left the baseline in place for a file that no longer
|
||||
/// exists, so the same removal was re-attempted and re-reported as an error on
|
||||
/// every pass, forever. Every other error is still an error, and keeping the
|
||||
/// baseline is exactly right there: the surviving file must not be mistaken for
|
||||
/// a new local creation on the next pass and uploaded back (bugs.md B42).
|
||||
fn removed_locally(result: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match result {
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// The body of [`Core::walk_local`], without a `Core`: recursively collect the
|
||||
/// local tree under `dir` into `out`, reporting each entry to `scanned`.
|
||||
///
|
||||
/// **Every failure here is fatal to the pass, deliberately.** A `read_dir` that
|
||||
/// fails, an entry that cannot be stat-ed, a name that is not UTF-8 — each of
|
||||
/// them would, if skipped, leave `out` describing a *smaller* tree than the one
|
||||
/// on disk. Reconciliation reads a path present in the baseline and absent from
|
||||
/// the scan as "the user deleted this", and propagates that to Drive. So an
|
||||
/// unreadable subdirectory would trash its remote counterpart, and the deeper
|
||||
/// the failure, the more it takes with it (`docs/BUGS.md` B55). Refusing the
|
||||
/// whole pass keeps the previous baseline and leaves the remote alone; the next
|
||||
/// pass retries once the transient condition clears.
|
||||
///
|
||||
/// Split out from the method so exactly that can be tested against a real tree
|
||||
/// with a real unreadable directory in it, which needs no `Core` at all.
|
||||
fn walk_local_tree(
|
||||
root: &Path,
|
||||
dir: &Path,
|
||||
rules: &IgnoreRules,
|
||||
writing: &HashSet<PathBuf>,
|
||||
out: &mut HashMap<String, LocalItem>,
|
||||
scanned: &mut dyn FnMut(&str),
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||
format!(
|
||||
"local path {} escaped sync root {}: {e}",
|
||||
path.display(),
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let rel = stripped.to_str().ok_or_else(|| {
|
||||
format!(
|
||||
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
// Ignore our own in-flight download temp files.
|
||||
if rel.contains(".pdfs-tmp-") {
|
||||
continue;
|
||||
}
|
||||
// Ignored paths are dropped here as well as from the classification
|
||||
// union — not for correctness (the union filter is what makes this
|
||||
// safe) but so an ignored `node_modules/` is never descended into or
|
||||
// counted against scan progress.
|
||||
if rules.is_ignored(rel, meta.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
if meta.is_dir() {
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
mtime_ns: None,
|
||||
size: 0,
|
||||
open_for_write: false,
|
||||
},
|
||||
);
|
||||
scanned(rel);
|
||||
walk_local_tree(root, &path, rules, writing, out, scanned)?;
|
||||
} else if meta.is_file() {
|
||||
// Check whether any process holds this file open for writing.
|
||||
// The canonical path is used because /proc/*/fd links resolve
|
||||
// to canonical targets.
|
||||
let ofw = std::fs::canonicalize(&path)
|
||||
.map(|canon| writing.contains(&canon))
|
||||
.unwrap_or(false);
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: false,
|
||||
mtime: system_mtime(&meta),
|
||||
mtime_ns: system_mtime_ns(&meta),
|
||||
size: meta.len() as i64,
|
||||
open_for_write: ofw,
|
||||
},
|
||||
);
|
||||
scanned(rel);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publish a conflict copy without overwriting an earlier conflict bearing the
|
||||
/// same timestamp. The original is removed only after the new copy is durable.
|
||||
fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||
@@ -1444,6 +1623,26 @@ fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||
unreachable!("u32 conflict suffix space exhausted")
|
||||
}
|
||||
|
||||
/// The remote uid of `rel`'s parent folder: from the pass's folder-uid map
|
||||
/// first, and from the baseline second for a parent the map never covered.
|
||||
///
|
||||
/// Used to name the folder whose cached listing a remote mutation has just
|
||||
/// invalidated (B86). `None` is not an error — it only means this pass cannot
|
||||
/// say which folder to invalidate, and the caller skips it.
|
||||
fn parent_uid_of(
|
||||
rel: &str,
|
||||
remote_dirs: &HashMap<String, NodeUid>,
|
||||
baseline: &HashMap<String, StoredSyncEntry>,
|
||||
) -> Option<NodeUid> {
|
||||
let parent = parent_rel(rel);
|
||||
remote_dirs.get(parent).cloned().or_else(|| {
|
||||
baseline
|
||||
.get(parent)
|
||||
.and_then(|e| e.remote_uid.as_deref())
|
||||
.and_then(parse_uid)
|
||||
})
|
||||
}
|
||||
|
||||
fn sync_parent_dir(path: &Path) -> std::io::Result<()> {
|
||||
match path.parent() {
|
||||
Some(parent) => std::fs::File::open(parent)?.sync_all(),
|
||||
@@ -1476,10 +1675,77 @@ fn system_mtime(meta: &std::fs::Metadata) -> i64 {
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// The same modification time in nanoseconds, when the filesystem reports one.
|
||||
///
|
||||
/// Whole seconds cannot distinguish an edit made in the same second as the last
|
||||
/// sync from no edit at all, and if the size did not change either, that file
|
||||
/// was never uploaded (bugs.md B25). `None` for a clock or filesystem that
|
||||
/// cannot answer, which falls the comparison back to seconds rather than
|
||||
/// asserting an mtime of zero.
|
||||
fn system_mtime_ns(meta: &std::fs::Metadata) -> Option<i64> {
|
||||
meta.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok())
|
||||
.and_then(|d| i64::try_from(d.as_nanos()).ok())
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A removal of something already gone has produced the state the sync
|
||||
/// engine wanted, so its baseline is dropped rather than retried forever.
|
||||
/// Anything else stays an error and keeps the baseline (bugs.md B42).
|
||||
#[test]
|
||||
fn an_already_absent_path_counts_as_removed() {
|
||||
let root = std::env::temp_dir().join(format!("pdfs-b42-{}", std::process::id()));
|
||||
let missing = root.join("never-existed");
|
||||
assert!(removed_locally(std::fs::remove_file(&missing)).is_ok());
|
||||
assert!(removed_locally(std::fs::remove_dir_all(&missing)).is_ok());
|
||||
|
||||
// A directory in the way of `remove_file` is a real failure, and must
|
||||
// stay one: the file survives, and its baseline has to survive with it.
|
||||
let occupied = root.join("a-directory");
|
||||
std::fs::create_dir_all(&occupied).unwrap();
|
||||
assert!(removed_locally(std::fs::remove_file(&occupied)).is_err());
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// The download guard: a destination that still matches the plan is
|
||||
/// replaced silently, one that has moved on since is kept (bugs.md B39).
|
||||
#[test]
|
||||
fn a_download_keeps_a_local_edit_it_did_not_plan_for() {
|
||||
let root = std::env::temp_dir().join(format!("pdfs-b39-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
let path = root.join("report.txt");
|
||||
|
||||
// Nothing there, nothing expected: the download simply creates it.
|
||||
assert!(keep_racing_local_edit(&path, None).is_ok());
|
||||
assert!(!path.exists());
|
||||
|
||||
// Unchanged since planning: no copy, the caller overwrites it.
|
||||
std::fs::write(&path, b"planned").unwrap();
|
||||
let planned = local_sig(&path).unwrap();
|
||||
keep_racing_local_edit(&path, Some(planned)).unwrap();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"planned");
|
||||
|
||||
// Changed since planning: the racing bytes are the only copy of
|
||||
// themselves, so they must survive the overwrite as a conflict copy.
|
||||
std::fs::write(&path, b"a much longer local edit").unwrap();
|
||||
keep_racing_local_edit(&path, Some(planned)).unwrap();
|
||||
let kept: Vec<_> = std::fs::read_dir(&root)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| std::fs::read(e.path()).unwrap_or_default())
|
||||
.collect();
|
||||
assert!(
|
||||
kept.iter()
|
||||
.any(|bytes| bytes == b"a much longer local edit"),
|
||||
"the racing edit must still exist somewhere: {kept:?}"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// Feed `n` `Reconcile` events `gap` apart on another thread, starting
|
||||
/// immediately. Returns the sender's join handle so the test can keep it
|
||||
/// alive (dropping the sender early would look like a disconnect).
|
||||
@@ -1565,6 +1831,7 @@ mod tests {
|
||||
|
||||
fn local_item(is_dir: bool) -> LocalItem {
|
||||
LocalItem {
|
||||
mtime_ns: None,
|
||||
is_dir,
|
||||
mtime: 1,
|
||||
size: 1,
|
||||
@@ -1722,6 +1989,44 @@ mod tests {
|
||||
assert!(guard_local_wipe(&base, &fresh).is_err());
|
||||
}
|
||||
|
||||
/// B54's remaining case, end to end: a folder whose local root is still
|
||||
/// *readable* but is no longer the tree it was — an empty directory left
|
||||
/// where a mount used to be, or a different device mounted over it.
|
||||
///
|
||||
/// Nothing fails here. The scan succeeds and honestly reports an empty
|
||||
/// directory, so the only thing standing between the user and a trashed
|
||||
/// remote folder is the guard, and it has to fire for a one-file baseline
|
||||
/// exactly as it does for a large one.
|
||||
#[test]
|
||||
fn a_replaced_mountpoint_trips_the_guard_at_every_baseline_size() {
|
||||
let root = sync_test_dir("wipe-replaced-mount");
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut local: HashMap<String, LocalItem> = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap();
|
||||
assert!(local.is_empty(), "the replacement root is genuinely empty");
|
||||
|
||||
for size in 1..=3 {
|
||||
let baseline: HashMap<String, ()> =
|
||||
(0..size).map(|i| (format!("file{i}.txt"), ())).collect();
|
||||
assert!(
|
||||
guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_err(),
|
||||
"a {size}-path baseline against an empty root is not a deletion"
|
||||
);
|
||||
}
|
||||
|
||||
// And the guard steps aside the moment the real tree is back, so a
|
||||
// remounted folder resumes rather than staying wedged.
|
||||
std::fs::write(root.join("file0.txt"), b"back").unwrap();
|
||||
let mut local: HashMap<String, LocalItem> = HashMap::new();
|
||||
walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap();
|
||||
let baseline: HashMap<String, ()> = [("file0.txt".to_string(), ())].into_iter().collect();
|
||||
assert!(guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_ok());
|
||||
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
/// A restored folder (features.md 5.2) starts with an empty baseline against
|
||||
/// an empty local directory and a full remote. That must reconcile as
|
||||
/// "download everything" rather than tripping the wipe guard — otherwise
|
||||
@@ -1764,6 +2069,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 0,
|
||||
local_size: 0,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: Some(mtime.to_string()),
|
||||
remote_hash: Some(size.to_string()),
|
||||
},
|
||||
@@ -1801,6 +2107,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 0,
|
||||
local_size: 0,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
},
|
||||
@@ -1815,6 +2122,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 10,
|
||||
local_size: 20,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: Some("1700".into()),
|
||||
remote_hash: Some("4096".into()),
|
||||
};
|
||||
@@ -1828,6 +2136,7 @@ mod tests {
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime: 0,
|
||||
local_size: 0,
|
||||
local_mtime_ns: None,
|
||||
remote_rev: None,
|
||||
remote_hash: None,
|
||||
};
|
||||
@@ -1863,6 +2172,7 @@ mod tests {
|
||||
uid: uid(),
|
||||
mtime: 0,
|
||||
size: 10,
|
||||
local: None,
|
||||
};
|
||||
assert_eq!(op.kind(), ActivityKind::Download);
|
||||
}
|
||||
@@ -1918,6 +2228,204 @@ mod tests {
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
/// B49's remaining cases. Preservation is the step that makes a download
|
||||
/// safe to publish, so every way it can fail has to leave the local file
|
||||
/// where it was — the download is aborted, not the file replaced.
|
||||
#[test]
|
||||
fn conflict_preservation_failures_all_keep_the_local_file() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
// An unwritable directory: the conflict copy cannot be created at all.
|
||||
let dir = sync_test_dir("conflict-eacces");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"the only copy").unwrap();
|
||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o500)).unwrap();
|
||||
let denied = preserve_conflict_copy(&source, 1700);
|
||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(denied.is_err(), "a read-only directory must fail the copy");
|
||||
assert_eq!(
|
||||
std::fs::read(&source).unwrap(),
|
||||
b"the only copy",
|
||||
"the local file survives byte-identical"
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
// A source that vanished between planning and preservation. The pass
|
||||
// must report it rather than publish the download over the gap.
|
||||
let dir = sync_test_dir("conflict-enoent");
|
||||
assert!(preserve_conflict_copy(&dir.join("gone.txt"), 1700).is_err());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
// A write error partway through the copy — induced by making the
|
||||
// destination name a directory, so `create_new` fails as something
|
||||
// other than `AlreadyExists`.
|
||||
let dir = sync_test_dir("conflict-eisdir");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"still the only copy").unwrap();
|
||||
std::fs::create_dir(conflict_path(&source, 1700)).unwrap();
|
||||
// Suffix 0 is taken by a directory; `create_new` on it reports
|
||||
// `AlreadyExists`, so preservation moves on to suffix 1 and succeeds.
|
||||
// What must not happen is the source being removed without a copy.
|
||||
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||
assert!(preserved.is_file());
|
||||
assert_eq!(std::fs::read(&preserved).unwrap(), b"still the only copy");
|
||||
assert!(!source.exists());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
/// A conflict copy keeps the bytes *and* the metadata that identifies them,
|
||||
/// so the next pass uploads the user's version rather than a file that looks
|
||||
/// freshly written.
|
||||
#[test]
|
||||
fn a_conflict_copy_is_byte_and_mtime_identical() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let dir = sync_test_dir("conflict-identity");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"local edit that must survive").unwrap();
|
||||
let before = std::fs::metadata(&source).unwrap();
|
||||
|
||||
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||
let after = std::fs::metadata(&preserved).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read(&preserved).unwrap(),
|
||||
b"local edit that must survive"
|
||||
);
|
||||
assert_eq!(after.len(), before.len());
|
||||
assert_eq!(after.modified().unwrap(), before.modified().unwrap());
|
||||
assert_eq!(after.permissions().mode(), before.permissions().mode());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
// ---- B55: an incomplete local scan is never a set of deletions ---------
|
||||
|
||||
fn no_progress() -> impl FnMut(&str) {
|
||||
|_: &str| {}
|
||||
}
|
||||
|
||||
/// A directory the pass cannot read must fail the pass. Omitting it would
|
||||
/// hand reconciliation a baseline-minus-subtree, which it reads as the user
|
||||
/// having deleted that subtree and propagates to Drive (B55).
|
||||
#[test]
|
||||
fn an_unreadable_subdirectory_fails_the_scan_instead_of_shrinking_it() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-eacces");
|
||||
std::fs::write(root.join("keep.txt"), b"a").unwrap();
|
||||
let locked = root.join("locked");
|
||||
std::fs::create_dir(&locked).unwrap();
|
||||
std::fs::write(locked.join("inner.txt"), b"b").unwrap();
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let error = result.expect_err("an unreadable directory must abort the pass");
|
||||
assert!(
|
||||
error.contains("locked"),
|
||||
"the error should name the subtree it could not read: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The same rule one level up: if the sync root itself cannot be read, the
|
||||
/// pass sees nothing, and "nothing" must never mean "delete everything".
|
||||
#[test]
|
||||
fn an_unreadable_root_fails_the_scan() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-root-eacces");
|
||||
std::fs::write(root.join("keep.txt"), b"a").unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
/// A root that is gone entirely — an unmounted device folder, a removed
|
||||
/// directory — is the sharpest version of the same case: every baseline path
|
||||
/// is "missing", and the scan must refuse rather than report an empty tree.
|
||||
#[test]
|
||||
fn an_absent_root_fails_the_scan() {
|
||||
let root = sync_test_dir("scan-absent");
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
assert!(walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress).is_err());
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
/// A name that is not valid UTF-8 cannot be keyed into the scan, so it
|
||||
/// cannot be compared against the baseline either. Refusing the pass is the
|
||||
/// only safe reading: skipping it would classify the file as deleted.
|
||||
#[test]
|
||||
fn a_non_utf8_name_fails_the_scan() {
|
||||
use std::ffi::OsStr;
|
||||
use std::os::unix::ffi::OsStrExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-non-utf8");
|
||||
std::fs::write(root.join(OsStr::from_bytes(b"bad\xffname")), b"x").unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let error = result.expect_err("a name the scan cannot key must abort the pass");
|
||||
assert!(error.contains("UTF-8"), "{error}");
|
||||
}
|
||||
|
||||
/// The complement: a scan that *can* read everything reports everything, so
|
||||
/// the strictness above does not come at the cost of a working pass. A
|
||||
/// symlink is skipped by design rather than by failure, and a dangling one
|
||||
/// stats fine through `symlink_metadata`.
|
||||
#[test]
|
||||
fn a_readable_tree_scans_completely() {
|
||||
let root = sync_test_dir("scan-complete");
|
||||
std::fs::write(root.join("a.txt"), b"aa").unwrap();
|
||||
std::fs::create_dir(root.join("sub")).unwrap();
|
||||
std::fs::write(root.join("sub/b.txt"), b"bbb").unwrap();
|
||||
std::os::unix::fs::symlink("nowhere", root.join("dangling")).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut |rel| {
|
||||
seen.push(rel.to_string())
|
||||
});
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
result.unwrap();
|
||||
let mut keys: Vec<&str> = out.keys().map(String::as_str).collect();
|
||||
keys.sort_unstable();
|
||||
assert_eq!(keys, vec!["a.txt", "sub", "sub/b.txt"]);
|
||||
assert_eq!(out["a.txt"].size, 2);
|
||||
assert_eq!(out["sub/b.txt"].size, 3);
|
||||
assert!(out["sub"].is_dir);
|
||||
assert_eq!(seen.len(), 3, "every reported entry ticks scan progress");
|
||||
}
|
||||
|
||||
fn sync_test_dir(label: &str) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
|
||||
@@ -32,17 +32,24 @@ pub(crate) enum SyncMsg {
|
||||
ReconcileAll,
|
||||
/// Re-read the folder set and adjust the filesystem watches (after add/remove).
|
||||
Rewatch,
|
||||
/// The daemon is tearing down: finish the current pass and end the engine
|
||||
/// thread, so the mount can join it (bugs.md B44).
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Start the sync engine: a control thread that owns the filesystem watcher and
|
||||
/// serialises reconcile passes, plus a periodic poll thread. The engine receives
|
||||
/// on `rx`; senders live in [`Core::sync_tx`].
|
||||
pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) {
|
||||
if let Err(e) = std::thread::Builder::new()
|
||||
pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) -> Option<std::thread::JoinHandle<()>> {
|
||||
match std::thread::Builder::new()
|
||||
.name("pdfs-sync".into())
|
||||
.spawn(move || engine_loop(core, rx))
|
||||
{
|
||||
warn!(error = %e, "failed to start sync engine");
|
||||
Ok(handle) => Some(handle),
|
||||
Err(e) => {
|
||||
warn!(error = %e, "failed to start sync engine");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,11 +81,11 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
// Periodic remote poll.
|
||||
{
|
||||
let tx = core.sync_tx.clone();
|
||||
let shutdown = core.shutdown.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-sync-poll".into())
|
||||
.spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(POLL_INTERVAL);
|
||||
while shutdown.sleep(POLL_INTERVAL) {
|
||||
if tx.send(SyncMsg::ReconcileAll).is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -94,8 +101,12 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
let mut ids: HashSet<i64> = HashSet::new();
|
||||
let mut all = false;
|
||||
let mut do_rewatch = false;
|
||||
classify(msg, &mut ids, &mut all, &mut do_rewatch);
|
||||
settle(&rx, &mut ids, &mut all, &mut do_rewatch);
|
||||
if !classify(msg, &mut ids, &mut all, &mut do_rewatch)
|
||||
|| !settle(&rx, &mut ids, &mut all, &mut do_rewatch)
|
||||
{
|
||||
info!("sync engine stopping");
|
||||
return;
|
||||
}
|
||||
|
||||
if do_rewatch {
|
||||
rewatch(&core, &mut watcher, &watched);
|
||||
@@ -116,9 +127,9 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
||||
{
|
||||
let tx = core.sync_tx.clone();
|
||||
let shutdown = core.shutdown.clone();
|
||||
std::thread::spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(POLL_INTERVAL);
|
||||
while shutdown.sleep(POLL_INTERVAL) {
|
||||
if tx.send(SyncMsg::ReconcileAll).is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -130,8 +141,12 @@ fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
||||
let mut ids: HashSet<i64> = HashSet::new();
|
||||
let mut all = false;
|
||||
let mut do_rewatch = false;
|
||||
classify(msg, &mut ids, &mut all, &mut do_rewatch);
|
||||
settle(&rx, &mut ids, &mut all, &mut do_rewatch);
|
||||
if !classify(msg, &mut ids, &mut all, &mut do_rewatch)
|
||||
|| !settle(&rx, &mut ids, &mut all, &mut do_rewatch)
|
||||
{
|
||||
info!("sync engine stopping");
|
||||
return;
|
||||
}
|
||||
if all {
|
||||
reconcile_all(core);
|
||||
} else {
|
||||
@@ -158,7 +173,7 @@ pub(super) fn settle_with(
|
||||
do_rewatch: &mut bool,
|
||||
quiet: Duration,
|
||||
cap: Duration,
|
||||
) {
|
||||
) -> bool {
|
||||
let deadline = Instant::now() + cap;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
@@ -170,28 +185,49 @@ pub(super) fn settle_with(
|
||||
match rx.recv_timeout(wait) {
|
||||
// Another event: the burst is still going, so the quiet window
|
||||
// starts again from here.
|
||||
Ok(m) => classify(m, ids, all, do_rewatch),
|
||||
// A stop lands mid-burst: give up the rest of the burst rather
|
||||
// than settling for it, and let the caller end the loop.
|
||||
Ok(m) => {
|
||||
if !classify(m, ids, all, do_rewatch) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Quiet for a full window: whatever was being written has stopped.
|
||||
Err(RecvTimeoutError::Timeout) => break,
|
||||
// Every sender is gone; the caller's loop is about to end anyway.
|
||||
Err(RecvTimeoutError::Disconnected) => break,
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// [`settle_with`] at the real timings.
|
||||
fn settle(rx: &Receiver<SyncMsg>, ids: &mut HashSet<i64>, all: &mut bool, do_rewatch: &mut bool) {
|
||||
settle_with(rx, ids, all, do_rewatch, DEBOUNCE, MAX_SETTLE);
|
||||
fn settle(
|
||||
rx: &Receiver<SyncMsg>,
|
||||
ids: &mut HashSet<i64>,
|
||||
all: &mut bool,
|
||||
do_rewatch: &mut bool,
|
||||
) -> bool {
|
||||
settle_with(rx, ids, all, do_rewatch, DEBOUNCE, MAX_SETTLE)
|
||||
}
|
||||
|
||||
pub(super) fn classify(msg: SyncMsg, ids: &mut HashSet<i64>, all: &mut bool, rewatch: &mut bool) {
|
||||
/// Fold one message into the pending work of the current burst. Returns `false`
|
||||
/// for [`SyncMsg::Stop`], which is not work but an instruction to end the loop.
|
||||
pub(super) fn classify(
|
||||
msg: SyncMsg,
|
||||
ids: &mut HashSet<i64>,
|
||||
all: &mut bool,
|
||||
rewatch: &mut bool,
|
||||
) -> bool {
|
||||
match msg {
|
||||
SyncMsg::Reconcile(id) => {
|
||||
ids.insert(id);
|
||||
}
|
||||
SyncMsg::ReconcileAll => *all = true,
|
||||
SyncMsg::Rewatch => *rewatch = true,
|
||||
SyncMsg::Stop => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Reconcile every mirror folder in turn.
|
||||
|
||||
@@ -2,9 +2,73 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
/// What a reconcile pass remembers about the local side of one path: enough to
|
||||
/// recognise the same content again, and no more.
|
||||
///
|
||||
/// `mtime_ns` is the whole reason this is a struct rather than a pair — see
|
||||
/// [`StoredSyncEntry::local_mtime_ns`](pdfs_core::db::StoredSyncEntry).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) struct LocalSig {
|
||||
pub(super) mtime: i64,
|
||||
pub(super) mtime_ns: Option<i64>,
|
||||
pub(super) size: i64,
|
||||
}
|
||||
|
||||
impl LocalSig {
|
||||
/// Whether these two observations describe the same local content.
|
||||
///
|
||||
/// Nanoseconds decide it when both sides have them; otherwise the comparison
|
||||
/// falls back to whole seconds, which is what a baseline row written before
|
||||
/// schema 28 — or a filesystem that reports no sub-second time — can support
|
||||
/// (bugs.md B25).
|
||||
pub(super) fn same_content(&self, other: &Self) -> bool {
|
||||
if self.size != other.size {
|
||||
return false;
|
||||
}
|
||||
match (self.mtime_ns, other.mtime_ns) {
|
||||
(Some(a), Some(b)) => a == b,
|
||||
_ => self.mtime == other.mtime,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&LocalItem> for LocalSig {
|
||||
fn from(item: &LocalItem) -> Self {
|
||||
Self {
|
||||
mtime: item.mtime,
|
||||
mtime_ns: item.mtime_ns,
|
||||
size: item.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&std::fs::Metadata> for LocalSig {
|
||||
fn from(meta: &std::fs::Metadata) -> Self {
|
||||
Self {
|
||||
mtime: system_mtime(meta),
|
||||
mtime_ns: system_mtime_ns(meta),
|
||||
size: meta.len() as i64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The baseline's record of the local side, as [`LocalSig`] compares it.
|
||||
impl From<&StoredSyncEntry> for LocalSig {
|
||||
fn from(entry: &StoredSyncEntry) -> Self {
|
||||
Self {
|
||||
mtime: entry.local_mtime,
|
||||
mtime_ns: entry.local_mtime_ns,
|
||||
size: entry.local_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct LocalItem {
|
||||
pub(super) is_dir: bool,
|
||||
pub(super) mtime: i64,
|
||||
/// [`LocalItem::mtime`] to nanosecond precision, when the filesystem
|
||||
/// reports one.
|
||||
pub(super) mtime_ns: Option<i64>,
|
||||
pub(super) size: i64,
|
||||
/// True when another process holds this file open for writing (detected via
|
||||
/// `/proc/*/fd`). An open-for-write file is kept in the map — so it is not
|
||||
@@ -91,11 +155,18 @@ pub(super) enum Pending {
|
||||
UploadRevision { rel: String, uid: NodeUid },
|
||||
/// Download remote `uid` to the local path, stamping `mtime`. `size` is the
|
||||
/// remote's reported size, used as the transfer's expected total.
|
||||
///
|
||||
/// `local` is what the planning walk saw at this path, or `None` if nothing
|
||||
/// was there. A download can run for minutes, so the apply step revalidates
|
||||
/// against it immediately before it renames over the destination: a writer
|
||||
/// that finished an edit in the meantime must not be overwritten without a
|
||||
/// conflict copy (bugs.md B39).
|
||||
Download {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
local: Option<LocalSig>,
|
||||
},
|
||||
/// Both sides changed: set the local copy aside, then download remote `uid`.
|
||||
Conflict {
|
||||
|
||||
@@ -218,7 +218,10 @@ pub(super) fn plan_file(
|
||||
}
|
||||
|
||||
let local_changed = local.is_some_and(|item| {
|
||||
baseline.is_none_or(|base| base.local_mtime != item.mtime || base.local_size != item.size)
|
||||
// Nanoseconds where both sides have them (bugs.md B25): whole seconds
|
||||
// could not see an edit made in the same second as the last sync that
|
||||
// left the file the same length, so it was never uploaded.
|
||||
baseline.is_none_or(|base| !LocalSig::from(item).same_content(&LocalSig::from(base)))
|
||||
});
|
||||
let remote_changed = remote.is_some_and(|item| {
|
||||
baseline.is_none_or(|base| remote_sig(base) != Some((item.mtime, item.size)))
|
||||
@@ -252,6 +255,7 @@ mod file_plan_tests {
|
||||
|
||||
fn local(mtime: i64, size: i64) -> LocalItem {
|
||||
LocalItem {
|
||||
mtime_ns: None,
|
||||
is_dir: false,
|
||||
mtime,
|
||||
size,
|
||||
@@ -278,12 +282,49 @@ mod file_plan_tests {
|
||||
rel_path: "file".into(),
|
||||
remote_uid: Some("v~l".into()),
|
||||
local_mtime,
|
||||
local_mtime_ns: None,
|
||||
local_size,
|
||||
remote_rev: Some(remote_mtime.to_string()),
|
||||
remote_hash: Some(remote_size.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whole seconds cannot see an edit made inside the same second that leaves the
|
||||
/// file the same length, so a mirror silently stopped uploading it. With a
|
||||
/// nanosecond time on both sides it is an ordinary local change (bugs.md B25).
|
||||
#[test]
|
||||
fn a_same_second_same_size_edit_is_still_a_local_change() {
|
||||
let mut base = baseline(10, 20, 30, 40);
|
||||
base.local_mtime_ns = Some(10_000_000_000);
|
||||
let edited = LocalItem {
|
||||
mtime_ns: Some(10_400_000_000),
|
||||
..local(10, 20)
|
||||
};
|
||||
let remote = remote(30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&edited), Some(&remote), Some(&base)),
|
||||
FilePlan::UploadRevision
|
||||
);
|
||||
|
||||
// The same file untouched still reads as unchanged.
|
||||
let same = LocalItem {
|
||||
mtime_ns: Some(10_000_000_000),
|
||||
..local(10, 20)
|
||||
};
|
||||
assert_eq!(
|
||||
plan_file(Some(&same), Some(&remote), Some(&base)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
|
||||
// And a row from before the column exists compares on seconds, exactly
|
||||
// as it did when it was written — no mass re-upload on upgrade.
|
||||
let old = baseline(10, 20, 30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&edited), Some(&remote), Some(&old)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_plan_covers_every_presence_and_change_shape() {
|
||||
let same_local = local(10, 20);
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
//! Importing a Google Photos Takeout export into Proton Photos.
|
||||
//!
|
||||
//! [`pdfs_core::takeout`] says what is in the archives; this is the part that
|
||||
//! talks to Proton. One import runs on its own thread and moves through four
|
||||
//! phases, reporting into the transfer registry as a job so the GUI's transfer
|
||||
//! list shows it without a bespoke widget:
|
||||
//!
|
||||
//! 1. **Scan** — read the archives' central directories and sidecars.
|
||||
//! 2. **Deduplicate against the account** — ask the photos volume which of the
|
||||
//! names already exist ([`ProtonPhotosClient::name_collisions`]), hash only
|
||||
//! those files, then confirm with the content digest. A library that is
|
||||
//! already half-synced by Proton Photos on a phone therefore costs one cheap
|
||||
//! request per 150 names and reads no media at all for the misses.
|
||||
//! 3. **Upload** the survivors, streaming each entry straight out of its zip —
|
||||
//! no extraction to disk, so the import needs no scratch space.
|
||||
//! 4. **Albums** — create the albums the export names that the account lacks,
|
||||
//! then add every photo of each album, the ones that were already on the
|
||||
//! account included.
|
||||
//!
|
||||
//! Re-running an interrupted import is the resume story: phase 2 finds
|
||||
//! everything the previous run uploaded and skips it. Nothing is persisted
|
||||
//! between runs.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::io::Read;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use pdfs_core::control::{ActivityKind, ImportSummary, TransferDirection};
|
||||
use pdfs_core::takeout::{Bucket, TakeoutPhoto, TakeoutScan, scan_archives};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use proton_drive_rs::{PhotoTag, PhotoTagsUpdate, PhotoUploadMetadata};
|
||||
use sha1::{Digest, Sha1};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::Core;
|
||||
use crate::transfers::CountingReader;
|
||||
|
||||
/// The one import this daemon may be running, and how the last one ended.
|
||||
///
|
||||
/// Process-global rather than a [`Core`] field because it must be *one* import
|
||||
/// per daemon however many `Core` clones exist (the forked on-demand mounts each
|
||||
/// hold one), and because a front-end asking for the result after the import
|
||||
/// finished must still find it.
|
||||
static IMPORT: std::sync::LazyLock<parking_lot::Mutex<ImportState>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(ImportState::default()));
|
||||
|
||||
#[derive(Default)]
|
||||
struct ImportState {
|
||||
running: bool,
|
||||
cancel: Option<Arc<AtomicBool>>,
|
||||
last: Option<ImportSummary>,
|
||||
}
|
||||
|
||||
/// Whether an import is running, and the last one's report.
|
||||
pub(crate) fn import_status() -> (bool, Option<ImportSummary>) {
|
||||
let state = IMPORT.lock();
|
||||
(state.running, state.last.clone())
|
||||
}
|
||||
|
||||
/// Ask the running import to stop. False when none is running.
|
||||
pub(crate) fn cancel_import() -> bool {
|
||||
match &IMPORT.lock().cancel {
|
||||
Some(flag) => {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// One photo to import, after the copies Google keeps per album are folded into
|
||||
/// a single upload that joins several albums.
|
||||
struct Planned {
|
||||
photo: TakeoutPhoto,
|
||||
/// Every album this photo belongs to, by export title.
|
||||
albums: Vec<String>,
|
||||
/// Filled in phase 2/3: where the photo ended up on the account.
|
||||
uid: Option<NodeUid>,
|
||||
}
|
||||
|
||||
/// Fold the scan into one entry per distinct photo.
|
||||
///
|
||||
/// Google stores a full copy of a photo in every album folder it appears in, so
|
||||
/// the same file is in the export two or three times over. They are folded on
|
||||
/// `(name, size)`: a byte-identical copy under the same name is the same photo,
|
||||
/// and the album memberships of the copies are unioned onto the survivor. Two
|
||||
/// genuinely different photos sharing both a name and an exact size would be
|
||||
/// merged — vanishingly unlikely, and the cost of being wrong is one photo
|
||||
/// filed under an extra album rather than lost data.
|
||||
fn plan(scan: &TakeoutScan) -> Vec<Planned> {
|
||||
let mut order: Vec<(String, u64)> = Vec::new();
|
||||
let mut by_key: HashMap<(String, u64), Planned> = HashMap::new();
|
||||
|
||||
for photo in &scan.photos {
|
||||
let key = (photo.name.clone(), photo.size);
|
||||
let entry = by_key.entry(key.clone()).or_insert_with(|| {
|
||||
order.push(key.clone());
|
||||
Planned {
|
||||
photo: photo.clone(),
|
||||
albums: Vec::new(),
|
||||
uid: None,
|
||||
}
|
||||
});
|
||||
if let Bucket::Album(title) = &photo.bucket
|
||||
&& !entry.albums.iter().any(|existing| existing == title)
|
||||
{
|
||||
entry.albums.push(title.clone());
|
||||
}
|
||||
// A copy that carries a capture time wins over one that does not: the
|
||||
// album copy and the year-folder copy do not always both have a sidecar.
|
||||
if entry.photo.capture_time.is_none() && photo.capture_time.is_some() {
|
||||
entry.photo.capture_time = photo.capture_time;
|
||||
}
|
||||
entry.photo.favorite |= photo.favorite;
|
||||
}
|
||||
|
||||
order
|
||||
.into_iter()
|
||||
.filter_map(|key| by_key.remove(&key))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Open one archive and stream `entry` through `sink`.
|
||||
fn read_entry<R>(
|
||||
archive: &mut zip::ZipArchive<std::io::BufReader<std::fs::File>>,
|
||||
entry: &str,
|
||||
sink: impl FnOnce(&mut (dyn Read + Send)) -> R,
|
||||
) -> CoreResult<R> {
|
||||
let mut file = archive
|
||||
.by_name(entry)
|
||||
.map_err(|e| CoreError::invalid(format!("{entry}: {e}")))?;
|
||||
Ok(sink(&mut file))
|
||||
}
|
||||
|
||||
impl Core {
|
||||
/// Import Google Photos Takeout archives. Blocking; run on its own thread.
|
||||
///
|
||||
/// `cancel` stops the run at the next photo boundary — an upload already on
|
||||
/// the wire is allowed to finish, so a cancelled import never leaves a
|
||||
/// half-written revision behind.
|
||||
pub(crate) fn import_takeout(
|
||||
&self,
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
cancel: Arc<AtomicBool>,
|
||||
) -> CoreResult<ImportSummary> {
|
||||
let job = self.transfers.begin_job("Importing Google Photos");
|
||||
job.detail("reading archives");
|
||||
|
||||
let scan = scan_archives(&archives)?;
|
||||
let mut planned = plan(&scan);
|
||||
let mut summary = ImportSummary {
|
||||
found: planned.len(),
|
||||
skipped_trashed: scan.skipped_trashed,
|
||||
..Default::default()
|
||||
};
|
||||
job.set_total(planned.len() as u64);
|
||||
info!(
|
||||
archives = archives.len(),
|
||||
photos = summary.found,
|
||||
albums = scan.albums.len(),
|
||||
trashed = summary.skipped_trashed,
|
||||
"takeout scan complete"
|
||||
);
|
||||
|
||||
let photos = self.photos();
|
||||
if self
|
||||
.rt
|
||||
.block_on(photos.get_photos_root())
|
||||
.map_err(|e| CoreError::from_api(&e, "resolve photos library"))?
|
||||
.is_none()
|
||||
{
|
||||
return Err(CoreError::invalid(
|
||||
"this account has no Proton Photos library yet; open Proton Photos once to create it, then import again",
|
||||
));
|
||||
}
|
||||
|
||||
// Phase 2: which of these are already on the account.
|
||||
job.detail("checking for photos you already have");
|
||||
self.mark_existing(&mut planned, &archives, &mut summary, &cancel)?;
|
||||
|
||||
if dry_run {
|
||||
summary.duplicates = planned.iter().filter(|p| p.uid.is_some()).count();
|
||||
return Ok(summary);
|
||||
}
|
||||
|
||||
// Phase 3: upload what is left, one archive at a time so each zip's
|
||||
// central directory is parsed once.
|
||||
self.upload_planned(&mut planned, &archives, &mut summary, &cancel, &job)?;
|
||||
|
||||
// Phase 4: albums. Attempted even after a cancel — the photos that did
|
||||
// upload should still land in their albums.
|
||||
if !scan.albums.is_empty() {
|
||||
job.detail("sorting photos into albums");
|
||||
if let Err(e) = self.link_albums(&planned, &scan.albums, &mut summary) {
|
||||
warn!(error = %e, "takeout album linking failed");
|
||||
}
|
||||
}
|
||||
|
||||
self.spawn_timeline_refresh();
|
||||
summary.cancelled = cancel.load(Ordering::Relaxed);
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Fill in `uid` for every planned photo the account already holds.
|
||||
///
|
||||
/// Two passes on purpose: the name-only query is one request per 150 photos
|
||||
/// and needs no file contents, so the expensive SHA-1 pass only runs for the
|
||||
/// handful of names that actually collide.
|
||||
fn mark_existing(
|
||||
&self,
|
||||
planned: &mut [Planned],
|
||||
archives: &[PathBuf],
|
||||
summary: &mut ImportSummary,
|
||||
cancel: &AtomicBool,
|
||||
) -> CoreResult<()> {
|
||||
let photos = self.photos();
|
||||
let names: Vec<String> = planned.iter().map(|p| p.photo.name.clone()).collect();
|
||||
let collisions = self
|
||||
.rt
|
||||
.block_on(photos.name_collisions(&names))
|
||||
.map_err(|e| CoreError::from_api(&e, "check for existing photos"))?;
|
||||
|
||||
// Hash the colliding files, grouped by archive so each zip opens once.
|
||||
let mut by_archive: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
|
||||
for (index, planned) in planned.iter().enumerate() {
|
||||
if collisions.get(index).copied().unwrap_or(false) {
|
||||
by_archive
|
||||
.entry(planned.photo.archive)
|
||||
.or_default()
|
||||
.push(index);
|
||||
}
|
||||
}
|
||||
|
||||
// Kept as a pair list, not a map: the duplicate lookup answers in input
|
||||
// order, so the order these were built in is what maps answers back to
|
||||
// planned photos.
|
||||
let mut digests: Vec<(usize, String)> = Vec::new();
|
||||
for (archive_index, indices) in by_archive {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let mut archive = open_archive(archives, archive_index)?;
|
||||
for index in indices {
|
||||
let entry = planned[index].photo.entry.clone();
|
||||
match read_entry(&mut archive, &entry, sha1_of) {
|
||||
Ok(Ok(digest)) => digests.push((index, digest)),
|
||||
Ok(Err(e)) | Err(e) => {
|
||||
warn!(entry, error = %e, "cannot hash takeout entry");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if digests.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let items: Vec<(String, String)> = digests
|
||||
.iter()
|
||||
.map(|(index, digest)| (planned[*index].photo.name.clone(), digest.clone()))
|
||||
.collect();
|
||||
let matches = self
|
||||
.rt
|
||||
.block_on(photos.find_duplicates_many(&items))
|
||||
.map_err(|e| CoreError::from_api(&e, "check for existing photos"))?;
|
||||
|
||||
for (slot, (index, _)) in digests.iter().enumerate() {
|
||||
if let Some(uid) = matches.get(slot).and_then(|uids| uids.first()) {
|
||||
planned[*index].uid = Some(uid.clone());
|
||||
summary.duplicates += 1;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Upload every planned photo that is not already on the account.
|
||||
fn upload_planned(
|
||||
&self,
|
||||
planned: &mut [Planned],
|
||||
archives: &[PathBuf],
|
||||
summary: &mut ImportSummary,
|
||||
cancel: &AtomicBool,
|
||||
job: &crate::transfers::JobGuard,
|
||||
) -> CoreResult<()> {
|
||||
let photos = self.photos();
|
||||
let mut favorites: Vec<NodeUid> = Vec::new();
|
||||
|
||||
let mut by_archive: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
|
||||
for (index, item) in planned.iter().enumerate() {
|
||||
if item.uid.is_none() {
|
||||
by_archive
|
||||
.entry(item.photo.archive)
|
||||
.or_default()
|
||||
.push(index);
|
||||
}
|
||||
}
|
||||
|
||||
for (archive_index, indices) in by_archive {
|
||||
if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() {
|
||||
break;
|
||||
}
|
||||
let mut archive = open_archive(archives, archive_index)?;
|
||||
|
||||
for index in indices {
|
||||
if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() {
|
||||
break;
|
||||
}
|
||||
let photo = planned[index].photo.clone();
|
||||
job.detail(photo.name.clone());
|
||||
|
||||
let guard = self.transfers.begin(
|
||||
photo.name.clone(),
|
||||
"",
|
||||
TransferDirection::Upload,
|
||||
photo.size,
|
||||
);
|
||||
let metadata = PhotoUploadMetadata {
|
||||
capture_time: photo.capture_time,
|
||||
..Default::default()
|
||||
};
|
||||
let uploaded = read_entry(&mut archive, &photo.entry, |reader| {
|
||||
let counting = CountingReader::new(reader, &guard);
|
||||
self.rt.block_on(photos.upload_photo_from(
|
||||
&photo.name,
|
||||
&photo.media_type,
|
||||
counting,
|
||||
photo.size as i64,
|
||||
Vec::new(),
|
||||
metadata,
|
||||
false,
|
||||
))
|
||||
});
|
||||
drop(guard);
|
||||
job.step();
|
||||
|
||||
match uploaded {
|
||||
Ok(Ok(uid)) => {
|
||||
summary.uploaded += 1;
|
||||
summary.bytes += photo.size;
|
||||
if photo.favorite {
|
||||
favorites.push(uid.clone());
|
||||
}
|
||||
planned[index].uid = Some(uid);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
summary.failed += 1;
|
||||
warn!(name = photo.name, error = %e, "takeout photo upload failed");
|
||||
}
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
warn!(name = photo.name, error = %e, "cannot read takeout entry");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Favourites are a separate call: the upload seal takes classification
|
||||
// tags, but `Favorite` is not one of them — it goes through the photos
|
||||
// volume's dedicated favourite endpoint.
|
||||
if !favorites.is_empty() {
|
||||
let updates: Vec<PhotoTagsUpdate> = favorites
|
||||
.into_iter()
|
||||
.map(|node_uid| PhotoTagsUpdate {
|
||||
node_uid,
|
||||
tags_to_add: vec![PhotoTag::Favorite],
|
||||
tags_to_remove: Vec::new(),
|
||||
})
|
||||
.collect();
|
||||
if let Err(e) = self.rt.block_on(photos.update_photos(&updates)) {
|
||||
warn!(error = %e, "marking imported favourites failed");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create the albums the export names that the account lacks, then add each
|
||||
/// album's photos to it.
|
||||
fn link_albums(
|
||||
&self,
|
||||
planned: &[Planned],
|
||||
albums: &[String],
|
||||
summary: &mut ImportSummary,
|
||||
) -> CoreResult<()> {
|
||||
let photos = self.photos();
|
||||
|
||||
// Existing albums by decrypted name, so a second import of the same
|
||||
// export adds to the album it made the first time instead of a twin.
|
||||
let mut existing: HashMap<String, NodeUid> = HashMap::new();
|
||||
let uids = self
|
||||
.rt
|
||||
.block_on(photos.enumerate_album_node_uids())
|
||||
.map_err(|e| CoreError::from_api(&e, "list albums"))?;
|
||||
if !uids.is_empty() {
|
||||
let albums_nodes = self
|
||||
.rt
|
||||
.block_on(photos.enumerate_nodes(&uids))
|
||||
.map_err(|e| CoreError::from_api(&e, "read albums"))?;
|
||||
for node in albums_nodes {
|
||||
existing.entry(node.name.clone()).or_insert(node.uid);
|
||||
}
|
||||
}
|
||||
|
||||
for title in albums {
|
||||
let members: Vec<NodeUid> = planned
|
||||
.iter()
|
||||
.filter(|item| item.albums.iter().any(|album| album == title))
|
||||
.filter_map(|item| item.uid.clone())
|
||||
.collect();
|
||||
if members.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let album_uid = match existing.get(title) {
|
||||
Some(uid) => uid.clone(),
|
||||
None => match self.rt.block_on(photos.create_album(title)) {
|
||||
Ok(uid) => {
|
||||
summary.albums_created += 1;
|
||||
existing.insert(title.clone(), uid.clone());
|
||||
uid
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(album = title, error = %e, "cannot create album");
|
||||
continue;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
match self
|
||||
.rt
|
||||
.block_on(photos.add_photos_to_album(&album_uid, &members))
|
||||
{
|
||||
Ok(outcomes) => {
|
||||
for (uid, outcome) in outcomes {
|
||||
match outcome {
|
||||
Ok(()) => summary.album_links += 1,
|
||||
// A photo already in the album is the expected
|
||||
// outcome of a re-run, not a failure worth counting.
|
||||
Err(e) => {
|
||||
warn!(album = title, %uid, error = %e, "cannot add photo to album")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(album = title, error = %e, "album addition failed"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run an import on a background thread, logging its outcome to the activity
|
||||
/// feed. Returns immediately.
|
||||
///
|
||||
/// Refuses while one is already running: two imports of the same export
|
||||
/// would race each other's duplicate detection and upload everything twice.
|
||||
pub(crate) fn spawn_takeout_import(
|
||||
&self,
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
) -> CoreResult<()> {
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
{
|
||||
let mut state = IMPORT.lock();
|
||||
if state.running {
|
||||
return Err(CoreError::conflict(
|
||||
"an import is already running; wait for it to finish or cancel it",
|
||||
));
|
||||
}
|
||||
state.running = true;
|
||||
state.cancel = Some(cancel.clone());
|
||||
state.last = None;
|
||||
}
|
||||
|
||||
let core = self.clone();
|
||||
std::thread::spawn(move || {
|
||||
let outcome = core.import_takeout(archives, dry_run, cancel);
|
||||
{
|
||||
let mut state = IMPORT.lock();
|
||||
state.running = false;
|
||||
state.cancel = None;
|
||||
state.last = outcome.as_ref().ok().cloned();
|
||||
}
|
||||
match outcome {
|
||||
Ok(summary) => {
|
||||
let mut parts = vec![format!("{} uploaded", summary.uploaded)];
|
||||
if summary.duplicates > 0 {
|
||||
parts.push(format!("{} already there", summary.duplicates));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
parts.push(format!("{} albums", summary.albums_created));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
parts.push(format!("{} failed", summary.failed));
|
||||
}
|
||||
if summary.cancelled {
|
||||
parts.push("cancelled".to_string());
|
||||
}
|
||||
core.log_activity(
|
||||
ActivityKind::Upload,
|
||||
"Google Photos import",
|
||||
parts.join(" · "),
|
||||
summary.failed == 0 && !summary.cancelled,
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "takeout import failed");
|
||||
core.log_activity(ActivityKind::Upload, "Google Photos import", &e, false);
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Open one of the import's archives by index.
|
||||
fn open_archive(
|
||||
archives: &[PathBuf],
|
||||
index: usize,
|
||||
) -> CoreResult<zip::ZipArchive<std::io::BufReader<std::fs::File>>> {
|
||||
let path = archives
|
||||
.get(index)
|
||||
.ok_or_else(|| CoreError::internal(format!("archive {index} is not in the import")))?;
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
zip::ZipArchive::new(std::io::BufReader::new(file))
|
||||
.map_err(|e| CoreError::invalid(format!("{} is not a readable zip: {e}", path.display())))
|
||||
}
|
||||
|
||||
/// The lowercase-hex SHA-1 of everything `reader` yields — the digest Proton's
|
||||
/// photo duplicate detection is keyed on.
|
||||
fn sha1_of(reader: &mut (dyn Read + Send)) -> CoreResult<String> {
|
||||
let mut hasher = Sha1::new();
|
||||
let mut buffer = vec![0u8; 256 * 1024];
|
||||
loop {
|
||||
let read = reader
|
||||
.read(&mut buffer)
|
||||
.map_err(|e| CoreError::invalid(format!("read failed: {e}")))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
}
|
||||
Ok(hex_lower(&hasher.finalize()))
|
||||
}
|
||||
|
||||
fn hex_lower(bytes: &[u8]) -> String {
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
out.push_str(&format!("{byte:02x}"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pdfs_core::takeout::TakeoutPhoto;
|
||||
|
||||
fn photo(name: &str, size: u64, bucket: Bucket) -> TakeoutPhoto {
|
||||
TakeoutPhoto {
|
||||
archive: 0,
|
||||
entry: format!("Takeout/Google Photos/x/{name}"),
|
||||
name: name.to_string(),
|
||||
size,
|
||||
media_type: "image/jpeg".to_string(),
|
||||
capture_time: None,
|
||||
favorite: false,
|
||||
bucket,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn album_copies_of_one_photo_fold_into_a_single_upload() {
|
||||
let scan = TakeoutScan {
|
||||
photos: vec![
|
||||
photo("IMG_1.jpg", 100, Bucket::Timeline),
|
||||
photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into())),
|
||||
photo("IMG_1.jpg", 100, Bucket::Album("Best of".into())),
|
||||
photo("IMG_2.jpg", 200, Bucket::Timeline),
|
||||
],
|
||||
albums: vec!["Iceland".into(), "Best of".into()],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let planned = plan(&scan);
|
||||
assert_eq!(planned.len(), 2, "the three copies are one upload");
|
||||
assert_eq!(planned[0].albums, vec!["Iceland", "Best of"]);
|
||||
assert!(planned[1].albums.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_name_different_size_stays_two_photos() {
|
||||
let scan = TakeoutScan {
|
||||
photos: vec![
|
||||
photo("IMG_1.jpg", 100, Bucket::Timeline),
|
||||
photo("IMG_1.jpg", 101, Bucket::Album("Iceland".into())),
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(plan(&scan).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_from_any_copy_survives_the_fold() {
|
||||
// Only the album copy carries a sidecar; the fold must not lose it.
|
||||
let mut bare = photo("IMG_1.jpg", 100, Bucket::Timeline);
|
||||
bare.capture_time = None;
|
||||
let mut rich = photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into()));
|
||||
rich.capture_time = Some(1_560_000_000);
|
||||
rich.favorite = true;
|
||||
|
||||
let planned = plan(&TakeoutScan {
|
||||
photos: vec![bare, rich],
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(planned[0].photo.capture_time, Some(1_560_000_000));
|
||||
assert!(planned[0].photo.favorite);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha1_matches_the_known_digest_of_abc() {
|
||||
let mut reader: &[u8] = b"abc";
|
||||
assert_eq!(
|
||||
sha1_of(&mut reader).unwrap(),
|
||||
"a9993e364706816aba3e25717850c26c9cd0d89d"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ use parking_lot::Mutex;
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use pdfs_core::control::{JobItem, TransferDirection, TransferItem};
|
||||
@@ -272,20 +272,50 @@ impl<R: Read> Read for OwnedCountingReader<R> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`Read`] that tallies bytes read through a [`TransferGuard`] (upload path).
|
||||
/// A [`Read`] that tallies bytes read through a [`TransferGuard`] (upload path),
|
||||
/// and can be told to stop.
|
||||
pub struct CountingReader<'a, R> {
|
||||
inner: R,
|
||||
guard: &'a TransferGuard,
|
||||
cancel: Option<Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
impl<'a, R: Read> CountingReader<'a, R> {
|
||||
pub fn new(inner: R, guard: &'a TransferGuard) -> Self {
|
||||
Self { inner, guard }
|
||||
Self {
|
||||
inner,
|
||||
guard,
|
||||
cancel: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Abandon the upload as soon as the SDK asks for its next bytes when
|
||||
/// `cancel` is set.
|
||||
///
|
||||
/// This is the only cancellation point an upload has: the SDK takes a plain
|
||||
/// blocking [`Read`] and pulls from it a block at a time, so refusing to
|
||||
/// hand over the next block is what stops it. Used when a newer revision of
|
||||
/// the same file is queued — the bytes on the wire have already been
|
||||
/// superseded, and finishing them only spends uplink on a revision the next
|
||||
/// op replaces.
|
||||
pub fn with_cancel(mut self, cancel: Arc<AtomicBool>) -> Self {
|
||||
self.cancel = Some(cancel);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for CountingReader<'_, R> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
if let Some(cancel) = &self.cancel
|
||||
&& cancel.load(Ordering::Relaxed)
|
||||
{
|
||||
// Not `Ok(0)`: a short read is how this reader says "end of file",
|
||||
// and the SDK would seal a truncated revision from it.
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Interrupted,
|
||||
"upload superseded by a newer write",
|
||||
));
|
||||
}
|
||||
let n = self.inner.read(buf)?;
|
||||
self.guard.add(n as u64);
|
||||
Ok(n)
|
||||
|
||||
@@ -171,7 +171,7 @@ impl Core {
|
||||
stats.bytes += size;
|
||||
match self.fetch_node(&uid) {
|
||||
Ok(node) => {
|
||||
let mut st = self.state.lock();
|
||||
let mut st = self.state();
|
||||
let ino = st.intern(parent_ino, node);
|
||||
if let Some(kids) = st.children.get_mut(&parent_ino)
|
||||
&& !kids.contains(&ino)
|
||||
@@ -216,7 +216,7 @@ impl Core {
|
||||
self.ensure_children(pino)
|
||||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||||
let existing = {
|
||||
let st = self.state.lock();
|
||||
let st = self.state();
|
||||
st.children.get(&pino).and_then(|kids| {
|
||||
kids.iter().find_map(|ino| {
|
||||
st.entries
|
||||
@@ -243,7 +243,7 @@ impl Core {
|
||||
let node = self
|
||||
.fetch_node(&new_uid)
|
||||
.map_err(|e| self.errno_error(e, "fetch node"))?;
|
||||
let mut st = self.state.lock();
|
||||
let mut st = self.state();
|
||||
let ino = st.intern(pino, node);
|
||||
if let Some(kids) = st.children.get_mut(&pino)
|
||||
&& !kids.contains(&ino)
|
||||
|
||||
+109
-10
@@ -14,6 +14,7 @@ use pages::photos::*;
|
||||
use pages::shared::*;
|
||||
use pages::shared_by_me::*;
|
||||
use pages::status::*;
|
||||
use pages::takeout::*;
|
||||
use pages::trash::*;
|
||||
use pages::verify::*;
|
||||
use widgets::details::*;
|
||||
@@ -46,10 +47,10 @@ use pdfs_core::config::AppDirs;
|
||||
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, DeviceInfo, DirEntry, ErrorKind,
|
||||
InvitationInfo, JobItem, PhotoItem, PhotoKind, PublicLinkInfo, RefreshScope, Request, Response,
|
||||
RestorableFolder, RestoreItem, SearchHit, ShareEntry, ShareEntryKind, SharedItem,
|
||||
SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem, pending_summary,
|
||||
send,
|
||||
ImportSummary, InvitationInfo, JobItem, PhotoItem, PhotoKind, PublicLinkInfo, RefreshScope,
|
||||
Request, Response, RestorableFolder, RestoreItem, SearchHit, ShareEntry, ShareEntryKind,
|
||||
SharedItem, SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem,
|
||||
pending_summary, send,
|
||||
};
|
||||
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
@@ -120,6 +121,7 @@ struct Ui {
|
||||
pub(crate) devices: DevicesState,
|
||||
pub(crate) locations: LocationsState,
|
||||
pub(crate) activity: ActivityState,
|
||||
pub(crate) takeout: TakeoutState,
|
||||
}
|
||||
|
||||
impl Ui {
|
||||
@@ -246,7 +248,10 @@ fn load_proton_theme() {
|
||||
.navigation-sidebar row {{ border-radius: 8px; margin: 2px 6px; }}\n\
|
||||
.navigation-sidebar row:selected {{ background: alpha({PROTON_PURPLE}, 0.16); color: {PROTON_PURPLE}; font-weight: 600; }}\n\
|
||||
.navigation-sidebar row:selected image {{ color: {PROTON_PURPLE}; }}\n\
|
||||
.file-tile:selected, .file-tile:hover:selected {{ background: alpha({PROTON_PURPLE}, 0.20); }}\n"
|
||||
.file-tile:selected, .file-tile:hover:selected {{ background: alpha({PROTON_PURPLE}, 0.20); }}\n\
|
||||
.bulk-bar {{ padding: 6px 8px; border-radius: 12px; background: alpha({PROTON_PURPLE}, 0.12); }}\n\
|
||||
.dropzone {{ border: 2px dashed alpha(currentColor, 0.25); border-radius: 16px; padding: 28px 18px; transition: background 160ms ease, border-color 160ms ease; }}\n\
|
||||
.dropzone-active {{ border-color: {PROTON_PURPLE}; background: alpha({PROTON_PURPLE}, 0.10); }}\n"
|
||||
);
|
||||
let provider = gtk4::CssProvider::new();
|
||||
provider.load_from_string(&css);
|
||||
@@ -285,6 +290,7 @@ fn build_window(app: &adw::Application) {
|
||||
let (locations_page, locations_widgets) = build_locations_page();
|
||||
let (activity_page, activity_widgets) = build_activity_page();
|
||||
let (trash_page, trash_widgets) = build_trash_page();
|
||||
let (takeout_page, takeout_widgets) = build_takeout_page();
|
||||
stack.add_named(&login_page, Some("login"));
|
||||
stack.add_named(&main_page, Some("main"));
|
||||
stack.add_named(&browser_page, Some("browser"));
|
||||
@@ -295,6 +301,7 @@ fn build_window(app: &adw::Application) {
|
||||
stack.add_named(&locations_page, Some("locations"));
|
||||
stack.add_named(&activity_page, Some("activity"));
|
||||
stack.add_named(&trash_page, Some("trash"));
|
||||
stack.add_named(&takeout_page, Some("takeout"));
|
||||
|
||||
// Sidebar: the signed-in destinations. Selecting a row swaps the page stack;
|
||||
// `sync_sidebar` pushes the other way when navigation happens elsewhere (e.g.
|
||||
@@ -362,6 +369,8 @@ fn build_window(app: &adw::Application) {
|
||||
budget_row: main_widgets.budget_row.clone(),
|
||||
mountpoint_row: main_widgets.mountpoint_row.clone(),
|
||||
settings_suppress: Cell::new(false),
|
||||
budget_source: RefCell::new(None),
|
||||
pins_expanded: Cell::new(false),
|
||||
pins_group: main_widgets.pins_group.clone(),
|
||||
pin_rows: RefCell::new(Vec::new()),
|
||||
pins_state: RefCell::new(None),
|
||||
@@ -382,6 +391,13 @@ fn build_window(app: &adw::Application) {
|
||||
upload: browser_widgets.upload.clone(),
|
||||
upload_folder: browser_widgets.upload_folder.clone(),
|
||||
search_source: RefCell::new(None),
|
||||
views: browser_widgets.views.clone(),
|
||||
bulk: browser_widgets.bulk.clone(),
|
||||
bulk_label: browser_widgets.bulk_label.clone(),
|
||||
bulk_trash: browser_widgets.bulk_trash.clone(),
|
||||
bulk_pin: browser_widgets.bulk_pin.clone(),
|
||||
bulk_unpin: browser_widgets.bulk_unpin.clone(),
|
||||
empty_actions: browser_widgets.empty_actions.clone(),
|
||||
},
|
||||
details: DetailsState {
|
||||
details: browser_widgets.details,
|
||||
@@ -407,6 +423,8 @@ fn build_window(app: &adw::Application) {
|
||||
retry: gallery_widgets.retry.clone(),
|
||||
more: gallery_widgets.more.clone(),
|
||||
upload: gallery_widgets.upload.clone(),
|
||||
import: gallery_widgets.import.clone(),
|
||||
empty_actions: gallery_widgets.empty_actions.clone(),
|
||||
title: gallery_widgets.title.clone(),
|
||||
subtitle: gallery_widgets.subtitle.clone(),
|
||||
albums: gallery_widgets.albums.clone(),
|
||||
@@ -494,6 +512,24 @@ fn build_window(app: &adw::Application) {
|
||||
inflight: Cell::new(false),
|
||||
key: RefCell::new(None),
|
||||
},
|
||||
takeout: TakeoutState {
|
||||
archives: RefCell::new(Vec::new()),
|
||||
list_group: takeout_widgets.list_group.clone(),
|
||||
rows: RefCell::new(Vec::new()),
|
||||
dropzone: takeout_widgets.dropzone.clone(),
|
||||
scan_button: takeout_widgets.scan_button.clone(),
|
||||
import_button: takeout_widgets.import_button.clone(),
|
||||
cancel_button: takeout_widgets.cancel_button.clone(),
|
||||
clear_button: takeout_widgets.clear_button.clone(),
|
||||
progress_group: takeout_widgets.progress_group.clone(),
|
||||
progress_label: takeout_widgets.progress_label.clone(),
|
||||
progress_bar: takeout_widgets.progress_bar.clone(),
|
||||
summary_group: takeout_widgets.summary_group.clone(),
|
||||
summary_rows: RefCell::new(Vec::new()),
|
||||
inflight: Cell::new(false),
|
||||
running: Cell::new(false),
|
||||
dry_run: Cell::new(false),
|
||||
},
|
||||
});
|
||||
wire_login(&ui);
|
||||
wire_logout(&ui, &main_widgets.logout_button);
|
||||
@@ -501,9 +537,16 @@ fn build_window(app: &adw::Application) {
|
||||
&ui,
|
||||
&main_widgets.purge_button,
|
||||
&main_widgets.mountpoint_button,
|
||||
&main_widgets.import_row,
|
||||
);
|
||||
wire_sidebar(&ui);
|
||||
wire_browser(&ui, &browser_widgets.grid, &browser_widgets.column_view);
|
||||
wire_bulk(
|
||||
&ui,
|
||||
&browser_widgets.bulk_clear,
|
||||
&browser_widgets.empty_upload,
|
||||
&browser_widgets.empty_new_folder,
|
||||
);
|
||||
wire_browser_actions(
|
||||
&ui,
|
||||
&browser_widgets.new_folder,
|
||||
@@ -513,6 +556,11 @@ fn build_window(app: &adw::Application) {
|
||||
wire_details(&ui);
|
||||
wire_search(&ui);
|
||||
wire_gallery(&ui, &gallery_widgets.list, &gallery_widgets.scroll);
|
||||
wire_gallery_empty(
|
||||
&ui,
|
||||
&gallery_widgets.empty_upload,
|
||||
&gallery_widgets.empty_import,
|
||||
);
|
||||
wire_albums(&ui);
|
||||
wire_trash(&ui, &trash_widgets.list, &trash_widgets.empty);
|
||||
wire_shared(&ui, &shared_widgets.retry, &shared_widgets.add_bookmark);
|
||||
@@ -520,6 +568,7 @@ fn build_window(app: &adw::Application) {
|
||||
wire_devices(&ui, &devices_widgets.retry, &devices_widgets.restore);
|
||||
wire_locations(&ui, &locations_widgets.retry, &locations_widgets.add_folder);
|
||||
wire_activity(&ui, &activity_widgets.retry);
|
||||
wire_takeout(&ui, &takeout_widgets);
|
||||
wire_refresh(
|
||||
&ui,
|
||||
&[
|
||||
@@ -559,6 +608,9 @@ fn build_window(app: &adw::Application) {
|
||||
// every visit to stay live.
|
||||
Some("activity") => load_activity(&ui_nav),
|
||||
Some("trash") => load_trash(&ui_nav),
|
||||
// The import runs in the daemon and outlives this page, so arriving
|
||||
// here asks straight away whether one is in flight.
|
||||
Some("takeout") => refresh_takeout(&ui_nav),
|
||||
_ => {}
|
||||
}
|
||||
});
|
||||
@@ -644,6 +696,17 @@ fn build_sidebar() -> (adw::NavigationPage, gtk4::ListBox) {
|
||||
(page, list)
|
||||
}
|
||||
|
||||
/// The sidebar destination a page belongs under. Most pages are their own
|
||||
/// destination; a sub-page reached from one (Import, off Settings) has no row of
|
||||
/// its own and answers with its parent, so the sidebar highlights where the user
|
||||
/// came from rather than clearing — which would read as "nowhere".
|
||||
fn destination_of(page: &str) -> &str {
|
||||
match page {
|
||||
"takeout" => "main",
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// Selecting a sidebar row navigates the page stack. The reverse direction (stack
|
||||
/// → sidebar highlight) is [`sync_sidebar`], so the two can't fight: this handler
|
||||
/// only ever writes the stack.
|
||||
@@ -654,7 +717,11 @@ fn wire_sidebar(ui: &Rc<Ui>) {
|
||||
let Some((page, _, _)) = DESTINATIONS.get(row.index() as usize) else {
|
||||
return;
|
||||
};
|
||||
if ui_row.stack.visible_child_name().as_deref() != Some(*page) {
|
||||
// Compared by *destination*, not by page: while a sub-page is open its
|
||||
// parent's row is the selected one, and re-asserting it here would kick
|
||||
// the user off the sub-page the moment they arrived.
|
||||
let current = ui_row.stack.visible_child_name();
|
||||
if current.as_deref().map(destination_of) != Some(*page) {
|
||||
ui_row.stack.set_visible_child_name(page);
|
||||
}
|
||||
});
|
||||
@@ -755,6 +822,7 @@ fn sync_sidebar(ui: &Rc<Ui>) {
|
||||
let Some(current) = ui.stack.visible_child_name() else {
|
||||
return;
|
||||
};
|
||||
let current = destination_of(¤t);
|
||||
let index = DESTINATIONS
|
||||
.iter()
|
||||
.position(|(page, _, _)| *page == current);
|
||||
@@ -776,6 +844,22 @@ fn toast(ui: &Rc<Ui>, message: &str) {
|
||||
ui.toasts.add_toast(adw::Toast::new(message));
|
||||
}
|
||||
|
||||
/// Show a toast that offers one follow-up action — an Undo, or a way to the page
|
||||
/// the outcome landed on.
|
||||
///
|
||||
/// Given a longer timeout than a plain toast: an action the user has to notice,
|
||||
/// read and reach is not the same ask as a line of confirmation they can ignore.
|
||||
fn toast_action(ui: &Rc<Ui>, message: &str, label: &str, action: impl Fn(&Rc<Ui>) + 'static) {
|
||||
let toast = adw::Toast::builder()
|
||||
.title(message)
|
||||
.button_label(label)
|
||||
.timeout(8)
|
||||
.build();
|
||||
let ui_action = ui.clone();
|
||||
toast.connect_button_clicked(move |_| action(&ui_action));
|
||||
ui.toasts.add_toast(toast);
|
||||
}
|
||||
|
||||
/// Show a toast for a failure. Same surface as [`toast`], but the message is
|
||||
/// prefixed with what was being attempted, since a bare daemon error ("no such
|
||||
/// file") reads as noise without it.
|
||||
@@ -913,7 +997,13 @@ fn install_shortcuts(ui: &Rc<Ui>, window: &adw::ApplicationWindow) {
|
||||
// browser-only bindings.
|
||||
gtk4::gdk::Key::F5 => reload_current_page(&ui),
|
||||
gtk4::gdk::Key::r | gtk4::gdk::Key::R if ctrl => reload_current_page(&ui),
|
||||
gtk4::gdk::Key::f | gtk4::gdk::Key::F if ctrl && on_browser => {
|
||||
// Search lives on Files, but the shortcut works from anywhere:
|
||||
// wanting to find a file is not a reason to first have to remember
|
||||
// which page owns the search box.
|
||||
gtk4::gdk::Key::f | gtk4::gdk::Key::F if ctrl => {
|
||||
if !on_browser {
|
||||
ui.stack.set_visible_child_name("browser");
|
||||
}
|
||||
ui.browser.search.grab_focus();
|
||||
}
|
||||
gtk4::gdk::Key::n | gtk4::gdk::Key::N if ctrl && on_browser => prompt_new_folder(&ui),
|
||||
@@ -923,12 +1013,21 @@ fn install_shortcuts(ui: &Rc<Ui>, window: &adw::ApplicationWindow) {
|
||||
prompt_rename(&ui, &entry);
|
||||
}
|
||||
}
|
||||
// Acts on the whole selection: Delete with five files highlighted
|
||||
// means those five, not whichever one the details pane happens to
|
||||
// be describing.
|
||||
gtk4::gdk::Key::Delete if on_browser => {
|
||||
if let Some(entry) = selected_entry(&ui) {
|
||||
prompt_delete(&ui, &entry);
|
||||
let entries = selected_entries(&ui);
|
||||
if !entries.is_empty() {
|
||||
prompt_delete_many(&ui, entries);
|
||||
}
|
||||
}
|
||||
gtk4::gdk::Key::Escape if on_browser && ui.browser.split.shows_sidebar() => {
|
||||
gtk4::gdk::Key::Escape
|
||||
if on_browser
|
||||
&& (ui.browser.split.shows_sidebar() || ui.browser.bulk.reveals_child()) =>
|
||||
{
|
||||
clear_selection(&ui);
|
||||
sync_bulk_bar(&ui);
|
||||
hide_details(&ui);
|
||||
}
|
||||
_ => return glib::Propagation::Proceed,
|
||||
|
||||
@@ -31,6 +31,21 @@ pub(crate) struct BrowserState {
|
||||
/// Pending debounce timer for the search box; replaced on every keystroke so
|
||||
/// only the last pause actually fires a [`Request::Search`].
|
||||
pub(crate) search_source: RefCell<Option<glib::SourceId>>,
|
||||
/// The grid/list view stack, read to find out which of the two selections is
|
||||
/// the one the user is actually working in.
|
||||
pub(crate) views: gtk4::Stack,
|
||||
/// The bulk-action bar, revealed once more than one entry is selected.
|
||||
pub(crate) bulk: gtk4::Revealer,
|
||||
pub(crate) bulk_label: gtk4::Label,
|
||||
/// Bulk buttons, sensitive only where they apply (offline state is a
|
||||
/// file-only notion, so a folder in the selection disables those two).
|
||||
pub(crate) bulk_trash: gtk4::Button,
|
||||
pub(crate) bulk_pin: gtk4::Button,
|
||||
pub(crate) bulk_unpin: gtk4::Button,
|
||||
/// Upload / New folder offered on the *empty folder* status page, so that
|
||||
/// state is a place to act rather than a dead end. Hidden on every other
|
||||
/// status (a load error is not the moment to offer an upload).
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
}
|
||||
|
||||
/// Idle pause after the last keystroke before a search query is sent, so typing
|
||||
@@ -76,9 +91,20 @@ pub(crate) struct BrowserWidgets {
|
||||
pub(crate) split: adw::OverlaySplitView,
|
||||
pub(crate) details: DetailsWidgets,
|
||||
/// The two selection models, so a selection change can drive the details pane
|
||||
/// and so an action can re-read the entry the user has highlighted.
|
||||
pub(crate) grid_selection: gtk4::SingleSelection,
|
||||
pub(crate) list_selection: gtk4::SingleSelection,
|
||||
/// and so an action can re-read the entries the user has highlighted.
|
||||
pub(crate) grid_selection: gtk4::MultiSelection,
|
||||
pub(crate) list_selection: gtk4::MultiSelection,
|
||||
/// The grid/list stack, so the page can tell which selection is live.
|
||||
pub(crate) views: gtk4::Stack,
|
||||
pub(crate) bulk: gtk4::Revealer,
|
||||
pub(crate) bulk_label: gtk4::Label,
|
||||
pub(crate) bulk_trash: gtk4::Button,
|
||||
pub(crate) bulk_pin: gtk4::Button,
|
||||
pub(crate) bulk_unpin: gtk4::Button,
|
||||
pub(crate) bulk_clear: gtk4::Button,
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
pub(crate) empty_upload: gtk4::Button,
|
||||
pub(crate) empty_new_folder: gtk4::Button,
|
||||
}
|
||||
|
||||
pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
@@ -166,19 +192,34 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
retry.add_css_class("pill");
|
||||
retry.add_css_class("suggested-action");
|
||||
retry.set_visible(false);
|
||||
// The empty-folder state's way out. `StatusPage` takes one child, so Retry
|
||||
// and these share a box; each is shown only for the state it belongs to.
|
||||
let empty_upload = gtk4::Button::builder().label("Upload files").build();
|
||||
empty_upload.add_css_class("pill");
|
||||
empty_upload.add_css_class("suggested-action");
|
||||
let empty_new_folder = gtk4::Button::builder().label("New folder").build();
|
||||
empty_new_folder.add_css_class("pill");
|
||||
let empty_actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
empty_actions.set_halign(gtk4::Align::Center);
|
||||
empty_actions.set_visible(false);
|
||||
empty_actions.append(&empty_upload);
|
||||
empty_actions.append(&empty_new_folder);
|
||||
|
||||
let status_child = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
status_child.append(&retry);
|
||||
status_child.append(&empty_actions);
|
||||
|
||||
let status = adw::StatusPage::builder()
|
||||
.icon_name("folder-symbolic")
|
||||
.vexpand(true)
|
||||
.child(&retry)
|
||||
.child(&status_child)
|
||||
.build();
|
||||
status.add_css_class("compact");
|
||||
|
||||
// Icon grid.
|
||||
let grid_selection = gtk4::SingleSelection::builder()
|
||||
.model(&model)
|
||||
.autoselect(false)
|
||||
.can_unselect(true)
|
||||
.build();
|
||||
// Icon grid. Multi-select: acting on a batch is the common case for trashing
|
||||
// and for taking a folder's worth of files offline, and doing it one
|
||||
// confirmation dialog at a time is not a workflow.
|
||||
let grid_selection = gtk4::MultiSelection::new(Some(model.clone()));
|
||||
let grid = gtk4::GridView::builder()
|
||||
.model(&grid_selection)
|
||||
.min_columns(2)
|
||||
@@ -190,12 +231,8 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
.child(&grid)
|
||||
.build();
|
||||
|
||||
// Column list.
|
||||
let list_selection = gtk4::SingleSelection::builder()
|
||||
.model(&model)
|
||||
.autoselect(false)
|
||||
.can_unselect(true)
|
||||
.build();
|
||||
// Column list, with its own selection over the same model.
|
||||
let list_selection = gtk4::MultiSelection::new(Some(model.clone()));
|
||||
let column_view = gtk4::ColumnView::builder().model(&list_selection).build();
|
||||
column_view.add_css_class("data-table");
|
||||
let column_scroll = gtk4::ScrolledWindow::builder()
|
||||
@@ -239,12 +276,51 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
.sidebar(&details_pane)
|
||||
.build();
|
||||
|
||||
// Bulk-action bar. A revealer rather than a hidden box so it slides in
|
||||
// instead of making the whole view jump when a second entry is selected.
|
||||
let bulk_label = gtk4::Label::builder().hexpand(true).xalign(0.0).build();
|
||||
let bulk_pin = gtk4::Button::builder()
|
||||
.label("Keep offline")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_pin.add_css_class("flat");
|
||||
let bulk_unpin = gtk4::Button::builder()
|
||||
.label("Remove offline copy")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_unpin.add_css_class("flat");
|
||||
let bulk_trash = gtk4::Button::builder()
|
||||
.label("Move to Trash")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_trash.add_css_class("destructive-action");
|
||||
let bulk_clear = gtk4::Button::builder()
|
||||
.icon_name("window-close-symbolic")
|
||||
.tooltip_text("Clear selection (Esc)")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_clear.add_css_class("flat");
|
||||
bulk_clear.add_css_class("circular");
|
||||
let bulk_box = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
bulk_box.add_css_class("toolbar");
|
||||
bulk_box.add_css_class("bulk-bar");
|
||||
bulk_box.append(&bulk_label);
|
||||
bulk_box.append(&bulk_pin);
|
||||
bulk_box.append(&bulk_unpin);
|
||||
bulk_box.append(&bulk_trash);
|
||||
bulk_box.append(&bulk_clear);
|
||||
let bulk = gtk4::Revealer::builder()
|
||||
.transition_type(gtk4::RevealerTransitionType::SlideDown)
|
||||
.child(&bulk_box)
|
||||
.build();
|
||||
|
||||
let inner = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
inner.set_margin_top(12);
|
||||
inner.set_margin_bottom(12);
|
||||
inner.set_margin_start(12);
|
||||
inner.set_margin_end(12);
|
||||
inner.append(&header);
|
||||
inner.append(&bulk);
|
||||
inner.append(&split);
|
||||
|
||||
(
|
||||
@@ -267,6 +343,16 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
details,
|
||||
grid_selection,
|
||||
list_selection,
|
||||
views: view_stack,
|
||||
bulk,
|
||||
bulk_label,
|
||||
bulk_trash,
|
||||
bulk_pin,
|
||||
bulk_unpin,
|
||||
bulk_clear,
|
||||
empty_actions,
|
||||
empty_upload,
|
||||
empty_new_folder,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -278,10 +364,273 @@ pub(crate) fn browser_status(ui: &Rc<Ui>, icon: &str, title: &str, description:
|
||||
ui.browser.status.set_title(title);
|
||||
ui.browser.status.set_description(Some(description));
|
||||
ui.browser.retry.set_visible(retry);
|
||||
// Only the empty-folder state offers a way to fill the folder; every other
|
||||
// status (loading, offline, error) turns them back off.
|
||||
ui.browser.empty_actions.set_visible(false);
|
||||
ui.browser.content.set_visible_child_name("status");
|
||||
hide_details(ui);
|
||||
}
|
||||
|
||||
/// The selection model of whichever view is on screen. The grid and the list
|
||||
/// each own one over the same model, so "what is selected" depends on which the
|
||||
/// user is looking at.
|
||||
pub(crate) fn active_selection(ui: &Rc<Ui>) -> gtk4::MultiSelection {
|
||||
if ui.browser.views.visible_child_name().as_deref() == Some("list") {
|
||||
ui.details.list_selection.clone()
|
||||
} else {
|
||||
ui.details.grid_selection.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Every entry highlighted in the view on screen, in model order.
|
||||
///
|
||||
/// Walks the model rather than the selection bitset: a listing is at most a few
|
||||
/// thousand rows, and asking each position whether it is selected keeps this
|
||||
/// free of bitset-iterator lifetimes for no measurable cost.
|
||||
pub(crate) fn selected_entries(ui: &Rc<Ui>) -> Vec<DirEntry> {
|
||||
let selection = active_selection(ui);
|
||||
let count = selection.n_items();
|
||||
(0..count)
|
||||
.filter(|i| selection.is_selected(*i))
|
||||
.filter_map(|i| entry_at(Some(&selection), i))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Drop the selection in both views, which also retracts the bulk bar.
|
||||
pub(crate) fn clear_selection(ui: &Rc<Ui>) {
|
||||
ui.details.grid_selection.unselect_all();
|
||||
ui.details.list_selection.unselect_all();
|
||||
}
|
||||
|
||||
/// Reflect the current selection in the bulk bar: how many are selected, and
|
||||
/// which bulk actions apply to that mix. Offline state is a file-only notion, so
|
||||
/// a folder anywhere in the selection disables the two pin buttons rather than
|
||||
/// half-failing once pressed.
|
||||
pub(crate) fn sync_bulk_bar(ui: &Rc<Ui>) {
|
||||
let entries = selected_entries(ui);
|
||||
// One selected entry is the details pane's job; the bar is for a batch.
|
||||
if entries.len() < 2 {
|
||||
ui.browser.bulk.set_reveal_child(false);
|
||||
return;
|
||||
}
|
||||
let files_only = entries.iter().all(|e| !e.is_dir);
|
||||
let any_pinned = entries.iter().any(|e| e.pinned);
|
||||
let any_unpinned = entries.iter().any(|e| !e.pinned);
|
||||
let mounted = *ui.mounted.borrow();
|
||||
ui.browser
|
||||
.bulk_label
|
||||
.set_label(&format!("{} selected", entries.len()));
|
||||
ui.browser.bulk_trash.set_sensitive(mounted);
|
||||
ui.browser
|
||||
.bulk_pin
|
||||
.set_sensitive(mounted && files_only && any_unpinned);
|
||||
ui.browser
|
||||
.bulk_unpin
|
||||
.set_sensitive(mounted && files_only && any_pinned);
|
||||
ui.browser.bulk.set_reveal_child(true);
|
||||
}
|
||||
|
||||
/// Confirm and move every selected entry to Trash.
|
||||
///
|
||||
/// One dialog for the batch, one pass over the daemon, one toast — and one Undo
|
||||
/// that restores the whole batch, because a mis-aimed bulk delete is exactly the
|
||||
/// action a user most needs to take back.
|
||||
pub(crate) fn prompt_delete_many(ui: &Rc<Ui>, entries: Vec<DirEntry>) {
|
||||
if entries.len() < 2 {
|
||||
if let Some(entry) = entries.first() {
|
||||
prompt_delete(ui, entry);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let win = ui_window(ui);
|
||||
let dialog = adw::AlertDialog::builder()
|
||||
.heading("Move to Trash")
|
||||
.body(format!(
|
||||
"Move {} items to Trash? Anything inside a selected folder goes with it.",
|
||||
entries.len()
|
||||
))
|
||||
.build();
|
||||
dialog.add_response("cancel", "Cancel");
|
||||
dialog.add_response("trash", "Move to Trash");
|
||||
dialog.set_response_appearance("trash", adw::ResponseAppearance::Destructive);
|
||||
dialog.set_default_response(Some("cancel"));
|
||||
dialog.set_close_response("cancel");
|
||||
|
||||
let ui = ui.clone();
|
||||
dialog.connect_response(None, move |_, resp| {
|
||||
if resp == "trash" {
|
||||
run_bulk_delete(&ui, entries.clone());
|
||||
}
|
||||
});
|
||||
dialog.present(win.as_ref());
|
||||
}
|
||||
|
||||
/// Trash a batch, one request at a time.
|
||||
///
|
||||
/// Sequential rather than parallel: the daemon serialises these anyway, and a
|
||||
/// burst of threads against one socket buys nothing but a harder failure to
|
||||
/// report. Partial success is reported as such — the entries that did move are
|
||||
/// still restorable through the Undo.
|
||||
pub(crate) fn run_bulk_delete(ui: &Rc<Ui>, entries: Vec<DirEntry>) {
|
||||
if !*ui.mounted.borrow() {
|
||||
toast_error(
|
||||
ui,
|
||||
"Couldn't move to Trash",
|
||||
"Proton Drive isn't connected.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
let socket = ui.dirs.control_socket();
|
||||
// A one-entry batch is still a batch, but it is worth naming in the toast:
|
||||
// "Moved “notes.txt” to Trash" tells the user which file far better than a
|
||||
// count of one does.
|
||||
let single = (entries.len() == 1).then(|| entries[0].name.clone());
|
||||
let paths: Vec<(String, String)> = entries
|
||||
.iter()
|
||||
.map(|e| (entry_rel(ui, e), e.uid.clone()))
|
||||
.collect();
|
||||
ui.busy_begin();
|
||||
clear_selection(ui);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let mut trashed: Vec<String> = Vec::new();
|
||||
let mut failure: Option<String> = None;
|
||||
for (path, uid) in paths {
|
||||
let rx = spawn_request(socket.clone(), Request::Delete { path });
|
||||
match rx.recv().await {
|
||||
Ok(Ok(Response::Ok { .. })) => trashed.push(uid),
|
||||
Ok(Ok(Response::Error { message, .. })) => {
|
||||
failure.get_or_insert(message);
|
||||
}
|
||||
_ => {
|
||||
failure.get_or_insert_with(|| "The mount service didn't respond.".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ui.busy_end();
|
||||
reload_listing(&ui);
|
||||
match (trashed.len(), failure) {
|
||||
(0, Some(message)) => toast_error(&ui, "Couldn't move to Trash", &message),
|
||||
(0, None) => {}
|
||||
(n, failure) => {
|
||||
let message = match (failure, &single) {
|
||||
(Some(_), _) => format!("Moved {n} items to Trash — some couldn't be moved"),
|
||||
(None, Some(name)) => format!("Moved “{name}” to Trash"),
|
||||
(None, None) => format!("Moved {n} items to Trash"),
|
||||
};
|
||||
toast_action(&ui, &message, "Undo", move |ui| {
|
||||
restore_uids(ui, trashed.clone(), n);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Put trashed nodes back where they came from — the Undo behind a trash toast.
|
||||
pub(crate) fn restore_uids(ui: &Rc<Ui>, uids: Vec<String>, count: usize) {
|
||||
run_mutation(
|
||||
ui,
|
||||
Request::Restore { uids },
|
||||
if count == 1 {
|
||||
"Restored from Trash".to_string()
|
||||
} else {
|
||||
format!("Restored {count} items from Trash")
|
||||
},
|
||||
"Couldn't restore from Trash",
|
||||
);
|
||||
}
|
||||
|
||||
/// Pin or unpin every selected file, one request at a time.
|
||||
pub(crate) fn run_bulk_pin(ui: &Rc<Ui>, entries: Vec<DirEntry>, pin: bool) {
|
||||
let socket = ui.dirs.control_socket();
|
||||
// Only the entries that need changing: re-pinning an already-pinned file is
|
||||
// a wasted round-trip, and it would inflate the count the toast reports.
|
||||
let paths: Vec<String> = entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.pinned != pin)
|
||||
.map(|e| entry_rel(ui, e))
|
||||
.collect();
|
||||
if paths.is_empty() {
|
||||
return;
|
||||
}
|
||||
ui.busy_begin();
|
||||
clear_selection(ui);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let mut done = 0usize;
|
||||
let mut failure: Option<String> = None;
|
||||
for path in paths {
|
||||
let req = if pin {
|
||||
Request::Pin { path }
|
||||
} else {
|
||||
Request::Unpin { path }
|
||||
};
|
||||
let rx = spawn_request(socket.clone(), req);
|
||||
match rx.recv().await {
|
||||
Ok(Ok(Response::Ok { .. })) => done += 1,
|
||||
Ok(Ok(Response::Error { message, .. })) => {
|
||||
failure.get_or_insert(message);
|
||||
}
|
||||
_ => {
|
||||
failure.get_or_insert_with(|| "The mount service didn't respond.".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ui.busy_end();
|
||||
load_browser(&ui);
|
||||
match (done, failure) {
|
||||
(0, Some(message)) => toast_error(&ui, "Couldn't change offline state", &message),
|
||||
(0, None) => {}
|
||||
(n, _) => toast(
|
||||
&ui,
|
||||
&if pin {
|
||||
format!("{n} files are now available offline")
|
||||
} else {
|
||||
format!("{n} files are no longer kept offline")
|
||||
},
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Wire the bulk-action bar and the empty-folder state's buttons.
|
||||
pub(crate) fn wire_bulk(
|
||||
ui: &Rc<Ui>,
|
||||
bulk_clear: >k4::Button,
|
||||
empty_upload: >k4::Button,
|
||||
empty_new_folder: >k4::Button,
|
||||
) {
|
||||
let ui_trash = ui.clone();
|
||||
ui.browser
|
||||
.bulk_trash
|
||||
.connect_clicked(move |_| prompt_delete_many(&ui_trash, selected_entries(&ui_trash)));
|
||||
let ui_pin = ui.clone();
|
||||
ui.browser
|
||||
.bulk_pin
|
||||
.connect_clicked(move |_| run_bulk_pin(&ui_pin, selected_entries(&ui_pin), true));
|
||||
let ui_unpin = ui.clone();
|
||||
ui.browser
|
||||
.bulk_unpin
|
||||
.connect_clicked(move |_| run_bulk_pin(&ui_unpin, selected_entries(&ui_unpin), false));
|
||||
let ui_clear = ui.clone();
|
||||
bulk_clear.connect_clicked(move |_| {
|
||||
clear_selection(&ui_clear);
|
||||
sync_bulk_bar(&ui_clear);
|
||||
});
|
||||
|
||||
// Switching views switches which selection is live, so the bar has to be
|
||||
// re-derived rather than left showing the other view's count.
|
||||
let ui_view = ui.clone();
|
||||
ui.browser
|
||||
.views
|
||||
.connect_visible_child_name_notify(move |_| sync_bulk_bar(&ui_view));
|
||||
|
||||
let ui_upload = ui.clone();
|
||||
empty_upload.connect_clicked(move |_| prompt_upload(&ui_upload));
|
||||
let ui_new = ui.clone();
|
||||
empty_new_folder.connect_clicked(move |_| prompt_new_folder(&ui_new));
|
||||
}
|
||||
|
||||
/// Swap the Files content area back to the grid/list views.
|
||||
pub(crate) fn browser_views(ui: &Rc<Ui>) {
|
||||
ui.browser.content.set_visible_child_name("views");
|
||||
@@ -474,7 +823,14 @@ pub(crate) fn attach_context_menu(ui: &Rc<Ui>, item: >k4::ListItem, anchor: &g
|
||||
gesture.connect_pressed(move |_, _, x, y| {
|
||||
if let Some(obj) = item.item().and_downcast::<BoxedAnyObject>() {
|
||||
let entry = obj.borrow::<DirEntry>().clone();
|
||||
show_context_menu(&ui, &entry, &target, x, y);
|
||||
// Right-clicking a row that is part of a multi-selection acts on the
|
||||
// batch; right-clicking outside one acts on the row, as before.
|
||||
let selected = selected_entries(&ui);
|
||||
if selected.len() > 1 && selected.iter().any(|e| e.uid == entry.uid) {
|
||||
show_bulk_context_menu(&ui, selected, &target, x, y);
|
||||
} else {
|
||||
show_context_menu(&ui, &entry, &target, x, y);
|
||||
}
|
||||
}
|
||||
});
|
||||
anchor.add_controller(gesture);
|
||||
@@ -588,6 +944,75 @@ pub(crate) fn show_context_menu(ui: &Rc<Ui>, entry: &DirEntry, anchor: >k4::Bo
|
||||
popover.popup();
|
||||
}
|
||||
|
||||
/// The context menu for a multi-selection: the batch-capable actions only.
|
||||
/// Rename, Move, Share and the revision history all need a single subject, so
|
||||
/// they are simply absent here rather than offered and then refused.
|
||||
pub(crate) fn show_bulk_context_menu(
|
||||
ui: &Rc<Ui>,
|
||||
entries: Vec<DirEntry>,
|
||||
anchor: >k4::Box,
|
||||
x: f64,
|
||||
y: f64,
|
||||
) {
|
||||
let menu = gtk4::Box::new(gtk4::Orientation::Vertical, 0);
|
||||
let popover = gtk4::Popover::builder()
|
||||
.has_arrow(false)
|
||||
.position(gtk4::PositionType::Bottom)
|
||||
.pointing_to(>k4::gdk::Rectangle::new(x as i32, y as i32, 1, 1))
|
||||
.child(&menu)
|
||||
.build();
|
||||
popover.set_parent(anchor);
|
||||
popover.connect_closed(|p| p.unparent());
|
||||
|
||||
let header = gtk4::Label::builder()
|
||||
.label(format!("{} selected", entries.len()))
|
||||
.halign(gtk4::Align::Start)
|
||||
.build();
|
||||
header.add_css_class("dim-label");
|
||||
header.add_css_class("caption");
|
||||
header.set_margin_top(6);
|
||||
header.set_margin_bottom(4);
|
||||
header.set_margin_start(10);
|
||||
menu.append(&header);
|
||||
|
||||
let files_only = entries.iter().all(|e| !e.is_dir);
|
||||
if files_only && entries.iter().any(|e| !e.pinned) {
|
||||
let pin = menu_item("Keep offline", "starred-symbolic");
|
||||
let ui_pin = ui.clone();
|
||||
let batch = entries.clone();
|
||||
let pop = popover.clone();
|
||||
pin.connect_clicked(move |_| {
|
||||
pop.popdown();
|
||||
run_bulk_pin(&ui_pin, batch.clone(), true);
|
||||
});
|
||||
menu.append(&pin);
|
||||
}
|
||||
if files_only && entries.iter().any(|e| e.pinned) {
|
||||
let unpin = menu_item("Remove offline copy", "non-starred-symbolic");
|
||||
let ui_unpin = ui.clone();
|
||||
let batch = entries.clone();
|
||||
let pop = popover.clone();
|
||||
unpin.connect_clicked(move |_| {
|
||||
pop.popdown();
|
||||
run_bulk_pin(&ui_unpin, batch.clone(), false);
|
||||
});
|
||||
menu.append(&unpin);
|
||||
}
|
||||
|
||||
menu.append(>k4::Separator::new(gtk4::Orientation::Horizontal));
|
||||
|
||||
let trash = menu_item("Move to Trash", "user-trash-symbolic");
|
||||
let ui_tr = ui.clone();
|
||||
let pop = popover.clone();
|
||||
trash.connect_clicked(move |_| {
|
||||
pop.popdown();
|
||||
prompt_delete_many(&ui_tr, entries.clone());
|
||||
});
|
||||
menu.append(&trash);
|
||||
|
||||
popover.popup();
|
||||
}
|
||||
|
||||
/// A left-aligned, flat icon+label button for the context menu.
|
||||
pub(crate) fn menu_item(label: &str, icon: &str) -> gtk4::Button {
|
||||
let row = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
@@ -976,8 +1401,6 @@ pub(crate) fn prompt_move(ui: &Rc<Ui>, entry: &DirEntry) {
|
||||
/// Confirm and move the entry to Trash through the daemon.
|
||||
pub(crate) fn prompt_delete(ui: &Rc<Ui>, entry: &DirEntry) {
|
||||
let win = ui_window(ui);
|
||||
let rel = entry_rel(ui, entry);
|
||||
let name = entry.name.clone();
|
||||
let dialog = adw::AlertDialog::builder()
|
||||
.heading("Move to Trash")
|
||||
.body(format!("Move “{}” to Trash?", entry.name))
|
||||
@@ -988,15 +1411,13 @@ pub(crate) fn prompt_delete(ui: &Rc<Ui>, entry: &DirEntry) {
|
||||
dialog.set_default_response(Some("cancel"));
|
||||
dialog.set_close_response("cancel");
|
||||
|
||||
// Routed through the batch path so a single trash gets the same Undo: one
|
||||
// mis-click on one file is no less worth taking back than fifty.
|
||||
let entry = entry.clone();
|
||||
let ui = ui.clone();
|
||||
dialog.connect_response(None, move |_, resp| {
|
||||
if resp == "trash" {
|
||||
run_mutation(
|
||||
&ui,
|
||||
Request::Delete { path: rel.clone() },
|
||||
format!("Moved “{name}” to Trash"),
|
||||
"Couldn't move to Trash",
|
||||
);
|
||||
run_bulk_delete(&ui, vec![entry.clone()]);
|
||||
}
|
||||
});
|
||||
dialog.present(win.as_ref());
|
||||
@@ -1346,6 +1767,7 @@ pub(crate) fn repaint_browser(ui: &Rc<Ui>, entries: &[DirEntry]) {
|
||||
"Upload a file or create a folder to get started.",
|
||||
false,
|
||||
);
|
||||
ui.browser.empty_actions.set_visible(true);
|
||||
return;
|
||||
}
|
||||
browser_views(ui);
|
||||
|
||||
@@ -9,5 +9,6 @@ pub(crate) mod photos;
|
||||
pub(crate) mod shared;
|
||||
pub(crate) mod shared_by_me;
|
||||
pub(crate) mod status;
|
||||
pub(crate) mod takeout;
|
||||
pub(crate) mod trash;
|
||||
pub(crate) mod verify;
|
||||
@@ -19,6 +19,12 @@ pub(crate) struct GalleryState {
|
||||
pub(crate) retry: gtk4::Button,
|
||||
pub(crate) more: gtk4::Button,
|
||||
pub(crate) upload: gtk4::Button,
|
||||
/// Opens the Google Photos Takeout import chooser.
|
||||
pub(crate) import: gtk4::Button,
|
||||
/// Upload / Import offered on the *empty timeline* status page, so a fresh
|
||||
/// account is a place to start rather than a dead end. Hidden on every other
|
||||
/// status: a load error is not the moment to offer an upload.
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
/// "Photos", or the album's name while one is open.
|
||||
pub(crate) title: gtk4::Label,
|
||||
/// "1,204 photos" under the page title.
|
||||
@@ -177,6 +183,10 @@ pub(crate) struct GalleryWidgets {
|
||||
pub(crate) scroll: gtk4::ScrolledWindow,
|
||||
pub(crate) retry: gtk4::Button,
|
||||
pub(crate) upload: gtk4::Button,
|
||||
pub(crate) import: gtk4::Button,
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
pub(crate) empty_upload: gtk4::Button,
|
||||
pub(crate) empty_import: gtk4::Button,
|
||||
pub(crate) refresh: gtk4::Button,
|
||||
/// The Albums grid, its own status page and the stack between them, plus the
|
||||
/// Photos/Albums switcher and the back button out of an album.
|
||||
@@ -227,10 +237,29 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) {
|
||||
retry.add_css_class("suggested-action");
|
||||
retry.set_visible(false);
|
||||
|
||||
// The empty-timeline state's two ways forward. `StatusPage` takes one child,
|
||||
// so Retry and these share a box and each is shown only for its own state.
|
||||
let empty_upload = gtk4::Button::builder().label("Upload photos").build();
|
||||
empty_upload.add_css_class("pill");
|
||||
empty_upload.add_css_class("suggested-action");
|
||||
let empty_import = gtk4::Button::builder()
|
||||
.label("Import from Google Photos")
|
||||
.build();
|
||||
empty_import.add_css_class("pill");
|
||||
let empty_actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
empty_actions.set_halign(gtk4::Align::Center);
|
||||
empty_actions.set_visible(false);
|
||||
empty_actions.append(&empty_upload);
|
||||
empty_actions.append(&empty_import);
|
||||
|
||||
let status_child = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
status_child.append(&retry);
|
||||
status_child.append(&empty_actions);
|
||||
|
||||
let status = adw::StatusPage::builder()
|
||||
.icon_name("image-x-generic-symbolic")
|
||||
.vexpand(true)
|
||||
.child(&retry)
|
||||
.child(&status_child)
|
||||
.build();
|
||||
status.add_css_class("compact");
|
||||
|
||||
@@ -288,12 +317,23 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) {
|
||||
.build();
|
||||
upload.add_css_class("pill");
|
||||
upload.add_css_class("suggested-action");
|
||||
|
||||
// Importing an export is a rarer, heavier action than adding one photo, so
|
||||
// it sits beside Upload as a plain button rather than a second accent one.
|
||||
let import = gtk4::Button::builder()
|
||||
.icon_name("folder-download-symbolic")
|
||||
.tooltip_text("Import a Google Photos Takeout export")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
import.add_css_class("flat");
|
||||
import.add_css_class("circular");
|
||||
let refresh = refresh_button();
|
||||
|
||||
let header_box = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
header_box.append(&back);
|
||||
header_box.append(&titles);
|
||||
header_box.append(&refresh);
|
||||
header_box.append(&import);
|
||||
header_box.append(&upload);
|
||||
|
||||
// All / Photos / Videos / Raw filter. Linked toggles acting as one segmented
|
||||
@@ -428,6 +468,10 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) {
|
||||
scroll,
|
||||
retry,
|
||||
upload,
|
||||
import,
|
||||
empty_actions,
|
||||
empty_upload,
|
||||
empty_import,
|
||||
refresh,
|
||||
tabs,
|
||||
favorites_btn,
|
||||
@@ -554,6 +598,15 @@ pub(crate) fn thousands(n: usize) -> String {
|
||||
/// Wire the gallery: install the section factory, the zoom gestures, the pager
|
||||
/// and the upload button. Activating a thumbnail downloads the photo and opens it
|
||||
/// in the in-app lightbox.
|
||||
/// Wire the empty-timeline buttons. Upload re-fires the header's own Upload
|
||||
/// rather than duplicating its file-chooser, so the two can't drift apart.
|
||||
pub(crate) fn wire_gallery_empty(ui: &Rc<Ui>, upload: >k4::Button, import: >k4::Button) {
|
||||
let ui_upload = ui.clone();
|
||||
upload.connect_clicked(move |_| ui_upload.gallery.upload.emit_clicked());
|
||||
let ui_import = ui.clone();
|
||||
import.connect_clicked(move |_| ui_import.stack.set_visible_child_name("takeout"));
|
||||
}
|
||||
|
||||
pub(crate) fn wire_gallery(ui: &Rc<Ui>, list: >k4::ListView, scroll: >k4::ScrolledWindow) {
|
||||
let factory = gtk4::SignalListItemFactory::new();
|
||||
factory.connect_setup(|_, item| {
|
||||
@@ -729,6 +782,14 @@ pub(crate) fn wire_gallery(ui: &Rc<Ui>, list: >k4::ListView, scroll: >k4::Sc
|
||||
load_gallery(&ui_dates, false);
|
||||
});
|
||||
|
||||
// The import is a staged, hours-long migration, so the button hands over to
|
||||
// the Import page rather than opening a file chooser here: the archives need
|
||||
// reviewing before anything is sent, and the run needs somewhere to report.
|
||||
let ui_import = ui.clone();
|
||||
ui.gallery
|
||||
.import
|
||||
.connect_clicked(move |_| ui_import.stack.set_visible_child_name("takeout"));
|
||||
|
||||
let ui_upload = ui.clone();
|
||||
ui.gallery.upload.connect_clicked(move |_| {
|
||||
let dialog = gtk4::FileDialog::builder()
|
||||
@@ -1460,6 +1521,12 @@ pub(crate) fn load_gallery(ui: &Rc<Ui>, append: bool) {
|
||||
)
|
||||
};
|
||||
gallery_status(&ui, "image-x-generic-symbolic", title, description, false);
|
||||
// Only the whole-timeline empty state: an empty *album* is
|
||||
// filled by adding existing photos to it, not by uploading
|
||||
// into it, so the buttons would point the wrong way.
|
||||
ui.gallery
|
||||
.empty_actions
|
||||
.set_visible(ui.gallery.album.borrow().is_none());
|
||||
return;
|
||||
}
|
||||
ui.gallery.content.set_visible_child_name("timeline");
|
||||
@@ -1502,6 +1569,9 @@ pub(crate) fn gallery_status(ui: &Rc<Ui>, icon: &str, title: &str, description:
|
||||
ui.gallery.status.set_title(title);
|
||||
ui.gallery.status.set_description(Some(description));
|
||||
ui.gallery.retry.set_visible(retry);
|
||||
// Only the empty timeline offers a way to fill it; every other status turns
|
||||
// those buttons back off.
|
||||
ui.gallery.empty_actions.set_visible(false);
|
||||
ui.gallery.more.set_visible(false);
|
||||
ui.gallery.content.set_visible_child_name("status");
|
||||
}
|
||||
|
||||
@@ -37,7 +37,16 @@ pub(crate) struct StatusState {
|
||||
/// Set while a settings widget is being populated programmatically, so its
|
||||
/// change handler skips the IPC/systemd side effect.
|
||||
pub(crate) settings_suppress: Cell<bool>,
|
||||
/// Pending debounce for the cache-budget editor. Each `+` click is a value
|
||||
/// change, and firing a `SetCacheBudget` (and a toast) per click means a
|
||||
/// stack of toasts and a series of caps the user never asked to apply — so
|
||||
/// only the value they settle on is sent.
|
||||
pub(crate) budget_source: RefCell<Option<glib::SourceId>>,
|
||||
pub(crate) pins_group: adw::PreferencesGroup,
|
||||
/// Whether the pin list is showing every pin or only the first
|
||||
/// [`PINS_COLLAPSED`]. A long pin list would otherwise push everything below
|
||||
/// it — including the Developer group — off the end of the page.
|
||||
pub(crate) pins_expanded: Cell<bool>,
|
||||
/// Rows currently shown under [`Self::pins_group`], retained so a refresh can
|
||||
/// diff against them and only rebuild when the pin set actually changes.
|
||||
pub(crate) pin_rows: RefCell<Vec<PinRow>>,
|
||||
@@ -104,6 +113,8 @@ pub(crate) struct MainWidgets {
|
||||
/// which is where the folder is changed.
|
||||
pub(crate) mountpoint_row: adw::ActionRow,
|
||||
pub(crate) mountpoint_button: gtk4::Button,
|
||||
/// Opens the Google Photos Takeout import page.
|
||||
pub(crate) import_row: adw::ActionRow,
|
||||
}
|
||||
|
||||
/// The main (logged-in) page: a libadwaita settings surface — account header,
|
||||
@@ -233,6 +244,22 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) {
|
||||
mountpoint_row.add_suffix(&mountpoint_button);
|
||||
system_group.add(&mountpoint_row);
|
||||
|
||||
// Import: a one-off migration rather than a setting, so it is a doorway to
|
||||
// its own page rather than controls inlined here — staging a set of archives
|
||||
// and watching an hours-long upload needs the room.
|
||||
let import_group = adw::PreferencesGroup::builder()
|
||||
.title("Import")
|
||||
.description("Bring a photo library from another service into Proton Photos.")
|
||||
.build();
|
||||
let import_row = adw::ActionRow::builder()
|
||||
.title("Import from Google Photos")
|
||||
.subtitle("Add a Google Takeout export to your timeline.")
|
||||
.activatable(true)
|
||||
.build();
|
||||
import_row.add_prefix(>k4::Image::from_icon_name("folder-download-symbolic"));
|
||||
import_row.add_suffix(>k4::Image::from_icon_name("go-next-symbolic"));
|
||||
import_group.add(&import_row);
|
||||
|
||||
// Pins group: filled in by refresh.
|
||||
let pins_group = adw::PreferencesGroup::builder()
|
||||
.title("Pinned files")
|
||||
@@ -265,6 +292,7 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) {
|
||||
inner.append(&transfers_group);
|
||||
inner.append(&storage_group);
|
||||
inner.append(&system_group);
|
||||
inner.append(&import_group);
|
||||
inner.append(&pins_group);
|
||||
inner.append(&dev_group);
|
||||
|
||||
@@ -292,10 +320,20 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) {
|
||||
purge_button,
|
||||
mountpoint_row,
|
||||
mountpoint_button,
|
||||
import_row,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// How many pins the Settings list shows before collapsing the rest behind a
|
||||
/// "Show all" row. Enough to recognise the list at a glance; short enough that
|
||||
/// the groups below it stay reachable.
|
||||
pub(crate) const PINS_COLLAPSED: usize = 6;
|
||||
|
||||
/// How long the cache-budget editor waits after the last change before applying
|
||||
/// it, so a run of `+` clicks is one request rather than one per click.
|
||||
pub(crate) const BUDGET_DEBOUNCE: Duration = Duration::from_millis(600);
|
||||
|
||||
/// Bytes per GiB, for the cache-budget editor's unit conversion.
|
||||
pub(crate) const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
|
||||
|
||||
@@ -308,6 +346,7 @@ pub(crate) fn wire_settings(
|
||||
ui: &Rc<Ui>,
|
||||
purge_button: >k4::Button,
|
||||
mountpoint_button: >k4::Button,
|
||||
import_row: &adw::ActionRow,
|
||||
) {
|
||||
let config = ui.dirs.load_config();
|
||||
|
||||
@@ -329,13 +368,22 @@ pub(crate) fn wire_settings(
|
||||
if ui_budget.status.settings_suppress.get() {
|
||||
return;
|
||||
}
|
||||
// Replace any pending apply, so only the value the user stops on is sent.
|
||||
if let Some(src) = ui_budget.status.budget_source.borrow_mut().take() {
|
||||
src.remove();
|
||||
}
|
||||
let bytes = (row.value() * GIB).round() as u64;
|
||||
settings_request(
|
||||
&ui_budget,
|
||||
Request::SetCacheBudget { bytes },
|
||||
"Cache budget updated",
|
||||
"Couldn't set cache budget",
|
||||
);
|
||||
let ui_fire = ui_budget.clone();
|
||||
let src = glib::timeout_add_local_once(BUDGET_DEBOUNCE, move || {
|
||||
ui_fire.status.budget_source.borrow_mut().take();
|
||||
settings_request(
|
||||
&ui_fire,
|
||||
Request::SetCacheBudget { bytes },
|
||||
"Cache budget updated",
|
||||
"Couldn't set cache budget",
|
||||
);
|
||||
});
|
||||
*ui_budget.status.budget_source.borrow_mut() = Some(src);
|
||||
});
|
||||
|
||||
// Purge: confirm, then drop all unpinned cached content via the daemon.
|
||||
@@ -382,6 +430,10 @@ pub(crate) fn wire_settings(
|
||||
// local path; this button is the way there.
|
||||
let ui_mp = ui.clone();
|
||||
mountpoint_button.connect_clicked(move |_| ui_mp.stack.set_visible_child_name("locations"));
|
||||
|
||||
// Import: a doorway, not a setting — everything about it lives on its page.
|
||||
let ui_import = ui.clone();
|
||||
import_row.connect_activated(move |_| ui_import.stack.set_visible_child_name("takeout"));
|
||||
}
|
||||
|
||||
/// Run a settings control-socket round-trip (budget / purge) on a worker thread,
|
||||
@@ -513,6 +565,12 @@ pub(crate) fn refresh(ui: &Rc<Ui>) {
|
||||
refresh_transfers(ui);
|
||||
// Both of these pages show work as it happens, so they follow the tick while
|
||||
// they are on screen. Every other page loads on navigation only.
|
||||
// A Takeout import outlives the page that started it, so it is polled
|
||||
// wherever the user has navigated to — otherwise leaving the page means
|
||||
// never being told it finished.
|
||||
if ui.stack.visible_child_name().as_deref() == Some("takeout") || ui.takeout.running.get() {
|
||||
refresh_takeout(ui);
|
||||
}
|
||||
match ui.stack.visible_child_name().as_deref() {
|
||||
Some("main") => refresh_quota(ui),
|
||||
Some("locations") => refresh_locations(ui),
|
||||
@@ -653,6 +711,10 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
// count falling to zero is what "the sync finished" looks like from here.
|
||||
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
||||
// starts its upload job mid-flight, which is not a thing finishing.
|
||||
// The Import page reads its progress off the same snapshot rather than
|
||||
// polling the daemon a second time for numbers already on the wire.
|
||||
takeout_progress(ui, jobs);
|
||||
|
||||
let previous = ui.status.active_transfers.replace(items.len());
|
||||
if items.is_empty() && previous > 0 {
|
||||
let files = if previous == 1 {
|
||||
@@ -808,11 +870,14 @@ pub(crate) fn refresh_status(ui: &Rc<Ui>) {
|
||||
(used as f64 / budget as f64).min(1.0)
|
||||
};
|
||||
ui.status.cache_bar.set_fraction(fraction);
|
||||
ui.status.cache_label.set_text(&format!(
|
||||
"{} of {} used",
|
||||
human_bytes(used),
|
||||
human_bytes(budget)
|
||||
));
|
||||
// A 0 budget means *unlimited*, not a zero-byte cap — "of 0 B
|
||||
// used" reads as a broken read-out, and there is no fraction to
|
||||
// draw against no limit.
|
||||
ui.status.cache_label.set_text(&if budget == 0 {
|
||||
format!("{} cached — no limit set", human_bytes(used))
|
||||
} else {
|
||||
format!("{} of {} used", human_bytes(used), human_bytes(budget))
|
||||
});
|
||||
repaint_pins(&ui, &pins, true);
|
||||
}
|
||||
// Daemon unreachable (still starting, or down): report not-mounted and
|
||||
@@ -865,7 +930,15 @@ pub(crate) fn repaint_pins(ui: &Rc<Ui>, pins: &[pdfs_core::cache::Pin], mounted:
|
||||
return;
|
||||
}
|
||||
|
||||
for pin in pins {
|
||||
// Only the first page of pins is rendered; the rest sit behind the row
|
||||
// below, so a device with hundreds of pins doesn't bury the groups after it.
|
||||
let expanded = ui.status.pins_expanded.get();
|
||||
let shown = if expanded {
|
||||
pins.len()
|
||||
} else {
|
||||
pins.len().min(PINS_COLLAPSED)
|
||||
};
|
||||
for pin in &pins[..shown] {
|
||||
let name = Path::new(&pin.path)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
@@ -903,4 +976,37 @@ pub(crate) fn repaint_pins(ui: &Rc<Ui>, pins: &[pdfs_core::cache::Pin], mounted:
|
||||
unpin: Some(unpin),
|
||||
});
|
||||
}
|
||||
|
||||
if pins.len() > PINS_COLLAPSED {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(if expanded {
|
||||
"Show fewer".to_string()
|
||||
} else {
|
||||
format!("Show all {} pinned files", pins.len())
|
||||
})
|
||||
.activatable(true)
|
||||
.build();
|
||||
row.add_suffix(>k4::Image::from_icon_name(if expanded {
|
||||
"go-up-symbolic"
|
||||
} else {
|
||||
"go-down-symbolic"
|
||||
}));
|
||||
let ui_more = ui.clone();
|
||||
row.connect_activated(move |_| {
|
||||
ui_more
|
||||
.status
|
||||
.pins_expanded
|
||||
.set(!ui_more.status.pins_expanded.get());
|
||||
// The diff guard compares pin *paths*; the same paths render
|
||||
// differently now, so the baseline has to be dropped for the next
|
||||
// repaint to actually rebuild.
|
||||
*ui_more.status.pins_state.borrow_mut() = None;
|
||||
refresh(&ui_more);
|
||||
});
|
||||
ui.status.pins_group.add(&row);
|
||||
ui.status
|
||||
.pin_rows
|
||||
.borrow_mut()
|
||||
.push(PinRow { row, unpin: None });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
//! The Import page: bringing a Google Photos **Takeout** export into Proton
|
||||
//! Photos.
|
||||
//!
|
||||
//! Reached from Settings → *Import from Google Photos* (and from the Gallery's
|
||||
//! import button), because it is a one-off migration rather than something the
|
||||
//! photo timeline does every day. It gets a page of its own instead of a dialog
|
||||
//! for two reasons: an export is a *set* of archives that has to be staged and
|
||||
//! checked before anything is sent, and the import that follows runs for hours,
|
||||
//! so its progress needs somewhere to live that the user can navigate away from
|
||||
//! and come back to.
|
||||
//!
|
||||
//! Archives arrive either by drag-and-drop onto the page or through the file
|
||||
//! chooser; both funnel into [`add_archives`], which is where the `.zip` filter
|
||||
//! and de-duplication live. Nothing is sent until the user presses Scan or
|
||||
//! Import — staging is local, so a mis-drop costs nothing.
|
||||
//!
|
||||
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
||||
//! stages paths, launches, polls [`Request::ImportStatus`] for the final report,
|
||||
//! and reads live progress off the import's job in the transfer snapshot the
|
||||
//! Settings tick already fetches (see [`takeout_progress`]).
|
||||
|
||||
use crate::*;
|
||||
|
||||
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
||||
/// Matching on it is how this page borrows the progress the Settings tick is
|
||||
/// already polling instead of adding a second poll of its own.
|
||||
const IMPORT_JOB: &str = "Importing Google Photos";
|
||||
|
||||
/// What the archive list says before anything is staged. `repaint_archives`
|
||||
/// swaps in the count and total once there is something to report, and swaps
|
||||
/// this back when the list is emptied again.
|
||||
/// Where the staged set is remembered between runs, under the app's state
|
||||
/// directory. Staging an export means picking dozens of files out of a file
|
||||
/// chooser; losing that to a window close (or a crash mid-import) and having to
|
||||
/// redo it is the kind of small cruelty that makes a migration not get finished.
|
||||
fn staged_path(ui: &Rc<Ui>) -> PathBuf {
|
||||
ui.dirs.state_dir().join("takeout-staged.json")
|
||||
}
|
||||
|
||||
/// Read the remembered set, dropping any archive that has since been moved or
|
||||
/// deleted — a stale path would only fail at launch, and silently is worse.
|
||||
fn load_staged(ui: &Rc<Ui>) -> Vec<PathBuf> {
|
||||
let Ok(text) = std::fs::read_to_string(staged_path(ui)) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let paths: Vec<String> = serde_json::from_str(&text).unwrap_or_default();
|
||||
paths
|
||||
.into_iter()
|
||||
.map(PathBuf::from)
|
||||
.filter(|p| p.is_file())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Remember the staged set. Best-effort: failing to write it costs the user a
|
||||
/// re-pick next time, which is not worth interrupting them over now.
|
||||
fn save_staged(ui: &Rc<Ui>) {
|
||||
let paths: Vec<String> = ui
|
||||
.takeout
|
||||
.archives
|
||||
.borrow()
|
||||
.iter()
|
||||
.map(|p| p.display().to_string())
|
||||
.collect();
|
||||
let path = staged_path(ui);
|
||||
let written = serde_json::to_string(&paths)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|text| std::fs::write(&path, text).map_err(|e| e.to_string()));
|
||||
if let Err(e) = written {
|
||||
tracing::warn!("cannot remember staged Takeout archives: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_LIST_DESCRIPTION: &str = "Photos already in Proton Photos are matched by name and content and skipped, so \
|
||||
re-running an interrupted import is safe.";
|
||||
|
||||
pub(crate) struct TakeoutState {
|
||||
/// Archives staged for this import, in the order they were added. Sent as
|
||||
/// one set: Google splits an export across numbered zips and routinely puts
|
||||
/// a photo in one part and its metadata sidecar in another, so a per-file
|
||||
/// import would lose album titles and capture times.
|
||||
pub(crate) archives: RefCell<Vec<PathBuf>>,
|
||||
/// The staged-archive list and the rows currently rendered in it.
|
||||
pub(crate) list_group: adw::PreferencesGroup,
|
||||
pub(crate) rows: RefCell<Vec<adw::PreferencesRow>>,
|
||||
/// The drop zone, whose `dropzone-active` class follows the drag.
|
||||
pub(crate) dropzone: gtk4::Box,
|
||||
pub(crate) scan_button: gtk4::Button,
|
||||
pub(crate) import_button: gtk4::Button,
|
||||
pub(crate) cancel_button: gtk4::Button,
|
||||
pub(crate) clear_button: gtk4::Button,
|
||||
/// Progress group, hidden unless an import is running.
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
/// Report group for the last finished run, hidden until there is one.
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
||||
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
||||
/// can't stack worker threads on the 2s tick.
|
||||
pub(crate) inflight: Cell<bool>,
|
||||
/// Whether the daemon reported an import running on the last poll.
|
||||
pub(crate) running: Cell<bool>,
|
||||
/// Whether the run in flight was launched as a dry run, so the report can
|
||||
/// say "would upload" rather than claiming photos were sent. The daemon's
|
||||
/// summary carries no such flag — it is the same shape either way.
|
||||
pub(crate) dry_run: Cell<bool>,
|
||||
}
|
||||
|
||||
/// Widgets the Import page hands back for wiring.
|
||||
pub(crate) struct TakeoutWidgets {
|
||||
pub(crate) list_group: adw::PreferencesGroup,
|
||||
pub(crate) dropzone: gtk4::Box,
|
||||
/// The whole page, which is the drop target: a drag released anywhere on it
|
||||
/// counts, not only inside the dashed rectangle.
|
||||
pub(crate) page: gtk4::Widget,
|
||||
pub(crate) choose_button: gtk4::Button,
|
||||
pub(crate) scan_button: gtk4::Button,
|
||||
pub(crate) import_button: gtk4::Button,
|
||||
pub(crate) cancel_button: gtk4::Button,
|
||||
pub(crate) clear_button: gtk4::Button,
|
||||
pub(crate) back_button: gtk4::Button,
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
}
|
||||
|
||||
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
// Header: a back button out to Settings, since this page is reached from
|
||||
// there and has no sidebar row of its own to navigate back with.
|
||||
let back_button = gtk4::Button::builder()
|
||||
.icon_name("go-previous-symbolic")
|
||||
.tooltip_text("Back to Settings")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
back_button.add_css_class("flat");
|
||||
|
||||
let title = gtk4::Label::builder()
|
||||
.label("Import from Google Photos")
|
||||
.halign(gtk4::Align::Start)
|
||||
.build();
|
||||
title.add_css_class("title-2");
|
||||
let subtitle = gtk4::Label::builder()
|
||||
.label("Add a Google Takeout export to your Proton Photos timeline.")
|
||||
.halign(gtk4::Align::Start)
|
||||
.wrap(true)
|
||||
.xalign(0.0)
|
||||
.build();
|
||||
subtitle.add_css_class("dim-label");
|
||||
|
||||
let titles = gtk4::Box::new(gtk4::Orientation::Vertical, 2);
|
||||
titles.set_hexpand(true);
|
||||
titles.append(&title);
|
||||
titles.append(&subtitle);
|
||||
|
||||
let header = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
header.append(&back_button);
|
||||
header.append(&titles);
|
||||
let header_clamp = adw::Clamp::builder().child(&header).build();
|
||||
|
||||
// Drop zone. It is a hint, not the only way in — the button under it does
|
||||
// the same thing for anyone not dragging, and the drop target covers the
|
||||
// whole page so a drag released just outside the rectangle still lands.
|
||||
let drop_icon = gtk4::Image::from_icon_name("folder-download-symbolic");
|
||||
drop_icon.set_pixel_size(48);
|
||||
drop_icon.add_css_class("dim-label");
|
||||
let drop_title = gtk4::Label::builder()
|
||||
.label("Drop your Takeout .zip files here")
|
||||
.build();
|
||||
drop_title.add_css_class("title-4");
|
||||
let drop_hint = gtk4::Label::builder()
|
||||
.label("Add every part of the export at once — a photo and its metadata often sit in different parts.")
|
||||
.wrap(true)
|
||||
.justify(gtk4::Justification::Center)
|
||||
.max_width_chars(48)
|
||||
.build();
|
||||
drop_hint.add_css_class("dim-label");
|
||||
let choose_button = gtk4::Button::builder()
|
||||
.label("Choose files…")
|
||||
.halign(gtk4::Align::Center)
|
||||
.build();
|
||||
choose_button.add_css_class("pill");
|
||||
|
||||
let dropzone = gtk4::Box::new(gtk4::Orientation::Vertical, 10);
|
||||
dropzone.set_halign(gtk4::Align::Fill);
|
||||
dropzone.add_css_class("dropzone");
|
||||
dropzone.append(&drop_icon);
|
||||
dropzone.append(&drop_title);
|
||||
dropzone.append(&drop_hint);
|
||||
dropzone.append(&choose_button);
|
||||
|
||||
// Staged archives. Filled by `repaint_archives`, which owns the description
|
||||
// too (it becomes the count + total once there is something to count); the
|
||||
// group carries a placeholder row while empty so it never renders as a bare
|
||||
// title.
|
||||
let list_group = adw::PreferencesGroup::builder()
|
||||
.title("Archives to import")
|
||||
.description(EMPTY_LIST_DESCRIPTION)
|
||||
.build();
|
||||
|
||||
let clear_button = gtk4::Button::builder()
|
||||
.label("Clear")
|
||||
.tooltip_text("Remove every staged archive")
|
||||
.build();
|
||||
clear_button.add_css_class("flat");
|
||||
|
||||
// Actions. Scan is a dry run: it reads the archives' directories and reports
|
||||
// what an import *would* do, without uploading a byte. It is offered first
|
||||
// because the import behind it is hours long and hard to take back.
|
||||
let scan_button = gtk4::Button::builder()
|
||||
.label("Scan")
|
||||
.tooltip_text("Check the archives and report what would be imported, without uploading")
|
||||
.build();
|
||||
scan_button.add_css_class("pill");
|
||||
let import_button = gtk4::Button::builder().label("Import").build();
|
||||
import_button.add_css_class("pill");
|
||||
import_button.add_css_class("suggested-action");
|
||||
let cancel_button = gtk4::Button::builder()
|
||||
.label("Stop import")
|
||||
.tooltip_text("Finish the photo on the wire, file what was uploaded, then stop")
|
||||
.build();
|
||||
cancel_button.add_css_class("pill");
|
||||
cancel_button.add_css_class("destructive-action");
|
||||
cancel_button.set_visible(false);
|
||||
|
||||
// Clear sits apart from the launch buttons, on the other end of the row: it
|
||||
// undoes staging, and putting it next to Import invites the wrong click.
|
||||
let actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
let spacer = gtk4::Box::new(gtk4::Orientation::Horizontal, 0);
|
||||
spacer.set_hexpand(true);
|
||||
actions.append(&clear_button);
|
||||
actions.append(&spacer);
|
||||
actions.append(&cancel_button);
|
||||
actions.append(&scan_button);
|
||||
actions.append(&import_button);
|
||||
|
||||
// Progress: the import's job from the transfer snapshot, so this page shows
|
||||
// the same numbers the Settings Activity list does.
|
||||
let progress_group = adw::PreferencesGroup::builder()
|
||||
.title("Import in progress")
|
||||
.visible(false)
|
||||
.build();
|
||||
let progress_box = gtk4::Box::new(gtk4::Orientation::Vertical, 6);
|
||||
progress_box.set_margin_top(8);
|
||||
progress_box.set_margin_bottom(8);
|
||||
let progress_label = gtk4::Label::builder().halign(gtk4::Align::Start).build();
|
||||
progress_label.add_css_class("dim-label");
|
||||
let progress_bar = gtk4::ProgressBar::new();
|
||||
progress_box.append(&progress_label);
|
||||
progress_box.append(&progress_bar);
|
||||
progress_group.add(
|
||||
&adw::PreferencesRow::builder()
|
||||
.activatable(false)
|
||||
.child(&progress_box)
|
||||
.build(),
|
||||
);
|
||||
|
||||
// Report for the last finished run, kept on screen after it ends: an import
|
||||
// outlives the window being on this page, and "what happened" is the whole
|
||||
// reason to come back to it.
|
||||
let summary_group = adw::PreferencesGroup::builder()
|
||||
.title("Last import")
|
||||
.visible(false)
|
||||
.build();
|
||||
|
||||
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
||||
groups.append(&dropzone);
|
||||
groups.append(&list_group);
|
||||
groups.append(&actions);
|
||||
groups.append(&progress_group);
|
||||
groups.append(&summary_group);
|
||||
let clamp = adw::Clamp::builder()
|
||||
.maximum_size(640)
|
||||
.child(&groups)
|
||||
.build();
|
||||
let scroll = gtk4::ScrolledWindow::builder()
|
||||
.vexpand(true)
|
||||
.hscrollbar_policy(gtk4::PolicyType::Never)
|
||||
.child(&clamp)
|
||||
.build();
|
||||
|
||||
let page = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
page.set_margin_top(18);
|
||||
page.set_margin_bottom(18);
|
||||
page.set_margin_start(18);
|
||||
page.set_margin_end(18);
|
||||
page.append(&header_clamp);
|
||||
page.append(&scroll);
|
||||
|
||||
let widgets = TakeoutWidgets {
|
||||
list_group: list_group.clone(),
|
||||
dropzone: dropzone.clone(),
|
||||
page: page.clone().upcast(),
|
||||
choose_button,
|
||||
scan_button: scan_button.clone(),
|
||||
import_button: import_button.clone(),
|
||||
cancel_button: cancel_button.clone(),
|
||||
clear_button,
|
||||
back_button,
|
||||
progress_group: progress_group.clone(),
|
||||
progress_label: progress_label.clone(),
|
||||
progress_bar: progress_bar.clone(),
|
||||
summary_group: summary_group.clone(),
|
||||
};
|
||||
(page.upcast(), widgets)
|
||||
}
|
||||
|
||||
pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
|
||||
// Drag-and-drop. The target sits on the whole page rather than the dashed
|
||||
// rectangle, so a drag released anywhere counts; the rectangle only lights
|
||||
// up to say where the drop is understood.
|
||||
let target = gtk4::DropTarget::new(
|
||||
gtk4::gdk::FileList::static_type(),
|
||||
gtk4::gdk::DragAction::COPY,
|
||||
);
|
||||
let ui_enter = ui.clone();
|
||||
target.connect_enter(move |_, _, _| {
|
||||
ui_enter.takeout.dropzone.add_css_class("dropzone-active");
|
||||
gtk4::gdk::DragAction::COPY
|
||||
});
|
||||
let ui_leave = ui.clone();
|
||||
target.connect_leave(move |_| {
|
||||
ui_leave
|
||||
.takeout
|
||||
.dropzone
|
||||
.remove_css_class("dropzone-active");
|
||||
});
|
||||
let ui_drop = ui.clone();
|
||||
target.connect_drop(move |_, value, _, _| {
|
||||
ui_drop.takeout.dropzone.remove_css_class("dropzone-active");
|
||||
let Ok(files) = value.get::<gtk4::gdk::FileList>() else {
|
||||
return false;
|
||||
};
|
||||
let paths: Vec<PathBuf> = files.files().iter().filter_map(|f| f.path()).collect();
|
||||
add_archives(&ui_drop, paths);
|
||||
true
|
||||
});
|
||||
widgets.page.add_controller(target);
|
||||
|
||||
let ui_choose = ui.clone();
|
||||
widgets.choose_button.connect_clicked(move |_| {
|
||||
// Multi-select on purpose: the whole export goes in one request.
|
||||
let dialog = gtk4::FileDialog::builder()
|
||||
.title("Select your Google Takeout archives")
|
||||
.build();
|
||||
let filter = gtk4::FileFilter::new();
|
||||
filter.set_name(Some("Takeout archives"));
|
||||
filter.add_pattern("*.zip");
|
||||
let filters = gio::ListStore::new::<gtk4::FileFilter>();
|
||||
filters.append(&filter);
|
||||
dialog.set_filters(Some(&filters));
|
||||
|
||||
let ui = ui_choose.clone();
|
||||
dialog.open_multiple(
|
||||
ui_window(&ui_choose).as_ref(),
|
||||
gio::Cancellable::NONE,
|
||||
move |res| {
|
||||
let Ok(files) = res else { return };
|
||||
let paths: Vec<PathBuf> = (0..files.n_items())
|
||||
.filter_map(|index| files.item(index))
|
||||
.filter_map(|object| object.downcast::<gio::File>().ok())
|
||||
.filter_map(|file| file.path())
|
||||
.collect();
|
||||
add_archives(&ui, paths);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
let ui_clear = ui.clone();
|
||||
widgets.clear_button.connect_clicked(move |_| {
|
||||
ui_clear.takeout.archives.borrow_mut().clear();
|
||||
repaint_archives(&ui_clear);
|
||||
});
|
||||
|
||||
let ui_scan = ui.clone();
|
||||
widgets
|
||||
.scan_button
|
||||
.connect_clicked(move |_| start_import(&ui_scan, true));
|
||||
|
||||
// The import itself is confirmed: it uploads a whole photo library, which is
|
||||
// hours of bandwidth and a large change to the account's timeline.
|
||||
let ui_import = ui.clone();
|
||||
widgets.import_button.connect_clicked(move |_| {
|
||||
let count = ui_import.takeout.archives.borrow().len();
|
||||
let ui = ui_import.clone();
|
||||
let dialog = adw::AlertDialog::builder()
|
||||
.heading("Import this export?")
|
||||
.body(format!(
|
||||
"{count} archive(s) will be uploaded to Proton Photos, creating albums as they \
|
||||
appear in the export. This can take hours; you can stop it at any point and \
|
||||
what has been uploaded stays. Photos you already have are skipped.",
|
||||
))
|
||||
.build();
|
||||
dialog.add_response("cancel", "Cancel");
|
||||
dialog.add_response("import", "Import");
|
||||
dialog.set_response_appearance("import", adw::ResponseAppearance::Suggested);
|
||||
dialog.set_default_response(Some("cancel"));
|
||||
dialog.set_close_response("cancel");
|
||||
dialog.connect_response(None, move |_, resp| {
|
||||
if resp == "import" {
|
||||
start_import(&ui, false);
|
||||
}
|
||||
});
|
||||
dialog.present(ui_window(&ui_import).as_ref());
|
||||
});
|
||||
|
||||
let ui_cancel = ui.clone();
|
||||
widgets.cancel_button.connect_clicked(move |_| {
|
||||
let ui = ui_cancel.clone();
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::CancelImport);
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => toast(&ui, "Stopping the import…"),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, "Couldn't stop the import", &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
"Couldn't stop the import",
|
||||
"The mount service didn't respond.",
|
||||
),
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let ui_back = ui.clone();
|
||||
widgets
|
||||
.back_button
|
||||
.connect_clicked(move |_| ui_back.stack.set_visible_child_name("main"));
|
||||
|
||||
// Restore whatever was staged when the window last closed, before the first
|
||||
// paint, so the page comes up as the user left it.
|
||||
*ui.takeout.archives.borrow_mut() = load_staged(ui);
|
||||
repaint_archives(ui);
|
||||
}
|
||||
|
||||
/// Stage `paths`, keeping only `.zip` files and skipping ones already staged.
|
||||
///
|
||||
/// Both the chooser and the drop target land here, so the filter is stated once.
|
||||
/// A dropped folder (an already-extracted export) is rejected rather than walked:
|
||||
/// the scan reads the archives' central directories, and an extracted tree has
|
||||
/// none — silently importing half of it would be worse than saying no.
|
||||
pub(crate) fn add_archives(ui: &Rc<Ui>, paths: Vec<PathBuf>) {
|
||||
let mut added = 0usize;
|
||||
let mut rejected = 0usize;
|
||||
{
|
||||
let mut staged = ui.takeout.archives.borrow_mut();
|
||||
for path in paths {
|
||||
let is_zip = path
|
||||
.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("zip"));
|
||||
if !is_zip || !path.is_file() {
|
||||
rejected += 1;
|
||||
continue;
|
||||
}
|
||||
if staged.contains(&path) {
|
||||
continue;
|
||||
}
|
||||
staged.push(path);
|
||||
added += 1;
|
||||
}
|
||||
// Numbered parts drop in whatever order the file manager hands them
|
||||
// over; sorting keeps the list readable and the request stable.
|
||||
staged.sort();
|
||||
}
|
||||
repaint_archives(ui);
|
||||
if rejected > 0 {
|
||||
toast(
|
||||
ui,
|
||||
"Only Takeout .zip files can be imported — extracted folders aren't supported.",
|
||||
);
|
||||
} else if added > 0 {
|
||||
toast(ui, &format!("Added {added} archive(s)"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild the staged-archive list and re-derive which actions are available.
|
||||
pub(crate) fn repaint_archives(ui: &Rc<Ui>) {
|
||||
for row in ui.takeout.rows.borrow_mut().drain(..) {
|
||||
ui.takeout.list_group.remove(&row);
|
||||
}
|
||||
|
||||
let staged = ui.takeout.archives.borrow().clone();
|
||||
let mut rows: Vec<adw::PreferencesRow> = Vec::new();
|
||||
if staged.is_empty() {
|
||||
ui.takeout
|
||||
.list_group
|
||||
.set_description(Some(EMPTY_LIST_DESCRIPTION));
|
||||
let row = dim_row("No archives added yet.");
|
||||
ui.takeout.list_group.add(&row);
|
||||
rows.push(row.upcast());
|
||||
} else {
|
||||
let mut total = 0u64;
|
||||
for path in &staged {
|
||||
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
|
||||
total += size;
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(
|
||||
path.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| path.display().to_string()),
|
||||
)
|
||||
.subtitle(human_bytes(size))
|
||||
.build();
|
||||
let remove = gtk4::Button::builder()
|
||||
.icon_name("edit-delete-symbolic")
|
||||
.tooltip_text("Remove from this import")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
remove.add_css_class("flat");
|
||||
let ui_remove = ui.clone();
|
||||
let target = path.clone();
|
||||
remove.connect_clicked(move |_| {
|
||||
ui_remove
|
||||
.takeout
|
||||
.archives
|
||||
.borrow_mut()
|
||||
.retain(|p| *p != target);
|
||||
repaint_archives(&ui_remove);
|
||||
});
|
||||
row.add_suffix(&remove);
|
||||
ui.takeout.list_group.add(&row);
|
||||
rows.push(row.upcast());
|
||||
}
|
||||
ui.takeout.list_group.set_description(Some(&format!(
|
||||
"{} archive(s), {} in total. {EMPTY_LIST_DESCRIPTION}",
|
||||
staged.len(),
|
||||
human_bytes(total)
|
||||
)));
|
||||
}
|
||||
*ui.takeout.rows.borrow_mut() = rows;
|
||||
|
||||
save_staged(ui);
|
||||
sync_takeout_actions(ui);
|
||||
}
|
||||
|
||||
/// Enable exactly the actions that make sense right now: nothing to launch
|
||||
/// without archives, nothing to launch while a run is in flight, and Stop only
|
||||
/// while there is something to stop.
|
||||
fn sync_takeout_actions(ui: &Rc<Ui>) {
|
||||
let has_files = !ui.takeout.archives.borrow().is_empty();
|
||||
let running = ui.takeout.running.get();
|
||||
ui.takeout.scan_button.set_sensitive(has_files && !running);
|
||||
ui.takeout
|
||||
.import_button
|
||||
.set_sensitive(has_files && !running);
|
||||
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
||||
ui.takeout.cancel_button.set_visible(running);
|
||||
}
|
||||
|
||||
/// Hand the staged set to the daemon. It acks immediately and works in the
|
||||
/// background — an export is far past any socket timeout — so success here means
|
||||
/// "started", and the page switches over to reporting progress.
|
||||
fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
let archives: Vec<String> = ui
|
||||
.takeout
|
||||
.archives
|
||||
.borrow()
|
||||
.iter()
|
||||
.map(|p| p.display().to_string())
|
||||
.collect();
|
||||
if archives.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
ui.takeout.dry_run.set(dry_run);
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(
|
||||
ui.dirs.control_socket(),
|
||||
Request::ImportTakeout { archives, dry_run },
|
||||
);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => {
|
||||
ui.takeout.running.set(true);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout.progress_label.set_text(if dry_run {
|
||||
"Reading archives…"
|
||||
} else {
|
||||
"Starting import…"
|
||||
});
|
||||
sync_takeout_actions(&ui);
|
||||
toast(
|
||||
&ui,
|
||||
if dry_run {
|
||||
"Scanning the export…"
|
||||
} else {
|
||||
"Import started — you can leave this page, it keeps running."
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, "Couldn't start the import", &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
"Couldn't start the import",
|
||||
"The mount service didn't respond.",
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Poll the daemon for the import's state while the page is on screen, and paint
|
||||
/// the report once a run has finished. Cheap and inflight-guarded, like the other
|
||||
/// tick polls.
|
||||
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
if ui.takeout.inflight.get() {
|
||||
return;
|
||||
}
|
||||
ui.takeout.inflight.set(true);
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::ImportStatus);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.takeout.inflight.set(false);
|
||||
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
|
||||
return;
|
||||
};
|
||||
let was_running = ui.takeout.running.replace(running);
|
||||
if running {
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
} else {
|
||||
ui.takeout.progress_group.set_visible(false);
|
||||
// An import runs for hours, so its ending is announced rather than
|
||||
// left to be discovered: a toast for whoever is looking at the
|
||||
// window, and a desktop notification for whoever is not.
|
||||
if was_running {
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
let headline = if dry {
|
||||
"Scan finished"
|
||||
} else {
|
||||
"Import finished"
|
||||
};
|
||||
toast(&ui, headline);
|
||||
if !dry {
|
||||
let body = match &summary {
|
||||
Some(s) if s.failed > 0 => {
|
||||
format!("{} photos added, {} failed.", s.uploaded, s.failed)
|
||||
}
|
||||
Some(s) if s.cancelled => {
|
||||
format!("Stopped after {} photos.", s.uploaded)
|
||||
}
|
||||
Some(s) => format!("{} photos added to Proton Photos.", s.uploaded),
|
||||
None => "The Google Photos import has finished.".to_string(),
|
||||
};
|
||||
notify("takeout-import", headline, &body);
|
||||
}
|
||||
}
|
||||
}
|
||||
if was_running != running {
|
||||
sync_takeout_actions(&ui);
|
||||
}
|
||||
match summary {
|
||||
Some(summary) => repaint_summary(&ui, &summary),
|
||||
None => ui.takeout.summary_group.set_visible(false),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Paint the last run's report.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
||||
ui.takeout.summary_group.remove(&row);
|
||||
}
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
ui.takeout
|
||||
.summary_group
|
||||
.set_title(if dry { "Scan result" } else { "Last import" });
|
||||
ui.takeout.summary_group.set_description(Some(if dry {
|
||||
"Nothing was uploaded — this is what an import would do."
|
||||
} else if summary.cancelled {
|
||||
"Stopped early. Everything uploaded so far is in your timeline; run it again to \
|
||||
continue where it left off."
|
||||
} else {
|
||||
"Finished."
|
||||
}));
|
||||
|
||||
// Only lines that carry information: a run with no failures should not have
|
||||
// to show a "0 failed" row for the user to read past.
|
||||
let mut lines: Vec<(String, String)> = vec![
|
||||
("Photos in the export".into(), summary.found.to_string()),
|
||||
(
|
||||
if dry { "Would upload" } else { "Uploaded" }.into(),
|
||||
summary.uploaded.to_string(),
|
||||
),
|
||||
];
|
||||
if summary.duplicates > 0 {
|
||||
lines.push((
|
||||
"Already in Proton Photos".into(),
|
||||
summary.duplicates.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
"Albums to create"
|
||||
} else {
|
||||
"Albums created"
|
||||
}
|
||||
.into(),
|
||||
summary.albums_created.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.album_links > 0 {
|
||||
lines.push((
|
||||
"Photos filed into albums".into(),
|
||||
summary.album_links.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.skipped_trashed > 0 {
|
||||
lines.push((
|
||||
"Skipped (in Google's trash)".into(),
|
||||
summary.skipped_trashed.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
"Data to upload"
|
||||
} else {
|
||||
"Data uploaded"
|
||||
}
|
||||
.into(),
|
||||
human_bytes(summary.bytes),
|
||||
));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
lines.push(("Failed".into(), summary.failed.to_string()));
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (title, value) in lines {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(title)
|
||||
.subtitle(value)
|
||||
.build();
|
||||
row.add_css_class("property");
|
||||
ui.takeout.summary_group.add(&row);
|
||||
rows.push(row);
|
||||
}
|
||||
*ui.takeout.summary_rows.borrow_mut() = rows;
|
||||
ui.takeout.summary_group.set_visible(true);
|
||||
}
|
||||
|
||||
/// Paint the import's progress from the transfer snapshot the Settings tick
|
||||
/// already fetched, so this page costs no extra poll. A job with no known total
|
||||
/// pulses instead of showing a fraction, the same as the Activity list.
|
||||
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
|
||||
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
|
||||
return;
|
||||
};
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
if job.total > 0 {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&format!("{} of {} — {}", job.done, job.total, job.detail));
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(if job.detail.is_empty() {
|
||||
"Working…"
|
||||
} else {
|
||||
&job.detail
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,8 @@ pub(crate) struct DetailsState {
|
||||
pub(crate) details_suppress: Cell<bool>,
|
||||
/// The grid's and list's selection models. Both wrap the one browser model, so
|
||||
/// a selection in either drives the details pane.
|
||||
pub(crate) grid_selection: gtk4::SingleSelection,
|
||||
pub(crate) list_selection: gtk4::SingleSelection,
|
||||
pub(crate) grid_selection: gtk4::MultiSelection,
|
||||
pub(crate) list_selection: gtk4::MultiSelection,
|
||||
}
|
||||
|
||||
/// The widgets in the browser's details pane that a selection repaints.
|
||||
@@ -158,24 +158,22 @@ pub(crate) fn wire_details(ui: &Rc<Ui>) {
|
||||
ui.details.list_selection.clone(),
|
||||
] {
|
||||
let ui_sel = ui.clone();
|
||||
selection.connect_selection_changed(move |sel, _, _| {
|
||||
match entry_at(sel.model().as_ref(), sel.selected()) {
|
||||
Some(entry) => show_details(&ui_sel, &entry),
|
||||
None => hide_details(&ui_sel),
|
||||
selection.connect_selection_changed(move |_, _, _| {
|
||||
// The details pane describes *one* entry. A batch is the bulk bar's
|
||||
// business, so the pane steps aside rather than picking a member of
|
||||
// the selection to speak for the rest.
|
||||
let entries = selected_entries(&ui_sel);
|
||||
match entries.as_slice() {
|
||||
[entry] => show_details(&ui_sel, entry),
|
||||
_ => hide_details(&ui_sel),
|
||||
}
|
||||
sync_bulk_bar(&ui_sel);
|
||||
});
|
||||
}
|
||||
|
||||
let ui_close = ui.clone();
|
||||
ui.details.details.close_button.connect_clicked(move |_| {
|
||||
ui_close
|
||||
.details
|
||||
.grid_selection
|
||||
.set_selected(gtk4::INVALID_LIST_POSITION);
|
||||
ui_close
|
||||
.details
|
||||
.list_selection
|
||||
.set_selected(gtk4::INVALID_LIST_POSITION);
|
||||
clear_selection(&ui_close);
|
||||
hide_details(&ui_close);
|
||||
});
|
||||
|
||||
@@ -289,8 +287,9 @@ pub(crate) fn hide_details(ui: &Rc<Ui>) {
|
||||
*ui.details.details_entry.borrow_mut() = None;
|
||||
}
|
||||
|
||||
/// The entry highlighted in whichever browser view is on screen, if any. Backs the
|
||||
/// F2 / Delete shortcuts.
|
||||
/// The single entry the details pane is showing, if any. Backs the actions that
|
||||
/// only make sense on one entry (rename, versions); the batch-capable ones ask
|
||||
/// [`selected_entries`] instead.
|
||||
pub(crate) fn selected_entry(ui: &Rc<Ui>) -> Option<DirEntry> {
|
||||
ui.details.details_entry.borrow().clone()
|
||||
}
|
||||
@@ -130,7 +130,7 @@ fn pins(socket: &Path) -> Result<Vec<Hit>, String> {
|
||||
Hit::Drive(SearchHit {
|
||||
name: file_name(&pin.path),
|
||||
path: pin.path,
|
||||
is_dir: pin.recursive,
|
||||
is_dir: pin.is_dir.unwrap_or(pin.recursive),
|
||||
size: 0,
|
||||
modified: 0,
|
||||
pinned: true,
|
||||
|
||||
@@ -147,29 +147,26 @@ impl Hit {
|
||||
}
|
||||
|
||||
/// The dimmed second line: the containing folder, as the user thinks of it.
|
||||
///
|
||||
/// A Drive hit is only "My files" when it really sits in the primary mount.
|
||||
/// A node inside a device folder resolves to a path on this machine
|
||||
/// (`~/Downloads/…`) — the daemon has already worked that out in
|
||||
/// `mounted_path`, and saying "My files / Downloads" for it names a folder
|
||||
/// the user cannot find and does not match where activating the row opens.
|
||||
fn location(&self) -> String {
|
||||
match self {
|
||||
Hit::Drive(h) => {
|
||||
let parent = parent_of(&h.path);
|
||||
if parent.is_empty() {
|
||||
"My files".to_string()
|
||||
} else {
|
||||
format!("My files / {}", parent.replace('/', " / "))
|
||||
Hit::Drive(h) => match h.mounted_path.as_deref() {
|
||||
Some(mounted) => home_relative(&parent_of(mounted)),
|
||||
None => {
|
||||
let parent = parent_of(&h.path);
|
||||
if parent.is_empty() {
|
||||
"My files".to_string()
|
||||
} else {
|
||||
format!("My files / {}", parent.replace('/', " / "))
|
||||
}
|
||||
}
|
||||
}
|
||||
Hit::Local(h) => {
|
||||
let parent = parent_of(&h.path);
|
||||
match dirs_home().and_then(|home| {
|
||||
Path::new(&parent)
|
||||
.strip_prefix(&home)
|
||||
.ok()
|
||||
.map(|rel| rel.display().to_string())
|
||||
}) {
|
||||
Some(rel) if rel.is_empty() => "Home".to_string(),
|
||||
Some(rel) => format!("Home / {}", rel.replace('/', " / ")),
|
||||
None => parent,
|
||||
}
|
||||
}
|
||||
},
|
||||
Hit::Local(h) => home_relative(&parent_of(&h.path)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -869,7 +866,7 @@ impl Ui {
|
||||
.map(|pin| SearchHit {
|
||||
name: file_name(&pin.path),
|
||||
path: pin.path,
|
||||
is_dir: pin.recursive,
|
||||
is_dir: pin.is_dir.unwrap_or(pin.recursive),
|
||||
size: 0,
|
||||
modified: 0,
|
||||
pinned: true,
|
||||
@@ -1358,6 +1355,21 @@ fn file_name(path: &str) -> String {
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// An absolute directory path as "Home / …", falling back to the path itself
|
||||
/// for anything outside the home directory.
|
||||
fn home_relative(parent: &str) -> String {
|
||||
match dirs_home().and_then(|home| {
|
||||
Path::new(parent)
|
||||
.strip_prefix(&home)
|
||||
.ok()
|
||||
.map(|rel| rel.display().to_string())
|
||||
}) {
|
||||
Some(rel) if rel.is_empty() => "Home".to_string(),
|
||||
Some(rel) => format!("Home / {}", rel.replace('/', " / ")),
|
||||
None => parent.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_of(path: &str) -> String {
|
||||
Path::new(path)
|
||||
.parent()
|
||||
|
||||
@@ -153,8 +153,9 @@ Because Proton Drive does not support partial byte writes, modified files must b
|
||||
2. **Close/Release (`queue_revision`):** When the application closes the file descriptor, the daemon:
|
||||
- Fetches any untouched gaps from the remote base file to compile the full file.
|
||||
- Durably publishes the scratch data and authored-range sidecar into `staging` under a `{uid}-{millis}-{counter}` name. Temporary data and metadata are synced before atomic rename, and their directory is synced before the source is removed.
|
||||
- Transactionally queues or supersedes a pending database operation (`PendingOp`), so insertion failure cannot erase the previously acknowledged upload. For `OP_REVISION` ops, execution is debounced by 2 seconds (`DRAIN_REVISION_DEBOUNCE = 2s`) to give rapid follow-up writes (e.g. `aria2c` preallocation followed by writing) time to supersede the staged blob before network transmission.
|
||||
3. **Async Drain Thread (`run_pending_drain`):** The background drain worker picks up the database operations queue, handles revision uploads, resolves conflicts, and cleans up staging files. Upon landing a revision upload (`refresh_after_upload`), it rebaselines both still-queued ops (`rebaseline_pending`) and open write handles targeting the same node to prevent false self-conflict copies on subsequent writes.
|
||||
- Transactionally queues or supersedes a pending database operation (`PendingOp`), so insertion failure cannot erase the previously acknowledged upload. For `OP_REVISION` ops, execution is debounced to give rapid follow-up writes (e.g. `aria2c` preallocation followed by writing) time to supersede the staged blob before network transmission. The debounce is adaptive (`Core::revision_debounce`): it starts at `DRAIN_REVISION_DEBOUNCE = 2s` and, once a node has been uploaded, widens toward how long that upload actually took, bounded by `DRAIN_REVISION_DEBOUNCE_MAX = 60s`. A file saved faster than it can be sent therefore supersedes in the queue rather than mid-upload.
|
||||
- Signals `Core::cancel_upload` for the node before touching the queue. A drain worker may already be reading the blob this write supersedes; the flag is read by the upload's `CountingReader`, which refuses the SDK's next block. `queue_trash` and `discard_queued_ops` do the same, because they unlink the blob outright.
|
||||
3. **Async Drain Threads (`run_pending_drain`):** `DRAIN_WORKERS = 3` background workers share the operations queue. They pick work up through `Db::claim_next_due_op`, which marks the row `claimed_at` in the same transaction that selects it and excludes any op whose uid another worker already holds — ordering only has to hold *per node*, and that exclusion is what guarantees it. Worker 0 additionally runs the queue's idle chores (LRU touch flush, `recover_fsynced_writes`). A claim is process-local state: the single-writer `flock` means one found at open belongs to a crashed run, and `Db::clear_op_claims` drops the lot so those ops are not invisible forever. Each worker handles revision uploads, resolves conflicts, and cleans up staging files. Upon landing a revision upload (`refresh_after_upload`), it rebaselines both still-queued ops (`rebaseline_pending`) and open write handles targeting the same node to prevent false self-conflict copies on subsequent writes.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
|
||||
+991
-19
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,517 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to the Linux client since the 1.0.0 release. Format follows
|
||||
[Keep a Changelog](https://keepachangelog.com/1.1.0/); versions follow semver, and every
|
||||
release is tagged `vX.Y.Z` with `Cargo.toml` and `packaging/PKGBUILD` in agreement.
|
||||
|
||||
`Schema:` lines record the SQLite `SCHEMA_VERSION` (`pdfs-core/src/db/migrations.rs`) a
|
||||
release ships. Migrations are forward-only — a database written by a newer client is a hard
|
||||
refuse-to-open, not a downgrade, so rolling back a release means restoring the cache from
|
||||
scratch (user data in `staging/` and `recovery/` is never touched by this).
|
||||
|
||||
## [1.9.0] — 2026-08-18
|
||||
|
||||
Google Photos import, and a pass over the desktop app's interaction gaps. No schema change.
|
||||
Schema: **28**. Builds against the published `proton-sdk` / `proton-drive-rs` **0.6.1** — the album
|
||||
writes this release needs are in the registry crates, so the local `[patch.crates-io]` shim the
|
||||
feature was developed against is gone and no local path dependency remains.
|
||||
|
||||
### Added
|
||||
- **Import a Google Photos Takeout export into Proton Photos** — `pdfs import-google-photos
|
||||
takeout-*.zip` (add `--dry-run` to see what it would do, `--wait` to follow it), or the import
|
||||
button in the Photos header, which multi-selects the export's zips. Albums in the export are
|
||||
recreated as Proton albums and their photos filed into them; year folders and `Archive/` import
|
||||
into the timeline; `Trash/` is skipped. Capture times and favourites come from Google's per-photo
|
||||
JSON sidecars, including the truncated and `(N)`-swapped sidecar names Google produces.
|
||||
- Photos already on the account are **not re-uploaded**: the import asks the photos volume which of
|
||||
the export's names exist (batched, 150 per request), reads and SHA-1s only those files, and skips
|
||||
the ones whose content digest matches too — Proton's own duplicate rule. A library already partly
|
||||
synced by Proton Photos on a phone therefore imports only what is missing, and an interrupted
|
||||
import resumes by being run again. Google keeps a full copy of a photo in every album folder it
|
||||
appears in; those fold into one upload that joins several albums.
|
||||
- The import streams each photo straight out of its zip, so an export needs no scratch space to
|
||||
unpack into, and reports as a job in `pdfs transfers` and the GUI's activity view. `pdfs
|
||||
import-status` shows the counts, `pdfs cancel-import` stops it — a cancelled run still files what
|
||||
it uploaded into its albums.
|
||||
- **An Import page in the GUI** (Settings → *Import from Google Photos*, or the import button in
|
||||
the Photos header). Archives are staged by drag-and-drop onto the page or through the file
|
||||
chooser, listed with their sizes so the set can be checked before anything is sent, and launched
|
||||
as either a Scan (dry run) or a confirmed Import. The page shows live progress off the import's
|
||||
job, keeps the finished run's report on screen, and can stop a run in flight; because the import
|
||||
outlives the page, it is polled wherever the user navigates and its completion is reported as a
|
||||
desktop notification.
|
||||
|
||||
- SDK (`proton-sdk-rs` **0.6.1**): album **writes** — `ProtonPhotosClient::create_album` and
|
||||
`add_photos_to_album`, ported from the TypeScript SDK (upstream C# has no album write API), plus
|
||||
batch duplicate detection (`find_duplicates_many`, `name_collisions`). The workspace deps moved
|
||||
`"0.6.0"` → `"0.6.1"` and `Cargo.lock` re-resolved to the registry crates. Adding a photo from an
|
||||
album *shared with us* (a different volume) is still unported and fails that photo's outcome.
|
||||
- **Multi-select in Files.** Both views select a batch (click-drag, Ctrl-click, Shift-click); a bar
|
||||
slides in reporting the count and offering Move to Trash and offline on/off for the whole
|
||||
selection. Delete acts on everything selected, Esc clears it, and right-clicking inside a
|
||||
selection offers the batch actions rather than the one row under the pointer. Actions that need a
|
||||
single subject — rename, move, share, revisions — stay single-entry, and the details pane steps
|
||||
aside for a batch instead of speaking for it.
|
||||
- **Undo on trash.** "Moved *X* to Trash" now carries an Undo that restores the batch, single files
|
||||
included.
|
||||
- **The empty states are a place to act**: an empty folder offers Upload files / New folder, and an
|
||||
empty photo timeline offers Upload photos / Import from Google Photos, instead of describing what
|
||||
the user could go and do elsewhere.
|
||||
- **Ctrl+F works from any page**, navigating to Files and focusing the search box, rather than being
|
||||
silently inert everywhere but Files.
|
||||
- Staged Takeout archives survive closing the window; archives that have since been moved or deleted
|
||||
are dropped from the restored set.
|
||||
|
||||
### Fixed
|
||||
- Settings showed the local cache as "X of 0 B used" when no cache budget was set. An unset budget
|
||||
means *unlimited*, so it now reads "X cached — no limit set" rather than implying a zero cap.
|
||||
- The cache-budget editor fired a `SetCacheBudget` round-trip — and a toast — on every step, so a
|
||||
run of `+` clicks applied a series of caps the user never asked for and stacked a column of
|
||||
toasts. It now waits for the value to settle.
|
||||
- A long pin list pushed everything below it off the end of Settings. It shows the first six with a
|
||||
"Show all N" row for the rest.
|
||||
|
||||
## [1.8.2] — 2026-08-17
|
||||
|
||||
Closes the two findings 1.8.1's live verification produced (B85, B86) and finishes the data-safety
|
||||
audit entries that had been sitting as "fixed in code, verification pending" since 1.0 (B49–B55).
|
||||
No schema change. Schema: **28**.
|
||||
|
||||
The read path no longer assumes a revision is a series of 4 MiB blocks; it takes the geometry from
|
||||
the revision. The B85 investigation also **contradicted its own bug report** — see Notes, because
|
||||
the conclusion recorded there was that reads of such files were losing data, and they were not.
|
||||
|
||||
### Fixed
|
||||
- A folder switched from mirror to on-demand now shows the files the mirror pass itself uploaded.
|
||||
The sync engine created nodes on the remote without invalidating the parent's cached child
|
||||
listing, and that listing is a database flag rather than an in-memory cache — so it survived a
|
||||
daemon restart, and the files stayed invisible until the folder was switched back. Every remote
|
||||
mutation the engine makes (folder create, file upload, and the three trash paths) now drops the
|
||||
parent's cached listing by uid, which reaches a folder that no mount has resident — the case a
|
||||
mirror folder always is (`docs/BUGS.md` B86).
|
||||
- `pdfs ls` and `pdfs refresh` accept a path under any mount the daemon owns, not only the primary
|
||||
one. Both previously answered "is not under the mountpoint" for anything inside a secondary
|
||||
on-demand mount, which left a folder serving a stale listing with no manual way to re-enumerate
|
||||
it. A path under nothing this daemon has mounted still gets the same error as before.
|
||||
- A file whose blocks are not 4 MiB is no longer read twice over. Block offsets came from
|
||||
`index * 4 MiB`, so on a revision laid out differently every block the client asked for
|
||||
straddled two of the revision's own, and each was fetched and decrypted to answer for one — then
|
||||
cached at boundaries the next read would not ask about. Offsets now come from the revision's
|
||||
recorded block sizes, and the on-disk cache, the in-memory ring and the in-flight fetch key
|
||||
address a block by the byte range it covers rather than by its index (`docs/BUGS.md` B85).
|
||||
|
||||
### Changed
|
||||
- The block cache records each block's plaintext start alongside its `(mtime, size)` tag, and each
|
||||
revision's block table in a small sidecar beside its blocks. Blocks cached by an earlier release
|
||||
carry no start; they are honoured wherever the old 4 MiB assumption and the new plan agree on
|
||||
the offset, which is every file this client uploaded — so the existing cache survives the
|
||||
upgrade intact and is quietly refetched only for the files it was placing wrongly.
|
||||
|
||||
### Notes
|
||||
- **B85's diagnosis was wrong, and the entry has been corrected rather than closed.** It read a
|
||||
set of journal lines as proof that this client was requesting wrong byte ranges and being saved
|
||||
only by B84's repair path. `RevisionReader::read_at` does not work that way: it plans over the
|
||||
revision's real block sizes and clamps every block to the range asked for, identically in the
|
||||
0.6.0 the client builds against and in every earlier version. A read can therefore never return
|
||||
more bytes than it requested — yet every quoted line shows exactly that (`blen=4194304
|
||||
got=4939506`), and the one place in the client that logs that shape can only reach it on the
|
||||
opposite condition. The numbers were either transcribed with two fields swapped or captured from
|
||||
a build not in the history; they want re-capturing from the journal before anything is concluded
|
||||
from them. What was real is the assumption itself, and that is what this release removes — a
|
||||
cost fix, not a correctness one. Nothing here changes what a read returns.
|
||||
- The first read of a file still plans on the 4 MiB assumption, and learns the real geometry as a
|
||||
side effect of opening the reader it needs anyway. Resolving a reader merely to *plan* would
|
||||
reintroduce the per-read key derivation the block cache exists to avoid, so the first read of a
|
||||
non-4-MiB file pays one straddling fetch and then self-corrects.
|
||||
- Seven data-safety audit entries (B49–B55) moved from "fixed in code, verification pending" to
|
||||
verified. The fixes were all genuinely present and needed no change; what they lacked was the
|
||||
injected-failure testing each entry asked for, and that is what was written — unwritable
|
||||
directories and `chmod 000` subtrees for permission failures, a real `/dev/shm`↔`/tmp` device
|
||||
boundary for the cross-filesystem fallbacks, SQLite's `max_page_count` for a full disk, an abort
|
||||
trigger for a failed insert, and truncated and unpublished sidecars for the crash boundaries.
|
||||
Two seams were added to make failure arrangeable at all: `stage_write_at`, mirroring the
|
||||
`preserve_write_at` that already existed, and `walk_local_tree` lifted out of the sync engine so
|
||||
the local scan can be driven against a real tree without a daemon.
|
||||
- **Two of those entries are not fully closed, and say so.** B51 and B52 concern crash consistency,
|
||||
and nothing short of a machine actually losing power tests that; the crash *boundaries* are
|
||||
covered by reconstructing on disk exactly what a crash at each point leaves. B54 also diverges
|
||||
from its required fix on purpose: it has no "deliberate delete" confirmation path, so a total
|
||||
local wipe is refused outright rather than offered for confirmation. That is stricter than asked
|
||||
for, and the cost is that a user who really did delete everything must remove and re-add the
|
||||
sync folder.
|
||||
- B86's fix routes only `ls` and `refresh` to the owning mount. The other path-addressed control
|
||||
requests — pin, rename, move, delete, sharing — still resolve against the primary mount only.
|
||||
That is a larger change and was not what blocked the repro; it is recorded as an open follow-up
|
||||
(`docs/BUGS.md` B88).
|
||||
- Three findings were opened rather than fixed: the geometry above is learned one read late, so a
|
||||
non-4-MiB file pays a single wasted fetch on its first read (B87); the remaining path-addressed
|
||||
control requests are still primary-only (B88); and the container-level `CLAUDE.md` still
|
||||
describes the SDK dependency as 0.1.11 when 1.7.1 bumped it to 0.6.0, which is what sent the
|
||||
B85 investigation to the wrong source tree to begin with (B89).
|
||||
|
||||
## [1.8.1] — 2026-08-16
|
||||
|
||||
Some files read short through the mount and reported success — an application got a truncated
|
||||
copy with nothing anywhere saying so (`docs/BUGS.md` B84). Found because GNOME's file indexer
|
||||
kept complaining that PNGs under `~/ProtonDrive/Pictures` were corrupt; it was right. The rest
|
||||
of this release is the long-standing architectural backlog from the July concurrency audits
|
||||
(B25, B32, B39, B40, B43, B44), each verified against the current code before being fixed.
|
||||
Schema: **28** — migration V28 adds `sync_entry.local_mtime_ns`.
|
||||
|
||||
### Fixed
|
||||
- A read no longer trusts a revision whose recorded block sizes disagree with the file's size.
|
||||
Such a file leaves the block/range path entirely: it is fetched through the whole-file,
|
||||
manifest-verified download — the same path `pdfs pin` uses, and the reason pinning was the
|
||||
workaround — and served from the resulting cached blob. One download per file however many
|
||||
reads discover the problem at once, and files above 512 MiB fail `EIO` with a note to pin
|
||||
them rather than pulling down gigabytes for one read.
|
||||
- A block that comes back shorter than the file's size implies is never served, cached or kept
|
||||
in memory. Assembling a read out of short blocks is what quietly produced the truncation:
|
||||
the missing bytes were simply not appended, the kernel handed userspace a short read, and
|
||||
every tool reads a short read as end-of-file.
|
||||
- A synced folder no longer reports the same deletion as an error on every pass. Deleting a
|
||||
local file that something else had already deleted counted as a failure, which kept its
|
||||
baseline and made the next pass try again — forever. A path that is already gone is the
|
||||
state the sync pass wanted (`docs/BUGS.md` B42).
|
||||
- A mirror folder no longer misses an edit made in the same second as its last sync. Change
|
||||
detection compared whole seconds, so rewriting a file in place without changing its length
|
||||
read as "unchanged" and never uploaded — the remote silently stayed behind until something
|
||||
else about the file moved. Baselines now record the modification time to nanoseconds
|
||||
(`docs/BUGS.md` B25).
|
||||
- A sync download no longer overwrites an edit that finished while it was running. The
|
||||
destination is re-checked in the last moment before the download is published, and a local
|
||||
file that has moved on since the pass planned for it is kept as a conflict copy — the same
|
||||
resolution a both-sides-changed conflict gets (`docs/BUGS.md` B39).
|
||||
- An upload interrupted by a writer no longer reports itself as synchronized. The baseline now
|
||||
records the content that actually went up rather than whatever is on disk when the transfer
|
||||
ends, so a revision torn by a concurrent write is replaced by a clean one on the next pass
|
||||
instead of standing as the file's content on Drive (`docs/BUGS.md` B40).
|
||||
- Listing a directory that changes while it is being read no longer skips or repeats entries.
|
||||
`readdir` resumed by index into a listing rebuilt on every call, so a create or a trash
|
||||
between two pages shifted everything after it. The listing is now frozen for the life of the
|
||||
directory handle (`docs/BUGS.md` B43).
|
||||
- Opening a file for writing while a previous close is still publishing no longer discards that
|
||||
close's revision. The base is re-checked at the moment the write handle is installed, rather
|
||||
than trusted across a copy that can be the size of the whole file (`docs/BUGS.md` B32).
|
||||
- Unmounting now stops and joins the daemon's background workers instead of leaving them
|
||||
running. The drain, sync engine, conflict sweep, online probe, local indexer and control
|
||||
listener share a stop signal and interruptible waits, so an in-process remount no longer
|
||||
starts a second set on top of the first (`docs/BUGS.md` B44).
|
||||
|
||||
### Notes
|
||||
- The size `stat` reports was always the correct one; it is the reader's belief about where
|
||||
blocks end that was wrong, so nothing about reported sizes changes.
|
||||
- Verified live on the affected account: four multi-block PNGs that had been reading short now
|
||||
deliver their full size, and the journal shows each one repaired. The diagnostics also
|
||||
answered the open question, and not as expected — the revision metadata is fine; those files
|
||||
simply do not have 4 MiB blocks, and this client assumes they do. Filed as `docs/BUGS.md`
|
||||
B85. Until that is fixed, a file with non-4-MiB blocks is downloaded whole on its first read.
|
||||
- Migration V28's new column is nullable rather than converted from the existing seconds. A
|
||||
baseline row written before this release has no sub-second time to recover, and inventing
|
||||
zero for it would make every already-synced file compare as changed and re-upload an entire
|
||||
mirror on first start; `NULL` keeps those rows comparing exactly as they did, and each gains
|
||||
a real value the next time its path syncs.
|
||||
- Three audit entries closed without a code change, having been fixed already or being correct
|
||||
as they stand: sync-folder removal is serialized against reconciliation (B41), the drain's
|
||||
database contention was removed by the read-only connection pool and the claim column (B30),
|
||||
and the refusals for symlinks, hard links, device nodes and `chmod` are deliberate — Drive
|
||||
cannot represent them, and a local-only representation no other Proton client can read would
|
||||
be worse than a clean refusal (B31).
|
||||
- Six of this release's fixes were then driven live against the installed 1.8.1 on a real
|
||||
account: the same-second edit uploads (B25), three rapid rewrites of a 64 MiB file publish the
|
||||
third one byte for byte (B32), a write racing a download survives as a conflict copy (B39), an
|
||||
upload torn by a concurrent write heals itself on the following pass (B40), a `getdents64`
|
||||
walk with a 160-byte buffer against a directory mutated between pages returns every entry
|
||||
exactly once (B43), and a restart of a daemon with six mounts and a busy drain stops in 110 ms
|
||||
and comes back at the same thread count (B44). B42 could not be driven from a single machine —
|
||||
see B86 below.
|
||||
- One new bug came out of the verification: an on-demand mount serves a folder listing that
|
||||
predates the sync engine's own uploads, so files the mirror pass created are invisible until
|
||||
the folder is switched back (`docs/BUGS.md` B86). Nothing is lost — they are on the server and
|
||||
come back on the next mirror pass — but they cannot be seen or named while the folder is
|
||||
on-demand, which is also what blocks B42's live verification.
|
||||
|
||||
## [1.8.0] — 2026-08-16
|
||||
|
||||
A queued write whose node had left the local tree was retried every five seconds forever and
|
||||
counted as an ordinary pending upload — 28 of them, holding 18.4 GiB, had been doing that for
|
||||
30 days on the machine that found it (`docs/BUGS.md` B83). Schema: 27 — migration V27 adds
|
||||
`pending_op.access_deferred_since`.
|
||||
|
||||
### Fixed
|
||||
- The drain no longer reads "I have never heard of this node" as "you may not write this
|
||||
node". A missing authority is now resolved against the remote: the node is re-interned if
|
||||
it is still there, and reported as a real failure if it is gone. Ops that had been stuck
|
||||
invisibly for weeks now say what is wrong with them on the first pass.
|
||||
- A create or mkdir resolves its *parent* as the authority, so the refetch asks about the
|
||||
folder rather than the node it has not made yet.
|
||||
|
||||
### Added
|
||||
- Access deferrals are bounded. A deferral still costs no attempt and records no error for
|
||||
the first five minutes — the case it exists for is a share downgraded mid-queue and undone
|
||||
a moment later — but past that window each recheck records a failure, so the op enters the
|
||||
ordinary backoff and appears in `failing_ops` and the `pdfs status` error text like any
|
||||
other stuck operation. The first deferral of a run is logged at `warn` rather than `debug`.
|
||||
|
||||
### Notes
|
||||
- No staged data is touched by any of this. A reported failure keeps its `pending_op` row and
|
||||
its staged blob exactly as before; the change is in what the user is told, and in the queue
|
||||
no longer waiting on a permission change that was never coming.
|
||||
|
||||
## [1.7.1] — 2026-08-16
|
||||
|
||||
Write-back throughput follow-up to 1.7.0, same audit
|
||||
(`docs/plans/audit-2026-08-15-perf-locks.md`). Schema: 26 — migration V26 adds
|
||||
`pending_op.claimed_at`.
|
||||
|
||||
### Added
|
||||
- Parallel write-back drain: three workers share the queue through a `pending_op.claimed_at`
|
||||
claim column (migration V26), so a large upload no longer holds every queued rename, trash
|
||||
and small write behind it. Ordering is preserved per node — the claim query never offers
|
||||
two workers ops for the same uid — and claims left by a crashed run are cleared at open.
|
||||
- An upload already on the wire is cancelled when a newer write, a trash or a discard
|
||||
supersedes it, instead of spending the uplink on a revision the next operation replaces.
|
||||
|
||||
### Changed
|
||||
- The revision debounce is adaptive: it widens toward how long that node's last upload
|
||||
actually took (bounded to 2–60 s), so a file saved faster than it can be sent supersedes
|
||||
in the queue rather than mid-upload.
|
||||
- Workspace deps bumped to `proton-sdk` / `proton-drive-rs` 0.6.0.
|
||||
|
||||
## [1.7.0] — 2026-08-16
|
||||
|
||||
Performance and robustness pass over the mount, from the architecture audit in
|
||||
`docs/plans/audit-2026-08-15-perf-locks.md`. Schema: 25.
|
||||
|
||||
### Added
|
||||
- Sequential-read prefetch (`Core::prefetch_ahead`): per-file sequential detection, depth
|
||||
doubling 2→8, reset to 0 after a seek, skipping blocks already cached, and bounded by its
|
||||
own permit budget so it can never queue ahead of a demand read.
|
||||
- Single-flight block fetching (`BlockFlight`) keyed on `(uid, mtime, size, block)`, so a
|
||||
demand read, kernel read-ahead and prefetch of the same block share one download+decrypt.
|
||||
- In-memory block LRU (`BlockRing`) in front of *both* block paths; a block read off disk is
|
||||
promoted into it, so the following ~31 kernel reads of that 4 MiB block cost nothing.
|
||||
- Read-only SQLite connection pool (`ReadPool` / `Db::read`) with every `SELECT`-only method
|
||||
routed at it, so a read no longer waits on the write connection.
|
||||
- `nodes.path` persisted and backfilled, replacing the recursive `path_of` walk.
|
||||
- Single-writer `flock` on the database; a `SQLITE_CORRUPT`/`NOTADB` file is quarantined as
|
||||
`cache.db.corrupt-<ts>` (WAL sidecars too) and rebuilt rather than failing the mount.
|
||||
- `VACUUM` and the deep `integrity_check` now ack immediately and run as background jobs;
|
||||
`pdfs doctor` polls for the verdict.
|
||||
- Staging reconcile at mount: staged blobs no queued operation refers to are re-queued, or
|
||||
named in the log when they cannot be addressed.
|
||||
- Queue/staging visibility — `Status` gained `parked_uploads`, `failing_ops`,
|
||||
`failing_error`, `staged_bytes`, `staged_oldest_secs`; `pdfs status` prints them when
|
||||
non-zero.
|
||||
- `statfs` implemented against the account quota; FUSE `init` negotiates `max_readahead`,
|
||||
`max_background`, `PARALLEL_DIROPS` and `CACHE_SYMLINKS`; `blksize` raised to 1 MiB.
|
||||
|
||||
### Changed
|
||||
- `State` no longer writes to SQLite under the inode lock — mutations queue into an outbox
|
||||
applied after the lock is released, so interning a 5000-child listing no longer commits a
|
||||
transaction with the whole mount frozen.
|
||||
- Re-listing an unchanged folder no longer touches its subtree; when a subtree really moved,
|
||||
one recursive walk carries both the new path and the inherited trashed flag.
|
||||
- `release(2)` no longer gap-fills from the network: it stages an incomplete blob from local
|
||||
caches only and the drain thread finishes it off the dispatch loop.
|
||||
- A partial write over an undrained one is merged instead of refused.
|
||||
- Size-upgrade waiters park on a queue instead of occupying a worker thread, capped at 8
|
||||
folders in flight; past the cap `stat` answers provisionally.
|
||||
- Block-cache disk writes moved off the read's critical path; no more fsync per cached block.
|
||||
- Cache LRU touches buffered in memory and flushed in one transaction.
|
||||
- Search: sync-folder list and pin lookups hoisted out of the per-hit loop.
|
||||
- SQLite tuned (`cache_size`, `mmap_size`, `temp_store=MEMORY`, `wal_autocheckpoint`) and
|
||||
three new indexes added.
|
||||
- Mount registry lock released before FUSE teardown/join; kernel notifications batched under
|
||||
the state lock and flushed after it.
|
||||
|
||||
### Fixed
|
||||
- **Data loss:** `ENOSPC` is now propagated from `write`/`fsync`/staging instead of being
|
||||
swallowed, with emergency eviction wired up and scratch marked durable before any staging
|
||||
error; a failed `record_pending_write` fails `release` rather than dropping the write.
|
||||
- **Data loss:** the conflict sweep no longer discards queued operations after the trash
|
||||
round trip.
|
||||
- **Data loss:** `rescue_scratch` moves a blob with an unreadable sidecar to `recovery/`
|
||||
unaccompanied instead of fabricating a sidecar for it.
|
||||
- Own-sealed revisions are persisted (7-day TTL), so a restart no longer reopens the
|
||||
transient-download fork window fixed in 1.1.0.
|
||||
- Depth-capped `is_pinned` (cycle-safe), daemon write timeout, and every client-supplied
|
||||
`limit` clamped.
|
||||
- `recover_fsynced_writes` no longer scans the recovery directory on every idle pass.
|
||||
|
||||
## [1.6.0] — 2026-08-12
|
||||
|
||||
Search-index correctness and ranking (`docs/plans/search-index.md`). Schema: 21 —
|
||||
migration V21 rebuilds `nodes_fts`.
|
||||
|
||||
### Fixed
|
||||
- **B80** — 29% of the account was missing from the search index. `node_is_indexable_tx`
|
||||
required a null-parent ancestor, which only the My Files root has, so every node under a
|
||||
device folder was dropped from `nodes_fts` on its first upsert after the v16 backfill
|
||||
(2,705 of 9,252 nodes on the account this was diagnosed against). The rule now matches the
|
||||
backfill, migration V21 replays it, nodes under a trashed folder stay out, and the walk is
|
||||
depth-capped.
|
||||
- **B81** — the prompt labelled every Drive hit "My files", including device folders; pin
|
||||
rows also took `is_dir` from `Pin::recursive`.
|
||||
- **B82** — a `parent_uid` cycle hung the daemon on any short search: the recursive CTEs in
|
||||
`path_of` and the pin walk use `UNION ALL`, which does not deduplicate. `path_of` is now
|
||||
depth-capped at 256 and returns the truncated path.
|
||||
|
||||
### Changed
|
||||
- Local search ranking: whole-query prefix matches get their own lane (newest first) ahead
|
||||
of trigram rank, which on a home directory full of dependency trees was burying the
|
||||
obvious answer — 73% of the 500 best-ranked candidates for "test" came from one Go module
|
||||
cache. The single-character lane, which recovers a typo that destroys every trigram, stays
|
||||
behind it.
|
||||
|
||||
## [1.5.0] — 2026-08-11
|
||||
|
||||
Schema: 20.
|
||||
|
||||
### Added
|
||||
- **`pdfs-prompt --dmenu`** — the same search through an external dmenu-style launcher
|
||||
(fuzzel, rofi, wofi, tofi, bemenu, dmenu) instead of the built-in GTK HUD, so the prompt
|
||||
matches a tiling-WM desktop. A launcher filters a fixed list rather than re-querying per
|
||||
keystroke, so searching is a two-step loop; `--query` skips the first step, fuzzel and rofi
|
||||
get file-type icons. Configurable as `"prompt": { "mode": "dmenu", "menu": [...] }`, with
|
||||
`--gtk` overriding for one invocation.
|
||||
- **Configurable open-with rules** (`pdfs-core/src/opener.rs`): an ordered list of name/class
|
||||
patterns in `config.json` under `open_with`, each mapped to argv and optionally wrapped in
|
||||
a terminal, first match wins, falling back to `xdg-open`. Lets "open a text file" mean
|
||||
`nvim` in a terminal without breaking every graphical caller. Absent config reproduces the
|
||||
old behaviour exactly.
|
||||
|
||||
### Fixed
|
||||
- The prompt and dmenu openers hand applications the cached file while matching rules
|
||||
against the *original* Drive filename, so e.g. `.md` files stop being routed by their
|
||||
cache-path extension.
|
||||
|
||||
## [1.4.0] — 2026-08-10
|
||||
|
||||
Schema: 20.
|
||||
|
||||
### Added
|
||||
- **File version history** (`pdfs-fuse/src/revisions.rs` + a GTK versions dialog): list a
|
||||
file's revisions, restore one, delete one, or write an old one out to a local path. A
|
||||
restore re-points the file server-side, so no content crosses the wire and nothing enters
|
||||
the drain queue; the daemon drops its cached content and repopulates on the next read.
|
||||
- **Persistent SDK entity cache** (`pdfs-core/src/sdkcache.rs`): the SDK's decrypted node
|
||||
metadata now survives a daemon restart instead of re-walking the tree. It lives in its own
|
||||
`sdk_cache.db`, deliberately off the daemon's hot `cache.db` mutex, holds no key material,
|
||||
needs no migrations, and is cleared on logout.
|
||||
- Per-node outcomes for batch operations (`pdfs-core/src/batch.rs`): `trash`/`restore`/
|
||||
`delete`/`move` report one result per node, so a node the server rejects no longer fails
|
||||
the whole batch.
|
||||
- Photo viewer page in the GUI and richer photo metadata in the details pane.
|
||||
|
||||
### Changed
|
||||
- SDK telemetry routed into `tracing`, so its transfer/block-storage/request spans land in
|
||||
the daemon log instead of a no-op sink.
|
||||
- A file that fits in one block uploads as a single atomic request rather than the
|
||||
draft/block/commit sequence.
|
||||
|
||||
## [1.3.0] — 2026-07-31
|
||||
|
||||
Schema: 19.
|
||||
|
||||
### Added
|
||||
- **Photo albums** — `PhotoAlbums` / `AlbumPhotos` control requests, daemon-side `albums.rs`
|
||||
and a GUI albums page (the former gallery view, renamed). Albums other people share with
|
||||
you are included; their photos are persisted per album because a shared album's photos
|
||||
never appear in your own timeline. Served stale-while-refreshing like the timeline.
|
||||
- **GUI "Locations" page** — one row per mount spec: the primary `~/ProtonDrive` mount
|
||||
alongside every folder this computer backs up, mirrored or on-demand. Called *Locations*
|
||||
rather than *Mounts* because a mirror folder is a plain directory with no FUSE session.
|
||||
It absorbs the old Computers page's actions and the mountpoint chooser from Settings.
|
||||
- `role` on shared-node metadata (`viewer`/`editor`/`admin`), defaulted for wire-compat with
|
||||
older clients and daemons.
|
||||
- FUSE acceptance coverage for the permission-based mounts.
|
||||
|
||||
## [1.2.1] — 2026-07-31
|
||||
|
||||
Schema: 18.
|
||||
|
||||
### Fixed
|
||||
- Migration bug that could leave the `ProtonDrive` mount unwritable after upgrade.
|
||||
- Permission error on on-demand mounts.
|
||||
- B78.
|
||||
|
||||
### Changed
|
||||
- Release pipeline: `.deb` signing added and `.rpm` signing fixed.
|
||||
|
||||
## [1.2.0] — 2026-07-29
|
||||
|
||||
Sharing and multi-mount release. Schema: 18.
|
||||
|
||||
### Added
|
||||
- **Shared with me**: shares others sent you are listed, verified and mountable, including on
|
||||
the primary mount.
|
||||
They appear under a synthetic `Shared with me` directory on a virtual volume
|
||||
(`pdfs-fuse/src/virtual.rs`), with name collisions and over-long components resolved by
|
||||
its own naming rules.
|
||||
- Per-share access model (`db/share_access.rs`): the effective role of each shared root is
|
||||
persisted and kept resident in mount state, so interning an inode never costs a query. The
|
||||
database stays the restart/offline authority — every role change is written through before
|
||||
the cache updates. A share you cannot write is presented POSIX read-only.
|
||||
- Unified location model (`pdfs-core/src/mounts.rs`, `db/mounts.rs`): one presentation shape
|
||||
for every local Proton Drive location, joining device sync state with a new `mount` table.
|
||||
`MountMode` is `Mirror` (full local copy, reconciled both ways) or `OnDemand` (FUSE session
|
||||
fetching on access), with an `Unknown` fallback so a mode written by a newer client does
|
||||
not break an older one.
|
||||
|
||||
## [1.1.1] — 2026-07-27
|
||||
|
||||
Schema: 16.
|
||||
|
||||
### Fixed
|
||||
- Trash refresh did not pull updates.
|
||||
- Packaging: `PKGBUILD` was left at 1.1.0 by the 1.1.1 version bump, failing the release
|
||||
workflow's tag/manifest/PKGBUILD agreement check.
|
||||
|
||||
## [1.1.0] — 2026-07-26
|
||||
|
||||
Conflict-handling and concurrency release. Schema: 16.
|
||||
|
||||
### Added
|
||||
- Conflict sweep (`pdfs-fuse/src/sweep.rs`, 5-minute cadence): removes conflict copies proven
|
||||
identical by size + `content_sha1`, surfaces divergent ones once as an activity entry, and
|
||||
never removes what it cannot prove.
|
||||
- Sweep mode and `.pdfsignore`-driven conflict handling; ignore rules are rebuilt per sync
|
||||
pass so edits take effect without a restart.
|
||||
- Substantially extended FUSE acceptance suite.
|
||||
|
||||
### Fixed
|
||||
- **B69** — spurious `(sync-conflict)` copies. Revision identity now uses the server's
|
||||
`revision_id` rather than `(mtime, size)`, which the server re-stamps on seal; the planner
|
||||
gained `AdoptBaseline` so an equal-size no-baseline pair is recorded instead of forked on
|
||||
first reconcile.
|
||||
- **B70** — browser downloads into an on-demand mount had their in-flight temporaries sealed
|
||||
as canonical revisions. Transient names are recognised and held until the name is finalized.
|
||||
- Assorted race conditions and concurrency bugs on the write/drain paths.
|
||||
|
||||
### Changed
|
||||
- Workspace deps bumped to `proton-sdk` / `proton-drive-rs` 0.2.2.
|
||||
|
||||
## [1.0.0] — 2026-07-22
|
||||
|
||||
First stable release: FUSE files-on-demand mount, sync daemon under `proton-drive.service`,
|
||||
`pdfs` CLI and the GTK4 app, on top of the pure-Rust Proton Drive SDK. Schema: 16.
|
||||
|
||||
### Fixed
|
||||
- The outstanding FUSE defects tracked in `docs/BUGS.md`, plus a truncate defect, validated
|
||||
by a new POSIX compliance suite for the filesystem.
|
||||
|
||||
[1.8.2]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.2
|
||||
[1.8.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.1
|
||||
[1.8.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.0
|
||||
[1.7.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.7.1
|
||||
[1.7.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.7.0
|
||||
[1.6.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.6.0
|
||||
[1.5.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.5.0
|
||||
[1.4.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.4.0
|
||||
[1.3.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.3.0
|
||||
[1.2.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.2.1
|
||||
[1.2.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.2.0
|
||||
[1.1.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.1.1
|
||||
[1.1.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.1.0
|
||||
[1.0.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.0.0
|
||||
+2
-1
@@ -14,7 +14,8 @@ Consolidated from the historical feature roadmap; current bug status lives in [`
|
||||
* **Custom FUSE Mount Options (Horizon 1)**: Expose custom mounting parameters in configuration (e.g. `ro` for read-only, `direct_io` to bypass kernel caching, `allow_other`). Auto-generate systemd user mount units upon sync folder configuration.
|
||||
* **Symbolic Link & Hard Link Virtualization (Horizon 2)**: Add symbolic link virtualization inside the metadata store (`NodeType::Symlink`) by intercepting `readlink(2)` and `symlink(2)`. Targets should sync remotely as small encrypted metadata placeholder files on Drive (`proton-vfs-symlink: <target>`).
|
||||
* **Desktop File Manager Integration (Horizon 2)**: Develop shell extension plugins for popular Linux file managers (Nautilus Python plugin, Dolphin Service Menus, Thunar custom actions) to offer context-menu shortcuts ("Pin", "Unpin", "Copy Share Link", "View Version History") and file overlay sync badges.
|
||||
* **Integrated Photo Gallery Enhancements (Horizon 2)**: Support creating and editing Proton Photo Albums (album *writes* are unported in the SDK too) and create an auto-upload Pictures folder pipeline. Exif display, the date-based timeline scrubber and favourites are implemented.
|
||||
* **Integrated Photo Gallery Enhancements (Horizon 2)**: Album *editing* — rename, cover photo, delete, and removing photos from an album — is still unported in the SDK (TS `updateAlbum` / `deleteAlbum` / `removePhotos`), as is adding a photo that lives on someone else's volume (TS `copyPhoto`). Album **creation** and adding own-volume photos landed with the Google Photos import. Still open: an auto-upload Pictures folder pipeline, and a GUI for creating albums by hand. Exif display, the date-based timeline scrubber and favourites are implemented.
|
||||
* **Google Photos import follow-ups**: the importer uploads photos one at a time — a large export would finish sooner with a bounded concurrent upload pool. Trash folders are recognized by a hardcoded list of localized names ([`takeout.rs`](../crates/pdfs-core/src/takeout.rs) `TRASH_FOLDERS`); a locale not on it imports its trash as ordinary photos. Motion-photo `.MP`/`.MV` sidecar videos are ignored rather than uploaded as related photos. Live validation against a real export is pending.
|
||||
* **Random-Access Media Streaming seek support (Horizon 2)**: Intercept out-of-order sparse reads (player seeks) in `pdfs-fuse` and prioritize downloading blocks around the seek offset, aborting/postponing sequential pre-fetches.
|
||||
* **Dynamic Sync Dashboard & Queue Visualization (Horizon 2)**: GUI real-time transfer list (progress bars, speed, ETA), sync history feed, storage usage breakdown, and global pause/resume sync controls.
|
||||
* **Interactive Terminal UI (TUI) Mode (Horizon 3)**: Implement `pdfs tui` using the `ratatui` crate to show transfer speed, sync status, active transfer queue with progress bars, and scrolling daemon logs.
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
# Build + install: cd packaging && makepkg -fi
|
||||
|
||||
pkgname=proton-drive-linux
|
||||
pkgver=1.5.0
|
||||
pkgver=1.9.0
|
||||
pkgrel=1
|
||||
pkgdesc="Proton Drive client for Linux: FUSE files-on-demand mount, CLI, GTK4 app + tray"
|
||||
arch=('x86_64')
|
||||
|
||||
Reference in new issue
Block a user