mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-06 09:12:10 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bd9481902 | ||
|
|
b2d5870834 | ||
|
|
078774f6a1 | ||
|
|
326a9b4e92 |
No files matched your search
Generated
+44
-8
@@ -3538,7 +3538,7 @@ checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-cli"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap 4.6.4",
|
||||
@@ -3553,7 +3553,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-core"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"directories",
|
||||
@@ -3566,15 +3566,17 @@ dependencies = [
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-fuse"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"fuser",
|
||||
"futures",
|
||||
@@ -3585,13 +3587,15 @@ dependencies = [
|
||||
"pdfs-core",
|
||||
"proton-drive-rs",
|
||||
"serde_json",
|
||||
"sha1 0.10.7",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-gui"
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-channel",
|
||||
@@ -3852,9 +3856,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-drive-rs"
|
||||
version = "0.6.0"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe0e39da5b61ceb862af2f4406870b9c1d9a1554889e8d40a32fff13882ec690"
|
||||
checksum = "58322795538d66137d2eb071590375c5287e69059c429e2309dc37368ab5b588"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
@@ -3879,9 +3883,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-sdk"
|
||||
version = "0.6.0"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97cde8cf8e3099d1dbb56302462a632a12e0ff818c012b9c9b0a7a5200544fd6"
|
||||
checksum = "d77482a3160d14926c65537c64bc95cf27d609597a524c97d97d97d1777e9603"
|
||||
dependencies = [
|
||||
"aes-gcm 0.11.0",
|
||||
"async-trait",
|
||||
@@ -5313,6 +5317,12 @@ dependencies = [
|
||||
"cipher 0.4.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
@@ -6254,6 +6264,20 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"typed-path",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.6"
|
||||
@@ -6266,6 +6290,18 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.1"
|
||||
|
||||
+9
-3
@@ -3,15 +3,15 @@ resolver = "2"
|
||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.8.1"
|
||||
version = "1.9.0"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||
rust-version = "1.96"
|
||||
|
||||
[workspace.dependencies]
|
||||
proton-sdk = { version = "0.6.0" }
|
||||
proton-drive-rs = { version = "0.6.0" }
|
||||
proton-sdk = { version = "0.6.1" }
|
||||
proton-drive-rs = { version = "0.6.1" }
|
||||
|
||||
# Internal crates.
|
||||
pdfs-core = { path = "crates/pdfs-core" }
|
||||
@@ -32,6 +32,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
directories = "5"
|
||||
ignore = "0.4"
|
||||
sha2 = "0.10"
|
||||
# Google Takeout exports are zip sets; the importer reads their central
|
||||
# directories and streams entries out without extracting to disk first.
|
||||
zip = { version = "8", default-features = false, features = ["deflate"] }
|
||||
# Proton's photo duplicate detection is keyed on the plaintext SHA-1, so an
|
||||
# import has to speak SHA-1 whatever its age.
|
||||
sha1 = "0.10"
|
||||
# Thumbnail generation for photos the server has none for. Decoders only for the
|
||||
# formats a camera or phone actually produces — the rest of the crate's codecs
|
||||
# would be dead weight in the daemon.
|
||||
|
||||
@@ -158,6 +158,28 @@ enum Command {
|
||||
},
|
||||
/// List the photo albums (including ones shared with this account).
|
||||
Albums,
|
||||
/// Import a Google Photos Takeout export into Proton Photos.
|
||||
///
|
||||
/// Pass every zip of the export at once — Google splits one export across
|
||||
/// numbered parts, and a photo's metadata often sits in a different part
|
||||
/// than the photo. Albums in the export are recreated as Proton albums, and
|
||||
/// photos already on the account (same name and same content) are skipped,
|
||||
/// so an interrupted import can simply be run again.
|
||||
ImportGooglePhotos {
|
||||
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
||||
#[arg(required = true)]
|
||||
archives: Vec<PathBuf>,
|
||||
/// Report what would be imported without uploading anything.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
/// Stay attached and print progress until the import finishes.
|
||||
#[arg(long)]
|
||||
wait: bool,
|
||||
},
|
||||
/// Show how the running (or last) Google Photos import is doing.
|
||||
ImportStatus,
|
||||
/// Stop the running Google Photos import.
|
||||
CancelImport,
|
||||
/// List the photos in one album (newest capture first).
|
||||
Album {
|
||||
/// Album node uid in `volume~link` form (from `pdfs albums`).
|
||||
@@ -589,6 +611,13 @@ fn main() -> Result<()> {
|
||||
} => cmd_photos(limit, offset, favorites),
|
||||
Command::Favorite { uid, remove } => cmd_favorite(uid, remove),
|
||||
Command::Albums => cmd_albums(),
|
||||
Command::ImportGooglePhotos {
|
||||
archives,
|
||||
dry_run,
|
||||
wait,
|
||||
} => cmd_import_google_photos(archives, dry_run, wait),
|
||||
Command::ImportStatus => cmd_import_status(),
|
||||
Command::CancelImport => cmd_cancel_import(),
|
||||
Command::Album { uid, limit, offset } => cmd_album(uid, limit, offset),
|
||||
Command::OpenPhoto { uid } => cmd_open_photo(uid),
|
||||
Command::Search { query, limit } => cmd_search(query, limit),
|
||||
@@ -1711,6 +1740,104 @@ fn cmd_albums() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it gets
|
||||
// absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
|
||||
match control_request(CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
})? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
if !wait {
|
||||
println!("Run `pdfs import-status` to follow it.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Poll rather than stream: the control protocol is request/response, and an
|
||||
// import runs for hours — a held-open socket would be the only long-lived
|
||||
// connection the daemon has.
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus {
|
||||
running: true,
|
||||
summary: _,
|
||||
} => continue,
|
||||
CtlResponse::ImportStatus { summary, .. } => {
|
||||
print_import_summary(summary.as_ref());
|
||||
return Ok(());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_import_status() -> Result<()> {
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus { running, summary } => {
|
||||
if running {
|
||||
println!("Import running. `pdfs transfers` shows its progress.");
|
||||
}
|
||||
print_import_summary(summary.as_ref());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_cancel_import() -> Result<()> {
|
||||
match control_request(CtlRequest::CancelImport)? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
let Some(summary) = summary else {
|
||||
println!("No import has run yet.");
|
||||
return;
|
||||
};
|
||||
println!("Photos in export : {}", summary.found);
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
println!("Already on Proton: {}", summary.duplicates);
|
||||
if summary.skipped_trashed > 0 {
|
||||
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
||||
}
|
||||
if summary.albums_created > 0 || summary.album_links > 0 {
|
||||
println!(
|
||||
"Albums : {} created, {} photos filed",
|
||||
summary.albums_created, summary.album_links
|
||||
);
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
println!("Failed : {}", summary.failed);
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
println!("Uploaded bytes : {}", summary.bytes);
|
||||
}
|
||||
if summary.cancelled {
|
||||
println!("Import was cancelled before it finished.");
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
||||
match control_request(CtlRequest::AlbumPhotos { uid, offset, limit })? {
|
||||
CtlResponse::Photos { items, .. } if items.is_empty() => println!("No photos."),
|
||||
|
||||
@@ -18,6 +18,8 @@ tracing.workspace = true
|
||||
directories.workspace = true
|
||||
ignore.workspace = true
|
||||
sha2.workspace = true
|
||||
sha1.workspace = true
|
||||
zip.workspace = true
|
||||
rusqlite.workspace = true
|
||||
keyring.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
+862
-67
File diff suppressed because it is too large.
Load diff
@@ -103,6 +103,33 @@ pub enum Request {
|
||||
/// re-encrypted for our timeline, which the SDK does not implement yet; the
|
||||
/// daemon reports that as an error rather than silently doing nothing.
|
||||
SetPhotoFavorite { uid: String, favorite: bool },
|
||||
/// Import Google Photos **Takeout** archives into Proton Photos.
|
||||
///
|
||||
/// `archives` are absolute paths to the export's `.zip` parts on the
|
||||
/// daemon's own filesystem — the whole set at once, because a photo and its
|
||||
/// metadata sidecar routinely land in different parts. Photos already on the
|
||||
/// account (matched by name *and* content digest) are skipped, so a library
|
||||
/// partly synced by Proton Photos elsewhere does not import twice, and an
|
||||
/// interrupted import resumes by simply being run again.
|
||||
///
|
||||
/// With `dry_run` the daemon scans and reports what it *would* do without
|
||||
/// uploading anything. Otherwise it acks immediately with
|
||||
/// [`Response::Ok`] and works in the background — an export is hours of
|
||||
/// upload, far past any socket timeout. Progress shows up in
|
||||
/// [`Request::GetQueueStatus`] as a job, and the final counts in
|
||||
/// [`Request::ImportStatus`].
|
||||
ImportTakeout {
|
||||
archives: Vec<String>,
|
||||
#[serde(default)]
|
||||
dry_run: bool,
|
||||
},
|
||||
/// How the running (or last finished) Takeout import is doing. Replies with
|
||||
/// [`Response::ImportStatus`].
|
||||
ImportStatus,
|
||||
/// Ask the running Takeout import to stop. It finishes the photo on the wire
|
||||
/// and files what it has uploaded into its albums, then reports
|
||||
/// `cancelled`. Replies with [`Response::Ok`].
|
||||
CancelImport,
|
||||
/// Upload the photo at `source_path` under the given name and media type.
|
||||
///
|
||||
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
||||
@@ -1052,6 +1079,35 @@ pub struct PhotoMonth {
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
/// How a Google Photos Takeout import is going, or how the last one ended
|
||||
/// ([`Response::ImportStatus`]).
|
||||
///
|
||||
/// The counts are cumulative for the run, so the same shape serves progress and
|
||||
/// the final report. `found` counts *distinct* photos: Google stores a separate
|
||||
/// copy of a photo in each album folder it appears in, and those fold into one
|
||||
/// upload that joins several albums.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ImportSummary {
|
||||
/// Distinct photos the export holds.
|
||||
pub found: usize,
|
||||
/// Photos uploaded by this run.
|
||||
pub uploaded: usize,
|
||||
/// Photos already on the account, matched by name *and* content digest.
|
||||
pub duplicates: usize,
|
||||
/// Photos that could not be read or uploaded.
|
||||
pub failed: usize,
|
||||
/// Photos in the export's trash bucket, which are never imported.
|
||||
pub skipped_trashed: usize,
|
||||
/// Albums created by this run.
|
||||
pub albums_created: usize,
|
||||
/// Album memberships added by this run.
|
||||
pub album_links: usize,
|
||||
/// Bytes uploaded.
|
||||
pub bytes: u64,
|
||||
/// True when the run stopped early because it was cancelled.
|
||||
pub cancelled: bool,
|
||||
}
|
||||
|
||||
/// One album in a [`Response::Albums`] listing.
|
||||
///
|
||||
/// An album is a folder node on the photos volume carrying album properties, so
|
||||
@@ -1270,6 +1326,14 @@ pub enum Response {
|
||||
available: bool,
|
||||
items: Vec<AlbumInfo>,
|
||||
},
|
||||
/// How the Takeout import is doing (reply to [`Request::ImportStatus`]).
|
||||
/// `running` is false once it has finished, when `summary` is the final
|
||||
/// report; `summary` is `None` when no import has run this session.
|
||||
ImportStatus {
|
||||
running: bool,
|
||||
#[serde(default)]
|
||||
summary: Option<ImportSummary>,
|
||||
},
|
||||
/// The months the timeline spans (reply to [`Request::PhotoMonths`]),
|
||||
/// newest first.
|
||||
PhotoMonths { months: Vec<PhotoMonth> },
|
||||
|
||||
@@ -1895,6 +1895,123 @@ fn failed_superseding_insert_keeps_the_old_pending_op() {
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||
}
|
||||
|
||||
/// B53's remaining half. The insert-failure regression above proves the *row*
|
||||
/// rolls back; this proves the two things that make the rollback usable — the
|
||||
/// old blob is not reported as superseded (so the caller does not delete the
|
||||
/// bytes the surviving row still points at), and the rollback survives a
|
||||
/// restart rather than living only in the connection that failed.
|
||||
#[test]
|
||||
fn a_rolled_back_supersede_keeps_its_blob_and_survives_reopen() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-db-b53-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("queue.sqlite");
|
||||
let op = |blob: &str| PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("durable").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some(blob.to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch(
|
||||
"CREATE TRIGGER reject_replacement BEFORE INSERT ON pending_op
|
||||
WHEN NEW.blob_path = '/staging/replacement'
|
||||
BEGIN SELECT RAISE(ABORT, 'injected insert failure'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// The failure must not hand back the old blob path. That value is the
|
||||
// caller's instruction to delete those bytes, and the row that owns
|
||||
// them is still queued.
|
||||
assert!(db.enqueue_op(&op("/staging/replacement")).is_err());
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch("DROP TRIGGER reject_replacement")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1, "old or new — never neither");
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||
|
||||
// And once the injected failure is gone, superseding works and *does*
|
||||
// report the blob it retired.
|
||||
let (_, superseded) = db.enqueue_op(&op("/staging/replacement")).unwrap();
|
||||
assert_eq!(superseded.as_deref(), Some("/staging/original"));
|
||||
assert_eq!(db.pending_ops().unwrap().len(), 1);
|
||||
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
/// A full disk, as SQLite reports it. `max_page_count` makes the database
|
||||
/// refuse to grow, which is the same `SQLITE_FULL` an out-of-space filesystem
|
||||
/// produces — and the queue must come out of it with the acknowledged work
|
||||
/// still in it (B53).
|
||||
#[test]
|
||||
fn a_full_database_rolls_the_supersede_back_rather_than_erasing_it() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let op = |blob: &str| PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("full").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some(blob.to_string()),
|
||||
meta_json: Some("{}".to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||
|
||||
// Cap the file at its current size, so any page the insert needs fails.
|
||||
let pages: i64 = db
|
||||
.with_conn(|conn| Ok(conn.query_row("PRAGMA page_count", [], |r| r.get(0))?))
|
||||
.unwrap();
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch(&format!("PRAGMA max_page_count = {pages}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// A payload far larger than one page, so growth is unavoidable.
|
||||
let mut fat = op("/staging/replacement");
|
||||
fat.meta_json = Some("x".repeat(512 * 1024));
|
||||
let result = db.enqueue_op(&fat);
|
||||
|
||||
db.with_conn(|conn| {
|
||||
conn.execute_batch("PRAGMA max_page_count = 1073741823")?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(result.is_err(), "a database that cannot grow must refuse");
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(
|
||||
ops[0].blob_path.as_deref(),
|
||||
Some("/staging/original"),
|
||||
"the acknowledged upload is still queued and still owns its bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_trash_replacement_returns_owned_blobs_after_commit() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
|
||||
@@ -20,6 +20,7 @@ pub mod search;
|
||||
pub mod service;
|
||||
pub mod shell;
|
||||
pub mod syncignore;
|
||||
pub mod takeout;
|
||||
|
||||
pub use access::{Access, access_for, perm_bits};
|
||||
pub use error::{CoreError, CoreResult, Error, Result};
|
||||
|
||||
@@ -0,0 +1,887 @@
|
||||
//! Reading a Google Photos **Takeout** export: what is in the archives, which
|
||||
//! album each photo belongs to, and when it was taken.
|
||||
//!
|
||||
//! This module is pure structure — it opens the archives' central directories
|
||||
//! and reads the small JSON sidecars, never the media bytes. The import itself
|
||||
//! (hashing, duplicate detection, upload, album creation) lives in the daemon;
|
||||
//! everything here is offline-testable and has no Proton dependency.
|
||||
//!
|
||||
//! # What a Takeout export looks like
|
||||
//!
|
||||
//! Google splits an export into numbered zips (`takeout-20260817T...-001.zip`,
|
||||
//! `-002.zip`, …) that are *independent* archives — no zip64 spanning — but a
|
||||
//! photo and its JSON sidecar can land in different parts, and one album's
|
||||
//! folder can be spread over several. So the scan treats the whole set as one
|
||||
//! namespace keyed by the path inside the export:
|
||||
//!
|
||||
//! ```text
|
||||
//! Takeout/Google Photos/Iceland 2019/IMG_0042.jpg
|
||||
//! Takeout/Google Photos/Iceland 2019/IMG_0042.jpg.supplemental-metadata.json
|
||||
//! Takeout/Google Photos/Iceland 2019/metadata.json <- album title
|
||||
//! Takeout/Google Photos/Photos from 2019/IMG_0043.jpg <- no album
|
||||
//! Takeout/Google Photos/Trash/IMG_0044.jpg <- skipped
|
||||
//! ```
|
||||
//!
|
||||
//! The service folder (`Google Photos`) is localized, as are the year folders
|
||||
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
||||
//! service name: the first component is dropped when it is the export root, the
|
||||
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
|
||||
//! **album** when it carries an album metadata sidecar naming it; anything else
|
||||
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
||||
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
||||
//! unavoidable, because a trashed photo is not marked as such anywhere else.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::{CoreError, CoreResult};
|
||||
|
||||
/// Folder names Google uses for the trash bucket, lowercased. A Takeout export
|
||||
/// only contains one when the user asked for it, and its photos are deleted —
|
||||
/// importing them would resurrect them, so they are skipped.
|
||||
///
|
||||
/// Localized, and necessarily incomplete: a locale not listed here imports its
|
||||
/// trash as ordinary timeline photos. Extend as reports come in rather than
|
||||
/// guessing at transliterations.
|
||||
pub const TRASH_FOLDERS: &[&str] = &[
|
||||
"trash",
|
||||
"bin",
|
||||
"papierkorb",
|
||||
"corbeille",
|
||||
"papelera",
|
||||
"papelera de reciclaje",
|
||||
"cestino",
|
||||
"lixeira",
|
||||
"prullenbak",
|
||||
"kosz",
|
||||
"koš",
|
||||
"skräpkorgen",
|
||||
"papperskorg",
|
||||
"papirkurv",
|
||||
"roskakori",
|
||||
"kuka",
|
||||
"回收站",
|
||||
"回收筒",
|
||||
"ゴミ箱",
|
||||
"휴지통",
|
||||
"корзина",
|
||||
"кошик",
|
||||
"çöp kutusu",
|
||||
"σκουπίδια",
|
||||
];
|
||||
|
||||
/// Sidecar file names that describe the *folder* (its album title) rather than
|
||||
/// one photo, lowercased. Google has renamed this file more than once.
|
||||
const ALBUM_METADATA_NAMES: &[&str] = &[
|
||||
"metadata.json",
|
||||
"album-metadata.json",
|
||||
"print-subscriptions.json",
|
||||
"shared_album_comments.json",
|
||||
"user-generated-memory-titles.json",
|
||||
];
|
||||
|
||||
/// The suffix Google appends to a photo's sidecar, lowercased. Truncation means
|
||||
/// only a *prefix* of it may survive (see [`strip_sidecar_suffix`]).
|
||||
const SUPPLEMENTAL: &str = ".supplemental-metadata";
|
||||
|
||||
/// File extensions imported as photos or videos. Anything else in the export —
|
||||
/// `archive_browser.html`, `.csv` activity logs, `print-subscriptions.json` —
|
||||
/// is not media and is ignored.
|
||||
const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
|
||||
("jpg", "image/jpeg"),
|
||||
("jpeg", "image/jpeg"),
|
||||
("png", "image/png"),
|
||||
("gif", "image/gif"),
|
||||
("webp", "image/webp"),
|
||||
("bmp", "image/bmp"),
|
||||
("tif", "image/tiff"),
|
||||
("tiff", "image/tiff"),
|
||||
("heic", "image/heic"),
|
||||
("heif", "image/heif"),
|
||||
("avif", "image/avif"),
|
||||
("dng", "image/x-adobe-dng"),
|
||||
("cr2", "image/x-canon-cr2"),
|
||||
("cr3", "image/x-canon-cr3"),
|
||||
("nef", "image/x-nikon-nef"),
|
||||
("arw", "image/x-sony-arw"),
|
||||
("orf", "image/x-olympus-orf"),
|
||||
("rw2", "image/x-panasonic-rw2"),
|
||||
("raf", "image/x-fuji-raf"),
|
||||
("mp4", "video/mp4"),
|
||||
("m4v", "video/x-m4v"),
|
||||
("mov", "video/quicktime"),
|
||||
("avi", "video/x-msvideo"),
|
||||
("mkv", "video/x-matroska"),
|
||||
("webm", "video/webm"),
|
||||
("3gp", "video/3gpp"),
|
||||
("mts", "video/mp2t"),
|
||||
("m2ts", "video/mp2t"),
|
||||
("mpg", "video/mpeg"),
|
||||
("mpeg", "video/mpeg"),
|
||||
];
|
||||
|
||||
/// The media type for `name`'s extension, or `None` when it is not media this
|
||||
/// import handles.
|
||||
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
||||
let ext = Path::new(name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())?
|
||||
.to_ascii_lowercase();
|
||||
MEDIA_EXTENSIONS
|
||||
.iter()
|
||||
.find(|(candidate, _)| *candidate == ext)
|
||||
.map(|(_, media_type)| *media_type)
|
||||
}
|
||||
|
||||
/// Where a photo sits in the export, which decides whether it joins an album.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Bucket {
|
||||
/// A named album folder — the photo joins that album *and* the timeline.
|
||||
Album(String),
|
||||
/// A year folder (`Photos from 2019`) or the export root: timeline only.
|
||||
Timeline,
|
||||
/// The archive bucket: timeline only, like Google shows it.
|
||||
Archived,
|
||||
}
|
||||
|
||||
/// One media file found in the export.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutPhoto {
|
||||
/// Index into the archive list the scan was given.
|
||||
pub archive: usize,
|
||||
/// The entry's full path inside that archive — how the importer reopens it.
|
||||
pub entry: String,
|
||||
/// The name to upload under: the sidecar's `title` when it has one (Google
|
||||
/// mangles long names on disk but keeps the original there), else the file
|
||||
/// name on disk.
|
||||
pub name: String,
|
||||
/// Uncompressed size in bytes, from the zip's central directory.
|
||||
pub size: u64,
|
||||
/// Media type from the extension.
|
||||
pub media_type: String,
|
||||
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
|
||||
/// `creationTime`). `None` when the photo has no sidecar — the upload then
|
||||
/// defaults it to now, as the SDK does.
|
||||
pub capture_time: Option<i64>,
|
||||
/// The sidecar's `favorited` flag.
|
||||
pub favorite: bool,
|
||||
/// Which bucket the file was found in.
|
||||
pub bucket: Bucket,
|
||||
}
|
||||
|
||||
/// Everything one scan found, ready to be turned into an import plan.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutScan {
|
||||
/// Every importable media file, in archive then entry order.
|
||||
pub photos: Vec<TakeoutPhoto>,
|
||||
/// Album names in the order first seen, so the importer creates them in a
|
||||
/// stable order.
|
||||
pub albums: Vec<String>,
|
||||
/// Media files skipped because they sit in the trash bucket.
|
||||
pub skipped_trashed: usize,
|
||||
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
||||
/// activity `.csv`s, `.mp` motion-photo parts).
|
||||
pub skipped_other: usize,
|
||||
}
|
||||
|
||||
impl TakeoutScan {
|
||||
/// Total bytes of media to be read, before duplicate detection.
|
||||
pub fn total_bytes(&self) -> u64 {
|
||||
self.photos.iter().map(|p| p.size).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
||||
/// file also carries geo data, people tags and view counts.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct PhotoSidecar {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
#[serde(rename = "photoTakenTime", default)]
|
||||
photo_taken_time: Option<SidecarTime>,
|
||||
#[serde(rename = "creationTime", default)]
|
||||
creation_time: Option<SidecarTime>,
|
||||
#[serde(default)]
|
||||
favorited: bool,
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, as a *string*.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarTime {
|
||||
#[serde(default)]
|
||||
timestamp: Option<String>,
|
||||
}
|
||||
|
||||
impl SidecarTime {
|
||||
fn epoch_seconds(&self) -> Option<i64> {
|
||||
self.timestamp.as_ref()?.parse().ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// A folder-level sidecar. `title` is the album name as the user typed it,
|
||||
/// which is what the folder name is derived (and sometimes truncated) from.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct AlbumSidecar {
|
||||
#[serde(default)]
|
||||
title: Option<String>,
|
||||
}
|
||||
|
||||
/// One entry as the scan sees it, independent of where it came from.
|
||||
struct RawEntry {
|
||||
archive: usize,
|
||||
entry: String,
|
||||
/// `<folder>/<file>` with the export root and service folder removed.
|
||||
relative: String,
|
||||
size: u64,
|
||||
}
|
||||
|
||||
impl RawEntry {
|
||||
/// The folder component of `relative`, or `""` at the service root.
|
||||
fn folder(&self) -> &str {
|
||||
match self.relative.rfind('/') {
|
||||
Some(index) => &self.relative[..index],
|
||||
None => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// The file name component of `relative`.
|
||||
fn file_name(&self) -> &str {
|
||||
match self.relative.rfind('/') {
|
||||
Some(index) => &self.relative[index + 1..],
|
||||
None => &self.relative,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan a set of Takeout zip archives into one plan.
|
||||
///
|
||||
/// The archives are treated as one export: an album folder split across parts
|
||||
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
||||
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
||||
/// this stays fast on a 200 GB export.
|
||||
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<RawEntry> = Vec::new();
|
||||
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
|
||||
for (index, path) in archives.iter().enumerate() {
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
|
||||
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
|
||||
})?;
|
||||
|
||||
for position in 0..zip.len() {
|
||||
let mut entry = zip.by_index(position).map_err(|e| {
|
||||
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
|
||||
})?;
|
||||
if entry.is_dir() {
|
||||
continue;
|
||||
}
|
||||
// `enclosed_name` refuses paths that escape the archive root, which
|
||||
// is exactly the check we want before using one as a key.
|
||||
let Some(name) = entry.enclosed_name() else {
|
||||
continue;
|
||||
};
|
||||
let name = name.to_string_lossy().replace('\\', "/");
|
||||
let Some(relative) = strip_export_prefix(&name) else {
|
||||
continue;
|
||||
};
|
||||
let size = entry.size();
|
||||
|
||||
// Sidecars are read now, while the entry is open: they are a few
|
||||
// hundred bytes each and reopening the archive per photo later would
|
||||
// be a second full pass.
|
||||
if relative.to_ascii_lowercase().ends_with(".json") {
|
||||
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.insert(relative.to_string(), body);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push(RawEntry {
|
||||
archive: index,
|
||||
entry: name.clone(),
|
||||
relative: relative.to_string(),
|
||||
size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(assemble(entries, &sidecars))
|
||||
}
|
||||
|
||||
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
||||
/// inside the export. Split out from [`scan_archives`] so the matching rules can
|
||||
/// be tested without building zip fixtures.
|
||||
fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> TakeoutScan {
|
||||
// Album titles per folder, from the folder-level sidecars.
|
||||
let mut album_titles: HashMap<String, String> = HashMap::new();
|
||||
for (path, body) in sidecars {
|
||||
let (folder, file) = split_folder(path);
|
||||
if !ALBUM_METADATA_NAMES.contains(&file.to_ascii_lowercase().as_str()) {
|
||||
continue;
|
||||
}
|
||||
if let Ok(album) = serde_json::from_slice::<AlbumSidecar>(body)
|
||||
&& let Some(title) = album.title.filter(|t| !t.trim().is_empty())
|
||||
{
|
||||
album_titles.insert(folder.to_string(), title);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-photo sidecars, indexed by folder so a truncated name only ever
|
||||
// matches within its own folder.
|
||||
let mut per_folder_sidecars: HashMap<String, BTreeMap<String, &Vec<u8>>> = HashMap::new();
|
||||
for (path, body) in sidecars {
|
||||
let (folder, file) = split_folder(path);
|
||||
if ALBUM_METADATA_NAMES.contains(&file.to_ascii_lowercase().as_str()) {
|
||||
continue;
|
||||
}
|
||||
per_folder_sidecars
|
||||
.entry(folder.to_string())
|
||||
.or_default()
|
||||
.insert(file.to_string(), body);
|
||||
}
|
||||
|
||||
let mut scan = TakeoutScan::default();
|
||||
let mut seen_albums: HashMap<String, ()> = HashMap::new();
|
||||
|
||||
for raw in &entries {
|
||||
let folder = raw.folder().to_string();
|
||||
let file_name = raw.file_name().to_string();
|
||||
|
||||
if is_trash_folder(&folder) {
|
||||
if media_type_of(&file_name).is_some() {
|
||||
scan.skipped_trashed += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let Some(media_type) = media_type_of(&file_name) else {
|
||||
scan.skipped_other += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
let sidecar = per_folder_sidecars
|
||||
.get(&folder)
|
||||
.and_then(|in_folder| find_sidecar(&file_name, in_folder))
|
||||
.and_then(|body| serde_json::from_slice::<PhotoSidecar>(body).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
let bucket = match album_titles.get(&folder) {
|
||||
Some(title) => Bucket::Album(title.clone()),
|
||||
None if is_archive_folder(&folder) => Bucket::Archived,
|
||||
None => Bucket::Timeline,
|
||||
};
|
||||
if let Bucket::Album(title) = &bucket
|
||||
&& seen_albums.insert(title.clone(), ()).is_none()
|
||||
{
|
||||
scan.albums.push(title.clone());
|
||||
}
|
||||
|
||||
// The sidecar's `title` is the name Google had before it mangled the
|
||||
// on-disk one (truncation, `(1)` suffixes, character replacement), so it
|
||||
// is the better name to upload under — but only when it still looks like
|
||||
// the same file, i.e. the extension agrees.
|
||||
let name = sidecar
|
||||
.title
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|title| !title.is_empty() && same_extension(title, &file_name))
|
||||
.unwrap_or(&file_name)
|
||||
.to_string();
|
||||
|
||||
scan.photos.push(TakeoutPhoto {
|
||||
archive: raw.archive,
|
||||
entry: raw.entry.clone(),
|
||||
name,
|
||||
size: raw.size,
|
||||
media_type: media_type.to_string(),
|
||||
capture_time: sidecar
|
||||
.photo_taken_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
.or_else(|| {
|
||||
sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
}),
|
||||
favorite: sidecar.favorited,
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
|
||||
scan
|
||||
}
|
||||
|
||||
/// Drop the export root (`Takeout/`) and the localized service folder from a
|
||||
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
|
||||
///
|
||||
/// `None` for a path with no service folder at all (the export's own
|
||||
/// `archive_browser.html`), which carries nothing to import.
|
||||
fn strip_export_prefix(path: &str) -> Option<&str> {
|
||||
let mut rest = path;
|
||||
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
|
||||
// the user may not have it — so the root is dropped only when a service
|
||||
// folder follows it, which the component count decides.
|
||||
let components = rest.split('/').count();
|
||||
if components >= 3 {
|
||||
rest = rest.split_once('/')?.1;
|
||||
}
|
||||
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
|
||||
let (_service, tail) = rest.split_once('/')?;
|
||||
Some(tail)
|
||||
}
|
||||
|
||||
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
||||
fn split_folder(path: &str) -> (&str, &str) {
|
||||
match path.rfind('/') {
|
||||
Some(index) => (&path[..index], &path[index + 1..]),
|
||||
None => ("", path),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_trash_folder(folder: &str) -> bool {
|
||||
let name = folder.rsplit('/').next().unwrap_or(folder).to_lowercase();
|
||||
TRASH_FOLDERS.contains(&name.as_str())
|
||||
}
|
||||
|
||||
/// Google's archive bucket, which is not an album but is not the plain timeline
|
||||
/// either. Only the English name is matched; a missed locale simply imports as
|
||||
/// [`Bucket::Timeline`], which is the same upload either way.
|
||||
fn is_archive_folder(folder: &str) -> bool {
|
||||
let name = folder.rsplit('/').next().unwrap_or(folder).to_lowercase();
|
||||
name == "archive"
|
||||
}
|
||||
|
||||
/// Whether two file names end in the same extension, case-insensitively.
|
||||
fn same_extension(a: &str, b: &str) -> bool {
|
||||
let ext = |name: &str| {
|
||||
Path::new(name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(str::to_ascii_lowercase)
|
||||
};
|
||||
ext(a) == ext(b)
|
||||
}
|
||||
|
||||
/// Strip a sidecar's `.json` and any `.supplemental-metadata` suffix, including
|
||||
/// the *truncated* forms Google produces.
|
||||
///
|
||||
/// Google caps the sidecar's file name at 51 characters, so a long photo name
|
||||
/// leaves only a prefix of the suffix behind — `IMG_1234.jpg.supplemental-me`,
|
||||
/// `…-metad`, or nothing at all. Any non-empty prefix of the suffix is stripped;
|
||||
/// a bare `.json` sidecar loses only that.
|
||||
fn strip_sidecar_suffix(sidecar: &str) -> String {
|
||||
let stem = sidecar
|
||||
.strip_suffix(".json")
|
||||
.or_else(|| sidecar.strip_suffix(".JSON"))
|
||||
.unwrap_or(sidecar);
|
||||
// Longest match first so `.supplemental-metadata` wins over `.s`.
|
||||
for length in (2..=SUPPLEMENTAL.len()).rev() {
|
||||
let candidate = &SUPPLEMENTAL[..length];
|
||||
if let Some(head) = stem.strip_suffix(candidate) {
|
||||
return head.to_string();
|
||||
}
|
||||
}
|
||||
stem.to_string()
|
||||
}
|
||||
|
||||
/// Move a trailing `(N)` from after the extension to before it: Google writes
|
||||
/// the sidecar of `IMG_1234(1).jpg` as `IMG_1234.jpg(1).json`.
|
||||
fn unswap_duplicate_marker(stem: &str) -> Option<String> {
|
||||
let marker_start = stem.rfind('(')?;
|
||||
let marker = &stem[marker_start..];
|
||||
if !marker.ends_with(')')
|
||||
|| !marker[1..marker.len() - 1]
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_digit())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let head = &stem[..marker_start];
|
||||
let dot = head.rfind('.')?;
|
||||
Some(format!("{}{}{}", &head[..dot], marker, &head[dot..]))
|
||||
}
|
||||
|
||||
/// Find the sidecar belonging to `file_name` among the JSON files of its folder.
|
||||
///
|
||||
/// Tried in order of confidence: the exact name, the `(N)`-swapped name, then a
|
||||
/// prefix match for the case where truncation cut into the photo name itself.
|
||||
fn find_sidecar<'a>(
|
||||
file_name: &str,
|
||||
in_folder: &BTreeMap<String, &'a Vec<u8>>,
|
||||
) -> Option<&'a Vec<u8>> {
|
||||
let mut prefix_match: Option<&'a Vec<u8>> = None;
|
||||
let mut prefix_len = 0usize;
|
||||
|
||||
for (sidecar_name, body) in in_folder {
|
||||
let stem = strip_sidecar_suffix(sidecar_name);
|
||||
if stem == file_name {
|
||||
return Some(body);
|
||||
}
|
||||
if unswap_duplicate_marker(&stem).as_deref() == Some(file_name) {
|
||||
return Some(body);
|
||||
}
|
||||
// Truncated: the stem is a prefix of the photo's name. Keep the longest
|
||||
// such prefix — a folder holding `IMG_1.jpg` and `IMG_12.jpg` must not
|
||||
// let the shorter stem claim the longer photo.
|
||||
if !stem.is_empty() && file_name.starts_with(&stem) && stem.len() > prefix_len {
|
||||
prefix_len = stem.len();
|
||||
prefix_match = Some(body);
|
||||
}
|
||||
}
|
||||
prefix_match
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn entry(archive: usize, path: &str, size: u64) -> RawEntry {
|
||||
RawEntry {
|
||||
archive,
|
||||
entry: path.to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().to_string(),
|
||||
size,
|
||||
}
|
||||
}
|
||||
|
||||
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
||||
let relative = strip_export_prefix(path).unwrap().to_string();
|
||||
map.insert(relative, body.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_export_root_and_localized_service_folder() {
|
||||
assert_eq!(
|
||||
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
||||
Some("Island 2019/IMG_1.jpg")
|
||||
);
|
||||
// Already unwrapped by the user: no `Takeout/` root left.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
||||
Some("IMG_1.jpg")
|
||||
);
|
||||
// Nothing below a service folder is nothing to import.
|
||||
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn album_folder_becomes_an_album_year_folder_does_not() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland 2019/metadata.json",
|
||||
r#"{"title":"Iceland 2019"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_2.jpg", 20),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.albums, vec!["Iceland 2019"]);
|
||||
assert_eq!(scan.photos[0].bucket, Bucket::Album("Iceland 2019".into()));
|
||||
assert_eq!(scan.photos[1].bucket, Bucket::Timeline);
|
||||
assert_eq!(scan.total_bytes(), 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trashed_photos_are_skipped_and_counted() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Papierkorb/IMG_1.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Bin/IMG_2.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_3.jpg", 10),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(scan.skipped_trashed, 2);
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_3.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_media_entries_are_ignored() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/notes.txt", 10),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.skipped_other, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_supplies_capture_time_and_favourite() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.supplemental-metadata.json",
|
||||
r#"{"title":"IMG_1.jpg","photoTakenTime":{"timestamp":"1560000000"},"favorited":true}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert!(scan.photos[0].favorite);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creation_time_is_the_fallback_capture_time() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_supplemental_suffixes_still_match() {
|
||||
// Google caps the sidecar name at 51 chars, cutting into the suffix.
|
||||
assert_eq!(
|
||||
strip_sidecar_suffix("IMG_1.jpg.supplemental-me.json"),
|
||||
"IMG_1.jpg"
|
||||
);
|
||||
assert_eq!(
|
||||
strip_sidecar_suffix("IMG_1.jpg.supplemental-metadata.json"),
|
||||
"IMG_1.jpg"
|
||||
);
|
||||
assert_eq!(strip_sidecar_suffix("IMG_1.jpg.json"), "IMG_1.jpg");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_marker_after_the_extension_is_moved_back() {
|
||||
assert_eq!(
|
||||
unswap_duplicate_marker("IMG_1.jpg(1)").as_deref(),
|
||||
Some("IMG_1(1).jpg")
|
||||
);
|
||||
assert_eq!(unswap_duplicate_marker("IMG_1.jpg"), None);
|
||||
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg(1).json",
|
||||
r#"{"photoTakenTime":{"timestamp":"1600000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1(1).jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_600_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longest_prefix_wins_when_the_photo_name_itself_was_truncated() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.json",
|
||||
r#"{"photoTakenTime":{"timestamp":"1"}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_12.json",
|
||||
r#"{"photoTakenTime":{"timestamp":"2"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_12345.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_title_replaces_a_mangled_on_disk_name() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/a_very_long_name_that_google_.jpg.json",
|
||||
r#"{"title":"a_very_long_name_that_google_truncated.jpg"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/a_very_long_name_that_google_.jpg",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
scan.photos[0].name,
|
||||
"a_very_long_name_that_google_truncated.jpg"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_title_with_a_different_extension_is_not_trusted() {
|
||||
// Motion photos name their `.MP` part after the `.jpg`; taking the title
|
||||
// verbatim would upload the video under an image name.
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.mp4.json",
|
||||
r#"{"title":"IMG_1.jpg"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.mp4",
|
||||
10,
|
||||
)],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.mp4");
|
||||
assert_eq!(scan.photos[0].media_type, "video/mp4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_album_split_across_archives_stays_one_album() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland 2019/metadata.json",
|
||||
r#"{"title":"Iceland 2019"}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", 10),
|
||||
entry(1, "Takeout/Google Photos/Iceland 2019/IMG_2.jpg", 10),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.albums, vec!["Iceland 2019"]);
|
||||
assert_eq!(scan.photos.len(), 2);
|
||||
assert_eq!(scan.photos[1].archive, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_bucket_imports_to_the_timeline() {
|
||||
let scan = assemble(
|
||||
vec![entry(0, "Takeout/Google Photos/Archive/IMG_1.jpg", 10)],
|
||||
&HashMap::new(),
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].bucket, Bucket::Archived);
|
||||
assert!(scan.albums.is_empty());
|
||||
}
|
||||
|
||||
/// Write a zip holding `files` and return its path. No `tempfile`
|
||||
/// dev-dependency: the daemon crates deliberately carry none, and one
|
||||
/// uniquely named file under the temp dir is all this needs.
|
||||
fn write_zip(tag: &str, files: &[(&str, &[u8])]) -> PathBuf {
|
||||
use std::io::Write;
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-takeout-test-{tag}-{}.zip",
|
||||
std::process::id()
|
||||
));
|
||||
let file = std::fs::File::create(&path).unwrap();
|
||||
let mut zip = zip::ZipWriter::new(file);
|
||||
let options: zip::write::FileOptions<'_, ()> =
|
||||
zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
for (name, body) in files {
|
||||
zip.start_file(*name, options).unwrap();
|
||||
zip.write_all(body).unwrap();
|
||||
}
|
||||
zip.finish().unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scans_a_two_part_export_where_the_sidecar_is_in_the_other_part() {
|
||||
// The case that makes the whole set one scan: part 1 has the photo,
|
||||
// part 2 has its metadata and the album title.
|
||||
let first = write_zip(
|
||||
"part1",
|
||||
&[(
|
||||
"Takeout/Google Photos/Iceland 2019/IMG_1.jpg",
|
||||
b"the photo bytes",
|
||||
)],
|
||||
);
|
||||
let second = write_zip(
|
||||
"part2",
|
||||
&[
|
||||
(
|
||||
"Takeout/Google Photos/Iceland 2019/metadata.json",
|
||||
br#"{"title":"Iceland 2019"}"#.as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Photos/Iceland 2019/IMG_1.jpg.supplemental-metadata.json",
|
||||
br#"{"title":"IMG_1.jpg","photoTakenTime":{"timestamp":"1560000000"}}"#
|
||||
.as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Photos/Trash/IMG_9.jpg",
|
||||
b"deleted".as_slice(),
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
let scan = scan_archives(&[first.clone(), second.clone()]).unwrap();
|
||||
let _ = std::fs::remove_file(&first);
|
||||
let _ = std::fs::remove_file(&second);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
|
||||
assert_eq!(scan.photos[0].archive, 0, "the photo is in part 1");
|
||||
assert_eq!(scan.photos[0].size, "the photo bytes".len() as u64);
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert_eq!(scan.photos[0].bucket, Bucket::Album("Iceland 2019".into()));
|
||||
assert_eq!(scan.albums, vec!["Iceland 2019"]);
|
||||
assert_eq!(scan.skipped_trashed, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_a_zip_is_a_readable_error() {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
||||
std::fs::write(&path, b"not a zip at all").unwrap();
|
||||
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
assert!(
|
||||
error.message.contains("not a readable zip"),
|
||||
"unexpected message: {}",
|
||||
error.message
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ description = "FUSE on-demand filesystem for Proton Drive"
|
||||
|
||||
[dependencies]
|
||||
pdfs-core.workspace = true
|
||||
zip.workspace = true
|
||||
sha1.workspace = true
|
||||
proton-drive-rs.workspace = true
|
||||
futures.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
@@ -104,6 +104,31 @@ fn read_request_line(reader: &mut impl BufRead) -> std::io::Result<Option<String
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
|
||||
/// Turn a CLI-supplied path into the mount that owns it and the path relative
|
||||
/// to *that* mount's root.
|
||||
///
|
||||
/// [`rel_to_mount`] is the older, primary-only reading, and it is what every
|
||||
/// mutating request still uses. It rejects anything outside the primary
|
||||
/// mountpoint, which leaves a secondary on-demand mount — a synced folder
|
||||
/// switched out of mirror mode — with no way to be named at all: `pdfs ls` and
|
||||
/// `pdfs refresh` both answered "is not under the mountpoint", so a folder
|
||||
/// serving a stale listing had no manual escape hatch (`docs/BUGS.md` B86).
|
||||
///
|
||||
/// A relative path keeps the old meaning, relative to the primary mount, since
|
||||
/// there is nothing in it to choose a mount by.
|
||||
fn route_to_mount(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Core, PathBuf)> {
|
||||
let p = Path::new(path);
|
||||
if !p.is_absolute() {
|
||||
return Ok((core.clone(), p.to_path_buf()));
|
||||
}
|
||||
if let Some(routed) = core.rooted_at(p) {
|
||||
return Ok(routed);
|
||||
}
|
||||
// Fall back to the primary reading so the error text stays the familiar one
|
||||
// when the path is simply outside everything this daemon has mounted.
|
||||
rel_to_mount(mountpoint, path).map(|rel| (core.clone(), rel))
|
||||
}
|
||||
|
||||
/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path
|
||||
/// must live under `mountpoint`; a relative path is taken as already relative to
|
||||
/// the mount root.
|
||||
@@ -193,8 +218,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Ok(CtlRequest::ListPins) => CtlResponse::Pins {
|
||||
pins: core.cache.list_pins(),
|
||||
},
|
||||
Ok(CtlRequest::ListDir { path }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => match core.list_dir(&rel) {
|
||||
Ok(CtlRequest::ListDir { path }) => match route_to_mount(core, mountpoint, &path) {
|
||||
Ok((core, rel)) => match core.list_dir(&rel) {
|
||||
Ok(entries) => CtlResponse::Entries { entries },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
@@ -202,9 +227,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Ok(CtlRequest::Refresh { scope }) => {
|
||||
let result = match &scope {
|
||||
RefreshScope::Dir { path } => {
|
||||
rel_to_mount(mountpoint, path).and_then(|rel| core.refresh_dir(&rel))
|
||||
}
|
||||
RefreshScope::Dir { path } => route_to_mount(core, mountpoint, path)
|
||||
.and_then(|(core, rel)| core.refresh_dir(&rel)),
|
||||
RefreshScope::Trash => {
|
||||
core.invalidate_trash();
|
||||
Ok(())
|
||||
@@ -305,6 +329,51 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
None => CtlResponse::error(CoreError::invalid(format!("bad uid: {uid}"))),
|
||||
},
|
||||
// The scan alone walks every archive's central directory, so even the
|
||||
// dry run goes to a background thread and is read back through
|
||||
// `ImportStatus` rather than answered inline.
|
||||
Ok(CtlRequest::ImportTakeout { archives, dry_run }) => {
|
||||
if archives.is_empty() {
|
||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||
} else {
|
||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)))
|
||||
} else {
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ImportStatus) => {
|
||||
let (running, summary) = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus { running, summary }
|
||||
}
|
||||
Ok(CtlRequest::CancelImport) => {
|
||||
if crate::takeout::cancel_import() {
|
||||
CtlResponse::Ok {
|
||||
message: "stopping the import".to_string(),
|
||||
}
|
||||
} else {
|
||||
CtlResponse::error(CoreError::invalid("no import is running"))
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::UploadPhoto {
|
||||
name,
|
||||
media_type,
|
||||
|
||||
+203
-14
@@ -55,7 +55,7 @@ use fuser::{
|
||||
};
|
||||
use futures::StreamExt as _;
|
||||
use pdfs_core::batch;
|
||||
use pdfs_core::cache::{BLOCK_SIZE, Baseline, ContentCache, StagedWrite, block_len};
|
||||
use pdfs_core::cache::{BLOCK_SIZE, Baseline, BlockGeometry, BlockSpan, ContentCache, StagedWrite};
|
||||
use pdfs_core::config::{AppDirs, SweepMode};
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo,
|
||||
@@ -96,6 +96,7 @@ mod shutdown;
|
||||
mod state;
|
||||
mod sweep;
|
||||
mod sync;
|
||||
mod takeout;
|
||||
mod transfers;
|
||||
mod upload;
|
||||
mod r#virtual;
|
||||
@@ -715,6 +716,26 @@ struct MountedState {
|
||||
/// [`Core::notifier`], still empty until its session is spawned.
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
/// This mount's in-flight size upgrades, keyed by *its* folder inodes — so
|
||||
/// a `Core` rebuilt onto this inode space by [`Core::rooted_at`] shares the
|
||||
/// batch the mount's own session is already running rather than starting a
|
||||
/// second one against inode numbers that mean nothing to it.
|
||||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||||
}
|
||||
|
||||
/// The per-mount half of a [`Core`] — every field [`Core::fork_state`] gives a
|
||||
/// fork a fresh copy of — recovered from the registry.
|
||||
///
|
||||
/// This is what lets a control request naming a path under a secondary mount be
|
||||
/// answered *in that mount's inode space* instead of being rejected for not
|
||||
/// being under the primary mountpoint (`docs/BUGS.md` B86). See
|
||||
/// [`Core::rooted_at`].
|
||||
struct MountParts {
|
||||
mountpoint: PathBuf,
|
||||
state: Arc<Mutex<State>>,
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||||
}
|
||||
|
||||
/// Every mounted inode space in the daemon, shared by the primary `Core` and
|
||||
@@ -737,6 +758,7 @@ impl StateRegistry {
|
||||
state: &Arc<Mutex<State>>,
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
size_upgrades: Arc<Mutex<HashMap<u64, Arc<SizeUpgrade>>>>,
|
||||
) {
|
||||
let mut states = self.0.lock();
|
||||
states.retain(|m| m.state.strong_count() > 0);
|
||||
@@ -751,6 +773,7 @@ impl StateRegistry {
|
||||
state: Arc::downgrade(state),
|
||||
notifier,
|
||||
session_live,
|
||||
size_upgrades,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -777,9 +800,9 @@ impl StateRegistry {
|
||||
///
|
||||
/// On-demand roots can be nested below another location. Selecting by the
|
||||
/// longest component prefix ensures the nested session wins rather than the
|
||||
/// broader primary mount. Callers currently need the selected mount only;
|
||||
/// a future path router can derive a relative suffix with
|
||||
/// `path.strip_prefix(mountpoint)` without storing a second path value here.
|
||||
/// broader primary mount. Callers that need the relative suffix derive it
|
||||
/// with `path.strip_prefix(mountpoint)`; [`StateRegistry::covering_parts`]
|
||||
/// is the variant that also hands back enough to *serve* a request there.
|
||||
fn covering(&self, path: &Path) -> Option<LiveMount> {
|
||||
self.live()
|
||||
.into_iter()
|
||||
@@ -789,6 +812,48 @@ impl StateRegistry {
|
||||
.max_by_key(|(mountpoint, _, _, _)| mountpoint.components().count())
|
||||
}
|
||||
|
||||
/// [`StateRegistry::register`] with an empty size-upgrade map, for the
|
||||
/// tests that exercise the reap-and-route rules and nothing else.
|
||||
#[cfg(test)]
|
||||
fn register_bare(
|
||||
&self,
|
||||
mountpoint: &Path,
|
||||
state: &Arc<Mutex<State>>,
|
||||
notifier: Arc<OnceLock<Notifier>>,
|
||||
session_live: Arc<AtomicBool>,
|
||||
) {
|
||||
self.register(
|
||||
mountpoint,
|
||||
state,
|
||||
notifier,
|
||||
session_live,
|
||||
Arc::new(Mutex::new(HashMap::new())),
|
||||
);
|
||||
}
|
||||
|
||||
/// [`StateRegistry::covering`], but returning every per-mount field a
|
||||
/// [`Core`] needs to be re-rooted onto that inode space.
|
||||
///
|
||||
/// Kept separate from `covering` so the common callers — which only ask
|
||||
/// *whether* a path is mounted — keep the cheaper tuple.
|
||||
fn covering_parts(&self, path: &Path) -> Option<MountParts> {
|
||||
let mut states = self.0.lock();
|
||||
states.retain(|m| m.state.strong_count() > 0);
|
||||
states
|
||||
.iter()
|
||||
.filter(|m| m.session_live.load(Ordering::Acquire) && path.starts_with(&m.mountpoint))
|
||||
.max_by_key(|m| m.mountpoint.components().count())
|
||||
.and_then(|m| {
|
||||
Some(MountParts {
|
||||
mountpoint: m.mountpoint.clone(),
|
||||
state: m.state.upgrade()?,
|
||||
notifier: m.notifier.clone(),
|
||||
session_live: m.session_live.clone(),
|
||||
size_upgrades: m.size_upgrades.clone(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn is_mounted_at(&self, path: &Path) -> bool {
|
||||
self.covering(path).is_some_and(|(mountpoint, _, _, live)| {
|
||||
mountpoint == path && live.load(Ordering::Acquire)
|
||||
@@ -908,9 +973,41 @@ impl Core {
|
||||
&self.state,
|
||||
self.notifier.clone(),
|
||||
self.session_live.clone(),
|
||||
self.size_upgrades.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
/// This `Core`, re-rooted onto whichever live mount most specifically covers
|
||||
/// the absolute path `abs`, together with `abs` relative to that mount's
|
||||
/// root.
|
||||
///
|
||||
/// Everything a `Core` holds is per-daemon except the five fields
|
||||
/// [`Core::fork_state`] replaces, so swapping exactly those turns the
|
||||
/// primary `Core` into the fork that owns the path — sharing the fork's
|
||||
/// inode space, notification channel and size-upgrade batches rather than
|
||||
/// shadowing them. Without this, every path-addressed control request is
|
||||
/// answered against the primary inode space, so anything under a secondary
|
||||
/// on-demand mount is rejected as "not under the mountpoint" and the user
|
||||
/// has no way to re-enumerate a folder they can see (`docs/BUGS.md` B86).
|
||||
///
|
||||
/// `None` when no live mount covers `abs`.
|
||||
fn rooted_at(&self, abs: &Path) -> Option<(Core, PathBuf)> {
|
||||
let parts = self.states.covering_parts(abs)?;
|
||||
let rel = abs.strip_prefix(&parts.mountpoint).ok()?.to_path_buf();
|
||||
let mut core = self.clone();
|
||||
if !Arc::ptr_eq(&parts.state, &self.state) {
|
||||
// A different inode space, so adopt its per-mount half wholesale.
|
||||
// `primary` is a property of the mount, and the only mount this
|
||||
// `Core` can be the primary of is the one it was built for.
|
||||
core.primary = false;
|
||||
core.state = parts.state;
|
||||
core.notifier = parts.notifier;
|
||||
core.session_live = parts.session_live;
|
||||
core.size_upgrades = parts.size_upgrades;
|
||||
}
|
||||
Some((core, rel))
|
||||
}
|
||||
|
||||
/// Run `apply` against every live inode space — this daemon's primary mount
|
||||
/// and each on-demand fork — one lock at a time.
|
||||
///
|
||||
@@ -3934,6 +4031,33 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the cached child listing of the folder `parent`, everywhere it is
|
||||
/// held — naming the folder by uid rather than by inode.
|
||||
///
|
||||
/// [`Core::invalidate_parent_listing`] is the path-based sibling, and it
|
||||
/// only reaches *this* mount's inode space. A remote mutation made by the
|
||||
/// sync engine needs more than that, for two reasons. The folder it changed
|
||||
/// is usually not resident in any inode space at the time — a mirror folder
|
||||
/// has no FUSE session at all — so there is no inode to name. And the stale
|
||||
/// listing that outlives the pass is not a hot-cache map but the `listed`
|
||||
/// flag on the folder's DB row, which survives a daemon restart and is what
|
||||
/// a later `ensure_children` trusts.
|
||||
///
|
||||
/// So the flag is cleared unconditionally, and the resident inode spaces are
|
||||
/// invalidated on top of it. That is what makes a mirror folder switched to
|
||||
/// on-demand re-enumerate instead of serving a snapshot that predates the
|
||||
/// engine's own uploads — `docs/BUGS.md` B86.
|
||||
pub(crate) fn invalidate_children_of(&self, parent: &NodeUid) {
|
||||
if let Err(e) = self.db.set_listed(parent, false) {
|
||||
warn!(uid = %parent, error = ?e, "could not clear cached listing");
|
||||
}
|
||||
self.for_each_state(|st| {
|
||||
if let Some(&ino) = st.by_uid.get(parent) {
|
||||
st.invalidate_listing(ino);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Rename a file or folder to `new_name`. `rel` is mountpoint-relative.
|
||||
/// Mirrors the FUSE `rename` write path: rename on the remote, forget the
|
||||
/// node so it re-interns under its new name, and drop the parent listing so
|
||||
@@ -5853,7 +5977,7 @@ mod tests {
|
||||
("/mnt/fork-a", &fork_a),
|
||||
("/mnt/fork-b", &fork_b),
|
||||
] {
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new(mountpoint),
|
||||
state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -5904,19 +6028,19 @@ mod tests {
|
||||
let nested_live = session_flag(false);
|
||||
let sibling_live = session_flag(true);
|
||||
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
primary_path,
|
||||
&primary,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
primary_live.clone(),
|
||||
);
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
nested_path,
|
||||
&nested,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
nested_live.clone(),
|
||||
);
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
sibling_path,
|
||||
&sibling,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -5994,6 +6118,71 @@ mod tests {
|
||||
assert_eq!(registry.live().len(), 2);
|
||||
}
|
||||
|
||||
/// The routing half of `docs/BUGS.md` B86: a path under a secondary mount
|
||||
/// has to resolve to *that* mount's inode space and a suffix relative to
|
||||
/// *its* root, or `pdfs ls` and `pdfs refresh` can only ever name the
|
||||
/// primary mount and a stale on-demand folder has no escape hatch.
|
||||
#[test]
|
||||
fn covering_parts_route_a_path_to_the_mount_that_owns_it() {
|
||||
let registry = StateRegistry::default();
|
||||
let (primary, _primary_dir) = rooted_state("primary-volume", "primary");
|
||||
let (nested, _nested_dir) = rooted_state("nested-volume", "nested");
|
||||
let primary = std::sync::Arc::new(parking_lot::Mutex::new(primary));
|
||||
let nested = std::sync::Arc::new(parking_lot::Mutex::new(nested));
|
||||
let primary_path = std::path::Path::new("/home/me/ProtonDrive");
|
||||
let nested_path = std::path::Path::new("/home/me/ProtonDrive/Device");
|
||||
let primary_upgrades = std::sync::Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
let nested_upgrades = std::sync::Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
|
||||
registry.register(
|
||||
primary_path,
|
||||
&primary,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
session_flag(true),
|
||||
primary_upgrades.clone(),
|
||||
);
|
||||
registry.register(
|
||||
nested_path,
|
||||
&nested,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
session_flag(true),
|
||||
nested_upgrades.clone(),
|
||||
);
|
||||
|
||||
let deep = nested_path.join("sub/file.txt");
|
||||
let parts = registry.covering_parts(&deep).expect("nested mount covers");
|
||||
assert_eq!(parts.mountpoint, nested_path, "the nested mount wins");
|
||||
assert!(
|
||||
std::sync::Arc::ptr_eq(&parts.state, &nested),
|
||||
"the request must be answered in the nested inode space"
|
||||
);
|
||||
assert!(
|
||||
std::sync::Arc::ptr_eq(&parts.size_upgrades, &nested_upgrades),
|
||||
"and share that mount's in-flight size upgrades, not the primary's"
|
||||
);
|
||||
assert_eq!(
|
||||
deep.strip_prefix(&parts.mountpoint).unwrap(),
|
||||
std::path::Path::new("sub/file.txt"),
|
||||
"the suffix is relative to the mount that owns it, not the primary"
|
||||
);
|
||||
|
||||
let shallow = primary_path.join("Documents");
|
||||
let parts = registry
|
||||
.covering_parts(&shallow)
|
||||
.expect("primary mount covers");
|
||||
assert!(
|
||||
std::sync::Arc::ptr_eq(&parts.state, &primary),
|
||||
"a path outside the nested root still routes to the primary"
|
||||
);
|
||||
|
||||
assert!(
|
||||
registry
|
||||
.covering_parts(std::path::Path::new("/tmp/outside"))
|
||||
.is_none(),
|
||||
"a path under no mount routes nowhere, and the caller reports that"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_registry_resolves_uids_from_any_resident_mount() {
|
||||
let registry = StateRegistry::default();
|
||||
@@ -6007,13 +6196,13 @@ mod tests {
|
||||
fork.children.insert(super::ROOT_INO, vec![fork_ino]);
|
||||
let primary = std::sync::Arc::new(parking_lot::Mutex::new(primary));
|
||||
let fork = std::sync::Arc::new(parking_lot::Mutex::new(fork));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/primary"),
|
||||
&primary,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
session_flag(true),
|
||||
);
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&fork,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6041,7 +6230,7 @@ mod tests {
|
||||
let child_ino = state.intern(super::ROOT_INO, child_node);
|
||||
state.children.insert(super::ROOT_INO, vec![child_ino]);
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6104,7 +6293,7 @@ mod tests {
|
||||
);
|
||||
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6372,7 +6561,7 @@ mod tests {
|
||||
state.children.insert(revoked_ino, vec![child_ino]);
|
||||
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
@@ -6426,7 +6615,7 @@ mod tests {
|
||||
state.children.insert(cycle_b, vec![cycle_a]);
|
||||
|
||||
let state = std::sync::Arc::new(parking_lot::Mutex::new(state));
|
||||
registry.register(
|
||||
registry.register_bare(
|
||||
std::path::Path::new("/mnt/device"),
|
||||
&state,
|
||||
std::sync::Arc::new(std::sync::OnceLock::new()),
|
||||
|
||||
+157
-88
@@ -102,22 +102,38 @@ static NEXT_OPEN_ID: AtomicU64 = AtomicU64::new(0);
|
||||
/// Validated by the same `(mtime, size)` pair the on-disk caches use: a node
|
||||
/// whose tag no longer matches has its blocks dropped rather than served, so a
|
||||
/// new revision can never be stitched together from the old one.
|
||||
/// A ring entry's identity: the node, and the plaintext range the block covers.
|
||||
///
|
||||
/// The range, rather than the block index, because the index alone stopped
|
||||
/// naming one range when the geometry started coming from the revision — a
|
||||
/// block cached under the [`BlockGeometry::uniform`] fallback and the block at
|
||||
/// the same index under the real geometry are different bytes
|
||||
/// (`docs/BUGS.md` B85). A range is the same identity under either plan, so a
|
||||
/// re-planned read misses rather than being served the wrong bytes.
|
||||
type RingKey = (NodeUid, BlockSpan);
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct BlockRing {
|
||||
blocks: HashMap<(NodeUid, u64), Arc<Vec<u8>>>,
|
||||
blocks: HashMap<RingKey, Arc<Vec<u8>>>,
|
||||
/// Keys least-recently-used first, for eviction.
|
||||
order: VecDeque<(NodeUid, u64)>,
|
||||
order: VecDeque<RingKey>,
|
||||
/// Per-node validity tag, `(mtime, size)`.
|
||||
tags: HashMap<NodeUid, (i64, u64)>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
impl BlockRing {
|
||||
fn get(&mut self, uid: &NodeUid, mtime: i64, size: u64, idx: u64) -> Option<Arc<Vec<u8>>> {
|
||||
fn get(
|
||||
&mut self,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: u64,
|
||||
span: BlockSpan,
|
||||
) -> Option<Arc<Vec<u8>>> {
|
||||
if self.tags.get(uid) != Some(&(mtime, size)) {
|
||||
return None;
|
||||
}
|
||||
let key = (uid.clone(), idx);
|
||||
let key = (uid.clone(), span);
|
||||
let hit = self.blocks.get(&key).cloned()?;
|
||||
// Move to the young end: a block being read repeatedly (the whole reason
|
||||
// this exists) must not age out under a sequential read passing through.
|
||||
@@ -128,14 +144,21 @@ impl BlockRing {
|
||||
Some(hit)
|
||||
}
|
||||
|
||||
fn insert(&mut self, uid: &NodeUid, mtime: i64, size: u64, idx: u64, bytes: Arc<Vec<u8>>) {
|
||||
fn insert(
|
||||
&mut self,
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
size: u64,
|
||||
span: BlockSpan,
|
||||
bytes: Arc<Vec<u8>>,
|
||||
) {
|
||||
if self.tags.get(uid) != Some(&(mtime, size)) {
|
||||
// Revision changed under us (or first sight): anything held for this
|
||||
// node describes the old one.
|
||||
self.drop_node(uid);
|
||||
self.tags.insert(uid.clone(), (mtime, size));
|
||||
}
|
||||
let key = (uid.clone(), idx);
|
||||
let key = (uid.clone(), span);
|
||||
if self.blocks.contains_key(&key) {
|
||||
return;
|
||||
}
|
||||
@@ -176,8 +199,10 @@ impl BlockRing {
|
||||
/// The result of fetching one block, shared between everyone who wanted it.
|
||||
type BlockResult = Result<Arc<Vec<u8>>, Errno>;
|
||||
|
||||
/// Identifies a block *of a revision*: node, validity tag, index.
|
||||
type BlockKey = (NodeUid, i64, u64, u64);
|
||||
/// Identifies a block *of a revision*: node, validity tag, and the plaintext
|
||||
/// range the block covers — see [`RingKey`] for why it is the range and not the
|
||||
/// index.
|
||||
type BlockKey = (NodeUid, i64, u64, BlockSpan);
|
||||
|
||||
/// A claim on a block key: who owns the fetch, and where its result is published.
|
||||
type Claim = (u64, tokio::sync::watch::Receiver<Option<BlockResult>>);
|
||||
@@ -417,11 +442,30 @@ impl Core {
|
||||
self.prefetch.streams.lock().remove(uid);
|
||||
}
|
||||
|
||||
/// How this revision's plaintext is divided into content blocks.
|
||||
///
|
||||
/// The revision's own geometry when a previous read has learned and recorded
|
||||
/// it, and the [`BLOCK_SIZE`] assumption when it has not — which is every
|
||||
/// first read, because learning it means opening a `RevisionReader`, and
|
||||
/// opening one per read is precisely the per-file key derivation the block
|
||||
/// cache exists to avoid (bugs.md B12, B85).
|
||||
///
|
||||
/// Being wrong here is not a correctness problem: `read_at` plans over the
|
||||
/// real sizes and clamps to what was asked for either way. It costs a
|
||||
/// straddling fetch and a cached block the next read will not ask for, and
|
||||
/// both settle once the geometry is known.
|
||||
fn block_geometry(&self, uid: &NodeUid, mtime: i64, fsize: u64) -> BlockGeometry {
|
||||
self.cache
|
||||
.block_geometry(uid, mtime, fsize)
|
||||
.unwrap_or_else(|| BlockGeometry::uniform(fsize))
|
||||
}
|
||||
|
||||
/// Serve bytes `[offset, offset + len)` of `uid`'s active revision, hitting
|
||||
/// the on-disk caches before the network: a whole-file blob (pinned files)
|
||||
/// first, then the block cache — fetching only the [`BLOCK_SIZE`]-aligned
|
||||
/// blocks that overlap the request and caching each. `mtime`/`fsize` validate
|
||||
/// both caches. Network I/O runs without any lock held.
|
||||
/// first, then the block cache — fetching only the blocks that overlap the
|
||||
/// request, on the revision's own boundaries ([`Core::block_geometry`]), and
|
||||
/// caching each. `mtime`/`fsize` validate both caches. Network I/O runs
|
||||
/// without any lock held.
|
||||
pub(super) fn read_range(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
@@ -500,7 +544,7 @@ impl Core {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Fetch block `bidx`, joining a fetch already in flight for it rather than
|
||||
/// Fetch the block at `span`, joining a fetch already in flight rather than
|
||||
/// issuing a second one, and publish the result to everyone who joined.
|
||||
///
|
||||
/// See [`BlockFlight`]. A follower whose leader disappears (a panic, or a
|
||||
@@ -512,10 +556,10 @@ impl Core {
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
bidx: u64,
|
||||
span: BlockSpan,
|
||||
cache_blocks: bool,
|
||||
) -> BlockResult {
|
||||
let key: BlockKey = (uid.clone(), mtime, fsize, bidx);
|
||||
let key: BlockKey = (uid.clone(), mtime, fsize, span);
|
||||
let claim = {
|
||||
let mut inflight = self.block_flight.inflight.lock();
|
||||
match inflight.get(&key) {
|
||||
@@ -547,12 +591,12 @@ impl Core {
|
||||
}
|
||||
}
|
||||
return self
|
||||
.read_block(reader, uid, mtime, fsize, bidx, cache_blocks)
|
||||
.read_block(reader, uid, mtime, fsize, span, cache_blocks)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let result = self
|
||||
.read_block(reader, uid, mtime, fsize, bidx, cache_blocks)
|
||||
.read_block(reader, uid, mtime, fsize, span, cache_blocks)
|
||||
.await;
|
||||
// Retire the claim before publishing, so a racer that wakes and re-enters
|
||||
// finds no stale entry to join.
|
||||
@@ -578,11 +622,9 @@ impl Core {
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
bidx: u64,
|
||||
span: BlockSpan,
|
||||
cache_blocks: bool,
|
||||
) -> BlockResult {
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
let blen = block_len(fsize, bidx);
|
||||
// The reader derives every block boundary from the revision's recorded
|
||||
// block sizes. If they do not add up to the size the node reports, this
|
||||
// file's boundaries are wrong, and the failure is not confined to short
|
||||
@@ -591,25 +633,33 @@ impl Core {
|
||||
// kind takes the file off the range path entirely (bugs.md B84).
|
||||
let bytes = if reader.size() != fsize {
|
||||
warn!(
|
||||
%uid, bidx, fsize,
|
||||
%uid, bidx = span.idx, fsize,
|
||||
reader_size = reader.size(),
|
||||
reader_blocks = reader.block_sizes().len(),
|
||||
"the revision's block table disagrees with the node size"
|
||||
);
|
||||
self.repair_block(uid, mtime, fsize, bidx).await?
|
||||
self.repair_block(uid, mtime, fsize, span).await?
|
||||
} else {
|
||||
let bytes = reader.read_at(bstart, blen).await.map_err(|e| {
|
||||
warn!(%uid, bstart, blen, error = %e, "block read failed");
|
||||
// Learned here rather than on a path of its own: this is where a
|
||||
// reader is open anyway, and recording the real geometry is what
|
||||
// lets the *next* read plan its blocks on the revision's own
|
||||
// boundaries instead of the 4 MiB fallback (bugs.md B85).
|
||||
if cache_blocks {
|
||||
self.cache
|
||||
.store_block_geometry(uid, mtime, fsize, reader.block_sizes());
|
||||
}
|
||||
let bytes = reader.read_at(span.start, span.len).await.map_err(|e| {
|
||||
warn!(%uid, start = span.start, len = span.len, error = %e, "block read failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
if bytes.len() as u64 == blen {
|
||||
if bytes.len() as u64 == span.len {
|
||||
bytes
|
||||
} else {
|
||||
warn!(
|
||||
%uid, bidx, blen, got = bytes.len(), fsize,
|
||||
"block came back shorter than the file's size implies"
|
||||
%uid, bidx = span.idx, blen = span.len, got = bytes.len(), fsize,
|
||||
"block came back shorter than the range asked for"
|
||||
);
|
||||
self.repair_block(uid, mtime, fsize, bidx).await?
|
||||
self.repair_block(uid, mtime, fsize, span).await?
|
||||
}
|
||||
};
|
||||
let bytes = Arc::new(bytes);
|
||||
@@ -618,16 +668,16 @@ impl Core {
|
||||
let uid = uid.clone();
|
||||
let bytes = bytes.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _ = cache.store_block(&uid, mtime, fsize, bidx, &bytes);
|
||||
let _ = cache.store_block(&uid, mtime, fsize, span, &bytes);
|
||||
});
|
||||
}
|
||||
self.block_ring
|
||||
.lock()
|
||||
.insert(uid, mtime, fsize, bidx, bytes.clone());
|
||||
.insert(uid, mtime, fsize, span, bytes.clone());
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Recover block `bidx` of a file whose per-block reads cannot be trusted,
|
||||
/// Recover the block at `span` of a file whose per-block reads cannot be
|
||||
/// by fetching the whole revision through the manifest-verified download and
|
||||
/// serving the block out of that.
|
||||
///
|
||||
@@ -648,10 +698,9 @@ impl Core {
|
||||
uid: &NodeUid,
|
||||
mtime: i64,
|
||||
fsize: u64,
|
||||
bidx: u64,
|
||||
span: BlockSpan,
|
||||
) -> Result<Vec<u8>, Errno> {
|
||||
let blen = block_len(fsize, bidx);
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
let (bstart, blen) = (span.start, span.len);
|
||||
if fsize > REPAIR_MAX {
|
||||
warn!(
|
||||
%uid, fsize,
|
||||
@@ -683,11 +732,11 @@ impl Core {
|
||||
}
|
||||
match self.cache.read_range(uid, mtime, fsize, bstart, blen) {
|
||||
Some(bytes) if bytes.len() as u64 == blen => {
|
||||
info!(%uid, bidx, "served a short block from a repaired whole-file download");
|
||||
info!(%uid, bidx = span.idx, "served a short block from a repaired whole-file download");
|
||||
Ok(bytes)
|
||||
}
|
||||
_ => {
|
||||
error!(%uid, bidx, fsize, "repairing a short block did not produce the block");
|
||||
error!(%uid, bidx = span.idx, fsize, "repairing a short block did not produce the block");
|
||||
Err(Errno::EIO)
|
||||
}
|
||||
}
|
||||
@@ -776,17 +825,19 @@ impl Core {
|
||||
if depth == 0 {
|
||||
return;
|
||||
}
|
||||
let last_block = fsize.saturating_sub(1) / BLOCK_SIZE;
|
||||
let wanted: Vec<u64> = ((last + 1)..=(last + depth).min(last_block))
|
||||
.filter(|&bidx| {
|
||||
let geometry = self.block_geometry(uid, mtime, fsize);
|
||||
let last_block = geometry.blocks().saturating_sub(1);
|
||||
let wanted: Vec<BlockSpan> = ((last + 1)..=(last + depth).min(last_block))
|
||||
.filter_map(|bidx| geometry.span(bidx))
|
||||
.filter(|&span| {
|
||||
// Blocks already in memory need nothing; blocks already on disk
|
||||
// are cheap enough to serve that spending a round-trip to move
|
||||
// them into the ring would be the wrong trade.
|
||||
self.block_ring
|
||||
.lock()
|
||||
.get(uid, mtime, fsize, bidx)
|
||||
.get(uid, mtime, fsize, span)
|
||||
.is_none()
|
||||
&& !(cache_blocks && self.cache.has_block(uid, mtime, fsize, bidx))
|
||||
&& !(cache_blocks && self.cache.has_block(uid, mtime, fsize, span))
|
||||
})
|
||||
.collect();
|
||||
if wanted.is_empty() {
|
||||
@@ -803,7 +854,7 @@ impl Core {
|
||||
// round-trips, and the reader catches up with the window before it
|
||||
// finishes.
|
||||
let mut set = tokio::task::JoinSet::new();
|
||||
for bidx in wanted {
|
||||
for span in wanted {
|
||||
let Ok(permit) = core.prefetch_budget.clone().try_acquire_owned() else {
|
||||
break; // the budget is spent; the demand path can have it
|
||||
};
|
||||
@@ -811,10 +862,10 @@ impl Core {
|
||||
set.spawn(async move {
|
||||
let _permit = permit;
|
||||
if let Err(error) = core
|
||||
.fetch_block(&reader, &uid, mtime, fsize, bidx, cache_blocks)
|
||||
.fetch_block(&reader, &uid, mtime, fsize, span, cache_blocks)
|
||||
.await
|
||||
{
|
||||
debug!(%uid, bidx, ?error, "prefetch failed");
|
||||
debug!(%uid, bidx = span.idx, ?error, "prefetch failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -844,8 +895,13 @@ impl Core {
|
||||
}
|
||||
let end = offset.saturating_add(len).min(fsize);
|
||||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||||
let first = offset / BLOCK_SIZE;
|
||||
let last = (end - 1) / BLOCK_SIZE;
|
||||
// On the revision's own boundaries where they are known, and on the
|
||||
// 4 MiB fallback where they are not (bugs.md B85).
|
||||
let wanted = self.block_geometry(uid, mtime, fsize).spans(offset, end);
|
||||
let (Some(&head), Some(&tail)) = (wanted.first(), wanted.last()) else {
|
||||
return Ok(out);
|
||||
};
|
||||
let (first, last) = (head.idx, tail.idx);
|
||||
|
||||
// Collect the blocks overlapping the request, serving any already cached
|
||||
// and fetching the rest concurrently. A multi-block read (e.g. a media
|
||||
@@ -853,31 +909,31 @@ impl Core {
|
||||
// would otherwise stall on each block round-trip in turn; downloading the
|
||||
// misses in parallel saturates the connection and bounds latency at the
|
||||
// slowest single block instead of their sum.
|
||||
let mut blocks: Vec<Option<Arc<Vec<u8>>>> = Vec::with_capacity((last - first + 1) as usize);
|
||||
let mut misses: Vec<u64> = Vec::new();
|
||||
for bidx in first..=last {
|
||||
let mut blocks: Vec<Option<Arc<Vec<u8>>>> = Vec::with_capacity(wanted.len());
|
||||
let mut misses: Vec<BlockSpan> = Vec::new();
|
||||
for &span in &wanted {
|
||||
// The in-memory ring comes first on both paths: a streaming read has
|
||||
// no other cache at all, and a cached one would otherwise pay two
|
||||
// opens, a JSON parse and a 4 MiB read+alloc for every 128 KiB the
|
||||
// kernel asks for (audit T3).
|
||||
let hit = self.block_ring.lock().get(uid, mtime, fsize, bidx);
|
||||
let hit = self.block_ring.lock().get(uid, mtime, fsize, span);
|
||||
let hit = hit.or_else(|| {
|
||||
if !cache_blocks {
|
||||
return None;
|
||||
}
|
||||
let bytes = Arc::new(self.cache.cached_block(uid, mtime, fsize, bidx)?);
|
||||
let bytes = Arc::new(self.cache.cached_block(uid, mtime, fsize, span)?);
|
||||
// Promote it, so the other ~31 reads of this block are answered
|
||||
// from memory.
|
||||
self.block_ring
|
||||
.lock()
|
||||
.insert(uid, mtime, fsize, bidx, bytes.clone());
|
||||
.insert(uid, mtime, fsize, span, bytes.clone());
|
||||
Some(bytes)
|
||||
});
|
||||
match hit {
|
||||
Some(b) => blocks.push(Some(b)),
|
||||
None => {
|
||||
blocks.push(None);
|
||||
misses.push(bidx);
|
||||
misses.push(span);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -890,12 +946,12 @@ impl Core {
|
||||
let reader = self.revision_reader(uid, mtime, fsize).await?;
|
||||
|
||||
let mut set = tokio::task::JoinSet::new();
|
||||
for &bidx in &misses {
|
||||
for &span in &misses {
|
||||
let (core, reader, uid) = (self.clone(), reader.clone(), uid.clone());
|
||||
set.spawn(async move {
|
||||
core.fetch_block(&reader, &uid, mtime, fsize, bidx, cache_blocks)
|
||||
core.fetch_block(&reader, &uid, mtime, fsize, span, cache_blocks)
|
||||
.await
|
||||
.map(|bytes| (bidx, bytes))
|
||||
.map(|bytes| (span.idx, bytes))
|
||||
});
|
||||
}
|
||||
let mut out = Vec::with_capacity(misses.len());
|
||||
@@ -913,18 +969,20 @@ impl Core {
|
||||
// ahead of a sequential reader instead of stalling once it catches up.
|
||||
self.prefetch_ahead(uid, mtime, fsize, first, last, cache_blocks);
|
||||
|
||||
for (i, block) in blocks.into_iter().enumerate() {
|
||||
let bidx = first + i as u64;
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
for (span, block) in wanted.into_iter().zip(blocks) {
|
||||
let bstart = span.start;
|
||||
// Every slot is populated: cache hits up front, misses by the fetch above.
|
||||
let block = block.expect("block fetched or cached");
|
||||
// Both sources guarantee the length the file's size implies — the
|
||||
// cache validates it on read, the network path in `read_block` — so
|
||||
// this refuses rather than quietly contributing fewer bytes than the
|
||||
// range covers, which is what turned a bad block into a truncated
|
||||
// file (bugs.md B84).
|
||||
if block.len() as u64 != block_len(fsize, bidx) {
|
||||
error!(%uid, bidx, len = block.len(), fsize, "assembling a read hit a short block");
|
||||
// Both sources guarantee the length the geometry says the block has
|
||||
// — the cache validates it on read, the network path in `read_block`
|
||||
// — so this refuses rather than quietly contributing fewer bytes
|
||||
// than the range covers, which is what turned a bad block into a
|
||||
// truncated file (bugs.md B84).
|
||||
if block.len() as u64 != span.len {
|
||||
error!(
|
||||
%uid, bidx = span.idx, len = block.len(), expected = span.len, fsize,
|
||||
"assembling a read hit a short block"
|
||||
);
|
||||
return Err(Errno::EIO);
|
||||
}
|
||||
let s = (offset.max(bstart) - bstart) as usize;
|
||||
@@ -1016,13 +1074,13 @@ impl Core {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||||
for bidx in (offset / BLOCK_SIZE)..=((end - 1) / BLOCK_SIZE) {
|
||||
let ring = self.block_ring.lock().get(uid, mtime, fsize, bidx);
|
||||
for span in self.block_geometry(uid, mtime, fsize).spans(offset, end) {
|
||||
let ring = self.block_ring.lock().get(uid, mtime, fsize, span);
|
||||
let block = match ring {
|
||||
Some(bytes) => bytes,
|
||||
None => Arc::new(self.cache.cached_block(uid, mtime, fsize, bidx)?),
|
||||
None => Arc::new(self.cache.cached_block(uid, mtime, fsize, span)?),
|
||||
};
|
||||
let bstart = bidx * BLOCK_SIZE;
|
||||
let bstart = span.start;
|
||||
let s = (offset.max(bstart) - bstart) as usize;
|
||||
let e = (end.min(bstart + block.len() as u64) - bstart) as usize;
|
||||
if s >= e {
|
||||
@@ -1142,31 +1200,42 @@ mod block_ring_tests {
|
||||
RING_BYTES / BLOCK_SIZE
|
||||
}
|
||||
|
||||
/// Block `idx` under the uniform geometry — the shape these tests are
|
||||
/// written against, since what they exercise is tagging and eviction rather
|
||||
/// than block layout.
|
||||
fn span(idx: u64) -> BlockSpan {
|
||||
BlockSpan {
|
||||
idx,
|
||||
start: idx * BLOCK_SIZE,
|
||||
len: BLOCK_SIZE,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stored_block_is_served_back() {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
ring.insert(&node, 7, 100, 3, block(0xab));
|
||||
assert_eq!(ring.get(&node, 7, 100, 3).map(|b| b[0]), Some(0xab));
|
||||
ring.insert(&node, 7, 100, span(3), block(0xab));
|
||||
assert_eq!(ring.get(&node, 7, 100, span(3)).map(|b| b[0]), Some(0xab));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_block_of_another_revision_is_never_served() {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
ring.insert(&node, 7, 100, 0, block(1));
|
||||
ring.insert(&node, 7, 100, span(0), block(1));
|
||||
// Same node, newer revision: the tag moved, so the old bytes go.
|
||||
assert!(ring.get(&node, 8, 100, 0).is_none());
|
||||
assert!(ring.get(&node, 7, 101, 0).is_none());
|
||||
assert!(ring.get(&node, 8, 100, span(0)).is_none());
|
||||
assert!(ring.get(&node, 7, 101, span(0)).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_revision_drops_what_the_old_one_left() {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
ring.insert(&node, 7, 100, 0, block(1));
|
||||
ring.insert(&node, 8, 100, 0, block(2));
|
||||
assert_eq!(ring.get(&node, 8, 100, 0).map(|b| b[0]), Some(2));
|
||||
ring.insert(&node, 7, 100, span(0), block(1));
|
||||
ring.insert(&node, 8, 100, span(0), block(2));
|
||||
assert_eq!(ring.get(&node, 8, 100, span(0)).map(|b| b[0]), Some(2));
|
||||
assert_eq!(ring.bytes, BLOCK_SIZE);
|
||||
}
|
||||
|
||||
@@ -1175,13 +1244,13 @@ mod block_ring_tests {
|
||||
let mut ring = BlockRing::default();
|
||||
let node = uid("a");
|
||||
for idx in 0..capacity() {
|
||||
ring.insert(&node, 1, u64::MAX, idx, block(idx as u8));
|
||||
ring.insert(&node, 1, u64::MAX, span(idx), block(idx as u8));
|
||||
}
|
||||
// Re-reading block 0 makes block 1 the oldest instead.
|
||||
assert!(ring.get(&node, 1, u64::MAX, 0).is_some());
|
||||
ring.insert(&node, 1, u64::MAX, capacity(), block(0xff));
|
||||
assert!(ring.get(&node, 1, u64::MAX, 0).is_some());
|
||||
assert!(ring.get(&node, 1, u64::MAX, 1).is_none());
|
||||
assert!(ring.get(&node, 1, u64::MAX, span(0)).is_some());
|
||||
ring.insert(&node, 1, u64::MAX, span(capacity()), block(0xff));
|
||||
assert!(ring.get(&node, 1, u64::MAX, span(0)).is_some());
|
||||
assert!(ring.get(&node, 1, u64::MAX, span(1)).is_none());
|
||||
assert_eq!(ring.bytes, RING_BYTES);
|
||||
}
|
||||
|
||||
@@ -1189,9 +1258,9 @@ mod block_ring_tests {
|
||||
fn a_node_whose_last_block_ages_out_keeps_no_tag() {
|
||||
let mut ring = BlockRing::default();
|
||||
let (old, new) = (uid("old"), uid("new"));
|
||||
ring.insert(&old, 1, u64::MAX, 0, block(1));
|
||||
ring.insert(&old, 1, u64::MAX, span(0), block(1));
|
||||
for idx in 0..capacity() {
|
||||
ring.insert(&new, 1, u64::MAX, idx, block(2));
|
||||
ring.insert(&new, 1, u64::MAX, span(idx), block(2));
|
||||
}
|
||||
assert!(!ring.tags.contains_key(&old));
|
||||
assert!(ring.tags.contains_key(&new));
|
||||
@@ -1201,11 +1270,11 @@ mod block_ring_tests {
|
||||
fn dropping_a_node_frees_only_its_own_bytes() {
|
||||
let mut ring = BlockRing::default();
|
||||
let (a, b) = (uid("a"), uid("b"));
|
||||
ring.insert(&a, 1, 100, 0, block(1));
|
||||
ring.insert(&b, 1, 100, 0, block(2));
|
||||
ring.insert(&a, 1, 100, span(0), block(1));
|
||||
ring.insert(&b, 1, 100, span(0), block(2));
|
||||
ring.drop_node(&a);
|
||||
assert!(ring.get(&a, 1, 100, 0).is_none());
|
||||
assert_eq!(ring.get(&b, 1, 100, 0).map(|x| x[0]), Some(2));
|
||||
assert!(ring.get(&a, 1, 100, span(0)).is_none());
|
||||
assert_eq!(ring.get(&b, 1, 100, span(0)).map(|x| x[0]), Some(2));
|
||||
assert_eq!(ring.bytes, BLOCK_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
+364
-64
@@ -525,6 +525,9 @@ impl Core {
|
||||
continue;
|
||||
}
|
||||
trashed.push(rel.clone());
|
||||
if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -777,6 +780,9 @@ impl Core {
|
||||
outcome.errors += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = parent_uid_of(rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -832,6 +838,9 @@ impl Core {
|
||||
outcome.errors += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = parent_uid_of(&rel, &remote_dirs, &baseline) {
|
||||
self.invalidate_children_of(&parent);
|
||||
}
|
||||
let _ = self.db.sync_entry_remove(folder_id, &rel);
|
||||
self.log_activity(
|
||||
ActivityKind::Trash,
|
||||
@@ -857,70 +866,9 @@ impl Core {
|
||||
writing: &HashSet<PathBuf>,
|
||||
out: &mut HashMap<String, LocalItem>,
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||
format!(
|
||||
"local path {} escaped sync root {}: {e}",
|
||||
path.display(),
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let rel = stripped.to_str().ok_or_else(|| {
|
||||
format!(
|
||||
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
// Ignore our own in-flight download temp files.
|
||||
if rel.contains(".pdfs-tmp-") {
|
||||
continue;
|
||||
}
|
||||
// Ignored paths are dropped here as well as from the classification
|
||||
// union — not for correctness (the union filter is what makes this
|
||||
// safe) but so an ignored `node_modules/` is never descended into or
|
||||
// counted against scan progress.
|
||||
if rules.is_ignored(rel, meta.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
if meta.is_dir() {
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
mtime_ns: None,
|
||||
size: 0,
|
||||
open_for_write: false,
|
||||
},
|
||||
);
|
||||
self.progress_scanned(folder_id, base_name(rel));
|
||||
self.walk_local(folder_id, root, &path, rules, writing, out)?;
|
||||
} else if meta.is_file() {
|
||||
// Check whether any process holds this file open for writing.
|
||||
// The canonical path is used because /proc/*/fd links resolve
|
||||
// to canonical targets.
|
||||
let ofw = std::fs::canonicalize(&path)
|
||||
.map(|canon| writing.contains(&canon))
|
||||
.unwrap_or(false);
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: false,
|
||||
mtime: system_mtime(&meta),
|
||||
mtime_ns: system_mtime_ns(&meta),
|
||||
size: meta.len() as i64,
|
||||
open_for_write: ofw,
|
||||
},
|
||||
);
|
||||
self.progress_scanned(folder_id, base_name(rel));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
walk_local_tree(root, dir, rules, writing, out, &mut |rel| {
|
||||
self.progress_scanned(folder_id, base_name(rel))
|
||||
})
|
||||
}
|
||||
|
||||
/// Map every remote folder under `folder` to its uid by rel path, ignoring
|
||||
@@ -1186,6 +1134,9 @@ impl Core {
|
||||
.create_folder(parent, base_name(rel), Some(now_secs()))
|
||||
.await
|
||||
.map_err(|e| format!("create remote folder {rel}: {e}"))?;
|
||||
// The parent has gained a child that no cached listing of it
|
||||
// knows about (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
self.baseline_dir(folder_id, rel, &uid)?;
|
||||
Ok(Applied::Dir(rel.clone(), uid))
|
||||
}
|
||||
@@ -1295,6 +1246,10 @@ impl Core {
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("upload {rel}: {e}"))?;
|
||||
// The parent has gained a child that no cached listing of it knows
|
||||
// about. Without this, a folder later switched to on-demand serves a
|
||||
// snapshot that predates this upload (B86).
|
||||
self.invalidate_children_of(parent);
|
||||
warn_if_torn(rel, &path, streamed);
|
||||
self.record_file_baseline(folder_id, rel, Some(streamed), &uid)
|
||||
.await
|
||||
@@ -1546,6 +1501,95 @@ fn removed_locally(result: std::io::Result<()>) -> std::io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The body of [`Core::walk_local`], without a `Core`: recursively collect the
|
||||
/// local tree under `dir` into `out`, reporting each entry to `scanned`.
|
||||
///
|
||||
/// **Every failure here is fatal to the pass, deliberately.** A `read_dir` that
|
||||
/// fails, an entry that cannot be stat-ed, a name that is not UTF-8 — each of
|
||||
/// them would, if skipped, leave `out` describing a *smaller* tree than the one
|
||||
/// on disk. Reconciliation reads a path present in the baseline and absent from
|
||||
/// the scan as "the user deleted this", and propagates that to Drive. So an
|
||||
/// unreadable subdirectory would trash its remote counterpart, and the deeper
|
||||
/// the failure, the more it takes with it (`docs/BUGS.md` B55). Refusing the
|
||||
/// whole pass keeps the previous baseline and leaves the remote alone; the next
|
||||
/// pass retries once the transient condition clears.
|
||||
///
|
||||
/// Split out from the method so exactly that can be tested against a real tree
|
||||
/// with a real unreadable directory in it, which needs no `Core` at all.
|
||||
fn walk_local_tree(
|
||||
root: &Path,
|
||||
dir: &Path,
|
||||
rules: &IgnoreRules,
|
||||
writing: &HashSet<PathBuf>,
|
||||
out: &mut HashMap<String, LocalItem>,
|
||||
scanned: &mut dyn FnMut(&str),
|
||||
) -> Result<(), String> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||
let path = entry.path();
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||
format!(
|
||||
"local path {} escaped sync root {}: {e}",
|
||||
path.display(),
|
||||
root.display()
|
||||
)
|
||||
})?;
|
||||
let rel = stripped.to_str().ok_or_else(|| {
|
||||
format!(
|
||||
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
// Ignore our own in-flight download temp files.
|
||||
if rel.contains(".pdfs-tmp-") {
|
||||
continue;
|
||||
}
|
||||
// Ignored paths are dropped here as well as from the classification
|
||||
// union — not for correctness (the union filter is what makes this
|
||||
// safe) but so an ignored `node_modules/` is never descended into or
|
||||
// counted against scan progress.
|
||||
if rules.is_ignored(rel, meta.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
if meta.is_dir() {
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
mtime_ns: None,
|
||||
size: 0,
|
||||
open_for_write: false,
|
||||
},
|
||||
);
|
||||
scanned(rel);
|
||||
walk_local_tree(root, &path, rules, writing, out, scanned)?;
|
||||
} else if meta.is_file() {
|
||||
// Check whether any process holds this file open for writing.
|
||||
// The canonical path is used because /proc/*/fd links resolve
|
||||
// to canonical targets.
|
||||
let ofw = std::fs::canonicalize(&path)
|
||||
.map(|canon| writing.contains(&canon))
|
||||
.unwrap_or(false);
|
||||
out.insert(
|
||||
rel.to_string(),
|
||||
LocalItem {
|
||||
is_dir: false,
|
||||
mtime: system_mtime(&meta),
|
||||
mtime_ns: system_mtime_ns(&meta),
|
||||
size: meta.len() as i64,
|
||||
open_for_write: ofw,
|
||||
},
|
||||
);
|
||||
scanned(rel);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publish a conflict copy without overwriting an earlier conflict bearing the
|
||||
/// same timestamp. The original is removed only after the new copy is durable.
|
||||
fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||
@@ -1579,6 +1623,26 @@ fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||
unreachable!("u32 conflict suffix space exhausted")
|
||||
}
|
||||
|
||||
/// The remote uid of `rel`'s parent folder: from the pass's folder-uid map
|
||||
/// first, and from the baseline second for a parent the map never covered.
|
||||
///
|
||||
/// Used to name the folder whose cached listing a remote mutation has just
|
||||
/// invalidated (B86). `None` is not an error — it only means this pass cannot
|
||||
/// say which folder to invalidate, and the caller skips it.
|
||||
fn parent_uid_of(
|
||||
rel: &str,
|
||||
remote_dirs: &HashMap<String, NodeUid>,
|
||||
baseline: &HashMap<String, StoredSyncEntry>,
|
||||
) -> Option<NodeUid> {
|
||||
let parent = parent_rel(rel);
|
||||
remote_dirs.get(parent).cloned().or_else(|| {
|
||||
baseline
|
||||
.get(parent)
|
||||
.and_then(|e| e.remote_uid.as_deref())
|
||||
.and_then(parse_uid)
|
||||
})
|
||||
}
|
||||
|
||||
fn sync_parent_dir(path: &Path) -> std::io::Result<()> {
|
||||
match path.parent() {
|
||||
Some(parent) => std::fs::File::open(parent)?.sync_all(),
|
||||
@@ -1925,6 +1989,44 @@ mod tests {
|
||||
assert!(guard_local_wipe(&base, &fresh).is_err());
|
||||
}
|
||||
|
||||
/// B54's remaining case, end to end: a folder whose local root is still
|
||||
/// *readable* but is no longer the tree it was — an empty directory left
|
||||
/// where a mount used to be, or a different device mounted over it.
|
||||
///
|
||||
/// Nothing fails here. The scan succeeds and honestly reports an empty
|
||||
/// directory, so the only thing standing between the user and a trashed
|
||||
/// remote folder is the guard, and it has to fire for a one-file baseline
|
||||
/// exactly as it does for a large one.
|
||||
#[test]
|
||||
fn a_replaced_mountpoint_trips_the_guard_at_every_baseline_size() {
|
||||
let root = sync_test_dir("wipe-replaced-mount");
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut local: HashMap<String, LocalItem> = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap();
|
||||
assert!(local.is_empty(), "the replacement root is genuinely empty");
|
||||
|
||||
for size in 1..=3 {
|
||||
let baseline: HashMap<String, ()> =
|
||||
(0..size).map(|i| (format!("file{i}.txt"), ())).collect();
|
||||
assert!(
|
||||
guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_err(),
|
||||
"a {size}-path baseline against an empty root is not a deletion"
|
||||
);
|
||||
}
|
||||
|
||||
// And the guard steps aside the moment the real tree is back, so a
|
||||
// remounted folder resumes rather than staying wedged.
|
||||
std::fs::write(root.join("file0.txt"), b"back").unwrap();
|
||||
let mut local: HashMap<String, LocalItem> = HashMap::new();
|
||||
walk_local_tree(&root, &root, &rules, &writing, &mut local, &mut progress).unwrap();
|
||||
let baseline: HashMap<String, ()> = [("file0.txt".to_string(), ())].into_iter().collect();
|
||||
assert!(guard_local_wipe(&filter_baseline(&baseline, &rules), &local).is_ok());
|
||||
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
}
|
||||
|
||||
/// A restored folder (features.md 5.2) starts with an empty baseline against
|
||||
/// an empty local directory and a full remote. That must reconcile as
|
||||
/// "download everything" rather than tripping the wipe guard — otherwise
|
||||
@@ -2126,6 +2228,204 @@ mod tests {
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
/// B49's remaining cases. Preservation is the step that makes a download
|
||||
/// safe to publish, so every way it can fail has to leave the local file
|
||||
/// where it was — the download is aborted, not the file replaced.
|
||||
#[test]
|
||||
fn conflict_preservation_failures_all_keep_the_local_file() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
// An unwritable directory: the conflict copy cannot be created at all.
|
||||
let dir = sync_test_dir("conflict-eacces");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"the only copy").unwrap();
|
||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o500)).unwrap();
|
||||
let denied = preserve_conflict_copy(&source, 1700);
|
||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(denied.is_err(), "a read-only directory must fail the copy");
|
||||
assert_eq!(
|
||||
std::fs::read(&source).unwrap(),
|
||||
b"the only copy",
|
||||
"the local file survives byte-identical"
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
// A source that vanished between planning and preservation. The pass
|
||||
// must report it rather than publish the download over the gap.
|
||||
let dir = sync_test_dir("conflict-enoent");
|
||||
assert!(preserve_conflict_copy(&dir.join("gone.txt"), 1700).is_err());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
// A write error partway through the copy — induced by making the
|
||||
// destination name a directory, so `create_new` fails as something
|
||||
// other than `AlreadyExists`.
|
||||
let dir = sync_test_dir("conflict-eisdir");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"still the only copy").unwrap();
|
||||
std::fs::create_dir(conflict_path(&source, 1700)).unwrap();
|
||||
// Suffix 0 is taken by a directory; `create_new` on it reports
|
||||
// `AlreadyExists`, so preservation moves on to suffix 1 and succeeds.
|
||||
// What must not happen is the source being removed without a copy.
|
||||
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||
assert!(preserved.is_file());
|
||||
assert_eq!(std::fs::read(&preserved).unwrap(), b"still the only copy");
|
||||
assert!(!source.exists());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
/// A conflict copy keeps the bytes *and* the metadata that identifies them,
|
||||
/// so the next pass uploads the user's version rather than a file that looks
|
||||
/// freshly written.
|
||||
#[test]
|
||||
fn a_conflict_copy_is_byte_and_mtime_identical() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let dir = sync_test_dir("conflict-identity");
|
||||
let source = dir.join("notes.txt");
|
||||
std::fs::write(&source, b"local edit that must survive").unwrap();
|
||||
let before = std::fs::metadata(&source).unwrap();
|
||||
|
||||
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||
let after = std::fs::metadata(&preserved).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read(&preserved).unwrap(),
|
||||
b"local edit that must survive"
|
||||
);
|
||||
assert_eq!(after.len(), before.len());
|
||||
assert_eq!(after.modified().unwrap(), before.modified().unwrap());
|
||||
assert_eq!(after.permissions().mode(), before.permissions().mode());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
// ---- B55: an incomplete local scan is never a set of deletions ---------
|
||||
|
||||
fn no_progress() -> impl FnMut(&str) {
|
||||
|_: &str| {}
|
||||
}
|
||||
|
||||
/// A directory the pass cannot read must fail the pass. Omitting it would
|
||||
/// hand reconciliation a baseline-minus-subtree, which it reads as the user
|
||||
/// having deleted that subtree and propagates to Drive (B55).
|
||||
#[test]
|
||||
fn an_unreadable_subdirectory_fails_the_scan_instead_of_shrinking_it() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-eacces");
|
||||
std::fs::write(root.join("keep.txt"), b"a").unwrap();
|
||||
let locked = root.join("locked");
|
||||
std::fs::create_dir(&locked).unwrap();
|
||||
std::fs::write(locked.join("inner.txt"), b"b").unwrap();
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
|
||||
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let error = result.expect_err("an unreadable directory must abort the pass");
|
||||
assert!(
|
||||
error.contains("locked"),
|
||||
"the error should name the subtree it could not read: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The same rule one level up: if the sync root itself cannot be read, the
|
||||
/// pass sees nothing, and "nothing" must never mean "delete everything".
|
||||
#[test]
|
||||
fn an_unreadable_root_fails_the_scan() {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-root-eacces");
|
||||
std::fs::write(root.join("keep.txt"), b"a").unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
/// A root that is gone entirely — an unmounted device folder, a removed
|
||||
/// directory — is the sharpest version of the same case: every baseline path
|
||||
/// is "missing", and the scan must refuse rather than report an empty tree.
|
||||
#[test]
|
||||
fn an_absent_root_fails_the_scan() {
|
||||
let root = sync_test_dir("scan-absent");
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
assert!(walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress).is_err());
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
/// A name that is not valid UTF-8 cannot be keyed into the scan, so it
|
||||
/// cannot be compared against the baseline either. Refusing the pass is the
|
||||
/// only safe reading: skipping it would classify the file as deleted.
|
||||
#[test]
|
||||
fn a_non_utf8_name_fails_the_scan() {
|
||||
use std::ffi::OsStr;
|
||||
use std::os::unix::ffi::OsStrExt as _;
|
||||
|
||||
let root = sync_test_dir("scan-non-utf8");
|
||||
std::fs::write(root.join(OsStr::from_bytes(b"bad\xffname")), b"x").unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut progress = no_progress();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut progress);
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
let error = result.expect_err("a name the scan cannot key must abort the pass");
|
||||
assert!(error.contains("UTF-8"), "{error}");
|
||||
}
|
||||
|
||||
/// The complement: a scan that *can* read everything reports everything, so
|
||||
/// the strictness above does not come at the cost of a working pass. A
|
||||
/// symlink is skipped by design rather than by failure, and a dangling one
|
||||
/// stats fine through `symlink_metadata`.
|
||||
#[test]
|
||||
fn a_readable_tree_scans_completely() {
|
||||
let root = sync_test_dir("scan-complete");
|
||||
std::fs::write(root.join("a.txt"), b"aa").unwrap();
|
||||
std::fs::create_dir(root.join("sub")).unwrap();
|
||||
std::fs::write(root.join("sub/b.txt"), b"bbb").unwrap();
|
||||
std::os::unix::fs::symlink("nowhere", root.join("dangling")).unwrap();
|
||||
|
||||
let rules = rules_for(&[]);
|
||||
let writing = HashSet::new();
|
||||
let mut out = HashMap::new();
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
let result = walk_local_tree(&root, &root, &rules, &writing, &mut out, &mut |rel| {
|
||||
seen.push(rel.to_string())
|
||||
});
|
||||
std::fs::remove_dir_all(&root).unwrap();
|
||||
|
||||
result.unwrap();
|
||||
let mut keys: Vec<&str> = out.keys().map(String::as_str).collect();
|
||||
keys.sort_unstable();
|
||||
assert_eq!(keys, vec!["a.txt", "sub", "sub/b.txt"]);
|
||||
assert_eq!(out["a.txt"].size, 2);
|
||||
assert_eq!(out["sub/b.txt"].size, 3);
|
||||
assert!(out["sub"].is_dir);
|
||||
assert_eq!(seen.len(), 3, "every reported entry ticks scan progress");
|
||||
}
|
||||
|
||||
fn sync_test_dir(label: &str) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
//! Importing a Google Photos Takeout export into Proton Photos.
|
||||
//!
|
||||
//! [`pdfs_core::takeout`] says what is in the archives; this is the part that
|
||||
//! talks to Proton. One import runs on its own thread and moves through four
|
||||
//! phases, reporting into the transfer registry as a job so the GUI's transfer
|
||||
//! list shows it without a bespoke widget:
|
||||
//!
|
||||
//! 1. **Scan** — read the archives' central directories and sidecars.
|
||||
//! 2. **Deduplicate against the account** — ask the photos volume which of the
|
||||
//! names already exist ([`ProtonPhotosClient::name_collisions`]), hash only
|
||||
//! those files, then confirm with the content digest. A library that is
|
||||
//! already half-synced by Proton Photos on a phone therefore costs one cheap
|
||||
//! request per 150 names and reads no media at all for the misses.
|
||||
//! 3. **Upload** the survivors, streaming each entry straight out of its zip —
|
||||
//! no extraction to disk, so the import needs no scratch space.
|
||||
//! 4. **Albums** — create the albums the export names that the account lacks,
|
||||
//! then add every photo of each album, the ones that were already on the
|
||||
//! account included.
|
||||
//!
|
||||
//! Re-running an interrupted import is the resume story: phase 2 finds
|
||||
//! everything the previous run uploaded and skips it. Nothing is persisted
|
||||
//! between runs.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::io::Read;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use pdfs_core::control::{ActivityKind, ImportSummary, TransferDirection};
|
||||
use pdfs_core::takeout::{Bucket, TakeoutPhoto, TakeoutScan, scan_archives};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use proton_drive_rs::{PhotoTag, PhotoTagsUpdate, PhotoUploadMetadata};
|
||||
use sha1::{Digest, Sha1};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::Core;
|
||||
use crate::transfers::CountingReader;
|
||||
|
||||
/// The one import this daemon may be running, and how the last one ended.
|
||||
///
|
||||
/// Process-global rather than a [`Core`] field because it must be *one* import
|
||||
/// per daemon however many `Core` clones exist (the forked on-demand mounts each
|
||||
/// hold one), and because a front-end asking for the result after the import
|
||||
/// finished must still find it.
|
||||
static IMPORT: std::sync::LazyLock<parking_lot::Mutex<ImportState>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(ImportState::default()));
|
||||
|
||||
#[derive(Default)]
|
||||
struct ImportState {
|
||||
running: bool,
|
||||
cancel: Option<Arc<AtomicBool>>,
|
||||
last: Option<ImportSummary>,
|
||||
}
|
||||
|
||||
/// Whether an import is running, and the last one's report.
|
||||
pub(crate) fn import_status() -> (bool, Option<ImportSummary>) {
|
||||
let state = IMPORT.lock();
|
||||
(state.running, state.last.clone())
|
||||
}
|
||||
|
||||
/// Ask the running import to stop. False when none is running.
|
||||
pub(crate) fn cancel_import() -> bool {
|
||||
match &IMPORT.lock().cancel {
|
||||
Some(flag) => {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// One photo to import, after the copies Google keeps per album are folded into
|
||||
/// a single upload that joins several albums.
|
||||
struct Planned {
|
||||
photo: TakeoutPhoto,
|
||||
/// Every album this photo belongs to, by export title.
|
||||
albums: Vec<String>,
|
||||
/// Filled in phase 2/3: where the photo ended up on the account.
|
||||
uid: Option<NodeUid>,
|
||||
}
|
||||
|
||||
/// Fold the scan into one entry per distinct photo.
|
||||
///
|
||||
/// Google stores a full copy of a photo in every album folder it appears in, so
|
||||
/// the same file is in the export two or three times over. They are folded on
|
||||
/// `(name, size)`: a byte-identical copy under the same name is the same photo,
|
||||
/// and the album memberships of the copies are unioned onto the survivor. Two
|
||||
/// genuinely different photos sharing both a name and an exact size would be
|
||||
/// merged — vanishingly unlikely, and the cost of being wrong is one photo
|
||||
/// filed under an extra album rather than lost data.
|
||||
fn plan(scan: &TakeoutScan) -> Vec<Planned> {
|
||||
let mut order: Vec<(String, u64)> = Vec::new();
|
||||
let mut by_key: HashMap<(String, u64), Planned> = HashMap::new();
|
||||
|
||||
for photo in &scan.photos {
|
||||
let key = (photo.name.clone(), photo.size);
|
||||
let entry = by_key.entry(key.clone()).or_insert_with(|| {
|
||||
order.push(key.clone());
|
||||
Planned {
|
||||
photo: photo.clone(),
|
||||
albums: Vec::new(),
|
||||
uid: None,
|
||||
}
|
||||
});
|
||||
if let Bucket::Album(title) = &photo.bucket
|
||||
&& !entry.albums.iter().any(|existing| existing == title)
|
||||
{
|
||||
entry.albums.push(title.clone());
|
||||
}
|
||||
// A copy that carries a capture time wins over one that does not: the
|
||||
// album copy and the year-folder copy do not always both have a sidecar.
|
||||
if entry.photo.capture_time.is_none() && photo.capture_time.is_some() {
|
||||
entry.photo.capture_time = photo.capture_time;
|
||||
}
|
||||
entry.photo.favorite |= photo.favorite;
|
||||
}
|
||||
|
||||
order
|
||||
.into_iter()
|
||||
.filter_map(|key| by_key.remove(&key))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Open one archive and stream `entry` through `sink`.
|
||||
fn read_entry<R>(
|
||||
archive: &mut zip::ZipArchive<std::io::BufReader<std::fs::File>>,
|
||||
entry: &str,
|
||||
sink: impl FnOnce(&mut (dyn Read + Send)) -> R,
|
||||
) -> CoreResult<R> {
|
||||
let mut file = archive
|
||||
.by_name(entry)
|
||||
.map_err(|e| CoreError::invalid(format!("{entry}: {e}")))?;
|
||||
Ok(sink(&mut file))
|
||||
}
|
||||
|
||||
impl Core {
|
||||
/// Import Google Photos Takeout archives. Blocking; run on its own thread.
|
||||
///
|
||||
/// `cancel` stops the run at the next photo boundary — an upload already on
|
||||
/// the wire is allowed to finish, so a cancelled import never leaves a
|
||||
/// half-written revision behind.
|
||||
pub(crate) fn import_takeout(
|
||||
&self,
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
cancel: Arc<AtomicBool>,
|
||||
) -> CoreResult<ImportSummary> {
|
||||
let job = self.transfers.begin_job("Importing Google Photos");
|
||||
job.detail("reading archives");
|
||||
|
||||
let scan = scan_archives(&archives)?;
|
||||
let mut planned = plan(&scan);
|
||||
let mut summary = ImportSummary {
|
||||
found: planned.len(),
|
||||
skipped_trashed: scan.skipped_trashed,
|
||||
..Default::default()
|
||||
};
|
||||
job.set_total(planned.len() as u64);
|
||||
info!(
|
||||
archives = archives.len(),
|
||||
photos = summary.found,
|
||||
albums = scan.albums.len(),
|
||||
trashed = summary.skipped_trashed,
|
||||
"takeout scan complete"
|
||||
);
|
||||
|
||||
let photos = self.photos();
|
||||
if self
|
||||
.rt
|
||||
.block_on(photos.get_photos_root())
|
||||
.map_err(|e| CoreError::from_api(&e, "resolve photos library"))?
|
||||
.is_none()
|
||||
{
|
||||
return Err(CoreError::invalid(
|
||||
"this account has no Proton Photos library yet; open Proton Photos once to create it, then import again",
|
||||
));
|
||||
}
|
||||
|
||||
// Phase 2: which of these are already on the account.
|
||||
job.detail("checking for photos you already have");
|
||||
self.mark_existing(&mut planned, &archives, &mut summary, &cancel)?;
|
||||
|
||||
if dry_run {
|
||||
summary.duplicates = planned.iter().filter(|p| p.uid.is_some()).count();
|
||||
return Ok(summary);
|
||||
}
|
||||
|
||||
// Phase 3: upload what is left, one archive at a time so each zip's
|
||||
// central directory is parsed once.
|
||||
self.upload_planned(&mut planned, &archives, &mut summary, &cancel, &job)?;
|
||||
|
||||
// Phase 4: albums. Attempted even after a cancel — the photos that did
|
||||
// upload should still land in their albums.
|
||||
if !scan.albums.is_empty() {
|
||||
job.detail("sorting photos into albums");
|
||||
if let Err(e) = self.link_albums(&planned, &scan.albums, &mut summary) {
|
||||
warn!(error = %e, "takeout album linking failed");
|
||||
}
|
||||
}
|
||||
|
||||
self.spawn_timeline_refresh();
|
||||
summary.cancelled = cancel.load(Ordering::Relaxed);
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Fill in `uid` for every planned photo the account already holds.
|
||||
///
|
||||
/// Two passes on purpose: the name-only query is one request per 150 photos
|
||||
/// and needs no file contents, so the expensive SHA-1 pass only runs for the
|
||||
/// handful of names that actually collide.
|
||||
fn mark_existing(
|
||||
&self,
|
||||
planned: &mut [Planned],
|
||||
archives: &[PathBuf],
|
||||
summary: &mut ImportSummary,
|
||||
cancel: &AtomicBool,
|
||||
) -> CoreResult<()> {
|
||||
let photos = self.photos();
|
||||
let names: Vec<String> = planned.iter().map(|p| p.photo.name.clone()).collect();
|
||||
let collisions = self
|
||||
.rt
|
||||
.block_on(photos.name_collisions(&names))
|
||||
.map_err(|e| CoreError::from_api(&e, "check for existing photos"))?;
|
||||
|
||||
// Hash the colliding files, grouped by archive so each zip opens once.
|
||||
let mut by_archive: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
|
||||
for (index, planned) in planned.iter().enumerate() {
|
||||
if collisions.get(index).copied().unwrap_or(false) {
|
||||
by_archive
|
||||
.entry(planned.photo.archive)
|
||||
.or_default()
|
||||
.push(index);
|
||||
}
|
||||
}
|
||||
|
||||
// Kept as a pair list, not a map: the duplicate lookup answers in input
|
||||
// order, so the order these were built in is what maps answers back to
|
||||
// planned photos.
|
||||
let mut digests: Vec<(usize, String)> = Vec::new();
|
||||
for (archive_index, indices) in by_archive {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let mut archive = open_archive(archives, archive_index)?;
|
||||
for index in indices {
|
||||
let entry = planned[index].photo.entry.clone();
|
||||
match read_entry(&mut archive, &entry, sha1_of) {
|
||||
Ok(Ok(digest)) => digests.push((index, digest)),
|
||||
Ok(Err(e)) | Err(e) => {
|
||||
warn!(entry, error = %e, "cannot hash takeout entry");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if digests.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let items: Vec<(String, String)> = digests
|
||||
.iter()
|
||||
.map(|(index, digest)| (planned[*index].photo.name.clone(), digest.clone()))
|
||||
.collect();
|
||||
let matches = self
|
||||
.rt
|
||||
.block_on(photos.find_duplicates_many(&items))
|
||||
.map_err(|e| CoreError::from_api(&e, "check for existing photos"))?;
|
||||
|
||||
for (slot, (index, _)) in digests.iter().enumerate() {
|
||||
if let Some(uid) = matches.get(slot).and_then(|uids| uids.first()) {
|
||||
planned[*index].uid = Some(uid.clone());
|
||||
summary.duplicates += 1;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Upload every planned photo that is not already on the account.
|
||||
fn upload_planned(
|
||||
&self,
|
||||
planned: &mut [Planned],
|
||||
archives: &[PathBuf],
|
||||
summary: &mut ImportSummary,
|
||||
cancel: &AtomicBool,
|
||||
job: &crate::transfers::JobGuard,
|
||||
) -> CoreResult<()> {
|
||||
let photos = self.photos();
|
||||
let mut favorites: Vec<NodeUid> = Vec::new();
|
||||
|
||||
let mut by_archive: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
|
||||
for (index, item) in planned.iter().enumerate() {
|
||||
if item.uid.is_none() {
|
||||
by_archive
|
||||
.entry(item.photo.archive)
|
||||
.or_default()
|
||||
.push(index);
|
||||
}
|
||||
}
|
||||
|
||||
for (archive_index, indices) in by_archive {
|
||||
if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() {
|
||||
break;
|
||||
}
|
||||
let mut archive = open_archive(archives, archive_index)?;
|
||||
|
||||
for index in indices {
|
||||
if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() {
|
||||
break;
|
||||
}
|
||||
let photo = planned[index].photo.clone();
|
||||
job.detail(photo.name.clone());
|
||||
|
||||
let guard = self.transfers.begin(
|
||||
photo.name.clone(),
|
||||
"",
|
||||
TransferDirection::Upload,
|
||||
photo.size,
|
||||
);
|
||||
let metadata = PhotoUploadMetadata {
|
||||
capture_time: photo.capture_time,
|
||||
..Default::default()
|
||||
};
|
||||
let uploaded = read_entry(&mut archive, &photo.entry, |reader| {
|
||||
let counting = CountingReader::new(reader, &guard);
|
||||
self.rt.block_on(photos.upload_photo_from(
|
||||
&photo.name,
|
||||
&photo.media_type,
|
||||
counting,
|
||||
photo.size as i64,
|
||||
Vec::new(),
|
||||
metadata,
|
||||
false,
|
||||
))
|
||||
});
|
||||
drop(guard);
|
||||
job.step();
|
||||
|
||||
match uploaded {
|
||||
Ok(Ok(uid)) => {
|
||||
summary.uploaded += 1;
|
||||
summary.bytes += photo.size;
|
||||
if photo.favorite {
|
||||
favorites.push(uid.clone());
|
||||
}
|
||||
planned[index].uid = Some(uid);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
summary.failed += 1;
|
||||
warn!(name = photo.name, error = %e, "takeout photo upload failed");
|
||||
}
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
warn!(name = photo.name, error = %e, "cannot read takeout entry");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Favourites are a separate call: the upload seal takes classification
|
||||
// tags, but `Favorite` is not one of them — it goes through the photos
|
||||
// volume's dedicated favourite endpoint.
|
||||
if !favorites.is_empty() {
|
||||
let updates: Vec<PhotoTagsUpdate> = favorites
|
||||
.into_iter()
|
||||
.map(|node_uid| PhotoTagsUpdate {
|
||||
node_uid,
|
||||
tags_to_add: vec![PhotoTag::Favorite],
|
||||
tags_to_remove: Vec::new(),
|
||||
})
|
||||
.collect();
|
||||
if let Err(e) = self.rt.block_on(photos.update_photos(&updates)) {
|
||||
warn!(error = %e, "marking imported favourites failed");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create the albums the export names that the account lacks, then add each
|
||||
/// album's photos to it.
|
||||
fn link_albums(
|
||||
&self,
|
||||
planned: &[Planned],
|
||||
albums: &[String],
|
||||
summary: &mut ImportSummary,
|
||||
) -> CoreResult<()> {
|
||||
let photos = self.photos();
|
||||
|
||||
// Existing albums by decrypted name, so a second import of the same
|
||||
// export adds to the album it made the first time instead of a twin.
|
||||
let mut existing: HashMap<String, NodeUid> = HashMap::new();
|
||||
let uids = self
|
||||
.rt
|
||||
.block_on(photos.enumerate_album_node_uids())
|
||||
.map_err(|e| CoreError::from_api(&e, "list albums"))?;
|
||||
if !uids.is_empty() {
|
||||
let albums_nodes = self
|
||||
.rt
|
||||
.block_on(photos.enumerate_nodes(&uids))
|
||||
.map_err(|e| CoreError::from_api(&e, "read albums"))?;
|
||||
for node in albums_nodes {
|
||||
existing.entry(node.name.clone()).or_insert(node.uid);
|
||||
}
|
||||
}
|
||||
|
||||
for title in albums {
|
||||
let members: Vec<NodeUid> = planned
|
||||
.iter()
|
||||
.filter(|item| item.albums.iter().any(|album| album == title))
|
||||
.filter_map(|item| item.uid.clone())
|
||||
.collect();
|
||||
if members.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let album_uid = match existing.get(title) {
|
||||
Some(uid) => uid.clone(),
|
||||
None => match self.rt.block_on(photos.create_album(title)) {
|
||||
Ok(uid) => {
|
||||
summary.albums_created += 1;
|
||||
existing.insert(title.clone(), uid.clone());
|
||||
uid
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(album = title, error = %e, "cannot create album");
|
||||
continue;
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
match self
|
||||
.rt
|
||||
.block_on(photos.add_photos_to_album(&album_uid, &members))
|
||||
{
|
||||
Ok(outcomes) => {
|
||||
for (uid, outcome) in outcomes {
|
||||
match outcome {
|
||||
Ok(()) => summary.album_links += 1,
|
||||
// A photo already in the album is the expected
|
||||
// outcome of a re-run, not a failure worth counting.
|
||||
Err(e) => {
|
||||
warn!(album = title, %uid, error = %e, "cannot add photo to album")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(album = title, error = %e, "album addition failed"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run an import on a background thread, logging its outcome to the activity
|
||||
/// feed. Returns immediately.
|
||||
///
|
||||
/// Refuses while one is already running: two imports of the same export
|
||||
/// would race each other's duplicate detection and upload everything twice.
|
||||
pub(crate) fn spawn_takeout_import(
|
||||
&self,
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
) -> CoreResult<()> {
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
{
|
||||
let mut state = IMPORT.lock();
|
||||
if state.running {
|
||||
return Err(CoreError::conflict(
|
||||
"an import is already running; wait for it to finish or cancel it",
|
||||
));
|
||||
}
|
||||
state.running = true;
|
||||
state.cancel = Some(cancel.clone());
|
||||
state.last = None;
|
||||
}
|
||||
|
||||
let core = self.clone();
|
||||
std::thread::spawn(move || {
|
||||
let outcome = core.import_takeout(archives, dry_run, cancel);
|
||||
{
|
||||
let mut state = IMPORT.lock();
|
||||
state.running = false;
|
||||
state.cancel = None;
|
||||
state.last = outcome.as_ref().ok().cloned();
|
||||
}
|
||||
match outcome {
|
||||
Ok(summary) => {
|
||||
let mut parts = vec![format!("{} uploaded", summary.uploaded)];
|
||||
if summary.duplicates > 0 {
|
||||
parts.push(format!("{} already there", summary.duplicates));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
parts.push(format!("{} albums", summary.albums_created));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
parts.push(format!("{} failed", summary.failed));
|
||||
}
|
||||
if summary.cancelled {
|
||||
parts.push("cancelled".to_string());
|
||||
}
|
||||
core.log_activity(
|
||||
ActivityKind::Upload,
|
||||
"Google Photos import",
|
||||
parts.join(" · "),
|
||||
summary.failed == 0 && !summary.cancelled,
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "takeout import failed");
|
||||
core.log_activity(ActivityKind::Upload, "Google Photos import", &e, false);
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Open one of the import's archives by index.
|
||||
fn open_archive(
|
||||
archives: &[PathBuf],
|
||||
index: usize,
|
||||
) -> CoreResult<zip::ZipArchive<std::io::BufReader<std::fs::File>>> {
|
||||
let path = archives
|
||||
.get(index)
|
||||
.ok_or_else(|| CoreError::internal(format!("archive {index} is not in the import")))?;
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
zip::ZipArchive::new(std::io::BufReader::new(file))
|
||||
.map_err(|e| CoreError::invalid(format!("{} is not a readable zip: {e}", path.display())))
|
||||
}
|
||||
|
||||
/// The lowercase-hex SHA-1 of everything `reader` yields — the digest Proton's
|
||||
/// photo duplicate detection is keyed on.
|
||||
fn sha1_of(reader: &mut (dyn Read + Send)) -> CoreResult<String> {
|
||||
let mut hasher = Sha1::new();
|
||||
let mut buffer = vec![0u8; 256 * 1024];
|
||||
loop {
|
||||
let read = reader
|
||||
.read(&mut buffer)
|
||||
.map_err(|e| CoreError::invalid(format!("read failed: {e}")))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
}
|
||||
Ok(hex_lower(&hasher.finalize()))
|
||||
}
|
||||
|
||||
fn hex_lower(bytes: &[u8]) -> String {
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
out.push_str(&format!("{byte:02x}"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pdfs_core::takeout::TakeoutPhoto;
|
||||
|
||||
fn photo(name: &str, size: u64, bucket: Bucket) -> TakeoutPhoto {
|
||||
TakeoutPhoto {
|
||||
archive: 0,
|
||||
entry: format!("Takeout/Google Photos/x/{name}"),
|
||||
name: name.to_string(),
|
||||
size,
|
||||
media_type: "image/jpeg".to_string(),
|
||||
capture_time: None,
|
||||
favorite: false,
|
||||
bucket,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn album_copies_of_one_photo_fold_into_a_single_upload() {
|
||||
let scan = TakeoutScan {
|
||||
photos: vec![
|
||||
photo("IMG_1.jpg", 100, Bucket::Timeline),
|
||||
photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into())),
|
||||
photo("IMG_1.jpg", 100, Bucket::Album("Best of".into())),
|
||||
photo("IMG_2.jpg", 200, Bucket::Timeline),
|
||||
],
|
||||
albums: vec!["Iceland".into(), "Best of".into()],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let planned = plan(&scan);
|
||||
assert_eq!(planned.len(), 2, "the three copies are one upload");
|
||||
assert_eq!(planned[0].albums, vec!["Iceland", "Best of"]);
|
||||
assert!(planned[1].albums.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_name_different_size_stays_two_photos() {
|
||||
let scan = TakeoutScan {
|
||||
photos: vec![
|
||||
photo("IMG_1.jpg", 100, Bucket::Timeline),
|
||||
photo("IMG_1.jpg", 101, Bucket::Album("Iceland".into())),
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(plan(&scan).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_from_any_copy_survives_the_fold() {
|
||||
// Only the album copy carries a sidecar; the fold must not lose it.
|
||||
let mut bare = photo("IMG_1.jpg", 100, Bucket::Timeline);
|
||||
bare.capture_time = None;
|
||||
let mut rich = photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into()));
|
||||
rich.capture_time = Some(1_560_000_000);
|
||||
rich.favorite = true;
|
||||
|
||||
let planned = plan(&TakeoutScan {
|
||||
photos: vec![bare, rich],
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(planned[0].photo.capture_time, Some(1_560_000_000));
|
||||
assert!(planned[0].photo.favorite);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha1_matches_the_known_digest_of_abc() {
|
||||
let mut reader: &[u8] = b"abc";
|
||||
assert_eq!(
|
||||
sha1_of(&mut reader).unwrap(),
|
||||
"a9993e364706816aba3e25717850c26c9cd0d89d"
|
||||
);
|
||||
}
|
||||
}
|
||||
+109
-10
@@ -14,6 +14,7 @@ use pages::photos::*;
|
||||
use pages::shared::*;
|
||||
use pages::shared_by_me::*;
|
||||
use pages::status::*;
|
||||
use pages::takeout::*;
|
||||
use pages::trash::*;
|
||||
use pages::verify::*;
|
||||
use widgets::details::*;
|
||||
@@ -46,10 +47,10 @@ use pdfs_core::config::AppDirs;
|
||||
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, DeviceInfo, DirEntry, ErrorKind,
|
||||
InvitationInfo, JobItem, PhotoItem, PhotoKind, PublicLinkInfo, RefreshScope, Request, Response,
|
||||
RestorableFolder, RestoreItem, SearchHit, ShareEntry, ShareEntryKind, SharedItem,
|
||||
SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem, pending_summary,
|
||||
send,
|
||||
ImportSummary, InvitationInfo, JobItem, PhotoItem, PhotoKind, PublicLinkInfo, RefreshScope,
|
||||
Request, Response, RestorableFolder, RestoreItem, SearchHit, ShareEntry, ShareEntryKind,
|
||||
SharedItem, SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem,
|
||||
pending_summary, send,
|
||||
};
|
||||
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
@@ -120,6 +121,7 @@ struct Ui {
|
||||
pub(crate) devices: DevicesState,
|
||||
pub(crate) locations: LocationsState,
|
||||
pub(crate) activity: ActivityState,
|
||||
pub(crate) takeout: TakeoutState,
|
||||
}
|
||||
|
||||
impl Ui {
|
||||
@@ -246,7 +248,10 @@ fn load_proton_theme() {
|
||||
.navigation-sidebar row {{ border-radius: 8px; margin: 2px 6px; }}\n\
|
||||
.navigation-sidebar row:selected {{ background: alpha({PROTON_PURPLE}, 0.16); color: {PROTON_PURPLE}; font-weight: 600; }}\n\
|
||||
.navigation-sidebar row:selected image {{ color: {PROTON_PURPLE}; }}\n\
|
||||
.file-tile:selected, .file-tile:hover:selected {{ background: alpha({PROTON_PURPLE}, 0.20); }}\n"
|
||||
.file-tile:selected, .file-tile:hover:selected {{ background: alpha({PROTON_PURPLE}, 0.20); }}\n\
|
||||
.bulk-bar {{ padding: 6px 8px; border-radius: 12px; background: alpha({PROTON_PURPLE}, 0.12); }}\n\
|
||||
.dropzone {{ border: 2px dashed alpha(currentColor, 0.25); border-radius: 16px; padding: 28px 18px; transition: background 160ms ease, border-color 160ms ease; }}\n\
|
||||
.dropzone-active {{ border-color: {PROTON_PURPLE}; background: alpha({PROTON_PURPLE}, 0.10); }}\n"
|
||||
);
|
||||
let provider = gtk4::CssProvider::new();
|
||||
provider.load_from_string(&css);
|
||||
@@ -285,6 +290,7 @@ fn build_window(app: &adw::Application) {
|
||||
let (locations_page, locations_widgets) = build_locations_page();
|
||||
let (activity_page, activity_widgets) = build_activity_page();
|
||||
let (trash_page, trash_widgets) = build_trash_page();
|
||||
let (takeout_page, takeout_widgets) = build_takeout_page();
|
||||
stack.add_named(&login_page, Some("login"));
|
||||
stack.add_named(&main_page, Some("main"));
|
||||
stack.add_named(&browser_page, Some("browser"));
|
||||
@@ -295,6 +301,7 @@ fn build_window(app: &adw::Application) {
|
||||
stack.add_named(&locations_page, Some("locations"));
|
||||
stack.add_named(&activity_page, Some("activity"));
|
||||
stack.add_named(&trash_page, Some("trash"));
|
||||
stack.add_named(&takeout_page, Some("takeout"));
|
||||
|
||||
// Sidebar: the signed-in destinations. Selecting a row swaps the page stack;
|
||||
// `sync_sidebar` pushes the other way when navigation happens elsewhere (e.g.
|
||||
@@ -362,6 +369,8 @@ fn build_window(app: &adw::Application) {
|
||||
budget_row: main_widgets.budget_row.clone(),
|
||||
mountpoint_row: main_widgets.mountpoint_row.clone(),
|
||||
settings_suppress: Cell::new(false),
|
||||
budget_source: RefCell::new(None),
|
||||
pins_expanded: Cell::new(false),
|
||||
pins_group: main_widgets.pins_group.clone(),
|
||||
pin_rows: RefCell::new(Vec::new()),
|
||||
pins_state: RefCell::new(None),
|
||||
@@ -382,6 +391,13 @@ fn build_window(app: &adw::Application) {
|
||||
upload: browser_widgets.upload.clone(),
|
||||
upload_folder: browser_widgets.upload_folder.clone(),
|
||||
search_source: RefCell::new(None),
|
||||
views: browser_widgets.views.clone(),
|
||||
bulk: browser_widgets.bulk.clone(),
|
||||
bulk_label: browser_widgets.bulk_label.clone(),
|
||||
bulk_trash: browser_widgets.bulk_trash.clone(),
|
||||
bulk_pin: browser_widgets.bulk_pin.clone(),
|
||||
bulk_unpin: browser_widgets.bulk_unpin.clone(),
|
||||
empty_actions: browser_widgets.empty_actions.clone(),
|
||||
},
|
||||
details: DetailsState {
|
||||
details: browser_widgets.details,
|
||||
@@ -407,6 +423,8 @@ fn build_window(app: &adw::Application) {
|
||||
retry: gallery_widgets.retry.clone(),
|
||||
more: gallery_widgets.more.clone(),
|
||||
upload: gallery_widgets.upload.clone(),
|
||||
import: gallery_widgets.import.clone(),
|
||||
empty_actions: gallery_widgets.empty_actions.clone(),
|
||||
title: gallery_widgets.title.clone(),
|
||||
subtitle: gallery_widgets.subtitle.clone(),
|
||||
albums: gallery_widgets.albums.clone(),
|
||||
@@ -494,6 +512,24 @@ fn build_window(app: &adw::Application) {
|
||||
inflight: Cell::new(false),
|
||||
key: RefCell::new(None),
|
||||
},
|
||||
takeout: TakeoutState {
|
||||
archives: RefCell::new(Vec::new()),
|
||||
list_group: takeout_widgets.list_group.clone(),
|
||||
rows: RefCell::new(Vec::new()),
|
||||
dropzone: takeout_widgets.dropzone.clone(),
|
||||
scan_button: takeout_widgets.scan_button.clone(),
|
||||
import_button: takeout_widgets.import_button.clone(),
|
||||
cancel_button: takeout_widgets.cancel_button.clone(),
|
||||
clear_button: takeout_widgets.clear_button.clone(),
|
||||
progress_group: takeout_widgets.progress_group.clone(),
|
||||
progress_label: takeout_widgets.progress_label.clone(),
|
||||
progress_bar: takeout_widgets.progress_bar.clone(),
|
||||
summary_group: takeout_widgets.summary_group.clone(),
|
||||
summary_rows: RefCell::new(Vec::new()),
|
||||
inflight: Cell::new(false),
|
||||
running: Cell::new(false),
|
||||
dry_run: Cell::new(false),
|
||||
},
|
||||
});
|
||||
wire_login(&ui);
|
||||
wire_logout(&ui, &main_widgets.logout_button);
|
||||
@@ -501,9 +537,16 @@ fn build_window(app: &adw::Application) {
|
||||
&ui,
|
||||
&main_widgets.purge_button,
|
||||
&main_widgets.mountpoint_button,
|
||||
&main_widgets.import_row,
|
||||
);
|
||||
wire_sidebar(&ui);
|
||||
wire_browser(&ui, &browser_widgets.grid, &browser_widgets.column_view);
|
||||
wire_bulk(
|
||||
&ui,
|
||||
&browser_widgets.bulk_clear,
|
||||
&browser_widgets.empty_upload,
|
||||
&browser_widgets.empty_new_folder,
|
||||
);
|
||||
wire_browser_actions(
|
||||
&ui,
|
||||
&browser_widgets.new_folder,
|
||||
@@ -513,6 +556,11 @@ fn build_window(app: &adw::Application) {
|
||||
wire_details(&ui);
|
||||
wire_search(&ui);
|
||||
wire_gallery(&ui, &gallery_widgets.list, &gallery_widgets.scroll);
|
||||
wire_gallery_empty(
|
||||
&ui,
|
||||
&gallery_widgets.empty_upload,
|
||||
&gallery_widgets.empty_import,
|
||||
);
|
||||
wire_albums(&ui);
|
||||
wire_trash(&ui, &trash_widgets.list, &trash_widgets.empty);
|
||||
wire_shared(&ui, &shared_widgets.retry, &shared_widgets.add_bookmark);
|
||||
@@ -520,6 +568,7 @@ fn build_window(app: &adw::Application) {
|
||||
wire_devices(&ui, &devices_widgets.retry, &devices_widgets.restore);
|
||||
wire_locations(&ui, &locations_widgets.retry, &locations_widgets.add_folder);
|
||||
wire_activity(&ui, &activity_widgets.retry);
|
||||
wire_takeout(&ui, &takeout_widgets);
|
||||
wire_refresh(
|
||||
&ui,
|
||||
&[
|
||||
@@ -559,6 +608,9 @@ fn build_window(app: &adw::Application) {
|
||||
// every visit to stay live.
|
||||
Some("activity") => load_activity(&ui_nav),
|
||||
Some("trash") => load_trash(&ui_nav),
|
||||
// The import runs in the daemon and outlives this page, so arriving
|
||||
// here asks straight away whether one is in flight.
|
||||
Some("takeout") => refresh_takeout(&ui_nav),
|
||||
_ => {}
|
||||
}
|
||||
});
|
||||
@@ -644,6 +696,17 @@ fn build_sidebar() -> (adw::NavigationPage, gtk4::ListBox) {
|
||||
(page, list)
|
||||
}
|
||||
|
||||
/// The sidebar destination a page belongs under. Most pages are their own
|
||||
/// destination; a sub-page reached from one (Import, off Settings) has no row of
|
||||
/// its own and answers with its parent, so the sidebar highlights where the user
|
||||
/// came from rather than clearing — which would read as "nowhere".
|
||||
fn destination_of(page: &str) -> &str {
|
||||
match page {
|
||||
"takeout" => "main",
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// Selecting a sidebar row navigates the page stack. The reverse direction (stack
|
||||
/// → sidebar highlight) is [`sync_sidebar`], so the two can't fight: this handler
|
||||
/// only ever writes the stack.
|
||||
@@ -654,7 +717,11 @@ fn wire_sidebar(ui: &Rc<Ui>) {
|
||||
let Some((page, _, _)) = DESTINATIONS.get(row.index() as usize) else {
|
||||
return;
|
||||
};
|
||||
if ui_row.stack.visible_child_name().as_deref() != Some(*page) {
|
||||
// Compared by *destination*, not by page: while a sub-page is open its
|
||||
// parent's row is the selected one, and re-asserting it here would kick
|
||||
// the user off the sub-page the moment they arrived.
|
||||
let current = ui_row.stack.visible_child_name();
|
||||
if current.as_deref().map(destination_of) != Some(*page) {
|
||||
ui_row.stack.set_visible_child_name(page);
|
||||
}
|
||||
});
|
||||
@@ -755,6 +822,7 @@ fn sync_sidebar(ui: &Rc<Ui>) {
|
||||
let Some(current) = ui.stack.visible_child_name() else {
|
||||
return;
|
||||
};
|
||||
let current = destination_of(¤t);
|
||||
let index = DESTINATIONS
|
||||
.iter()
|
||||
.position(|(page, _, _)| *page == current);
|
||||
@@ -776,6 +844,22 @@ fn toast(ui: &Rc<Ui>, message: &str) {
|
||||
ui.toasts.add_toast(adw::Toast::new(message));
|
||||
}
|
||||
|
||||
/// Show a toast that offers one follow-up action — an Undo, or a way to the page
|
||||
/// the outcome landed on.
|
||||
///
|
||||
/// Given a longer timeout than a plain toast: an action the user has to notice,
|
||||
/// read and reach is not the same ask as a line of confirmation they can ignore.
|
||||
fn toast_action(ui: &Rc<Ui>, message: &str, label: &str, action: impl Fn(&Rc<Ui>) + 'static) {
|
||||
let toast = adw::Toast::builder()
|
||||
.title(message)
|
||||
.button_label(label)
|
||||
.timeout(8)
|
||||
.build();
|
||||
let ui_action = ui.clone();
|
||||
toast.connect_button_clicked(move |_| action(&ui_action));
|
||||
ui.toasts.add_toast(toast);
|
||||
}
|
||||
|
||||
/// Show a toast for a failure. Same surface as [`toast`], but the message is
|
||||
/// prefixed with what was being attempted, since a bare daemon error ("no such
|
||||
/// file") reads as noise without it.
|
||||
@@ -913,7 +997,13 @@ fn install_shortcuts(ui: &Rc<Ui>, window: &adw::ApplicationWindow) {
|
||||
// browser-only bindings.
|
||||
gtk4::gdk::Key::F5 => reload_current_page(&ui),
|
||||
gtk4::gdk::Key::r | gtk4::gdk::Key::R if ctrl => reload_current_page(&ui),
|
||||
gtk4::gdk::Key::f | gtk4::gdk::Key::F if ctrl && on_browser => {
|
||||
// Search lives on Files, but the shortcut works from anywhere:
|
||||
// wanting to find a file is not a reason to first have to remember
|
||||
// which page owns the search box.
|
||||
gtk4::gdk::Key::f | gtk4::gdk::Key::F if ctrl => {
|
||||
if !on_browser {
|
||||
ui.stack.set_visible_child_name("browser");
|
||||
}
|
||||
ui.browser.search.grab_focus();
|
||||
}
|
||||
gtk4::gdk::Key::n | gtk4::gdk::Key::N if ctrl && on_browser => prompt_new_folder(&ui),
|
||||
@@ -923,12 +1013,21 @@ fn install_shortcuts(ui: &Rc<Ui>, window: &adw::ApplicationWindow) {
|
||||
prompt_rename(&ui, &entry);
|
||||
}
|
||||
}
|
||||
// Acts on the whole selection: Delete with five files highlighted
|
||||
// means those five, not whichever one the details pane happens to
|
||||
// be describing.
|
||||
gtk4::gdk::Key::Delete if on_browser => {
|
||||
if let Some(entry) = selected_entry(&ui) {
|
||||
prompt_delete(&ui, &entry);
|
||||
let entries = selected_entries(&ui);
|
||||
if !entries.is_empty() {
|
||||
prompt_delete_many(&ui, entries);
|
||||
}
|
||||
}
|
||||
gtk4::gdk::Key::Escape if on_browser && ui.browser.split.shows_sidebar() => {
|
||||
gtk4::gdk::Key::Escape
|
||||
if on_browser
|
||||
&& (ui.browser.split.shows_sidebar() || ui.browser.bulk.reveals_child()) =>
|
||||
{
|
||||
clear_selection(&ui);
|
||||
sync_bulk_bar(&ui);
|
||||
hide_details(&ui);
|
||||
}
|
||||
_ => return glib::Propagation::Proceed,
|
||||
|
||||
@@ -31,6 +31,21 @@ pub(crate) struct BrowserState {
|
||||
/// Pending debounce timer for the search box; replaced on every keystroke so
|
||||
/// only the last pause actually fires a [`Request::Search`].
|
||||
pub(crate) search_source: RefCell<Option<glib::SourceId>>,
|
||||
/// The grid/list view stack, read to find out which of the two selections is
|
||||
/// the one the user is actually working in.
|
||||
pub(crate) views: gtk4::Stack,
|
||||
/// The bulk-action bar, revealed once more than one entry is selected.
|
||||
pub(crate) bulk: gtk4::Revealer,
|
||||
pub(crate) bulk_label: gtk4::Label,
|
||||
/// Bulk buttons, sensitive only where they apply (offline state is a
|
||||
/// file-only notion, so a folder in the selection disables those two).
|
||||
pub(crate) bulk_trash: gtk4::Button,
|
||||
pub(crate) bulk_pin: gtk4::Button,
|
||||
pub(crate) bulk_unpin: gtk4::Button,
|
||||
/// Upload / New folder offered on the *empty folder* status page, so that
|
||||
/// state is a place to act rather than a dead end. Hidden on every other
|
||||
/// status (a load error is not the moment to offer an upload).
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
}
|
||||
|
||||
/// Idle pause after the last keystroke before a search query is sent, so typing
|
||||
@@ -76,9 +91,20 @@ pub(crate) struct BrowserWidgets {
|
||||
pub(crate) split: adw::OverlaySplitView,
|
||||
pub(crate) details: DetailsWidgets,
|
||||
/// The two selection models, so a selection change can drive the details pane
|
||||
/// and so an action can re-read the entry the user has highlighted.
|
||||
pub(crate) grid_selection: gtk4::SingleSelection,
|
||||
pub(crate) list_selection: gtk4::SingleSelection,
|
||||
/// and so an action can re-read the entries the user has highlighted.
|
||||
pub(crate) grid_selection: gtk4::MultiSelection,
|
||||
pub(crate) list_selection: gtk4::MultiSelection,
|
||||
/// The grid/list stack, so the page can tell which selection is live.
|
||||
pub(crate) views: gtk4::Stack,
|
||||
pub(crate) bulk: gtk4::Revealer,
|
||||
pub(crate) bulk_label: gtk4::Label,
|
||||
pub(crate) bulk_trash: gtk4::Button,
|
||||
pub(crate) bulk_pin: gtk4::Button,
|
||||
pub(crate) bulk_unpin: gtk4::Button,
|
||||
pub(crate) bulk_clear: gtk4::Button,
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
pub(crate) empty_upload: gtk4::Button,
|
||||
pub(crate) empty_new_folder: gtk4::Button,
|
||||
}
|
||||
|
||||
pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
@@ -166,19 +192,34 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
retry.add_css_class("pill");
|
||||
retry.add_css_class("suggested-action");
|
||||
retry.set_visible(false);
|
||||
// The empty-folder state's way out. `StatusPage` takes one child, so Retry
|
||||
// and these share a box; each is shown only for the state it belongs to.
|
||||
let empty_upload = gtk4::Button::builder().label("Upload files").build();
|
||||
empty_upload.add_css_class("pill");
|
||||
empty_upload.add_css_class("suggested-action");
|
||||
let empty_new_folder = gtk4::Button::builder().label("New folder").build();
|
||||
empty_new_folder.add_css_class("pill");
|
||||
let empty_actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
empty_actions.set_halign(gtk4::Align::Center);
|
||||
empty_actions.set_visible(false);
|
||||
empty_actions.append(&empty_upload);
|
||||
empty_actions.append(&empty_new_folder);
|
||||
|
||||
let status_child = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
status_child.append(&retry);
|
||||
status_child.append(&empty_actions);
|
||||
|
||||
let status = adw::StatusPage::builder()
|
||||
.icon_name("folder-symbolic")
|
||||
.vexpand(true)
|
||||
.child(&retry)
|
||||
.child(&status_child)
|
||||
.build();
|
||||
status.add_css_class("compact");
|
||||
|
||||
// Icon grid.
|
||||
let grid_selection = gtk4::SingleSelection::builder()
|
||||
.model(&model)
|
||||
.autoselect(false)
|
||||
.can_unselect(true)
|
||||
.build();
|
||||
// Icon grid. Multi-select: acting on a batch is the common case for trashing
|
||||
// and for taking a folder's worth of files offline, and doing it one
|
||||
// confirmation dialog at a time is not a workflow.
|
||||
let grid_selection = gtk4::MultiSelection::new(Some(model.clone()));
|
||||
let grid = gtk4::GridView::builder()
|
||||
.model(&grid_selection)
|
||||
.min_columns(2)
|
||||
@@ -190,12 +231,8 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
.child(&grid)
|
||||
.build();
|
||||
|
||||
// Column list.
|
||||
let list_selection = gtk4::SingleSelection::builder()
|
||||
.model(&model)
|
||||
.autoselect(false)
|
||||
.can_unselect(true)
|
||||
.build();
|
||||
// Column list, with its own selection over the same model.
|
||||
let list_selection = gtk4::MultiSelection::new(Some(model.clone()));
|
||||
let column_view = gtk4::ColumnView::builder().model(&list_selection).build();
|
||||
column_view.add_css_class("data-table");
|
||||
let column_scroll = gtk4::ScrolledWindow::builder()
|
||||
@@ -239,12 +276,51 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
.sidebar(&details_pane)
|
||||
.build();
|
||||
|
||||
// Bulk-action bar. A revealer rather than a hidden box so it slides in
|
||||
// instead of making the whole view jump when a second entry is selected.
|
||||
let bulk_label = gtk4::Label::builder().hexpand(true).xalign(0.0).build();
|
||||
let bulk_pin = gtk4::Button::builder()
|
||||
.label("Keep offline")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_pin.add_css_class("flat");
|
||||
let bulk_unpin = gtk4::Button::builder()
|
||||
.label("Remove offline copy")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_unpin.add_css_class("flat");
|
||||
let bulk_trash = gtk4::Button::builder()
|
||||
.label("Move to Trash")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_trash.add_css_class("destructive-action");
|
||||
let bulk_clear = gtk4::Button::builder()
|
||||
.icon_name("window-close-symbolic")
|
||||
.tooltip_text("Clear selection (Esc)")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
bulk_clear.add_css_class("flat");
|
||||
bulk_clear.add_css_class("circular");
|
||||
let bulk_box = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
bulk_box.add_css_class("toolbar");
|
||||
bulk_box.add_css_class("bulk-bar");
|
||||
bulk_box.append(&bulk_label);
|
||||
bulk_box.append(&bulk_pin);
|
||||
bulk_box.append(&bulk_unpin);
|
||||
bulk_box.append(&bulk_trash);
|
||||
bulk_box.append(&bulk_clear);
|
||||
let bulk = gtk4::Revealer::builder()
|
||||
.transition_type(gtk4::RevealerTransitionType::SlideDown)
|
||||
.child(&bulk_box)
|
||||
.build();
|
||||
|
||||
let inner = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
inner.set_margin_top(12);
|
||||
inner.set_margin_bottom(12);
|
||||
inner.set_margin_start(12);
|
||||
inner.set_margin_end(12);
|
||||
inner.append(&header);
|
||||
inner.append(&bulk);
|
||||
inner.append(&split);
|
||||
|
||||
(
|
||||
@@ -267,6 +343,16 @@ pub(crate) fn build_browser_page() -> (gtk4::Widget, BrowserWidgets) {
|
||||
details,
|
||||
grid_selection,
|
||||
list_selection,
|
||||
views: view_stack,
|
||||
bulk,
|
||||
bulk_label,
|
||||
bulk_trash,
|
||||
bulk_pin,
|
||||
bulk_unpin,
|
||||
bulk_clear,
|
||||
empty_actions,
|
||||
empty_upload,
|
||||
empty_new_folder,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -278,10 +364,273 @@ pub(crate) fn browser_status(ui: &Rc<Ui>, icon: &str, title: &str, description:
|
||||
ui.browser.status.set_title(title);
|
||||
ui.browser.status.set_description(Some(description));
|
||||
ui.browser.retry.set_visible(retry);
|
||||
// Only the empty-folder state offers a way to fill the folder; every other
|
||||
// status (loading, offline, error) turns them back off.
|
||||
ui.browser.empty_actions.set_visible(false);
|
||||
ui.browser.content.set_visible_child_name("status");
|
||||
hide_details(ui);
|
||||
}
|
||||
|
||||
/// The selection model of whichever view is on screen. The grid and the list
|
||||
/// each own one over the same model, so "what is selected" depends on which the
|
||||
/// user is looking at.
|
||||
pub(crate) fn active_selection(ui: &Rc<Ui>) -> gtk4::MultiSelection {
|
||||
if ui.browser.views.visible_child_name().as_deref() == Some("list") {
|
||||
ui.details.list_selection.clone()
|
||||
} else {
|
||||
ui.details.grid_selection.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Every entry highlighted in the view on screen, in model order.
|
||||
///
|
||||
/// Walks the model rather than the selection bitset: a listing is at most a few
|
||||
/// thousand rows, and asking each position whether it is selected keeps this
|
||||
/// free of bitset-iterator lifetimes for no measurable cost.
|
||||
pub(crate) fn selected_entries(ui: &Rc<Ui>) -> Vec<DirEntry> {
|
||||
let selection = active_selection(ui);
|
||||
let count = selection.n_items();
|
||||
(0..count)
|
||||
.filter(|i| selection.is_selected(*i))
|
||||
.filter_map(|i| entry_at(Some(&selection), i))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Drop the selection in both views, which also retracts the bulk bar.
|
||||
pub(crate) fn clear_selection(ui: &Rc<Ui>) {
|
||||
ui.details.grid_selection.unselect_all();
|
||||
ui.details.list_selection.unselect_all();
|
||||
}
|
||||
|
||||
/// Reflect the current selection in the bulk bar: how many are selected, and
|
||||
/// which bulk actions apply to that mix. Offline state is a file-only notion, so
|
||||
/// a folder anywhere in the selection disables the two pin buttons rather than
|
||||
/// half-failing once pressed.
|
||||
pub(crate) fn sync_bulk_bar(ui: &Rc<Ui>) {
|
||||
let entries = selected_entries(ui);
|
||||
// One selected entry is the details pane's job; the bar is for a batch.
|
||||
if entries.len() < 2 {
|
||||
ui.browser.bulk.set_reveal_child(false);
|
||||
return;
|
||||
}
|
||||
let files_only = entries.iter().all(|e| !e.is_dir);
|
||||
let any_pinned = entries.iter().any(|e| e.pinned);
|
||||
let any_unpinned = entries.iter().any(|e| !e.pinned);
|
||||
let mounted = *ui.mounted.borrow();
|
||||
ui.browser
|
||||
.bulk_label
|
||||
.set_label(&format!("{} selected", entries.len()));
|
||||
ui.browser.bulk_trash.set_sensitive(mounted);
|
||||
ui.browser
|
||||
.bulk_pin
|
||||
.set_sensitive(mounted && files_only && any_unpinned);
|
||||
ui.browser
|
||||
.bulk_unpin
|
||||
.set_sensitive(mounted && files_only && any_pinned);
|
||||
ui.browser.bulk.set_reveal_child(true);
|
||||
}
|
||||
|
||||
/// Confirm and move every selected entry to Trash.
|
||||
///
|
||||
/// One dialog for the batch, one pass over the daemon, one toast — and one Undo
|
||||
/// that restores the whole batch, because a mis-aimed bulk delete is exactly the
|
||||
/// action a user most needs to take back.
|
||||
pub(crate) fn prompt_delete_many(ui: &Rc<Ui>, entries: Vec<DirEntry>) {
|
||||
if entries.len() < 2 {
|
||||
if let Some(entry) = entries.first() {
|
||||
prompt_delete(ui, entry);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let win = ui_window(ui);
|
||||
let dialog = adw::AlertDialog::builder()
|
||||
.heading("Move to Trash")
|
||||
.body(format!(
|
||||
"Move {} items to Trash? Anything inside a selected folder goes with it.",
|
||||
entries.len()
|
||||
))
|
||||
.build();
|
||||
dialog.add_response("cancel", "Cancel");
|
||||
dialog.add_response("trash", "Move to Trash");
|
||||
dialog.set_response_appearance("trash", adw::ResponseAppearance::Destructive);
|
||||
dialog.set_default_response(Some("cancel"));
|
||||
dialog.set_close_response("cancel");
|
||||
|
||||
let ui = ui.clone();
|
||||
dialog.connect_response(None, move |_, resp| {
|
||||
if resp == "trash" {
|
||||
run_bulk_delete(&ui, entries.clone());
|
||||
}
|
||||
});
|
||||
dialog.present(win.as_ref());
|
||||
}
|
||||
|
||||
/// Trash a batch, one request at a time.
|
||||
///
|
||||
/// Sequential rather than parallel: the daemon serialises these anyway, and a
|
||||
/// burst of threads against one socket buys nothing but a harder failure to
|
||||
/// report. Partial success is reported as such — the entries that did move are
|
||||
/// still restorable through the Undo.
|
||||
pub(crate) fn run_bulk_delete(ui: &Rc<Ui>, entries: Vec<DirEntry>) {
|
||||
if !*ui.mounted.borrow() {
|
||||
toast_error(
|
||||
ui,
|
||||
"Couldn't move to Trash",
|
||||
"Proton Drive isn't connected.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
let socket = ui.dirs.control_socket();
|
||||
// A one-entry batch is still a batch, but it is worth naming in the toast:
|
||||
// "Moved “notes.txt” to Trash" tells the user which file far better than a
|
||||
// count of one does.
|
||||
let single = (entries.len() == 1).then(|| entries[0].name.clone());
|
||||
let paths: Vec<(String, String)> = entries
|
||||
.iter()
|
||||
.map(|e| (entry_rel(ui, e), e.uid.clone()))
|
||||
.collect();
|
||||
ui.busy_begin();
|
||||
clear_selection(ui);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let mut trashed: Vec<String> = Vec::new();
|
||||
let mut failure: Option<String> = None;
|
||||
for (path, uid) in paths {
|
||||
let rx = spawn_request(socket.clone(), Request::Delete { path });
|
||||
match rx.recv().await {
|
||||
Ok(Ok(Response::Ok { .. })) => trashed.push(uid),
|
||||
Ok(Ok(Response::Error { message, .. })) => {
|
||||
failure.get_or_insert(message);
|
||||
}
|
||||
_ => {
|
||||
failure.get_or_insert_with(|| "The mount service didn't respond.".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ui.busy_end();
|
||||
reload_listing(&ui);
|
||||
match (trashed.len(), failure) {
|
||||
(0, Some(message)) => toast_error(&ui, "Couldn't move to Trash", &message),
|
||||
(0, None) => {}
|
||||
(n, failure) => {
|
||||
let message = match (failure, &single) {
|
||||
(Some(_), _) => format!("Moved {n} items to Trash — some couldn't be moved"),
|
||||
(None, Some(name)) => format!("Moved “{name}” to Trash"),
|
||||
(None, None) => format!("Moved {n} items to Trash"),
|
||||
};
|
||||
toast_action(&ui, &message, "Undo", move |ui| {
|
||||
restore_uids(ui, trashed.clone(), n);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Put trashed nodes back where they came from — the Undo behind a trash toast.
|
||||
pub(crate) fn restore_uids(ui: &Rc<Ui>, uids: Vec<String>, count: usize) {
|
||||
run_mutation(
|
||||
ui,
|
||||
Request::Restore { uids },
|
||||
if count == 1 {
|
||||
"Restored from Trash".to_string()
|
||||
} else {
|
||||
format!("Restored {count} items from Trash")
|
||||
},
|
||||
"Couldn't restore from Trash",
|
||||
);
|
||||
}
|
||||
|
||||
/// Pin or unpin every selected file, one request at a time.
|
||||
pub(crate) fn run_bulk_pin(ui: &Rc<Ui>, entries: Vec<DirEntry>, pin: bool) {
|
||||
let socket = ui.dirs.control_socket();
|
||||
// Only the entries that need changing: re-pinning an already-pinned file is
|
||||
// a wasted round-trip, and it would inflate the count the toast reports.
|
||||
let paths: Vec<String> = entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.pinned != pin)
|
||||
.map(|e| entry_rel(ui, e))
|
||||
.collect();
|
||||
if paths.is_empty() {
|
||||
return;
|
||||
}
|
||||
ui.busy_begin();
|
||||
clear_selection(ui);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let mut done = 0usize;
|
||||
let mut failure: Option<String> = None;
|
||||
for path in paths {
|
||||
let req = if pin {
|
||||
Request::Pin { path }
|
||||
} else {
|
||||
Request::Unpin { path }
|
||||
};
|
||||
let rx = spawn_request(socket.clone(), req);
|
||||
match rx.recv().await {
|
||||
Ok(Ok(Response::Ok { .. })) => done += 1,
|
||||
Ok(Ok(Response::Error { message, .. })) => {
|
||||
failure.get_or_insert(message);
|
||||
}
|
||||
_ => {
|
||||
failure.get_or_insert_with(|| "The mount service didn't respond.".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ui.busy_end();
|
||||
load_browser(&ui);
|
||||
match (done, failure) {
|
||||
(0, Some(message)) => toast_error(&ui, "Couldn't change offline state", &message),
|
||||
(0, None) => {}
|
||||
(n, _) => toast(
|
||||
&ui,
|
||||
&if pin {
|
||||
format!("{n} files are now available offline")
|
||||
} else {
|
||||
format!("{n} files are no longer kept offline")
|
||||
},
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Wire the bulk-action bar and the empty-folder state's buttons.
|
||||
pub(crate) fn wire_bulk(
|
||||
ui: &Rc<Ui>,
|
||||
bulk_clear: >k4::Button,
|
||||
empty_upload: >k4::Button,
|
||||
empty_new_folder: >k4::Button,
|
||||
) {
|
||||
let ui_trash = ui.clone();
|
||||
ui.browser
|
||||
.bulk_trash
|
||||
.connect_clicked(move |_| prompt_delete_many(&ui_trash, selected_entries(&ui_trash)));
|
||||
let ui_pin = ui.clone();
|
||||
ui.browser
|
||||
.bulk_pin
|
||||
.connect_clicked(move |_| run_bulk_pin(&ui_pin, selected_entries(&ui_pin), true));
|
||||
let ui_unpin = ui.clone();
|
||||
ui.browser
|
||||
.bulk_unpin
|
||||
.connect_clicked(move |_| run_bulk_pin(&ui_unpin, selected_entries(&ui_unpin), false));
|
||||
let ui_clear = ui.clone();
|
||||
bulk_clear.connect_clicked(move |_| {
|
||||
clear_selection(&ui_clear);
|
||||
sync_bulk_bar(&ui_clear);
|
||||
});
|
||||
|
||||
// Switching views switches which selection is live, so the bar has to be
|
||||
// re-derived rather than left showing the other view's count.
|
||||
let ui_view = ui.clone();
|
||||
ui.browser
|
||||
.views
|
||||
.connect_visible_child_name_notify(move |_| sync_bulk_bar(&ui_view));
|
||||
|
||||
let ui_upload = ui.clone();
|
||||
empty_upload.connect_clicked(move |_| prompt_upload(&ui_upload));
|
||||
let ui_new = ui.clone();
|
||||
empty_new_folder.connect_clicked(move |_| prompt_new_folder(&ui_new));
|
||||
}
|
||||
|
||||
/// Swap the Files content area back to the grid/list views.
|
||||
pub(crate) fn browser_views(ui: &Rc<Ui>) {
|
||||
ui.browser.content.set_visible_child_name("views");
|
||||
@@ -474,7 +823,14 @@ pub(crate) fn attach_context_menu(ui: &Rc<Ui>, item: >k4::ListItem, anchor: &g
|
||||
gesture.connect_pressed(move |_, _, x, y| {
|
||||
if let Some(obj) = item.item().and_downcast::<BoxedAnyObject>() {
|
||||
let entry = obj.borrow::<DirEntry>().clone();
|
||||
show_context_menu(&ui, &entry, &target, x, y);
|
||||
// Right-clicking a row that is part of a multi-selection acts on the
|
||||
// batch; right-clicking outside one acts on the row, as before.
|
||||
let selected = selected_entries(&ui);
|
||||
if selected.len() > 1 && selected.iter().any(|e| e.uid == entry.uid) {
|
||||
show_bulk_context_menu(&ui, selected, &target, x, y);
|
||||
} else {
|
||||
show_context_menu(&ui, &entry, &target, x, y);
|
||||
}
|
||||
}
|
||||
});
|
||||
anchor.add_controller(gesture);
|
||||
@@ -588,6 +944,75 @@ pub(crate) fn show_context_menu(ui: &Rc<Ui>, entry: &DirEntry, anchor: >k4::Bo
|
||||
popover.popup();
|
||||
}
|
||||
|
||||
/// The context menu for a multi-selection: the batch-capable actions only.
|
||||
/// Rename, Move, Share and the revision history all need a single subject, so
|
||||
/// they are simply absent here rather than offered and then refused.
|
||||
pub(crate) fn show_bulk_context_menu(
|
||||
ui: &Rc<Ui>,
|
||||
entries: Vec<DirEntry>,
|
||||
anchor: >k4::Box,
|
||||
x: f64,
|
||||
y: f64,
|
||||
) {
|
||||
let menu = gtk4::Box::new(gtk4::Orientation::Vertical, 0);
|
||||
let popover = gtk4::Popover::builder()
|
||||
.has_arrow(false)
|
||||
.position(gtk4::PositionType::Bottom)
|
||||
.pointing_to(>k4::gdk::Rectangle::new(x as i32, y as i32, 1, 1))
|
||||
.child(&menu)
|
||||
.build();
|
||||
popover.set_parent(anchor);
|
||||
popover.connect_closed(|p| p.unparent());
|
||||
|
||||
let header = gtk4::Label::builder()
|
||||
.label(format!("{} selected", entries.len()))
|
||||
.halign(gtk4::Align::Start)
|
||||
.build();
|
||||
header.add_css_class("dim-label");
|
||||
header.add_css_class("caption");
|
||||
header.set_margin_top(6);
|
||||
header.set_margin_bottom(4);
|
||||
header.set_margin_start(10);
|
||||
menu.append(&header);
|
||||
|
||||
let files_only = entries.iter().all(|e| !e.is_dir);
|
||||
if files_only && entries.iter().any(|e| !e.pinned) {
|
||||
let pin = menu_item("Keep offline", "starred-symbolic");
|
||||
let ui_pin = ui.clone();
|
||||
let batch = entries.clone();
|
||||
let pop = popover.clone();
|
||||
pin.connect_clicked(move |_| {
|
||||
pop.popdown();
|
||||
run_bulk_pin(&ui_pin, batch.clone(), true);
|
||||
});
|
||||
menu.append(&pin);
|
||||
}
|
||||
if files_only && entries.iter().any(|e| e.pinned) {
|
||||
let unpin = menu_item("Remove offline copy", "non-starred-symbolic");
|
||||
let ui_unpin = ui.clone();
|
||||
let batch = entries.clone();
|
||||
let pop = popover.clone();
|
||||
unpin.connect_clicked(move |_| {
|
||||
pop.popdown();
|
||||
run_bulk_pin(&ui_unpin, batch.clone(), false);
|
||||
});
|
||||
menu.append(&unpin);
|
||||
}
|
||||
|
||||
menu.append(>k4::Separator::new(gtk4::Orientation::Horizontal));
|
||||
|
||||
let trash = menu_item("Move to Trash", "user-trash-symbolic");
|
||||
let ui_tr = ui.clone();
|
||||
let pop = popover.clone();
|
||||
trash.connect_clicked(move |_| {
|
||||
pop.popdown();
|
||||
prompt_delete_many(&ui_tr, entries.clone());
|
||||
});
|
||||
menu.append(&trash);
|
||||
|
||||
popover.popup();
|
||||
}
|
||||
|
||||
/// A left-aligned, flat icon+label button for the context menu.
|
||||
pub(crate) fn menu_item(label: &str, icon: &str) -> gtk4::Button {
|
||||
let row = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
@@ -976,8 +1401,6 @@ pub(crate) fn prompt_move(ui: &Rc<Ui>, entry: &DirEntry) {
|
||||
/// Confirm and move the entry to Trash through the daemon.
|
||||
pub(crate) fn prompt_delete(ui: &Rc<Ui>, entry: &DirEntry) {
|
||||
let win = ui_window(ui);
|
||||
let rel = entry_rel(ui, entry);
|
||||
let name = entry.name.clone();
|
||||
let dialog = adw::AlertDialog::builder()
|
||||
.heading("Move to Trash")
|
||||
.body(format!("Move “{}” to Trash?", entry.name))
|
||||
@@ -988,15 +1411,13 @@ pub(crate) fn prompt_delete(ui: &Rc<Ui>, entry: &DirEntry) {
|
||||
dialog.set_default_response(Some("cancel"));
|
||||
dialog.set_close_response("cancel");
|
||||
|
||||
// Routed through the batch path so a single trash gets the same Undo: one
|
||||
// mis-click on one file is no less worth taking back than fifty.
|
||||
let entry = entry.clone();
|
||||
let ui = ui.clone();
|
||||
dialog.connect_response(None, move |_, resp| {
|
||||
if resp == "trash" {
|
||||
run_mutation(
|
||||
&ui,
|
||||
Request::Delete { path: rel.clone() },
|
||||
format!("Moved “{name}” to Trash"),
|
||||
"Couldn't move to Trash",
|
||||
);
|
||||
run_bulk_delete(&ui, vec![entry.clone()]);
|
||||
}
|
||||
});
|
||||
dialog.present(win.as_ref());
|
||||
@@ -1346,6 +1767,7 @@ pub(crate) fn repaint_browser(ui: &Rc<Ui>, entries: &[DirEntry]) {
|
||||
"Upload a file or create a folder to get started.",
|
||||
false,
|
||||
);
|
||||
ui.browser.empty_actions.set_visible(true);
|
||||
return;
|
||||
}
|
||||
browser_views(ui);
|
||||
|
||||
@@ -9,5 +9,6 @@ pub(crate) mod photos;
|
||||
pub(crate) mod shared;
|
||||
pub(crate) mod shared_by_me;
|
||||
pub(crate) mod status;
|
||||
pub(crate) mod takeout;
|
||||
pub(crate) mod trash;
|
||||
pub(crate) mod verify;
|
||||
@@ -19,6 +19,12 @@ pub(crate) struct GalleryState {
|
||||
pub(crate) retry: gtk4::Button,
|
||||
pub(crate) more: gtk4::Button,
|
||||
pub(crate) upload: gtk4::Button,
|
||||
/// Opens the Google Photos Takeout import chooser.
|
||||
pub(crate) import: gtk4::Button,
|
||||
/// Upload / Import offered on the *empty timeline* status page, so a fresh
|
||||
/// account is a place to start rather than a dead end. Hidden on every other
|
||||
/// status: a load error is not the moment to offer an upload.
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
/// "Photos", or the album's name while one is open.
|
||||
pub(crate) title: gtk4::Label,
|
||||
/// "1,204 photos" under the page title.
|
||||
@@ -177,6 +183,10 @@ pub(crate) struct GalleryWidgets {
|
||||
pub(crate) scroll: gtk4::ScrolledWindow,
|
||||
pub(crate) retry: gtk4::Button,
|
||||
pub(crate) upload: gtk4::Button,
|
||||
pub(crate) import: gtk4::Button,
|
||||
pub(crate) empty_actions: gtk4::Box,
|
||||
pub(crate) empty_upload: gtk4::Button,
|
||||
pub(crate) empty_import: gtk4::Button,
|
||||
pub(crate) refresh: gtk4::Button,
|
||||
/// The Albums grid, its own status page and the stack between them, plus the
|
||||
/// Photos/Albums switcher and the back button out of an album.
|
||||
@@ -227,10 +237,29 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) {
|
||||
retry.add_css_class("suggested-action");
|
||||
retry.set_visible(false);
|
||||
|
||||
// The empty-timeline state's two ways forward. `StatusPage` takes one child,
|
||||
// so Retry and these share a box and each is shown only for its own state.
|
||||
let empty_upload = gtk4::Button::builder().label("Upload photos").build();
|
||||
empty_upload.add_css_class("pill");
|
||||
empty_upload.add_css_class("suggested-action");
|
||||
let empty_import = gtk4::Button::builder()
|
||||
.label("Import from Google Photos")
|
||||
.build();
|
||||
empty_import.add_css_class("pill");
|
||||
let empty_actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
empty_actions.set_halign(gtk4::Align::Center);
|
||||
empty_actions.set_visible(false);
|
||||
empty_actions.append(&empty_upload);
|
||||
empty_actions.append(&empty_import);
|
||||
|
||||
let status_child = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
status_child.append(&retry);
|
||||
status_child.append(&empty_actions);
|
||||
|
||||
let status = adw::StatusPage::builder()
|
||||
.icon_name("image-x-generic-symbolic")
|
||||
.vexpand(true)
|
||||
.child(&retry)
|
||||
.child(&status_child)
|
||||
.build();
|
||||
status.add_css_class("compact");
|
||||
|
||||
@@ -288,12 +317,23 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) {
|
||||
.build();
|
||||
upload.add_css_class("pill");
|
||||
upload.add_css_class("suggested-action");
|
||||
|
||||
// Importing an export is a rarer, heavier action than adding one photo, so
|
||||
// it sits beside Upload as a plain button rather than a second accent one.
|
||||
let import = gtk4::Button::builder()
|
||||
.icon_name("folder-download-symbolic")
|
||||
.tooltip_text("Import a Google Photos Takeout export")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
import.add_css_class("flat");
|
||||
import.add_css_class("circular");
|
||||
let refresh = refresh_button();
|
||||
|
||||
let header_box = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
header_box.append(&back);
|
||||
header_box.append(&titles);
|
||||
header_box.append(&refresh);
|
||||
header_box.append(&import);
|
||||
header_box.append(&upload);
|
||||
|
||||
// All / Photos / Videos / Raw filter. Linked toggles acting as one segmented
|
||||
@@ -428,6 +468,10 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) {
|
||||
scroll,
|
||||
retry,
|
||||
upload,
|
||||
import,
|
||||
empty_actions,
|
||||
empty_upload,
|
||||
empty_import,
|
||||
refresh,
|
||||
tabs,
|
||||
favorites_btn,
|
||||
@@ -554,6 +598,15 @@ pub(crate) fn thousands(n: usize) -> String {
|
||||
/// Wire the gallery: install the section factory, the zoom gestures, the pager
|
||||
/// and the upload button. Activating a thumbnail downloads the photo and opens it
|
||||
/// in the in-app lightbox.
|
||||
/// Wire the empty-timeline buttons. Upload re-fires the header's own Upload
|
||||
/// rather than duplicating its file-chooser, so the two can't drift apart.
|
||||
pub(crate) fn wire_gallery_empty(ui: &Rc<Ui>, upload: >k4::Button, import: >k4::Button) {
|
||||
let ui_upload = ui.clone();
|
||||
upload.connect_clicked(move |_| ui_upload.gallery.upload.emit_clicked());
|
||||
let ui_import = ui.clone();
|
||||
import.connect_clicked(move |_| ui_import.stack.set_visible_child_name("takeout"));
|
||||
}
|
||||
|
||||
pub(crate) fn wire_gallery(ui: &Rc<Ui>, list: >k4::ListView, scroll: >k4::ScrolledWindow) {
|
||||
let factory = gtk4::SignalListItemFactory::new();
|
||||
factory.connect_setup(|_, item| {
|
||||
@@ -729,6 +782,14 @@ pub(crate) fn wire_gallery(ui: &Rc<Ui>, list: >k4::ListView, scroll: >k4::Sc
|
||||
load_gallery(&ui_dates, false);
|
||||
});
|
||||
|
||||
// The import is a staged, hours-long migration, so the button hands over to
|
||||
// the Import page rather than opening a file chooser here: the archives need
|
||||
// reviewing before anything is sent, and the run needs somewhere to report.
|
||||
let ui_import = ui.clone();
|
||||
ui.gallery
|
||||
.import
|
||||
.connect_clicked(move |_| ui_import.stack.set_visible_child_name("takeout"));
|
||||
|
||||
let ui_upload = ui.clone();
|
||||
ui.gallery.upload.connect_clicked(move |_| {
|
||||
let dialog = gtk4::FileDialog::builder()
|
||||
@@ -1460,6 +1521,12 @@ pub(crate) fn load_gallery(ui: &Rc<Ui>, append: bool) {
|
||||
)
|
||||
};
|
||||
gallery_status(&ui, "image-x-generic-symbolic", title, description, false);
|
||||
// Only the whole-timeline empty state: an empty *album* is
|
||||
// filled by adding existing photos to it, not by uploading
|
||||
// into it, so the buttons would point the wrong way.
|
||||
ui.gallery
|
||||
.empty_actions
|
||||
.set_visible(ui.gallery.album.borrow().is_none());
|
||||
return;
|
||||
}
|
||||
ui.gallery.content.set_visible_child_name("timeline");
|
||||
@@ -1502,6 +1569,9 @@ pub(crate) fn gallery_status(ui: &Rc<Ui>, icon: &str, title: &str, description:
|
||||
ui.gallery.status.set_title(title);
|
||||
ui.gallery.status.set_description(Some(description));
|
||||
ui.gallery.retry.set_visible(retry);
|
||||
// Only the empty timeline offers a way to fill it; every other status turns
|
||||
// those buttons back off.
|
||||
ui.gallery.empty_actions.set_visible(false);
|
||||
ui.gallery.more.set_visible(false);
|
||||
ui.gallery.content.set_visible_child_name("status");
|
||||
}
|
||||
|
||||
@@ -37,7 +37,16 @@ pub(crate) struct StatusState {
|
||||
/// Set while a settings widget is being populated programmatically, so its
|
||||
/// change handler skips the IPC/systemd side effect.
|
||||
pub(crate) settings_suppress: Cell<bool>,
|
||||
/// Pending debounce for the cache-budget editor. Each `+` click is a value
|
||||
/// change, and firing a `SetCacheBudget` (and a toast) per click means a
|
||||
/// stack of toasts and a series of caps the user never asked to apply — so
|
||||
/// only the value they settle on is sent.
|
||||
pub(crate) budget_source: RefCell<Option<glib::SourceId>>,
|
||||
pub(crate) pins_group: adw::PreferencesGroup,
|
||||
/// Whether the pin list is showing every pin or only the first
|
||||
/// [`PINS_COLLAPSED`]. A long pin list would otherwise push everything below
|
||||
/// it — including the Developer group — off the end of the page.
|
||||
pub(crate) pins_expanded: Cell<bool>,
|
||||
/// Rows currently shown under [`Self::pins_group`], retained so a refresh can
|
||||
/// diff against them and only rebuild when the pin set actually changes.
|
||||
pub(crate) pin_rows: RefCell<Vec<PinRow>>,
|
||||
@@ -104,6 +113,8 @@ pub(crate) struct MainWidgets {
|
||||
/// which is where the folder is changed.
|
||||
pub(crate) mountpoint_row: adw::ActionRow,
|
||||
pub(crate) mountpoint_button: gtk4::Button,
|
||||
/// Opens the Google Photos Takeout import page.
|
||||
pub(crate) import_row: adw::ActionRow,
|
||||
}
|
||||
|
||||
/// The main (logged-in) page: a libadwaita settings surface — account header,
|
||||
@@ -233,6 +244,22 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) {
|
||||
mountpoint_row.add_suffix(&mountpoint_button);
|
||||
system_group.add(&mountpoint_row);
|
||||
|
||||
// Import: a one-off migration rather than a setting, so it is a doorway to
|
||||
// its own page rather than controls inlined here — staging a set of archives
|
||||
// and watching an hours-long upload needs the room.
|
||||
let import_group = adw::PreferencesGroup::builder()
|
||||
.title("Import")
|
||||
.description("Bring a photo library from another service into Proton Photos.")
|
||||
.build();
|
||||
let import_row = adw::ActionRow::builder()
|
||||
.title("Import from Google Photos")
|
||||
.subtitle("Add a Google Takeout export to your timeline.")
|
||||
.activatable(true)
|
||||
.build();
|
||||
import_row.add_prefix(>k4::Image::from_icon_name("folder-download-symbolic"));
|
||||
import_row.add_suffix(>k4::Image::from_icon_name("go-next-symbolic"));
|
||||
import_group.add(&import_row);
|
||||
|
||||
// Pins group: filled in by refresh.
|
||||
let pins_group = adw::PreferencesGroup::builder()
|
||||
.title("Pinned files")
|
||||
@@ -265,6 +292,7 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) {
|
||||
inner.append(&transfers_group);
|
||||
inner.append(&storage_group);
|
||||
inner.append(&system_group);
|
||||
inner.append(&import_group);
|
||||
inner.append(&pins_group);
|
||||
inner.append(&dev_group);
|
||||
|
||||
@@ -292,10 +320,20 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) {
|
||||
purge_button,
|
||||
mountpoint_row,
|
||||
mountpoint_button,
|
||||
import_row,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// How many pins the Settings list shows before collapsing the rest behind a
|
||||
/// "Show all" row. Enough to recognise the list at a glance; short enough that
|
||||
/// the groups below it stay reachable.
|
||||
pub(crate) const PINS_COLLAPSED: usize = 6;
|
||||
|
||||
/// How long the cache-budget editor waits after the last change before applying
|
||||
/// it, so a run of `+` clicks is one request rather than one per click.
|
||||
pub(crate) const BUDGET_DEBOUNCE: Duration = Duration::from_millis(600);
|
||||
|
||||
/// Bytes per GiB, for the cache-budget editor's unit conversion.
|
||||
pub(crate) const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
|
||||
|
||||
@@ -308,6 +346,7 @@ pub(crate) fn wire_settings(
|
||||
ui: &Rc<Ui>,
|
||||
purge_button: >k4::Button,
|
||||
mountpoint_button: >k4::Button,
|
||||
import_row: &adw::ActionRow,
|
||||
) {
|
||||
let config = ui.dirs.load_config();
|
||||
|
||||
@@ -329,13 +368,22 @@ pub(crate) fn wire_settings(
|
||||
if ui_budget.status.settings_suppress.get() {
|
||||
return;
|
||||
}
|
||||
// Replace any pending apply, so only the value the user stops on is sent.
|
||||
if let Some(src) = ui_budget.status.budget_source.borrow_mut().take() {
|
||||
src.remove();
|
||||
}
|
||||
let bytes = (row.value() * GIB).round() as u64;
|
||||
settings_request(
|
||||
&ui_budget,
|
||||
Request::SetCacheBudget { bytes },
|
||||
"Cache budget updated",
|
||||
"Couldn't set cache budget",
|
||||
);
|
||||
let ui_fire = ui_budget.clone();
|
||||
let src = glib::timeout_add_local_once(BUDGET_DEBOUNCE, move || {
|
||||
ui_fire.status.budget_source.borrow_mut().take();
|
||||
settings_request(
|
||||
&ui_fire,
|
||||
Request::SetCacheBudget { bytes },
|
||||
"Cache budget updated",
|
||||
"Couldn't set cache budget",
|
||||
);
|
||||
});
|
||||
*ui_budget.status.budget_source.borrow_mut() = Some(src);
|
||||
});
|
||||
|
||||
// Purge: confirm, then drop all unpinned cached content via the daemon.
|
||||
@@ -382,6 +430,10 @@ pub(crate) fn wire_settings(
|
||||
// local path; this button is the way there.
|
||||
let ui_mp = ui.clone();
|
||||
mountpoint_button.connect_clicked(move |_| ui_mp.stack.set_visible_child_name("locations"));
|
||||
|
||||
// Import: a doorway, not a setting — everything about it lives on its page.
|
||||
let ui_import = ui.clone();
|
||||
import_row.connect_activated(move |_| ui_import.stack.set_visible_child_name("takeout"));
|
||||
}
|
||||
|
||||
/// Run a settings control-socket round-trip (budget / purge) on a worker thread,
|
||||
@@ -513,6 +565,12 @@ pub(crate) fn refresh(ui: &Rc<Ui>) {
|
||||
refresh_transfers(ui);
|
||||
// Both of these pages show work as it happens, so they follow the tick while
|
||||
// they are on screen. Every other page loads on navigation only.
|
||||
// A Takeout import outlives the page that started it, so it is polled
|
||||
// wherever the user has navigated to — otherwise leaving the page means
|
||||
// never being told it finished.
|
||||
if ui.stack.visible_child_name().as_deref() == Some("takeout") || ui.takeout.running.get() {
|
||||
refresh_takeout(ui);
|
||||
}
|
||||
match ui.stack.visible_child_name().as_deref() {
|
||||
Some("main") => refresh_quota(ui),
|
||||
Some("locations") => refresh_locations(ui),
|
||||
@@ -653,6 +711,10 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
// count falling to zero is what "the sync finished" looks like from here.
|
||||
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
||||
// starts its upload job mid-flight, which is not a thing finishing.
|
||||
// The Import page reads its progress off the same snapshot rather than
|
||||
// polling the daemon a second time for numbers already on the wire.
|
||||
takeout_progress(ui, jobs);
|
||||
|
||||
let previous = ui.status.active_transfers.replace(items.len());
|
||||
if items.is_empty() && previous > 0 {
|
||||
let files = if previous == 1 {
|
||||
@@ -808,11 +870,14 @@ pub(crate) fn refresh_status(ui: &Rc<Ui>) {
|
||||
(used as f64 / budget as f64).min(1.0)
|
||||
};
|
||||
ui.status.cache_bar.set_fraction(fraction);
|
||||
ui.status.cache_label.set_text(&format!(
|
||||
"{} of {} used",
|
||||
human_bytes(used),
|
||||
human_bytes(budget)
|
||||
));
|
||||
// A 0 budget means *unlimited*, not a zero-byte cap — "of 0 B
|
||||
// used" reads as a broken read-out, and there is no fraction to
|
||||
// draw against no limit.
|
||||
ui.status.cache_label.set_text(&if budget == 0 {
|
||||
format!("{} cached — no limit set", human_bytes(used))
|
||||
} else {
|
||||
format!("{} of {} used", human_bytes(used), human_bytes(budget))
|
||||
});
|
||||
repaint_pins(&ui, &pins, true);
|
||||
}
|
||||
// Daemon unreachable (still starting, or down): report not-mounted and
|
||||
@@ -865,7 +930,15 @@ pub(crate) fn repaint_pins(ui: &Rc<Ui>, pins: &[pdfs_core::cache::Pin], mounted:
|
||||
return;
|
||||
}
|
||||
|
||||
for pin in pins {
|
||||
// Only the first page of pins is rendered; the rest sit behind the row
|
||||
// below, so a device with hundreds of pins doesn't bury the groups after it.
|
||||
let expanded = ui.status.pins_expanded.get();
|
||||
let shown = if expanded {
|
||||
pins.len()
|
||||
} else {
|
||||
pins.len().min(PINS_COLLAPSED)
|
||||
};
|
||||
for pin in &pins[..shown] {
|
||||
let name = Path::new(&pin.path)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
@@ -903,4 +976,37 @@ pub(crate) fn repaint_pins(ui: &Rc<Ui>, pins: &[pdfs_core::cache::Pin], mounted:
|
||||
unpin: Some(unpin),
|
||||
});
|
||||
}
|
||||
|
||||
if pins.len() > PINS_COLLAPSED {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(if expanded {
|
||||
"Show fewer".to_string()
|
||||
} else {
|
||||
format!("Show all {} pinned files", pins.len())
|
||||
})
|
||||
.activatable(true)
|
||||
.build();
|
||||
row.add_suffix(>k4::Image::from_icon_name(if expanded {
|
||||
"go-up-symbolic"
|
||||
} else {
|
||||
"go-down-symbolic"
|
||||
}));
|
||||
let ui_more = ui.clone();
|
||||
row.connect_activated(move |_| {
|
||||
ui_more
|
||||
.status
|
||||
.pins_expanded
|
||||
.set(!ui_more.status.pins_expanded.get());
|
||||
// The diff guard compares pin *paths*; the same paths render
|
||||
// differently now, so the baseline has to be dropped for the next
|
||||
// repaint to actually rebuild.
|
||||
*ui_more.status.pins_state.borrow_mut() = None;
|
||||
refresh(&ui_more);
|
||||
});
|
||||
ui.status.pins_group.add(&row);
|
||||
ui.status
|
||||
.pin_rows
|
||||
.borrow_mut()
|
||||
.push(PinRow { row, unpin: None });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
//! The Import page: bringing a Google Photos **Takeout** export into Proton
|
||||
//! Photos.
|
||||
//!
|
||||
//! Reached from Settings → *Import from Google Photos* (and from the Gallery's
|
||||
//! import button), because it is a one-off migration rather than something the
|
||||
//! photo timeline does every day. It gets a page of its own instead of a dialog
|
||||
//! for two reasons: an export is a *set* of archives that has to be staged and
|
||||
//! checked before anything is sent, and the import that follows runs for hours,
|
||||
//! so its progress needs somewhere to live that the user can navigate away from
|
||||
//! and come back to.
|
||||
//!
|
||||
//! Archives arrive either by drag-and-drop onto the page or through the file
|
||||
//! chooser; both funnel into [`add_archives`], which is where the `.zip` filter
|
||||
//! and de-duplication live. Nothing is sent until the user presses Scan or
|
||||
//! Import — staging is local, so a mis-drop costs nothing.
|
||||
//!
|
||||
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
||||
//! stages paths, launches, polls [`Request::ImportStatus`] for the final report,
|
||||
//! and reads live progress off the import's job in the transfer snapshot the
|
||||
//! Settings tick already fetches (see [`takeout_progress`]).
|
||||
|
||||
use crate::*;
|
||||
|
||||
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
||||
/// Matching on it is how this page borrows the progress the Settings tick is
|
||||
/// already polling instead of adding a second poll of its own.
|
||||
const IMPORT_JOB: &str = "Importing Google Photos";
|
||||
|
||||
/// What the archive list says before anything is staged. `repaint_archives`
|
||||
/// swaps in the count and total once there is something to report, and swaps
|
||||
/// this back when the list is emptied again.
|
||||
/// Where the staged set is remembered between runs, under the app's state
|
||||
/// directory. Staging an export means picking dozens of files out of a file
|
||||
/// chooser; losing that to a window close (or a crash mid-import) and having to
|
||||
/// redo it is the kind of small cruelty that makes a migration not get finished.
|
||||
fn staged_path(ui: &Rc<Ui>) -> PathBuf {
|
||||
ui.dirs.state_dir().join("takeout-staged.json")
|
||||
}
|
||||
|
||||
/// Read the remembered set, dropping any archive that has since been moved or
|
||||
/// deleted — a stale path would only fail at launch, and silently is worse.
|
||||
fn load_staged(ui: &Rc<Ui>) -> Vec<PathBuf> {
|
||||
let Ok(text) = std::fs::read_to_string(staged_path(ui)) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let paths: Vec<String> = serde_json::from_str(&text).unwrap_or_default();
|
||||
paths
|
||||
.into_iter()
|
||||
.map(PathBuf::from)
|
||||
.filter(|p| p.is_file())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Remember the staged set. Best-effort: failing to write it costs the user a
|
||||
/// re-pick next time, which is not worth interrupting them over now.
|
||||
fn save_staged(ui: &Rc<Ui>) {
|
||||
let paths: Vec<String> = ui
|
||||
.takeout
|
||||
.archives
|
||||
.borrow()
|
||||
.iter()
|
||||
.map(|p| p.display().to_string())
|
||||
.collect();
|
||||
let path = staged_path(ui);
|
||||
let written = serde_json::to_string(&paths)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|text| std::fs::write(&path, text).map_err(|e| e.to_string()));
|
||||
if let Err(e) = written {
|
||||
tracing::warn!("cannot remember staged Takeout archives: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
const EMPTY_LIST_DESCRIPTION: &str = "Photos already in Proton Photos are matched by name and content and skipped, so \
|
||||
re-running an interrupted import is safe.";
|
||||
|
||||
pub(crate) struct TakeoutState {
|
||||
/// Archives staged for this import, in the order they were added. Sent as
|
||||
/// one set: Google splits an export across numbered zips and routinely puts
|
||||
/// a photo in one part and its metadata sidecar in another, so a per-file
|
||||
/// import would lose album titles and capture times.
|
||||
pub(crate) archives: RefCell<Vec<PathBuf>>,
|
||||
/// The staged-archive list and the rows currently rendered in it.
|
||||
pub(crate) list_group: adw::PreferencesGroup,
|
||||
pub(crate) rows: RefCell<Vec<adw::PreferencesRow>>,
|
||||
/// The drop zone, whose `dropzone-active` class follows the drag.
|
||||
pub(crate) dropzone: gtk4::Box,
|
||||
pub(crate) scan_button: gtk4::Button,
|
||||
pub(crate) import_button: gtk4::Button,
|
||||
pub(crate) cancel_button: gtk4::Button,
|
||||
pub(crate) clear_button: gtk4::Button,
|
||||
/// Progress group, hidden unless an import is running.
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
/// Report group for the last finished run, hidden until there is one.
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
||||
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
||||
/// can't stack worker threads on the 2s tick.
|
||||
pub(crate) inflight: Cell<bool>,
|
||||
/// Whether the daemon reported an import running on the last poll.
|
||||
pub(crate) running: Cell<bool>,
|
||||
/// Whether the run in flight was launched as a dry run, so the report can
|
||||
/// say "would upload" rather than claiming photos were sent. The daemon's
|
||||
/// summary carries no such flag — it is the same shape either way.
|
||||
pub(crate) dry_run: Cell<bool>,
|
||||
}
|
||||
|
||||
/// Widgets the Import page hands back for wiring.
|
||||
pub(crate) struct TakeoutWidgets {
|
||||
pub(crate) list_group: adw::PreferencesGroup,
|
||||
pub(crate) dropzone: gtk4::Box,
|
||||
/// The whole page, which is the drop target: a drag released anywhere on it
|
||||
/// counts, not only inside the dashed rectangle.
|
||||
pub(crate) page: gtk4::Widget,
|
||||
pub(crate) choose_button: gtk4::Button,
|
||||
pub(crate) scan_button: gtk4::Button,
|
||||
pub(crate) import_button: gtk4::Button,
|
||||
pub(crate) cancel_button: gtk4::Button,
|
||||
pub(crate) clear_button: gtk4::Button,
|
||||
pub(crate) back_button: gtk4::Button,
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
}
|
||||
|
||||
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
// Header: a back button out to Settings, since this page is reached from
|
||||
// there and has no sidebar row of its own to navigate back with.
|
||||
let back_button = gtk4::Button::builder()
|
||||
.icon_name("go-previous-symbolic")
|
||||
.tooltip_text("Back to Settings")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
back_button.add_css_class("flat");
|
||||
|
||||
let title = gtk4::Label::builder()
|
||||
.label("Import from Google Photos")
|
||||
.halign(gtk4::Align::Start)
|
||||
.build();
|
||||
title.add_css_class("title-2");
|
||||
let subtitle = gtk4::Label::builder()
|
||||
.label("Add a Google Takeout export to your Proton Photos timeline.")
|
||||
.halign(gtk4::Align::Start)
|
||||
.wrap(true)
|
||||
.xalign(0.0)
|
||||
.build();
|
||||
subtitle.add_css_class("dim-label");
|
||||
|
||||
let titles = gtk4::Box::new(gtk4::Orientation::Vertical, 2);
|
||||
titles.set_hexpand(true);
|
||||
titles.append(&title);
|
||||
titles.append(&subtitle);
|
||||
|
||||
let header = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
header.append(&back_button);
|
||||
header.append(&titles);
|
||||
let header_clamp = adw::Clamp::builder().child(&header).build();
|
||||
|
||||
// Drop zone. It is a hint, not the only way in — the button under it does
|
||||
// the same thing for anyone not dragging, and the drop target covers the
|
||||
// whole page so a drag released just outside the rectangle still lands.
|
||||
let drop_icon = gtk4::Image::from_icon_name("folder-download-symbolic");
|
||||
drop_icon.set_pixel_size(48);
|
||||
drop_icon.add_css_class("dim-label");
|
||||
let drop_title = gtk4::Label::builder()
|
||||
.label("Drop your Takeout .zip files here")
|
||||
.build();
|
||||
drop_title.add_css_class("title-4");
|
||||
let drop_hint = gtk4::Label::builder()
|
||||
.label("Add every part of the export at once — a photo and its metadata often sit in different parts.")
|
||||
.wrap(true)
|
||||
.justify(gtk4::Justification::Center)
|
||||
.max_width_chars(48)
|
||||
.build();
|
||||
drop_hint.add_css_class("dim-label");
|
||||
let choose_button = gtk4::Button::builder()
|
||||
.label("Choose files…")
|
||||
.halign(gtk4::Align::Center)
|
||||
.build();
|
||||
choose_button.add_css_class("pill");
|
||||
|
||||
let dropzone = gtk4::Box::new(gtk4::Orientation::Vertical, 10);
|
||||
dropzone.set_halign(gtk4::Align::Fill);
|
||||
dropzone.add_css_class("dropzone");
|
||||
dropzone.append(&drop_icon);
|
||||
dropzone.append(&drop_title);
|
||||
dropzone.append(&drop_hint);
|
||||
dropzone.append(&choose_button);
|
||||
|
||||
// Staged archives. Filled by `repaint_archives`, which owns the description
|
||||
// too (it becomes the count + total once there is something to count); the
|
||||
// group carries a placeholder row while empty so it never renders as a bare
|
||||
// title.
|
||||
let list_group = adw::PreferencesGroup::builder()
|
||||
.title("Archives to import")
|
||||
.description(EMPTY_LIST_DESCRIPTION)
|
||||
.build();
|
||||
|
||||
let clear_button = gtk4::Button::builder()
|
||||
.label("Clear")
|
||||
.tooltip_text("Remove every staged archive")
|
||||
.build();
|
||||
clear_button.add_css_class("flat");
|
||||
|
||||
// Actions. Scan is a dry run: it reads the archives' directories and reports
|
||||
// what an import *would* do, without uploading a byte. It is offered first
|
||||
// because the import behind it is hours long and hard to take back.
|
||||
let scan_button = gtk4::Button::builder()
|
||||
.label("Scan")
|
||||
.tooltip_text("Check the archives and report what would be imported, without uploading")
|
||||
.build();
|
||||
scan_button.add_css_class("pill");
|
||||
let import_button = gtk4::Button::builder().label("Import").build();
|
||||
import_button.add_css_class("pill");
|
||||
import_button.add_css_class("suggested-action");
|
||||
let cancel_button = gtk4::Button::builder()
|
||||
.label("Stop import")
|
||||
.tooltip_text("Finish the photo on the wire, file what was uploaded, then stop")
|
||||
.build();
|
||||
cancel_button.add_css_class("pill");
|
||||
cancel_button.add_css_class("destructive-action");
|
||||
cancel_button.set_visible(false);
|
||||
|
||||
// Clear sits apart from the launch buttons, on the other end of the row: it
|
||||
// undoes staging, and putting it next to Import invites the wrong click.
|
||||
let actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8);
|
||||
let spacer = gtk4::Box::new(gtk4::Orientation::Horizontal, 0);
|
||||
spacer.set_hexpand(true);
|
||||
actions.append(&clear_button);
|
||||
actions.append(&spacer);
|
||||
actions.append(&cancel_button);
|
||||
actions.append(&scan_button);
|
||||
actions.append(&import_button);
|
||||
|
||||
// Progress: the import's job from the transfer snapshot, so this page shows
|
||||
// the same numbers the Settings Activity list does.
|
||||
let progress_group = adw::PreferencesGroup::builder()
|
||||
.title("Import in progress")
|
||||
.visible(false)
|
||||
.build();
|
||||
let progress_box = gtk4::Box::new(gtk4::Orientation::Vertical, 6);
|
||||
progress_box.set_margin_top(8);
|
||||
progress_box.set_margin_bottom(8);
|
||||
let progress_label = gtk4::Label::builder().halign(gtk4::Align::Start).build();
|
||||
progress_label.add_css_class("dim-label");
|
||||
let progress_bar = gtk4::ProgressBar::new();
|
||||
progress_box.append(&progress_label);
|
||||
progress_box.append(&progress_bar);
|
||||
progress_group.add(
|
||||
&adw::PreferencesRow::builder()
|
||||
.activatable(false)
|
||||
.child(&progress_box)
|
||||
.build(),
|
||||
);
|
||||
|
||||
// Report for the last finished run, kept on screen after it ends: an import
|
||||
// outlives the window being on this page, and "what happened" is the whole
|
||||
// reason to come back to it.
|
||||
let summary_group = adw::PreferencesGroup::builder()
|
||||
.title("Last import")
|
||||
.visible(false)
|
||||
.build();
|
||||
|
||||
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
||||
groups.append(&dropzone);
|
||||
groups.append(&list_group);
|
||||
groups.append(&actions);
|
||||
groups.append(&progress_group);
|
||||
groups.append(&summary_group);
|
||||
let clamp = adw::Clamp::builder()
|
||||
.maximum_size(640)
|
||||
.child(&groups)
|
||||
.build();
|
||||
let scroll = gtk4::ScrolledWindow::builder()
|
||||
.vexpand(true)
|
||||
.hscrollbar_policy(gtk4::PolicyType::Never)
|
||||
.child(&clamp)
|
||||
.build();
|
||||
|
||||
let page = gtk4::Box::new(gtk4::Orientation::Vertical, 12);
|
||||
page.set_margin_top(18);
|
||||
page.set_margin_bottom(18);
|
||||
page.set_margin_start(18);
|
||||
page.set_margin_end(18);
|
||||
page.append(&header_clamp);
|
||||
page.append(&scroll);
|
||||
|
||||
let widgets = TakeoutWidgets {
|
||||
list_group: list_group.clone(),
|
||||
dropzone: dropzone.clone(),
|
||||
page: page.clone().upcast(),
|
||||
choose_button,
|
||||
scan_button: scan_button.clone(),
|
||||
import_button: import_button.clone(),
|
||||
cancel_button: cancel_button.clone(),
|
||||
clear_button,
|
||||
back_button,
|
||||
progress_group: progress_group.clone(),
|
||||
progress_label: progress_label.clone(),
|
||||
progress_bar: progress_bar.clone(),
|
||||
summary_group: summary_group.clone(),
|
||||
};
|
||||
(page.upcast(), widgets)
|
||||
}
|
||||
|
||||
pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
|
||||
// Drag-and-drop. The target sits on the whole page rather than the dashed
|
||||
// rectangle, so a drag released anywhere counts; the rectangle only lights
|
||||
// up to say where the drop is understood.
|
||||
let target = gtk4::DropTarget::new(
|
||||
gtk4::gdk::FileList::static_type(),
|
||||
gtk4::gdk::DragAction::COPY,
|
||||
);
|
||||
let ui_enter = ui.clone();
|
||||
target.connect_enter(move |_, _, _| {
|
||||
ui_enter.takeout.dropzone.add_css_class("dropzone-active");
|
||||
gtk4::gdk::DragAction::COPY
|
||||
});
|
||||
let ui_leave = ui.clone();
|
||||
target.connect_leave(move |_| {
|
||||
ui_leave
|
||||
.takeout
|
||||
.dropzone
|
||||
.remove_css_class("dropzone-active");
|
||||
});
|
||||
let ui_drop = ui.clone();
|
||||
target.connect_drop(move |_, value, _, _| {
|
||||
ui_drop.takeout.dropzone.remove_css_class("dropzone-active");
|
||||
let Ok(files) = value.get::<gtk4::gdk::FileList>() else {
|
||||
return false;
|
||||
};
|
||||
let paths: Vec<PathBuf> = files.files().iter().filter_map(|f| f.path()).collect();
|
||||
add_archives(&ui_drop, paths);
|
||||
true
|
||||
});
|
||||
widgets.page.add_controller(target);
|
||||
|
||||
let ui_choose = ui.clone();
|
||||
widgets.choose_button.connect_clicked(move |_| {
|
||||
// Multi-select on purpose: the whole export goes in one request.
|
||||
let dialog = gtk4::FileDialog::builder()
|
||||
.title("Select your Google Takeout archives")
|
||||
.build();
|
||||
let filter = gtk4::FileFilter::new();
|
||||
filter.set_name(Some("Takeout archives"));
|
||||
filter.add_pattern("*.zip");
|
||||
let filters = gio::ListStore::new::<gtk4::FileFilter>();
|
||||
filters.append(&filter);
|
||||
dialog.set_filters(Some(&filters));
|
||||
|
||||
let ui = ui_choose.clone();
|
||||
dialog.open_multiple(
|
||||
ui_window(&ui_choose).as_ref(),
|
||||
gio::Cancellable::NONE,
|
||||
move |res| {
|
||||
let Ok(files) = res else { return };
|
||||
let paths: Vec<PathBuf> = (0..files.n_items())
|
||||
.filter_map(|index| files.item(index))
|
||||
.filter_map(|object| object.downcast::<gio::File>().ok())
|
||||
.filter_map(|file| file.path())
|
||||
.collect();
|
||||
add_archives(&ui, paths);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
let ui_clear = ui.clone();
|
||||
widgets.clear_button.connect_clicked(move |_| {
|
||||
ui_clear.takeout.archives.borrow_mut().clear();
|
||||
repaint_archives(&ui_clear);
|
||||
});
|
||||
|
||||
let ui_scan = ui.clone();
|
||||
widgets
|
||||
.scan_button
|
||||
.connect_clicked(move |_| start_import(&ui_scan, true));
|
||||
|
||||
// The import itself is confirmed: it uploads a whole photo library, which is
|
||||
// hours of bandwidth and a large change to the account's timeline.
|
||||
let ui_import = ui.clone();
|
||||
widgets.import_button.connect_clicked(move |_| {
|
||||
let count = ui_import.takeout.archives.borrow().len();
|
||||
let ui = ui_import.clone();
|
||||
let dialog = adw::AlertDialog::builder()
|
||||
.heading("Import this export?")
|
||||
.body(format!(
|
||||
"{count} archive(s) will be uploaded to Proton Photos, creating albums as they \
|
||||
appear in the export. This can take hours; you can stop it at any point and \
|
||||
what has been uploaded stays. Photos you already have are skipped.",
|
||||
))
|
||||
.build();
|
||||
dialog.add_response("cancel", "Cancel");
|
||||
dialog.add_response("import", "Import");
|
||||
dialog.set_response_appearance("import", adw::ResponseAppearance::Suggested);
|
||||
dialog.set_default_response(Some("cancel"));
|
||||
dialog.set_close_response("cancel");
|
||||
dialog.connect_response(None, move |_, resp| {
|
||||
if resp == "import" {
|
||||
start_import(&ui, false);
|
||||
}
|
||||
});
|
||||
dialog.present(ui_window(&ui_import).as_ref());
|
||||
});
|
||||
|
||||
let ui_cancel = ui.clone();
|
||||
widgets.cancel_button.connect_clicked(move |_| {
|
||||
let ui = ui_cancel.clone();
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::CancelImport);
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => toast(&ui, "Stopping the import…"),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, "Couldn't stop the import", &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
"Couldn't stop the import",
|
||||
"The mount service didn't respond.",
|
||||
),
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let ui_back = ui.clone();
|
||||
widgets
|
||||
.back_button
|
||||
.connect_clicked(move |_| ui_back.stack.set_visible_child_name("main"));
|
||||
|
||||
// Restore whatever was staged when the window last closed, before the first
|
||||
// paint, so the page comes up as the user left it.
|
||||
*ui.takeout.archives.borrow_mut() = load_staged(ui);
|
||||
repaint_archives(ui);
|
||||
}
|
||||
|
||||
/// Stage `paths`, keeping only `.zip` files and skipping ones already staged.
|
||||
///
|
||||
/// Both the chooser and the drop target land here, so the filter is stated once.
|
||||
/// A dropped folder (an already-extracted export) is rejected rather than walked:
|
||||
/// the scan reads the archives' central directories, and an extracted tree has
|
||||
/// none — silently importing half of it would be worse than saying no.
|
||||
pub(crate) fn add_archives(ui: &Rc<Ui>, paths: Vec<PathBuf>) {
|
||||
let mut added = 0usize;
|
||||
let mut rejected = 0usize;
|
||||
{
|
||||
let mut staged = ui.takeout.archives.borrow_mut();
|
||||
for path in paths {
|
||||
let is_zip = path
|
||||
.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("zip"));
|
||||
if !is_zip || !path.is_file() {
|
||||
rejected += 1;
|
||||
continue;
|
||||
}
|
||||
if staged.contains(&path) {
|
||||
continue;
|
||||
}
|
||||
staged.push(path);
|
||||
added += 1;
|
||||
}
|
||||
// Numbered parts drop in whatever order the file manager hands them
|
||||
// over; sorting keeps the list readable and the request stable.
|
||||
staged.sort();
|
||||
}
|
||||
repaint_archives(ui);
|
||||
if rejected > 0 {
|
||||
toast(
|
||||
ui,
|
||||
"Only Takeout .zip files can be imported — extracted folders aren't supported.",
|
||||
);
|
||||
} else if added > 0 {
|
||||
toast(ui, &format!("Added {added} archive(s)"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild the staged-archive list and re-derive which actions are available.
|
||||
pub(crate) fn repaint_archives(ui: &Rc<Ui>) {
|
||||
for row in ui.takeout.rows.borrow_mut().drain(..) {
|
||||
ui.takeout.list_group.remove(&row);
|
||||
}
|
||||
|
||||
let staged = ui.takeout.archives.borrow().clone();
|
||||
let mut rows: Vec<adw::PreferencesRow> = Vec::new();
|
||||
if staged.is_empty() {
|
||||
ui.takeout
|
||||
.list_group
|
||||
.set_description(Some(EMPTY_LIST_DESCRIPTION));
|
||||
let row = dim_row("No archives added yet.");
|
||||
ui.takeout.list_group.add(&row);
|
||||
rows.push(row.upcast());
|
||||
} else {
|
||||
let mut total = 0u64;
|
||||
for path in &staged {
|
||||
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
|
||||
total += size;
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(
|
||||
path.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| path.display().to_string()),
|
||||
)
|
||||
.subtitle(human_bytes(size))
|
||||
.build();
|
||||
let remove = gtk4::Button::builder()
|
||||
.icon_name("edit-delete-symbolic")
|
||||
.tooltip_text("Remove from this import")
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
remove.add_css_class("flat");
|
||||
let ui_remove = ui.clone();
|
||||
let target = path.clone();
|
||||
remove.connect_clicked(move |_| {
|
||||
ui_remove
|
||||
.takeout
|
||||
.archives
|
||||
.borrow_mut()
|
||||
.retain(|p| *p != target);
|
||||
repaint_archives(&ui_remove);
|
||||
});
|
||||
row.add_suffix(&remove);
|
||||
ui.takeout.list_group.add(&row);
|
||||
rows.push(row.upcast());
|
||||
}
|
||||
ui.takeout.list_group.set_description(Some(&format!(
|
||||
"{} archive(s), {} in total. {EMPTY_LIST_DESCRIPTION}",
|
||||
staged.len(),
|
||||
human_bytes(total)
|
||||
)));
|
||||
}
|
||||
*ui.takeout.rows.borrow_mut() = rows;
|
||||
|
||||
save_staged(ui);
|
||||
sync_takeout_actions(ui);
|
||||
}
|
||||
|
||||
/// Enable exactly the actions that make sense right now: nothing to launch
|
||||
/// without archives, nothing to launch while a run is in flight, and Stop only
|
||||
/// while there is something to stop.
|
||||
fn sync_takeout_actions(ui: &Rc<Ui>) {
|
||||
let has_files = !ui.takeout.archives.borrow().is_empty();
|
||||
let running = ui.takeout.running.get();
|
||||
ui.takeout.scan_button.set_sensitive(has_files && !running);
|
||||
ui.takeout
|
||||
.import_button
|
||||
.set_sensitive(has_files && !running);
|
||||
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
||||
ui.takeout.cancel_button.set_visible(running);
|
||||
}
|
||||
|
||||
/// Hand the staged set to the daemon. It acks immediately and works in the
|
||||
/// background — an export is far past any socket timeout — so success here means
|
||||
/// "started", and the page switches over to reporting progress.
|
||||
fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
let archives: Vec<String> = ui
|
||||
.takeout
|
||||
.archives
|
||||
.borrow()
|
||||
.iter()
|
||||
.map(|p| p.display().to_string())
|
||||
.collect();
|
||||
if archives.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
ui.takeout.dry_run.set(dry_run);
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(
|
||||
ui.dirs.control_socket(),
|
||||
Request::ImportTakeout { archives, dry_run },
|
||||
);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => {
|
||||
ui.takeout.running.set(true);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout.progress_label.set_text(if dry_run {
|
||||
"Reading archives…"
|
||||
} else {
|
||||
"Starting import…"
|
||||
});
|
||||
sync_takeout_actions(&ui);
|
||||
toast(
|
||||
&ui,
|
||||
if dry_run {
|
||||
"Scanning the export…"
|
||||
} else {
|
||||
"Import started — you can leave this page, it keeps running."
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, "Couldn't start the import", &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
"Couldn't start the import",
|
||||
"The mount service didn't respond.",
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Poll the daemon for the import's state while the page is on screen, and paint
|
||||
/// the report once a run has finished. Cheap and inflight-guarded, like the other
|
||||
/// tick polls.
|
||||
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
if ui.takeout.inflight.get() {
|
||||
return;
|
||||
}
|
||||
ui.takeout.inflight.set(true);
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::ImportStatus);
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.takeout.inflight.set(false);
|
||||
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
|
||||
return;
|
||||
};
|
||||
let was_running = ui.takeout.running.replace(running);
|
||||
if running {
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
} else {
|
||||
ui.takeout.progress_group.set_visible(false);
|
||||
// An import runs for hours, so its ending is announced rather than
|
||||
// left to be discovered: a toast for whoever is looking at the
|
||||
// window, and a desktop notification for whoever is not.
|
||||
if was_running {
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
let headline = if dry {
|
||||
"Scan finished"
|
||||
} else {
|
||||
"Import finished"
|
||||
};
|
||||
toast(&ui, headline);
|
||||
if !dry {
|
||||
let body = match &summary {
|
||||
Some(s) if s.failed > 0 => {
|
||||
format!("{} photos added, {} failed.", s.uploaded, s.failed)
|
||||
}
|
||||
Some(s) if s.cancelled => {
|
||||
format!("Stopped after {} photos.", s.uploaded)
|
||||
}
|
||||
Some(s) => format!("{} photos added to Proton Photos.", s.uploaded),
|
||||
None => "The Google Photos import has finished.".to_string(),
|
||||
};
|
||||
notify("takeout-import", headline, &body);
|
||||
}
|
||||
}
|
||||
}
|
||||
if was_running != running {
|
||||
sync_takeout_actions(&ui);
|
||||
}
|
||||
match summary {
|
||||
Some(summary) => repaint_summary(&ui, &summary),
|
||||
None => ui.takeout.summary_group.set_visible(false),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Paint the last run's report.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
||||
ui.takeout.summary_group.remove(&row);
|
||||
}
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
ui.takeout
|
||||
.summary_group
|
||||
.set_title(if dry { "Scan result" } else { "Last import" });
|
||||
ui.takeout.summary_group.set_description(Some(if dry {
|
||||
"Nothing was uploaded — this is what an import would do."
|
||||
} else if summary.cancelled {
|
||||
"Stopped early. Everything uploaded so far is in your timeline; run it again to \
|
||||
continue where it left off."
|
||||
} else {
|
||||
"Finished."
|
||||
}));
|
||||
|
||||
// Only lines that carry information: a run with no failures should not have
|
||||
// to show a "0 failed" row for the user to read past.
|
||||
let mut lines: Vec<(String, String)> = vec![
|
||||
("Photos in the export".into(), summary.found.to_string()),
|
||||
(
|
||||
if dry { "Would upload" } else { "Uploaded" }.into(),
|
||||
summary.uploaded.to_string(),
|
||||
),
|
||||
];
|
||||
if summary.duplicates > 0 {
|
||||
lines.push((
|
||||
"Already in Proton Photos".into(),
|
||||
summary.duplicates.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
"Albums to create"
|
||||
} else {
|
||||
"Albums created"
|
||||
}
|
||||
.into(),
|
||||
summary.albums_created.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.album_links > 0 {
|
||||
lines.push((
|
||||
"Photos filed into albums".into(),
|
||||
summary.album_links.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.skipped_trashed > 0 {
|
||||
lines.push((
|
||||
"Skipped (in Google's trash)".into(),
|
||||
summary.skipped_trashed.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
"Data to upload"
|
||||
} else {
|
||||
"Data uploaded"
|
||||
}
|
||||
.into(),
|
||||
human_bytes(summary.bytes),
|
||||
));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
lines.push(("Failed".into(), summary.failed.to_string()));
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (title, value) in lines {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(title)
|
||||
.subtitle(value)
|
||||
.build();
|
||||
row.add_css_class("property");
|
||||
ui.takeout.summary_group.add(&row);
|
||||
rows.push(row);
|
||||
}
|
||||
*ui.takeout.summary_rows.borrow_mut() = rows;
|
||||
ui.takeout.summary_group.set_visible(true);
|
||||
}
|
||||
|
||||
/// Paint the import's progress from the transfer snapshot the Settings tick
|
||||
/// already fetched, so this page costs no extra poll. A job with no known total
|
||||
/// pulses instead of showing a fraction, the same as the Activity list.
|
||||
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
|
||||
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
|
||||
return;
|
||||
};
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
if job.total > 0 {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&format!("{} of {} — {}", job.done, job.total, job.detail));
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(if job.detail.is_empty() {
|
||||
"Working…"
|
||||
} else {
|
||||
&job.detail
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,8 @@ pub(crate) struct DetailsState {
|
||||
pub(crate) details_suppress: Cell<bool>,
|
||||
/// The grid's and list's selection models. Both wrap the one browser model, so
|
||||
/// a selection in either drives the details pane.
|
||||
pub(crate) grid_selection: gtk4::SingleSelection,
|
||||
pub(crate) list_selection: gtk4::SingleSelection,
|
||||
pub(crate) grid_selection: gtk4::MultiSelection,
|
||||
pub(crate) list_selection: gtk4::MultiSelection,
|
||||
}
|
||||
|
||||
/// The widgets in the browser's details pane that a selection repaints.
|
||||
@@ -158,24 +158,22 @@ pub(crate) fn wire_details(ui: &Rc<Ui>) {
|
||||
ui.details.list_selection.clone(),
|
||||
] {
|
||||
let ui_sel = ui.clone();
|
||||
selection.connect_selection_changed(move |sel, _, _| {
|
||||
match entry_at(sel.model().as_ref(), sel.selected()) {
|
||||
Some(entry) => show_details(&ui_sel, &entry),
|
||||
None => hide_details(&ui_sel),
|
||||
selection.connect_selection_changed(move |_, _, _| {
|
||||
// The details pane describes *one* entry. A batch is the bulk bar's
|
||||
// business, so the pane steps aside rather than picking a member of
|
||||
// the selection to speak for the rest.
|
||||
let entries = selected_entries(&ui_sel);
|
||||
match entries.as_slice() {
|
||||
[entry] => show_details(&ui_sel, entry),
|
||||
_ => hide_details(&ui_sel),
|
||||
}
|
||||
sync_bulk_bar(&ui_sel);
|
||||
});
|
||||
}
|
||||
|
||||
let ui_close = ui.clone();
|
||||
ui.details.details.close_button.connect_clicked(move |_| {
|
||||
ui_close
|
||||
.details
|
||||
.grid_selection
|
||||
.set_selected(gtk4::INVALID_LIST_POSITION);
|
||||
ui_close
|
||||
.details
|
||||
.list_selection
|
||||
.set_selected(gtk4::INVALID_LIST_POSITION);
|
||||
clear_selection(&ui_close);
|
||||
hide_details(&ui_close);
|
||||
});
|
||||
|
||||
@@ -289,8 +287,9 @@ pub(crate) fn hide_details(ui: &Rc<Ui>) {
|
||||
*ui.details.details_entry.borrow_mut() = None;
|
||||
}
|
||||
|
||||
/// The entry highlighted in whichever browser view is on screen, if any. Backs the
|
||||
/// F2 / Delete shortcuts.
|
||||
/// The single entry the details pane is showing, if any. Backs the actions that
|
||||
/// only make sense on one entry (rename, versions); the batch-capable ones ask
|
||||
/// [`selected_entries`] instead.
|
||||
pub(crate) fn selected_entry(ui: &Rc<Ui>) -> Option<DirEntry> {
|
||||
ui.details.details_entry.borrow().clone()
|
||||
}
|
||||
+410
-17
@@ -12,13 +12,165 @@ Conventions:
|
||||
|
||||
---
|
||||
|
||||
## B89 — The recorded SDK dependency is three minor versions out of date in the working memory
|
||||
|
||||
**Status:** Open — documentation only, but it misdirects every reader of the SDK code.
|
||||
**Found:** 2026-08-17, while checking B85's evidence against the reader implementation.
|
||||
**Where:** `proton-dev/CLAUDE.md` (the container-level file, not in this repo).
|
||||
|
||||
`CLAUDE.md` states that `proton-drive-linux/Cargo.toml` declares `proton-sdk = "0.1"` /
|
||||
`proton-drive-rs = "0.1"` and that `Cargo.lock` pins **0.1.11** from the registry, against a local
|
||||
`proton-sdk-rs/` at 0.1.12. Neither half is true any more: the lock pins **0.6.0**, bumped by
|
||||
1.7.1 (see its changelog entry, "Workspace deps bumped to `proton-sdk` / `proton-drive-rs` 0.6.0")
|
||||
and never reflected in the working memory.
|
||||
|
||||
**Why it matters beyond tidiness.** The whole point of that section is to tell you which source
|
||||
you are reasoning about when you read SDK code. Verifying B85 meant establishing what `read_at`
|
||||
does in the version the client actually builds against; following `CLAUDE.md` sends you to
|
||||
`proton-drive-rs-0.1.11` in the registry cache, and to a local checkout described as one patch
|
||||
ahead of it when it is in fact five minor versions behind. The conclusion happened to be the same
|
||||
in every version — but that was luck, and the next question of this shape may not be.
|
||||
|
||||
**Fix:** correct the versions, and prefer describing where to *look* (`grep -A2 'name =
|
||||
"proton-drive-rs"' Cargo.lock`) over restating a number that goes stale on every dependency bump.
|
||||
|
||||
---
|
||||
|
||||
## B88 — `pdfs pin`, `rename`, `move`, `delete` and sharing still cannot name a path under a secondary mount
|
||||
|
||||
**Status:** Open — B86's fix covers `ls` and `refresh` only.
|
||||
**Found:** 2026-08-17, while fixing B86.
|
||||
**Where:** `crates/pdfs-fuse/src/control.rs`, every handler still calling `rel_to_mount`.
|
||||
|
||||
B86's second half added `Core::rooted_at` and `control.rs::route_to_mount`, which resolve a path
|
||||
against whichever mount the daemon owns it under and answer the request in that mount's inode
|
||||
space. Only `ListDir` and `RefreshScope::Dir` were routed through it, because those two were what
|
||||
the B86 repro needed: an escape hatch for a folder serving a stale listing.
|
||||
|
||||
Every other path-addressed control request — `Pin`, `Unpin`, `Rename`, `Move`, `Delete`,
|
||||
`CreateFolder`, `OpenFile`, the sharing and public-link requests, `ListRevisions` — still uses the
|
||||
primary-only `rel_to_mount`, so an absolute path inside a secondary on-demand mount is rejected
|
||||
with "is not under the mountpoint". The user can now *see* such a folder's contents and cannot act
|
||||
on them by path.
|
||||
|
||||
**Why it was left.** Routing a mutating request is not the same change as routing a read. Each of
|
||||
those handlers resolves an inode, checks access, performs a remote operation and then invalidates
|
||||
local state, and several of them invalidate `self.state()` — the primary — explicitly. Routing
|
||||
them means auditing each for which mount's state it is really talking about, which is a larger and
|
||||
riskier change than the one B86 needed. Doing it half-way would be worse than not doing it.
|
||||
|
||||
**Required fix/test:** move the remaining handlers onto `route_to_mount`, auditing each for
|
||||
per-mount state assumptions as it goes; the `for_each_state`/uid-keyed helpers are the pattern for
|
||||
the ones that touch more than their own inode space. Test: a `pdfs pin` and a `pdfs rename` by
|
||||
absolute path inside a secondary on-demand mount, plus a path under no mount still erroring.
|
||||
|
||||
---
|
||||
|
||||
## B87 — A revision's block geometry is learned a read too late
|
||||
|
||||
**Status:** Open — a bounded, self-correcting cost, accepted deliberately in 1.8.2.
|
||||
**Found:** 2026-08-17, while fixing B85. This is the residue that fix leaves.
|
||||
**Where:** `crates/pdfs-fuse/src/reads.rs` (`Core::block_geometry`, `read_block`).
|
||||
|
||||
B85's fix takes block boundaries from the revision instead of assuming 4 MiB. The geometry comes
|
||||
from `RevisionReader::block_sizes()`, and a reader is expensive to open — link details, ancestor
|
||||
keys, an S2K node-key unlock and the block table, which is exactly the per-read work the block
|
||||
cache exists to avoid (B12). So the geometry is *learned* as a side effect of the first read,
|
||||
which has to open a reader anyway, and written to a `<key>.geom` sidecar for later reads.
|
||||
|
||||
The first read of a file therefore still plans on `BlockGeometry::uniform`. On a file whose blocks
|
||||
are not 4 MiB that plan is wrong, and it costs exactly one straddling fetch: the reader fetches
|
||||
two of the revision's blocks to answer for one client block, and the block written to the cache is
|
||||
at an offset the *next* read — now planning correctly — will not ask for, so it ages out unused.
|
||||
Every read after the first is right.
|
||||
|
||||
**Why it is not being fixed now.** The alternatives are all worse than the cost. Resolving a
|
||||
reader before planning reintroduces B12's per-read key derivation on every file. Persisting the
|
||||
geometry at upload only covers files this client wrote, which are the ones the fallback is already
|
||||
correct for. Fetching the block table alone still costs the node-key unlock.
|
||||
|
||||
**Worth doing if:** non-4-MiB revisions turn out to be common on real accounts rather than a
|
||||
handful of files on one, or if the block table becomes reachable without unlocking the node key.
|
||||
Measure before deciding — one wasted fetch per file per revision is not obviously worth paying
|
||||
anything to remove.
|
||||
|
||||
---
|
||||
|
||||
## B86 — An on-demand mount serves a listing that predates the mirror engine's own uploads
|
||||
|
||||
**Status:** Fixed in code (2026-08-17); live re-run of the repro pending.
|
||||
**Found:** 2026-08-16, while trying to drive B42's remote-delete path from one machine.
|
||||
**Where:** `crates/pdfs-fuse/src/sync.rs` upload path (no listing invalidation) meets the cached
|
||||
children snapshot the on-demand mount enumerates from.
|
||||
|
||||
**Repro.** With folder 6 (`~/pdfs-live-mirror`) in mirror mode: create `c1.txt` and `c2.txt`,
|
||||
`pdfs sync now 6` — the activity feed reports `c1.txt new file`, `c2.txt new file` and
|
||||
`sync_entry` gains rows carrying real `remote_uid`s. Switch the folder to on-demand and list it:
|
||||
|
||||
```
|
||||
$ ls ~/pdfs-live-mirror
|
||||
b25.txt
|
||||
b40.bin # c1.txt, c2.txt and the conflict copy are missing
|
||||
```
|
||||
|
||||
`ls ~/pdfs-live-mirror/c1.txt` returns `ENOENT`. Restarting the daemon does not help — the stale
|
||||
children snapshot is in the database, not in memory. Switching the folder back to mirror proves
|
||||
the files were on the server the whole time: the pass downloads all five, and `b40.bin` comes back
|
||||
md5-identical.
|
||||
|
||||
**Effect.** No data loss — the mirror side is correct and the files are intact remotely — but a
|
||||
user who switches a synced folder to on-demand sees an incomplete folder, and anything that walks
|
||||
it (a backup, an indexer, `rm -r`) sees the same incomplete folder. It also blocks B42's live
|
||||
verification, because `pdfs rm` cannot name a file the mount will not admit exists.
|
||||
|
||||
**Second, smaller gap in the same repro:** `pdfs refresh` and `pdfs ls` reject any path that is
|
||||
not under the *primary* mountpoint (`Error: … is not under the mountpoint`), so a secondary
|
||||
on-demand mount has no manual escape hatch either.
|
||||
|
||||
**Required fix/test:** invalidate (or update) the parent's cached children when the sync engine
|
||||
creates a remote node, the same way the FUSE write path does; and let `refresh`/`ls` resolve a
|
||||
path under any mount the daemon owns, not just the primary. Test: the repro above, asserting the
|
||||
new files appear immediately after the mode switch.
|
||||
|
||||
### Fix (2026-08-17)
|
||||
|
||||
Two halves, matching the two halves of the report.
|
||||
|
||||
**The stale listing.** `Core::invalidate_children_of` (`pdfs-fuse/src/lib.rs`) drops a folder's
|
||||
cached children *by uid*. The existing `invalidate_parent_listing` could not serve here for two
|
||||
reasons: it names the folder by inode, and a mirror folder has no inode space at all at the moment
|
||||
the engine writes to it; and the listing that outlived the pass was not a hot-cache map but the
|
||||
`listed` flag on the folder's DB row — which is why restarting the daemon did not help. The new
|
||||
helper clears that flag unconditionally and then invalidates whichever mounts happen to hold the
|
||||
folder resident. The sync engine calls it after every remote mutation it makes: folder creation
|
||||
and file upload (`apply_one`, `upload_new`), and the three remote-trash sites, whose parent uid
|
||||
comes from the pass's folder-uid map or the baseline (`parent_uid_of`).
|
||||
|
||||
**The missing escape hatch.** `StateRegistry` now carries each mount's `size_upgrades` map as well
|
||||
as its state, notifier and liveness flag — that set is exactly the per-mount half of a `Core`
|
||||
(everything `fork_state` replaces) — and `StateRegistry::covering_parts` returns it for the mount
|
||||
that most specifically covers a path. `Core::rooted_at` swaps those fields into a clone of the
|
||||
`Core`, giving a view rooted at the owning mount plus the path relative to *its* root. `pdfs ls`
|
||||
and `pdfs refresh` route through it (`control.rs::route_to_mount`), so a secondary on-demand mount
|
||||
can be named at last; a path under nothing this daemon has mounted still gets the familiar "is not
|
||||
under the mountpoint" error. The other path-addressed control requests deliberately still use the
|
||||
primary-only `rel_to_mount` — routing the mutating ones is a larger change and was not what
|
||||
blocked the repro.
|
||||
|
||||
Test: `covering_parts_route_a_path_to_the_mount_that_owns_it` covers the routing rule (nested
|
||||
mount wins, suffix relative to it, its own inode space and upgrade batches, a path under no mount
|
||||
routes nowhere). The listing-invalidation half is the live repro, which is what is still pending.
|
||||
|
||||
---
|
||||
|
||||
## B84 — A block read that comes back short truncates the file, silently
|
||||
|
||||
**Status:** Fixed and **live-verified 2026-08-16**. **Data correctness** — an application reading a large
|
||||
file through the mount could get a truncated copy that reported success. The truncation is now
|
||||
impossible: a disagreement is repaired from the whole-file download, or the read fails `EIO`.
|
||||
Unverified because the repair path has not yet been driven against the three known files on a
|
||||
live mount, and the metadata question below is still open.
|
||||
Re-verified on 1.8.1 after the release build: `118454731_p0.png`, `121677947_p0.png`,
|
||||
`123349685_p0.png` and `128451683_p0.png` each delivered exactly their `stat` size
|
||||
(19471492, 11811519, 22055175, 11393178 bytes) with no `Invalid argument` reply in the journal.
|
||||
**Found:** 2026-08-16, chasing what looked like log noise. GNOME's file indexer
|
||||
(`localsearch-3`) was filling the journal with `libpng error: Read Error`,
|
||||
`libpng error: IDAT: CRC error` and `File too small to be a PNG` on files under
|
||||
@@ -127,7 +279,9 @@ download on first read.
|
||||
|
||||
## B85 — The read path assumes 4 MiB blocks; some revisions do not have them
|
||||
|
||||
**Status:** Open — **data correctness**, currently masked by B84's repair path
|
||||
**Status:** Fixed in code (2026-08-17) — the geometry now comes from the revision. See
|
||||
"Correction to the diagnosis" below: the premise that this was *losing* data does not survive
|
||||
reading the SDK, and the fix is a cost fix, not a correctness one.
|
||||
**Found:** 2026-08-16, while live-verifying B84 (see its journal evidence)
|
||||
**Where:** `crates/pdfs-fuse/src/reads.rs`, `crates/pdfs-core/src/cache.rs` (`BLOCK_SIZE`,
|
||||
`block_len`)
|
||||
@@ -171,6 +325,65 @@ size recorded per block after some transformation. Worth answering before choosi
|
||||
"blocks are whatever the revision says" and "blocks are a different constant" lead to different
|
||||
designs.
|
||||
|
||||
### Correction to the diagnosis (2026-08-17)
|
||||
|
||||
**The logged numbers cannot have come from `read_at`, and the "this client asks for the wrong
|
||||
range" reading is wrong.** `RevisionReader::read_at(offset, length)` does not assume any geometry:
|
||||
it computes `end = min(offset + length, file_size)`, plans over the *real* block sizes
|
||||
(`plan_blocks`), and splices each block clamped to `[offset, end)` (`splice_block`). Both are
|
||||
present and identical in the version the client actually builds against — `Cargo.lock` pins
|
||||
`proton-drive-rs` **0.6.0**, not the 0.1.11 the top-level `CLAUDE.md` still describes — and in
|
||||
0.1.11 as well. So a `read_at(bstart, blen)` can return at most `blen` bytes, and every quoted
|
||||
line has `got > blen`:
|
||||
|
||||
```
|
||||
bidx=0 blen=4194304 got=4939506 # got exceeds the length requested
|
||||
bidx=1 blen=4194304 got=5670228 # …as does this one
|
||||
```
|
||||
|
||||
The only site in the client that logs this shape (`reads.rs`, the `blen`/`got`/`fsize` warning)
|
||||
reaches it solely on the `bytes.len() != blen` branch of a `read_at(bstart, blen)`. Those two
|
||||
facts cannot both hold. Either the numbers were transcribed with `blen` and `got` swapped, or they
|
||||
came from a build that is not in the history. **This wants re-capturing from the journal before
|
||||
anything is concluded from it** — and in particular, the entry's claim that B84's repair path is
|
||||
"currently masking data loss" here is not supported.
|
||||
|
||||
What *is* true, and is what got fixed: the client assumed the geometry. `idx * BLOCK_SIZE` and
|
||||
`block_len(size, idx)` encoded "4 MiB blocks with a remainder" in the range planner, the in-memory
|
||||
ring, the in-flight fetch key and the on-disk block cache. When a revision is not shaped that way,
|
||||
nothing breaks — but every client block straddles two server blocks, so the reader fetches and
|
||||
decrypts two to answer for one, and the block written to disk lines up with nothing the next read
|
||||
asks for. That is a real and unbounded cost on exactly the files the entry found, and it defeats
|
||||
the property the block cache was built around.
|
||||
|
||||
### Fix (2026-08-17)
|
||||
|
||||
`BlockGeometry` (`pdfs-core/src/cache.rs`) is a prefix sum over a revision's block sizes, with
|
||||
`BlockGeometry::uniform(size)` reproducing the old constant-block arithmetic exactly. Blocks are
|
||||
now addressed by `BlockSpan { idx, start, len }` rather than by index, everywhere: the range
|
||||
planner (`spans(offset, end)`), the in-memory ring, the in-flight-fetch key, and the on-disk block
|
||||
cache. The span is the identity because the index alone stopped naming one byte range — block 1
|
||||
under the fallback and block 1 under the real geometry are different bytes — so the block
|
||||
sidecar records the block's plaintext `start` and a re-planned read misses rather than being
|
||||
served the wrong offset. Sidecars written before that field existed read as "wherever the 4 MiB
|
||||
assumption put it", which keeps the existing cache intact for every file this client uploaded.
|
||||
|
||||
The geometry itself is learned as a side effect of the first read (which has to open a
|
||||
`RevisionReader` anyway) and written to a `<key>.geom` sidecar under the block cache;
|
||||
`Core::block_geometry` reads it back and falls back to `uniform` when it is absent. Resolving a
|
||||
reader just to *plan* would reintroduce the per-read key derivation the block cache exists to
|
||||
avoid (B12), so the first read of a file still plans on the fallback and self-corrects. A recorded
|
||||
table whose sizes do not sum to the node's size is refused outright, and eviction takes the
|
||||
geometry with the blocks it describes.
|
||||
|
||||
The open question above is still open, and no longer blocks anything: "blocks are whatever the
|
||||
revision says" is now what the code does, so a different constant is just another table.
|
||||
|
||||
Tests: `the_uniform_geometry_is_the_old_block_arithmetic`,
|
||||
`a_non_uniform_geometry_places_blocks_by_prefix_sum` (built on the exact sizes from the log
|
||||
above), `a_cached_block_is_not_served_to_a_different_geometry`,
|
||||
`a_recorded_geometry_is_returned_only_for_the_revision_it_describes`.
|
||||
|
||||
---
|
||||
|
||||
## B83 — An access-deferred op retries forever and is reported as an ordinary queued upload
|
||||
@@ -1795,8 +2008,16 @@ suite run against a heavily loaded drain.
|
||||
|
||||
## B25 — Weak Conflict Baseline Detection `(mtime, size)` (HIGH-05)
|
||||
|
||||
**Status:** Fixed (unverified) 2026-08-16 — the FUSE-write half was closed by **B69**; the
|
||||
mirror half is closed here.
|
||||
**Status:** Fixed and **live-verified 2026-08-16** on 1.8.1 — the FUSE-write half was closed by
|
||||
**B69**; the mirror half is closed here.
|
||||
|
||||
**Live verification (2026-08-16, `~/pdfs-live-mirror`, folder 6).** `b25.txt` was written with
|
||||
mtime `12:00:10.100000000`, synced (baseline `local_mtime=1786874410`,
|
||||
`local_mtime_ns=1786874410100000000`), then rewritten with sixteen different bytes and stamped
|
||||
`12:00:10.900000000` — identical whole second, identical length. The next pass classified it as
|
||||
locally changed and uploaded `b25.txt new version`; the baseline advanced to
|
||||
`…410900000000`, and reading the file back through an on-demand mount of the same folder returned
|
||||
`BBBBBBBBBBBBBBBB`. Under the old whole-second comparison this edit was invisible.
|
||||
**Found:** 2026-07-21, multi-agent sync engine audit (`audit_bugs.md` HIGH-05)
|
||||
**Where:** `crates/pdfs-core/src/cache.rs:L157`, `Baseline`
|
||||
|
||||
@@ -1961,7 +2182,15 @@ that made these look like incidents in the journal.
|
||||
|
||||
## B32 — Race Condition in Concurrent Handle Release and Open (MED-05)
|
||||
|
||||
**Status:** Fixed (unverified) 2026-08-16
|
||||
**Status:** Fixed and **live-verified 2026-08-16** on 1.8.1
|
||||
|
||||
**Live verification (2026-08-16).** Three 64 MiB random files were written over the same mount
|
||||
path (`ProtonDrive/pdfs-live-20260816/b32.bin`) back to back, each `cp` closing before the next
|
||||
began, while the drain was still working. The queue coalesced them into a single op (29 queued =
|
||||
28 known-stuck + 1). After the drain retired it, `pdfs refresh` dropped the cached listing and a
|
||||
full re-read from Drive returned `4ad41482f5199a401e425381d7621626` — the md5 of the *third*
|
||||
write, byte for byte. No open re-used a stale staged base, and the retry warning never had to
|
||||
fire.
|
||||
**Found:** 2026-07-21, multi-agent concurrency audit (`audit_bugs.md` MED-05)
|
||||
**Where:** `crates/pdfs-fuse/src/lib.rs:L4042, L4574`
|
||||
|
||||
@@ -2101,7 +2330,22 @@ pass from being treated as successfully settled.
|
||||
|
||||
## B39 — Sync Download Can Overwrite a Concurrent Local Edit (HIGH-09)
|
||||
|
||||
**Status:** Fixed (unverified) 2026-08-16
|
||||
**Status:** Fixed and **live-verified 2026-08-16** on 1.8.1
|
||||
|
||||
**Live verification (2026-08-16).** Folder 6 was switched to on-demand (evicting its local
|
||||
copies) and then back to mirror, which queues a download of the 128 MiB `b40.bin`. Three seconds
|
||||
into that download, `RACING-LOCAL-EDIT-B39` was written to the destination path. The pass logged
|
||||
|
||||
```
|
||||
WARN pdfs_fuse::sync: sync: local file changed while downloading; keeping it as a conflict copy
|
||||
path=/home/narl/pdfs-live-mirror/b40.bin expected=None
|
||||
actual=Some(LocalSig { mtime: 1786916720, mtime_ns: Some(1786916720414151922), size: 21 })
|
||||
```
|
||||
|
||||
and finished with the remote copy at `b40.bin` and the racing 21 bytes intact at
|
||||
`b40 (sync-conflict 1786916794).bin` — which the following pass then uploaded as a file of its
|
||||
own. Before the fix the rename would have destroyed those bytes with no trace.
|
||||
|
||||
**Found:** 2026-07-22, sync concurrency audit
|
||||
**Where:** `crates/pdfs-fuse/src/sync.rs`, download classification and apply path
|
||||
|
||||
@@ -2136,7 +2380,25 @@ the guard directly instead.
|
||||
|
||||
## B40 — Sync Upload Can Stream Torn Live Content (HIGH-10)
|
||||
|
||||
**Status:** Fixed (unverified) 2026-08-16
|
||||
**Status:** Fixed and **live-verified 2026-08-16** on 1.8.1
|
||||
|
||||
**Live verification (2026-08-16).** A 128 MiB `b40.bin` was dropped into folder 6 and a pass
|
||||
forced; six seconds into the upload, sixteen bytes were overwritten at offset 100,000,000. The
|
||||
pass logged
|
||||
|
||||
```
|
||||
WARN pdfs_fuse::sync: sync: file changed while uploading; the remote revision may be torn and
|
||||
will be replaced on the next pass rel="b40.bin"
|
||||
streamed=LocalSig { mtime: 1786916599, mtime_ns: Some(1786916599336211782), size: 134217728 }
|
||||
now=Some(LocalSig { mtime: 1786916605, mtime_ns: Some(1786916605686365985), size: 134217728 })
|
||||
```
|
||||
|
||||
and — this is the part that matters — recorded the baseline against the *streamed* signature
|
||||
(`…599`), not the file on disk. The very next pass therefore read the file as locally changed and
|
||||
uploaded `b40.bin new version`. Round-tripping the folder through on-demand and back later
|
||||
downloaded the remote copy at md5 `cfb45003003fb2ad50c2c365f8313849` — the post-mutation local
|
||||
content, exactly. The torn revision healed itself without anyone asking.
|
||||
|
||||
**Found:** 2026-07-22, sync concurrency audit
|
||||
**Where:** `crates/pdfs-fuse/src/sync.rs`, upload apply and baseline update
|
||||
|
||||
@@ -2227,11 +2489,29 @@ the state the engine was asking for. Covered by
|
||||
`sync::tests::an_already_absent_path_counts_as_removed`. Unverified because no live sync pass has
|
||||
been driven over an externally-deleted file yet.
|
||||
|
||||
**Live verification attempted 2026-08-16, blocked.** Driving this needs the *remote* side of a
|
||||
mirror folder to lose a file, and a mirror folder's remote lives in the device share, which no
|
||||
mountpoint exposes — `pdfs rm` only accepts paths under a mountpoint, and the obvious workaround
|
||||
(switch the folder to on-demand, trash the file through its own mount, switch back) runs into
|
||||
**B86**: the on-demand mount serves a listing that predates the mirror engine's uploads, so the
|
||||
file to be trashed is not visible to `rm` at all. Verifying B42 live therefore needs either a
|
||||
second client on the account or B86 fixed first. The ENOENT branch remains covered only by its
|
||||
unit test.
|
||||
|
||||
---
|
||||
|
||||
## B43 — FUSE Directory Cookies Are Not Stable (MED-09)
|
||||
|
||||
**Status:** Fixed (unverified) 2026-08-16
|
||||
**Status:** Fixed and **live-verified 2026-08-16** on 1.8.1
|
||||
|
||||
**Live verification (2026-08-16).** A probe opened
|
||||
`ProtonDrive/pdfs-live-20260816` (12 files) and called `getdents64` with a 160-byte buffer, so the
|
||||
kernel had to page through the listing; between every page it created a new file in the directory
|
||||
and unlinked an older one. Three pages, 14 entries returned (12 + `.` + `..`), zero duplicates,
|
||||
zero pre-existing entries missed — and none of the files created mid-enumeration appeared, which
|
||||
is the frozen snapshot behaving as specified. The same probe against the pre-fix index-cookie
|
||||
readdir is what this entry was written about.
|
||||
|
||||
**Found:** 2026-07-22, FUSE/POSIX audit
|
||||
**Where:** `crates/pdfs-fuse/src/filesystem.rs`, `ProtonFs::serve_readdir`
|
||||
|
||||
@@ -2267,7 +2547,15 @@ entries between `getdents` calls, asserting no entry is skipped or repeated.
|
||||
|
||||
## B44 — FUSE Background Workers Have No Coordinated Shutdown (MED-10)
|
||||
|
||||
**Status:** Fixed (unverified) 2026-08-16
|
||||
**Status:** Fixed and **live-verified 2026-08-16** on 1.8.1
|
||||
|
||||
**Live verification (2026-08-16).** `systemctl --user restart proton-drive.service` on a daemon
|
||||
with six live mounts, a busy drain and 18.4 GiB staged: stop to `Stopped` took 110 ms, the journal
|
||||
shows the six unmounts followed by `sync engine stopping` and `daemon stopping`, and systemd never
|
||||
reached its SIGKILL timeout. The old PID was gone immediately, and the fresh daemon settled at 42
|
||||
threads — the same count the outgoing one had, so no loop was left behind or spawned twice. A
|
||||
second restart mid-session behaved identically.
|
||||
|
||||
**Found:** 2026-07-22, daemon lifecycle audit
|
||||
**Where:** `crates/pdfs-fuse/src/mount.rs`, mount lifecycle; drain/index/sync loops
|
||||
|
||||
@@ -2497,7 +2785,8 @@ reused, making a session-long tombstone safe.
|
||||
|
||||
## B49 — Failed Conflict Preservation Still Overwrites the Local File (CRIT-05)
|
||||
|
||||
**Status:** Fixed in code with deterministic regressions; live/fault verification pending
|
||||
**Status:** Fixed, with the injected-failure suite the entry asked for (2026-08-17). Only the
|
||||
end-to-end live conflict remains unexercised.
|
||||
**Found:** 2026-07-22, 1.0 data-safety audit
|
||||
**Where:** `crates/pdfs-fuse/src/sync.rs`, download/conflict handling
|
||||
|
||||
@@ -2510,11 +2799,23 @@ use collision-resistant names. Test existing names, permissions, `ENOSPC`,
|
||||
`EIO`, and cross-filesystem behavior. This is B39's failure path; B25 remains
|
||||
the separate weak-baseline problem.
|
||||
|
||||
**Verification (2026-08-17).** `preserve_conflict_copy` was already correct — it copies to a
|
||||
`create_new` destination, syncs data and directory, and removes the source last, so every failure
|
||||
returns `Err` and `apply_one` never reaches the download. What was missing was proof, so:
|
||||
`conflict_preservation_failures_all_keep_the_local_file` drives an unwritable directory
|
||||
(`EACCES`), a source that vanished between planning and preservation (`ENOENT`), and a destination
|
||||
name already taken by a directory — and asserts the local file is still there, byte-identical,
|
||||
after each. `a_conflict_copy_is_byte_and_mtime_identical` covers the other half: the copy carries
|
||||
the bytes, mtime and mode, so the next pass uploads the user's version rather than something that
|
||||
looks freshly written. The existing collision test covers the suffix search.
|
||||
|
||||
Not covered: a genuine `ENOSPC` or `EIO` from the storage layer, and the end-to-end live conflict.
|
||||
|
||||
---
|
||||
|
||||
## B50 — Failed Orphan Staging Deletes the Sole Scratch Copy (CRIT-06)
|
||||
|
||||
**Status:** Fixed in code; injected filesystem-failure verification pending
|
||||
**Status:** Fixed, with injected filesystem-failure verification (2026-08-17).
|
||||
**Found:** 2026-07-22, 1.0 data-safety audit
|
||||
**Where:** `crates/pdfs-fuse/src/lib.rs`, `Core::stage_orphaned_write`
|
||||
|
||||
@@ -2526,11 +2827,29 @@ metadata are committed. Inject `ENOSPC`, `EIO`, `EROFS`, permission, and
|
||||
cross-device failures and assert a byte-identical copy remains discoverable.
|
||||
Related: B23, B27, and B28.
|
||||
|
||||
**Verification (2026-08-17).** Four new tests in `pdfs-core/src/cache.rs`.
|
||||
`orphan_preservation_keeps_the_scratch_copy_when_staging_is_unwritable` makes staging mode `0500`
|
||||
so both the rename and the copy fallback fail, and asserts the scratch file survives
|
||||
byte-identical, is the path the error names, and carries its recovery marker.
|
||||
`a_retained_orphan_write_is_discoverable_after_a_restart` reopens the cache after that failure and
|
||||
asserts `recovered_writes` finds the bytes *with their metadata* — present is not the same as
|
||||
discoverable, and discoverable is what the entry asked for.
|
||||
`orphan_preservation_across_a_device_boundary_publishes_before_removing` and
|
||||
`a_failed_cross_device_preservation_keeps_the_marked_source` drive the copy fallback across a real
|
||||
device boundary (`/dev/shm` vs `/tmp`), asserting the ordering — destination and sidecar published
|
||||
and synced first, source removed last — and that a publication which does not finish leaves the
|
||||
marked source in place. The cross-device pair skips with a printed note on a machine that has only
|
||||
one filesystem, rather than passing silently.
|
||||
|
||||
Not covered: a genuine `ENOSPC`/`EIO` from the block layer; `EROFS` is approximated by the
|
||||
unwritable-directory case rather than a read-only mount.
|
||||
|
||||
---
|
||||
|
||||
## B51 — Scratch Sidecar Does Not Establish a Durable Recovery Record (CRIT-07)
|
||||
|
||||
**Status:** Fixed in code; power-loss verification pending
|
||||
**Status:** Fixed, with crash-boundary and corrupt-sidecar verification (2026-08-17). True
|
||||
power-loss testing (a machine actually losing power mid-write) is still not covered.
|
||||
**Found:** 2026-07-22, 1.0 crash-consistency audit
|
||||
**Where:** `crates/pdfs-core/src/cache.rs`, scratch durability markers
|
||||
|
||||
@@ -2543,11 +2862,24 @@ a sparse partial write as complete and replace untouched ranges with zeros.
|
||||
then sync the directory before acknowledging durability. Crash-test every
|
||||
boundary and corrupt/partial sidecars. B20 and B23 do not cover this invariant.
|
||||
|
||||
**Verification (2026-08-17).** `a_half_written_scratch_sidecar_is_never_read_as_a_record` puts on
|
||||
disk exactly what a crash inside `mark_scratch_durable` leaves — a `.json.tmp` and no published
|
||||
sidecar — and asserts the next open recovers nothing from it, so an unpublished temporary can
|
||||
never be replayed as a write. `a_corrupt_scratch_sidecar_still_rescues_its_bytes` truncates a
|
||||
published sidecar to a JSON prefix and asserts the blob is still moved to `recovery/`
|
||||
unaccompanied: metadata loss must not take the user's bytes with it.
|
||||
`remarking_a_scratch_file_replaces_its_record_atomically` covers the repeated-`fsync` case — each
|
||||
call leaves one complete parseable record and no debris.
|
||||
|
||||
Not covered: real power loss. Nothing short of a crashing VM tests that, and the entry should not
|
||||
be closed as if it did.
|
||||
|
||||
---
|
||||
|
||||
## B52 — Staged-Write Publication Is Not Crash-Safe (CRIT-08)
|
||||
|
||||
**Status:** Fixed in code; cross-filesystem and power-loss verification pending
|
||||
**Status:** Fixed, with cross-filesystem verification against a real device boundary
|
||||
(2026-08-17). True power-loss testing is still not covered.
|
||||
**Found:** 2026-07-22, 1.0 crash-consistency audit
|
||||
**Where:** `crates/pdfs-core/src/cache.rs`, staged-write publication
|
||||
|
||||
@@ -2560,11 +2892,24 @@ renames and directory syncs; delete the source only after verification. Inject
|
||||
failure after every write, sync, rename, DB enqueue, and deletion. Related: B50
|
||||
and B51.
|
||||
|
||||
**Verification (2026-08-17).** `stage_write` gained a `stage_write_at` seam — the public entry
|
||||
point picks a timestamped destination, which is right for production and impossible to arrange a
|
||||
failure at, the same reason `preserve_write_at` exists.
|
||||
`staging_across_a_device_boundary_removes_the_source_last` drives the copy path across a real
|
||||
device boundary and asserts the destination and sidecar are both on disk before the source goes.
|
||||
`a_failed_staging_sidecar_keeps_the_cross_device_source` blocks the sidecar's temporary name and
|
||||
asserts the source survives — a staged blob nothing describes is not a substitute for the bytes.
|
||||
`an_unaccompanied_staged_blob_survives_a_restart` asserts startup leaves an undescribed staging
|
||||
blob (and a leftover `.json.tmp`) alone rather than tidying user data away.
|
||||
|
||||
Not covered: real power loss, and injected failure at *every* one of the listed points — the
|
||||
sidecar and source-removal boundaries are covered, the intermediate `fsync` calls are not.
|
||||
|
||||
---
|
||||
|
||||
## B53 — Superseding a Pending Operation Is Not Transactional (CRIT-09)
|
||||
|
||||
**Status:** Fixed with rollback regression; crash/ENOSPC verification pending
|
||||
**Status:** Fixed, with rollback, restart and full-database verification (2026-08-17).
|
||||
**Found:** 2026-07-22, 1.0 SQLite durability audit
|
||||
**Where:** `crates/pdfs-core/src/db/ops.rs`, `Db::enqueue_op`
|
||||
|
||||
@@ -2577,11 +2922,22 @@ blob until commit and make cleanup recoverable. Inject insert/commit failures
|
||||
and prove that old or new operation—never neither—remains queued. B27 concerns
|
||||
a retained failing row; this issue erases the row.
|
||||
|
||||
**Verification (2026-08-17).** The existing `failed_superseding_insert_keeps_the_old_pending_op`
|
||||
proves the row rolls back. Two tests were added for the parts that make the rollback usable.
|
||||
`a_rolled_back_supersede_keeps_its_blob_and_survives_reopen` asserts the failed call does *not*
|
||||
report the old blob as superseded — that value is the caller's instruction to delete those bytes,
|
||||
and the row that owns them is still queued — and that the rollback is still there after closing
|
||||
and reopening a file-backed database, so it is not an artefact of the connection that failed.
|
||||
`a_full_database_rolls_the_supersede_back_rather_than_erasing_it` caps `max_page_count` at the
|
||||
current size, which is the same `SQLITE_FULL` an out-of-space filesystem produces, and asserts the
|
||||
acknowledged upload is still queued and still owns its bytes.
|
||||
|
||||
---
|
||||
|
||||
## B54 — Total-Wipe Guard Does Not Protect a One-Entry Baseline (CRIT-10)
|
||||
|
||||
**Status:** Fixed with one-entry regression; live mode-matrix verification pending
|
||||
**Status:** Fixed, with one-entry, absent-root, unreadable-root and replaced-mountpoint
|
||||
regressions (2026-08-17). The live mode matrix is still the only thing unexercised.
|
||||
**Found:** 2026-07-22, 1.0 mirror-sync audit
|
||||
**Where:** `crates/pdfs-fuse/src/sync/planner.rs`
|
||||
|
||||
@@ -2594,11 +2950,31 @@ independently complete scan plus an explicit deliberate-delete signal for a
|
||||
total wipe. Test absent roots, replaced mountpoints, permissions, and one-file
|
||||
folders in every mode combination.
|
||||
|
||||
**Verification (2026-08-17).** The guard reads `!baseline.is_empty()`, so every non-empty baseline
|
||||
is protected; `guard_local_wipe_blocks_only_a_total_disappearance` already carried the one-entry
|
||||
case. The "independently complete scan" half is B55's fix — an absent, unreadable or partially
|
||||
readable root now fails the pass before the guard is reached, which those tests cover.
|
||||
|
||||
What was missing is the case where nothing fails: a root that is readable but is no longer the
|
||||
tree it was, an empty directory left where a mount used to be.
|
||||
`a_replaced_mountpoint_trips_the_guard_at_every_baseline_size` walks a real empty root, confirms
|
||||
the scan honestly reports nothing, and asserts the guard fires for baselines of one, two and three
|
||||
paths — then that it steps aside the moment the tree is back, so a remounted folder resumes
|
||||
instead of staying wedged.
|
||||
|
||||
**Deliberate divergence from the required fix:** there is no explicit deliberate-delete signal.
|
||||
The guard refuses a total wipe outright rather than offering a way to confirm one. That is
|
||||
stricter than asked for, and it means a user who really did delete everything must remove and
|
||||
re-add the sync folder. Worth revisiting if anyone hits it; not worth building a confirmation path
|
||||
nobody has asked for yet.
|
||||
|
||||
Not covered: the live mode matrix (`scripts/fuse-acceptance.sh --managed-live`).
|
||||
|
||||
---
|
||||
|
||||
## B55 — Incomplete Local Scans Are Interpreted as Deletions (CRIT-11)
|
||||
|
||||
**Status:** Fixed in code; injected iterator/stat failure tests pending
|
||||
**Status:** Fixed, with injected iterator/stat failure tests (2026-08-17).
|
||||
**Found:** 2026-07-22, 1.0 mirror-sync audit
|
||||
**Where:** `crates/pdfs-fuse/src/sync.rs`, local scan/reconciliation
|
||||
|
||||
@@ -2611,6 +2987,23 @@ subtree must make the pass non-destructive and retain its prior baseline. Test
|
||||
`readdir`, `stat`, permission, transient-I/O, and disappearing-entry failures.
|
||||
B42 covers the inverse local-delete failure.
|
||||
|
||||
**Verification (2026-08-17).** The scan body was lifted out of `Core::walk_local` into a free
|
||||
`walk_local_tree`, so it can be driven against a real tree with a real unreadable directory in it
|
||||
and no `Core` at all. Every failure in it is fatal to the pass by construction — a `read_dir` that
|
||||
fails, an entry that cannot be stat-ed, a name that is not UTF-8 — because a skipped path is a
|
||||
path reconciliation reads as a deletion and propagates to Drive.
|
||||
|
||||
Tests: `an_unreadable_subdirectory_fails_the_scan_instead_of_shrinking_it` (chmod `000` on a
|
||||
subdirectory holding a file, asserting the error names the subtree),
|
||||
`an_unreadable_root_fails_the_scan`, `an_absent_root_fails_the_scan` (the unmounted-device case),
|
||||
`a_non_utf8_name_fails_the_scan`, and `a_readable_tree_scans_completely` as the complement — the
|
||||
strictness must not cost a working pass, so a healthy tree still reports every entry and still
|
||||
skips symlinks by design.
|
||||
|
||||
Not covered: transient I/O errors, and an entry that disappears between `read_dir` and `stat`.
|
||||
Both are races that would need a filesystem shim to force; the code path they take is the same
|
||||
`symlink_metadata` failure the permission test drives.
|
||||
|
||||
---
|
||||
|
||||
## B56 — Database Schema Version Parsing Fails Open (HIGH-13)
|
||||
|
||||
@@ -9,6 +9,146 @@ release ships. Migrations are forward-only — a database written by a newer cli
|
||||
refuse-to-open, not a downgrade, so rolling back a release means restoring the cache from
|
||||
scratch (user data in `staging/` and `recovery/` is never touched by this).
|
||||
|
||||
## [1.9.0] — 2026-08-18
|
||||
|
||||
Google Photos import, and a pass over the desktop app's interaction gaps. No schema change.
|
||||
Schema: **28**. Builds against the published `proton-sdk` / `proton-drive-rs` **0.6.1** — the album
|
||||
writes this release needs are in the registry crates, so the local `[patch.crates-io]` shim the
|
||||
feature was developed against is gone and no local path dependency remains.
|
||||
|
||||
### Added
|
||||
- **Import a Google Photos Takeout export into Proton Photos** — `pdfs import-google-photos
|
||||
takeout-*.zip` (add `--dry-run` to see what it would do, `--wait` to follow it), or the import
|
||||
button in the Photos header, which multi-selects the export's zips. Albums in the export are
|
||||
recreated as Proton albums and their photos filed into them; year folders and `Archive/` import
|
||||
into the timeline; `Trash/` is skipped. Capture times and favourites come from Google's per-photo
|
||||
JSON sidecars, including the truncated and `(N)`-swapped sidecar names Google produces.
|
||||
- Photos already on the account are **not re-uploaded**: the import asks the photos volume which of
|
||||
the export's names exist (batched, 150 per request), reads and SHA-1s only those files, and skips
|
||||
the ones whose content digest matches too — Proton's own duplicate rule. A library already partly
|
||||
synced by Proton Photos on a phone therefore imports only what is missing, and an interrupted
|
||||
import resumes by being run again. Google keeps a full copy of a photo in every album folder it
|
||||
appears in; those fold into one upload that joins several albums.
|
||||
- The import streams each photo straight out of its zip, so an export needs no scratch space to
|
||||
unpack into, and reports as a job in `pdfs transfers` and the GUI's activity view. `pdfs
|
||||
import-status` shows the counts, `pdfs cancel-import` stops it — a cancelled run still files what
|
||||
it uploaded into its albums.
|
||||
- **An Import page in the GUI** (Settings → *Import from Google Photos*, or the import button in
|
||||
the Photos header). Archives are staged by drag-and-drop onto the page or through the file
|
||||
chooser, listed with their sizes so the set can be checked before anything is sent, and launched
|
||||
as either a Scan (dry run) or a confirmed Import. The page shows live progress off the import's
|
||||
job, keeps the finished run's report on screen, and can stop a run in flight; because the import
|
||||
outlives the page, it is polled wherever the user navigates and its completion is reported as a
|
||||
desktop notification.
|
||||
|
||||
- SDK (`proton-sdk-rs` **0.6.1**): album **writes** — `ProtonPhotosClient::create_album` and
|
||||
`add_photos_to_album`, ported from the TypeScript SDK (upstream C# has no album write API), plus
|
||||
batch duplicate detection (`find_duplicates_many`, `name_collisions`). The workspace deps moved
|
||||
`"0.6.0"` → `"0.6.1"` and `Cargo.lock` re-resolved to the registry crates. Adding a photo from an
|
||||
album *shared with us* (a different volume) is still unported and fails that photo's outcome.
|
||||
- **Multi-select in Files.** Both views select a batch (click-drag, Ctrl-click, Shift-click); a bar
|
||||
slides in reporting the count and offering Move to Trash and offline on/off for the whole
|
||||
selection. Delete acts on everything selected, Esc clears it, and right-clicking inside a
|
||||
selection offers the batch actions rather than the one row under the pointer. Actions that need a
|
||||
single subject — rename, move, share, revisions — stay single-entry, and the details pane steps
|
||||
aside for a batch instead of speaking for it.
|
||||
- **Undo on trash.** "Moved *X* to Trash" now carries an Undo that restores the batch, single files
|
||||
included.
|
||||
- **The empty states are a place to act**: an empty folder offers Upload files / New folder, and an
|
||||
empty photo timeline offers Upload photos / Import from Google Photos, instead of describing what
|
||||
the user could go and do elsewhere.
|
||||
- **Ctrl+F works from any page**, navigating to Files and focusing the search box, rather than being
|
||||
silently inert everywhere but Files.
|
||||
- Staged Takeout archives survive closing the window; archives that have since been moved or deleted
|
||||
are dropped from the restored set.
|
||||
|
||||
### Fixed
|
||||
- Settings showed the local cache as "X of 0 B used" when no cache budget was set. An unset budget
|
||||
means *unlimited*, so it now reads "X cached — no limit set" rather than implying a zero cap.
|
||||
- The cache-budget editor fired a `SetCacheBudget` round-trip — and a toast — on every step, so a
|
||||
run of `+` clicks applied a series of caps the user never asked for and stacked a column of
|
||||
toasts. It now waits for the value to settle.
|
||||
- A long pin list pushed everything below it off the end of Settings. It shows the first six with a
|
||||
"Show all N" row for the rest.
|
||||
|
||||
## [1.8.2] — 2026-08-17
|
||||
|
||||
Closes the two findings 1.8.1's live verification produced (B85, B86) and finishes the data-safety
|
||||
audit entries that had been sitting as "fixed in code, verification pending" since 1.0 (B49–B55).
|
||||
No schema change. Schema: **28**.
|
||||
|
||||
The read path no longer assumes a revision is a series of 4 MiB blocks; it takes the geometry from
|
||||
the revision. The B85 investigation also **contradicted its own bug report** — see Notes, because
|
||||
the conclusion recorded there was that reads of such files were losing data, and they were not.
|
||||
|
||||
### Fixed
|
||||
- A folder switched from mirror to on-demand now shows the files the mirror pass itself uploaded.
|
||||
The sync engine created nodes on the remote without invalidating the parent's cached child
|
||||
listing, and that listing is a database flag rather than an in-memory cache — so it survived a
|
||||
daemon restart, and the files stayed invisible until the folder was switched back. Every remote
|
||||
mutation the engine makes (folder create, file upload, and the three trash paths) now drops the
|
||||
parent's cached listing by uid, which reaches a folder that no mount has resident — the case a
|
||||
mirror folder always is (`docs/BUGS.md` B86).
|
||||
- `pdfs ls` and `pdfs refresh` accept a path under any mount the daemon owns, not only the primary
|
||||
one. Both previously answered "is not under the mountpoint" for anything inside a secondary
|
||||
on-demand mount, which left a folder serving a stale listing with no manual way to re-enumerate
|
||||
it. A path under nothing this daemon has mounted still gets the same error as before.
|
||||
- A file whose blocks are not 4 MiB is no longer read twice over. Block offsets came from
|
||||
`index * 4 MiB`, so on a revision laid out differently every block the client asked for
|
||||
straddled two of the revision's own, and each was fetched and decrypted to answer for one — then
|
||||
cached at boundaries the next read would not ask about. Offsets now come from the revision's
|
||||
recorded block sizes, and the on-disk cache, the in-memory ring and the in-flight fetch key
|
||||
address a block by the byte range it covers rather than by its index (`docs/BUGS.md` B85).
|
||||
|
||||
### Changed
|
||||
- The block cache records each block's plaintext start alongside its `(mtime, size)` tag, and each
|
||||
revision's block table in a small sidecar beside its blocks. Blocks cached by an earlier release
|
||||
carry no start; they are honoured wherever the old 4 MiB assumption and the new plan agree on
|
||||
the offset, which is every file this client uploaded — so the existing cache survives the
|
||||
upgrade intact and is quietly refetched only for the files it was placing wrongly.
|
||||
|
||||
### Notes
|
||||
- **B85's diagnosis was wrong, and the entry has been corrected rather than closed.** It read a
|
||||
set of journal lines as proof that this client was requesting wrong byte ranges and being saved
|
||||
only by B84's repair path. `RevisionReader::read_at` does not work that way: it plans over the
|
||||
revision's real block sizes and clamps every block to the range asked for, identically in the
|
||||
0.6.0 the client builds against and in every earlier version. A read can therefore never return
|
||||
more bytes than it requested — yet every quoted line shows exactly that (`blen=4194304
|
||||
got=4939506`), and the one place in the client that logs that shape can only reach it on the
|
||||
opposite condition. The numbers were either transcribed with two fields swapped or captured from
|
||||
a build not in the history; they want re-capturing from the journal before anything is concluded
|
||||
from them. What was real is the assumption itself, and that is what this release removes — a
|
||||
cost fix, not a correctness one. Nothing here changes what a read returns.
|
||||
- The first read of a file still plans on the 4 MiB assumption, and learns the real geometry as a
|
||||
side effect of opening the reader it needs anyway. Resolving a reader merely to *plan* would
|
||||
reintroduce the per-read key derivation the block cache exists to avoid, so the first read of a
|
||||
non-4-MiB file pays one straddling fetch and then self-corrects.
|
||||
- Seven data-safety audit entries (B49–B55) moved from "fixed in code, verification pending" to
|
||||
verified. The fixes were all genuinely present and needed no change; what they lacked was the
|
||||
injected-failure testing each entry asked for, and that is what was written — unwritable
|
||||
directories and `chmod 000` subtrees for permission failures, a real `/dev/shm`↔`/tmp` device
|
||||
boundary for the cross-filesystem fallbacks, SQLite's `max_page_count` for a full disk, an abort
|
||||
trigger for a failed insert, and truncated and unpublished sidecars for the crash boundaries.
|
||||
Two seams were added to make failure arrangeable at all: `stage_write_at`, mirroring the
|
||||
`preserve_write_at` that already existed, and `walk_local_tree` lifted out of the sync engine so
|
||||
the local scan can be driven against a real tree without a daemon.
|
||||
- **Two of those entries are not fully closed, and say so.** B51 and B52 concern crash consistency,
|
||||
and nothing short of a machine actually losing power tests that; the crash *boundaries* are
|
||||
covered by reconstructing on disk exactly what a crash at each point leaves. B54 also diverges
|
||||
from its required fix on purpose: it has no "deliberate delete" confirmation path, so a total
|
||||
local wipe is refused outright rather than offered for confirmation. That is stricter than asked
|
||||
for, and the cost is that a user who really did delete everything must remove and re-add the
|
||||
sync folder.
|
||||
- B86's fix routes only `ls` and `refresh` to the owning mount. The other path-addressed control
|
||||
requests — pin, rename, move, delete, sharing — still resolve against the primary mount only.
|
||||
That is a larger change and was not what blocked the repro; it is recorded as an open follow-up
|
||||
(`docs/BUGS.md` B88).
|
||||
- Three findings were opened rather than fixed: the geometry above is learned one read late, so a
|
||||
non-4-MiB file pays a single wasted fetch on its first read (B87); the remaining path-addressed
|
||||
control requests are still primary-only (B88); and the container-level `CLAUDE.md` still
|
||||
describes the SDK dependency as 0.1.11 when 1.7.1 bumped it to 0.6.0, which is what sent the
|
||||
B85 investigation to the wrong source tree to begin with (B89).
|
||||
|
||||
## [1.8.1] — 2026-08-16
|
||||
|
||||
Some files read short through the mount and reported success — an application got a truncated
|
||||
@@ -77,6 +217,19 @@ Schema: **28** — migration V28 adds `sync_entry.local_mtime_ns`.
|
||||
and the refusals for symlinks, hard links, device nodes and `chmod` are deliberate — Drive
|
||||
cannot represent them, and a local-only representation no other Proton client can read would
|
||||
be worse than a clean refusal (B31).
|
||||
- Six of this release's fixes were then driven live against the installed 1.8.1 on a real
|
||||
account: the same-second edit uploads (B25), three rapid rewrites of a 64 MiB file publish the
|
||||
third one byte for byte (B32), a write racing a download survives as a conflict copy (B39), an
|
||||
upload torn by a concurrent write heals itself on the following pass (B40), a `getdents64`
|
||||
walk with a 160-byte buffer against a directory mutated between pages returns every entry
|
||||
exactly once (B43), and a restart of a daemon with six mounts and a busy drain stops in 110 ms
|
||||
and comes back at the same thread count (B44). B42 could not be driven from a single machine —
|
||||
see B86 below.
|
||||
- One new bug came out of the verification: an on-demand mount serves a folder listing that
|
||||
predates the sync engine's own uploads, so files the mirror pass created are invisible until
|
||||
the folder is switched back (`docs/BUGS.md` B86). Nothing is lost — they are on the server and
|
||||
come back on the next mirror pass — but they cannot be seen or named while the folder is
|
||||
on-demand, which is also what blocks B42's live verification.
|
||||
|
||||
## [1.8.0] — 2026-08-16
|
||||
|
||||
@@ -348,6 +501,9 @@ First stable release: FUSE files-on-demand mount, sync daemon under `proton-driv
|
||||
- The outstanding FUSE defects tracked in `docs/BUGS.md`, plus a truncate defect, validated
|
||||
by a new POSIX compliance suite for the filesystem.
|
||||
|
||||
[1.8.2]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.2
|
||||
[1.8.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.1
|
||||
[1.8.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.8.0
|
||||
[1.7.1]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.7.1
|
||||
[1.7.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.7.0
|
||||
[1.6.0]: https://github.com/narrrl/proton-drive-linux/releases/tag/v1.6.0
|
||||
|
||||
+2
-1
@@ -14,7 +14,8 @@ Consolidated from the historical feature roadmap; current bug status lives in [`
|
||||
* **Custom FUSE Mount Options (Horizon 1)**: Expose custom mounting parameters in configuration (e.g. `ro` for read-only, `direct_io` to bypass kernel caching, `allow_other`). Auto-generate systemd user mount units upon sync folder configuration.
|
||||
* **Symbolic Link & Hard Link Virtualization (Horizon 2)**: Add symbolic link virtualization inside the metadata store (`NodeType::Symlink`) by intercepting `readlink(2)` and `symlink(2)`. Targets should sync remotely as small encrypted metadata placeholder files on Drive (`proton-vfs-symlink: <target>`).
|
||||
* **Desktop File Manager Integration (Horizon 2)**: Develop shell extension plugins for popular Linux file managers (Nautilus Python plugin, Dolphin Service Menus, Thunar custom actions) to offer context-menu shortcuts ("Pin", "Unpin", "Copy Share Link", "View Version History") and file overlay sync badges.
|
||||
* **Integrated Photo Gallery Enhancements (Horizon 2)**: Support creating and editing Proton Photo Albums (album *writes* are unported in the SDK too) and create an auto-upload Pictures folder pipeline. Exif display, the date-based timeline scrubber and favourites are implemented.
|
||||
* **Integrated Photo Gallery Enhancements (Horizon 2)**: Album *editing* — rename, cover photo, delete, and removing photos from an album — is still unported in the SDK (TS `updateAlbum` / `deleteAlbum` / `removePhotos`), as is adding a photo that lives on someone else's volume (TS `copyPhoto`). Album **creation** and adding own-volume photos landed with the Google Photos import. Still open: an auto-upload Pictures folder pipeline, and a GUI for creating albums by hand. Exif display, the date-based timeline scrubber and favourites are implemented.
|
||||
* **Google Photos import follow-ups**: the importer uploads photos one at a time — a large export would finish sooner with a bounded concurrent upload pool. Trash folders are recognized by a hardcoded list of localized names ([`takeout.rs`](../crates/pdfs-core/src/takeout.rs) `TRASH_FOLDERS`); a locale not on it imports its trash as ordinary photos. Motion-photo `.MP`/`.MV` sidecar videos are ignored rather than uploaded as related photos. Live validation against a real export is pending.
|
||||
* **Random-Access Media Streaming seek support (Horizon 2)**: Intercept out-of-order sparse reads (player seeks) in `pdfs-fuse` and prioritize downloading blocks around the seek offset, aborting/postponing sequential pre-fetches.
|
||||
* **Dynamic Sync Dashboard & Queue Visualization (Horizon 2)**: GUI real-time transfer list (progress bars, speed, ETA), sync history feed, storage usage breakdown, and global pause/resume sync controls.
|
||||
* **Interactive Terminal UI (TUI) Mode (Horizon 3)**: Implement `pdfs tui` using the `ratatui` crate to show transfer speed, sync status, active transfer queue with progress bars, and scrolling daemon logs.
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
# Build + install: cd packaging && makepkg -fi
|
||||
|
||||
pkgname=proton-drive-linux
|
||||
pkgver=1.8.1
|
||||
pkgver=1.9.0
|
||||
pkgrel=1
|
||||
pkgdesc="Proton Drive client for Linux: FUSE files-on-demand mount, CLI, GTK4 app + tray"
|
||||
arch=('x86_64')
|
||||
|
||||
Reference in new issue
Block a user