diff --git a/Cargo.lock b/Cargo.lock index 5486f06..b928753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,7 +187,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -198,7 +198,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1568,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3223,7 +3223,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3566,10 +3566,12 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha1 0.10.7", "sha2 0.10.9", "thiserror 2.0.19", "tokio", "tracing", + "zip", ] [[package]] @@ -3585,8 +3587,10 @@ dependencies = [ "pdfs-core", "proton-drive-rs", "serde_json", + "sha1 0.10.7", "tokio", "tracing", + "zip", ] [[package]] @@ -3852,9 +3856,7 @@ dependencies = [ [[package]] name = "proton-drive-rs" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0e39da5b61ceb862af2f4406870b9c1d9a1554889e8d40a32fff13882ec690" +version = "0.6.1" dependencies = [ "async-trait", "base64", @@ -3879,9 +3881,7 @@ dependencies = [ [[package]] name = "proton-sdk" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cde8cf8e3099d1dbb56302462a632a12e0ff818c012b9c9b0a7a5200544fd6" +version = "0.6.1" dependencies = [ "aes-gcm 0.11.0", "async-trait", @@ -3999,7 +3999,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4424,7 +4424,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4481,7 +4481,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4833,7 +4833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4982,10 +4982,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5313,6 +5313,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -5327,7 +5333,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5652,7 +5658,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -6254,6 +6260,20 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.6" @@ -6266,6 +6286,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index cb0d1e8..1713c6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } directories = "5" ignore = "0.4" sha2 = "0.10" +# Google Takeout exports are zip sets; the importer reads their central +# directories and streams entries out without extracting to disk first. +zip = { version = "8", default-features = false, features = ["deflate"] } +# Proton's photo duplicate detection is keyed on the plaintext SHA-1, so an +# import has to speak SHA-1 whatever its age. +sha1 = "0.10" # Thumbnail generation for photos the server has none for. Decoders only for the # formats a camera or phone actually produces — the rest of the crate's codecs # would be dead weight in the daemon. @@ -44,3 +50,10 @@ async-trait = "0.1" futures = "0.3" fuser = "0.17" libc = "0.2" + +# LOCAL ONLY — do not commit. The Takeout import needs album writes and batch +# duplicate detection, which are in the working tree of proton-sdk-rs but not in +# a published 0.6.x. Releases build against the registry versions. +[patch.crates-io] +proton-sdk = { path = "../proton-sdk-rs/crates/proton-sdk" } +proton-drive-rs = { path = "../proton-sdk-rs/crates/proton-drive-rs" } diff --git a/crates/pdfs-cli/src/main.rs b/crates/pdfs-cli/src/main.rs index 8ea4a50..9e6029a 100644 --- a/crates/pdfs-cli/src/main.rs +++ b/crates/pdfs-cli/src/main.rs @@ -158,6 +158,28 @@ enum Command { }, /// List the photo albums (including ones shared with this account). Albums, + /// Import a Google Photos Takeout export into Proton Photos. + /// + /// Pass every zip of the export at once — Google splits one export across + /// numbered parts, and a photo's metadata often sits in a different part + /// than the photo. Albums in the export are recreated as Proton albums, and + /// photos already on the account (same name and same content) are skipped, + /// so an interrupted import can simply be run again. + ImportGooglePhotos { + /// The export's `.zip` files (e.g. `takeout-*.zip`). + #[arg(required = true)] + archives: Vec, + /// Report what would be imported without uploading anything. + #[arg(long)] + dry_run: bool, + /// Stay attached and print progress until the import finishes. + #[arg(long)] + wait: bool, + }, + /// Show how the running (or last) Google Photos import is doing. + ImportStatus, + /// Stop the running Google Photos import. + CancelImport, /// List the photos in one album (newest capture first). Album { /// Album node uid in `volume~link` form (from `pdfs albums`). @@ -589,6 +611,13 @@ fn main() -> Result<()> { } => cmd_photos(limit, offset, favorites), Command::Favorite { uid, remove } => cmd_favorite(uid, remove), Command::Albums => cmd_albums(), + Command::ImportGooglePhotos { + archives, + dry_run, + wait, + } => cmd_import_google_photos(archives, dry_run, wait), + Command::ImportStatus => cmd_import_status(), + Command::CancelImport => cmd_cancel_import(), Command::Album { uid, limit, offset } => cmd_album(uid, limit, offset), Command::OpenPhoto { uid } => cmd_open_photo(uid), Command::Search { query, limit } => cmd_search(query, limit), @@ -1711,6 +1740,104 @@ fn cmd_albums() -> Result<()> { Ok(()) } +fn cmd_import_google_photos(archives: Vec, dry_run: bool, wait: bool) -> Result<()> { + // The daemon opens the archives itself off the shared filesystem, so it gets + // absolute paths — the same arrangement `pdfs upload` uses. + let mut absolute = Vec::with_capacity(archives.len()); + for archive in &archives { + let path = std::fs::canonicalize(archive) + .with_context(|| format!("resolve {}", archive.display()))?; + absolute.push( + path.to_str() + .ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))? + .to_owned(), + ); + } + + match control_request(CtlRequest::ImportTakeout { + archives: absolute, + dry_run, + })? { + CtlResponse::Ok { message } => println!("{message}"), + CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), + other => bail!("unexpected response: {other:?}"), + } + if !wait { + println!("Run `pdfs import-status` to follow it."); + return Ok(()); + } + + // Poll rather than stream: the control protocol is request/response, and an + // import runs for hours — a held-open socket would be the only long-lived + // connection the daemon has. + loop { + std::thread::sleep(std::time::Duration::from_secs(2)); + match control_request(CtlRequest::ImportStatus)? { + CtlResponse::ImportStatus { + running: true, + summary: _, + } => continue, + CtlResponse::ImportStatus { summary, .. } => { + print_import_summary(summary.as_ref()); + return Ok(()); + } + CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), + other => bail!("unexpected response: {other:?}"), + } + } +} + +fn cmd_import_status() -> Result<()> { + match control_request(CtlRequest::ImportStatus)? { + CtlResponse::ImportStatus { running, summary } => { + if running { + println!("Import running. `pdfs transfers` shows its progress."); + } + print_import_summary(summary.as_ref()); + } + CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), + other => bail!("unexpected response: {other:?}"), + } + Ok(()) +} + +fn cmd_cancel_import() -> Result<()> { + match control_request(CtlRequest::CancelImport)? { + CtlResponse::Ok { message } => println!("{message}"), + CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), + other => bail!("unexpected response: {other:?}"), + } + Ok(()) +} + +fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) { + let Some(summary) = summary else { + println!("No import has run yet."); + return; + }; + println!("Photos in export : {}", summary.found); + println!("Uploaded : {}", summary.uploaded); + println!("Already on Proton: {}", summary.duplicates); + if summary.skipped_trashed > 0 { + println!("Skipped (trash) : {}", summary.skipped_trashed); + } + if summary.albums_created > 0 || summary.album_links > 0 { + println!( + "Albums : {} created, {} photos filed", + summary.albums_created, summary.album_links + ); + } + if summary.failed > 0 { + println!("Failed : {}", summary.failed); + } + if summary.bytes > 0 { + println!("Uploaded bytes : {}", summary.bytes); + } + if summary.cancelled { + println!("Import was cancelled before it finished."); + } +} + fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> { match control_request(CtlRequest::AlbumPhotos { uid, offset, limit })? { CtlResponse::Photos { items, .. } if items.is_empty() => println!("No photos."), diff --git a/crates/pdfs-core/Cargo.toml b/crates/pdfs-core/Cargo.toml index 71cbef4..5eb3c2e 100644 --- a/crates/pdfs-core/Cargo.toml +++ b/crates/pdfs-core/Cargo.toml @@ -18,6 +18,8 @@ tracing.workspace = true directories.workspace = true ignore.workspace = true sha2.workspace = true +sha1.workspace = true +zip.workspace = true rusqlite.workspace = true keyring.workspace = true parking_lot.workspace = true diff --git a/crates/pdfs-core/src/control.rs b/crates/pdfs-core/src/control.rs index f751745..5f1dd3a 100644 --- a/crates/pdfs-core/src/control.rs +++ b/crates/pdfs-core/src/control.rs @@ -103,6 +103,33 @@ pub enum Request { /// re-encrypted for our timeline, which the SDK does not implement yet; the /// daemon reports that as an error rather than silently doing nothing. SetPhotoFavorite { uid: String, favorite: bool }, + /// Import Google Photos **Takeout** archives into Proton Photos. + /// + /// `archives` are absolute paths to the export's `.zip` parts on the + /// daemon's own filesystem — the whole set at once, because a photo and its + /// metadata sidecar routinely land in different parts. Photos already on the + /// account (matched by name *and* content digest) are skipped, so a library + /// partly synced by Proton Photos elsewhere does not import twice, and an + /// interrupted import resumes by simply being run again. + /// + /// With `dry_run` the daemon scans and reports what it *would* do without + /// uploading anything. Otherwise it acks immediately with + /// [`Response::Ok`] and works in the background — an export is hours of + /// upload, far past any socket timeout. Progress shows up in + /// [`Request::GetQueueStatus`] as a job, and the final counts in + /// [`Request::ImportStatus`]. + ImportTakeout { + archives: Vec, + #[serde(default)] + dry_run: bool, + }, + /// How the running (or last finished) Takeout import is doing. Replies with + /// [`Response::ImportStatus`]. + ImportStatus, + /// Ask the running Takeout import to stop. It finishes the photo on the wire + /// and files what it has uploaded into its albums, then reports + /// `cancelled`. Replies with [`Response::Ok`]. + CancelImport, /// Upload the photo at `source_path` under the given name and media type. /// /// A path, not the bytes: this protocol is line-delimited JSON, and @@ -1052,6 +1079,35 @@ pub struct PhotoMonth { pub count: usize, } +/// How a Google Photos Takeout import is going, or how the last one ended +/// ([`Response::ImportStatus`]). +/// +/// The counts are cumulative for the run, so the same shape serves progress and +/// the final report. `found` counts *distinct* photos: Google stores a separate +/// copy of a photo in each album folder it appears in, and those fold into one +/// upload that joins several albums. +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)] +pub struct ImportSummary { + /// Distinct photos the export holds. + pub found: usize, + /// Photos uploaded by this run. + pub uploaded: usize, + /// Photos already on the account, matched by name *and* content digest. + pub duplicates: usize, + /// Photos that could not be read or uploaded. + pub failed: usize, + /// Photos in the export's trash bucket, which are never imported. + pub skipped_trashed: usize, + /// Albums created by this run. + pub albums_created: usize, + /// Album memberships added by this run. + pub album_links: usize, + /// Bytes uploaded. + pub bytes: u64, + /// True when the run stopped early because it was cancelled. + pub cancelled: bool, +} + /// One album in a [`Response::Albums`] listing. /// /// An album is a folder node on the photos volume carrying album properties, so @@ -1270,6 +1326,14 @@ pub enum Response { available: bool, items: Vec, }, + /// How the Takeout import is doing (reply to [`Request::ImportStatus`]). + /// `running` is false once it has finished, when `summary` is the final + /// report; `summary` is `None` when no import has run this session. + ImportStatus { + running: bool, + #[serde(default)] + summary: Option, + }, /// The months the timeline spans (reply to [`Request::PhotoMonths`]), /// newest first. PhotoMonths { months: Vec }, diff --git a/crates/pdfs-core/src/lib.rs b/crates/pdfs-core/src/lib.rs index da71514..8b5a854 100644 --- a/crates/pdfs-core/src/lib.rs +++ b/crates/pdfs-core/src/lib.rs @@ -20,6 +20,7 @@ pub mod search; pub mod service; pub mod shell; pub mod syncignore; +pub mod takeout; pub use access::{Access, access_for, perm_bits}; pub use error::{CoreError, CoreResult, Error, Result}; diff --git a/crates/pdfs-core/src/takeout.rs b/crates/pdfs-core/src/takeout.rs new file mode 100644 index 0000000..9e21bdf --- /dev/null +++ b/crates/pdfs-core/src/takeout.rs @@ -0,0 +1,887 @@ +//! Reading a Google Photos **Takeout** export: what is in the archives, which +//! album each photo belongs to, and when it was taken. +//! +//! This module is pure structure — it opens the archives' central directories +//! and reads the small JSON sidecars, never the media bytes. The import itself +//! (hashing, duplicate detection, upload, album creation) lives in the daemon; +//! everything here is offline-testable and has no Proton dependency. +//! +//! # What a Takeout export looks like +//! +//! Google splits an export into numbered zips (`takeout-20260817T...-001.zip`, +//! `-002.zip`, …) that are *independent* archives — no zip64 spanning — but a +//! photo and its JSON sidecar can land in different parts, and one album's +//! folder can be spread over several. So the scan treats the whole set as one +//! namespace keyed by the path inside the export: +//! +//! ```text +//! Takeout/Google Photos/Iceland 2019/IMG_0042.jpg +//! Takeout/Google Photos/Iceland 2019/IMG_0042.jpg.supplemental-metadata.json +//! Takeout/Google Photos/Iceland 2019/metadata.json <- album title +//! Takeout/Google Photos/Photos from 2019/IMG_0043.jpg <- no album +//! Takeout/Google Photos/Trash/IMG_0044.jpg <- skipped +//! ``` +//! +//! The service folder (`Google Photos`) is localized, as are the year folders +//! (`Photos from 2019`) and the trash folder, so nothing here matches on the +//! service name: the first component is dropped when it is the export root, the +//! second is the service, and the rest is `/`. A folder is an +//! **album** when it carries an album metadata sidecar naming it; anything else +//! is a plain bucket that imports into the timeline only. Trash is matched by +//! name against [`TRASH_FOLDERS`] — the one place a localized name is +//! unavoidable, because a trashed photo is not marked as such anywhere else. + +use std::collections::{BTreeMap, HashMap}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::error::{CoreError, CoreResult}; + +/// Folder names Google uses for the trash bucket, lowercased. A Takeout export +/// only contains one when the user asked for it, and its photos are deleted — +/// importing them would resurrect them, so they are skipped. +/// +/// Localized, and necessarily incomplete: a locale not listed here imports its +/// trash as ordinary timeline photos. Extend as reports come in rather than +/// guessing at transliterations. +pub const TRASH_FOLDERS: &[&str] = &[ + "trash", + "bin", + "papierkorb", + "corbeille", + "papelera", + "papelera de reciclaje", + "cestino", + "lixeira", + "prullenbak", + "kosz", + "koš", + "skräpkorgen", + "papperskorg", + "papirkurv", + "roskakori", + "kuka", + "回收站", + "回收筒", + "ゴミ箱", + "휴지통", + "корзина", + "кошик", + "çöp kutusu", + "σκουπίδια", +]; + +/// Sidecar file names that describe the *folder* (its album title) rather than +/// one photo, lowercased. Google has renamed this file more than once. +const ALBUM_METADATA_NAMES: &[&str] = &[ + "metadata.json", + "album-metadata.json", + "print-subscriptions.json", + "shared_album_comments.json", + "user-generated-memory-titles.json", +]; + +/// The suffix Google appends to a photo's sidecar, lowercased. Truncation means +/// only a *prefix* of it may survive (see [`strip_sidecar_suffix`]). +const SUPPLEMENTAL: &str = ".supplemental-metadata"; + +/// File extensions imported as photos or videos. Anything else in the export — +/// `archive_browser.html`, `.csv` activity logs, `print-subscriptions.json` — +/// is not media and is ignored. +const MEDIA_EXTENSIONS: &[(&str, &str)] = &[ + ("jpg", "image/jpeg"), + ("jpeg", "image/jpeg"), + ("png", "image/png"), + ("gif", "image/gif"), + ("webp", "image/webp"), + ("bmp", "image/bmp"), + ("tif", "image/tiff"), + ("tiff", "image/tiff"), + ("heic", "image/heic"), + ("heif", "image/heif"), + ("avif", "image/avif"), + ("dng", "image/x-adobe-dng"), + ("cr2", "image/x-canon-cr2"), + ("cr3", "image/x-canon-cr3"), + ("nef", "image/x-nikon-nef"), + ("arw", "image/x-sony-arw"), + ("orf", "image/x-olympus-orf"), + ("rw2", "image/x-panasonic-rw2"), + ("raf", "image/x-fuji-raf"), + ("mp4", "video/mp4"), + ("m4v", "video/x-m4v"), + ("mov", "video/quicktime"), + ("avi", "video/x-msvideo"), + ("mkv", "video/x-matroska"), + ("webm", "video/webm"), + ("3gp", "video/3gpp"), + ("mts", "video/mp2t"), + ("m2ts", "video/mp2t"), + ("mpg", "video/mpeg"), + ("mpeg", "video/mpeg"), +]; + +/// The media type for `name`'s extension, or `None` when it is not media this +/// import handles. +pub fn media_type_of(name: &str) -> Option<&'static str> { + let ext = Path::new(name) + .extension() + .and_then(|e| e.to_str())? + .to_ascii_lowercase(); + MEDIA_EXTENSIONS + .iter() + .find(|(candidate, _)| *candidate == ext) + .map(|(_, media_type)| *media_type) +} + +/// Where a photo sits in the export, which decides whether it joins an album. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Bucket { + /// A named album folder — the photo joins that album *and* the timeline. + Album(String), + /// A year folder (`Photos from 2019`) or the export root: timeline only. + Timeline, + /// The archive bucket: timeline only, like Google shows it. + Archived, +} + +/// One media file found in the export. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TakeoutPhoto { + /// Index into the archive list the scan was given. + pub archive: usize, + /// The entry's full path inside that archive — how the importer reopens it. + pub entry: String, + /// The name to upload under: the sidecar's `title` when it has one (Google + /// mangles long names on disk but keeps the original there), else the file + /// name on disk. + pub name: String, + /// Uncompressed size in bytes, from the zip's central directory. + pub size: u64, + /// Media type from the extension. + pub media_type: String, + /// Capture time in epoch seconds, from `photoTakenTime` (falling back to + /// `creationTime`). `None` when the photo has no sidecar — the upload then + /// defaults it to now, as the SDK does. + pub capture_time: Option, + /// The sidecar's `favorited` flag. + pub favorite: bool, + /// Which bucket the file was found in. + pub bucket: Bucket, +} + +/// Everything one scan found, ready to be turned into an import plan. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct TakeoutScan { + /// Every importable media file, in archive then entry order. + pub photos: Vec, + /// Album names in the order first seen, so the importer creates them in a + /// stable order. + pub albums: Vec, + /// Media files skipped because they sit in the trash bucket. + pub skipped_trashed: usize, + /// Entries that were neither media nor a sidecar (`archive_browser.html`, + /// activity `.csv`s, `.mp` motion-photo parts). + pub skipped_other: usize, +} + +impl TakeoutScan { + /// Total bytes of media to be read, before duplicate detection. + pub fn total_bytes(&self) -> u64 { + self.photos.iter().map(|p| p.size).sum() + } +} + +/// Google's per-photo sidecar. Only the fields the import uses are read; the +/// file also carries geo data, people tags and view counts. +#[derive(Debug, Default, Deserialize)] +struct PhotoSidecar { + #[serde(default)] + title: Option, + #[serde(rename = "photoTakenTime", default)] + photo_taken_time: Option, + #[serde(rename = "creationTime", default)] + creation_time: Option, + #[serde(default)] + favorited: bool, +} + +/// A sidecar timestamp: epoch seconds, as a *string*. +#[derive(Debug, Deserialize)] +struct SidecarTime { + #[serde(default)] + timestamp: Option, +} + +impl SidecarTime { + fn epoch_seconds(&self) -> Option { + self.timestamp.as_ref()?.parse().ok() + } +} + +/// A folder-level sidecar. `title` is the album name as the user typed it, +/// which is what the folder name is derived (and sometimes truncated) from. +#[derive(Debug, Deserialize)] +struct AlbumSidecar { + #[serde(default)] + title: Option, +} + +/// One entry as the scan sees it, independent of where it came from. +struct RawEntry { + archive: usize, + entry: String, + /// `/` with the export root and service folder removed. + relative: String, + size: u64, +} + +impl RawEntry { + /// The folder component of `relative`, or `""` at the service root. + fn folder(&self) -> &str { + match self.relative.rfind('/') { + Some(index) => &self.relative[..index], + None => "", + } + } + + /// The file name component of `relative`. + fn file_name(&self) -> &str { + match self.relative.rfind('/') { + Some(index) => &self.relative[index + 1..], + None => &self.relative, + } + } +} + +/// Scan a set of Takeout zip archives into one plan. +/// +/// The archives are treated as one export: an album folder split across parts +/// is one album, and a sidecar in part 3 is matched to its photo in part 1. +/// Reads only the central directory and the JSON sidecars — no media bytes, so +/// this stays fast on a 200 GB export. +pub fn scan_archives(archives: &[PathBuf]) -> CoreResult { + let mut entries: Vec = Vec::new(); + let mut sidecars: HashMap> = HashMap::new(); + + for (index, path) in archives.iter().enumerate() { + let file = std::fs::File::open(path) + .map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?; + let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| { + CoreError::invalid(format!("{} is not a readable zip: {e}", path.display())) + })?; + + for position in 0..zip.len() { + let mut entry = zip.by_index(position).map_err(|e| { + CoreError::invalid(format!("{}: unreadable entry: {e}", path.display())) + })?; + if entry.is_dir() { + continue; + } + // `enclosed_name` refuses paths that escape the archive root, which + // is exactly the check we want before using one as a key. + let Some(name) = entry.enclosed_name() else { + continue; + }; + let name = name.to_string_lossy().replace('\\', "/"); + let Some(relative) = strip_export_prefix(&name) else { + continue; + }; + let size = entry.size(); + + // Sidecars are read now, while the entry is open: they are a few + // hundred bytes each and reopening the archive per photo later would + // be a second full pass. + if relative.to_ascii_lowercase().ends_with(".json") { + let mut body = Vec::with_capacity(size.min(64 * 1024) as usize); + if entry.read_to_end(&mut body).is_ok() { + sidecars.insert(relative.to_string(), body); + } + continue; + } + + entries.push(RawEntry { + archive: index, + entry: name.clone(), + relative: relative.to_string(), + size, + }); + } + } + + Ok(assemble(entries, &sidecars)) +} + +/// Build the scan from raw entries plus the sidecar bodies keyed by their path +/// inside the export. Split out from [`scan_archives`] so the matching rules can +/// be tested without building zip fixtures. +fn assemble(entries: Vec, sidecars: &HashMap>) -> TakeoutScan { + // Album titles per folder, from the folder-level sidecars. + let mut album_titles: HashMap = HashMap::new(); + for (path, body) in sidecars { + let (folder, file) = split_folder(path); + if !ALBUM_METADATA_NAMES.contains(&file.to_ascii_lowercase().as_str()) { + continue; + } + if let Ok(album) = serde_json::from_slice::(body) + && let Some(title) = album.title.filter(|t| !t.trim().is_empty()) + { + album_titles.insert(folder.to_string(), title); + } + } + + // Per-photo sidecars, indexed by folder so a truncated name only ever + // matches within its own folder. + let mut per_folder_sidecars: HashMap>> = HashMap::new(); + for (path, body) in sidecars { + let (folder, file) = split_folder(path); + if ALBUM_METADATA_NAMES.contains(&file.to_ascii_lowercase().as_str()) { + continue; + } + per_folder_sidecars + .entry(folder.to_string()) + .or_default() + .insert(file.to_string(), body); + } + + let mut scan = TakeoutScan::default(); + let mut seen_albums: HashMap = HashMap::new(); + + for raw in &entries { + let folder = raw.folder().to_string(); + let file_name = raw.file_name().to_string(); + + if is_trash_folder(&folder) { + if media_type_of(&file_name).is_some() { + scan.skipped_trashed += 1; + } + continue; + } + let Some(media_type) = media_type_of(&file_name) else { + scan.skipped_other += 1; + continue; + }; + + let sidecar = per_folder_sidecars + .get(&folder) + .and_then(|in_folder| find_sidecar(&file_name, in_folder)) + .and_then(|body| serde_json::from_slice::(body).ok()) + .unwrap_or_default(); + + let bucket = match album_titles.get(&folder) { + Some(title) => Bucket::Album(title.clone()), + None if is_archive_folder(&folder) => Bucket::Archived, + None => Bucket::Timeline, + }; + if let Bucket::Album(title) = &bucket + && seen_albums.insert(title.clone(), ()).is_none() + { + scan.albums.push(title.clone()); + } + + // The sidecar's `title` is the name Google had before it mangled the + // on-disk one (truncation, `(1)` suffixes, character replacement), so it + // is the better name to upload under — but only when it still looks like + // the same file, i.e. the extension agrees. + let name = sidecar + .title + .as_deref() + .map(str::trim) + .filter(|title| !title.is_empty() && same_extension(title, &file_name)) + .unwrap_or(&file_name) + .to_string(); + + scan.photos.push(TakeoutPhoto { + archive: raw.archive, + entry: raw.entry.clone(), + name, + size: raw.size, + media_type: media_type.to_string(), + capture_time: sidecar + .photo_taken_time + .as_ref() + .and_then(SidecarTime::epoch_seconds) + .or_else(|| { + sidecar + .creation_time + .as_ref() + .and_then(SidecarTime::epoch_seconds) + }), + favorite: sidecar.favorited, + bucket, + }); + } + + scan +} + +/// Drop the export root (`Takeout/`) and the localized service folder from a +/// path inside the archive, leaving `/` or ``. +/// +/// `None` for a path with no service folder at all (the export's own +/// `archive_browser.html`), which carries nothing to import. +fn strip_export_prefix(path: &str) -> Option<&str> { + let mut rest = path; + // The root is usually `Takeout/`, but an export unzipped and re-zipped by + // the user may not have it — so the root is dropped only when a service + // folder follows it, which the component count decides. + let components = rest.split('/').count(); + if components >= 3 { + rest = rest.split_once('/')?.1; + } + // Drop the service folder (`Google Photos`, `Google Fotos`, …). + let (_service, tail) = rest.split_once('/')?; + Some(tail) +} + +/// Split `/` into its parts; the folder is `""` at the root. +fn split_folder(path: &str) -> (&str, &str) { + match path.rfind('/') { + Some(index) => (&path[..index], &path[index + 1..]), + None => ("", path), + } +} + +fn is_trash_folder(folder: &str) -> bool { + let name = folder.rsplit('/').next().unwrap_or(folder).to_lowercase(); + TRASH_FOLDERS.contains(&name.as_str()) +} + +/// Google's archive bucket, which is not an album but is not the plain timeline +/// either. Only the English name is matched; a missed locale simply imports as +/// [`Bucket::Timeline`], which is the same upload either way. +fn is_archive_folder(folder: &str) -> bool { + let name = folder.rsplit('/').next().unwrap_or(folder).to_lowercase(); + name == "archive" +} + +/// Whether two file names end in the same extension, case-insensitively. +fn same_extension(a: &str, b: &str) -> bool { + let ext = |name: &str| { + Path::new(name) + .extension() + .and_then(|e| e.to_str()) + .map(str::to_ascii_lowercase) + }; + ext(a) == ext(b) +} + +/// Strip a sidecar's `.json` and any `.supplemental-metadata` suffix, including +/// the *truncated* forms Google produces. +/// +/// Google caps the sidecar's file name at 51 characters, so a long photo name +/// leaves only a prefix of the suffix behind — `IMG_1234.jpg.supplemental-me`, +/// `…-metad`, or nothing at all. Any non-empty prefix of the suffix is stripped; +/// a bare `.json` sidecar loses only that. +fn strip_sidecar_suffix(sidecar: &str) -> String { + let stem = sidecar + .strip_suffix(".json") + .or_else(|| sidecar.strip_suffix(".JSON")) + .unwrap_or(sidecar); + // Longest match first so `.supplemental-metadata` wins over `.s`. + for length in (2..=SUPPLEMENTAL.len()).rev() { + let candidate = &SUPPLEMENTAL[..length]; + if let Some(head) = stem.strip_suffix(candidate) { + return head.to_string(); + } + } + stem.to_string() +} + +/// Move a trailing `(N)` from after the extension to before it: Google writes +/// the sidecar of `IMG_1234(1).jpg` as `IMG_1234.jpg(1).json`. +fn unswap_duplicate_marker(stem: &str) -> Option { + let marker_start = stem.rfind('(')?; + let marker = &stem[marker_start..]; + if !marker.ends_with(')') + || !marker[1..marker.len() - 1] + .chars() + .all(|c| c.is_ascii_digit()) + { + return None; + } + let head = &stem[..marker_start]; + let dot = head.rfind('.')?; + Some(format!("{}{}{}", &head[..dot], marker, &head[dot..])) +} + +/// Find the sidecar belonging to `file_name` among the JSON files of its folder. +/// +/// Tried in order of confidence: the exact name, the `(N)`-swapped name, then a +/// prefix match for the case where truncation cut into the photo name itself. +fn find_sidecar<'a>( + file_name: &str, + in_folder: &BTreeMap>, +) -> Option<&'a Vec> { + let mut prefix_match: Option<&'a Vec> = None; + let mut prefix_len = 0usize; + + for (sidecar_name, body) in in_folder { + let stem = strip_sidecar_suffix(sidecar_name); + if stem == file_name { + return Some(body); + } + if unswap_duplicate_marker(&stem).as_deref() == Some(file_name) { + return Some(body); + } + // Truncated: the stem is a prefix of the photo's name. Keep the longest + // such prefix — a folder holding `IMG_1.jpg` and `IMG_12.jpg` must not + // let the shorter stem claim the longer photo. + if !stem.is_empty() && file_name.starts_with(&stem) && stem.len() > prefix_len { + prefix_len = stem.len(); + prefix_match = Some(body); + } + } + prefix_match +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(archive: usize, path: &str, size: u64) -> RawEntry { + RawEntry { + archive, + entry: path.to_string(), + relative: strip_export_prefix(path).unwrap().to_string(), + size, + } + } + + fn sidecar(map: &mut HashMap>, path: &str, body: &str) { + let relative = strip_export_prefix(path).unwrap().to_string(); + map.insert(relative, body.as_bytes().to_vec()); + } + + #[test] + fn strips_export_root_and_localized_service_folder() { + assert_eq!( + strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"), + Some("Island 2019/IMG_1.jpg") + ); + // Already unwrapped by the user: no `Takeout/` root left. + assert_eq!( + strip_export_prefix("Google Photos/IMG_1.jpg"), + Some("IMG_1.jpg") + ); + // Nothing below a service folder is nothing to import. + assert_eq!(strip_export_prefix("archive_browser.html"), None); + } + + #[test] + fn album_folder_becomes_an_album_year_folder_does_not() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Iceland 2019/metadata.json", + r#"{"title":"Iceland 2019"}"#, + ); + let scan = assemble( + vec![ + entry(0, "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", 10), + entry(0, "Takeout/Google Photos/Photos from 2019/IMG_2.jpg", 20), + ], + &sidecars, + ); + + assert_eq!(scan.albums, vec!["Iceland 2019"]); + assert_eq!(scan.photos[0].bucket, Bucket::Album("Iceland 2019".into())); + assert_eq!(scan.photos[1].bucket, Bucket::Timeline); + assert_eq!(scan.total_bytes(), 30); + } + + #[test] + fn trashed_photos_are_skipped_and_counted() { + let scan = assemble( + vec![ + entry(0, "Takeout/Google Photos/Papierkorb/IMG_1.jpg", 10), + entry(0, "Takeout/Google Photos/Bin/IMG_2.jpg", 10), + entry(0, "Takeout/Google Photos/Photos from 2019/IMG_3.jpg", 10), + ], + &HashMap::new(), + ); + + assert_eq!(scan.skipped_trashed, 2); + assert_eq!(scan.photos.len(), 1); + assert_eq!(scan.photos[0].name, "IMG_3.jpg"); + } + + #[test] + fn non_media_entries_are_ignored() { + let scan = assemble( + vec![ + entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10), + entry(0, "Takeout/Google Photos/Photos from 2019/notes.txt", 10), + ], + &HashMap::new(), + ); + + assert_eq!(scan.photos.len(), 1); + assert_eq!(scan.skipped_other, 1); + } + + #[test] + fn sidecar_supplies_capture_time_and_favourite() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_1.jpg.supplemental-metadata.json", + r#"{"title":"IMG_1.jpg","photoTakenTime":{"timestamp":"1560000000"},"favorited":true}"#, + ); + let scan = assemble( + vec![entry( + 0, + "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", + 10, + )], + &sidecars, + ); + + assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000)); + assert!(scan.photos[0].favorite); + } + + #[test] + fn creation_time_is_the_fallback_capture_time() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json", + r#"{"creationTime":{"timestamp":"1500000000"}}"#, + ); + let scan = assemble( + vec![entry( + 0, + "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", + 10, + )], + &sidecars, + ); + + assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000)); + } + + #[test] + fn truncated_supplemental_suffixes_still_match() { + // Google caps the sidecar name at 51 chars, cutting into the suffix. + assert_eq!( + strip_sidecar_suffix("IMG_1.jpg.supplemental-me.json"), + "IMG_1.jpg" + ); + assert_eq!( + strip_sidecar_suffix("IMG_1.jpg.supplemental-metadata.json"), + "IMG_1.jpg" + ); + assert_eq!(strip_sidecar_suffix("IMG_1.jpg.json"), "IMG_1.jpg"); + } + + #[test] + fn duplicate_marker_after_the_extension_is_moved_back() { + assert_eq!( + unswap_duplicate_marker("IMG_1.jpg(1)").as_deref(), + Some("IMG_1(1).jpg") + ); + assert_eq!(unswap_duplicate_marker("IMG_1.jpg"), None); + + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_1.jpg(1).json", + r#"{"photoTakenTime":{"timestamp":"1600000000"}}"#, + ); + let scan = assemble( + vec![entry( + 0, + "Takeout/Google Photos/Photos from 2019/IMG_1(1).jpg", + 10, + )], + &sidecars, + ); + + assert_eq!(scan.photos[0].capture_time, Some(1_600_000_000)); + } + + #[test] + fn longest_prefix_wins_when_the_photo_name_itself_was_truncated() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_1.json", + r#"{"photoTakenTime":{"timestamp":"1"}}"#, + ); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_12.json", + r#"{"photoTakenTime":{"timestamp":"2"}}"#, + ); + let scan = assemble( + vec![entry( + 0, + "Takeout/Google Photos/Photos from 2019/IMG_12345.jpg", + 10, + )], + &sidecars, + ); + + assert_eq!(scan.photos[0].capture_time, Some(2)); + } + + #[test] + fn sidecar_title_replaces_a_mangled_on_disk_name() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/a_very_long_name_that_google_.jpg.json", + r#"{"title":"a_very_long_name_that_google_truncated.jpg"}"#, + ); + let scan = assemble( + vec![entry( + 0, + "Takeout/Google Photos/Photos from 2019/a_very_long_name_that_google_.jpg", + 10, + )], + &sidecars, + ); + + assert_eq!( + scan.photos[0].name, + "a_very_long_name_that_google_truncated.jpg" + ); + } + + #[test] + fn a_title_with_a_different_extension_is_not_trusted() { + // Motion photos name their `.MP` part after the `.jpg`; taking the title + // verbatim would upload the video under an image name. + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_1.mp4.json", + r#"{"title":"IMG_1.jpg"}"#, + ); + let scan = assemble( + vec![entry( + 0, + "Takeout/Google Photos/Photos from 2019/IMG_1.mp4", + 10, + )], + &sidecars, + ); + + assert_eq!(scan.photos[0].name, "IMG_1.mp4"); + assert_eq!(scan.photos[0].media_type, "video/mp4"); + } + + #[test] + fn an_album_split_across_archives_stays_one_album() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Iceland 2019/metadata.json", + r#"{"title":"Iceland 2019"}"#, + ); + let scan = assemble( + vec![ + entry(0, "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", 10), + entry(1, "Takeout/Google Photos/Iceland 2019/IMG_2.jpg", 10), + ], + &sidecars, + ); + + assert_eq!(scan.albums, vec!["Iceland 2019"]); + assert_eq!(scan.photos.len(), 2); + assert_eq!(scan.photos[1].archive, 1); + } + + #[test] + fn archive_bucket_imports_to_the_timeline() { + let scan = assemble( + vec![entry(0, "Takeout/Google Photos/Archive/IMG_1.jpg", 10)], + &HashMap::new(), + ); + + assert_eq!(scan.photos[0].bucket, Bucket::Archived); + assert!(scan.albums.is_empty()); + } + + /// Write a zip holding `files` and return its path. No `tempfile` + /// dev-dependency: the daemon crates deliberately carry none, and one + /// uniquely named file under the temp dir is all this needs. + fn write_zip(tag: &str, files: &[(&str, &[u8])]) -> PathBuf { + use std::io::Write; + let path = std::env::temp_dir().join(format!( + "pdfs-takeout-test-{tag}-{}.zip", + std::process::id() + )); + let file = std::fs::File::create(&path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let options: zip::write::FileOptions<'_, ()> = + zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); + for (name, body) in files { + zip.start_file(*name, options).unwrap(); + zip.write_all(body).unwrap(); + } + zip.finish().unwrap(); + path + } + + #[test] + fn scans_a_two_part_export_where_the_sidecar_is_in_the_other_part() { + // The case that makes the whole set one scan: part 1 has the photo, + // part 2 has its metadata and the album title. + let first = write_zip( + "part1", + &[( + "Takeout/Google Photos/Iceland 2019/IMG_1.jpg", + b"the photo bytes", + )], + ); + let second = write_zip( + "part2", + &[ + ( + "Takeout/Google Photos/Iceland 2019/metadata.json", + br#"{"title":"Iceland 2019"}"#.as_slice(), + ), + ( + "Takeout/Google Photos/Iceland 2019/IMG_1.jpg.supplemental-metadata.json", + br#"{"title":"IMG_1.jpg","photoTakenTime":{"timestamp":"1560000000"}}"# + .as_slice(), + ), + ( + "Takeout/Google Photos/Trash/IMG_9.jpg", + b"deleted".as_slice(), + ), + ], + ); + + let scan = scan_archives(&[first.clone(), second.clone()]).unwrap(); + let _ = std::fs::remove_file(&first); + let _ = std::fs::remove_file(&second); + + assert_eq!(scan.photos.len(), 1); + assert_eq!(scan.photos[0].name, "IMG_1.jpg"); + assert_eq!(scan.photos[0].archive, 0, "the photo is in part 1"); + assert_eq!(scan.photos[0].size, "the photo bytes".len() as u64); + assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000)); + assert_eq!(scan.photos[0].bucket, Bucket::Album("Iceland 2019".into())); + assert_eq!(scan.albums, vec!["Iceland 2019"]); + assert_eq!(scan.skipped_trashed, 1); + } + + #[test] + fn a_file_that_is_not_a_zip_is_a_readable_error() { + let path = + std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id())); + std::fs::write(&path, b"not a zip at all").unwrap(); + let error = scan_archives(std::slice::from_ref(&path)).unwrap_err(); + let _ = std::fs::remove_file(&path); + assert!( + error.message.contains("not a readable zip"), + "unexpected message: {}", + error.message + ); + } +} diff --git a/crates/pdfs-fuse/Cargo.toml b/crates/pdfs-fuse/Cargo.toml index 83c8d62..f4564d7 100644 --- a/crates/pdfs-fuse/Cargo.toml +++ b/crates/pdfs-fuse/Cargo.toml @@ -9,6 +9,8 @@ description = "FUSE on-demand filesystem for Proton Drive" [dependencies] pdfs-core.workspace = true +zip.workspace = true +sha1.workspace = true proton-drive-rs.workspace = true futures.workspace = true tokio.workspace = true diff --git a/crates/pdfs-fuse/src/control.rs b/crates/pdfs-fuse/src/control.rs index 93542c0..3425a69 100644 --- a/crates/pdfs-fuse/src/control.rs +++ b/crates/pdfs-fuse/src/control.rs @@ -329,6 +329,51 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U }, None => CtlResponse::error(CoreError::invalid(format!("bad uid: {uid}"))), }, + // The scan alone walks every archive's central directory, so even the + // dry run goes to a background thread and is read back through + // `ImportStatus` rather than answered inline. + Ok(CtlRequest::ImportTakeout { archives, dry_run }) => { + if archives.is_empty() { + CtlResponse::error(CoreError::invalid("no Takeout archives given")) + } else { + let paths: Vec = archives.into_iter().map(PathBuf::from).collect(); + let missing: Vec = paths + .iter() + .filter(|path| !path.is_file()) + .map(|path| path.display().to_string()) + .collect(); + if !missing.is_empty() { + CtlResponse::error(CoreError::not_found(format!( + "cannot read: {}", + missing.join(", ") + ))) + } else { + let count = paths.len(); + match core.spawn_takeout_import(paths, dry_run) { + Ok(()) => CtlResponse::Ok { + message: format!( + "{} {count} archive(s)", + if dry_run { "scanning" } else { "importing" } + ), + }, + Err(e) => CtlResponse::error(e), + } + } + } + } + Ok(CtlRequest::ImportStatus) => { + let (running, summary) = crate::takeout::import_status(); + CtlResponse::ImportStatus { running, summary } + } + Ok(CtlRequest::CancelImport) => { + if crate::takeout::cancel_import() { + CtlResponse::Ok { + message: "stopping the import".to_string(), + } + } else { + CtlResponse::error(CoreError::invalid("no import is running")) + } + } Ok(CtlRequest::UploadPhoto { name, media_type, diff --git a/crates/pdfs-fuse/src/lib.rs b/crates/pdfs-fuse/src/lib.rs index 4799fa9..8d6c580 100644 --- a/crates/pdfs-fuse/src/lib.rs +++ b/crates/pdfs-fuse/src/lib.rs @@ -96,6 +96,7 @@ mod shutdown; mod state; mod sweep; mod sync; +mod takeout; mod transfers; mod upload; mod r#virtual; diff --git a/crates/pdfs-fuse/src/takeout.rs b/crates/pdfs-fuse/src/takeout.rs new file mode 100644 index 0000000..2d99c9f --- /dev/null +++ b/crates/pdfs-fuse/src/takeout.rs @@ -0,0 +1,632 @@ +//! Importing a Google Photos Takeout export into Proton Photos. +//! +//! [`pdfs_core::takeout`] says what is in the archives; this is the part that +//! talks to Proton. One import runs on its own thread and moves through four +//! phases, reporting into the transfer registry as a job so the GUI's transfer +//! list shows it without a bespoke widget: +//! +//! 1. **Scan** — read the archives' central directories and sidecars. +//! 2. **Deduplicate against the account** — ask the photos volume which of the +//! names already exist ([`ProtonPhotosClient::name_collisions`]), hash only +//! those files, then confirm with the content digest. A library that is +//! already half-synced by Proton Photos on a phone therefore costs one cheap +//! request per 150 names and reads no media at all for the misses. +//! 3. **Upload** the survivors, streaming each entry straight out of its zip — +//! no extraction to disk, so the import needs no scratch space. +//! 4. **Albums** — create the albums the export names that the account lacks, +//! then add every photo of each album, the ones that were already on the +//! account included. +//! +//! Re-running an interrupted import is the resume story: phase 2 finds +//! everything the previous run uploaded and skips it. Nothing is persisted +//! between runs. + +use std::collections::{BTreeMap, HashMap}; +use std::io::Read; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use pdfs_core::control::{ActivityKind, ImportSummary, TransferDirection}; +use pdfs_core::takeout::{Bucket, TakeoutPhoto, TakeoutScan, scan_archives}; +use pdfs_core::{CoreError, CoreResult}; +use proton_drive_rs::proton_sdk::ids::NodeUid; +use proton_drive_rs::{PhotoTag, PhotoTagsUpdate, PhotoUploadMetadata}; +use sha1::{Digest, Sha1}; +use tracing::{info, warn}; + +use super::Core; +use crate::transfers::CountingReader; + +/// The one import this daemon may be running, and how the last one ended. +/// +/// Process-global rather than a [`Core`] field because it must be *one* import +/// per daemon however many `Core` clones exist (the forked on-demand mounts each +/// hold one), and because a front-end asking for the result after the import +/// finished must still find it. +static IMPORT: std::sync::LazyLock> = + std::sync::LazyLock::new(|| parking_lot::Mutex::new(ImportState::default())); + +#[derive(Default)] +struct ImportState { + running: bool, + cancel: Option>, + last: Option, +} + +/// Whether an import is running, and the last one's report. +pub(crate) fn import_status() -> (bool, Option) { + let state = IMPORT.lock(); + (state.running, state.last.clone()) +} + +/// Ask the running import to stop. False when none is running. +pub(crate) fn cancel_import() -> bool { + match &IMPORT.lock().cancel { + Some(flag) => { + flag.store(true, Ordering::Relaxed); + true + } + None => false, + } +} + +/// One photo to import, after the copies Google keeps per album are folded into +/// a single upload that joins several albums. +struct Planned { + photo: TakeoutPhoto, + /// Every album this photo belongs to, by export title. + albums: Vec, + /// Filled in phase 2/3: where the photo ended up on the account. + uid: Option, +} + +/// Fold the scan into one entry per distinct photo. +/// +/// Google stores a full copy of a photo in every album folder it appears in, so +/// the same file is in the export two or three times over. They are folded on +/// `(name, size)`: a byte-identical copy under the same name is the same photo, +/// and the album memberships of the copies are unioned onto the survivor. Two +/// genuinely different photos sharing both a name and an exact size would be +/// merged — vanishingly unlikely, and the cost of being wrong is one photo +/// filed under an extra album rather than lost data. +fn plan(scan: &TakeoutScan) -> Vec { + let mut order: Vec<(String, u64)> = Vec::new(); + let mut by_key: HashMap<(String, u64), Planned> = HashMap::new(); + + for photo in &scan.photos { + let key = (photo.name.clone(), photo.size); + let entry = by_key.entry(key.clone()).or_insert_with(|| { + order.push(key.clone()); + Planned { + photo: photo.clone(), + albums: Vec::new(), + uid: None, + } + }); + if let Bucket::Album(title) = &photo.bucket + && !entry.albums.iter().any(|existing| existing == title) + { + entry.albums.push(title.clone()); + } + // A copy that carries a capture time wins over one that does not: the + // album copy and the year-folder copy do not always both have a sidecar. + if entry.photo.capture_time.is_none() && photo.capture_time.is_some() { + entry.photo.capture_time = photo.capture_time; + } + entry.photo.favorite |= photo.favorite; + } + + order + .into_iter() + .filter_map(|key| by_key.remove(&key)) + .collect() +} + +/// Open one archive and stream `entry` through `sink`. +fn read_entry( + archive: &mut zip::ZipArchive>, + entry: &str, + sink: impl FnOnce(&mut (dyn Read + Send)) -> R, +) -> CoreResult { + let mut file = archive + .by_name(entry) + .map_err(|e| CoreError::invalid(format!("{entry}: {e}")))?; + Ok(sink(&mut file)) +} + +impl Core { + /// Import Google Photos Takeout archives. Blocking; run on its own thread. + /// + /// `cancel` stops the run at the next photo boundary — an upload already on + /// the wire is allowed to finish, so a cancelled import never leaves a + /// half-written revision behind. + pub(crate) fn import_takeout( + &self, + archives: Vec, + dry_run: bool, + cancel: Arc, + ) -> CoreResult { + let job = self.transfers.begin_job("Importing Google Photos"); + job.detail("reading archives"); + + let scan = scan_archives(&archives)?; + let mut planned = plan(&scan); + let mut summary = ImportSummary { + found: planned.len(), + skipped_trashed: scan.skipped_trashed, + ..Default::default() + }; + job.set_total(planned.len() as u64); + info!( + archives = archives.len(), + photos = summary.found, + albums = scan.albums.len(), + trashed = summary.skipped_trashed, + "takeout scan complete" + ); + + let photos = self.photos(); + if self + .rt + .block_on(photos.get_photos_root()) + .map_err(|e| CoreError::from_api(&e, "resolve photos library"))? + .is_none() + { + return Err(CoreError::invalid( + "this account has no Proton Photos library yet; open Proton Photos once to create it, then import again", + )); + } + + // Phase 2: which of these are already on the account. + job.detail("checking for photos you already have"); + self.mark_existing(&mut planned, &archives, &mut summary, &cancel)?; + + if dry_run { + summary.duplicates = planned.iter().filter(|p| p.uid.is_some()).count(); + return Ok(summary); + } + + // Phase 3: upload what is left, one archive at a time so each zip's + // central directory is parsed once. + self.upload_planned(&mut planned, &archives, &mut summary, &cancel, &job)?; + + // Phase 4: albums. Attempted even after a cancel — the photos that did + // upload should still land in their albums. + if !scan.albums.is_empty() { + job.detail("sorting photos into albums"); + if let Err(e) = self.link_albums(&planned, &scan.albums, &mut summary) { + warn!(error = %e, "takeout album linking failed"); + } + } + + self.spawn_timeline_refresh(); + summary.cancelled = cancel.load(Ordering::Relaxed); + Ok(summary) + } + + /// Fill in `uid` for every planned photo the account already holds. + /// + /// Two passes on purpose: the name-only query is one request per 150 photos + /// and needs no file contents, so the expensive SHA-1 pass only runs for the + /// handful of names that actually collide. + fn mark_existing( + &self, + planned: &mut [Planned], + archives: &[PathBuf], + summary: &mut ImportSummary, + cancel: &AtomicBool, + ) -> CoreResult<()> { + let photos = self.photos(); + let names: Vec = planned.iter().map(|p| p.photo.name.clone()).collect(); + let collisions = self + .rt + .block_on(photos.name_collisions(&names)) + .map_err(|e| CoreError::from_api(&e, "check for existing photos"))?; + + // Hash the colliding files, grouped by archive so each zip opens once. + let mut by_archive: BTreeMap> = BTreeMap::new(); + for (index, planned) in planned.iter().enumerate() { + if collisions.get(index).copied().unwrap_or(false) { + by_archive + .entry(planned.photo.archive) + .or_default() + .push(index); + } + } + + // Kept as a pair list, not a map: the duplicate lookup answers in input + // order, so the order these were built in is what maps answers back to + // planned photos. + let mut digests: Vec<(usize, String)> = Vec::new(); + for (archive_index, indices) in by_archive { + if cancel.load(Ordering::Relaxed) { + break; + } + let mut archive = open_archive(archives, archive_index)?; + for index in indices { + let entry = planned[index].photo.entry.clone(); + match read_entry(&mut archive, &entry, sha1_of) { + Ok(Ok(digest)) => digests.push((index, digest)), + Ok(Err(e)) | Err(e) => { + warn!(entry, error = %e, "cannot hash takeout entry"); + } + } + } + } + if digests.is_empty() { + return Ok(()); + } + + let items: Vec<(String, String)> = digests + .iter() + .map(|(index, digest)| (planned[*index].photo.name.clone(), digest.clone())) + .collect(); + let matches = self + .rt + .block_on(photos.find_duplicates_many(&items)) + .map_err(|e| CoreError::from_api(&e, "check for existing photos"))?; + + for (slot, (index, _)) in digests.iter().enumerate() { + if let Some(uid) = matches.get(slot).and_then(|uids| uids.first()) { + planned[*index].uid = Some(uid.clone()); + summary.duplicates += 1; + } + } + Ok(()) + } + + /// Upload every planned photo that is not already on the account. + fn upload_planned( + &self, + planned: &mut [Planned], + archives: &[PathBuf], + summary: &mut ImportSummary, + cancel: &AtomicBool, + job: &crate::transfers::JobGuard, + ) -> CoreResult<()> { + let photos = self.photos(); + let mut favorites: Vec = Vec::new(); + + let mut by_archive: BTreeMap> = BTreeMap::new(); + for (index, item) in planned.iter().enumerate() { + if item.uid.is_none() { + by_archive + .entry(item.photo.archive) + .or_default() + .push(index); + } + } + + for (archive_index, indices) in by_archive { + if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() { + break; + } + let mut archive = open_archive(archives, archive_index)?; + + for index in indices { + if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() { + break; + } + let photo = planned[index].photo.clone(); + job.detail(photo.name.clone()); + + let guard = self.transfers.begin( + photo.name.clone(), + "", + TransferDirection::Upload, + photo.size, + ); + let metadata = PhotoUploadMetadata { + capture_time: photo.capture_time, + ..Default::default() + }; + let uploaded = read_entry(&mut archive, &photo.entry, |reader| { + let counting = CountingReader::new(reader, &guard); + self.rt.block_on(photos.upload_photo_from( + &photo.name, + &photo.media_type, + counting, + photo.size as i64, + Vec::new(), + metadata, + false, + )) + }); + drop(guard); + job.step(); + + match uploaded { + Ok(Ok(uid)) => { + summary.uploaded += 1; + summary.bytes += photo.size; + if photo.favorite { + favorites.push(uid.clone()); + } + planned[index].uid = Some(uid); + } + Ok(Err(e)) => { + summary.failed += 1; + warn!(name = photo.name, error = %e, "takeout photo upload failed"); + } + Err(e) => { + summary.failed += 1; + warn!(name = photo.name, error = %e, "cannot read takeout entry"); + } + } + } + } + + // Favourites are a separate call: the upload seal takes classification + // tags, but `Favorite` is not one of them — it goes through the photos + // volume's dedicated favourite endpoint. + if !favorites.is_empty() { + let updates: Vec = favorites + .into_iter() + .map(|node_uid| PhotoTagsUpdate { + node_uid, + tags_to_add: vec![PhotoTag::Favorite], + tags_to_remove: Vec::new(), + }) + .collect(); + if let Err(e) = self.rt.block_on(photos.update_photos(&updates)) { + warn!(error = %e, "marking imported favourites failed"); + } + } + Ok(()) + } + + /// Create the albums the export names that the account lacks, then add each + /// album's photos to it. + fn link_albums( + &self, + planned: &[Planned], + albums: &[String], + summary: &mut ImportSummary, + ) -> CoreResult<()> { + let photos = self.photos(); + + // Existing albums by decrypted name, so a second import of the same + // export adds to the album it made the first time instead of a twin. + let mut existing: HashMap = HashMap::new(); + let uids = self + .rt + .block_on(photos.enumerate_album_node_uids()) + .map_err(|e| CoreError::from_api(&e, "list albums"))?; + if !uids.is_empty() { + let albums_nodes = self + .rt + .block_on(photos.enumerate_nodes(&uids)) + .map_err(|e| CoreError::from_api(&e, "read albums"))?; + for node in albums_nodes { + existing.entry(node.name.clone()).or_insert(node.uid); + } + } + + for title in albums { + let members: Vec = planned + .iter() + .filter(|item| item.albums.iter().any(|album| album == title)) + .filter_map(|item| item.uid.clone()) + .collect(); + if members.is_empty() { + continue; + } + + let album_uid = match existing.get(title) { + Some(uid) => uid.clone(), + None => match self.rt.block_on(photos.create_album(title)) { + Ok(uid) => { + summary.albums_created += 1; + existing.insert(title.clone(), uid.clone()); + uid + } + Err(e) => { + warn!(album = title, error = %e, "cannot create album"); + continue; + } + }, + }; + + match self + .rt + .block_on(photos.add_photos_to_album(&album_uid, &members)) + { + Ok(outcomes) => { + for (uid, outcome) in outcomes { + match outcome { + Ok(()) => summary.album_links += 1, + // A photo already in the album is the expected + // outcome of a re-run, not a failure worth counting. + Err(e) => { + warn!(album = title, %uid, error = %e, "cannot add photo to album") + } + } + } + } + Err(e) => warn!(album = title, error = %e, "album addition failed"), + } + } + Ok(()) + } + + /// Run an import on a background thread, logging its outcome to the activity + /// feed. Returns immediately. + /// + /// Refuses while one is already running: two imports of the same export + /// would race each other's duplicate detection and upload everything twice. + pub(crate) fn spawn_takeout_import( + &self, + archives: Vec, + dry_run: bool, + ) -> CoreResult<()> { + let cancel = Arc::new(AtomicBool::new(false)); + { + let mut state = IMPORT.lock(); + if state.running { + return Err(CoreError::conflict( + "an import is already running; wait for it to finish or cancel it", + )); + } + state.running = true; + state.cancel = Some(cancel.clone()); + state.last = None; + } + + let core = self.clone(); + std::thread::spawn(move || { + let outcome = core.import_takeout(archives, dry_run, cancel); + { + let mut state = IMPORT.lock(); + state.running = false; + state.cancel = None; + state.last = outcome.as_ref().ok().cloned(); + } + match outcome { + Ok(summary) => { + let mut parts = vec![format!("{} uploaded", summary.uploaded)]; + if summary.duplicates > 0 { + parts.push(format!("{} already there", summary.duplicates)); + } + if summary.albums_created > 0 { + parts.push(format!("{} albums", summary.albums_created)); + } + if summary.failed > 0 { + parts.push(format!("{} failed", summary.failed)); + } + if summary.cancelled { + parts.push("cancelled".to_string()); + } + core.log_activity( + ActivityKind::Upload, + "Google Photos import", + parts.join(" · "), + summary.failed == 0 && !summary.cancelled, + ); + } + Err(e) => { + warn!(error = %e, "takeout import failed"); + core.log_activity(ActivityKind::Upload, "Google Photos import", &e, false); + } + } + }); + Ok(()) + } +} + +/// Open one of the import's archives by index. +fn open_archive( + archives: &[PathBuf], + index: usize, +) -> CoreResult>> { + let path = archives + .get(index) + .ok_or_else(|| CoreError::internal(format!("archive {index} is not in the import")))?; + let file = std::fs::File::open(path) + .map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?; + zip::ZipArchive::new(std::io::BufReader::new(file)) + .map_err(|e| CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))) +} + +/// The lowercase-hex SHA-1 of everything `reader` yields — the digest Proton's +/// photo duplicate detection is keyed on. +fn sha1_of(reader: &mut (dyn Read + Send)) -> CoreResult { + let mut hasher = Sha1::new(); + let mut buffer = vec![0u8; 256 * 1024]; + loop { + let read = reader + .read(&mut buffer) + .map_err(|e| CoreError::invalid(format!("read failed: {e}")))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(hex_lower(&hasher.finalize())) +} + +fn hex_lower(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push_str(&format!("{byte:02x}")); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use pdfs_core::takeout::TakeoutPhoto; + + fn photo(name: &str, size: u64, bucket: Bucket) -> TakeoutPhoto { + TakeoutPhoto { + archive: 0, + entry: format!("Takeout/Google Photos/x/{name}"), + name: name.to_string(), + size, + media_type: "image/jpeg".to_string(), + capture_time: None, + favorite: false, + bucket, + } + } + + #[test] + fn album_copies_of_one_photo_fold_into_a_single_upload() { + let scan = TakeoutScan { + photos: vec![ + photo("IMG_1.jpg", 100, Bucket::Timeline), + photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into())), + photo("IMG_1.jpg", 100, Bucket::Album("Best of".into())), + photo("IMG_2.jpg", 200, Bucket::Timeline), + ], + albums: vec!["Iceland".into(), "Best of".into()], + ..Default::default() + }; + + let planned = plan(&scan); + assert_eq!(planned.len(), 2, "the three copies are one upload"); + assert_eq!(planned[0].albums, vec!["Iceland", "Best of"]); + assert!(planned[1].albums.is_empty()); + } + + #[test] + fn same_name_different_size_stays_two_photos() { + let scan = TakeoutScan { + photos: vec![ + photo("IMG_1.jpg", 100, Bucket::Timeline), + photo("IMG_1.jpg", 101, Bucket::Album("Iceland".into())), + ], + ..Default::default() + }; + + assert_eq!(plan(&scan).len(), 2); + } + + #[test] + fn metadata_from_any_copy_survives_the_fold() { + // Only the album copy carries a sidecar; the fold must not lose it. + let mut bare = photo("IMG_1.jpg", 100, Bucket::Timeline); + bare.capture_time = None; + let mut rich = photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into())); + rich.capture_time = Some(1_560_000_000); + rich.favorite = true; + + let planned = plan(&TakeoutScan { + photos: vec![bare, rich], + ..Default::default() + }); + + assert_eq!(planned[0].photo.capture_time, Some(1_560_000_000)); + assert!(planned[0].photo.favorite); + } + + #[test] + fn sha1_matches_the_known_digest_of_abc() { + let mut reader: &[u8] = b"abc"; + assert_eq!( + sha1_of(&mut reader).unwrap(), + "a9993e364706816aba3e25717850c26c9cd0d89d" + ); + } +} diff --git a/crates/pdfs-gui/src/app/main.rs b/crates/pdfs-gui/src/app/main.rs index 7aa88e8..5891044 100644 --- a/crates/pdfs-gui/src/app/main.rs +++ b/crates/pdfs-gui/src/app/main.rs @@ -14,6 +14,7 @@ use pages::photos::*; use pages::shared::*; use pages::shared_by_me::*; use pages::status::*; +use pages::takeout::*; use pages::trash::*; use pages::verify::*; use widgets::details::*; @@ -46,10 +47,10 @@ use pdfs_core::config::AppDirs; use pdfs_core::control::{ ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, DeviceInfo, DirEntry, ErrorKind, - InvitationInfo, JobItem, PhotoItem, PhotoKind, PublicLinkInfo, RefreshScope, Request, Response, - RestorableFolder, RestoreItem, SearchHit, ShareEntry, ShareEntryKind, SharedItem, - SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem, pending_summary, - send, + ImportSummary, InvitationInfo, JobItem, PhotoItem, PhotoKind, PublicLinkInfo, RefreshScope, + Request, Response, RestorableFolder, RestoreItem, SearchHit, ShareEntry, ShareEntryKind, + SharedItem, SyncFolderInfo, SyncPhase, SyncProgress, TransferDirection, TransferItem, + pending_summary, send, }; use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec}; @@ -120,6 +121,7 @@ struct Ui { pub(crate) devices: DevicesState, pub(crate) locations: LocationsState, pub(crate) activity: ActivityState, + pub(crate) takeout: TakeoutState, } impl Ui { @@ -246,7 +248,9 @@ fn load_proton_theme() { .navigation-sidebar row {{ border-radius: 8px; margin: 2px 6px; }}\n\ .navigation-sidebar row:selected {{ background: alpha({PROTON_PURPLE}, 0.16); color: {PROTON_PURPLE}; font-weight: 600; }}\n\ .navigation-sidebar row:selected image {{ color: {PROTON_PURPLE}; }}\n\ - .file-tile:selected, .file-tile:hover:selected {{ background: alpha({PROTON_PURPLE}, 0.20); }}\n" + .file-tile:selected, .file-tile:hover:selected {{ background: alpha({PROTON_PURPLE}, 0.20); }}\n\ + .dropzone {{ border: 2px dashed alpha(currentColor, 0.25); border-radius: 16px; padding: 28px 18px; transition: background 160ms ease, border-color 160ms ease; }}\n\ + .dropzone-active {{ border-color: {PROTON_PURPLE}; background: alpha({PROTON_PURPLE}, 0.10); }}\n" ); let provider = gtk4::CssProvider::new(); provider.load_from_string(&css); @@ -285,6 +289,7 @@ fn build_window(app: &adw::Application) { let (locations_page, locations_widgets) = build_locations_page(); let (activity_page, activity_widgets) = build_activity_page(); let (trash_page, trash_widgets) = build_trash_page(); + let (takeout_page, takeout_widgets) = build_takeout_page(); stack.add_named(&login_page, Some("login")); stack.add_named(&main_page, Some("main")); stack.add_named(&browser_page, Some("browser")); @@ -295,6 +300,7 @@ fn build_window(app: &adw::Application) { stack.add_named(&locations_page, Some("locations")); stack.add_named(&activity_page, Some("activity")); stack.add_named(&trash_page, Some("trash")); + stack.add_named(&takeout_page, Some("takeout")); // Sidebar: the signed-in destinations. Selecting a row swaps the page stack; // `sync_sidebar` pushes the other way when navigation happens elsewhere (e.g. @@ -407,6 +413,7 @@ fn build_window(app: &adw::Application) { retry: gallery_widgets.retry.clone(), more: gallery_widgets.more.clone(), upload: gallery_widgets.upload.clone(), + import: gallery_widgets.import.clone(), title: gallery_widgets.title.clone(), subtitle: gallery_widgets.subtitle.clone(), albums: gallery_widgets.albums.clone(), @@ -494,6 +501,24 @@ fn build_window(app: &adw::Application) { inflight: Cell::new(false), key: RefCell::new(None), }, + takeout: TakeoutState { + archives: RefCell::new(Vec::new()), + list_group: takeout_widgets.list_group.clone(), + rows: RefCell::new(Vec::new()), + dropzone: takeout_widgets.dropzone.clone(), + scan_button: takeout_widgets.scan_button.clone(), + import_button: takeout_widgets.import_button.clone(), + cancel_button: takeout_widgets.cancel_button.clone(), + clear_button: takeout_widgets.clear_button.clone(), + progress_group: takeout_widgets.progress_group.clone(), + progress_label: takeout_widgets.progress_label.clone(), + progress_bar: takeout_widgets.progress_bar.clone(), + summary_group: takeout_widgets.summary_group.clone(), + summary_rows: RefCell::new(Vec::new()), + inflight: Cell::new(false), + running: Cell::new(false), + dry_run: Cell::new(false), + }, }); wire_login(&ui); wire_logout(&ui, &main_widgets.logout_button); @@ -501,6 +526,7 @@ fn build_window(app: &adw::Application) { &ui, &main_widgets.purge_button, &main_widgets.mountpoint_button, + &main_widgets.import_row, ); wire_sidebar(&ui); wire_browser(&ui, &browser_widgets.grid, &browser_widgets.column_view); @@ -520,6 +546,7 @@ fn build_window(app: &adw::Application) { wire_devices(&ui, &devices_widgets.retry, &devices_widgets.restore); wire_locations(&ui, &locations_widgets.retry, &locations_widgets.add_folder); wire_activity(&ui, &activity_widgets.retry); + wire_takeout(&ui, &takeout_widgets); wire_refresh( &ui, &[ @@ -559,6 +586,9 @@ fn build_window(app: &adw::Application) { // every visit to stay live. Some("activity") => load_activity(&ui_nav), Some("trash") => load_trash(&ui_nav), + // The import runs in the daemon and outlives this page, so arriving + // here asks straight away whether one is in flight. + Some("takeout") => refresh_takeout(&ui_nav), _ => {} } }); @@ -644,6 +674,17 @@ fn build_sidebar() -> (adw::NavigationPage, gtk4::ListBox) { (page, list) } +/// The sidebar destination a page belongs under. Most pages are their own +/// destination; a sub-page reached from one (Import, off Settings) has no row of +/// its own and answers with its parent, so the sidebar highlights where the user +/// came from rather than clearing — which would read as "nowhere". +fn destination_of(page: &str) -> &str { + match page { + "takeout" => "main", + other => other, + } +} + /// Selecting a sidebar row navigates the page stack. The reverse direction (stack /// → sidebar highlight) is [`sync_sidebar`], so the two can't fight: this handler /// only ever writes the stack. @@ -654,7 +695,11 @@ fn wire_sidebar(ui: &Rc) { let Some((page, _, _)) = DESTINATIONS.get(row.index() as usize) else { return; }; - if ui_row.stack.visible_child_name().as_deref() != Some(*page) { + // Compared by *destination*, not by page: while a sub-page is open its + // parent's row is the selected one, and re-asserting it here would kick + // the user off the sub-page the moment they arrived. + let current = ui_row.stack.visible_child_name(); + if current.as_deref().map(destination_of) != Some(*page) { ui_row.stack.set_visible_child_name(page); } }); @@ -755,6 +800,7 @@ fn sync_sidebar(ui: &Rc) { let Some(current) = ui.stack.visible_child_name() else { return; }; + let current = destination_of(¤t); let index = DESTINATIONS .iter() .position(|(page, _, _)| *page == current); diff --git a/crates/pdfs-gui/src/app/pages/mod.rs b/crates/pdfs-gui/src/app/pages/mod.rs index edcccd3..bce4ae6 100644 --- a/crates/pdfs-gui/src/app/pages/mod.rs +++ b/crates/pdfs-gui/src/app/pages/mod.rs @@ -9,5 +9,6 @@ pub(crate) mod photos; pub(crate) mod shared; pub(crate) mod shared_by_me; pub(crate) mod status; +pub(crate) mod takeout; pub(crate) mod trash; pub(crate) mod verify; diff --git a/crates/pdfs-gui/src/app/pages/photos.rs b/crates/pdfs-gui/src/app/pages/photos.rs index a20b3fd..b022f58 100644 --- a/crates/pdfs-gui/src/app/pages/photos.rs +++ b/crates/pdfs-gui/src/app/pages/photos.rs @@ -19,6 +19,8 @@ pub(crate) struct GalleryState { pub(crate) retry: gtk4::Button, pub(crate) more: gtk4::Button, pub(crate) upload: gtk4::Button, + /// Opens the Google Photos Takeout import chooser. + pub(crate) import: gtk4::Button, /// "Photos", or the album's name while one is open. pub(crate) title: gtk4::Label, /// "1,204 photos" under the page title. @@ -177,6 +179,7 @@ pub(crate) struct GalleryWidgets { pub(crate) scroll: gtk4::ScrolledWindow, pub(crate) retry: gtk4::Button, pub(crate) upload: gtk4::Button, + pub(crate) import: gtk4::Button, pub(crate) refresh: gtk4::Button, /// The Albums grid, its own status page and the stack between them, plus the /// Photos/Albums switcher and the back button out of an album. @@ -288,12 +291,23 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) { .build(); upload.add_css_class("pill"); upload.add_css_class("suggested-action"); + + // Importing an export is a rarer, heavier action than adding one photo, so + // it sits beside Upload as a plain button rather than a second accent one. + let import = gtk4::Button::builder() + .icon_name("folder-download-symbolic") + .tooltip_text("Import a Google Photos Takeout export") + .valign(gtk4::Align::Center) + .build(); + import.add_css_class("flat"); + import.add_css_class("circular"); let refresh = refresh_button(); let header_box = gtk4::Box::new(gtk4::Orientation::Horizontal, 8); header_box.append(&back); header_box.append(&titles); header_box.append(&refresh); + header_box.append(&import); header_box.append(&upload); // All / Photos / Videos / Raw filter. Linked toggles acting as one segmented @@ -428,6 +442,7 @@ pub(crate) fn build_gallery_page() -> (gtk4::Widget, GalleryWidgets) { scroll, retry, upload, + import, refresh, tabs, favorites_btn, @@ -729,6 +744,14 @@ pub(crate) fn wire_gallery(ui: &Rc, list: >k4::ListView, scroll: >k4::Sc load_gallery(&ui_dates, false); }); + // The import is a staged, hours-long migration, so the button hands over to + // the Import page rather than opening a file chooser here: the archives need + // reviewing before anything is sent, and the run needs somewhere to report. + let ui_import = ui.clone(); + ui.gallery + .import + .connect_clicked(move |_| ui_import.stack.set_visible_child_name("takeout")); + let ui_upload = ui.clone(); ui.gallery.upload.connect_clicked(move |_| { let dialog = gtk4::FileDialog::builder() diff --git a/crates/pdfs-gui/src/app/pages/status.rs b/crates/pdfs-gui/src/app/pages/status.rs index 11e63ed..dcb24e3 100644 --- a/crates/pdfs-gui/src/app/pages/status.rs +++ b/crates/pdfs-gui/src/app/pages/status.rs @@ -104,6 +104,8 @@ pub(crate) struct MainWidgets { /// which is where the folder is changed. pub(crate) mountpoint_row: adw::ActionRow, pub(crate) mountpoint_button: gtk4::Button, + /// Opens the Google Photos Takeout import page. + pub(crate) import_row: adw::ActionRow, } /// The main (logged-in) page: a libadwaita settings surface — account header, @@ -233,6 +235,22 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) { mountpoint_row.add_suffix(&mountpoint_button); system_group.add(&mountpoint_row); + // Import: a one-off migration rather than a setting, so it is a doorway to + // its own page rather than controls inlined here — staging a set of archives + // and watching an hours-long upload needs the room. + let import_group = adw::PreferencesGroup::builder() + .title("Import") + .description("Bring a photo library from another service into Proton Photos.") + .build(); + let import_row = adw::ActionRow::builder() + .title("Import from Google Photos") + .subtitle("Add a Google Takeout export to your timeline.") + .activatable(true) + .build(); + import_row.add_prefix(>k4::Image::from_icon_name("folder-download-symbolic")); + import_row.add_suffix(>k4::Image::from_icon_name("go-next-symbolic")); + import_group.add(&import_row); + // Pins group: filled in by refresh. let pins_group = adw::PreferencesGroup::builder() .title("Pinned files") @@ -265,6 +283,7 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) { inner.append(&transfers_group); inner.append(&storage_group); inner.append(&system_group); + inner.append(&import_group); inner.append(&pins_group); inner.append(&dev_group); @@ -292,6 +311,7 @@ pub(crate) fn build_main_page() -> (gtk4::Widget, MainWidgets) { purge_button, mountpoint_row, mountpoint_button, + import_row, }, ) } @@ -308,6 +328,7 @@ pub(crate) fn wire_settings( ui: &Rc, purge_button: >k4::Button, mountpoint_button: >k4::Button, + import_row: &adw::ActionRow, ) { let config = ui.dirs.load_config(); @@ -382,6 +403,10 @@ pub(crate) fn wire_settings( // local path; this button is the way there. let ui_mp = ui.clone(); mountpoint_button.connect_clicked(move |_| ui_mp.stack.set_visible_child_name("locations")); + + // Import: a doorway, not a setting — everything about it lives on its page. + let ui_import = ui.clone(); + import_row.connect_activated(move |_| ui_import.stack.set_visible_child_name("takeout")); } /// Run a settings control-socket round-trip (budget / purge) on a worker thread, @@ -513,6 +538,12 @@ pub(crate) fn refresh(ui: &Rc) { refresh_transfers(ui); // Both of these pages show work as it happens, so they follow the tick while // they are on screen. Every other page loads on navigation only. + // A Takeout import outlives the page that started it, so it is polled + // wherever the user has navigated to — otherwise leaving the page means + // never being told it finished. + if ui.stack.visible_child_name().as_deref() == Some("takeout") || ui.takeout.running.get() { + refresh_takeout(ui); + } match ui.stack.visible_child_name().as_deref() { Some("main") => refresh_quota(ui), Some("locations") => refresh_locations(ui), @@ -653,6 +684,10 @@ pub(crate) fn repaint_transfers(ui: &Rc, items: &[TransferItem], jobs: &[Job // count falling to zero is what "the sync finished" looks like from here. // Jobs are deliberately not counted — a bulk upload retires its scan job and // starts its upload job mid-flight, which is not a thing finishing. + // The Import page reads its progress off the same snapshot rather than + // polling the daemon a second time for numbers already on the wire. + takeout_progress(ui, jobs); + let previous = ui.status.active_transfers.replace(items.len()); if items.is_empty() && previous > 0 { let files = if previous == 1 { @@ -808,11 +843,14 @@ pub(crate) fn refresh_status(ui: &Rc) { (used as f64 / budget as f64).min(1.0) }; ui.status.cache_bar.set_fraction(fraction); - ui.status.cache_label.set_text(&format!( - "{} of {} used", - human_bytes(used), - human_bytes(budget) - )); + // A 0 budget means *unlimited*, not a zero-byte cap — "of 0 B + // used" reads as a broken read-out, and there is no fraction to + // draw against no limit. + ui.status.cache_label.set_text(&if budget == 0 { + format!("{} cached — no limit set", human_bytes(used)) + } else { + format!("{} of {} used", human_bytes(used), human_bytes(budget)) + }); repaint_pins(&ui, &pins, true); } // Daemon unreachable (still starting, or down): report not-mounted and diff --git a/crates/pdfs-gui/src/app/pages/takeout.rs b/crates/pdfs-gui/src/app/pages/takeout.rs new file mode 100644 index 0000000..5f2625f --- /dev/null +++ b/crates/pdfs-gui/src/app/pages/takeout.rs @@ -0,0 +1,734 @@ +//! The Import page: bringing a Google Photos **Takeout** export into Proton +//! Photos. +//! +//! Reached from Settings → *Import from Google Photos* (and from the Gallery's +//! import button), because it is a one-off migration rather than something the +//! photo timeline does every day. It gets a page of its own instead of a dialog +//! for two reasons: an export is a *set* of archives that has to be staged and +//! checked before anything is sent, and the import that follows runs for hours, +//! so its progress needs somewhere to live that the user can navigate away from +//! and come back to. +//! +//! Archives arrive either by drag-and-drop onto the page or through the file +//! chooser; both funnel into [`add_archives`], which is where the `.zip` filter +//! and de-duplication live. Nothing is sent until the user presses Scan or +//! Import — staging is local, so a mis-drop costs nothing. +//! +//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only +//! stages paths, launches, polls [`Request::ImportStatus`] for the final report, +//! and reads live progress off the import's job in the transfer snapshot the +//! Settings tick already fetches (see [`takeout_progress`]). + +use crate::*; + +/// The title the daemon gives the import's job in [`Request::GetQueueStatus`]. +/// Matching on it is how this page borrows the progress the Settings tick is +/// already polling instead of adding a second poll of its own. +const IMPORT_JOB: &str = "Importing Google Photos"; + +/// What the archive list says before anything is staged. `repaint_archives` +/// swaps in the count and total once there is something to report, and swaps +/// this back when the list is emptied again. +const EMPTY_LIST_DESCRIPTION: &str = "Photos already in Proton Photos are matched by name and content and skipped, so \ + re-running an interrupted import is safe."; + +pub(crate) struct TakeoutState { + /// Archives staged for this import, in the order they were added. Sent as + /// one set: Google splits an export across numbered zips and routinely puts + /// a photo in one part and its metadata sidecar in another, so a per-file + /// import would lose album titles and capture times. + pub(crate) archives: RefCell>, + /// The staged-archive list and the rows currently rendered in it. + pub(crate) list_group: adw::PreferencesGroup, + pub(crate) rows: RefCell>, + /// The drop zone, whose `dropzone-active` class follows the drag. + pub(crate) dropzone: gtk4::Box, + pub(crate) scan_button: gtk4::Button, + pub(crate) import_button: gtk4::Button, + pub(crate) cancel_button: gtk4::Button, + pub(crate) clear_button: gtk4::Button, + /// Progress group, hidden unless an import is running. + pub(crate) progress_group: adw::PreferencesGroup, + pub(crate) progress_label: gtk4::Label, + pub(crate) progress_bar: gtk4::ProgressBar, + /// Report group for the last finished run, hidden until there is one. + pub(crate) summary_group: adw::PreferencesGroup, + pub(crate) summary_rows: RefCell>, + /// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon + /// can't stack worker threads on the 2s tick. + pub(crate) inflight: Cell, + /// Whether the daemon reported an import running on the last poll. + pub(crate) running: Cell, + /// Whether the run in flight was launched as a dry run, so the report can + /// say "would upload" rather than claiming photos were sent. The daemon's + /// summary carries no such flag — it is the same shape either way. + pub(crate) dry_run: Cell, +} + +/// Widgets the Import page hands back for wiring. +pub(crate) struct TakeoutWidgets { + pub(crate) list_group: adw::PreferencesGroup, + pub(crate) dropzone: gtk4::Box, + /// The whole page, which is the drop target: a drag released anywhere on it + /// counts, not only inside the dashed rectangle. + pub(crate) page: gtk4::Widget, + pub(crate) choose_button: gtk4::Button, + pub(crate) scan_button: gtk4::Button, + pub(crate) import_button: gtk4::Button, + pub(crate) cancel_button: gtk4::Button, + pub(crate) clear_button: gtk4::Button, + pub(crate) back_button: gtk4::Button, + pub(crate) progress_group: adw::PreferencesGroup, + pub(crate) progress_label: gtk4::Label, + pub(crate) progress_bar: gtk4::ProgressBar, + pub(crate) summary_group: adw::PreferencesGroup, +} + +pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) { + // Header: a back button out to Settings, since this page is reached from + // there and has no sidebar row of its own to navigate back with. + let back_button = gtk4::Button::builder() + .icon_name("go-previous-symbolic") + .tooltip_text("Back to Settings") + .valign(gtk4::Align::Center) + .build(); + back_button.add_css_class("flat"); + + let title = gtk4::Label::builder() + .label("Import from Google Photos") + .halign(gtk4::Align::Start) + .build(); + title.add_css_class("title-2"); + let subtitle = gtk4::Label::builder() + .label("Add a Google Takeout export to your Proton Photos timeline.") + .halign(gtk4::Align::Start) + .wrap(true) + .xalign(0.0) + .build(); + subtitle.add_css_class("dim-label"); + + let titles = gtk4::Box::new(gtk4::Orientation::Vertical, 2); + titles.set_hexpand(true); + titles.append(&title); + titles.append(&subtitle); + + let header = gtk4::Box::new(gtk4::Orientation::Horizontal, 8); + header.append(&back_button); + header.append(&titles); + let header_clamp = adw::Clamp::builder().child(&header).build(); + + // Drop zone. It is a hint, not the only way in — the button under it does + // the same thing for anyone not dragging, and the drop target covers the + // whole page so a drag released just outside the rectangle still lands. + let drop_icon = gtk4::Image::from_icon_name("folder-download-symbolic"); + drop_icon.set_pixel_size(48); + drop_icon.add_css_class("dim-label"); + let drop_title = gtk4::Label::builder() + .label("Drop your Takeout .zip files here") + .build(); + drop_title.add_css_class("title-4"); + let drop_hint = gtk4::Label::builder() + .label("Add every part of the export at once — a photo and its metadata often sit in different parts.") + .wrap(true) + .justify(gtk4::Justification::Center) + .max_width_chars(48) + .build(); + drop_hint.add_css_class("dim-label"); + let choose_button = gtk4::Button::builder() + .label("Choose files…") + .halign(gtk4::Align::Center) + .build(); + choose_button.add_css_class("pill"); + + let dropzone = gtk4::Box::new(gtk4::Orientation::Vertical, 10); + dropzone.set_halign(gtk4::Align::Fill); + dropzone.add_css_class("dropzone"); + dropzone.append(&drop_icon); + dropzone.append(&drop_title); + dropzone.append(&drop_hint); + dropzone.append(&choose_button); + + // Staged archives. Filled by `repaint_archives`, which owns the description + // too (it becomes the count + total once there is something to count); the + // group carries a placeholder row while empty so it never renders as a bare + // title. + let list_group = adw::PreferencesGroup::builder() + .title("Archives to import") + .description(EMPTY_LIST_DESCRIPTION) + .build(); + + let clear_button = gtk4::Button::builder() + .label("Clear") + .tooltip_text("Remove every staged archive") + .build(); + clear_button.add_css_class("flat"); + + // Actions. Scan is a dry run: it reads the archives' directories and reports + // what an import *would* do, without uploading a byte. It is offered first + // because the import behind it is hours long and hard to take back. + let scan_button = gtk4::Button::builder() + .label("Scan") + .tooltip_text("Check the archives and report what would be imported, without uploading") + .build(); + scan_button.add_css_class("pill"); + let import_button = gtk4::Button::builder().label("Import").build(); + import_button.add_css_class("pill"); + import_button.add_css_class("suggested-action"); + let cancel_button = gtk4::Button::builder() + .label("Stop import") + .tooltip_text("Finish the photo on the wire, file what was uploaded, then stop") + .build(); + cancel_button.add_css_class("pill"); + cancel_button.add_css_class("destructive-action"); + cancel_button.set_visible(false); + + // Clear sits apart from the launch buttons, on the other end of the row: it + // undoes staging, and putting it next to Import invites the wrong click. + let actions = gtk4::Box::new(gtk4::Orientation::Horizontal, 8); + let spacer = gtk4::Box::new(gtk4::Orientation::Horizontal, 0); + spacer.set_hexpand(true); + actions.append(&clear_button); + actions.append(&spacer); + actions.append(&cancel_button); + actions.append(&scan_button); + actions.append(&import_button); + + // Progress: the import's job from the transfer snapshot, so this page shows + // the same numbers the Settings Activity list does. + let progress_group = adw::PreferencesGroup::builder() + .title("Import in progress") + .visible(false) + .build(); + let progress_box = gtk4::Box::new(gtk4::Orientation::Vertical, 6); + progress_box.set_margin_top(8); + progress_box.set_margin_bottom(8); + let progress_label = gtk4::Label::builder().halign(gtk4::Align::Start).build(); + progress_label.add_css_class("dim-label"); + let progress_bar = gtk4::ProgressBar::new(); + progress_box.append(&progress_label); + progress_box.append(&progress_bar); + progress_group.add( + &adw::PreferencesRow::builder() + .activatable(false) + .child(&progress_box) + .build(), + ); + + // Report for the last finished run, kept on screen after it ends: an import + // outlives the window being on this page, and "what happened" is the whole + // reason to come back to it. + let summary_group = adw::PreferencesGroup::builder() + .title("Last import") + .visible(false) + .build(); + + let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18); + groups.append(&dropzone); + groups.append(&list_group); + groups.append(&actions); + groups.append(&progress_group); + groups.append(&summary_group); + let clamp = adw::Clamp::builder() + .maximum_size(640) + .child(&groups) + .build(); + let scroll = gtk4::ScrolledWindow::builder() + .vexpand(true) + .hscrollbar_policy(gtk4::PolicyType::Never) + .child(&clamp) + .build(); + + let page = gtk4::Box::new(gtk4::Orientation::Vertical, 12); + page.set_margin_top(18); + page.set_margin_bottom(18); + page.set_margin_start(18); + page.set_margin_end(18); + page.append(&header_clamp); + page.append(&scroll); + + let widgets = TakeoutWidgets { + list_group: list_group.clone(), + dropzone: dropzone.clone(), + page: page.clone().upcast(), + choose_button, + scan_button: scan_button.clone(), + import_button: import_button.clone(), + cancel_button: cancel_button.clone(), + clear_button, + back_button, + progress_group: progress_group.clone(), + progress_label: progress_label.clone(), + progress_bar: progress_bar.clone(), + summary_group: summary_group.clone(), + }; + (page.upcast(), widgets) +} + +pub(crate) fn wire_takeout(ui: &Rc, widgets: &TakeoutWidgets) { + // Drag-and-drop. The target sits on the whole page rather than the dashed + // rectangle, so a drag released anywhere counts; the rectangle only lights + // up to say where the drop is understood. + let target = gtk4::DropTarget::new( + gtk4::gdk::FileList::static_type(), + gtk4::gdk::DragAction::COPY, + ); + let ui_enter = ui.clone(); + target.connect_enter(move |_, _, _| { + ui_enter.takeout.dropzone.add_css_class("dropzone-active"); + gtk4::gdk::DragAction::COPY + }); + let ui_leave = ui.clone(); + target.connect_leave(move |_| { + ui_leave + .takeout + .dropzone + .remove_css_class("dropzone-active"); + }); + let ui_drop = ui.clone(); + target.connect_drop(move |_, value, _, _| { + ui_drop.takeout.dropzone.remove_css_class("dropzone-active"); + let Ok(files) = value.get::() else { + return false; + }; + let paths: Vec = files.files().iter().filter_map(|f| f.path()).collect(); + add_archives(&ui_drop, paths); + true + }); + widgets.page.add_controller(target); + + let ui_choose = ui.clone(); + widgets.choose_button.connect_clicked(move |_| { + // Multi-select on purpose: the whole export goes in one request. + let dialog = gtk4::FileDialog::builder() + .title("Select your Google Takeout archives") + .build(); + let filter = gtk4::FileFilter::new(); + filter.set_name(Some("Takeout archives")); + filter.add_pattern("*.zip"); + let filters = gio::ListStore::new::(); + filters.append(&filter); + dialog.set_filters(Some(&filters)); + + let ui = ui_choose.clone(); + dialog.open_multiple( + ui_window(&ui_choose).as_ref(), + gio::Cancellable::NONE, + move |res| { + let Ok(files) = res else { return }; + let paths: Vec = (0..files.n_items()) + .filter_map(|index| files.item(index)) + .filter_map(|object| object.downcast::().ok()) + .filter_map(|file| file.path()) + .collect(); + add_archives(&ui, paths); + }, + ); + }); + + let ui_clear = ui.clone(); + widgets.clear_button.connect_clicked(move |_| { + ui_clear.takeout.archives.borrow_mut().clear(); + repaint_archives(&ui_clear); + }); + + let ui_scan = ui.clone(); + widgets + .scan_button + .connect_clicked(move |_| start_import(&ui_scan, true)); + + // The import itself is confirmed: it uploads a whole photo library, which is + // hours of bandwidth and a large change to the account's timeline. + let ui_import = ui.clone(); + widgets.import_button.connect_clicked(move |_| { + let count = ui_import.takeout.archives.borrow().len(); + let ui = ui_import.clone(); + let dialog = adw::AlertDialog::builder() + .heading("Import this export?") + .body(format!( + "{count} archive(s) will be uploaded to Proton Photos, creating albums as they \ + appear in the export. This can take hours; you can stop it at any point and \ + what has been uploaded stays. Photos you already have are skipped.", + )) + .build(); + dialog.add_response("cancel", "Cancel"); + dialog.add_response("import", "Import"); + dialog.set_response_appearance("import", adw::ResponseAppearance::Suggested); + dialog.set_default_response(Some("cancel")); + dialog.set_close_response("cancel"); + dialog.connect_response(None, move |_, resp| { + if resp == "import" { + start_import(&ui, false); + } + }); + dialog.present(ui_window(&ui_import).as_ref()); + }); + + let ui_cancel = ui.clone(); + widgets.cancel_button.connect_clicked(move |_| { + let ui = ui_cancel.clone(); + ui.busy_begin(); + let rx = spawn_request(ui.dirs.control_socket(), Request::CancelImport); + glib::spawn_future_local(async move { + let result = rx.recv().await; + ui.busy_end(); + match result { + Ok(Ok(Response::Ok { .. })) => toast(&ui, "Stopping the import…"), + Ok(Ok(Response::Error { message, kind })) => { + toast_failure(&ui, "Couldn't stop the import", &message, kind); + } + _ => toast_error( + &ui, + "Couldn't stop the import", + "The mount service didn't respond.", + ), + } + }); + }); + + let ui_back = ui.clone(); + widgets + .back_button + .connect_clicked(move |_| ui_back.stack.set_visible_child_name("main")); + + repaint_archives(ui); +} + +/// Stage `paths`, keeping only `.zip` files and skipping ones already staged. +/// +/// Both the chooser and the drop target land here, so the filter is stated once. +/// A dropped folder (an already-extracted export) is rejected rather than walked: +/// the scan reads the archives' central directories, and an extracted tree has +/// none — silently importing half of it would be worse than saying no. +pub(crate) fn add_archives(ui: &Rc, paths: Vec) { + let mut added = 0usize; + let mut rejected = 0usize; + { + let mut staged = ui.takeout.archives.borrow_mut(); + for path in paths { + let is_zip = path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("zip")); + if !is_zip || !path.is_file() { + rejected += 1; + continue; + } + if staged.contains(&path) { + continue; + } + staged.push(path); + added += 1; + } + // Numbered parts drop in whatever order the file manager hands them + // over; sorting keeps the list readable and the request stable. + staged.sort(); + } + repaint_archives(ui); + if rejected > 0 { + toast( + ui, + "Only Takeout .zip files can be imported — extracted folders aren't supported.", + ); + } else if added > 0 { + toast(ui, &format!("Added {added} archive(s)")); + } +} + +/// Rebuild the staged-archive list and re-derive which actions are available. +pub(crate) fn repaint_archives(ui: &Rc) { + for row in ui.takeout.rows.borrow_mut().drain(..) { + ui.takeout.list_group.remove(&row); + } + + let staged = ui.takeout.archives.borrow().clone(); + let mut rows: Vec = Vec::new(); + if staged.is_empty() { + ui.takeout + .list_group + .set_description(Some(EMPTY_LIST_DESCRIPTION)); + let row = dim_row("No archives added yet."); + ui.takeout.list_group.add(&row); + rows.push(row.upcast()); + } else { + let mut total = 0u64; + for path in &staged { + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + total += size; + let row = adw::ActionRow::builder() + .title( + path.file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| path.display().to_string()), + ) + .subtitle(human_bytes(size)) + .build(); + let remove = gtk4::Button::builder() + .icon_name("edit-delete-symbolic") + .tooltip_text("Remove from this import") + .valign(gtk4::Align::Center) + .build(); + remove.add_css_class("flat"); + let ui_remove = ui.clone(); + let target = path.clone(); + remove.connect_clicked(move |_| { + ui_remove + .takeout + .archives + .borrow_mut() + .retain(|p| *p != target); + repaint_archives(&ui_remove); + }); + row.add_suffix(&remove); + ui.takeout.list_group.add(&row); + rows.push(row.upcast()); + } + ui.takeout.list_group.set_description(Some(&format!( + "{} archive(s), {} in total. {EMPTY_LIST_DESCRIPTION}", + staged.len(), + human_bytes(total) + ))); + } + *ui.takeout.rows.borrow_mut() = rows; + + sync_takeout_actions(ui); +} + +/// Enable exactly the actions that make sense right now: nothing to launch +/// without archives, nothing to launch while a run is in flight, and Stop only +/// while there is something to stop. +fn sync_takeout_actions(ui: &Rc) { + let has_files = !ui.takeout.archives.borrow().is_empty(); + let running = ui.takeout.running.get(); + ui.takeout.scan_button.set_sensitive(has_files && !running); + ui.takeout + .import_button + .set_sensitive(has_files && !running); + ui.takeout.clear_button.set_sensitive(has_files && !running); + ui.takeout.cancel_button.set_visible(running); +} + +/// Hand the staged set to the daemon. It acks immediately and works in the +/// background — an export is far past any socket timeout — so success here means +/// "started", and the page switches over to reporting progress. +fn start_import(ui: &Rc, dry_run: bool) { + let archives: Vec = ui + .takeout + .archives + .borrow() + .iter() + .map(|p| p.display().to_string()) + .collect(); + if archives.is_empty() { + return; + } + + ui.takeout.dry_run.set(dry_run); + ui.busy_begin(); + let rx = spawn_request( + ui.dirs.control_socket(), + Request::ImportTakeout { archives, dry_run }, + ); + let ui = ui.clone(); + glib::spawn_future_local(async move { + let result = rx.recv().await; + ui.busy_end(); + match result { + Ok(Ok(Response::Ok { .. })) => { + ui.takeout.running.set(true); + ui.takeout.summary_group.set_visible(false); + ui.takeout.progress_group.set_visible(true); + ui.takeout.progress_bar.set_fraction(0.0); + ui.takeout.progress_label.set_text(if dry_run { + "Reading archives…" + } else { + "Starting import…" + }); + sync_takeout_actions(&ui); + toast( + &ui, + if dry_run { + "Scanning the export…" + } else { + "Import started — you can leave this page, it keeps running." + }, + ); + } + Ok(Ok(Response::Error { message, kind })) => { + toast_failure(&ui, "Couldn't start the import", &message, kind); + } + _ => toast_error( + &ui, + "Couldn't start the import", + "The mount service didn't respond.", + ), + } + }); +} + +/// Poll the daemon for the import's state while the page is on screen, and paint +/// the report once a run has finished. Cheap and inflight-guarded, like the other +/// tick polls. +pub(crate) fn refresh_takeout(ui: &Rc) { + if ui.takeout.inflight.get() { + return; + } + ui.takeout.inflight.set(true); + let rx = spawn_request(ui.dirs.control_socket(), Request::ImportStatus); + let ui = ui.clone(); + glib::spawn_future_local(async move { + let result = rx.recv().await; + ui.takeout.inflight.set(false); + let Ok(Ok(Response::ImportStatus { running, summary })) = result else { + return; + }; + let was_running = ui.takeout.running.replace(running); + if running { + ui.takeout.progress_group.set_visible(true); + } else { + ui.takeout.progress_group.set_visible(false); + // An import runs for hours, so its ending is announced rather than + // left to be discovered: a toast for whoever is looking at the + // window, and a desktop notification for whoever is not. + if was_running { + let dry = ui.takeout.dry_run.get(); + let headline = if dry { + "Scan finished" + } else { + "Import finished" + }; + toast(&ui, headline); + if !dry { + let body = match &summary { + Some(s) if s.failed > 0 => { + format!("{} photos added, {} failed.", s.uploaded, s.failed) + } + Some(s) if s.cancelled => { + format!("Stopped after {} photos.", s.uploaded) + } + Some(s) => format!("{} photos added to Proton Photos.", s.uploaded), + None => "The Google Photos import has finished.".to_string(), + }; + notify("takeout-import", headline, &body); + } + } + } + if was_running != running { + sync_takeout_actions(&ui); + } + match summary { + Some(summary) => repaint_summary(&ui, &summary), + None => ui.takeout.summary_group.set_visible(false), + } + }); +} + +/// Paint the last run's report. +fn repaint_summary(ui: &Rc, summary: &ImportSummary) { + for row in ui.takeout.summary_rows.borrow_mut().drain(..) { + ui.takeout.summary_group.remove(&row); + } + let dry = ui.takeout.dry_run.get(); + ui.takeout + .summary_group + .set_title(if dry { "Scan result" } else { "Last import" }); + ui.takeout.summary_group.set_description(Some(if dry { + "Nothing was uploaded — this is what an import would do." + } else if summary.cancelled { + "Stopped early. Everything uploaded so far is in your timeline; run it again to \ + continue where it left off." + } else { + "Finished." + })); + + // Only lines that carry information: a run with no failures should not have + // to show a "0 failed" row for the user to read past. + let mut lines: Vec<(String, String)> = vec![ + ("Photos in the export".into(), summary.found.to_string()), + ( + if dry { "Would upload" } else { "Uploaded" }.into(), + summary.uploaded.to_string(), + ), + ]; + if summary.duplicates > 0 { + lines.push(( + "Already in Proton Photos".into(), + summary.duplicates.to_string(), + )); + } + if summary.albums_created > 0 { + lines.push(( + if dry { + "Albums to create" + } else { + "Albums created" + } + .into(), + summary.albums_created.to_string(), + )); + } + if summary.album_links > 0 { + lines.push(( + "Photos filed into albums".into(), + summary.album_links.to_string(), + )); + } + if summary.skipped_trashed > 0 { + lines.push(( + "Skipped (in Google's trash)".into(), + summary.skipped_trashed.to_string(), + )); + } + if summary.bytes > 0 { + lines.push(( + if dry { + "Data to upload" + } else { + "Data uploaded" + } + .into(), + human_bytes(summary.bytes), + )); + } + if summary.failed > 0 { + lines.push(("Failed".into(), summary.failed.to_string())); + } + + let mut rows = Vec::new(); + for (title, value) in lines { + let row = adw::ActionRow::builder() + .title(title) + .subtitle(value) + .build(); + row.add_css_class("property"); + ui.takeout.summary_group.add(&row); + rows.push(row); + } + *ui.takeout.summary_rows.borrow_mut() = rows; + ui.takeout.summary_group.set_visible(true); +} + +/// Paint the import's progress from the transfer snapshot the Settings tick +/// already fetched, so this page costs no extra poll. A job with no known total +/// pulses instead of showing a fraction, the same as the Activity list. +pub(crate) fn takeout_progress(ui: &Rc, jobs: &[JobItem]) { + let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else { + return; + }; + ui.takeout.progress_group.set_visible(true); + if job.total > 0 { + ui.takeout + .progress_bar + .set_fraction((job.done as f64 / job.total as f64).min(1.0)); + ui.takeout + .progress_label + .set_text(&format!("{} of {} — {}", job.done, job.total, job.detail)); + } else { + ui.takeout.progress_bar.pulse(); + ui.takeout + .progress_label + .set_text(if job.detail.is_empty() { + "Working…" + } else { + &job.detail + }); + } +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5384099..0ff4b5c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,44 @@ release ships. Migrations are forward-only — a database written by a newer cli refuse-to-open, not a downgrade, so rolling back a release means restoring the cache from scratch (user data in `staging/` and `recovery/` is never touched by this). +## [Unreleased] + +Google Photos import. No schema change. Schema: **28**. + +### Added +- **Import a Google Photos Takeout export into Proton Photos** — `pdfs import-google-photos + takeout-*.zip` (add `--dry-run` to see what it would do, `--wait` to follow it), or the import + button in the Photos header, which multi-selects the export's zips. Albums in the export are + recreated as Proton albums and their photos filed into them; year folders and `Archive/` import + into the timeline; `Trash/` is skipped. Capture times and favourites come from Google's per-photo + JSON sidecars, including the truncated and `(N)`-swapped sidecar names Google produces. +- Photos already on the account are **not re-uploaded**: the import asks the photos volume which of + the export's names exist (batched, 150 per request), reads and SHA-1s only those files, and skips + the ones whose content digest matches too — Proton's own duplicate rule. A library already partly + synced by Proton Photos on a phone therefore imports only what is missing, and an interrupted + import resumes by being run again. Google keeps a full copy of a photo in every album folder it + appears in; those fold into one upload that joins several albums. +- The import streams each photo straight out of its zip, so an export needs no scratch space to + unpack into, and reports as a job in `pdfs transfers` and the GUI's activity view. `pdfs + import-status` shows the counts, `pdfs cancel-import` stops it — a cancelled run still files what + it uploaded into its albums. +- **An Import page in the GUI** (Settings → *Import from Google Photos*, or the import button in + the Photos header). Archives are staged by drag-and-drop onto the page or through the file + chooser, listed with their sizes so the set can be checked before anything is sent, and launched + as either a Scan (dry run) or a confirmed Import. The page shows live progress off the import's + job, keeps the finished run's report on screen, and can stop a run in flight; because the import + outlives the page, it is polled wherever the user navigates and its completion is reported as a + desktop notification. + +- SDK (`proton-sdk-rs`): album **writes** — `ProtonPhotosClient::create_album` and + `add_photos_to_album`, ported from the TypeScript SDK (upstream C# has no album write API), plus + batch duplicate detection (`find_duplicates_many`, `name_collisions`). Adding a photo from an + album *shared with us* (a different volume) is still unported and fails that photo's outcome. + +### Fixed +- Settings showed the local cache as "X of 0 B used" when no cache budget was set. An unset budget + means *unlimited*, so it now reads "X cached — no limit set" rather than implying a zero cap. + ## [1.8.2] — 2026-08-17 Closes the two findings 1.8.1's live verification produced (B85, B86) and finishes the data-safety diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 83efb59..97af1dc 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -14,7 +14,8 @@ Consolidated from the historical feature roadmap; current bug status lives in [` * **Custom FUSE Mount Options (Horizon 1)**: Expose custom mounting parameters in configuration (e.g. `ro` for read-only, `direct_io` to bypass kernel caching, `allow_other`). Auto-generate systemd user mount units upon sync folder configuration. * **Symbolic Link & Hard Link Virtualization (Horizon 2)**: Add symbolic link virtualization inside the metadata store (`NodeType::Symlink`) by intercepting `readlink(2)` and `symlink(2)`. Targets should sync remotely as small encrypted metadata placeholder files on Drive (`proton-vfs-symlink: `). * **Desktop File Manager Integration (Horizon 2)**: Develop shell extension plugins for popular Linux file managers (Nautilus Python plugin, Dolphin Service Menus, Thunar custom actions) to offer context-menu shortcuts ("Pin", "Unpin", "Copy Share Link", "View Version History") and file overlay sync badges. -* **Integrated Photo Gallery Enhancements (Horizon 2)**: Support creating and editing Proton Photo Albums (album *writes* are unported in the SDK too) and create an auto-upload Pictures folder pipeline. Exif display, the date-based timeline scrubber and favourites are implemented. +* **Integrated Photo Gallery Enhancements (Horizon 2)**: Album *editing* — rename, cover photo, delete, and removing photos from an album — is still unported in the SDK (TS `updateAlbum` / `deleteAlbum` / `removePhotos`), as is adding a photo that lives on someone else's volume (TS `copyPhoto`). Album **creation** and adding own-volume photos landed with the Google Photos import. Still open: an auto-upload Pictures folder pipeline, and a GUI for creating albums by hand. Exif display, the date-based timeline scrubber and favourites are implemented. +* **Google Photos import follow-ups**: the importer uploads photos one at a time — a large export would finish sooner with a bounded concurrent upload pool. Trash folders are recognized by a hardcoded list of localized names ([`takeout.rs`](../crates/pdfs-core/src/takeout.rs) `TRASH_FOLDERS`); a locale not on it imports its trash as ordinary photos. Motion-photo `.MP`/`.MV` sidecar videos are ignored rather than uploaded as related photos. Live validation against a real export is pending. * **Random-Access Media Streaming seek support (Horizon 2)**: Intercept out-of-order sparse reads (player seeks) in `pdfs-fuse` and prioritize downloading blocks around the seek offset, aborting/postponing sequential pre-fetches. * **Dynamic Sync Dashboard & Queue Visualization (Horizon 2)**: GUI real-time transfer list (progress bars, speed, ETA), sync history feed, storage usage breakdown, and global pause/resume sync controls. * **Interactive Terminal UI (TUI) Mode (Horizon 3)**: Implement `pdfs tui` using the `ratatui` crate to show transfer speed, sync status, active transfer queue with progress bars, and scrolling daemon logs.