diff --git a/Cargo.lock b/Cargo.lock index 95e7d89..0f2ca1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3640,6 +3640,7 @@ dependencies = [ "parking_lot 0.13.0", "pdfs-core", "proton-drive-rs", + "serde", "serde_json", "sha1 0.11.0", "tokio", @@ -3890,9 +3891,9 @@ dependencies = [ [[package]] name = "proton-drive-rs" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a22e621abbc1c61b4640728a7de969c16b66d5d983b59c638d501fda25a7b9" +checksum = "4265c2cf3bdc59ee273d7363c8b6a7cb4e98be6102ffa18144c4d8ebda00737b" dependencies = [ "async-trait", "base64 0.23.1", @@ -3918,9 +3919,9 @@ dependencies = [ [[package]] name = "proton-sdk" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd55a5e8b6600f3aa282ebbdf13ff366431ff9aae84d4eb20b0ef06d598919fe" +checksum = "fb98ab3e336ed95f82368a50348dad00e5fc919e982d748e83e844afcadbd5d8" dependencies = [ "aes-gcm 0.11.1", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 6929f66..719fb60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"] categories = ["filesystem", "command-line-utilities"] [workspace.dependencies] -proton-sdk = { version = "0.11.0" } -proton-drive-rs = { version = "0.11.0" } +proton-sdk = { version = "0.12.0" } +proton-drive-rs = { version = "0.12.0" } chrono = { version = "0.4", default-features = false, features = ["std"] } # Internal crates. diff --git a/crates/pdfs-cli/src/main.rs b/crates/pdfs-cli/src/main.rs index 87fb67a..c4b86eb 100644 --- a/crates/pdfs-cli/src/main.rs +++ b/crates/pdfs-cli/src/main.rs @@ -228,14 +228,19 @@ enum Command { /// numbered parts, and a photo's metadata often sits in a different part /// than the photo. Albums in the export are recreated as Proton albums, and /// photos already on the account (same name and same content) are skipped, - /// so an interrupted import can simply be run again. + /// so an interrupted import can simply be run again, or resumed with + /// `--resume` without naming the archives again. ImportGooglePhotos { /// The export's `.zip` files (e.g. `takeout-*.zip`). - #[arg(required = true, value_hint = ValueHint::FilePath)] + #[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)] archives: Vec, /// Report what would be imported without uploading anything. - #[arg(long)] + #[arg(long, conflicts_with = "resume")] dry_run: bool, + /// Start the last import again with the same archives, after it was + /// interrupted, stopped or failed. + #[arg(long, conflicts_with = "archives")] + resume: bool, /// Stay attached and print progress until the import finishes. #[arg(long)] wait: bool, @@ -1030,8 +1035,9 @@ fn main() -> Result<()> { Command::ImportGooglePhotos { archives, dry_run, + resume, wait, - } => cmd_import_google_photos(archives, dry_run, wait), + } => cmd_import_google_photos(archives, dry_run, resume, wait), Command::RedatePhotos { dry_run, limit, @@ -2916,24 +2922,34 @@ fn photo_count(n: usize) -> String { } } -fn cmd_import_google_photos(archives: Vec, dry_run: bool, wait: bool) -> Result<()> { - // The daemon opens the archives itself off the shared filesystem, so it gets - // absolute paths — the same arrangement `pdfs upload` uses. - let mut absolute = Vec::with_capacity(archives.len()); - for archive in &archives { - let path = std::fs::canonicalize(archive) - .with_context(|| format!("resolve {}", archive.display()))?; - absolute.push( - path.to_str() - .ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))? - .to_owned(), - ); - } +fn cmd_import_google_photos( + archives: Vec, + dry_run: bool, + resume: bool, + wait: bool, +) -> Result<()> { + let request = if resume { + CtlRequest::ResumeImport + } else { + // The daemon opens the archives itself off the shared filesystem, so it + // gets absolute paths — the same arrangement `pdfs upload` uses. + let mut absolute = Vec::with_capacity(archives.len()); + for archive in &archives { + let path = std::fs::canonicalize(archive) + .with_context(|| format!("resolve {}", archive.display()))?; + absolute.push( + path.to_str() + .ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))? + .to_owned(), + ); + } + CtlRequest::ImportTakeout { + archives: absolute, + dry_run, + } + }; - match control_request(CtlRequest::ImportTakeout { - archives: absolute, - dry_run, - })? { + match control_request(request)? { CtlResponse::Ok { message } => println!("{message}"), CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), other => bail!("unexpected response: {other:?}"), @@ -2945,20 +2961,34 @@ fn cmd_import_google_photos(archives: Vec, dry_run: bool, wait: bool) - // Poll rather than stream: the control protocol is request/response, and an // import runs for hours — a held-open socket would be the only long-lived - // connection the daemon has. + // connection the daemon has. A line is printed when the step changes and + // every few percent of it, not on every poll. + let mut last_line = String::new(); + let mut last_printed = std::time::Instant::now(); loop { std::thread::sleep(std::time::Duration::from_secs(2)); match control_request(CtlRequest::ImportStatus)? { CtlResponse::ImportStatus { running: true, - summary: _, - } => continue, - CtlResponse::ImportStatus { summary, .. } => { - print_import_summary(summary.as_ref()); - return finished_run( - "the import", - summary.map(|s| (s.failed, s.cancelled, s.error)), - ); + progress, + .. + } => { + let line = progress.as_ref().map(progress_line).unwrap_or_default(); + let step_changed = line.split(':').next() != last_line.split(':').next(); + if !line.is_empty() + && line != last_line + && (step_changed || last_printed.elapsed().as_secs() >= 30) + { + println!("{line}"); + last_line = line; + last_printed = std::time::Instant::now(); + } + } + CtlResponse::ImportStatus { + summary, resumable, .. + } => { + print_import_summary(summary.as_ref(), &resumable); + return import_outcome(summary.as_ref()); } CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), other => bail!("unexpected response: {other:?}"), @@ -2966,6 +2996,47 @@ fn cmd_import_google_photos(archives: Vec, dry_run: bool, wait: bool) - } } +/// One line saying where a running import is. +fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String { + use pdfs_core::control::ImportPhase; + let step = match progress.phase { + ImportPhase::Scanning => "Reading archives", + ImportPhase::Checking => "Checking for photos already on Proton", + ImportPhase::Uploading => "Uploading", + ImportPhase::Albums => "Filing photos into albums", + }; + match (progress.total, &progress.current) { + (0, _) => format!("{step}…"), + (total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done), + (total, None) => format!("{step}: {} of {total}", progress.done), + } +} + +/// How an import `--wait` followed ended, as the exit status: an error unless +/// it did everything it set out to do, so a script can tell. +fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> { + let Some(summary) = summary else { + bail!("the import ended without a result; the service may have restarted"); + }; + if let Some(error) = &summary.error { + bail!("the import failed: {error}"); + } + if summary.interrupted { + bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue"); + } + if summary.cancelled { + bail!("the import was cancelled"); + } + if summary.failed > 0 || !summary.unreadable_archives.is_empty() { + bail!( + "the import finished, but {} photo(s) and {} archive(s) failed", + summary.failed, + summary.unreadable_archives.len() + ); + } + Ok(()) +} + fn cmd_redate_photos( dry_run: bool, limit: Option, @@ -3100,11 +3171,19 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) { fn cmd_import_status() -> Result<()> { match control_request(CtlRequest::ImportStatus)? { - CtlResponse::ImportStatus { running, summary } => { + CtlResponse::ImportStatus { + running, + summary, + progress, + resumable, + } => { if running { - println!("Import running. `pdfs transfers` shows its progress."); + match progress.as_ref() { + Some(progress) => println!("Import running. {}", progress_line(progress)), + None => println!("Import running."), + } } - print_import_summary(summary.as_ref()); + print_import_summary(summary.as_ref(), &resumable); } CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)), other => bail!("unexpected response: {other:?}"), @@ -3121,18 +3200,28 @@ fn cmd_cancel_import() -> Result<()> { Ok(()) } -fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) { +fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) { let Some(summary) = summary else { println!("No import has run yet."); return; }; + if summary.dry_run { + println!("Scan only; nothing was uploaded."); + } if let Some(error) = &summary.error { - println!("Failed : {error}"); - return; + println!("Stopped by : {error}"); } println!("Photos in export : {}", summary.found); - println!("Uploaded : {}", summary.uploaded); + if summary.dry_run { + println!("Would upload : {}", summary.to_upload); + println!("Bytes to upload : {}", summary.to_upload_bytes); + } else { + println!("Uploaded : {}", summary.uploaded); + } println!("Already on Proton: {}", summary.duplicates); + if summary.live_photos > 0 { + println!("Live photos : {}", summary.live_photos); + } if summary.skipped_trashed > 0 { println!("Skipped (trash) : {}", summary.skipped_trashed); } @@ -3145,12 +3234,20 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) { if summary.failed > 0 { println!("Failed : {}", summary.failed); } + for archive in &summary.unreadable_archives { + println!("Unreadable : {archive}"); + } if summary.bytes > 0 { println!("Uploaded bytes : {}", summary.bytes); } - if summary.cancelled { + if summary.interrupted { + println!("The service stopped before the import finished."); + } else if summary.cancelled { println!("Import was cancelled before it finished."); } + if !resumable.is_empty() { + println!("Run `pdfs import-google-photos --resume` to continue it."); + } } fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> { @@ -4509,6 +4606,51 @@ mod login_wait_tests { assert!(finished_run("the import", None).is_err()); } + #[test] + fn a_waited_for_import_that_did_not_finish_exits_with_an_error() { + use pdfs_core::control::ImportSummary; + let done = ImportSummary { + uploaded: 4, + ..Default::default() + }; + assert!(import_outcome(Some(&done)).is_ok()); + assert!(import_outcome(None).is_err()); + for broken in [ + ImportSummary { + interrupted: true, + ..Default::default() + }, + ImportSummary { + unreadable_archives: vec!["/exports/takeout-002.zip".into()], + ..Default::default() + }, + ImportSummary { + error: Some("check for existing photos: offline".into()), + ..Default::default() + }, + ] { + assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}"); + } + let interrupted = import_outcome(Some(&ImportSummary { + interrupted: true, + ..Default::default() + })); + assert!(interrupted.unwrap_err().to_string().contains("--resume")); + } + + #[test] + fn resume_needs_no_archives_and_excludes_them() { + let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap(); + assert!(matches!( + cli.command, + Command::ImportGooglePhotos { resume: true, .. } + )); + assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err()); + assert!( + Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err() + ); + } + #[test] fn waiting_for_a_session_stops_hammering_the_keyring() { let mut checks_left = 20; diff --git a/crates/pdfs-core/src/config.rs b/crates/pdfs-core/src/config.rs index df8ad87..30ca326 100644 --- a/crates/pdfs-core/src/config.rs +++ b/crates/pdfs-core/src/config.rs @@ -562,6 +562,12 @@ impl AppDirs { self.state_dir().join("control.sock") } + /// The Google Photos import's last run: its archives and how it ended, so + /// an import cut short by a restart can be resumed. + pub fn takeout_import_path(&self) -> PathBuf { + self.state_dir().join("takeout-import.json") + } + /// Unix domain socket the tray uses to ensure single instance. pub fn tray_socket(&self) -> PathBuf { self.state_dir().join("tray.sock") diff --git a/crates/pdfs-core/src/control.rs b/crates/pdfs-core/src/control.rs index 34cb9fc..7ef2881 100644 --- a/crates/pdfs-core/src/control.rs +++ b/crates/pdfs-core/src/control.rs @@ -279,6 +279,11 @@ pub enum Request { /// and files what it has uploaded into its albums, then reports /// `cancelled`. Replies with [`Response::Ok`]. CancelImport, + /// Start the last import again with the same archives, after it was + /// interrupted, cancelled or failed. Photos it already uploaded are found + /// on the account and skipped. Replies like [`Request::ImportTakeout`]; + /// [`ErrorKind::NotFound`] when there is nothing to resume. + ResumeImport, /// Upload the photo at `source_path` under the given name and media type. /// /// A path, not the bytes: this protocol is line-delimited JSON, and @@ -1689,10 +1694,75 @@ pub struct ImportSummary { pub bytes: u64, /// True when the run stopped early because it was cancelled. pub cancelled: bool, - /// Why the run as a whole failed, when it did, such as an archive that - /// is not a zip. The counts are then empty. + /// Why the run as a whole stopped early, when it did, such as the + /// account being unreachable. The counts say how far it got. #[serde(default)] pub error: Option, + /// What kind of failure [`error`](Self::error) is, so a front-end can word + /// it in the user's language. The text itself is the daemon's English. + #[serde(default)] + pub error_kind: Option, + /// True for a scan that uploads nothing ([`Request::ImportTakeout`] with + /// `dry_run`). [`to_upload`](Self::to_upload) then says what an import + /// would upload. + #[serde(default)] + pub dry_run: bool, + /// Photos not yet on the account: what a dry run would upload, and what a + /// real run set out to upload once the duplicates were known. + #[serde(default)] + pub to_upload: usize, + /// Bytes of [`to_upload`](Self::to_upload). + #[serde(default)] + pub to_upload_bytes: u64, + /// Live and motion photos whose video was filed with its still. + #[serde(default)] + pub live_photos: usize, + /// True when the service stopped while the import ran (a shutdown or a + /// restart). [`Request::ResumeImport`] picks it up again. + #[serde(default)] + pub interrupted: bool, + /// Archives that could not be read and were left out, by path. + #[serde(default)] + pub unreadable_archives: Vec, +} + +impl ImportSummary { + /// Whether the run did all it set out to do. + pub fn complete(&self) -> bool { + self.error.is_none() + && !self.cancelled + && !self.interrupted + && self.failed == 0 + && self.unreadable_archives.is_empty() + } +} + +/// Which step a running Takeout import is on. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ImportPhase { + /// Reading the archives' directories and metadata. + #[default] + Scanning, + /// Asking the account which photos it already has. + Checking, + /// Uploading the photos that are new. + Uploading, + /// Creating albums and filing photos into them. + Albums, +} + +/// Where a running import is ([`Response::ImportStatus`]). +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)] +pub struct ImportProgress { + pub phase: ImportPhase, + /// Steps done in this phase. + pub done: u64, + /// Steps in this phase; `0` while not yet known. + pub total: u64, + /// The photo being uploaded, during [`ImportPhase::Uploading`]. + #[serde(default)] + pub current: Option, } /// One album in a [`Response::Albums`] listing. @@ -2208,8 +2278,17 @@ pub enum Response { /// report; `summary` is `None` when no import has run this session. ImportStatus { running: bool, + /// The counts so far while running, the final report after. #[serde(default)] summary: Option, + /// The current step, while running. + #[serde(default)] + progress: Option, + /// The archives of the last run, when it can be resumed with + /// [`Request::ResumeImport`]: it was a real import that did not + /// finish. Survives a restart of the service. + #[serde(default)] + resumable: Vec, }, /// How the photo re-date is doing (reply to [`Request::RedateStatus`]). /// `running` is false once it has finished, when `summary` is the final diff --git a/crates/pdfs-core/src/takeout.rs b/crates/pdfs-core/src/takeout.rs index 16d0542..fba5934 100644 --- a/crates/pdfs-core/src/takeout.rs +++ b/crates/pdfs-core/src/takeout.rs @@ -25,7 +25,9 @@ //! The service folder (`Google Photos`) is localized, as are the year folders //! (`Photos from 2019`) and the trash folder, so nothing here matches on the //! service name: the first component is dropped when it is the export root, the -//! second is the service, and the rest is `/`. A folder is an +//! second is the service, and the rest is `/`. The Photos service +//! is the one whose sidecars carry a `photoTakenTime`; other services in the same +//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an //! **album** when it carries an album metadata sidecar naming it; anything else //! is a plain bucket that imports into the timeline only. Trash is matched by //! name against [`TRASH_FOLDERS`] — the one place a localized name is @@ -121,8 +123,14 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[ ("m2ts", "video/mp2t"), ("mpg", "video/mpeg"), ("mpeg", "video/mpeg"), + // The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`. + ("mp", "video/mp4"), ]; +/// Sidecars bigger than this are not read. A real one is a few kilobytes; a +/// `.json` this large is some other service's data and not worth the memory. +const SIDECAR_MAX_BYTES: u64 = 1024 * 1024; + /// The media type for `name`'s extension, or `None` when it is not media this /// import handles. pub fn media_type_of(name: &str) -> Option<&'static str> { @@ -147,6 +155,13 @@ pub enum Bucket { Archived, } +/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct GeoPoint { + pub latitude: f64, + pub longitude: f64, +} + /// One media file found in the export. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TakeoutPhoto { @@ -162,14 +177,49 @@ pub struct TakeoutPhoto { pub size: u64, /// Media type from the extension. pub media_type: String, - /// Capture time in epoch seconds, from `photoTakenTime` (falling back to - /// `creationTime`). `None` when the photo has no sidecar — the upload then - /// defaults it to now, as the SDK does. + /// Capture time in epoch seconds, from the sidecar's `photoTakenTime`, + /// else from a date in the file name. `None` when neither has one; the + /// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time). pub capture_time: Option, + /// The sidecar's `creationTime`: when the photo reached Google Photos, not + /// when it was taken. Only a last resort for the capture time. + #[serde(default)] + pub creation_time: Option, + /// Where the photo was taken, from the sidecar's `geoData` (or + /// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped. + #[serde(default)] + pub location: Option, /// The sidecar's `favorited` flag. pub favorite: bool, /// Which bucket the file was found in. pub bucket: Bucket, + /// Pairs the halves of a live or motion photo: a still and a video in the + /// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`, + /// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half. + #[serde(default)] + pub live_key: Option, +} + +impl TakeoutPhoto { + /// Whether this is the video half of a pair, as opposed to the still. + pub fn is_video(&self) -> bool { + self.media_type.starts_with("video/") + } + + /// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather + /// than an Apple live photo. + pub fn is_motion_photo(&self) -> bool { + self.live_key + .as_deref() + .is_some_and(|key| key.ends_with(".mp")) + } +} + +/// An archive the scan had to leave out, and why. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UnreadableArchive { + pub path: PathBuf, + pub reason: String, } /// Everything one scan found, ready to be turned into an import plan. @@ -183,8 +233,16 @@ pub struct TakeoutScan { /// Media files skipped because they sit in the trash bucket. pub skipped_trashed: usize, /// Entries that were neither media nor a sidecar (`archive_browser.html`, - /// activity `.csv`s, `.mp` motion-photo parts). + /// activity `.csv`s), and media from other Google services in the same + /// export (`Google Drive/`, `YouTube/`). pub skipped_other: usize, + /// Archives that could not be opened or are not a zip. The rest of the + /// export is still imported. + #[serde(default)] + pub unreadable_archives: Vec, + /// Entries inside an otherwise readable archive that could not be read. + #[serde(default)] + pub unreadable_entries: usize, } impl TakeoutScan { @@ -195,7 +253,7 @@ impl TakeoutScan { } /// Google's per-photo sidecar. Only the fields the import uses are read; the -/// file also carries geo data, people tags and view counts. +/// file also carries people tags, a description and view counts. #[derive(Debug, Default, Deserialize)] struct PhotoSidecar { #[serde(default)] @@ -205,19 +263,69 @@ struct PhotoSidecar { #[serde(rename = "creationTime", default)] creation_time: Option, #[serde(default)] - favorited: bool, + favorited: Option, + /// The position as edited in Google Photos, else as the camera wrote it. + #[serde(rename = "geoData", default)] + geo_data: Option, + #[serde(rename = "geoDataExif", default)] + geo_data_exif: Option, } -/// A sidecar timestamp: epoch seconds, as a *string*. +impl PhotoSidecar { + fn location(&self) -> Option { + self.geo_data + .as_ref() + .and_then(SidecarGeo::point) + .or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point)) + } +} + +/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number. #[derive(Debug, Deserialize)] struct SidecarTime { #[serde(default)] - timestamp: Option, + timestamp: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum SidecarNumber { + Number(i64), + Text(String), } impl SidecarTime { + /// The timestamp, when it is a real moment. `0` and negative values are + /// Google's "unknown" and would file the photo under 1970. fn epoch_seconds(&self) -> Option { - self.timestamp.as_ref()?.parse().ok() + let seconds = match self.timestamp.as_ref()? { + SidecarNumber::Number(seconds) => *seconds, + SidecarNumber::Text(text) => text.trim().parse().ok()?, + }; + (seconds > 0).then_some(seconds) + } +} + +/// `geoData` in a sidecar. All-zero means Google has no position. +#[derive(Debug, Deserialize)] +struct SidecarGeo { + #[serde(default)] + latitude: Option, + #[serde(default)] + longitude: Option, +} + +impl SidecarGeo { + fn point(&self) -> Option { + let (latitude, longitude) = (self.latitude?, self.longitude?); + let on_globe = latitude.is_finite() + && longitude.is_finite() + && (-90.0..=90.0).contains(&latitude) + && (-180.0..=180.0).contains(&longitude); + (on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint { + latitude, + longitude, + }) } } @@ -262,21 +370,58 @@ impl RawEntry { /// is one album, and a sidecar in part 3 is matched to its photo in part 1. /// Reads only the central directory and the JSON sidecars — no media bytes, so /// this stays fast on a 200 GB export. +/// +/// An archive that cannot be opened is left out and listed in +/// [`TakeoutScan::unreadable_archives`]; the rest still imports. pub fn scan_archives(archives: &[PathBuf]) -> CoreResult { - let mut entries: Vec = Vec::new(); - let mut sidecars: HashMap> = HashMap::new(); + scan_archives_until(archives, || false) +} + +/// [`scan_archives`] that gives up when `stop` returns true, checked between +/// archives and every few hundred entries. +pub fn scan_archives_until( + archives: &[PathBuf], + stop: impl Fn() -> bool, +) -> CoreResult { + let mut entries: Vec<(String, RawEntry)> = Vec::new(); + let mut sidecars: Vec<(String, String, Vec)> = Vec::new(); + let mut unreadable_archives = Vec::new(); + let mut unreadable_entries = 0; for (index, path) in archives.iter().enumerate() { - let file = std::fs::File::open(path) - .map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?; - let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| { - CoreError::invalid(format!("{} is not a readable zip: {e}", path.display())) - })?; + if stop() { + return Err(CoreError::conflict("the import was cancelled")); + } + let opened = std::fs::File::open(path) + .map_err(|e| e.to_string()) + .and_then(|file| { + zip::ZipArchive::new(std::io::BufReader::new(file)) + .map_err(|e| format!("not a readable zip: {e}")) + }); + let mut zip = match opened { + Ok(zip) => zip, + Err(reason) => { + tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive"); + unreadable_archives.push(UnreadableArchive { + path: path.clone(), + reason, + }); + continue; + } + }; for position in 0..zip.len() { - let mut entry = zip.by_index(position).map_err(|e| { - CoreError::invalid(format!("{}: unreadable entry: {e}", path.display())) - })?; + if position % 512 == 0 && stop() { + return Err(CoreError::conflict("the import was cancelled")); + } + let mut entry = match zip.by_index(position) { + Ok(entry) => entry, + Err(e) => { + tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry"); + unreadable_entries += 1; + continue; + } + }; if entry.is_dir() { continue; } @@ -286,32 +431,92 @@ pub fn scan_archives(archives: &[PathBuf]) -> CoreResult { continue; }; let name = name.to_string_lossy().replace('\\', "/"); - let Some(relative) = strip_export_prefix(&name) else { + let Some((service, relative)) = strip_export_prefix(&name) else { continue; }; + let (service, relative) = (service.to_string(), relative.to_string()); let size = entry.size(); // Sidecars are read now, while the entry is open: they are a few // hundred bytes each and reopening the archive per photo later would // be a second full pass. if relative.to_ascii_lowercase().ends_with(".json") { - let mut body = Vec::with_capacity(size.min(64 * 1024) as usize); - if entry.read_to_end(&mut body).is_ok() { - sidecars.insert(relative.to_string(), body); + if size <= SIDECAR_MAX_BYTES { + let mut body = Vec::with_capacity(size as usize); + if entry.read_to_end(&mut body).is_ok() { + sidecars.push((service, relative, body)); + } else { + unreadable_entries += 1; + } } continue; } - - entries.push(RawEntry { - archive: index, - entry: name.clone(), - relative: relative.to_string(), - size, - }); + entries.push(( + service, + RawEntry { + archive: index, + entry: name, + relative, + size, + }, + )); } } - Ok(assemble(entries, &sidecars)) + let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars); + let mut scan = assemble(entries, &sidecars); + scan.skipped_other += other_services; + scan.unreadable_archives = unreadable_archives; + scan.unreadable_entries = unreadable_entries; + Ok(scan) +} + +/// Drop what other Google services put into the same export. +/// +/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to +/// `Google Photos/`, and their media must not land in the photo timeline. The +/// service folder's name is localized (`Google Fotos`), so the Photos service +/// is recognized by its content instead: it is the folder whose sidecars carry +/// a `photoTakenTime`. When no folder does — an export without sidecars, or +/// one re-zipped without the service level — everything is kept, as before. +/// +/// Returns the entries and sidecars to import, and how many media files were +/// left out. +fn keep_photos_services( + entries: Vec<(String, RawEntry)>, + sidecars: Vec<(String, String, Vec)>, +) -> (Vec, HashMap>, usize) { + #[derive(Deserialize)] + struct Marker { + #[serde(rename = "photoTakenTime")] + photo_taken_time: Option, + } + let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for (service, _, body) in &sidecars { + if photos_services.contains(service.as_str()) { + continue; + } + if serde_json::from_slice::(body).is_ok_and(|m| m.photo_taken_time.is_some()) { + photos_services.insert(service.as_str()); + } + } + let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service); + + let mut skipped = 0; + let mut kept_entries = Vec::with_capacity(entries.len()); + for (service, raw) in entries { + if keep(&service) { + kept_entries.push(raw); + } else if media_type_of(raw.file_name()).is_some() { + skipped += 1; + } + } + let kept_sidecars = sidecars + .iter() + .filter(|(service, _, _)| keep(service)) + .map(|(_, relative, body)| (relative.clone(), body.clone())) + .collect(); + (kept_entries, kept_sidecars, skipped) } /// Build the scan from raw entries plus the sidecar bodies keyed by their path @@ -403,19 +608,19 @@ fn assemble(entries: Vec, sidecars: &HashMap>) -> Take .unwrap_or(&file_name) .to_string(); - // Sidecar first, then the date stamped into the name. + // Sidecar first, then the date stamped into the name. `creationTime` + // is when the photo reached Google, which for a scan or an old photo + // uploaded late is years off; the importer uses it only after EXIF. let capture_time = sidecar .photo_taken_time .as_ref() .and_then(SidecarTime::epoch_seconds) - .or_else(|| { - sidecar - .creation_time - .as_ref() - .and_then(SidecarTime::epoch_seconds) - }) .or_else(|| capture_time_from_name(&name)) .or_else(|| capture_time_from_name(&file_name)); + let creation_time = sidecar + .creation_time + .as_ref() + .and_then(SidecarTime::epoch_seconds); scan.photos.push(TakeoutPhoto { archive: raw.archive, @@ -424,31 +629,57 @@ fn assemble(entries: Vec, sidecars: &HashMap>) -> Take size: raw.size, media_type: media_type.to_string(), capture_time, - favorite: sidecar.favorited, + creation_time, + location: sidecar.location(), + favorite: sidecar.favorited.unwrap_or(false), bucket, + live_key: live_key(&folder, &file_name), }); } scan } -/// Drop the export root (`Takeout/`) and the localized service folder from a -/// path inside the archive, leaving `/` or ``. +/// Split a path inside the archive into the localized service folder and the +/// rest, dropping the export root (`Takeout/`): `("Google Fotos", +/// "/")`. /// /// `None` for a path with no service folder at all (the export's own /// `archive_browser.html`), which carries nothing to import. -fn strip_export_prefix(path: &str) -> Option<&str> { +fn strip_export_prefix(path: &str) -> Option<(&str, &str)> { let mut rest = path; - // The root is usually `Takeout/`, but an export unzipped and re-zipped by - // the user may not have it — so the root is dropped only when a service - // folder follows it, which the component count decides. + // The root is `Takeout/`, but an export unzipped and re-zipped by the user + // may not have it. `Google Photos//` then has three parts like + // `Takeout/Google Photos/`, so the count alone cannot tell them + // apart: a root of another name is only taken for one with four parts. + let (first, after) = rest.split_once('/')?; let components = rest.split('/').count(); - if components >= 3 { - rest = rest.split_once('/')?.1; + if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 { + rest = after; } - // Drop the service folder (`Google Photos`, `Google Fotos`, …). - let (_service, tail) = rest.split_once('/')?; - Some(tail) + // The service folder (`Google Photos`, `Google Fotos`, …). + rest.split_once('/') +} + +/// The key that pairs the still and the video of one live or motion photo, or +/// `None` when `file_name` cannot be either half. +/// +/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem. +/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the +/// video's whole name. The folder is part of the key so two cameras' `IMG_1` +/// in different albums never pair. +fn live_key(folder: &str, file_name: &str) -> Option { + let lower = file_name.to_lowercase(); + let (stem, ext) = lower.rsplit_once('.')?; + let key = match ext { + "heic" | "heif" | "jpg" | "jpeg" => stem, + // Apple's video half. A plain `.mp4` beside a photo of the same name is + // far more often an unrelated clip than half of a live photo. + "mov" => stem, + "mp" => lower.as_str(), + _ => return None, + }; + (!key.is_empty()).then(|| format!("{folder}/{key}")) } /// Split `/` into its parts; the folder is `""` at the root. @@ -858,13 +1089,13 @@ mod tests { RawEntry { archive, entry: path.to_string(), - relative: strip_export_prefix(path).unwrap().to_string(), + relative: strip_export_prefix(path).unwrap().1.to_string(), size, } } fn sidecar(map: &mut HashMap>, path: &str, body: &str) { - let relative = strip_export_prefix(path).unwrap().to_string(); + let relative = strip_export_prefix(path).unwrap().1.to_string(); map.insert(relative, body.as_bytes().to_vec()); } @@ -872,12 +1103,17 @@ mod tests { fn strips_export_root_and_localized_service_folder() { assert_eq!( strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"), - Some("Island 2019/IMG_1.jpg") + Some(("Google Fotos", "Island 2019/IMG_1.jpg")) ); // Already unwrapped by the user: no `Takeout/` root left. assert_eq!( strip_export_prefix("Google Photos/IMG_1.jpg"), - Some("IMG_1.jpg") + Some(("Google Photos", "IMG_1.jpg")) + ); + // ... and an album keeps its folder. + assert_eq!( + strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"), + Some(("Google Photos", "Iceland/IMG_1.jpg")) ); // Nothing below a service folder is nothing to import. assert_eq!(strip_export_prefix("archive_browser.html"), None); @@ -957,23 +1193,144 @@ mod tests { } #[test] - fn creation_time_is_the_fallback_capture_time() { + fn creation_time_is_kept_apart_as_the_last_resort() { + // `creationTime` is when the photo reached Google. It must not beat the + // date in the name, and the importer tries EXIF before it. let mut sidecars = HashMap::new(); sidecar( &mut sidecars, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json", r#"{"creationTime":{"timestamp":"1500000000"}}"#, ); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json", + r#"{"creationTime":{"timestamp":"1700000000"}}"#, + ); let scan = assemble( - vec![entry( - 0, - "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", - 10, - )], + vec![ + entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10), + entry( + 0, + "Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg", + 10, + ), + ], &sidecars, ); - assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000)); + assert_eq!(scan.photos[0].capture_time, None); + assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000)); + assert_eq!( + scan.photos[1].capture_time, + capture_time_from_name("IMG_20190612_101500.jpg") + ); + assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000)); + } + + #[test] + fn sidecar_timestamps_tolerate_numbers_zero_and_null() { + let mut sidecars = HashMap::new(); + for (name, body) in [ + ("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#), + ( + "B.jpg", + r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#, + ), + ( + "C.jpg", + r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#, + ), + ] { + sidecar( + &mut sidecars, + &format!("Takeout/Google Photos/Photos from 2019/{name}.json"), + body, + ); + } + let scan = assemble( + ["A.jpg", "B.jpg", "C.jpg"] + .iter() + .map(|name| { + entry( + 0, + &format!("Takeout/Google Photos/Photos from 2019/{name}"), + 1, + ) + }) + .collect(), + &sidecars, + ); + + assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000)); + assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970"); + assert!(!scan.photos[1].favorite); + assert_eq!(scan.photos[2].capture_time, None); + } + + #[test] + fn location_comes_from_geo_data_and_zero_means_none() { + let mut sidecars = HashMap::new(); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Iceland/IMG_1.jpg.json", + r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#, + ); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Iceland/IMG_2.jpg.json", + r#"{"geoData":{"latitude":0.0,"longitude":0.0}, + "geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#, + ); + sidecar( + &mut sidecars, + "Takeout/Google Photos/Iceland/IMG_3.jpg.json", + r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#, + ); + let scan = assemble( + (1..=3) + .map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1)) + .collect(), + &sidecars, + ); + + assert_eq!( + scan.photos[0].location, + Some(GeoPoint { + latitude: 64.1466, + longitude: -21.9426 + }) + ); + assert_eq!( + scan.photos[1].location.map(|p| p.latitude), + Some(63.4186), + "geoDataExif fills in for an empty geoData" + ); + assert_eq!(scan.photos[2].location, None); + } + + #[test] + fn live_and_motion_photo_halves_share_a_key() { + let scan = assemble( + vec![ + entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1), + entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1), + entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1), + entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1), + entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1), + entry(0, "Takeout/Google Photos/Trip/scan.png", 1), + ], + &HashMap::new(), + ); + let key = |i: usize| scan.photos[i].live_key.clone(); + + assert_eq!(key(0), key(1)); + assert!(!scan.photos[0].is_motion_photo()); + assert_eq!(key(2), key(3)); + assert!(scan.photos[3].is_motion_photo()); + assert!(scan.photos[3].is_video(), "the .MP half is a video"); + assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair"); + assert_eq!(key(5), None); } #[test] @@ -1387,16 +1744,77 @@ mod tests { } #[test] - fn a_file_that_is_not_a_zip_is_a_readable_error() { - let path = + fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() { + let broken = std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id())); - std::fs::write(&path, b"not a zip at all").unwrap(); - let error = scan_archives(std::slice::from_ref(&path)).unwrap_err(); - let _ = std::fs::remove_file(&path); + std::fs::write(&broken, b"not a zip at all").unwrap(); + let good = write_zip( + "beside-broken", + &[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")], + ); + + let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap(); + let _ = std::fs::remove_file(&broken); + let _ = std::fs::remove_file(&good); + + assert_eq!(scan.photos.len(), 1); + assert_eq!(scan.photos[0].archive, 1); + assert_eq!(scan.unreadable_archives.len(), 1); + assert_eq!(scan.unreadable_archives[0].path, broken); assert!( - error.message.contains("not a readable zip"), - "unexpected message: {}", - error.message + scan.unreadable_archives[0] + .reason + .contains("not a readable zip"), + "unexpected reason: {}", + scan.unreadable_archives[0].reason ); } + + #[test] + fn other_services_in_the_same_export_are_not_imported() { + let zip = write_zip( + "services", + &[ + ( + "Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg", + b"x".as_slice(), + ), + ( + "Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json", + br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(), + ), + ("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()), + ("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()), + ], + ); + + let scan = scan_archives(std::slice::from_ref(&zip)).unwrap(); + let _ = std::fs::remove_file(&zip); + + assert_eq!(scan.photos.len(), 1); + assert_eq!(scan.photos[0].name, "IMG_1.jpg"); + assert_eq!(scan.skipped_other, 2); + } + + #[test] + fn an_export_without_sidecars_keeps_every_service() { + let zip = write_zip( + "no-sidecars", + &[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")], + ); + let scan = scan_archives(std::slice::from_ref(&zip)).unwrap(); + let _ = std::fs::remove_file(&zip); + assert_eq!(scan.photos.len(), 1); + } + + #[test] + fn a_stop_request_ends_the_scan() { + let zip = write_zip( + "stop", + &[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")], + ); + let result = scan_archives_until(std::slice::from_ref(&zip), || true); + let _ = std::fs::remove_file(&zip); + assert!(result.is_err()); + } } diff --git a/crates/pdfs-fuse/Cargo.toml b/crates/pdfs-fuse/Cargo.toml index 48a5d70..6a36a0e 100644 --- a/crates/pdfs-fuse/Cargo.toml +++ b/crates/pdfs-fuse/Cargo.toml @@ -20,6 +20,7 @@ tokio.workspace = true tracing.workspace = true fuser.workspace = true libc.workspace = true +serde.workspace = true serde_json.workspace = true base64.workspace = true image.workspace = true diff --git a/crates/pdfs-fuse/src/control.rs b/crates/pdfs-fuse/src/control.rs index d5cc9ea..34dfa7c 100644 --- a/crates/pdfs-fuse/src/control.rs +++ b/crates/pdfs-fuse/src/control.rs @@ -877,35 +877,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U if archives.is_empty() { CtlResponse::error(CoreError::invalid("no Takeout archives given")) } else { - let paths: Vec = archives.into_iter().map(PathBuf::from).collect(); - let missing: Vec = paths - .iter() - .filter(|path| !path.is_file()) - .map(|path| path.display().to_string()) - .collect(); - if !missing.is_empty() { - CtlResponse::error(CoreError::not_found(format!( - "cannot read: {}", - missing.join(", ") - ))) - } else { - let count = paths.len(); - match core.spawn_takeout_import(paths, dry_run) { - Ok(()) => CtlResponse::Ok { - message: format!( - "{} {count} archive(s)", - if dry_run { "scanning" } else { "importing" } - ), - }, - Err(e) => CtlResponse::error(e), - } - } + let paths = archives.into_iter().map(PathBuf::from).collect(); + start_takeout_import(core, paths, dry_run) } } Ok(CtlRequest::ImportStatus) => { - let (running, summary) = crate::takeout::import_status(); - CtlResponse::ImportStatus { running, summary } + let report = crate::takeout::import_status(); + CtlResponse::ImportStatus { + running: report.running, + summary: report.summary, + progress: report.progress, + resumable: report.resumable, + } } + Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() { + None => CtlResponse::error(CoreError::not_found("there is no import to resume")), + Some(paths) => start_takeout_import(core, paths, false), + }, Ok(CtlRequest::CancelImport) => { if crate::takeout::cancel_import() { CtlResponse::Ok { @@ -2009,6 +1997,31 @@ pub(crate) fn run_control_socket( } } +/// Check that every archive is there, then start the import in the background. +fn start_takeout_import(core: &Core, paths: Vec, dry_run: bool) -> CtlResponse { + let missing: Vec = paths + .iter() + .filter(|path| !path.is_file()) + .map(|path| path.display().to_string()) + .collect(); + if !missing.is_empty() { + return CtlResponse::error(CoreError::not_found(format!( + "cannot read: {}", + missing.join(", ") + ))); + } + let count = paths.len(); + match core.spawn_takeout_import(paths, dry_run) { + Ok(()) => CtlResponse::Ok { + message: format!( + "{} {count} archive(s)", + if dry_run { "scanning" } else { "importing" } + ), + }, + Err(e) => CtlResponse::error(e), + } +} + #[cfg(test)] mod request_limit_tests { use super::*; diff --git a/crates/pdfs-fuse/src/photos.rs b/crates/pdfs-fuse/src/photos.rs index 557b1f0..9d055d3 100644 --- a/crates/pdfs-fuse/src/photos.rs +++ b/crates/pdfs-fuse/src/photos.rs @@ -1193,7 +1193,7 @@ impl Core { if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) { return Ok(None); } - if !self.rt.block_on(self.refresh_timeline())? { + if !self.refresh_timeline_joined()? { return Ok(None); } } else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) { @@ -2640,6 +2640,36 @@ impl Core { /// Refresh the timeline off the request path, so a stale page is still served /// at DB speed. At most one refresh runs at a time. + /// Refresh the timeline and wait for it, sharing a refresh already under + /// way instead of starting a second one beside it. + /// + /// The first request after an empty cache has nothing to serve until the + /// fetch lands, so it waits. Two such requests (the gallery and its retry, + /// or a background refresh started by an import) used to each fetch the + /// whole library at once, sharing the session's rate limit and taking + /// minutes. Returns whether the account has a photos library. + pub(crate) fn refresh_timeline_joined(&self) -> CoreResult { + if self.timeline_refreshing.swap(true, Ordering::SeqCst) { + while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + // A refresh that landed stamped the timeline fresh. One that failed + // did not, and the caller gets this refresh's own error instead of + // an empty timeline. + if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) + || self.timeline_refreshing.swap(true, Ordering::SeqCst) + { + return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0)); + } + } + let refreshed = self.rt.block_on(self.refresh_timeline()); + self.timeline_refreshing.store(false, Ordering::SeqCst); + if refreshed.is_ok() { + self.events.publish(&[Topic::Photos]); + } + refreshed + } + pub(crate) fn spawn_timeline_refresh(&self) { if self.timeline_refreshing.swap(true, Ordering::SeqCst) { return; diff --git a/crates/pdfs-fuse/src/takeout.rs b/crates/pdfs-fuse/src/takeout.rs index 4a23d86..3b45f32 100644 --- a/crates/pdfs-fuse/src/takeout.rs +++ b/crates/pdfs-fuse/src/takeout.rs @@ -2,36 +2,47 @@ //! //! [`pdfs_core::takeout`] says what is in the archives; this is the part that //! talks to Proton. One import runs on its own thread and moves through four -//! phases, reporting into the transfer registry as a job so the GUI's transfer -//! list shows it without a bespoke widget: +//! phases ([`ImportPhase`]), which [`Request::ImportStatus`] reports together +//! with the running counts: //! -//! 1. **Scan** — read the archives' central directories and sidecars. -//! 2. **Deduplicate against the account** — ask the photos volume which of the -//! names already exist ([`ProtonPhotosClient::name_collisions`]), hash only -//! those files, then confirm with the content digest. A library that is -//! already half-synced by Proton Photos on a phone therefore costs one cheap -//! request per 150 names and reads no media at all for the misses. -//! 3. **Upload** the survivors, streaming each entry straight out of its zip — -//! no extraction to disk, so the import needs no scratch space. +//! 1. **Scan** — read the archives' central directories and sidecars. An +//! archive that cannot be read is left out and reported; the rest imports. +//! 2. **Check** — ask the photos volume which of the names already exist +//! ([`ProtonPhotosClient::name_collisions`]), hash only those files, then +//! confirm with the content digest. A library that is already half-synced +//! by Proton Photos on a phone therefore costs one cheap request per 150 +//! names and reads no media at all for the misses. +//! 3. **Upload** the survivors, streaming each entry out of its zip. A photo +//! that can have a thumbnail is copied to a staging file first, one at a +//! time. The video half of a live or motion photo goes after its still and +//! is filed with it. //! 4. **Albums** — create the albums the export names that the account lacks, //! then add every photo of each album, the ones that were already on the //! account included. //! -//! Re-running an interrupted import is the resume story: phase 2 finds -//! everything the previous run uploaded and skips it. Nothing is persisted -//! between runs. +//! Resuming is re-running: phase 2 finds everything an earlier run uploaded +//! and skips it. The archives and the outcome of the last run are kept in the +//! state directory, so an import cut short by a reboot can be picked up again +//! with [`Request::ResumeImport`] without choosing the archives again. +//! +//! [`Request::ImportStatus`]: pdfs_core::control::Request::ImportStatus +//! [`Request::ResumeImport`]: pdfs_core::control::Request::ResumeImport +//! [`ProtonPhotosClient::name_collisions`]: proton_drive_rs::ProtonPhotosClient::name_collisions use std::collections::{BTreeMap, HashMap}; use std::io::Read; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use pdfs_core::control::{ActivityKind, ImportSummary, TransferDirection}; -use pdfs_core::takeout::{Bucket, TakeoutPhoto, TakeoutScan, scan_archives}; +use pdfs_core::control::{ + ActivityKind, ImportPhase, ImportProgress, ImportSummary, TransferDirection, +}; +use pdfs_core::takeout::{Bucket, TakeoutPhoto, TakeoutScan, scan_archives_until}; use pdfs_core::{CoreError, CoreResult}; use proton_drive_rs::proton_sdk::ids::NodeUid; -use proton_drive_rs::{PhotoTag, PhotoTagsUpdate, PhotoUploadMetadata}; +use proton_drive_rs::{PhotoLocation, PhotoTag, PhotoTagsUpdate, PhotoUploadMetadata}; +use serde::{Deserialize, Serialize}; use sha1::{Digest, Sha1}; use tracing::{info, warn}; @@ -42,23 +53,161 @@ use crate::transfers::CountingReader; /// The one import this daemon may be running, and how the last one ended. /// /// Process-global rather than a [`Core`] field because it must be *one* import -/// per daemon however many `Core` clones exist (the forked on-demand mounts each -/// hold one), and because a front-end asking for the result after the import -/// finished must still find it. +/// per daemon however many `Core` clones exist, and because a front-end asking +/// for the result after the import finished must still find it. Loaded from +/// the state directory on first use, so the last run survives a restart. static IMPORT: std::sync::LazyLock> = - std::sync::LazyLock::new(|| parking_lot::Mutex::new(ImportState::default())); + std::sync::LazyLock::new(|| parking_lot::Mutex::new(ImportState::load(saved_path()))); + +/// Photos uploaded between two timeline refreshes, so a long import shows up +/// in the gallery as it goes rather than only at the end. +const TIMELINE_REFRESH_EVERY: usize = 250; + +/// Photos uploaded between two saves of the running counts, so a run cut short +/// by a power loss still reports roughly how far it got. +const CHECKPOINT_EVERY: usize = 25; + +/// How far apart, in seconds, the still and the video of one live photo may +/// claim to have been taken before they are treated as two unrelated files +/// that happen to share a name. +const LIVE_PAIR_MAX_SKEW: i64 = 120; #[derive(Default)] struct ImportState { running: bool, cancel: Option>, - last: Option, + archives: Vec, + dry_run: bool, + /// Live counts while running, the final report after. + summary: Option, + progress: Option, + /// Where the state is saved; `None` keeps it in memory only (tests). + path: Option, } -/// Whether an import is running, and the last one's report. -pub(crate) fn import_status() -> (bool, Option) { +/// What [`ImportState`] keeps on disk. +#[derive(Serialize, Deserialize, Default)] +struct SavedImport { + archives: Vec, + dry_run: bool, + running: bool, + summary: Option, +} + +fn saved_path() -> Option { + if cfg!(test) { + return None; + } + pdfs_core::config::AppDirs::new() + .ok() + .map(|dirs| dirs.takeout_import_path()) +} + +impl ImportState { + /// The state as the last daemon left it. A run that was still marked as + /// running did not get to finish: the daemon stopped under it. + fn load(path: Option) -> Self { + let saved = path + .as_deref() + .and_then(|path| std::fs::read(path).ok()) + .and_then(|bytes| { + serde_json::from_slice::(&bytes) + .inspect_err(|error| warn!(%error, "ignoring an unreadable import state")) + .ok() + }) + .unwrap_or_default(); + let mut summary = saved.summary; + if saved.running { + summary.get_or_insert_with(Default::default).interrupted = true; + } + Self { + archives: saved.archives, + dry_run: saved.dry_run, + summary, + path, + ..Default::default() + } + } + + /// Write the state out. Atomic, so a crash mid-write keeps the old one. + fn save(&self) { + let Some(path) = &self.path else { + return; + }; + let saved = SavedImport { + archives: self.archives.clone(), + dry_run: self.dry_run, + running: self.running, + summary: self.summary.clone(), + }; + let written = serde_json::to_vec_pretty(&saved) + .map_err(std::io::Error::other) + .and_then(|bytes| write_atomically(path, &bytes)); + if let Err(error) = written { + warn!(path = %path.display(), %error, "cannot save the import state"); + } + } + + /// The archives of the last run when it can be resumed: a real import that + /// did not finish everything it set out to do. + fn resumable(&self) -> Vec { + let unfinished = self + .summary + .as_ref() + .is_some_and(|summary| !summary.complete()); + if self.running || self.dry_run || !unfinished { + return Vec::new(); + } + self.archives + .iter() + .map(|path| path.display().to_string()) + .collect() + } +} + +fn write_atomically(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let tmp = path.with_extension(format!("tmp-{}", std::process::id())); + { + use std::io::Write; + let mut file = std::fs::File::create(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + } + std::fs::rename(&tmp, path)?; + if let Some(parent) = path.parent() { + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +/// What [`Request::ImportStatus`](pdfs_core::control::Request::ImportStatus) +/// answers. +pub(crate) struct ImportStatusReport { + pub running: bool, + pub summary: Option, + pub progress: Option, + pub resumable: Vec, +} + +/// Whether an import is running, its counts and step, and what can be resumed. +pub(crate) fn import_status() -> ImportStatusReport { let state = IMPORT.lock(); - (state.running, state.last.clone()) + ImportStatusReport { + running: state.running, + summary: state.summary.clone(), + progress: state.running.then(|| state.progress.clone()).flatten(), + resumable: state.resumable(), + } +} + +/// The archives [`Request::ResumeImport`](pdfs_core::control::Request::ResumeImport) +/// would import again, or `None` when there is nothing to resume. +pub(crate) fn resumable_archives() -> Option> { + let state = IMPORT.lock(); + (!state.resumable().is_empty()).then(|| state.archives.clone()) } /// Ask the running import to stop. False when none is running. @@ -72,17 +221,65 @@ pub(crate) fn cancel_import() -> bool { } } +/// Publishes a running import's step and counts into [`IMPORT`]. +struct Reporter; + +impl Reporter { + fn phase(&self, phase: ImportPhase, total: u64) { + IMPORT.lock().progress = Some(ImportProgress { + phase, + done: 0, + total, + current: None, + }); + } + + fn step(&self) { + if let Some(progress) = IMPORT.lock().progress.as_mut() { + progress.done += 1; + } + } + + fn current(&self, name: &str) { + if let Some(progress) = IMPORT.lock().progress.as_mut() { + progress.current = Some(name.to_string()); + } + } + + fn counts(&self, summary: &ImportSummary) { + IMPORT.lock().summary = Some(summary.clone()); + } + + /// Save the running state, still marked as running. + fn checkpoint(&self) { + IMPORT.lock().save(); + } +} + +/// How a planned photo relates to a live or motion photo pair. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LiveRole { + /// The still of a pair: uploaded first, tagged as a live photo. + Still(PhotoTag), + /// The video of a pair, filed with the still at this index of the plan. + Video(usize), +} + /// One photo to import, after the copies Google keeps per album are folded into /// a single upload that joins several albums. struct Planned { photo: TakeoutPhoto, /// Every album this photo belongs to, by export title. albums: Vec, + /// The live-photo keys of every folded copy (each copy sits in its own + /// folder, so each has its own key). + live_keys: Vec, + live: Option, /// Filled in phase 2/3: where the photo ended up on the account. uid: Option, } -/// Fold the scan into one entry per distinct photo. +/// Fold the scan into one entry per distinct photo, then pair live photos. /// /// Google stores a full copy of a photo in every album folder it appears in, so /// the same file is in the export two or three times over. They are folded on @@ -102,6 +299,8 @@ fn plan(scan: &TakeoutScan) -> Vec { Planned { photo: photo.clone(), albums: Vec::new(), + live_keys: Vec::new(), + live: None, uid: None, } }); @@ -110,18 +309,94 @@ fn plan(scan: &TakeoutScan) -> Vec { { entry.albums.push(title.clone()); } - // A copy that carries a capture time wins over one that does not: the - // album copy and the year-folder copy do not always both have a sidecar. + if let Some(live_key) = &photo.live_key + && !entry.live_keys.contains(live_key) + { + entry.live_keys.push(live_key.clone()); + } + // A copy that carries metadata wins over one that does not: the album + // copy and the year-folder copy do not always both have a sidecar. if entry.photo.capture_time.is_none() && photo.capture_time.is_some() { entry.photo.capture_time = photo.capture_time; } + if entry.photo.creation_time.is_none() && photo.creation_time.is_some() { + entry.photo.creation_time = photo.creation_time; + } + if entry.photo.location.is_none() && photo.location.is_some() { + entry.photo.location = photo.location; + } entry.photo.favorite |= photo.favorite; } - order + let mut planned: Vec = order .into_iter() .filter_map(|key| by_key.remove(&key)) - .collect() + .collect(); + pair_live_photos(&mut planned); + planned +} + +/// Pair the still and the video of each live or motion photo. +/// +/// The halves share a live key (same folder, same stem). The video takes the +/// still's date and place when it has none of its own, because a `.MOV` half +/// usually has no sidecar. Two files whose capture times disagree by minutes +/// are not a pair, whatever their names. +fn pair_live_photos(planned: &mut [Planned]) { + let mut stills: HashMap<&str, usize> = HashMap::new(); + for (index, item) in planned.iter().enumerate() { + if !item.photo.is_video() { + for key in &item.live_keys { + stills.entry(key.as_str()).or_insert(index); + } + } + } + + let mut pairs: Vec<(usize, usize)> = Vec::new(); + let mut taken = vec![false; planned.len()]; + for (video, item) in planned.iter().enumerate() { + if !item.photo.is_video() { + continue; + } + let Some(&still) = item + .live_keys + .iter() + .find_map(|key| stills.get(key.as_str())) + else { + continue; + }; + if taken[still] { + continue; + } + if let (Some(a), Some(b)) = (item.photo.capture_time, planned[still].photo.capture_time) + && (a - b).abs() > LIVE_PAIR_MAX_SKEW + { + continue; + } + taken[still] = true; + pairs.push((still, video)); + } + + for (still, video) in pairs { + let tag = if planned[video].photo.is_motion_photo() { + PhotoTag::MotionPhoto + } else { + PhotoTag::LivePhoto + }; + let (capture_time, location) = ( + planned[still].photo.capture_time, + planned[still].photo.location, + ); + planned[still].live = Some(LiveRole::Still(tag)); + let video = &mut planned[video]; + video.live = Some(LiveRole::Video(still)); + if video.photo.capture_time.is_none() { + video.photo.capture_time = capture_time; + } + if video.photo.location.is_none() { + video.photo.location = location; + } + } } /// Open one archive and stream `entry` through `sink`. @@ -161,35 +436,123 @@ impl Core { Ok(path) } + /// Whether the import should stop taking on new work. + fn import_should_stop(&self, cancel: &AtomicBool) -> bool { + cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() + } + /// Import Google Photos Takeout archives. Blocking; run on its own thread. /// + /// Never fails as a whole: whatever stops the run early is recorded in the + /// summary's `error` next to the counts of what it did get done, and the + /// photos it uploaded are still filed into their albums. + /// /// `cancel` stops the run at the next photo boundary — an upload already on /// the wire is allowed to finish, so a cancelled import never leaves a /// half-written revision behind. - pub(crate) fn import_takeout( + fn import_takeout( &self, - archives: Vec, + archives: &[PathBuf], dry_run: bool, - cancel: Arc, - ) -> CoreResult { - let job = self.transfers.begin_job("Importing Google Photos"); - job.detail("reading archives"); - - let scan = scan_archives(&archives)?; - let mut planned = plan(&scan); + cancel: &AtomicBool, + job: &crate::transfers::JobGuard, + ) -> ImportSummary { + let report = Reporter; let mut summary = ImportSummary { - found: planned.len(), - skipped_trashed: scan.skipped_trashed, + dry_run, ..Default::default() }; - job.set_total(planned.len() as u64); + let mut planned = Vec::new(); + let mut albums = Vec::new(); + + let outcome = self.run_import( + archives, + cancel, + &mut summary, + &mut planned, + &mut albums, + &report, + job, + ); + // Whether the run left work undone because it was asked to stop. A + // cancel that arrives after the last photo is not a cancelled run. + let stopped_early = match &outcome { + Ok(stopped) => *stopped, + Err(e) => { + warn!(error = %e, "takeout import stopped"); + summary.error = Some(e.message.clone()); + summary.error_kind = Some(e.kind); + false + } + }; + summary.interrupted = stopped_early && self.shutdown.is_stopping(); + summary.cancelled = stopped_early && !summary.interrupted; + + // Albums for whatever made it up, after a cancel or an error too — but + // not while the daemon is shutting down, which must not wait on it. A + // resumed run files them. + if !dry_run && !summary.interrupted && planned.iter().any(|p| p.uid.is_some()) { + if !albums.is_empty() { + report.phase(ImportPhase::Albums, albums.len() as u64); + job.detail("sorting photos into albums"); + if let Err(e) = self.link_albums(&planned, &albums, &mut summary, &report) { + warn!(error = %e, "takeout album linking failed"); + } + // Album linking stops for a shutdown too; a resumed run files + // the rest. + summary.interrupted |= self.shutdown.is_stopping(); + } + report.counts(&summary); + } + if summary.uploaded > 0 { + self.spawn_timeline_refresh(); + } + summary + } + + #[allow(clippy::too_many_arguments)] + fn run_import( + &self, + archives: &[PathBuf], + cancel: &AtomicBool, + summary: &mut ImportSummary, + planned: &mut Vec, + albums: &mut Vec, + report: &Reporter, + job: &crate::transfers::JobGuard, + ) -> CoreResult { + // Phase 1: scan. + report.phase(ImportPhase::Scanning, 0); + job.detail("reading archives"); + let scan = match scan_archives_until(archives, || self.import_should_stop(cancel)) { + Ok(scan) => scan, + Err(_) if self.import_should_stop(cancel) => return Ok(true), + Err(e) => return Err(e), + }; + *planned = plan(&scan); + *albums = scan.albums.clone(); + summary.found = planned.len(); + summary.skipped_trashed = scan.skipped_trashed; + summary.unreadable_archives = scan + .unreadable_archives + .iter() + .map(|archive| archive.path.display().to_string()) + .collect(); + report.counts(summary); info!( archives = archives.len(), + unreadable = scan.unreadable_archives.len(), photos = summary.found, albums = scan.albums.len(), trashed = summary.skipped_trashed, + other = scan.skipped_other, "takeout scan complete" ); + if scan.photos.is_empty() && scan.unreadable_archives.len() == archives.len() { + return Err(CoreError::invalid( + "none of the archives could be read; are they Google Takeout .zip files?", + )); + } let photos = self.photos(); if self @@ -205,29 +568,33 @@ impl Core { // Phase 2: which of these are already on the account. job.detail("checking for photos you already have"); - self.mark_existing(&mut planned, &archives, &mut summary, &cancel)?; - - if dry_run { - summary.duplicates = planned.iter().filter(|p| p.uid.is_some()).count(); - return Ok(summary); + self.mark_existing(planned, archives, summary, cancel, report)?; + if self.import_should_stop(cancel) { + return Ok(true); } - // Phase 3: upload what is left, one archive at a time so each zip's - // central directory is parsed once. - self.upload_planned(&mut planned, &archives, &mut summary, &cancel, &job)?; - - // Phase 4: albums. Attempted even after a cancel — the photos that did - // upload should still land in their albums. - if !scan.albums.is_empty() { - job.detail("sorting photos into albums"); - if let Err(e) = self.link_albums(&planned, &scan.albums, &mut summary) { - warn!(error = %e, "takeout album linking failed"); - } + let remaining = planned.iter().filter(|p| p.uid.is_none()); + summary.to_upload = remaining.clone().count(); + summary.to_upload_bytes = remaining.map(|p| p.photo.size).sum(); + report.counts(summary); + if summary.dry_run { + return Ok(false); } - self.spawn_timeline_refresh(); - summary.cancelled = cancel.load(Ordering::Relaxed); - Ok(summary) + // Favorites the export knows about for photos the account already had: + // a first import may have missed them, and setting one is idempotent. + let existing_favorites: Vec = planned + .iter() + .filter(|p| p.photo.favorite) + .filter_map(|p| p.uid.clone()) + .collect(); + self.mark_favorites(existing_favorites); + + // Phase 3: upload what is left. + report.checkpoint(); + report.phase(ImportPhase::Uploading, summary.to_upload as u64); + job.set_total(summary.to_upload as u64); + Ok(self.upload_planned(planned, archives, summary, cancel, job, report)) } /// Fill in `uid` for every planned photo the account already holds. @@ -241,7 +608,9 @@ impl Core { archives: &[PathBuf], summary: &mut ImportSummary, cancel: &AtomicBool, + report: &Reporter, ) -> CoreResult<()> { + report.phase(ImportPhase::Checking, 0); let photos = self.photos(); let names: Vec = planned.iter().map(|p| p.photo.name.clone()).collect(); let collisions = self @@ -259,17 +628,29 @@ impl Core { .push(index); } } + report.phase( + ImportPhase::Checking, + by_archive.values().map(Vec::len).sum::() as u64, + ); // Kept as a pair list, not a map: the duplicate lookup answers in input // order, so the order these were built in is what maps answers back to // planned photos. let mut digests: Vec<(usize, String)> = Vec::new(); - for (archive_index, indices) in by_archive { - if cancel.load(Ordering::Relaxed) { - break; - } - let mut archive = open_archive(archives, archive_index)?; + 'archives: for (archive_index, indices) in by_archive { + let mut archive = match open_archive(archives, archive_index) { + Ok(archive) => archive, + Err(e) => { + // Not hashed means not recognised as a duplicate; the upload + // phase then fails these photos on the same archive. + warn!(error = %e, "cannot reopen a takeout archive to check for duplicates"); + continue; + } + }; for index in indices { + if self.import_should_stop(cancel) { + break 'archives; + } let entry = planned[index].photo.entry.clone(); match read_entry(&mut archive, &entry, sha1_of) { Ok(Ok(digest)) => digests.push((index, digest)), @@ -277,9 +658,10 @@ impl Core { warn!(entry, error = %e, "cannot hash takeout entry"); } } + report.step(); } } - if digests.is_empty() { + if digests.is_empty() || self.import_should_stop(cancel) { return Ok(()); } @@ -298,10 +680,16 @@ impl Core { summary.duplicates += 1; } } + report.counts(summary); Ok(()) } /// Upload every planned photo that is not already on the account. + /// + /// Two rounds: everything but the videos of live photos first, then those + /// videos, so each one's still has a uid to be filed under. Within a round + /// the photos go archive by archive, so each zip's directory is read once. + /// Returns whether it stopped with photos left, on a cancel or a shutdown. fn upload_planned( &self, planned: &mut [Planned], @@ -309,149 +697,209 @@ impl Core { summary: &mut ImportSummary, cancel: &AtomicBool, job: &crate::transfers::JobGuard, - ) -> CoreResult<()> { - let photos = self.photos(); + report: &Reporter, + ) -> bool { let mut favorites: Vec = Vec::new(); + let mut stopped = false; - let mut by_archive: BTreeMap> = BTreeMap::new(); - for (index, item) in planned.iter().enumerate() { - if item.uid.is_none() { - by_archive - .entry(item.photo.archive) - .or_default() - .push(index); + for videos_round in [false, true] { + let mut by_archive: BTreeMap> = BTreeMap::new(); + for (index, item) in planned.iter().enumerate() { + let is_paired_video = matches!(item.live, Some(LiveRole::Video(_))); + if item.uid.is_none() && is_paired_video == videos_round { + by_archive + .entry(item.photo.archive) + .or_default() + .push(index); + } } - } - for (archive_index, indices) in by_archive { - if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() { - break; - } - let mut archive = open_archive(archives, archive_index)?; - - for index in indices { - if cancel.load(Ordering::Relaxed) || self.shutdown.is_stopping() { + for (archive_index, indices) in by_archive { + if self.import_should_stop(cancel) { + stopped = true; break; } - let photo = planned[index].photo.clone(); - job.detail(photo.name.clone()); - - let guard = self.transfers.begin( - photo.name.clone(), - "", - TransferDirection::Upload, - photo.size, - ); - let metadata = PhotoUploadMetadata { - capture_time: photo.capture_time, - last_modification_time: photo - .capture_time - .and_then(pdfs_core::timestamp::modification_time), - ..Default::default() - }; - // A thumbnail is made from a file, so a photo that can have one - // is copied out of the archive first and uploaded from the copy. - // Without one, Proton's apps show a blank tile until they make - // their own. - let staged = can_make_thumbnail(&photo.name, photo.size) - .then(|| self.stage_takeout_entry(&mut archive, &photo.entry)) - .and_then(|staged| { - staged - .inspect_err(|e| { - warn!(name = photo.name, error = %e, - "copying a takeout photo out failed; uploading it without a thumbnail") - }) - .ok() - }); - let uploaded = match &staged { - Some(path) => { - let thumbnails = self.upload_thumbnails(path, &photo.name); - // No sidecar and no date in the name: the photo's own - // EXIF, before Drive files it under today. - let capture_time = metadata - .capture_time - .or_else(|| pdfs_core::exiftime::capture_time_of(path)); - let metadata = PhotoUploadMetadata { - capture_time, - last_modification_time: capture_time - .and_then(pdfs_core::timestamp::modification_time), - ..metadata - }; - std::fs::File::open(path) - .map_err(|e| { - CoreError::internal(format!("open {}: {e}", path.display())) - }) - .map(|file| { - let counting = - CountingReader::new(std::io::BufReader::new(file), &guard); - self.rt.block_on(photos.upload_photo_from( - &photo.name, - &photo.media_type, - counting, - photo.size as i64, - thumbnails, - metadata, - false, - )) - }) - } - None => read_entry(&mut archive, &photo.entry, |reader| { - let counting = CountingReader::new(reader, &guard); - self.rt.block_on(photos.upload_photo_from( - &photo.name, - &photo.media_type, - counting, - photo.size as i64, - Vec::new(), - metadata, - false, - )) - }), - }; - if let Some(path) = staged { - let _ = std::fs::remove_file(path); - } - drop(guard); - job.step(); - - match uploaded { - Ok(Ok(uid)) => { - summary.uploaded += 1; - summary.bytes += photo.size; - if photo.favorite { - favorites.push(uid.clone()); - } - planned[index].uid = Some(uid); - } - Ok(Err(e)) => { - summary.failed += 1; - warn!(name = photo.name, error = %e, "takeout photo upload failed"); - } + let mut archive = match open_archive(archives, archive_index) { + Ok(archive) => archive, Err(e) => { - summary.failed += 1; - warn!(name = photo.name, error = %e, "cannot read takeout entry"); + // The archive was readable at scan time; whatever broke + // it since costs its own photos, not the whole run. + warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive"); + summary.failed += indices.len(); + for _ in &indices { + job.step(); + report.step(); + } + report.counts(summary); + continue; } + }; + + for index in indices { + if self.import_should_stop(cancel) { + stopped = true; + break; + } + let main_uid = match planned[index].live { + Some(LiveRole::Video(still)) => planned[still].uid.clone(), + _ => None, + }; + let photo = planned[index].photo.clone(); + job.detail(photo.name.clone()); + report.current(&photo.name); + + let uploaded = + self.upload_takeout_photo(&mut archive, &planned[index], main_uid.clone()); + job.step(); + report.step(); + + match uploaded { + Ok(uid) => { + self.note_self_change(&uid); + summary.uploaded += 1; + summary.bytes += photo.size; + if main_uid.is_some() { + summary.live_photos += 1; + } + if photo.favorite { + favorites.push(uid.clone()); + } + planned[index].uid = Some(uid); + if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) { + self.spawn_timeline_refresh(); + } + if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) { + report.counts(summary); + report.checkpoint(); + } + } + Err(e) => { + summary.failed += 1; + warn!(name = photo.name, error = %e, "takeout photo upload failed"); + } + } + report.counts(summary); } } } - // Favorites are a separate call: the upload seal takes classification - // tags, but `Favorite` is not one of them — it goes through the photos - // volume's dedicated favorite endpoint. - if !favorites.is_empty() { - let updates: Vec = favorites - .into_iter() - .map(|node_uid| PhotoTagsUpdate { - node_uid, - tags_to_add: vec![PhotoTag::Favorite], - tags_to_remove: Vec::new(), - }) - .collect(); - if let Err(e) = self.rt.block_on(photos.update_photos(&updates)) { - warn!(error = %e, "marking imported favorites failed"); + self.mark_favorites(favorites); + stopped + } + + /// Upload one planned photo out of its open archive. + fn upload_takeout_photo( + &self, + archive: &mut zip::ZipArchive>, + item: &Planned, + main_uid: Option, + ) -> CoreResult { + let photo = &item.photo; + let photos = self.photos(); + let guard = self.transfers.begin( + photo.name.clone(), + "", + TransferDirection::Upload, + photo.size, + ); + let tags = match item.live { + Some(LiveRole::Still(tag)) => vec![tag], + _ => Vec::new(), + }; + let metadata = |capture_time: Option| PhotoUploadMetadata { + capture_time, + last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time), + main_photo_uid: main_uid.clone(), + tags: tags.clone(), + location: photo + .location + .and_then(|p| PhotoLocation::from_claimed(Some(p.latitude), Some(p.longitude))), + // A draft under this name on the photos volume is this import's own, + // left by a run that was cut short — possibly under an earlier + // login, whose session id no longer matches. + replace_existing_draft: true, + }; + + // A thumbnail is made from a file, so a photo that can have one is + // copied out of the archive first and uploaded from the copy. Without + // one, Proton's apps show a blank tile until they make their own. + let staged = can_make_thumbnail(&photo.name, photo.size) + .then(|| self.stage_takeout_entry(archive, &photo.entry)) + .and_then(|staged| { + staged + .inspect_err(|e| { + warn!(name = photo.name, error = %e, + "copying a takeout photo out failed; uploading it without a thumbnail") + }) + .ok() + }); + let uploaded = match &staged { + Some(path) => { + let thumbnails = self.upload_thumbnails(path, &photo.name); + // No sidecar date and none in the name: the photo's own EXIF, + // then when Google received it, before Drive files it under + // today. + let capture_time = photo + .capture_time + .or_else(|| pdfs_core::exiftime::capture_time_of(path)) + .or(photo.creation_time); + std::fs::File::open(path) + .map_err(|e| CoreError::internal(format!("open {}: {e}", path.display()))) + .and_then(|file| { + let counting = CountingReader::new(std::io::BufReader::new(file), &guard); + self.rt + .block_on(photos.upload_photo_from( + &photo.name, + &photo.media_type, + counting, + photo.size as i64, + thumbnails, + metadata(capture_time), + false, + )) + .map_err(|e| CoreError::from_api(&e, "upload")) + }) } + None => read_entry(archive, &photo.entry, |reader| { + let counting = CountingReader::new(reader, &guard); + self.rt + .block_on(photos.upload_photo_from( + &photo.name, + &photo.media_type, + counting, + photo.size as i64, + Vec::new(), + metadata(photo.capture_time.or(photo.creation_time)), + false, + )) + .map_err(|e| CoreError::from_api(&e, "upload")) + }) + .and_then(|result| result), + }; + if let Some(path) = staged { + let _ = std::fs::remove_file(path); + } + uploaded + } + + /// Mark photos as favorites. A separate call: the upload seal takes + /// classification tags, but `Favorite` is not one of them — it goes through + /// the photos volume's dedicated favorite endpoint. + fn mark_favorites(&self, uids: Vec) { + if uids.is_empty() { + return; + } + let updates: Vec = uids + .into_iter() + .map(|node_uid| PhotoTagsUpdate { + node_uid, + tags_to_add: vec![PhotoTag::Favorite], + tags_to_remove: Vec::new(), + }) + .collect(); + if let Err(e) = self.rt.block_on(self.photos().update_photos(&updates)) { + warn!(error = %e, "marking imported favorites failed"); } - Ok(()) } /// Create the albums the export names that the account lacks, then add each @@ -461,6 +909,7 @@ impl Core { planned: &[Planned], albums: &[String], summary: &mut ImportSummary, + report: &Reporter, ) -> CoreResult<()> { let photos = self.photos(); @@ -482,6 +931,10 @@ impl Core { } for title in albums { + if self.shutdown.is_stopping() { + break; + } + report.step(); let members: Vec = planned .iter() .filter(|item| item.albums.iter().any(|album| album == title)) @@ -517,13 +970,14 @@ impl Core { // A photo already in the album is the expected // outcome of a re-run, not a failure worth counting. Err(e) => { - warn!(album = title, %uid, error = %e, "cannot add photo to album") + tracing::debug!(album = title, %uid, error = %e, "cannot add photo to album") } } } } Err(e) => warn!(album = title, error = %e, "album addition failed"), } + report.counts(summary); } Ok(()) } @@ -548,58 +1002,87 @@ impl Core { } state.running = true; state.cancel = Some(cancel.clone()); - state.last = None; + state.archives = archives.clone(); + state.dry_run = dry_run; + // A placeholder until the scan has counts, so a front-end can + // tell a scan from an import from the start. + state.summary = Some(ImportSummary { + dry_run, + ..Default::default() + }); + state.progress = Some(ImportProgress::default()); + state.save(); } let core = self.clone(); std::thread::spawn(move || { - let outcome = core.import_takeout(archives, dry_run, cancel); + // The job is dropped only after the final state is stored: the + // job leaving the transfer snapshot is the front-ends' cue to ask + // for the result, which must then be there. + let job = core.transfers.begin_job("Importing Google Photos"); + let run = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + core.import_takeout(&archives, dry_run, &cancel, &job) + })); + // A panic must not leave the import marked as running for the + // life of the daemon, refusing every later one. + let summary = run.unwrap_or_else(|_| ImportSummary { + dry_run, + error: Some("the import stopped on an internal error".to_string()), + error_kind: Some(pdfs_core::control::ErrorKind::Internal), + ..IMPORT.lock().summary.clone().unwrap_or_default() + }); { let mut state = IMPORT.lock(); state.running = false; state.cancel = None; - // A failed run is reported too: without it, `--wait` read - // "no import has run" and exited as if all was well. - state.last = Some(match &outcome { - Ok(summary) => summary.clone(), - Err(e) => ImportSummary { - error: Some(e.to_string()), - ..Default::default() - }, - }); + state.progress = None; + state.summary = Some(summary.clone()); + state.save(); } - match outcome { - Ok(summary) => { - let mut parts = vec![format!("{} uploaded", summary.uploaded)]; - if summary.duplicates > 0 { - parts.push(format!("{} already there", summary.duplicates)); - } - if summary.albums_created > 0 { - parts.push(format!("{} albums", summary.albums_created)); - } - if summary.failed > 0 { - parts.push(format!("{} failed", summary.failed)); - } - if summary.cancelled { - parts.push("cancelled".to_string()); - } - core.log_activity( - ActivityKind::Upload, - "Google Photos import", - parts.join(" · "), - summary.failed == 0 && !summary.cancelled, - ); - } - Err(e) => { - warn!(error = %e, "takeout import failed"); - core.log_activity(ActivityKind::Upload, "Google Photos import", &e, false); - } + drop(job); + if !dry_run { + core.log_activity( + ActivityKind::Upload, + "Google Photos import", + activity_line(&summary), + summary.complete(), + ); } }); Ok(()) } } +/// The activity-feed line for a finished import. The feed is the daemon's own +/// log, in English like the rest of it. +fn activity_line(summary: &ImportSummary) -> String { + let mut parts = vec![format!("{} uploaded", summary.uploaded)]; + if summary.duplicates > 0 { + parts.push(format!("{} already there", summary.duplicates)); + } + if summary.albums_created > 0 { + parts.push(format!("{} albums", summary.albums_created)); + } + if summary.failed > 0 { + parts.push(format!("{} failed", summary.failed)); + } + if !summary.unreadable_archives.is_empty() { + parts.push(format!( + "{} archives unreadable", + summary.unreadable_archives.len() + )); + } + if summary.interrupted { + parts.push("interrupted".to_string()); + } else if summary.cancelled { + parts.push("cancelled".to_string()); + } + if let Some(error) = &summary.error { + parts.push(error.clone()); + } + parts.join(" · ") +} + /// Open one of the import's archives by index. fn open_archive( archives: &[PathBuf], @@ -642,18 +1125,37 @@ pub(crate) fn hex_lower(bytes: &[u8]) -> String { #[cfg(test)] mod tests { use super::*; - use pdfs_core::takeout::TakeoutPhoto; + use pdfs_core::takeout::{GeoPoint, TakeoutPhoto}; fn photo(name: &str, size: u64, bucket: Bucket) -> TakeoutPhoto { + let folder = match &bucket { + Bucket::Album(title) => title.clone(), + _ => "Photos from 2019".to_string(), + }; + let (stem, ext) = name.rsplit_once('.').unwrap(); + let media_type = match ext.to_ascii_lowercase().as_str() { + "mov" | "mp4" | "mp" => "video/mp4", + _ => "image/jpeg", + }; TakeoutPhoto { archive: 0, - entry: format!("Takeout/Google Photos/x/{name}"), + entry: format!("Takeout/Google Photos/{folder}/{name}"), name: name.to_string(), size, - media_type: "image/jpeg".to_string(), + media_type: media_type.to_string(), capture_time: None, + creation_time: None, + location: None, favorite: false, bucket, + live_key: Some(format!( + "{folder}/{}", + if ext.eq_ignore_ascii_case("mp") { + name.to_lowercase() + } else { + stem.to_lowercase() + } + )), } } @@ -692,10 +1194,14 @@ mod tests { #[test] fn metadata_from_any_copy_survives_the_fold() { // Only the album copy carries a sidecar; the fold must not lose it. - let mut bare = photo("IMG_1.jpg", 100, Bucket::Timeline); - bare.capture_time = None; + let bare = photo("IMG_1.jpg", 100, Bucket::Timeline); let mut rich = photo("IMG_1.jpg", 100, Bucket::Album("Iceland".into())); rich.capture_time = Some(1_560_000_000); + rich.creation_time = Some(1_600_000_000); + rich.location = Some(GeoPoint { + latitude: 64.1, + longitude: -21.9, + }); rich.favorite = true; let planned = plan(&TakeoutScan { @@ -704,9 +1210,158 @@ mod tests { }); assert_eq!(planned[0].photo.capture_time, Some(1_560_000_000)); + assert_eq!(planned[0].photo.creation_time, Some(1_600_000_000)); + assert!(planned[0].photo.location.is_some()); assert!(planned[0].photo.favorite); } + #[test] + fn a_live_photo_video_is_filed_with_its_still_and_takes_its_date() { + let mut still = photo("IMG_1.HEIC", 100, Bucket::Timeline); + still.capture_time = Some(1_560_000_000); + still.location = Some(GeoPoint { + latitude: 64.1, + longitude: -21.9, + }); + let video = photo("IMG_1.MOV", 300, Bucket::Timeline); + let plain = photo("IMG_2.jpg", 100, Bucket::Timeline); + + let planned = plan(&TakeoutScan { + photos: vec![video, still, plain], + ..Default::default() + }); + + assert_eq!(planned[0].live, Some(LiveRole::Video(1))); + assert_eq!(planned[1].live, Some(LiveRole::Still(PhotoTag::LivePhoto))); + assert_eq!(planned[2].live, None); + assert_eq!(planned[0].photo.capture_time, Some(1_560_000_000)); + assert!(planned[0].photo.location.is_some()); + } + + #[test] + fn a_pixel_motion_photo_pairs_on_the_mp_name() { + let planned = plan(&TakeoutScan { + photos: vec![ + photo("PXL_1.MP.jpg", 100, Bucket::Timeline), + photo("PXL_1.MP", 300, Bucket::Timeline), + ], + ..Default::default() + }); + assert_eq!( + planned[0].live, + Some(LiveRole::Still(PhotoTag::MotionPhoto)) + ); + assert_eq!(planned[1].live, Some(LiveRole::Video(0))); + } + + #[test] + fn halves_taken_minutes_apart_are_not_a_pair() { + let mut still = photo("IMG_1.jpg", 100, Bucket::Timeline); + still.capture_time = Some(1_560_000_000); + let mut video = photo("IMG_1.MOV", 300, Bucket::Timeline); + video.capture_time = Some(1_560_000_000 + 3_600); + + let planned = plan(&TakeoutScan { + photos: vec![still, video], + ..Default::default() + }); + assert!(planned.iter().all(|p| p.live.is_none())); + } + + #[test] + fn an_album_copy_pairs_through_any_of_its_folders() { + // The still was first seen in an album, the video only in the year + // folder; the album copy of the still also sits in the year folder. + let planned = plan(&TakeoutScan { + photos: vec![ + photo("IMG_1.HEIC", 100, Bucket::Album("Trip".into())), + photo("IMG_1.HEIC", 100, Bucket::Timeline), + photo("IMG_1.MOV", 300, Bucket::Timeline), + ], + ..Default::default() + }); + assert_eq!(planned.len(), 2); + assert_eq!(planned[1].live, Some(LiveRole::Video(0))); + } + + #[test] + fn a_run_the_daemon_never_finished_loads_as_interrupted_and_resumable() { + let path = + std::env::temp_dir().join(format!("pdfs-takeout-state-{}.json", std::process::id())); + let saved = SavedImport { + archives: vec![PathBuf::from("/exports/takeout-001.zip")], + dry_run: false, + running: true, + summary: None, + }; + std::fs::write(&path, serde_json::to_vec(&saved).unwrap()).unwrap(); + + let state = ImportState::load(Some(path.clone())); + let _ = std::fs::remove_file(&path); + + assert!(!state.running); + assert!(state.summary.as_ref().unwrap().interrupted); + assert_eq!(state.resumable(), vec!["/exports/takeout-001.zip"]); + } + + #[test] + fn a_finished_import_or_a_scan_is_not_resumable() { + let done = ImportState { + archives: vec![PathBuf::from("/a.zip")], + summary: Some(ImportSummary { + uploaded: 3, + ..Default::default() + }), + ..Default::default() + }; + assert!(done.resumable().is_empty()); + + let scan = ImportState { + archives: vec![PathBuf::from("/a.zip")], + dry_run: true, + summary: Some(ImportSummary { + cancelled: true, + ..Default::default() + }), + ..Default::default() + }; + assert!(scan.resumable().is_empty()); + + let failed = ImportState { + archives: vec![PathBuf::from("/a.zip")], + summary: Some(ImportSummary { + failed: 1, + ..Default::default() + }), + ..Default::default() + }; + assert_eq!(failed.resumable(), vec!["/a.zip"]); + } + + #[test] + fn state_survives_a_save_and_load() { + let path = std::env::temp_dir().join(format!( + "pdfs-takeout-roundtrip-{}.json", + std::process::id() + )); + let state = ImportState { + archives: vec![PathBuf::from("/a.zip")], + summary: Some(ImportSummary { + uploaded: 2, + cancelled: true, + ..Default::default() + }), + path: Some(path.clone()), + ..Default::default() + }; + state.save(); + let loaded = ImportState::load(Some(path.clone())); + let _ = std::fs::remove_file(&path); + + assert_eq!(loaded.summary, state.summary); + assert_eq!(loaded.archives, state.archives); + } + #[test] fn sha1_matches_the_known_digest_of_abc() { let mut reader: &[u8] = b"abc"; diff --git a/crates/pdfs-gui/src/app/main.rs b/crates/pdfs-gui/src/app/main.rs index 453e316..f8396c5 100644 --- a/crates/pdfs-gui/src/app/main.rs +++ b/crates/pdfs-gui/src/app/main.rs @@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState}; use pdfs_core::control::{ ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo, - DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress, - PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope, - Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry, - ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress, - ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send, + DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem, + LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, + PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo, + SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, + SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send, }; use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec}; @@ -668,9 +668,11 @@ fn build_window(app: &adw::Application) { progress_bar: takeout_widgets.progress_bar.clone(), summary_group: takeout_widgets.summary_group.clone(), summary_rows: RefCell::new(Vec::new()), + resume_group: takeout_widgets.resume_group.clone(), + resume_row: takeout_widgets.resume_row.clone(), + resume_button: takeout_widgets.resume_button.clone(), inflight: Cell::new(false), running: Cell::new(false), - dry_run: Cell::new(false), }, update, }); diff --git a/crates/pdfs-gui/src/app/pages/status.rs b/crates/pdfs-gui/src/app/pages/status.rs index b4b2f65..0176ee2 100644 --- a/crates/pdfs-gui/src/app/pages/status.rs +++ b/crates/pdfs-gui/src/app/pages/status.rs @@ -1115,9 +1115,6 @@ pub(crate) fn repaint_transfers(ui: &Rc, items: &[TransferItem], jobs: &[Job // count falling to zero is what "the sync finished" looks like from here. // Jobs are deliberately not counted — a bulk upload retires its scan job and // starts its upload job mid-flight, which is not a thing finishing. - // The Import page reads its progress off the same snapshot rather than - // polling the daemon a second time for numbers already on the wire. - takeout_progress(ui, jobs); let previous = ui.status.active_transfers.replace(items.len()); if !items.is_empty() { @@ -1211,13 +1208,18 @@ pub(crate) fn repaint_transfers(ui: &Rc, items: &[TransferItem], jobs: &[Job pub(crate) fn job_line(j: &JobItem) -> ActivityLine { let done = j.done.to_string(); let total = j.total.to_string(); + let (title, detail) = if j.title == IMPORT_JOB { + import_job_text(&j.detail) + } else { + (j.title.clone(), j.detail.clone()) + }; let args: [(&str, &str); 4] = [ - ("title", &j.title), - ("detail", &j.detail), + ("title", &title), + ("detail", &detail), ("done", &done), ("total", &total), ]; - let text = match (j.total > 0, j.detail.is_empty()) { + let text = match (j.total > 0, detail.is_empty()) { // Translators: a background job's progress. {title} is the job, such as // "Scanning Photos"; {done} and {total} are counts. (true, true) => gettext_f("{title} — {done} of {total}", &args), diff --git a/crates/pdfs-gui/src/app/pages/takeout.rs b/crates/pdfs-gui/src/app/pages/takeout.rs index 62a6049..37a4370 100644 --- a/crates/pdfs-gui/src/app/pages/takeout.rs +++ b/crates/pdfs-gui/src/app/pages/takeout.rs @@ -15,17 +15,30 @@ //! Import — staging is local, so a mis-drop costs nothing. //! //! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only -//! stages paths, launches, asks [`Request::ImportStatus`] for the final report, -//! and reads live progress off the import's job in the transfer snapshot the -//! daemon already pushes (see [`takeout_progress`]). +//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the +//! running counts and the final report. It asks again whenever the import's job +//! moves in the transfer snapshot the daemon already pushes. A run the service +//! did not get to finish can be picked up with [`Request::ResumeImport`]. use crate::*; /// The title the daemon gives the import's job in [`Request::GetQueueStatus`]. -/// Matching on it is how this page borrows the progress the daemon already -/// pushes instead of asking for it separately. +/// Seeing it move is the cue to ask for the import's status; the Activity list +/// shows it under its translated name ([`import_job_text`]). pub(crate) const IMPORT_JOB: &str = "Importing Google Photos"; +/// The import job's title and detail in the user's language. The daemon names +/// its steps in English; the file being uploaded passes through as it is. +pub(crate) fn import_job_text(detail: &str) -> (String, String) { + let detail = match detail { + "reading archives" => gettext("Reading the archives"), + "checking for photos you already have" => gettext("Checking for photos you already have"), + "sorting photos into albums" => gettext("Sorting photos into albums"), + other => other.to_string(), + }; + (gettext("Importing from Google Photos"), detail) +} + /// What the archive list says before anything is staged. `repaint_archives` /// swaps in the count and total once there is something to report, and swaps /// this back when the list is emptied again. @@ -95,18 +108,19 @@ pub(crate) struct TakeoutState { pub(crate) progress_group: adw::PreferencesGroup, pub(crate) progress_label: gtk4::Label, pub(crate) progress_bar: gtk4::ProgressBar, - /// Report group for the last finished run, hidden until there is one. + /// The running counts, or the report for the last finished run; hidden + /// until there is one. pub(crate) summary_group: adw::PreferencesGroup, pub(crate) summary_rows: RefCell>, + /// Offers to resume an import the service did not get to finish. + pub(crate) resume_group: adw::PreferencesGroup, + pub(crate) resume_row: adw::ActionRow, + pub(crate) resume_button: gtk4::Button, /// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon /// can't stack worker threads on the 2s tick. pub(crate) inflight: Cell, /// Whether the daemon reported an import running on the last poll. pub(crate) running: Cell, - /// Whether the run in flight was launched as a dry run, so the report can - /// say "would upload" rather than claiming photos were sent. The daemon's - /// summary carries no such flag — it is the same shape either way. - pub(crate) dry_run: Cell, } /// Widgets the Import page hands back for wiring. @@ -126,6 +140,9 @@ pub(crate) struct TakeoutWidgets { pub(crate) progress_label: gtk4::Label, pub(crate) progress_bar: gtk4::ProgressBar, pub(crate) summary_group: adw::PreferencesGroup, + pub(crate) resume_group: adw::PreferencesGroup, + pub(crate) resume_row: adw::ActionRow, + pub(crate) resume_button: gtk4::Button, } pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) { @@ -261,7 +278,23 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) { .visible(false) .build(); + // An import cut short by a restart or a lost connection, offered back at the + // top of the page: picking the archives again is exactly what a reboot in + // the middle of a migration should not cost. + let resume_button = gtk4::Button::builder() + .label(pgettext("verb", "Resume")) + .valign(gtk4::Align::Center) + .build(); + resume_button.add_css_class("suggested-action"); + let resume_row = adw::ActionRow::builder() + .title(gettext("The last import didn't finish")) + .build(); + resume_row.add_suffix(&resume_button); + let resume_group = adw::PreferencesGroup::builder().visible(false).build(); + resume_group.add(&resume_row); + let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18); + groups.append(&resume_group); groups.append(&dropzone); groups.append(&list_group); groups.append(&actions); @@ -301,6 +334,9 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) { progress_label: progress_label.clone(), progress_bar: progress_bar.clone(), summary_group: summary_group.clone(), + resume_group, + resume_row, + resume_button, }; (frame.upcast(), widgets) } @@ -427,6 +463,28 @@ pub(crate) fn wire_takeout(ui: &Rc, widgets: &TakeoutWidgets) { }); }); + let ui_resume = ui.clone(); + widgets.resume_button.connect_clicked(move |_| { + let ui = ui_resume.clone(); + ui.busy_begin(); + let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport); + glib::spawn_future_local(async move { + let result = rx.recv().await; + ui.busy_end(); + match result { + Ok(Ok(Response::Ok { .. })) => import_started(&ui, false), + Ok(Ok(Response::Error { message, kind })) => { + toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind); + } + _ => toast_error( + &ui, + &gettext("Couldn't resume the import"), + &gettext("The Proton Drive service didn't respond."), + ), + } + }); + }); + let ui_back = ui.clone(); widgets .back_button @@ -559,6 +617,7 @@ fn sync_takeout_actions(ui: &Rc) { .import_button .set_sensitive(has_files && !running); ui.takeout.clear_button.set_sensitive(has_files && !running); + ui.takeout.resume_button.set_sensitive(!running); ui.takeout.cancel_button.set_visible(running); } @@ -577,7 +636,6 @@ fn start_import(ui: &Rc, dry_run: bool) { return; } - ui.takeout.dry_run.set(dry_run); ui.busy_begin(); let rx = spawn_request( ui.dirs.control_socket(), @@ -588,27 +646,7 @@ fn start_import(ui: &Rc, dry_run: bool) { let result = rx.recv().await; ui.busy_end(); match result { - Ok(Ok(Response::Ok { .. })) => { - ui.takeout.running.set(true); - ui.gallery.import_banner.set_revealed(true); - ui.takeout.summary_group.set_visible(false); - ui.takeout.progress_group.set_visible(true); - ui.takeout.progress_bar.set_fraction(0.0); - ui.takeout.progress_label.set_text(&if dry_run { - gettext("Reading archives…") - } else { - gettext("Starting import…") - }); - sync_takeout_actions(&ui); - toast( - &ui, - &if dry_run { - gettext("Scanning the export…") - } else { - gettext("Import started — you can leave this page, it keeps running.") - }, - ); - } + Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run), Ok(Ok(Response::Error { message, kind })) => { toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind); } @@ -621,9 +659,39 @@ fn start_import(ui: &Rc, dry_run: bool) { }); } -/// Ask the daemon for the import's state, and paint the report once a run has -/// finished. Asked on arrival and while import progress keeps arriving. Cheap -/// and inflight-guarded. +/// Switch the page over to reporting progress once the daemon took the run. +fn import_started(ui: &Rc, dry_run: bool) { + ui.takeout.running.set(true); + if !dry_run { + ui.gallery.import_banner.set_revealed(true); + } + ui.takeout.resume_group.set_visible(false); + ui.takeout.summary_group.set_visible(false); + ui.takeout.progress_group.set_visible(true); + ui.takeout.progress_group.set_title(&if dry_run { + gettext("Scan in progress") + } else { + gettext("Import in progress") + }); + ui.takeout.progress_bar.set_fraction(0.0); + ui.takeout + .progress_label + .set_text(&gettext("Reading the archives…")); + sync_takeout_actions(ui); + toast( + ui, + &if dry_run { + gettext("Scanning the export…") + } else { + gettext("Import started — you can leave this page, it keeps running.") + }, + ); +} + +/// Ask the daemon for the import's state and paint it: the step and counts +/// while it runs, the report once it has ended, and the offer to resume a run +/// that did not finish. Asked on arrival and while import progress keeps +/// arriving. Cheap and inflight-guarded. pub(crate) fn refresh_takeout(ui: &Rc) { if ui.takeout.inflight.get() { return; @@ -634,124 +702,250 @@ pub(crate) fn refresh_takeout(ui: &Rc) { glib::spawn_future_local(async move { let result = rx.recv().await; ui.takeout.inflight.set(false); - let Ok(Ok(Response::ImportStatus { running, summary })) = result else { + let Ok(Ok(Response::ImportStatus { + running, + summary, + progress, + resumable, + })) = result + else { return; }; let was_running = ui.takeout.running.replace(running); - ui.gallery.import_banner.set_revealed(running); + let dry = summary.as_ref().is_some_and(|s| s.dry_run); + ui.gallery.import_banner.set_revealed(running && !dry); + ui.takeout.progress_group.set_visible(running); if running { - ui.takeout.progress_group.set_visible(true); - } else { - ui.takeout.progress_group.set_visible(false); - // An import runs for hours, so its ending is announced rather than - // left to be discovered: a toast for whoever is looking at the - // window, and a desktop notification for whoever is not. - let failed = summary.as_ref().and_then(|s| s.error.clone()); - if was_running && let Some(error) = &failed { - let headline = gettext("Couldn't import from Google Photos"); - toast_error(&ui, &headline, error); - notify("takeout-import", &headline, error); - } else if was_running { - let dry = ui.takeout.dry_run.get(); - let headline = if dry { - gettext("Scan finished") - } else { - gettext("Import finished") - }; - toast(&ui, &headline); - if !dry { - let body = match &summary { - // Translators: {n} is the number of photos uploaded, {failed} the number that failed. - Some(s) if s.failed > 0 => ngettext_f( - "{n} photo added, {failed} failed.", - "{n} photos added, {failed} failed.", - s.uploaded as u64, - &[("failed", &s.failed.to_string())], - ), - Some(s) if s.cancelled => ngettext_f( - "Stopped after {n} photo.", - "Stopped after {n} photos.", - s.uploaded as u64, - &[], - ), - Some(s) => ngettext_f( - "{n} photo added to Proton Photos.", - "{n} photos added to Proton Photos.", - s.uploaded as u64, - &[], - ), - None => gettext("The Google Photos import has finished."), - }; - notify("takeout-import", &headline, &body); - } - } + ui.takeout.progress_group.set_title(&if dry { + gettext("Scan in progress") + } else { + gettext("Import in progress") + }); + paint_progress(&ui, progress.as_ref()); + } else if was_running && let Some(summary) = &summary { + announce_end(&ui, summary); } if was_running != running { sync_takeout_actions(&ui); } - match summary { - Some(summary) => repaint_summary(&ui, &summary), + match &summary { + Some(summary) => repaint_summary(&ui, summary, running), None => ui.takeout.summary_group.set_visible(false), } + paint_resume(&ui, running, summary.as_ref(), &resumable); }); } -/// Paint the last run's report. -fn repaint_summary(ui: &Rc, summary: &ImportSummary) { +/// Tell the user how a run ended. An import runs for hours, so its ending is +/// announced rather than left to be discovered: a toast for whoever is looking +/// at the window, and a desktop notification for whoever is not. +fn announce_end(ui: &Rc, summary: &ImportSummary) { + if summary.dry_run { + toast(ui, &gettext("Scan finished")); + return; + } + if let Some(error) = &summary.error { + let kind = summary.error_kind.unwrap_or_default(); + let headline = gettext("Couldn't import from Google Photos"); + toast_failure(ui, &headline, error, kind); + notify( + "takeout-import", + &headline, + &gettext("You can resume it once the problem is solved."), + ); + return; + } + let (headline, body) = if summary.interrupted { + ( + gettext("Import interrupted"), + gettext("You can resume it where it stopped."), + ) + } else if summary.cancelled { + ( + gettext("Import stopped"), + ngettext_f( + "Stopped after {n} photo.", + "Stopped after {n} photos.", + summary.uploaded as u64, + &[], + ), + ) + } else if summary.failed > 0 { + ( + gettext("Import finished"), + // Translators: {n} is the number of photos uploaded, {failed} the number that failed. + ngettext_f( + "{n} photo added, {failed} failed.", + "{n} photos added, {failed} failed.", + summary.uploaded as u64, + &[("failed", &summary.failed.to_string())], + ), + ) + } else { + ( + gettext("Import finished"), + ngettext_f( + "{n} photo added to Proton Photos.", + "{n} photos added to Proton Photos.", + summary.uploaded as u64, + &[], + ), + ) + }; + toast(ui, &headline); + notify("takeout-import", &headline, &body); + + // A complete import is done with its archives; keeping them staged would + // only invite running the same export again. + if summary.complete() { + ui.takeout.archives.borrow_mut().clear(); + repaint_archives(ui); + } +} + +/// Paint where a running import is. +fn paint_progress(ui: &Rc, progress: Option<&ImportProgress>) { + let Some(progress) = progress else { + ui.takeout.progress_bar.pulse(); + return; + }; + let done = progress.done.to_string(); + let total = progress.total.to_string(); + let current = progress.current.clone().unwrap_or_default(); + let counted = progress.total > 0; + let args = [ + ("done", done.as_str()), + ("total", total.as_str()), + ("name", current.as_str()), + ]; + let text = match (progress.phase, counted) { + (ImportPhase::Scanning, _) => gettext("Reading the archives…"), + (ImportPhase::Checking, false) => gettext("Checking for photos you already have…"), + // Translators: {done} and {total} are counts of photos. + (ImportPhase::Checking, true) => gettext_f( + "Checking for photos you already have — {done} of {total}", + &args, + ), + (ImportPhase::Uploading, false) => gettext("Uploading…"), + (ImportPhase::Uploading, true) if current.is_empty() => { + // Translators: {done} and {total} are counts of photos. + gettext_f("Uploading {done} of {total}", &args) + } + // Translators: {done} and {total} are counts of photos; {name} is the file being uploaded. + (ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args), + (ImportPhase::Albums, false) => gettext("Sorting photos into albums…"), + // Translators: {done} and {total} are counts of albums. + (ImportPhase::Albums, true) => { + gettext_f("Sorting photos into albums — {done} of {total}", &args) + } + }; + ui.takeout.progress_label.set_text(&text); + if counted { + ui.takeout + .progress_bar + .set_fraction((progress.done as f64 / progress.total as f64).min(1.0)); + } else { + ui.takeout.progress_bar.pulse(); + } +} + +/// Offer to resume the last run when the service says it can be. +fn paint_resume(ui: &Rc, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) { + let offer = !running && !resumable.is_empty(); + ui.takeout.resume_group.set_visible(offer); + if !offer { + return; + } + let uploaded = summary.map_or(0, |s| s.uploaded) as u64; + // Translators: {n} is the number of photos uploaded before the import stopped. + ui.takeout.resume_row.set_subtitle(&ngettext_f( + "{n} photo uploaded before it stopped. Resuming skips what is already there.", + "{n} photos uploaded before it stopped. Resuming skips what is already there.", + uploaded, + &[], + )); +} + +/// Paint the counts of a running import, or the report of the last one. +fn repaint_summary(ui: &Rc, summary: &ImportSummary, running: bool) { for row in ui.takeout.summary_rows.borrow_mut().drain(..) { ui.takeout.summary_group.remove(&row); } - let dry = ui.takeout.dry_run.get(); - ui.takeout.summary_group.set_title(&if dry { + let dry = summary.dry_run; + ui.takeout.summary_group.set_title(&if running { + gettext("So far") + } else if dry { gettext("Scan result") } else { gettext("Last import") }); - ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error { - // The reason is the service's own text, which is not translated. A - // group's description is markup, and the reason may name a path. - glib::markup_escape_text(&gettext_f( - "{what}: {detail}", - &[ - ("what", &gettext("Couldn't import from Google Photos")), - ("detail", error), - ], - )) - .to_string() + let description = if running { + None + } else if let Some(error) = &summary.error { + // The reason is the service's own text, which is not translated; the + // headline in front of it is. A group's description is markup, and the + // reason may name a path. + let what = gettext("Couldn't import from Google Photos"); + Some( + glib::markup_escape_text(&gettext_f( + "{what}: {detail}", + &[ + ( + "what", + error_headline(summary.error_kind.unwrap_or_default(), &what), + ), + ("detail", error), + ], + )) + .to_string(), + ) } else if dry { - gettext("Nothing was uploaded — this is what an import would do.") + Some(gettext( + "Nothing was uploaded — this is what an import would do.", + )) + } else if summary.interrupted { + Some(gettext( + "The Proton Drive service stopped while the import ran. Resume it to continue where it left off.", + )) } else if summary.cancelled { - gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.") + Some(gettext( + "Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.", + )) + } else if !summary.complete() { + Some(gettext( + "Finished with problems. Run it again to retry what didn't upload.", + )) } else { - gettext("Finished.") - })); + Some(gettext("Finished.")) + }; + ui.takeout + .summary_group + .set_description(description.as_deref()); // Only lines that carry information: a run with no failures should not have // to show a "0 failed" row for the user to read past. - let mut lines: Vec<(String, String)> = vec![ - (gettext("Photos in the export"), summary.found.to_string()), - ( - if dry { - gettext("Would upload") - } else { - gettext("Uploaded") - }, - summary.uploaded.to_string(), - ), - ]; + let mut lines: Vec<(String, String)> = + vec![(gettext("Photos in the export"), summary.found.to_string())]; + if dry { + lines.push((gettext("Would upload"), summary.to_upload.to_string())); + } else { + lines.push((gettext("Uploaded"), summary.uploaded.to_string())); + } if summary.duplicates > 0 { lines.push(( gettext("Already in Proton Photos"), summary.duplicates.to_string(), )); } + if summary.live_photos > 0 { + lines.push(( + gettext("Live photos kept together"), + summary.live_photos.to_string(), + )); + } if summary.albums_created > 0 { lines.push(( - if dry { - gettext("Albums to create") - } else { - gettext("Albums created") - }, + gettext("Albums created"), summary.albums_created.to_string(), )); } @@ -767,25 +961,31 @@ fn repaint_summary(ui: &Rc, summary: &ImportSummary) { summary.skipped_trashed.to_string(), )); } - if summary.bytes > 0 { + if dry && summary.to_upload_bytes > 0 { lines.push(( - if dry { - gettext("Data to upload") - } else { - gettext("Data uploaded") - }, - human_bytes(summary.bytes), + gettext("Data to upload"), + human_bytes(summary.to_upload_bytes), )); + } else if summary.bytes > 0 { + lines.push((gettext("Data uploaded"), human_bytes(summary.bytes))); } if summary.failed > 0 { lines.push((pgettext("state", "Failed"), summary.failed.to_string())); } + for path in &summary.unreadable_archives { + let name = Path::new(path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| path.clone()); + lines.push((gettext("Couldn't read this archive"), name)); + } let mut rows = Vec::new(); for (title, value) in lines { let row = adw::ActionRow::builder() .title(title) .subtitle(value) + .use_markup(false) .build(); row.add_css_class("property"); ui.takeout.summary_group.add(&row); @@ -794,35 +994,3 @@ fn repaint_summary(ui: &Rc, summary: &ImportSummary) { *ui.takeout.summary_rows.borrow_mut() = rows; ui.takeout.summary_group.set_visible(true); } - -/// Paint the import's progress from the transfer snapshot the daemon pushed, -/// so this page costs no extra request. A job with no known total -/// pulses instead of showing a fraction, the same as the Activity list. -pub(crate) fn takeout_progress(ui: &Rc, jobs: &[JobItem]) { - let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else { - return; - }; - ui.takeout.progress_group.set_visible(true); - if job.total > 0 { - ui.takeout - .progress_bar - .set_fraction((job.done as f64 / job.total as f64).min(1.0)); - let done = job.done.to_string(); - let total = job.total.to_string(); - // Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on. - let text = gettext_f( - "{done} of {total} — {detail}", - &[("done", &done), ("total", &total), ("detail", &job.detail)], - ); - ui.takeout.progress_label.set_text(&text); - } else { - ui.takeout.progress_bar.pulse(); - ui.takeout - .progress_label - .set_text(&if job.detail.is_empty() { - gettext("Working…") - } else { - job.detail.clone() - }); - } -} diff --git a/docs/BUGS.md b/docs/BUGS.md index b0ae67e..d24fb3b 100644 --- a/docs/BUGS.md +++ b/docs/BUGS.md @@ -109,6 +109,10 @@ a live account since. | Entry | Title | |---|---| +| [B289](#b289--the-takeout-import-mixes-in-other-services-and-drops-location-and-live-photos) | The Takeout import mixes in other services and drops location and live photos | +| [B288](#b288--an-interrupted-takeout-import-reads-as-finished-and-one-error-ends-it) | An interrupted Takeout import reads as finished, and one error ends it | +| [B287](#b287--a-photo-draft-left-by-an-interrupted-upload-blocks-that-name) | A photo draft left by an interrupted upload blocks that name | +| [B286](#b286--resuming-a-takeout-import-fails-on-a-draft-with-no-content-hash) | Resuming a Takeout import fails on a draft with no content hash | | [B285](#b285--a-restarted-secret-service-leaves-the-process-on-a-dead-session) | A restarted Secret Service leaves the process on a dead session | | [B284](#b284--a-keyring-entry-reported-missing-discards-refreshed-session-tokens) | A keyring entry reported missing discards refreshed session tokens | | [B283](#b283--videos-have-no-thumbnail-outside-photos-and-a-ts-recording-looks-like-source-code) | Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code | @@ -388,6 +392,127 @@ a live account since. Newest first. +### B289 — The Takeout import mixes in other services and drops location and live photos + +**Status:** Fixed, not verified live +**Found:** 2026-11-15, auditing the Google Photos import alongside +[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94). +**Where:** `crates/pdfs-core/src/takeout.rs`; `plan` and `upload_takeout_photo` in +`crates/pdfs-fuse/src/takeout.rs` + +**Repro:** Import a Takeout export that also holds `Drive/` or `YouTube/`, with Pixel motion +photos, iPhone live photos and photos that have a place in Google Photos. + +**Cause.** Five gaps in what the import carried over: +- The service folder was dropped without being checked, so media from other Google services in + the same export landed in the photo timeline. +- `geoData` was never read, although `docs/PHOTOS.md` said the import takes locations from the + export, and the SDK had no way to upload a location. +- The two halves of a live or motion photo uploaded as unrelated photos, the `.MOV` half usually + without a date, and the `.MP` half of a Pixel motion photo was skipped as unknown. +- `creationTime`, the day the photo reached Google, outranked the date in the file name and the + photo's own EXIF. +- A sidecar with `null` in `favorited`, a numeric `timestamp` or a `0` timestamp lost its other + fields or filed the photo under 1970. + +**Fix:** The Photos service is the folder whose sidecars carry `photoTakenTime`, and only it is +imported (everything, when no folder does). `geoData`, else `geoDataExif`, becomes the photo's +`Location` attribute through SDK 0.12.0's `PhotoUploadMetadata::location`; `0, 0` is no position. +Stills and videos with the same stem in one folder (`IMG_1.HEIC` + `IMG_1.MOV`, +`PXL_1.MP.jpg` + `PXL_1.MP`) are paired unless their dates are minutes apart: the still is +tagged as a live or motion photo, the video is uploaded after it with `main_photo_uid` and takes +its date and place. The capture time is `photoTakenTime`, then the name, then EXIF, then +`creationTime`. Favorites are also set on photos that were already on the account. + +**Verified:** `takeout::tests` in `pdfs-core` (services, locations, timestamps, live keys) and +`pdfs-fuse` (pairing, skew, folded copies). Not run against a real export and account. + +--- + +### B288 — An interrupted Takeout import reads as finished, and one error ends it + +**Status:** Fixed, not verified live +**Found:** 2026-11-15, auditing the Google Photos import alongside +[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94). +**Where:** `crates/pdfs-fuse/src/takeout.rs`; `refresh_takeout` in +`crates/pdfs-gui/src/app/pages/takeout.rs`; `cmd_import_google_photos` in +`crates/pdfs-cli/src/main.rs` + +**Repro:** Start an import of several archives, one of them damaged, and restart the service +while it uploads. + +**Cause.** +- The import state lived only in memory. After a restart `ImportStatus` had no summary, and the + app announced "Import finished". The archives had to be chosen again to continue. +- A daemon shutdown broke the upload loop but still filed albums and logged a success. +- One unreadable archive failed the scan, and a failed request in the middle of the run + discarded every count and skipped the albums for what had uploaded. +- The progress total counted photos already on the account, so the bar stopped short of the + end; a dry run reported "Would upload: 0"; the job's phases reached the app in English. + +**Fix:** The last run's archives and summary are kept in `takeout-import.json` in the state +directory. A run still marked running at start-up is `interrupted`, and `ImportStatus` lists +its archives as `resumable`; `ResumeImport` (`pdfs import-google-photos --resume`, **Resume** in +the app) starts it again, and the duplicate check skips what is there. An unreadable archive is +listed in `unreadable_archives` and the rest imports. A failure keeps the counts and still files +albums. `ImportStatus` carries the phase and its own progress, the summary says `dry_run`, +`to_upload` and `error_kind`, and the app words them in the user's language. The inline +timeline refresh of an empty gallery waits for a refresh already running instead of starting a +second one (`refresh_timeline_joined`). + +**Verified:** `takeout::tests` in `pdfs-fuse` (saved state, resumable runs) and `pdfs-core` +(unreadable archive), and the CLI tests for `--resume` and the exit status. Not run live. + +--- + +### B287 — A photo draft left by an interrupted upload blocks that name + +**Status:** Fixed, not verified live +**Found:** 2026-11-15, reading the SDK upload path for +[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94). +**Where:** `upload_photo_from`, `upload_file_from_inner` and `abandon_file_draft` in +`proton-drive-rs` (SDK 0.12.0) + +**Repro:** Interrupt a photo upload after its draft is created, sign in again, and upload a +photo with the same name. + +**Cause.** Photo uploads replaced a leftover draft only when its `ClientUID` was the current +session's. A new login is a new session, so the next upload of that name failed with 2500. A +failed upload also never deleted its own draft, which the C# and TypeScript SDKs do. + +**Fix:** SDK 0.12.0 adds `PhotoUploadMetadata::replace_existing_draft`, which the Takeout import +sets, and deletes the draft of a new file or photo whose upload fails. After a failed seal it +only does so on a definite rejection, because a seal lost in transit may have been applied. + +**Verified:** SDK unit test `draft_cleanup_only_after_a_definite_seal_rejection`. Not run live. + +--- + +### B286 — Resuming a Takeout import fails on a draft with no content hash + +**Status:** Fixed, not verified live +**Found:** 2026-11-15, [issue #94](https://github.com/narrrl/proton-drive-linux/issues/94) +(3.7.0, Fedora). An import stopped by a restart failed every re-run at once with +`check for existing photos: serialization error: invalid type: null, expected a string at line +1 column 2063`. +**Where:** `FoundPhotoDuplicateDto` in `proton-drive-rs` (SDK 0.11.0) + +**Repro:** Interrupt a photo upload after its draft is created, then run the import again. + +**Cause.** The duplicates endpoint sends `ContentHash` (and can send `Hash`) as `null` for a +draft, as its API schema says. The SDK declared both as `String` with `#[serde(default)]`, +which covers a missing field but not `null`, so one draft failed the whole batch of 150 names +and the import ended before uploading anything. + +**Fix:** SDK 0.12.0 makes both fields optional and skips drafts, trashed links and candidates +without hashes, like the TypeScript SDK. Every other defaulted DTO field reads `null` as its +default too, and an undecodable response is logged with its path. + +**Verified:** SDK unit tests `duplicate_response_tolerates_null_fields_and_skips_drafts` and +`duplicate_response_tolerates_a_null_list`. Not run against the reporter's account. + +--- + ### B285 — A restarted Secret Service leaves the process on a dead session **Status:** Fixed (unverified) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6b7a2e1..ae0e96d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -112,14 +112,49 @@ Existing folders stay device backups. Older builds cannot open the upgraded data carry their cover as their thumbnail, so they show it at once on other devices too. A cover image added to a folder later is picked up once the song changes or the service restarts. +- **Resume a Google Photos import** (B288). The archives and the outcome of the last import are + kept, so an import cut short by a restart or a reboot no longer has to be set up again. The + Import page offers **Resume**, and `pdfs import-google-photos --resume` does the same. Photos + that are already uploaded are skipped as before. + ### Changed -- **`proton-sdk` and `proton-drive-rs` 0.11.0.** Timestamps are converted where they enter +- **`proton-sdk` and `proton-drive-rs` 0.12.0.** Timestamps are converted where they enter from the SDK, and a photo keeps its own modification time now that it no longer follows the - capture time. [The SDK review](SDK-0.11.0.md) lists the new features and breaking changes. + capture time. 0.12.0 fixes the photo duplicate check and failed-upload drafts below. + [The 0.11.0 review](SDK-0.11.0.md) and [the 0.12.0 review](SDK-0.12.0.md) list the new + features and breaking changes. + +- **The Google Photos import brings more of the export along** (B289). + - Places from Google Photos are kept, so imported photos show up under **Places**. + - Live photos and Pixel motion photos stay one photo: the video is filed with its still and + takes its date. The `.MP` half of a motion photo is no longer skipped. + - Only the Google Photos part of an export is imported. Files from Google Drive, YouTube and + other services in the same Takeout stay out of the timeline. + - A photo without a date of its own is dated by its file name and then by its EXIF before + the day it reached Google Photos. + - Favorites are also set on photos that were already in Proton Photos. + +- **The import reports what it is doing** (B288). The Import page shows the current step + (reading, checking for photos you already have, uploading, sorting into albums), the counts + so far, and a progress bar that reaches the end. A scan says how many photos and how much + data an import would upload. `pdfs import-google-photos --wait` prints its progress, and + `pdfs import-status` shows the step. ### Fixed +- **Resuming a Google Photos import no longer fails at once** (#94, B286, B287). A photo upload + that was cut short left a draft behind, and every later run stopped with + `check for existing photos: serialization error: invalid type: null`. Such drafts are now + skipped by the duplicate check and replaced by the next upload of that photo, also after + signing in again. A failed upload removes its draft. + +- **One bad archive or error no longer ends the whole import** (B288). An archive that cannot be + read is left out and named in the report, and the rest of the export imports. When an import + stops early, it keeps the counts of what it did and still files the uploaded photos into their + albums. A restart of the service while importing is reported as an interruption, not as a + finished import. + - **Photos zoomed all the way out shows the whole timeline.** The first 200 photos fit on one screen there, and with nothing to scroll no more were loaded. More now load whenever the photos don't fill the window, also after a zoom or a resize. diff --git a/docs/CLI.md b/docs/CLI.md index 6b766b1..8a4d4ea 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -196,6 +196,7 @@ commands are for scripts and for cases where the mount is not available. | `pdfs add-to-album UID PHOTOS...` | Add photos to an album. | | `pdfs remove-from-album UID PHOTOS...` | Take photos out of an album. | | `pdfs import-google-photos ARCHIVES... [--dry-run] [--wait]` | Import a Google Takeout export. Pass every `.zip` of the export. | +| `pdfs import-google-photos --resume [--wait]` | Start the last import again with the same archives after it was interrupted, stopped or failed. | | `pdfs import-status` / `pdfs cancel-import` | Follow or stop a running import. | | `pdfs redate-photos [--dry-run] [--from DATE] [--to DATE] [--limit N] [--wait]` | Fix photos filed under their upload day when the file name carries the real date. | | `pdfs redate-status` / `pdfs cancel-redate` | Follow or stop a running re-date. | diff --git a/docs/PHOTOS.md b/docs/PHOTOS.md index 80ee746..80be461 100644 --- a/docs/PHOTOS.md +++ b/docs/PHOTOS.md @@ -55,6 +55,7 @@ server learns which areas you look at. 2. In the app, open the **Upload** menu on the Photos page and choose **Import from Google Photos…**. Select all the `.zip` files at once, because a photo and its details are often in different parts of the export. +3. Choose **Scan** to see what an import would do without uploading anything, then **Import**. From a terminal: @@ -63,8 +64,22 @@ pdfs import-google-photos --dry-run ~/Downloads/takeout-*.zip # show what woul pdfs import-google-photos --wait ~/Downloads/takeout-*.zip ``` -The import recreates your albums and takes capture dates and locations from the export. Photos -already in your library are skipped, so an interrupted import can simply be run again. +What the import takes from the export: + +- **Albums** are recreated. A photo that is in several albums is uploaded once and added to each. +- **Dates** come from the export's details, else from a date in the file name, else from the + photo's EXIF. +- **Places** come from the export, so imported photos appear under [Places](#places). +- **Favorites** are kept. +- **Live photos and motion photos** stay one photo: the video is filed with its still. + +What it leaves out: photos in Google's trash, and files from other Google services (Drive, +YouTube) that are in the same export. A `.zip` that cannot be read is left out and named in the +report; the rest still imports. + +Photos already in your library are skipped, so an interrupted import can simply be run again. +If the import was cut short, for example by a restart, the Import page offers **Resume**, and +`pdfs import-google-photos --resume` does the same without naming the archives again. `pdfs import-status` shows how far an import is, and `pdfs cancel-import` stops it. ## Fixing photos filed under the wrong date diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4ca340e..3dd278b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -63,7 +63,8 @@ them along. | Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK | | Photo name search | Not started | Photos search by date and filters only | | Automatic photo upload from a folder | Not started | | -| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos; motion-photo `.MP` parts are skipped | +| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos | +| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions | | Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content | ## Performance and correctness diff --git a/docs/SDK-0.12.0.md b/docs/SDK-0.12.0.md new file mode 100644 index 0000000..e69de29 diff --git a/po/de.po b/po/de.po index 2863084..0ff0a83 100644 --- a/po/de.po +++ b/po/de.po @@ -5,7 +5,7 @@ msgid "" msgstr "" "Project-Id-Version: proton-drive-linux 2.1.0\n" "Report-Msgid-Bugs-To: https://github.com/narrrl/proton-drive-linux/issues\n" -"POT-Creation-Date: 2026-11-14 15:41+0100\n" +"POT-Creation-Date: 2026-11-15 16:54+0100\n" "PO-Revision-Date: 2026-09-23 12:00+0200\n" "Last-Translator: Proton Drive for Linux contributors\n" "Language-Team: German\n" @@ -57,7 +57,7 @@ msgstr "%-d. %b %Y" msgid "%A, %B %-d, %Y, %H:%M" msgstr "%A, %-d. %B %Y, %H:%M" -#: crates/pdfs-gui/src/app/main.rs:879 +#: crates/pdfs-gui/src/app/main.rs:881 #: crates/pdfs-gui/src/app/pages/activity_text.rs:543 #: crates/pdfs-gui/src/app/pages/browser.rs:473 #: crates/pdfs-gui/src/app/pages/browser.rs:880 @@ -70,75 +70,75 @@ msgstr "%A, %-d. %B %Y, %H:%M" msgid "My files" msgstr "Meine Dateien" -#: crates/pdfs-gui/src/app/main.rs:880 crates/pdfs-gui/src/app/main.rs:1550 +#: crates/pdfs-gui/src/app/main.rs:882 crates/pdfs-gui/src/app/main.rs:1552 #: crates/pdfs-gui/src/app/pages/albums.rs:636 #: crates/pdfs-gui/src/app/pages/photos.rs:929 #: crates/pdfs-gui/src/app/pages/places.rs:473 msgid "Photos" msgstr "Fotos" -#: crates/pdfs-gui/src/app/main.rs:883 +#: crates/pdfs-gui/src/app/main.rs:885 #: crates/pdfs-gui/src/app/pages/shared.rs:168 #: crates/pdfs-gui/src/app/pages/shared.rs:493 msgid "Shared with me" msgstr "Mit mir geteilt" -#: crates/pdfs-gui/src/app/main.rs:888 +#: crates/pdfs-gui/src/app/main.rs:890 #: crates/pdfs-gui/src/app/pages/shared_by_me.rs:120 msgid "Shared by me" msgstr "Von mir geteilt" -#: crates/pdfs-gui/src/app/main.rs:891 +#: crates/pdfs-gui/src/app/main.rs:893 #: crates/pdfs-gui/src/app/pages/devices.rs:140 #: crates/pdfs-gui/src/app/pages/devices.rs:505 msgid "Computers" msgstr "Computer" -#: crates/pdfs-gui/src/app/main.rs:892 +#: crates/pdfs-gui/src/app/main.rs:894 #: crates/pdfs-gui/src/app/pages/trash.rs:101 msgid "Trash" msgstr "Papierkorb" #. Translators: sidebar entry for the page listing this computer's synced folders. -#: crates/pdfs-gui/src/app/main.rs:894 +#: crates/pdfs-gui/src/app/main.rs:896 msgid "Sync" msgstr "Synchronisierung" #. Translators: a failed action's toast. {what} says what failed, such as #. "Couldn't rename"; {detail} is the reason, usually untranslated text #. from the mount service. -#: crates/pdfs-gui/src/app/main.rs:1244 +#: crates/pdfs-gui/src/app/main.rs:1246 #: crates/pdfs-gui/src/app/pages/photo_viewer.rs:915 -#: crates/pdfs-gui/src/app/pages/takeout.rs:714 +#: crates/pdfs-gui/src/app/pages/takeout.rs:891 #, rust-format msgid "{what}: {detail}" msgstr "{what}: {detail}" -#: crates/pdfs-gui/src/app/main.rs:1267 +#: crates/pdfs-gui/src/app/main.rs:1269 msgid "You're offline" msgstr "Keine Internetverbindung" -#: crates/pdfs-gui/src/app/main.rs:1268 +#: crates/pdfs-gui/src/app/main.rs:1270 msgid "That's not there any more" msgstr "Das ist nicht mehr vorhanden" -#: crates/pdfs-gui/src/app/main.rs:1269 +#: crates/pdfs-gui/src/app/main.rs:1271 msgid "You don't have access to that" msgstr "Kein Zugriff darauf" -#: crates/pdfs-gui/src/app/main.rs:1270 +#: crates/pdfs-gui/src/app/main.rs:1272 msgid "Something changed this first" msgstr "Etwas anderes hat dies zuerst geändert" -#: crates/pdfs-gui/src/app/main.rs:1271 +#: crates/pdfs-gui/src/app/main.rs:1273 msgid "Your Proton Drive is full" msgstr "Das Proton Drive ist voll" -#: crates/pdfs-gui/src/app/main.rs:1358 +#: crates/pdfs-gui/src/app/main.rs:1360 msgid "Proton Drive for Linux" msgstr "Proton Drive for Linux" -#: crates/pdfs-gui/src/app/main.rs:1366 +#: crates/pdfs-gui/src/app/main.rs:1368 msgid "" "Files-on-demand Proton Drive for the Linux desktop.\n" "\n" @@ -150,23 +150,23 @@ msgstr "" "Inoffizieller Client – nicht mit Proton AG verbunden, von Proton AG " "unterstützt oder befürwortet." -#: crates/pdfs-gui/src/app/main.rs:1371 +#: crates/pdfs-gui/src/app/main.rs:1373 msgid "Source Code" msgstr "Quellcode" -#: crates/pdfs-gui/src/app/main.rs:1375 +#: crates/pdfs-gui/src/app/main.rs:1377 msgid "Proton Terms of Service" msgstr "Nutzungsbedingungen von Proton" -#: crates/pdfs-gui/src/app/main.rs:1379 +#: crates/pdfs-gui/src/app/main.rs:1381 msgid "Proton Privacy Policy" msgstr "Datenschutzerklärung von Proton" -#: crates/pdfs-gui/src/app/main.rs:1384 +#: crates/pdfs-gui/src/app/main.rs:1386 msgid "Disclaimer" msgstr "Haftungsausschluss" -#: crates/pdfs-gui/src/app/main.rs:1388 +#: crates/pdfs-gui/src/app/main.rs:1390 msgid "" "This is an unofficial, community-built client. It is not affiliated with, " "endorsed by, or supported by Proton AG.\n" @@ -191,7 +191,7 @@ msgstr "" "bereitgestellt. Daten, deren Verlust nicht hinnehmbar ist, sollten " "zusätzlich gesichert werden." -#: crates/pdfs-gui/src/app/main.rs:1396 +#: crates/pdfs-gui/src/app/main.rs:1398 msgid "" "Places are named with data from GeoNames, licensed under GeoNames, lizenziert unter CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -211,7 +211,7 @@ msgstr "" "href=\"https://www.naturalearthdata.com/\">Natural Earth gezeichnet, die " "gemeinfrei sind." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames, licensed under CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -208,7 +208,7 @@ msgstr "" "\">Natural Earth land outlines and country borders, which are in the " "public domain." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames, con licencia CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -209,7 +209,7 @@ msgstr "" "fronteras de países de Natural " "Earth, que son de dominio público." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames, sous licence CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -210,7 +210,7 @@ msgstr "" "frontières des pays de Natural " "Earth, qui sont dans le domaine public." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames, con licenza CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -208,7 +208,7 @@ msgstr "" "confini nazionali di Natural " "Earth, che sono di pubblico dominio." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames, gelicentieerd onder CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -209,7 +209,7 @@ msgstr "" "href=\"https://www.naturalearthdata.com/\">Natural Earth, die in het " "publieke domein vallen." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the \n" "Language-Team: LANGUAGE \n" @@ -60,7 +60,7 @@ msgstr "" msgid "%A, %B %-d, %Y, %H:%M" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:879 +#: crates/pdfs-gui/src/app/main.rs:881 #: crates/pdfs-gui/src/app/pages/activity_text.rs:543 #: crates/pdfs-gui/src/app/pages/browser.rs:473 #: crates/pdfs-gui/src/app/pages/browser.rs:880 @@ -73,75 +73,75 @@ msgstr "" msgid "My files" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:880 crates/pdfs-gui/src/app/main.rs:1550 +#: crates/pdfs-gui/src/app/main.rs:882 crates/pdfs-gui/src/app/main.rs:1552 #: crates/pdfs-gui/src/app/pages/albums.rs:636 #: crates/pdfs-gui/src/app/pages/photos.rs:929 #: crates/pdfs-gui/src/app/pages/places.rs:473 msgid "Photos" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:883 +#: crates/pdfs-gui/src/app/main.rs:885 #: crates/pdfs-gui/src/app/pages/shared.rs:168 #: crates/pdfs-gui/src/app/pages/shared.rs:493 msgid "Shared with me" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:888 +#: crates/pdfs-gui/src/app/main.rs:890 #: crates/pdfs-gui/src/app/pages/shared_by_me.rs:120 msgid "Shared by me" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:891 +#: crates/pdfs-gui/src/app/main.rs:893 #: crates/pdfs-gui/src/app/pages/devices.rs:140 #: crates/pdfs-gui/src/app/pages/devices.rs:505 msgid "Computers" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:892 +#: crates/pdfs-gui/src/app/main.rs:894 #: crates/pdfs-gui/src/app/pages/trash.rs:101 msgid "Trash" msgstr "" #. Translators: sidebar entry for the page listing this computer's synced folders. -#: crates/pdfs-gui/src/app/main.rs:894 +#: crates/pdfs-gui/src/app/main.rs:896 msgid "Sync" msgstr "" #. Translators: a failed action's toast. {what} says what failed, such as #. "Couldn't rename"; {detail} is the reason, usually untranslated text #. from the mount service. -#: crates/pdfs-gui/src/app/main.rs:1244 +#: crates/pdfs-gui/src/app/main.rs:1246 #: crates/pdfs-gui/src/app/pages/photo_viewer.rs:915 -#: crates/pdfs-gui/src/app/pages/takeout.rs:714 +#: crates/pdfs-gui/src/app/pages/takeout.rs:891 #, rust-format msgid "{what}: {detail}" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1267 +#: crates/pdfs-gui/src/app/main.rs:1269 msgid "You're offline" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1268 +#: crates/pdfs-gui/src/app/main.rs:1270 msgid "That's not there any more" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1269 +#: crates/pdfs-gui/src/app/main.rs:1271 msgid "You don't have access to that" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1270 +#: crates/pdfs-gui/src/app/main.rs:1272 msgid "Something changed this first" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1271 +#: crates/pdfs-gui/src/app/main.rs:1273 msgid "Your Proton Drive is full" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1358 +#: crates/pdfs-gui/src/app/main.rs:1360 msgid "Proton Drive for Linux" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1366 +#: crates/pdfs-gui/src/app/main.rs:1368 msgid "" "Files-on-demand Proton Drive for the Linux desktop.\n" "\n" @@ -149,23 +149,23 @@ msgid "" "AG." msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1371 +#: crates/pdfs-gui/src/app/main.rs:1373 msgid "Source Code" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1375 +#: crates/pdfs-gui/src/app/main.rs:1377 msgid "Proton Terms of Service" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1379 +#: crates/pdfs-gui/src/app/main.rs:1381 msgid "Proton Privacy Policy" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1384 +#: crates/pdfs-gui/src/app/main.rs:1386 msgid "Disclaimer" msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1388 +#: crates/pdfs-gui/src/app/main.rs:1390 msgid "" "This is an unofficial, community-built client. It is not affiliated with, " "endorsed by, or supported by Proton AG.\n" @@ -178,21 +178,21 @@ msgid "" "of any data you cannot afford to lose." msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1396 +#: crates/pdfs-gui/src/app/main.rs:1398 msgid "" "Places are named with data from GeoNames, licensed under CC BY 4.0." msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " "public domain." msgstr "" -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames udostępnionych na licencji CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -212,7 +212,7 @@ msgstr "" "www.naturalearthdata.com/\">Natural Earth, które należą do domeny " "publicznej." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the GeoNames, licensed under GeoNames, licenciados sob CC BY 4.0." -#: crates/pdfs-gui/src/app/main.rs:1404 +#: crates/pdfs-gui/src/app/main.rs:1406 msgid "" "The places map is drawn from Natural Earth land outlines and country borders, which are in the " @@ -208,7 +208,7 @@ msgstr "" "fronteiras de países do Natural Earth, que são de domínio público." -#: crates/pdfs-gui/src/app/main.rs:1412 +#: crates/pdfs-gui/src/app/main.rs:1414 msgid "" "The street map, when turned on, comes from OpenFreeMap in the