fix(fuse,core,gui,cli): resumable, robust Google Photos import (#94)

A Takeout import cut short by a restart could never be resumed: the
duplicate check failed on the draft the interrupted upload left behind
("invalid type: null, expected a string"). SDK 0.12.0 skips such drafts,
lets the import replace them across logins, and removes the draft of a
failed upload (B285, B286).

The import keeps its archives and outcome in the state directory, so a
run the service did not finish is reported as interrupted and can be
resumed from the app or with `pdfs import-google-photos --resume`. An
unreadable archive is left out instead of failing the scan, a failure
keeps its counts and still files albums, and a cancel or shutdown is
only reported when work was left undone (B287).

ImportStatus carries the phase, its progress, the running counts and
what can be resumed; the summary says dry_run, to_upload and
error_kind. The app shows these in the user's language, and --wait
prints progress. An empty gallery waits for a timeline refresh already
running instead of starting a second one.

The scan imports only the Photos service of an export, reads geoData
into the photo's Location, pairs live and motion photos (.MOV, .MP) with
their stills, and dates a photo by name and EXIF before Google's upload
time (B288).
This commit is contained in:
nvrl committed 2026-11-15 17:17:47 +01:00
1 parent 2ec3b7ff62
commit 2f2fe96cdf
28 files changed
+5196 -2544

No files matched your search

+180 -38
View File
@@ -228,14 +228,19 @@ enum Command {
/// numbered parts, and a photo's metadata often sits in a different part
/// than the photo. Albums in the export are recreated as Proton albums, and
/// photos already on the account (same name and same content) are skipped,
/// so an interrupted import can simply be run again.
/// so an interrupted import can simply be run again, or resumed with
/// `--resume` without naming the archives again.
ImportGooglePhotos {
/// The export's `.zip` files (e.g. `takeout-*.zip`).
#[arg(required = true, value_hint = ValueHint::FilePath)]
#[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)]
archives: Vec<PathBuf>,
/// Report what would be imported without uploading anything.
#[arg(long)]
#[arg(long, conflicts_with = "resume")]
dry_run: bool,
/// Start the last import again with the same archives, after it was
/// interrupted, stopped or failed.
#[arg(long, conflicts_with = "archives")]
resume: bool,
/// Stay attached and print progress until the import finishes.
#[arg(long)]
wait: bool,
@@ -1030,8 +1035,9 @@ fn main() -> Result<()> {
Command::ImportGooglePhotos {
archives,
dry_run,
resume,
wait,
} => cmd_import_google_photos(archives, dry_run, wait),
} => cmd_import_google_photos(archives, dry_run, resume, wait),
Command::RedatePhotos {
dry_run,
limit,
@@ -2916,24 +2922,34 @@ fn photo_count(n: usize) -> String {
}
}
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
// The daemon opens the archives itself off the shared filesystem, so it gets
// absolute paths — the same arrangement `pdfs upload` uses.
let mut absolute = Vec::with_capacity(archives.len());
for archive in &archives {
let path = std::fs::canonicalize(archive)
.with_context(|| format!("resolve {}", archive.display()))?;
absolute.push(
path.to_str()
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
.to_owned(),
);
}
fn cmd_import_google_photos(
archives: Vec<PathBuf>,
dry_run: bool,
resume: bool,
wait: bool,
) -> Result<()> {
let request = if resume {
CtlRequest::ResumeImport
} else {
// The daemon opens the archives itself off the shared filesystem, so it
// gets absolute paths — the same arrangement `pdfs upload` uses.
let mut absolute = Vec::with_capacity(archives.len());
for archive in &archives {
let path = std::fs::canonicalize(archive)
.with_context(|| format!("resolve {}", archive.display()))?;
absolute.push(
path.to_str()
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
.to_owned(),
);
}
CtlRequest::ImportTakeout {
archives: absolute,
dry_run,
}
};
match control_request(CtlRequest::ImportTakeout {
archives: absolute,
dry_run,
})? {
match control_request(request)? {
CtlResponse::Ok { message } => println!("{message}"),
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
other => bail!("unexpected response: {other:?}"),
@@ -2945,20 +2961,34 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
// Poll rather than stream: the control protocol is request/response, and an
// import runs for hours — a held-open socket would be the only long-lived
// connection the daemon has.
// connection the daemon has. A line is printed when the step changes and
// every few percent of it, not on every poll.
let mut last_line = String::new();
let mut last_printed = std::time::Instant::now();
loop {
std::thread::sleep(std::time::Duration::from_secs(2));
match control_request(CtlRequest::ImportStatus)? {
CtlResponse::ImportStatus {
running: true,
summary: _,
} => continue,
CtlResponse::ImportStatus { summary, .. } => {
print_import_summary(summary.as_ref());
return finished_run(
"the import",
summary.map(|s| (s.failed, s.cancelled, s.error)),
);
progress,
..
} => {
let line = progress.as_ref().map(progress_line).unwrap_or_default();
let step_changed = line.split(':').next() != last_line.split(':').next();
if !line.is_empty()
&& line != last_line
&& (step_changed || last_printed.elapsed().as_secs() >= 30)
{
println!("{line}");
last_line = line;
last_printed = std::time::Instant::now();
}
}
CtlResponse::ImportStatus {
summary, resumable, ..
} => {
print_import_summary(summary.as_ref(), &resumable);
return import_outcome(summary.as_ref());
}
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
other => bail!("unexpected response: {other:?}"),
@@ -2966,6 +2996,47 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
}
}
/// One line saying where a running import is.
fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String {
use pdfs_core::control::ImportPhase;
let step = match progress.phase {
ImportPhase::Scanning => "Reading archives",
ImportPhase::Checking => "Checking for photos already on Proton",
ImportPhase::Uploading => "Uploading",
ImportPhase::Albums => "Filing photos into albums",
};
match (progress.total, &progress.current) {
(0, _) => format!("{step}…"),
(total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done),
(total, None) => format!("{step}: {} of {total}", progress.done),
}
}
/// How an import `--wait` followed ended, as the exit status: an error unless
/// it did everything it set out to do, so a script can tell.
fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> {
let Some(summary) = summary else {
bail!("the import ended without a result; the service may have restarted");
};
if let Some(error) = &summary.error {
bail!("the import failed: {error}");
}
if summary.interrupted {
bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue");
}
if summary.cancelled {
bail!("the import was cancelled");
}
if summary.failed > 0 || !summary.unreadable_archives.is_empty() {
bail!(
"the import finished, but {} photo(s) and {} archive(s) failed",
summary.failed,
summary.unreadable_archives.len()
);
}
Ok(())
}
fn cmd_redate_photos(
dry_run: bool,
limit: Option<usize>,
@@ -3100,11 +3171,19 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) {
fn cmd_import_status() -> Result<()> {
match control_request(CtlRequest::ImportStatus)? {
CtlResponse::ImportStatus { running, summary } => {
CtlResponse::ImportStatus {
running,
summary,
progress,
resumable,
} => {
if running {
println!("Import running. `pdfs transfers` shows its progress.");
match progress.as_ref() {
Some(progress) => println!("Import running. {}", progress_line(progress)),
None => println!("Import running."),
}
}
print_import_summary(summary.as_ref());
print_import_summary(summary.as_ref(), &resumable);
}
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
other => bail!("unexpected response: {other:?}"),
@@ -3121,18 +3200,28 @@ fn cmd_cancel_import() -> Result<()> {
Ok(())
}
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) {
let Some(summary) = summary else {
println!("No import has run yet.");
return;
};
if summary.dry_run {
println!("Scan only; nothing was uploaded.");
}
if let Some(error) = &summary.error {
println!("Failed : {error}");
return;
println!("Stopped by : {error}");
}
println!("Photos in export : {}", summary.found);
println!("Uploaded : {}", summary.uploaded);
if summary.dry_run {
println!("Would upload : {}", summary.to_upload);
println!("Bytes to upload : {}", summary.to_upload_bytes);
} else {
println!("Uploaded : {}", summary.uploaded);
}
println!("Already on Proton: {}", summary.duplicates);
if summary.live_photos > 0 {
println!("Live photos : {}", summary.live_photos);
}
if summary.skipped_trashed > 0 {
println!("Skipped (trash) : {}", summary.skipped_trashed);
}
@@ -3145,12 +3234,20 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
if summary.failed > 0 {
println!("Failed : {}", summary.failed);
}
for archive in &summary.unreadable_archives {
println!("Unreadable : {archive}");
}
if summary.bytes > 0 {
println!("Uploaded bytes : {}", summary.bytes);
}
if summary.cancelled {
if summary.interrupted {
println!("The service stopped before the import finished.");
} else if summary.cancelled {
println!("Import was cancelled before it finished.");
}
if !resumable.is_empty() {
println!("Run `pdfs import-google-photos --resume` to continue it.");
}
}
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
@@ -4509,6 +4606,51 @@ mod login_wait_tests {
assert!(finished_run("the import", None).is_err());
}
#[test]
fn a_waited_for_import_that_did_not_finish_exits_with_an_error() {
use pdfs_core::control::ImportSummary;
let done = ImportSummary {
uploaded: 4,
..Default::default()
};
assert!(import_outcome(Some(&done)).is_ok());
assert!(import_outcome(None).is_err());
for broken in [
ImportSummary {
interrupted: true,
..Default::default()
},
ImportSummary {
unreadable_archives: vec!["/exports/takeout-002.zip".into()],
..Default::default()
},
ImportSummary {
error: Some("check for existing photos: offline".into()),
..Default::default()
},
] {
assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}");
}
let interrupted = import_outcome(Some(&ImportSummary {
interrupted: true,
..Default::default()
}));
assert!(interrupted.unwrap_err().to_string().contains("--resume"));
}
#[test]
fn resume_needs_no_archives_and_excludes_them() {
let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap();
assert!(matches!(
cli.command,
Command::ImportGooglePhotos { resume: true, .. }
));
assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err());
assert!(
Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err()
);
}
#[test]
fn waiting_for_a_session_stops_hammering_the_keyring() {
let mut checks_left = 20;
+6
View File
@@ -562,6 +562,12 @@ impl AppDirs {
self.state_dir().join("control.sock")
}
/// The Google Photos import's last run: its archives and how it ended, so
/// an import cut short by a restart can be resumed.
pub fn takeout_import_path(&self) -> PathBuf {
self.state_dir().join("takeout-import.json")
}
/// Unix domain socket the tray uses to ensure single instance.
pub fn tray_socket(&self) -> PathBuf {
self.state_dir().join("tray.sock")
+81 -2
View File
@@ -279,6 +279,11 @@ pub enum Request {
/// and files what it has uploaded into its albums, then reports
/// `cancelled`. Replies with [`Response::Ok`].
CancelImport,
/// Start the last import again with the same archives, after it was
/// interrupted, cancelled or failed. Photos it already uploaded are found
/// on the account and skipped. Replies like [`Request::ImportTakeout`];
/// [`ErrorKind::NotFound`] when there is nothing to resume.
ResumeImport,
/// Upload the photo at `source_path` under the given name and media type.
///
/// A path, not the bytes: this protocol is line-delimited JSON, and
@@ -1689,10 +1694,75 @@ pub struct ImportSummary {
pub bytes: u64,
/// True when the run stopped early because it was cancelled.
pub cancelled: bool,
/// Why the run as a whole failed, when it did, such as an archive that
/// is not a zip. The counts are then empty.
/// Why the run as a whole stopped early, when it did, such as the
/// account being unreachable. The counts say how far it got.
#[serde(default)]
pub error: Option<String>,
/// What kind of failure [`error`](Self::error) is, so a front-end can word
/// it in the user's language. The text itself is the daemon's English.
#[serde(default)]
pub error_kind: Option<ErrorKind>,
/// True for a scan that uploads nothing ([`Request::ImportTakeout`] with
/// `dry_run`). [`to_upload`](Self::to_upload) then says what an import
/// would upload.
#[serde(default)]
pub dry_run: bool,
/// Photos not yet on the account: what a dry run would upload, and what a
/// real run set out to upload once the duplicates were known.
#[serde(default)]
pub to_upload: usize,
/// Bytes of [`to_upload`](Self::to_upload).
#[serde(default)]
pub to_upload_bytes: u64,
/// Live and motion photos whose video was filed with its still.
#[serde(default)]
pub live_photos: usize,
/// True when the service stopped while the import ran (a shutdown or a
/// restart). [`Request::ResumeImport`] picks it up again.
#[serde(default)]
pub interrupted: bool,
/// Archives that could not be read and were left out, by path.
#[serde(default)]
pub unreadable_archives: Vec<String>,
}
impl ImportSummary {
/// Whether the run did all it set out to do.
pub fn complete(&self) -> bool {
self.error.is_none()
&& !self.cancelled
&& !self.interrupted
&& self.failed == 0
&& self.unreadable_archives.is_empty()
}
}
/// Which step a running Takeout import is on.
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ImportPhase {
/// Reading the archives' directories and metadata.
#[default]
Scanning,
/// Asking the account which photos it already has.
Checking,
/// Uploading the photos that are new.
Uploading,
/// Creating albums and filing photos into them.
Albums,
}
/// Where a running import is ([`Response::ImportStatus`]).
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
pub struct ImportProgress {
pub phase: ImportPhase,
/// Steps done in this phase.
pub done: u64,
/// Steps in this phase; `0` while not yet known.
pub total: u64,
/// The photo being uploaded, during [`ImportPhase::Uploading`].
#[serde(default)]
pub current: Option<String>,
}
/// One album in a [`Response::Albums`] listing.
@@ -2208,8 +2278,17 @@ pub enum Response {
/// report; `summary` is `None` when no import has run this session.
ImportStatus {
running: bool,
/// The counts so far while running, the final report after.
#[serde(default)]
summary: Option<ImportSummary>,
/// The current step, while running.
#[serde(default)]
progress: Option<ImportProgress>,
/// The archives of the last run, when it can be resumed with
/// [`Request::ResumeImport`]: it was a real import that did not
/// finish. Survives a restart of the service.
#[serde(default)]
resumable: Vec<String>,
},
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
/// `running` is false once it has finished, when `summary` is the final
+488 -70
View File
@@ -25,7 +25,9 @@
//! The service folder (`Google Photos`) is localized, as are the year folders
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
//! service name: the first component is dropped when it is the export root, the
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
//! second is the service, and the rest is `<folder>/<file>`. The Photos service
//! is the one whose sidecars carry a `photoTakenTime`; other services in the same
//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an
//! **album** when it carries an album metadata sidecar naming it; anything else
//! is a plain bucket that imports into the timeline only. Trash is matched by
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
@@ -121,8 +123,14 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
("m2ts", "video/mp2t"),
("mpg", "video/mpeg"),
("mpeg", "video/mpeg"),
// The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`.
("mp", "video/mp4"),
];
/// Sidecars bigger than this are not read. A real one is a few kilobytes; a
/// `.json` this large is some other service's data and not worth the memory.
const SIDECAR_MAX_BYTES: u64 = 1024 * 1024;
/// The media type for `name`'s extension, or `None` when it is not media this
/// import handles.
pub fn media_type_of(name: &str) -> Option<&'static str> {
@@ -147,6 +155,13 @@ pub enum Bucket {
Archived,
}
/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct GeoPoint {
pub latitude: f64,
pub longitude: f64,
}
/// One media file found in the export.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TakeoutPhoto {
@@ -162,14 +177,49 @@ pub struct TakeoutPhoto {
pub size: u64,
/// Media type from the extension.
pub media_type: String,
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
/// `creationTime`). `None` when the photo has no sidecar — the upload then
/// defaults it to now, as the SDK does.
/// Capture time in epoch seconds, from the sidecar's `photoTakenTime`,
/// else from a date in the file name. `None` when neither has one; the
/// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time).
pub capture_time: Option<i64>,
/// The sidecar's `creationTime`: when the photo reached Google Photos, not
/// when it was taken. Only a last resort for the capture time.
#[serde(default)]
pub creation_time: Option<i64>,
/// Where the photo was taken, from the sidecar's `geoData` (or
/// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped.
#[serde(default)]
pub location: Option<GeoPoint>,
/// The sidecar's `favorited` flag.
pub favorite: bool,
/// Which bucket the file was found in.
pub bucket: Bucket,
/// Pairs the halves of a live or motion photo: a still and a video in the
/// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`,
/// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half.
#[serde(default)]
pub live_key: Option<String>,
}
impl TakeoutPhoto {
/// Whether this is the video half of a pair, as opposed to the still.
pub fn is_video(&self) -> bool {
self.media_type.starts_with("video/")
}
/// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather
/// than an Apple live photo.
pub fn is_motion_photo(&self) -> bool {
self.live_key
.as_deref()
.is_some_and(|key| key.ends_with(".mp"))
}
}
/// An archive the scan had to leave out, and why.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnreadableArchive {
pub path: PathBuf,
pub reason: String,
}
/// Everything one scan found, ready to be turned into an import plan.
@@ -183,8 +233,16 @@ pub struct TakeoutScan {
/// Media files skipped because they sit in the trash bucket.
pub skipped_trashed: usize,
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
/// activity `.csv`s, `.mp` motion-photo parts).
/// activity `.csv`s), and media from other Google services in the same
/// export (`Google Drive/`, `YouTube/`).
pub skipped_other: usize,
/// Archives that could not be opened or are not a zip. The rest of the
/// export is still imported.
#[serde(default)]
pub unreadable_archives: Vec<UnreadableArchive>,
/// Entries inside an otherwise readable archive that could not be read.
#[serde(default)]
pub unreadable_entries: usize,
}
impl TakeoutScan {
@@ -195,7 +253,7 @@ impl TakeoutScan {
}
/// Google's per-photo sidecar. Only the fields the import uses are read; the
/// file also carries geo data, people tags and view counts.
/// file also carries people tags, a description and view counts.
#[derive(Debug, Default, Deserialize)]
struct PhotoSidecar {
#[serde(default)]
@@ -205,19 +263,69 @@ struct PhotoSidecar {
#[serde(rename = "creationTime", default)]
creation_time: Option<SidecarTime>,
#[serde(default)]
favorited: bool,
favorited: Option<bool>,
/// The position as edited in Google Photos, else as the camera wrote it.
#[serde(rename = "geoData", default)]
geo_data: Option<SidecarGeo>,
#[serde(rename = "geoDataExif", default)]
geo_data_exif: Option<SidecarGeo>,
}
/// A sidecar timestamp: epoch seconds, as a *string*.
impl PhotoSidecar {
fn location(&self) -> Option<GeoPoint> {
self.geo_data
.as_ref()
.and_then(SidecarGeo::point)
.or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point))
}
}
/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number.
#[derive(Debug, Deserialize)]
struct SidecarTime {
#[serde(default)]
timestamp: Option<String>,
timestamp: Option<SidecarNumber>,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum SidecarNumber {
Number(i64),
Text(String),
}
impl SidecarTime {
/// The timestamp, when it is a real moment. `0` and negative values are
/// Google's "unknown" and would file the photo under 1970.
fn epoch_seconds(&self) -> Option<i64> {
self.timestamp.as_ref()?.parse().ok()
let seconds = match self.timestamp.as_ref()? {
SidecarNumber::Number(seconds) => *seconds,
SidecarNumber::Text(text) => text.trim().parse().ok()?,
};
(seconds > 0).then_some(seconds)
}
}
/// `geoData` in a sidecar. All-zero means Google has no position.
#[derive(Debug, Deserialize)]
struct SidecarGeo {
#[serde(default)]
latitude: Option<f64>,
#[serde(default)]
longitude: Option<f64>,
}
impl SidecarGeo {
fn point(&self) -> Option<GeoPoint> {
let (latitude, longitude) = (self.latitude?, self.longitude?);
let on_globe = latitude.is_finite()
&& longitude.is_finite()
&& (-90.0..=90.0).contains(&latitude)
&& (-180.0..=180.0).contains(&longitude);
(on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint {
latitude,
longitude,
})
}
}
@@ -262,21 +370,58 @@ impl RawEntry {
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
/// Reads only the central directory and the JSON sidecars — no media bytes, so
/// this stays fast on a 200 GB export.
///
/// An archive that cannot be opened is left out and listed in
/// [`TakeoutScan::unreadable_archives`]; the rest still imports.
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
let mut entries: Vec<RawEntry> = Vec::new();
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
scan_archives_until(archives, || false)
}
/// [`scan_archives`] that gives up when `stop` returns true, checked between
/// archives and every few hundred entries.
pub fn scan_archives_until(
archives: &[PathBuf],
stop: impl Fn() -> bool,
) -> CoreResult<TakeoutScan> {
let mut entries: Vec<(String, RawEntry)> = Vec::new();
let mut sidecars: Vec<(String, String, Vec<u8>)> = Vec::new();
let mut unreadable_archives = Vec::new();
let mut unreadable_entries = 0;
for (index, path) in archives.iter().enumerate() {
let file = std::fs::File::open(path)
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
})?;
if stop() {
return Err(CoreError::conflict("the import was cancelled"));
}
let opened = std::fs::File::open(path)
.map_err(|e| e.to_string())
.and_then(|file| {
zip::ZipArchive::new(std::io::BufReader::new(file))
.map_err(|e| format!("not a readable zip: {e}"))
});
let mut zip = match opened {
Ok(zip) => zip,
Err(reason) => {
tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive");
unreadable_archives.push(UnreadableArchive {
path: path.clone(),
reason,
});
continue;
}
};
for position in 0..zip.len() {
let mut entry = zip.by_index(position).map_err(|e| {
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
})?;
if position % 512 == 0 && stop() {
return Err(CoreError::conflict("the import was cancelled"));
}
let mut entry = match zip.by_index(position) {
Ok(entry) => entry,
Err(e) => {
tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry");
unreadable_entries += 1;
continue;
}
};
if entry.is_dir() {
continue;
}
@@ -286,32 +431,92 @@ pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
continue;
};
let name = name.to_string_lossy().replace('\\', "/");
let Some(relative) = strip_export_prefix(&name) else {
let Some((service, relative)) = strip_export_prefix(&name) else {
continue;
};
let (service, relative) = (service.to_string(), relative.to_string());
let size = entry.size();
// Sidecars are read now, while the entry is open: they are a few
// hundred bytes each and reopening the archive per photo later would
// be a second full pass.
if relative.to_ascii_lowercase().ends_with(".json") {
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
if entry.read_to_end(&mut body).is_ok() {
sidecars.insert(relative.to_string(), body);
if size <= SIDECAR_MAX_BYTES {
let mut body = Vec::with_capacity(size as usize);
if entry.read_to_end(&mut body).is_ok() {
sidecars.push((service, relative, body));
} else {
unreadable_entries += 1;
}
}
continue;
}
entries.push(RawEntry {
archive: index,
entry: name.clone(),
relative: relative.to_string(),
size,
});
entries.push((
service,
RawEntry {
archive: index,
entry: name,
relative,
size,
},
));
}
}
Ok(assemble(entries, &sidecars))
let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars);
let mut scan = assemble(entries, &sidecars);
scan.skipped_other += other_services;
scan.unreadable_archives = unreadable_archives;
scan.unreadable_entries = unreadable_entries;
Ok(scan)
}
/// Drop what other Google services put into the same export.
///
/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to
/// `Google Photos/`, and their media must not land in the photo timeline. The
/// service folder's name is localized (`Google Fotos`), so the Photos service
/// is recognized by its content instead: it is the folder whose sidecars carry
/// a `photoTakenTime`. When no folder does — an export without sidecars, or
/// one re-zipped without the service level — everything is kept, as before.
///
/// Returns the entries and sidecars to import, and how many media files were
/// left out.
fn keep_photos_services(
entries: Vec<(String, RawEntry)>,
sidecars: Vec<(String, String, Vec<u8>)>,
) -> (Vec<RawEntry>, HashMap<String, Vec<u8>>, usize) {
#[derive(Deserialize)]
struct Marker {
#[serde(rename = "photoTakenTime")]
photo_taken_time: Option<serde::de::IgnoredAny>,
}
let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new();
for (service, _, body) in &sidecars {
if photos_services.contains(service.as_str()) {
continue;
}
if serde_json::from_slice::<Marker>(body).is_ok_and(|m| m.photo_taken_time.is_some()) {
photos_services.insert(service.as_str());
}
}
let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service);
let mut skipped = 0;
let mut kept_entries = Vec::with_capacity(entries.len());
for (service, raw) in entries {
if keep(&service) {
kept_entries.push(raw);
} else if media_type_of(raw.file_name()).is_some() {
skipped += 1;
}
}
let kept_sidecars = sidecars
.iter()
.filter(|(service, _, _)| keep(service))
.map(|(_, relative, body)| (relative.clone(), body.clone()))
.collect();
(kept_entries, kept_sidecars, skipped)
}
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
@@ -403,19 +608,19 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
.unwrap_or(&file_name)
.to_string();
// Sidecar first, then the date stamped into the name.
// Sidecar first, then the date stamped into the name. `creationTime`
// is when the photo reached Google, which for a scan or an old photo
// uploaded late is years off; the importer uses it only after EXIF.
let capture_time = sidecar
.photo_taken_time
.as_ref()
.and_then(SidecarTime::epoch_seconds)
.or_else(|| {
sidecar
.creation_time
.as_ref()
.and_then(SidecarTime::epoch_seconds)
})
.or_else(|| capture_time_from_name(&name))
.or_else(|| capture_time_from_name(&file_name));
let creation_time = sidecar
.creation_time
.as_ref()
.and_then(SidecarTime::epoch_seconds);
scan.photos.push(TakeoutPhoto {
archive: raw.archive,
@@ -424,31 +629,57 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
size: raw.size,
media_type: media_type.to_string(),
capture_time,
favorite: sidecar.favorited,
creation_time,
location: sidecar.location(),
favorite: sidecar.favorited.unwrap_or(false),
bucket,
live_key: live_key(&folder, &file_name),
});
}
scan
}
/// Drop the export root (`Takeout/`) and the localized service folder from a
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
/// Split a path inside the archive into the localized service folder and the
/// rest, dropping the export root (`Takeout/`): `("Google Fotos",
/// "<folder>/<file>")`.
///
/// `None` for a path with no service folder at all (the export's own
/// `archive_browser.html`), which carries nothing to import.
fn strip_export_prefix(path: &str) -> Option<&str> {
fn strip_export_prefix(path: &str) -> Option<(&str, &str)> {
let mut rest = path;
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
// the user may not have it — so the root is dropped only when a service
// folder follows it, which the component count decides.
// The root is `Takeout/`, but an export unzipped and re-zipped by the user
// may not have it. `Google Photos/<album>/<file>` then has three parts like
// `Takeout/Google Photos/<file>`, so the count alone cannot tell them
// apart: a root of another name is only taken for one with four parts.
let (first, after) = rest.split_once('/')?;
let components = rest.split('/').count();
if components >= 3 {
rest = rest.split_once('/')?.1;
if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 {
rest = after;
}
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
let (_service, tail) = rest.split_once('/')?;
Some(tail)
// The service folder (`Google Photos`, `Google Fotos`, …).
rest.split_once('/')
}
/// The key that pairs the still and the video of one live or motion photo, or
/// `None` when `file_name` cannot be either half.
///
/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem.
/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the
/// video's whole name. The folder is part of the key so two cameras' `IMG_1`
/// in different albums never pair.
fn live_key(folder: &str, file_name: &str) -> Option<String> {
let lower = file_name.to_lowercase();
let (stem, ext) = lower.rsplit_once('.')?;
let key = match ext {
"heic" | "heif" | "jpg" | "jpeg" => stem,
// Apple's video half. A plain `.mp4` beside a photo of the same name is
// far more often an unrelated clip than half of a live photo.
"mov" => stem,
"mp" => lower.as_str(),
_ => return None,
};
(!key.is_empty()).then(|| format!("{folder}/{key}"))
}
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
@@ -858,13 +1089,13 @@ mod tests {
RawEntry {
archive,
entry: path.to_string(),
relative: strip_export_prefix(path).unwrap().to_string(),
relative: strip_export_prefix(path).unwrap().1.to_string(),
size,
}
}
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
let relative = strip_export_prefix(path).unwrap().to_string();
let relative = strip_export_prefix(path).unwrap().1.to_string();
map.insert(relative, body.as_bytes().to_vec());
}
@@ -872,12 +1103,17 @@ mod tests {
fn strips_export_root_and_localized_service_folder() {
assert_eq!(
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
Some("Island 2019/IMG_1.jpg")
Some(("Google Fotos", "Island 2019/IMG_1.jpg"))
);
// Already unwrapped by the user: no `Takeout/` root left.
assert_eq!(
strip_export_prefix("Google Photos/IMG_1.jpg"),
Some("IMG_1.jpg")
Some(("Google Photos", "IMG_1.jpg"))
);
// ... and an album keeps its folder.
assert_eq!(
strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"),
Some(("Google Photos", "Iceland/IMG_1.jpg"))
);
// Nothing below a service folder is nothing to import.
assert_eq!(strip_export_prefix("archive_browser.html"), None);
@@ -957,23 +1193,144 @@ mod tests {
}
#[test]
fn creation_time_is_the_fallback_capture_time() {
fn creation_time_is_kept_apart_as_the_last_resort() {
// `creationTime` is when the photo reached Google. It must not beat the
// date in the name, and the importer tries EXIF before it.
let mut sidecars = HashMap::new();
sidecar(
&mut sidecars,
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
);
sidecar(
&mut sidecars,
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json",
r#"{"creationTime":{"timestamp":"1700000000"}}"#,
);
let scan = assemble(
vec![entry(
0,
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
10,
)],
vec![
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
entry(
0,
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg",
10,
),
],
&sidecars,
);
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
assert_eq!(scan.photos[0].capture_time, None);
assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000));
assert_eq!(
scan.photos[1].capture_time,
capture_time_from_name("IMG_20190612_101500.jpg")
);
assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000));
}
#[test]
fn sidecar_timestamps_tolerate_numbers_zero_and_null() {
let mut sidecars = HashMap::new();
for (name, body) in [
("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#),
(
"B.jpg",
r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#,
),
(
"C.jpg",
r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#,
),
] {
sidecar(
&mut sidecars,
&format!("Takeout/Google Photos/Photos from 2019/{name}.json"),
body,
);
}
let scan = assemble(
["A.jpg", "B.jpg", "C.jpg"]
.iter()
.map(|name| {
entry(
0,
&format!("Takeout/Google Photos/Photos from 2019/{name}"),
1,
)
})
.collect(),
&sidecars,
);
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970");
assert!(!scan.photos[1].favorite);
assert_eq!(scan.photos[2].capture_time, None);
}
#[test]
fn location_comes_from_geo_data_and_zero_means_none() {
let mut sidecars = HashMap::new();
sidecar(
&mut sidecars,
"Takeout/Google Photos/Iceland/IMG_1.jpg.json",
r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#,
);
sidecar(
&mut sidecars,
"Takeout/Google Photos/Iceland/IMG_2.jpg.json",
r#"{"geoData":{"latitude":0.0,"longitude":0.0},
"geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#,
);
sidecar(
&mut sidecars,
"Takeout/Google Photos/Iceland/IMG_3.jpg.json",
r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#,
);
let scan = assemble(
(1..=3)
.map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1))
.collect(),
&sidecars,
);
assert_eq!(
scan.photos[0].location,
Some(GeoPoint {
latitude: 64.1466,
longitude: -21.9426
})
);
assert_eq!(
scan.photos[1].location.map(|p| p.latitude),
Some(63.4186),
"geoDataExif fills in for an empty geoData"
);
assert_eq!(scan.photos[2].location, None);
}
#[test]
fn live_and_motion_photo_halves_share_a_key() {
let scan = assemble(
vec![
entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1),
entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1),
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1),
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1),
entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1),
entry(0, "Takeout/Google Photos/Trip/scan.png", 1),
],
&HashMap::new(),
);
let key = |i: usize| scan.photos[i].live_key.clone();
assert_eq!(key(0), key(1));
assert!(!scan.photos[0].is_motion_photo());
assert_eq!(key(2), key(3));
assert!(scan.photos[3].is_motion_photo());
assert!(scan.photos[3].is_video(), "the .MP half is a video");
assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair");
assert_eq!(key(5), None);
}
#[test]
@@ -1387,16 +1744,77 @@ mod tests {
}
#[test]
fn a_file_that_is_not_a_zip_is_a_readable_error() {
let path =
fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() {
let broken =
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
std::fs::write(&path, b"not a zip at all").unwrap();
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
let _ = std::fs::remove_file(&path);
std::fs::write(&broken, b"not a zip at all").unwrap();
let good = write_zip(
"beside-broken",
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
);
let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap();
let _ = std::fs::remove_file(&broken);
let _ = std::fs::remove_file(&good);
assert_eq!(scan.photos.len(), 1);
assert_eq!(scan.photos[0].archive, 1);
assert_eq!(scan.unreadable_archives.len(), 1);
assert_eq!(scan.unreadable_archives[0].path, broken);
assert!(
error.message.contains("not a readable zip"),
"unexpected message: {}",
error.message
scan.unreadable_archives[0]
.reason
.contains("not a readable zip"),
"unexpected reason: {}",
scan.unreadable_archives[0].reason
);
}
#[test]
fn other_services_in_the_same_export_are_not_imported() {
let zip = write_zip(
"services",
&[
(
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg",
b"x".as_slice(),
),
(
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json",
br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(),
),
("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()),
("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()),
],
);
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
let _ = std::fs::remove_file(&zip);
assert_eq!(scan.photos.len(), 1);
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
assert_eq!(scan.skipped_other, 2);
}
#[test]
fn an_export_without_sidecars_keeps_every_service() {
let zip = write_zip(
"no-sidecars",
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
);
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
let _ = std::fs::remove_file(&zip);
assert_eq!(scan.photos.len(), 1);
}
#[test]
fn a_stop_request_ends_the_scan() {
let zip = write_zip(
"stop",
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
);
let result = scan_archives_until(std::slice::from_ref(&zip), || true);
let _ = std::fs::remove_file(&zip);
assert!(result.is_err());
}
}
+1
View File
@@ -20,6 +20,7 @@ tokio.workspace = true
tracing.workspace = true
fuser.workspace = true
libc.workspace = true
serde.workspace = true
serde_json.workspace = true
base64.workspace = true
image.workspace = true
+38 -25
View File
@@ -877,35 +877,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
if archives.is_empty() {
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
} else {
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
let missing: Vec<String> = paths
.iter()
.filter(|path| !path.is_file())
.map(|path| path.display().to_string())
.collect();
if !missing.is_empty() {
CtlResponse::error(CoreError::not_found(format!(
"cannot read: {}",
missing.join(", ")
)))
} else {
let count = paths.len();
match core.spawn_takeout_import(paths, dry_run) {
Ok(()) => CtlResponse::Ok {
message: format!(
"{} {count} archive(s)",
if dry_run { "scanning" } else { "importing" }
),
},
Err(e) => CtlResponse::error(e),
}
}
let paths = archives.into_iter().map(PathBuf::from).collect();
start_takeout_import(core, paths, dry_run)
}
}
Ok(CtlRequest::ImportStatus) => {
let (running, summary) = crate::takeout::import_status();
CtlResponse::ImportStatus { running, summary }
let report = crate::takeout::import_status();
CtlResponse::ImportStatus {
running: report.running,
summary: report.summary,
progress: report.progress,
resumable: report.resumable,
}
}
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
Some(paths) => start_takeout_import(core, paths, false),
},
Ok(CtlRequest::CancelImport) => {
if crate::takeout::cancel_import() {
CtlResponse::Ok {
@@ -2009,6 +1997,31 @@ pub(crate) fn run_control_socket(
}
}
/// Check that every archive is there, then start the import in the background.
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
let missing: Vec<String> = paths
.iter()
.filter(|path| !path.is_file())
.map(|path| path.display().to_string())
.collect();
if !missing.is_empty() {
return CtlResponse::error(CoreError::not_found(format!(
"cannot read: {}",
missing.join(", ")
)));
}
let count = paths.len();
match core.spawn_takeout_import(paths, dry_run) {
Ok(()) => CtlResponse::Ok {
message: format!(
"{} {count} archive(s)",
if dry_run { "scanning" } else { "importing" }
),
},
Err(e) => CtlResponse::error(e),
}
}
#[cfg(test)]
mod request_limit_tests {
use super::*;
+31 -1
View File
@@ -1193,7 +1193,7 @@ impl Core {
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
return Ok(None);
}
if !self.rt.block_on(self.refresh_timeline())? {
if !self.refresh_timeline_joined()? {
return Ok(None);
}
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
@@ -2640,6 +2640,36 @@ impl Core {
/// Refresh the timeline off the request path, so a stale page is still served
/// at DB speed. At most one refresh runs at a time.
/// Refresh the timeline and wait for it, sharing a refresh already under
/// way instead of starting a second one beside it.
///
/// The first request after an empty cache has nothing to serve until the
/// fetch lands, so it waits. Two such requests (the gallery and its retry,
/// or a background refresh started by an import) used to each fetch the
/// whole library at once, sharing the session's rate limit and taking
/// minutes. Returns whether the account has a photos library.
pub(crate) fn refresh_timeline_joined(&self) -> CoreResult<bool> {
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() {
std::thread::sleep(std::time::Duration::from_millis(200));
}
// A refresh that landed stamped the timeline fresh. One that failed
// did not, and the caller gets this refresh's own error instead of
// an empty timeline.
if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL)
|| self.timeline_refreshing.swap(true, Ordering::SeqCst)
{
return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0));
}
}
let refreshed = self.rt.block_on(self.refresh_timeline());
self.timeline_refreshing.store(false, Ordering::SeqCst);
if refreshed.is_ok() {
self.events.publish(&[Topic::Photos]);
}
refreshed
}
pub(crate) fn spawn_timeline_refresh(&self) {
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
return;
File diff suppressed because it is too large. Load diff
+8 -6
View File
@@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState};
use pdfs_core::control::{
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress,
PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope,
Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry,
ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem,
LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo,
PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo,
SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase,
SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
};
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
@@ -668,9 +668,11 @@ fn build_window(app: &adw::Application) {
progress_bar: takeout_widgets.progress_bar.clone(),
summary_group: takeout_widgets.summary_group.clone(),
summary_rows: RefCell::new(Vec::new()),
resume_group: takeout_widgets.resume_group.clone(),
resume_row: takeout_widgets.resume_row.clone(),
resume_button: takeout_widgets.resume_button.clone(),
inflight: Cell::new(false),
running: Cell::new(false),
dry_run: Cell::new(false),
},
update,
});
+8 -6
View File
@@ -1115,9 +1115,6 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
// count falling to zero is what "the sync finished" looks like from here.
// Jobs are deliberately not counted — a bulk upload retires its scan job and
// starts its upload job mid-flight, which is not a thing finishing.
// The Import page reads its progress off the same snapshot rather than
// polling the daemon a second time for numbers already on the wire.
takeout_progress(ui, jobs);
let previous = ui.status.active_transfers.replace(items.len());
if !items.is_empty() {
@@ -1211,13 +1208,18 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
let done = j.done.to_string();
let total = j.total.to_string();
let (title, detail) = if j.title == IMPORT_JOB {
import_job_text(&j.detail)
} else {
(j.title.clone(), j.detail.clone())
};
let args: [(&str, &str); 4] = [
("title", &j.title),
("detail", &j.detail),
("title", &title),
("detail", &detail),
("done", &done),
("total", &total),
];
let text = match (j.total > 0, j.detail.is_empty()) {
let text = match (j.total > 0, detail.is_empty()) {
// Translators: a background job's progress. {title} is the job, such as
// "Scanning Photos"; {done} and {total} are counts.
(true, true) => gettext_f("{title} — {done} of {total}", &args),
+326 -158
View File
@@ -15,17 +15,30 @@
//! Import — staging is local, so a mis-drop costs nothing.
//!
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
//! stages paths, launches, asks [`Request::ImportStatus`] for the final report,
//! and reads live progress off the import's job in the transfer snapshot the
//! daemon already pushes (see [`takeout_progress`]).
//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the
//! running counts and the final report. It asks again whenever the import's job
//! moves in the transfer snapshot the daemon already pushes. A run the service
//! did not get to finish can be picked up with [`Request::ResumeImport`].
use crate::*;
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
/// Matching on it is how this page borrows the progress the daemon already
/// pushes instead of asking for it separately.
/// Seeing it move is the cue to ask for the import's status; the Activity list
/// shows it under its translated name ([`import_job_text`]).
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
/// The import job's title and detail in the user's language. The daemon names
/// its steps in English; the file being uploaded passes through as it is.
pub(crate) fn import_job_text(detail: &str) -> (String, String) {
let detail = match detail {
"reading archives" => gettext("Reading the archives"),
"checking for photos you already have" => gettext("Checking for photos you already have"),
"sorting photos into albums" => gettext("Sorting photos into albums"),
other => other.to_string(),
};
(gettext("Importing from Google Photos"), detail)
}
/// What the archive list says before anything is staged. `repaint_archives`
/// swaps in the count and total once there is something to report, and swaps
/// this back when the list is emptied again.
@@ -95,18 +108,19 @@ pub(crate) struct TakeoutState {
pub(crate) progress_group: adw::PreferencesGroup,
pub(crate) progress_label: gtk4::Label,
pub(crate) progress_bar: gtk4::ProgressBar,
/// Report group for the last finished run, hidden until there is one.
/// The running counts, or the report for the last finished run; hidden
/// until there is one.
pub(crate) summary_group: adw::PreferencesGroup,
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
/// Offers to resume an import the service did not get to finish.
pub(crate) resume_group: adw::PreferencesGroup,
pub(crate) resume_row: adw::ActionRow,
pub(crate) resume_button: gtk4::Button,
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
/// can't stack worker threads on the 2s tick.
pub(crate) inflight: Cell<bool>,
/// Whether the daemon reported an import running on the last poll.
pub(crate) running: Cell<bool>,
/// Whether the run in flight was launched as a dry run, so the report can
/// say "would upload" rather than claiming photos were sent. The daemon's
/// summary carries no such flag — it is the same shape either way.
pub(crate) dry_run: Cell<bool>,
}
/// Widgets the Import page hands back for wiring.
@@ -126,6 +140,9 @@ pub(crate) struct TakeoutWidgets {
pub(crate) progress_label: gtk4::Label,
pub(crate) progress_bar: gtk4::ProgressBar,
pub(crate) summary_group: adw::PreferencesGroup,
pub(crate) resume_group: adw::PreferencesGroup,
pub(crate) resume_row: adw::ActionRow,
pub(crate) resume_button: gtk4::Button,
}
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
@@ -261,7 +278,23 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
.visible(false)
.build();
// An import cut short by a restart or a lost connection, offered back at the
// top of the page: picking the archives again is exactly what a reboot in
// the middle of a migration should not cost.
let resume_button = gtk4::Button::builder()
.label(pgettext("verb", "Resume"))
.valign(gtk4::Align::Center)
.build();
resume_button.add_css_class("suggested-action");
let resume_row = adw::ActionRow::builder()
.title(gettext("The last import didn't finish"))
.build();
resume_row.add_suffix(&resume_button);
let resume_group = adw::PreferencesGroup::builder().visible(false).build();
resume_group.add(&resume_row);
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
groups.append(&resume_group);
groups.append(&dropzone);
groups.append(&list_group);
groups.append(&actions);
@@ -301,6 +334,9 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
progress_label: progress_label.clone(),
progress_bar: progress_bar.clone(),
summary_group: summary_group.clone(),
resume_group,
resume_row,
resume_button,
};
(frame.upcast(), widgets)
}
@@ -427,6 +463,28 @@ pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
});
});
let ui_resume = ui.clone();
widgets.resume_button.connect_clicked(move |_| {
let ui = ui_resume.clone();
ui.busy_begin();
let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport);
glib::spawn_future_local(async move {
let result = rx.recv().await;
ui.busy_end();
match result {
Ok(Ok(Response::Ok { .. })) => import_started(&ui, false),
Ok(Ok(Response::Error { message, kind })) => {
toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind);
}
_ => toast_error(
&ui,
&gettext("Couldn't resume the import"),
&gettext("The Proton Drive service didn't respond."),
),
}
});
});
let ui_back = ui.clone();
widgets
.back_button
@@ -559,6 +617,7 @@ fn sync_takeout_actions(ui: &Rc<Ui>) {
.import_button
.set_sensitive(has_files && !running);
ui.takeout.clear_button.set_sensitive(has_files && !running);
ui.takeout.resume_button.set_sensitive(!running);
ui.takeout.cancel_button.set_visible(running);
}
@@ -577,7 +636,6 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
return;
}
ui.takeout.dry_run.set(dry_run);
ui.busy_begin();
let rx = spawn_request(
ui.dirs.control_socket(),
@@ -588,27 +646,7 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
let result = rx.recv().await;
ui.busy_end();
match result {
Ok(Ok(Response::Ok { .. })) => {
ui.takeout.running.set(true);
ui.gallery.import_banner.set_revealed(true);
ui.takeout.summary_group.set_visible(false);
ui.takeout.progress_group.set_visible(true);
ui.takeout.progress_bar.set_fraction(0.0);
ui.takeout.progress_label.set_text(&if dry_run {
gettext("Reading archives…")
} else {
gettext("Starting import…")
});
sync_takeout_actions(&ui);
toast(
&ui,
&if dry_run {
gettext("Scanning the export…")
} else {
gettext("Import started — you can leave this page, it keeps running.")
},
);
}
Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run),
Ok(Ok(Response::Error { message, kind })) => {
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
}
@@ -621,9 +659,39 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
});
}
/// Ask the daemon for the import's state, and paint the report once a run has
/// finished. Asked on arrival and while import progress keeps arriving. Cheap
/// and inflight-guarded.
/// Switch the page over to reporting progress once the daemon took the run.
fn import_started(ui: &Rc<Ui>, dry_run: bool) {
ui.takeout.running.set(true);
if !dry_run {
ui.gallery.import_banner.set_revealed(true);
}
ui.takeout.resume_group.set_visible(false);
ui.takeout.summary_group.set_visible(false);
ui.takeout.progress_group.set_visible(true);
ui.takeout.progress_group.set_title(&if dry_run {
gettext("Scan in progress")
} else {
gettext("Import in progress")
});
ui.takeout.progress_bar.set_fraction(0.0);
ui.takeout
.progress_label
.set_text(&gettext("Reading the archives…"));
sync_takeout_actions(ui);
toast(
ui,
&if dry_run {
gettext("Scanning the export…")
} else {
gettext("Import started — you can leave this page, it keeps running.")
},
);
}
/// Ask the daemon for the import's state and paint it: the step and counts
/// while it runs, the report once it has ended, and the offer to resume a run
/// that did not finish. Asked on arrival and while import progress keeps
/// arriving. Cheap and inflight-guarded.
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
if ui.takeout.inflight.get() {
return;
@@ -634,124 +702,250 @@ pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
glib::spawn_future_local(async move {
let result = rx.recv().await;
ui.takeout.inflight.set(false);
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
let Ok(Ok(Response::ImportStatus {
running,
summary,
progress,
resumable,
})) = result
else {
return;
};
let was_running = ui.takeout.running.replace(running);
ui.gallery.import_banner.set_revealed(running);
let dry = summary.as_ref().is_some_and(|s| s.dry_run);
ui.gallery.import_banner.set_revealed(running && !dry);
ui.takeout.progress_group.set_visible(running);
if running {
ui.takeout.progress_group.set_visible(true);
} else {
ui.takeout.progress_group.set_visible(false);
// An import runs for hours, so its ending is announced rather than
// left to be discovered: a toast for whoever is looking at the
// window, and a desktop notification for whoever is not.
let failed = summary.as_ref().and_then(|s| s.error.clone());
if was_running && let Some(error) = &failed {
let headline = gettext("Couldn't import from Google Photos");
toast_error(&ui, &headline, error);
notify("takeout-import", &headline, error);
} else if was_running {
let dry = ui.takeout.dry_run.get();
let headline = if dry {
gettext("Scan finished")
} else {
gettext("Import finished")
};
toast(&ui, &headline);
if !dry {
let body = match &summary {
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
Some(s) if s.failed > 0 => ngettext_f(
"{n} photo added, {failed} failed.",
"{n} photos added, {failed} failed.",
s.uploaded as u64,
&[("failed", &s.failed.to_string())],
),
Some(s) if s.cancelled => ngettext_f(
"Stopped after {n} photo.",
"Stopped after {n} photos.",
s.uploaded as u64,
&[],
),
Some(s) => ngettext_f(
"{n} photo added to Proton Photos.",
"{n} photos added to Proton Photos.",
s.uploaded as u64,
&[],
),
None => gettext("The Google Photos import has finished."),
};
notify("takeout-import", &headline, &body);
}
}
ui.takeout.progress_group.set_title(&if dry {
gettext("Scan in progress")
} else {
gettext("Import in progress")
});
paint_progress(&ui, progress.as_ref());
} else if was_running && let Some(summary) = &summary {
announce_end(&ui, summary);
}
if was_running != running {
sync_takeout_actions(&ui);
}
match summary {
Some(summary) => repaint_summary(&ui, &summary),
match &summary {
Some(summary) => repaint_summary(&ui, summary, running),
None => ui.takeout.summary_group.set_visible(false),
}
paint_resume(&ui, running, summary.as_ref(), &resumable);
});
}
/// Paint the last run's report.
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
/// Tell the user how a run ended. An import runs for hours, so its ending is
/// announced rather than left to be discovered: a toast for whoever is looking
/// at the window, and a desktop notification for whoever is not.
fn announce_end(ui: &Rc<Ui>, summary: &ImportSummary) {
if summary.dry_run {
toast(ui, &gettext("Scan finished"));
return;
}
if let Some(error) = &summary.error {
let kind = summary.error_kind.unwrap_or_default();
let headline = gettext("Couldn't import from Google Photos");
toast_failure(ui, &headline, error, kind);
notify(
"takeout-import",
&headline,
&gettext("You can resume it once the problem is solved."),
);
return;
}
let (headline, body) = if summary.interrupted {
(
gettext("Import interrupted"),
gettext("You can resume it where it stopped."),
)
} else if summary.cancelled {
(
gettext("Import stopped"),
ngettext_f(
"Stopped after {n} photo.",
"Stopped after {n} photos.",
summary.uploaded as u64,
&[],
),
)
} else if summary.failed > 0 {
(
gettext("Import finished"),
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
ngettext_f(
"{n} photo added, {failed} failed.",
"{n} photos added, {failed} failed.",
summary.uploaded as u64,
&[("failed", &summary.failed.to_string())],
),
)
} else {
(
gettext("Import finished"),
ngettext_f(
"{n} photo added to Proton Photos.",
"{n} photos added to Proton Photos.",
summary.uploaded as u64,
&[],
),
)
};
toast(ui, &headline);
notify("takeout-import", &headline, &body);
// A complete import is done with its archives; keeping them staged would
// only invite running the same export again.
if summary.complete() {
ui.takeout.archives.borrow_mut().clear();
repaint_archives(ui);
}
}
/// Paint where a running import is.
fn paint_progress(ui: &Rc<Ui>, progress: Option<&ImportProgress>) {
let Some(progress) = progress else {
ui.takeout.progress_bar.pulse();
return;
};
let done = progress.done.to_string();
let total = progress.total.to_string();
let current = progress.current.clone().unwrap_or_default();
let counted = progress.total > 0;
let args = [
("done", done.as_str()),
("total", total.as_str()),
("name", current.as_str()),
];
let text = match (progress.phase, counted) {
(ImportPhase::Scanning, _) => gettext("Reading the archives…"),
(ImportPhase::Checking, false) => gettext("Checking for photos you already have…"),
// Translators: {done} and {total} are counts of photos.
(ImportPhase::Checking, true) => gettext_f(
"Checking for photos you already have — {done} of {total}",
&args,
),
(ImportPhase::Uploading, false) => gettext("Uploading…"),
(ImportPhase::Uploading, true) if current.is_empty() => {
// Translators: {done} and {total} are counts of photos.
gettext_f("Uploading {done} of {total}", &args)
}
// Translators: {done} and {total} are counts of photos; {name} is the file being uploaded.
(ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args),
(ImportPhase::Albums, false) => gettext("Sorting photos into albums…"),
// Translators: {done} and {total} are counts of albums.
(ImportPhase::Albums, true) => {
gettext_f("Sorting photos into albums — {done} of {total}", &args)
}
};
ui.takeout.progress_label.set_text(&text);
if counted {
ui.takeout
.progress_bar
.set_fraction((progress.done as f64 / progress.total as f64).min(1.0));
} else {
ui.takeout.progress_bar.pulse();
}
}
/// Offer to resume the last run when the service says it can be.
fn paint_resume(ui: &Rc<Ui>, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) {
let offer = !running && !resumable.is_empty();
ui.takeout.resume_group.set_visible(offer);
if !offer {
return;
}
let uploaded = summary.map_or(0, |s| s.uploaded) as u64;
// Translators: {n} is the number of photos uploaded before the import stopped.
ui.takeout.resume_row.set_subtitle(&ngettext_f(
"{n} photo uploaded before it stopped. Resuming skips what is already there.",
"{n} photos uploaded before it stopped. Resuming skips what is already there.",
uploaded,
&[],
));
}
/// Paint the counts of a running import, or the report of the last one.
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
ui.takeout.summary_group.remove(&row);
}
let dry = ui.takeout.dry_run.get();
ui.takeout.summary_group.set_title(&if dry {
let dry = summary.dry_run;
ui.takeout.summary_group.set_title(&if running {
gettext("So far")
} else if dry {
gettext("Scan result")
} else {
gettext("Last import")
});
ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error {
// The reason is the service's own text, which is not translated. A
// group's description is markup, and the reason may name a path.
glib::markup_escape_text(&gettext_f(
"{what}: {detail}",
&[
("what", &gettext("Couldn't import from Google Photos")),
("detail", error),
],
))
.to_string()
let description = if running {
None
} else if let Some(error) = &summary.error {
// The reason is the service's own text, which is not translated; the
// headline in front of it is. A group's description is markup, and the
// reason may name a path.
let what = gettext("Couldn't import from Google Photos");
Some(
glib::markup_escape_text(&gettext_f(
"{what}: {detail}",
&[
(
"what",
error_headline(summary.error_kind.unwrap_or_default(), &what),
),
("detail", error),
],
))
.to_string(),
)
} else if dry {
gettext("Nothing was uploaded — this is what an import would do.")
Some(gettext(
"Nothing was uploaded — this is what an import would do.",
))
} else if summary.interrupted {
Some(gettext(
"The Proton Drive service stopped while the import ran. Resume it to continue where it left off.",
))
} else if summary.cancelled {
gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.")
Some(gettext(
"Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.",
))
} else if !summary.complete() {
Some(gettext(
"Finished with problems. Run it again to retry what didn't upload.",
))
} else {
gettext("Finished.")
}));
Some(gettext("Finished."))
};
ui.takeout
.summary_group
.set_description(description.as_deref());
// Only lines that carry information: a run with no failures should not have
// to show a "0 failed" row for the user to read past.
let mut lines: Vec<(String, String)> = vec![
(gettext("Photos in the export"), summary.found.to_string()),
(
if dry {
gettext("Would upload")
} else {
gettext("Uploaded")
},
summary.uploaded.to_string(),
),
];
let mut lines: Vec<(String, String)> =
vec![(gettext("Photos in the export"), summary.found.to_string())];
if dry {
lines.push((gettext("Would upload"), summary.to_upload.to_string()));
} else {
lines.push((gettext("Uploaded"), summary.uploaded.to_string()));
}
if summary.duplicates > 0 {
lines.push((
gettext("Already in Proton Photos"),
summary.duplicates.to_string(),
));
}
if summary.live_photos > 0 {
lines.push((
gettext("Live photos kept together"),
summary.live_photos.to_string(),
));
}
if summary.albums_created > 0 {
lines.push((
if dry {
gettext("Albums to create")
} else {
gettext("Albums created")
},
gettext("Albums created"),
summary.albums_created.to_string(),
));
}
@@ -767,25 +961,31 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
summary.skipped_trashed.to_string(),
));
}
if summary.bytes > 0 {
if dry && summary.to_upload_bytes > 0 {
lines.push((
if dry {
gettext("Data to upload")
} else {
gettext("Data uploaded")
},
human_bytes(summary.bytes),
gettext("Data to upload"),
human_bytes(summary.to_upload_bytes),
));
} else if summary.bytes > 0 {
lines.push((gettext("Data uploaded"), human_bytes(summary.bytes)));
}
if summary.failed > 0 {
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
}
for path in &summary.unreadable_archives {
let name = Path::new(path)
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| path.clone());
lines.push((gettext("Couldn't read this archive"), name));
}
let mut rows = Vec::new();
for (title, value) in lines {
let row = adw::ActionRow::builder()
.title(title)
.subtitle(value)
.use_markup(false)
.build();
row.add_css_class("property");
ui.takeout.summary_group.add(&row);
@@ -794,35 +994,3 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
*ui.takeout.summary_rows.borrow_mut() = rows;
ui.takeout.summary_group.set_visible(true);
}
/// Paint the import's progress from the transfer snapshot the daemon pushed,
/// so this page costs no extra request. A job with no known total
/// pulses instead of showing a fraction, the same as the Activity list.
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
return;
};
ui.takeout.progress_group.set_visible(true);
if job.total > 0 {
ui.takeout
.progress_bar
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
let done = job.done.to_string();
let total = job.total.to_string();
// Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on.
let text = gettext_f(
"{done} of {total} — {detail}",
&[("done", &done), ("total", &total), ("detail", &job.detail)],
);
ui.takeout.progress_label.set_text(&text);
} else {
ui.takeout.progress_bar.pulse();
ui.takeout
.progress_label
.set_text(&if job.detail.is_empty() {
gettext("Working…")
} else {
job.detail.clone()
});
}
}