mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-15 17:41:59 +01:00
fix(fuse,core,gui,cli): resumable, robust Google Photos import (#94)
A Takeout import cut short by a restart could never be resumed: the
duplicate check failed on the draft the interrupted upload left behind
("invalid type: null, expected a string"). SDK 0.12.0 skips such drafts,
lets the import replace them across logins, and removes the draft of a
failed upload (B285, B286).
The import keeps its archives and outcome in the state directory, so a
run the service did not finish is reported as interrupted and can be
resumed from the app or with `pdfs import-google-photos --resume`. An
unreadable archive is left out instead of failing the scan, a failure
keeps its counts and still files albums, and a cancel or shutdown is
only reported when work was left undone (B287).
ImportStatus carries the phase, its progress, the running counts and
what can be resumed; the summary says dry_run, to_upload and
error_kind. The app shows these in the user's language, and --wait
prints progress. An empty gallery waits for a timeline refresh already
running instead of starting a second one.
The scan imports only the Photos service of an export, reads geoData
into the photo's Location, pairs live and motion photos (.MOV, .MP) with
their stills, and dates a photo by name and EXIF before Google's upload
time (B288).
This commit is contained in:
1 parent
2ec3b7ff62
commit
2f2fe96cdf
28 files changed
+5196
-2544
No files matched your search
+180
-38
@@ -228,14 +228,19 @@ enum Command {
|
||||
/// numbered parts, and a photo's metadata often sits in a different part
|
||||
/// than the photo. Albums in the export are recreated as Proton albums, and
|
||||
/// photos already on the account (same name and same content) are skipped,
|
||||
/// so an interrupted import can simply be run again.
|
||||
/// so an interrupted import can simply be run again, or resumed with
|
||||
/// `--resume` without naming the archives again.
|
||||
ImportGooglePhotos {
|
||||
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
||||
#[arg(required = true, value_hint = ValueHint::FilePath)]
|
||||
#[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)]
|
||||
archives: Vec<PathBuf>,
|
||||
/// Report what would be imported without uploading anything.
|
||||
#[arg(long)]
|
||||
#[arg(long, conflicts_with = "resume")]
|
||||
dry_run: bool,
|
||||
/// Start the last import again with the same archives, after it was
|
||||
/// interrupted, stopped or failed.
|
||||
#[arg(long, conflicts_with = "archives")]
|
||||
resume: bool,
|
||||
/// Stay attached and print progress until the import finishes.
|
||||
#[arg(long)]
|
||||
wait: bool,
|
||||
@@ -1030,8 +1035,9 @@ fn main() -> Result<()> {
|
||||
Command::ImportGooglePhotos {
|
||||
archives,
|
||||
dry_run,
|
||||
resume,
|
||||
wait,
|
||||
} => cmd_import_google_photos(archives, dry_run, wait),
|
||||
} => cmd_import_google_photos(archives, dry_run, resume, wait),
|
||||
Command::RedatePhotos {
|
||||
dry_run,
|
||||
limit,
|
||||
@@ -2916,24 +2922,34 @@ fn photo_count(n: usize) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it gets
|
||||
// absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
fn cmd_import_google_photos(
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
resume: bool,
|
||||
wait: bool,
|
||||
) -> Result<()> {
|
||||
let request = if resume {
|
||||
CtlRequest::ResumeImport
|
||||
} else {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it
|
||||
// gets absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
}
|
||||
};
|
||||
|
||||
match control_request(CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
})? {
|
||||
match control_request(request)? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -2945,20 +2961,34 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
||||
|
||||
// Poll rather than stream: the control protocol is request/response, and an
|
||||
// import runs for hours — a held-open socket would be the only long-lived
|
||||
// connection the daemon has.
|
||||
// connection the daemon has. A line is printed when the step changes and
|
||||
// every few percent of it, not on every poll.
|
||||
let mut last_line = String::new();
|
||||
let mut last_printed = std::time::Instant::now();
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus {
|
||||
running: true,
|
||||
summary: _,
|
||||
} => continue,
|
||||
CtlResponse::ImportStatus { summary, .. } => {
|
||||
print_import_summary(summary.as_ref());
|
||||
return finished_run(
|
||||
"the import",
|
||||
summary.map(|s| (s.failed, s.cancelled, s.error)),
|
||||
);
|
||||
progress,
|
||||
..
|
||||
} => {
|
||||
let line = progress.as_ref().map(progress_line).unwrap_or_default();
|
||||
let step_changed = line.split(':').next() != last_line.split(':').next();
|
||||
if !line.is_empty()
|
||||
&& line != last_line
|
||||
&& (step_changed || last_printed.elapsed().as_secs() >= 30)
|
||||
{
|
||||
println!("{line}");
|
||||
last_line = line;
|
||||
last_printed = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
CtlResponse::ImportStatus {
|
||||
summary, resumable, ..
|
||||
} => {
|
||||
print_import_summary(summary.as_ref(), &resumable);
|
||||
return import_outcome(summary.as_ref());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -2966,6 +2996,47 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
||||
}
|
||||
}
|
||||
|
||||
/// One line saying where a running import is.
|
||||
fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String {
|
||||
use pdfs_core::control::ImportPhase;
|
||||
let step = match progress.phase {
|
||||
ImportPhase::Scanning => "Reading archives",
|
||||
ImportPhase::Checking => "Checking for photos already on Proton",
|
||||
ImportPhase::Uploading => "Uploading",
|
||||
ImportPhase::Albums => "Filing photos into albums",
|
||||
};
|
||||
match (progress.total, &progress.current) {
|
||||
(0, _) => format!("{step}…"),
|
||||
(total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done),
|
||||
(total, None) => format!("{step}: {} of {total}", progress.done),
|
||||
}
|
||||
}
|
||||
|
||||
/// How an import `--wait` followed ended, as the exit status: an error unless
|
||||
/// it did everything it set out to do, so a script can tell.
|
||||
fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> {
|
||||
let Some(summary) = summary else {
|
||||
bail!("the import ended without a result; the service may have restarted");
|
||||
};
|
||||
if let Some(error) = &summary.error {
|
||||
bail!("the import failed: {error}");
|
||||
}
|
||||
if summary.interrupted {
|
||||
bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue");
|
||||
}
|
||||
if summary.cancelled {
|
||||
bail!("the import was cancelled");
|
||||
}
|
||||
if summary.failed > 0 || !summary.unreadable_archives.is_empty() {
|
||||
bail!(
|
||||
"the import finished, but {} photo(s) and {} archive(s) failed",
|
||||
summary.failed,
|
||||
summary.unreadable_archives.len()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_redate_photos(
|
||||
dry_run: bool,
|
||||
limit: Option<usize>,
|
||||
@@ -3100,11 +3171,19 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) {
|
||||
|
||||
fn cmd_import_status() -> Result<()> {
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus { running, summary } => {
|
||||
CtlResponse::ImportStatus {
|
||||
running,
|
||||
summary,
|
||||
progress,
|
||||
resumable,
|
||||
} => {
|
||||
if running {
|
||||
println!("Import running. `pdfs transfers` shows its progress.");
|
||||
match progress.as_ref() {
|
||||
Some(progress) => println!("Import running. {}", progress_line(progress)),
|
||||
None => println!("Import running."),
|
||||
}
|
||||
}
|
||||
print_import_summary(summary.as_ref());
|
||||
print_import_summary(summary.as_ref(), &resumable);
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -3121,18 +3200,28 @@ fn cmd_cancel_import() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) {
|
||||
let Some(summary) = summary else {
|
||||
println!("No import has run yet.");
|
||||
return;
|
||||
};
|
||||
if summary.dry_run {
|
||||
println!("Scan only; nothing was uploaded.");
|
||||
}
|
||||
if let Some(error) = &summary.error {
|
||||
println!("Failed : {error}");
|
||||
return;
|
||||
println!("Stopped by : {error}");
|
||||
}
|
||||
println!("Photos in export : {}", summary.found);
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
if summary.dry_run {
|
||||
println!("Would upload : {}", summary.to_upload);
|
||||
println!("Bytes to upload : {}", summary.to_upload_bytes);
|
||||
} else {
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
}
|
||||
println!("Already on Proton: {}", summary.duplicates);
|
||||
if summary.live_photos > 0 {
|
||||
println!("Live photos : {}", summary.live_photos);
|
||||
}
|
||||
if summary.skipped_trashed > 0 {
|
||||
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
||||
}
|
||||
@@ -3145,12 +3234,20 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
if summary.failed > 0 {
|
||||
println!("Failed : {}", summary.failed);
|
||||
}
|
||||
for archive in &summary.unreadable_archives {
|
||||
println!("Unreadable : {archive}");
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
println!("Uploaded bytes : {}", summary.bytes);
|
||||
}
|
||||
if summary.cancelled {
|
||||
if summary.interrupted {
|
||||
println!("The service stopped before the import finished.");
|
||||
} else if summary.cancelled {
|
||||
println!("Import was cancelled before it finished.");
|
||||
}
|
||||
if !resumable.is_empty() {
|
||||
println!("Run `pdfs import-google-photos --resume` to continue it.");
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
||||
@@ -4509,6 +4606,51 @@ mod login_wait_tests {
|
||||
assert!(finished_run("the import", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_waited_for_import_that_did_not_finish_exits_with_an_error() {
|
||||
use pdfs_core::control::ImportSummary;
|
||||
let done = ImportSummary {
|
||||
uploaded: 4,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(import_outcome(Some(&done)).is_ok());
|
||||
assert!(import_outcome(None).is_err());
|
||||
for broken in [
|
||||
ImportSummary {
|
||||
interrupted: true,
|
||||
..Default::default()
|
||||
},
|
||||
ImportSummary {
|
||||
unreadable_archives: vec!["/exports/takeout-002.zip".into()],
|
||||
..Default::default()
|
||||
},
|
||||
ImportSummary {
|
||||
error: Some("check for existing photos: offline".into()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}");
|
||||
}
|
||||
let interrupted = import_outcome(Some(&ImportSummary {
|
||||
interrupted: true,
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(interrupted.unwrap_err().to_string().contains("--resume"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_needs_no_archives_and_excludes_them() {
|
||||
let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap();
|
||||
assert!(matches!(
|
||||
cli.command,
|
||||
Command::ImportGooglePhotos { resume: true, .. }
|
||||
));
|
||||
assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err());
|
||||
assert!(
|
||||
Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiting_for_a_session_stops_hammering_the_keyring() {
|
||||
let mut checks_left = 20;
|
||||
|
||||
@@ -562,6 +562,12 @@ impl AppDirs {
|
||||
self.state_dir().join("control.sock")
|
||||
}
|
||||
|
||||
/// The Google Photos import's last run: its archives and how it ended, so
|
||||
/// an import cut short by a restart can be resumed.
|
||||
pub fn takeout_import_path(&self) -> PathBuf {
|
||||
self.state_dir().join("takeout-import.json")
|
||||
}
|
||||
|
||||
/// Unix domain socket the tray uses to ensure single instance.
|
||||
pub fn tray_socket(&self) -> PathBuf {
|
||||
self.state_dir().join("tray.sock")
|
||||
|
||||
@@ -279,6 +279,11 @@ pub enum Request {
|
||||
/// and files what it has uploaded into its albums, then reports
|
||||
/// `cancelled`. Replies with [`Response::Ok`].
|
||||
CancelImport,
|
||||
/// Start the last import again with the same archives, after it was
|
||||
/// interrupted, cancelled or failed. Photos it already uploaded are found
|
||||
/// on the account and skipped. Replies like [`Request::ImportTakeout`];
|
||||
/// [`ErrorKind::NotFound`] when there is nothing to resume.
|
||||
ResumeImport,
|
||||
/// Upload the photo at `source_path` under the given name and media type.
|
||||
///
|
||||
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
||||
@@ -1689,10 +1694,75 @@ pub struct ImportSummary {
|
||||
pub bytes: u64,
|
||||
/// True when the run stopped early because it was cancelled.
|
||||
pub cancelled: bool,
|
||||
/// Why the run as a whole failed, when it did, such as an archive that
|
||||
/// is not a zip. The counts are then empty.
|
||||
/// Why the run as a whole stopped early, when it did, such as the
|
||||
/// account being unreachable. The counts say how far it got.
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
/// What kind of failure [`error`](Self::error) is, so a front-end can word
|
||||
/// it in the user's language. The text itself is the daemon's English.
|
||||
#[serde(default)]
|
||||
pub error_kind: Option<ErrorKind>,
|
||||
/// True for a scan that uploads nothing ([`Request::ImportTakeout`] with
|
||||
/// `dry_run`). [`to_upload`](Self::to_upload) then says what an import
|
||||
/// would upload.
|
||||
#[serde(default)]
|
||||
pub dry_run: bool,
|
||||
/// Photos not yet on the account: what a dry run would upload, and what a
|
||||
/// real run set out to upload once the duplicates were known.
|
||||
#[serde(default)]
|
||||
pub to_upload: usize,
|
||||
/// Bytes of [`to_upload`](Self::to_upload).
|
||||
#[serde(default)]
|
||||
pub to_upload_bytes: u64,
|
||||
/// Live and motion photos whose video was filed with its still.
|
||||
#[serde(default)]
|
||||
pub live_photos: usize,
|
||||
/// True when the service stopped while the import ran (a shutdown or a
|
||||
/// restart). [`Request::ResumeImport`] picks it up again.
|
||||
#[serde(default)]
|
||||
pub interrupted: bool,
|
||||
/// Archives that could not be read and were left out, by path.
|
||||
#[serde(default)]
|
||||
pub unreadable_archives: Vec<String>,
|
||||
}
|
||||
|
||||
impl ImportSummary {
|
||||
/// Whether the run did all it set out to do.
|
||||
pub fn complete(&self) -> bool {
|
||||
self.error.is_none()
|
||||
&& !self.cancelled
|
||||
&& !self.interrupted
|
||||
&& self.failed == 0
|
||||
&& self.unreadable_archives.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Which step a running Takeout import is on.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportPhase {
|
||||
/// Reading the archives' directories and metadata.
|
||||
#[default]
|
||||
Scanning,
|
||||
/// Asking the account which photos it already has.
|
||||
Checking,
|
||||
/// Uploading the photos that are new.
|
||||
Uploading,
|
||||
/// Creating albums and filing photos into them.
|
||||
Albums,
|
||||
}
|
||||
|
||||
/// Where a running import is ([`Response::ImportStatus`]).
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ImportProgress {
|
||||
pub phase: ImportPhase,
|
||||
/// Steps done in this phase.
|
||||
pub done: u64,
|
||||
/// Steps in this phase; `0` while not yet known.
|
||||
pub total: u64,
|
||||
/// The photo being uploaded, during [`ImportPhase::Uploading`].
|
||||
#[serde(default)]
|
||||
pub current: Option<String>,
|
||||
}
|
||||
|
||||
/// One album in a [`Response::Albums`] listing.
|
||||
@@ -2208,8 +2278,17 @@ pub enum Response {
|
||||
/// report; `summary` is `None` when no import has run this session.
|
||||
ImportStatus {
|
||||
running: bool,
|
||||
/// The counts so far while running, the final report after.
|
||||
#[serde(default)]
|
||||
summary: Option<ImportSummary>,
|
||||
/// The current step, while running.
|
||||
#[serde(default)]
|
||||
progress: Option<ImportProgress>,
|
||||
/// The archives of the last run, when it can be resumed with
|
||||
/// [`Request::ResumeImport`]: it was a real import that did not
|
||||
/// finish. Survives a restart of the service.
|
||||
#[serde(default)]
|
||||
resumable: Vec<String>,
|
||||
},
|
||||
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
|
||||
/// `running` is false once it has finished, when `summary` is the final
|
||||
|
||||
+488
-70
@@ -25,7 +25,9 @@
|
||||
//! The service folder (`Google Photos`) is localized, as are the year folders
|
||||
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
||||
//! service name: the first component is dropped when it is the export root, the
|
||||
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
|
||||
//! second is the service, and the rest is `<folder>/<file>`. The Photos service
|
||||
//! is the one whose sidecars carry a `photoTakenTime`; other services in the same
|
||||
//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an
|
||||
//! **album** when it carries an album metadata sidecar naming it; anything else
|
||||
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
||||
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
||||
@@ -121,8 +123,14 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
|
||||
("m2ts", "video/mp2t"),
|
||||
("mpg", "video/mpeg"),
|
||||
("mpeg", "video/mpeg"),
|
||||
// The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`.
|
||||
("mp", "video/mp4"),
|
||||
];
|
||||
|
||||
/// Sidecars bigger than this are not read. A real one is a few kilobytes; a
|
||||
/// `.json` this large is some other service's data and not worth the memory.
|
||||
const SIDECAR_MAX_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
/// The media type for `name`'s extension, or `None` when it is not media this
|
||||
/// import handles.
|
||||
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
||||
@@ -147,6 +155,13 @@ pub enum Bucket {
|
||||
Archived,
|
||||
}
|
||||
|
||||
/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct GeoPoint {
|
||||
pub latitude: f64,
|
||||
pub longitude: f64,
|
||||
}
|
||||
|
||||
/// One media file found in the export.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutPhoto {
|
||||
@@ -162,14 +177,49 @@ pub struct TakeoutPhoto {
|
||||
pub size: u64,
|
||||
/// Media type from the extension.
|
||||
pub media_type: String,
|
||||
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
|
||||
/// `creationTime`). `None` when the photo has no sidecar — the upload then
|
||||
/// defaults it to now, as the SDK does.
|
||||
/// Capture time in epoch seconds, from the sidecar's `photoTakenTime`,
|
||||
/// else from a date in the file name. `None` when neither has one; the
|
||||
/// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time).
|
||||
pub capture_time: Option<i64>,
|
||||
/// The sidecar's `creationTime`: when the photo reached Google Photos, not
|
||||
/// when it was taken. Only a last resort for the capture time.
|
||||
#[serde(default)]
|
||||
pub creation_time: Option<i64>,
|
||||
/// Where the photo was taken, from the sidecar's `geoData` (or
|
||||
/// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped.
|
||||
#[serde(default)]
|
||||
pub location: Option<GeoPoint>,
|
||||
/// The sidecar's `favorited` flag.
|
||||
pub favorite: bool,
|
||||
/// Which bucket the file was found in.
|
||||
pub bucket: Bucket,
|
||||
/// Pairs the halves of a live or motion photo: a still and a video in the
|
||||
/// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||
/// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half.
|
||||
#[serde(default)]
|
||||
pub live_key: Option<String>,
|
||||
}
|
||||
|
||||
impl TakeoutPhoto {
|
||||
/// Whether this is the video half of a pair, as opposed to the still.
|
||||
pub fn is_video(&self) -> bool {
|
||||
self.media_type.starts_with("video/")
|
||||
}
|
||||
|
||||
/// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather
|
||||
/// than an Apple live photo.
|
||||
pub fn is_motion_photo(&self) -> bool {
|
||||
self.live_key
|
||||
.as_deref()
|
||||
.is_some_and(|key| key.ends_with(".mp"))
|
||||
}
|
||||
}
|
||||
|
||||
/// An archive the scan had to leave out, and why.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UnreadableArchive {
|
||||
pub path: PathBuf,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// Everything one scan found, ready to be turned into an import plan.
|
||||
@@ -183,8 +233,16 @@ pub struct TakeoutScan {
|
||||
/// Media files skipped because they sit in the trash bucket.
|
||||
pub skipped_trashed: usize,
|
||||
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
||||
/// activity `.csv`s, `.mp` motion-photo parts).
|
||||
/// activity `.csv`s), and media from other Google services in the same
|
||||
/// export (`Google Drive/`, `YouTube/`).
|
||||
pub skipped_other: usize,
|
||||
/// Archives that could not be opened or are not a zip. The rest of the
|
||||
/// export is still imported.
|
||||
#[serde(default)]
|
||||
pub unreadable_archives: Vec<UnreadableArchive>,
|
||||
/// Entries inside an otherwise readable archive that could not be read.
|
||||
#[serde(default)]
|
||||
pub unreadable_entries: usize,
|
||||
}
|
||||
|
||||
impl TakeoutScan {
|
||||
@@ -195,7 +253,7 @@ impl TakeoutScan {
|
||||
}
|
||||
|
||||
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
||||
/// file also carries geo data, people tags and view counts.
|
||||
/// file also carries people tags, a description and view counts.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct PhotoSidecar {
|
||||
#[serde(default)]
|
||||
@@ -205,19 +263,69 @@ struct PhotoSidecar {
|
||||
#[serde(rename = "creationTime", default)]
|
||||
creation_time: Option<SidecarTime>,
|
||||
#[serde(default)]
|
||||
favorited: bool,
|
||||
favorited: Option<bool>,
|
||||
/// The position as edited in Google Photos, else as the camera wrote it.
|
||||
#[serde(rename = "geoData", default)]
|
||||
geo_data: Option<SidecarGeo>,
|
||||
#[serde(rename = "geoDataExif", default)]
|
||||
geo_data_exif: Option<SidecarGeo>,
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, as a *string*.
|
||||
impl PhotoSidecar {
|
||||
fn location(&self) -> Option<GeoPoint> {
|
||||
self.geo_data
|
||||
.as_ref()
|
||||
.and_then(SidecarGeo::point)
|
||||
.or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point))
|
||||
}
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarTime {
|
||||
#[serde(default)]
|
||||
timestamp: Option<String>,
|
||||
timestamp: Option<SidecarNumber>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum SidecarNumber {
|
||||
Number(i64),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
impl SidecarTime {
|
||||
/// The timestamp, when it is a real moment. `0` and negative values are
|
||||
/// Google's "unknown" and would file the photo under 1970.
|
||||
fn epoch_seconds(&self) -> Option<i64> {
|
||||
self.timestamp.as_ref()?.parse().ok()
|
||||
let seconds = match self.timestamp.as_ref()? {
|
||||
SidecarNumber::Number(seconds) => *seconds,
|
||||
SidecarNumber::Text(text) => text.trim().parse().ok()?,
|
||||
};
|
||||
(seconds > 0).then_some(seconds)
|
||||
}
|
||||
}
|
||||
|
||||
/// `geoData` in a sidecar. All-zero means Google has no position.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarGeo {
|
||||
#[serde(default)]
|
||||
latitude: Option<f64>,
|
||||
#[serde(default)]
|
||||
longitude: Option<f64>,
|
||||
}
|
||||
|
||||
impl SidecarGeo {
|
||||
fn point(&self) -> Option<GeoPoint> {
|
||||
let (latitude, longitude) = (self.latitude?, self.longitude?);
|
||||
let on_globe = latitude.is_finite()
|
||||
&& longitude.is_finite()
|
||||
&& (-90.0..=90.0).contains(&latitude)
|
||||
&& (-180.0..=180.0).contains(&longitude);
|
||||
(on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint {
|
||||
latitude,
|
||||
longitude,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,21 +370,58 @@ impl RawEntry {
|
||||
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
||||
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
||||
/// this stays fast on a 200 GB export.
|
||||
///
|
||||
/// An archive that cannot be opened is left out and listed in
|
||||
/// [`TakeoutScan::unreadable_archives`]; the rest still imports.
|
||||
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<RawEntry> = Vec::new();
|
||||
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
scan_archives_until(archives, || false)
|
||||
}
|
||||
|
||||
/// [`scan_archives`] that gives up when `stop` returns true, checked between
|
||||
/// archives and every few hundred entries.
|
||||
pub fn scan_archives_until(
|
||||
archives: &[PathBuf],
|
||||
stop: impl Fn() -> bool,
|
||||
) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<(String, RawEntry)> = Vec::new();
|
||||
let mut sidecars: Vec<(String, String, Vec<u8>)> = Vec::new();
|
||||
let mut unreadable_archives = Vec::new();
|
||||
let mut unreadable_entries = 0;
|
||||
|
||||
for (index, path) in archives.iter().enumerate() {
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
|
||||
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
|
||||
})?;
|
||||
if stop() {
|
||||
return Err(CoreError::conflict("the import was cancelled"));
|
||||
}
|
||||
let opened = std::fs::File::open(path)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|file| {
|
||||
zip::ZipArchive::new(std::io::BufReader::new(file))
|
||||
.map_err(|e| format!("not a readable zip: {e}"))
|
||||
});
|
||||
let mut zip = match opened {
|
||||
Ok(zip) => zip,
|
||||
Err(reason) => {
|
||||
tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive");
|
||||
unreadable_archives.push(UnreadableArchive {
|
||||
path: path.clone(),
|
||||
reason,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for position in 0..zip.len() {
|
||||
let mut entry = zip.by_index(position).map_err(|e| {
|
||||
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
|
||||
})?;
|
||||
if position % 512 == 0 && stop() {
|
||||
return Err(CoreError::conflict("the import was cancelled"));
|
||||
}
|
||||
let mut entry = match zip.by_index(position) {
|
||||
Ok(entry) => entry,
|
||||
Err(e) => {
|
||||
tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry");
|
||||
unreadable_entries += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.is_dir() {
|
||||
continue;
|
||||
}
|
||||
@@ -286,32 +431,92 @@ pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
continue;
|
||||
};
|
||||
let name = name.to_string_lossy().replace('\\', "/");
|
||||
let Some(relative) = strip_export_prefix(&name) else {
|
||||
let Some((service, relative)) = strip_export_prefix(&name) else {
|
||||
continue;
|
||||
};
|
||||
let (service, relative) = (service.to_string(), relative.to_string());
|
||||
let size = entry.size();
|
||||
|
||||
// Sidecars are read now, while the entry is open: they are a few
|
||||
// hundred bytes each and reopening the archive per photo later would
|
||||
// be a second full pass.
|
||||
if relative.to_ascii_lowercase().ends_with(".json") {
|
||||
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.insert(relative.to_string(), body);
|
||||
if size <= SIDECAR_MAX_BYTES {
|
||||
let mut body = Vec::with_capacity(size as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.push((service, relative, body));
|
||||
} else {
|
||||
unreadable_entries += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push(RawEntry {
|
||||
archive: index,
|
||||
entry: name.clone(),
|
||||
relative: relative.to_string(),
|
||||
size,
|
||||
});
|
||||
entries.push((
|
||||
service,
|
||||
RawEntry {
|
||||
archive: index,
|
||||
entry: name,
|
||||
relative,
|
||||
size,
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(assemble(entries, &sidecars))
|
||||
let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars);
|
||||
let mut scan = assemble(entries, &sidecars);
|
||||
scan.skipped_other += other_services;
|
||||
scan.unreadable_archives = unreadable_archives;
|
||||
scan.unreadable_entries = unreadable_entries;
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
/// Drop what other Google services put into the same export.
|
||||
///
|
||||
/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to
|
||||
/// `Google Photos/`, and their media must not land in the photo timeline. The
|
||||
/// service folder's name is localized (`Google Fotos`), so the Photos service
|
||||
/// is recognized by its content instead: it is the folder whose sidecars carry
|
||||
/// a `photoTakenTime`. When no folder does — an export without sidecars, or
|
||||
/// one re-zipped without the service level — everything is kept, as before.
|
||||
///
|
||||
/// Returns the entries and sidecars to import, and how many media files were
|
||||
/// left out.
|
||||
fn keep_photos_services(
|
||||
entries: Vec<(String, RawEntry)>,
|
||||
sidecars: Vec<(String, String, Vec<u8>)>,
|
||||
) -> (Vec<RawEntry>, HashMap<String, Vec<u8>>, usize) {
|
||||
#[derive(Deserialize)]
|
||||
struct Marker {
|
||||
#[serde(rename = "photoTakenTime")]
|
||||
photo_taken_time: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new();
|
||||
for (service, _, body) in &sidecars {
|
||||
if photos_services.contains(service.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if serde_json::from_slice::<Marker>(body).is_ok_and(|m| m.photo_taken_time.is_some()) {
|
||||
photos_services.insert(service.as_str());
|
||||
}
|
||||
}
|
||||
let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service);
|
||||
|
||||
let mut skipped = 0;
|
||||
let mut kept_entries = Vec::with_capacity(entries.len());
|
||||
for (service, raw) in entries {
|
||||
if keep(&service) {
|
||||
kept_entries.push(raw);
|
||||
} else if media_type_of(raw.file_name()).is_some() {
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
let kept_sidecars = sidecars
|
||||
.iter()
|
||||
.filter(|(service, _, _)| keep(service))
|
||||
.map(|(_, relative, body)| (relative.clone(), body.clone()))
|
||||
.collect();
|
||||
(kept_entries, kept_sidecars, skipped)
|
||||
}
|
||||
|
||||
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
||||
@@ -403,19 +608,19 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
||||
.unwrap_or(&file_name)
|
||||
.to_string();
|
||||
|
||||
// Sidecar first, then the date stamped into the name.
|
||||
// Sidecar first, then the date stamped into the name. `creationTime`
|
||||
// is when the photo reached Google, which for a scan or an old photo
|
||||
// uploaded late is years off; the importer uses it only after EXIF.
|
||||
let capture_time = sidecar
|
||||
.photo_taken_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
.or_else(|| {
|
||||
sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
})
|
||||
.or_else(|| capture_time_from_name(&name))
|
||||
.or_else(|| capture_time_from_name(&file_name));
|
||||
let creation_time = sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds);
|
||||
|
||||
scan.photos.push(TakeoutPhoto {
|
||||
archive: raw.archive,
|
||||
@@ -424,31 +629,57 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
||||
size: raw.size,
|
||||
media_type: media_type.to_string(),
|
||||
capture_time,
|
||||
favorite: sidecar.favorited,
|
||||
creation_time,
|
||||
location: sidecar.location(),
|
||||
favorite: sidecar.favorited.unwrap_or(false),
|
||||
bucket,
|
||||
live_key: live_key(&folder, &file_name),
|
||||
});
|
||||
}
|
||||
|
||||
scan
|
||||
}
|
||||
|
||||
/// Drop the export root (`Takeout/`) and the localized service folder from a
|
||||
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
|
||||
/// Split a path inside the archive into the localized service folder and the
|
||||
/// rest, dropping the export root (`Takeout/`): `("Google Fotos",
|
||||
/// "<folder>/<file>")`.
|
||||
///
|
||||
/// `None` for a path with no service folder at all (the export's own
|
||||
/// `archive_browser.html`), which carries nothing to import.
|
||||
fn strip_export_prefix(path: &str) -> Option<&str> {
|
||||
fn strip_export_prefix(path: &str) -> Option<(&str, &str)> {
|
||||
let mut rest = path;
|
||||
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
|
||||
// the user may not have it — so the root is dropped only when a service
|
||||
// folder follows it, which the component count decides.
|
||||
// The root is `Takeout/`, but an export unzipped and re-zipped by the user
|
||||
// may not have it. `Google Photos/<album>/<file>` then has three parts like
|
||||
// `Takeout/Google Photos/<file>`, so the count alone cannot tell them
|
||||
// apart: a root of another name is only taken for one with four parts.
|
||||
let (first, after) = rest.split_once('/')?;
|
||||
let components = rest.split('/').count();
|
||||
if components >= 3 {
|
||||
rest = rest.split_once('/')?.1;
|
||||
if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 {
|
||||
rest = after;
|
||||
}
|
||||
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
|
||||
let (_service, tail) = rest.split_once('/')?;
|
||||
Some(tail)
|
||||
// The service folder (`Google Photos`, `Google Fotos`, …).
|
||||
rest.split_once('/')
|
||||
}
|
||||
|
||||
/// The key that pairs the still and the video of one live or motion photo, or
|
||||
/// `None` when `file_name` cannot be either half.
|
||||
///
|
||||
/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem.
|
||||
/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the
|
||||
/// video's whole name. The folder is part of the key so two cameras' `IMG_1`
|
||||
/// in different albums never pair.
|
||||
fn live_key(folder: &str, file_name: &str) -> Option<String> {
|
||||
let lower = file_name.to_lowercase();
|
||||
let (stem, ext) = lower.rsplit_once('.')?;
|
||||
let key = match ext {
|
||||
"heic" | "heif" | "jpg" | "jpeg" => stem,
|
||||
// Apple's video half. A plain `.mp4` beside a photo of the same name is
|
||||
// far more often an unrelated clip than half of a live photo.
|
||||
"mov" => stem,
|
||||
"mp" => lower.as_str(),
|
||||
_ => return None,
|
||||
};
|
||||
(!key.is_empty()).then(|| format!("{folder}/{key}"))
|
||||
}
|
||||
|
||||
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
||||
@@ -858,13 +1089,13 @@ mod tests {
|
||||
RawEntry {
|
||||
archive,
|
||||
entry: path.to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().1.to_string(),
|
||||
size,
|
||||
}
|
||||
}
|
||||
|
||||
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
||||
let relative = strip_export_prefix(path).unwrap().to_string();
|
||||
let relative = strip_export_prefix(path).unwrap().1.to_string();
|
||||
map.insert(relative, body.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
@@ -872,12 +1103,17 @@ mod tests {
|
||||
fn strips_export_root_and_localized_service_folder() {
|
||||
assert_eq!(
|
||||
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
||||
Some("Island 2019/IMG_1.jpg")
|
||||
Some(("Google Fotos", "Island 2019/IMG_1.jpg"))
|
||||
);
|
||||
// Already unwrapped by the user: no `Takeout/` root left.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
||||
Some("IMG_1.jpg")
|
||||
Some(("Google Photos", "IMG_1.jpg"))
|
||||
);
|
||||
// ... and an album keeps its folder.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"),
|
||||
Some(("Google Photos", "Iceland/IMG_1.jpg"))
|
||||
);
|
||||
// Nothing below a service folder is nothing to import.
|
||||
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
||||
@@ -957,23 +1193,144 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creation_time_is_the_fallback_capture_time() {
|
||||
fn creation_time_is_kept_apart_as_the_last_resort() {
|
||||
// `creationTime` is when the photo reached Google. It must not beat the
|
||||
// date in the name, and the importer tries EXIF before it.
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1700000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
|
||||
entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg",
|
||||
10,
|
||||
),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
|
||||
assert_eq!(scan.photos[0].capture_time, None);
|
||||
assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000));
|
||||
assert_eq!(
|
||||
scan.photos[1].capture_time,
|
||||
capture_time_from_name("IMG_20190612_101500.jpg")
|
||||
);
|
||||
assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_timestamps_tolerate_numbers_zero_and_null() {
|
||||
let mut sidecars = HashMap::new();
|
||||
for (name, body) in [
|
||||
("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#),
|
||||
(
|
||||
"B.jpg",
|
||||
r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#,
|
||||
),
|
||||
(
|
||||
"C.jpg",
|
||||
r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#,
|
||||
),
|
||||
] {
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
&format!("Takeout/Google Photos/Photos from 2019/{name}.json"),
|
||||
body,
|
||||
);
|
||||
}
|
||||
let scan = assemble(
|
||||
["A.jpg", "B.jpg", "C.jpg"]
|
||||
.iter()
|
||||
.map(|name| {
|
||||
entry(
|
||||
0,
|
||||
&format!("Takeout/Google Photos/Photos from 2019/{name}"),
|
||||
1,
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970");
|
||||
assert!(!scan.photos[1].favorite);
|
||||
assert_eq!(scan.photos[2].capture_time, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_comes_from_geo_data_and_zero_means_none() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_1.jpg.json",
|
||||
r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_2.jpg.json",
|
||||
r#"{"geoData":{"latitude":0.0,"longitude":0.0},
|
||||
"geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_3.jpg.json",
|
||||
r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
(1..=3)
|
||||
.map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1))
|
||||
.collect(),
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
scan.photos[0].location,
|
||||
Some(GeoPoint {
|
||||
latitude: 64.1466,
|
||||
longitude: -21.9426
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
scan.photos[1].location.map(|p| p.latitude),
|
||||
Some(63.4186),
|
||||
"geoDataExif fills in for an empty geoData"
|
||||
);
|
||||
assert_eq!(scan.photos[2].location, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_and_motion_photo_halves_share_a_key() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1),
|
||||
entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/scan.png", 1),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
let key = |i: usize| scan.photos[i].live_key.clone();
|
||||
|
||||
assert_eq!(key(0), key(1));
|
||||
assert!(!scan.photos[0].is_motion_photo());
|
||||
assert_eq!(key(2), key(3));
|
||||
assert!(scan.photos[3].is_motion_photo());
|
||||
assert!(scan.photos[3].is_video(), "the .MP half is a video");
|
||||
assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair");
|
||||
assert_eq!(key(5), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1387,16 +1744,77 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_a_zip_is_a_readable_error() {
|
||||
let path =
|
||||
fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() {
|
||||
let broken =
|
||||
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
||||
std::fs::write(&path, b"not a zip at all").unwrap();
|
||||
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
std::fs::write(&broken, b"not a zip at all").unwrap();
|
||||
let good = write_zip(
|
||||
"beside-broken",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
|
||||
let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap();
|
||||
let _ = std::fs::remove_file(&broken);
|
||||
let _ = std::fs::remove_file(&good);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].archive, 1);
|
||||
assert_eq!(scan.unreadable_archives.len(), 1);
|
||||
assert_eq!(scan.unreadable_archives[0].path, broken);
|
||||
assert!(
|
||||
error.message.contains("not a readable zip"),
|
||||
"unexpected message: {}",
|
||||
error.message
|
||||
scan.unreadable_archives[0]
|
||||
.reason
|
||||
.contains("not a readable zip"),
|
||||
"unexpected reason: {}",
|
||||
scan.unreadable_archives[0].reason
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_services_in_the_same_export_are_not_imported() {
|
||||
let zip = write_zip(
|
||||
"services",
|
||||
&[
|
||||
(
|
||||
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg",
|
||||
b"x".as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json",
|
||||
br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(),
|
||||
),
|
||||
("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()),
|
||||
("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()),
|
||||
],
|
||||
);
|
||||
|
||||
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
|
||||
assert_eq!(scan.skipped_other, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_export_without_sidecars_keeps_every_service() {
|
||||
let zip = write_zip(
|
||||
"no-sidecars",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_request_ends_the_scan() {
|
||||
let zip = write_zip(
|
||||
"stop",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
let result = scan_archives_until(std::slice::from_ref(&zip), || true);
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
fuser.workspace = true
|
||||
libc.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
base64.workspace = true
|
||||
image.workspace = true
|
||||
|
||||
@@ -877,35 +877,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
if archives.is_empty() {
|
||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||
} else {
|
||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)))
|
||||
} else {
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
let paths = archives.into_iter().map(PathBuf::from).collect();
|
||||
start_takeout_import(core, paths, dry_run)
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ImportStatus) => {
|
||||
let (running, summary) = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus { running, summary }
|
||||
let report = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus {
|
||||
running: report.running,
|
||||
summary: report.summary,
|
||||
progress: report.progress,
|
||||
resumable: report.resumable,
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
|
||||
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
|
||||
Some(paths) => start_takeout_import(core, paths, false),
|
||||
},
|
||||
Ok(CtlRequest::CancelImport) => {
|
||||
if crate::takeout::cancel_import() {
|
||||
CtlResponse::Ok {
|
||||
@@ -2009,6 +1997,31 @@ pub(crate) fn run_control_socket(
|
||||
}
|
||||
}
|
||||
|
||||
/// Check that every archive is there, then start the import in the background.
|
||||
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)));
|
||||
}
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_limit_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1193,7 +1193,7 @@ impl Core {
|
||||
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !self.rt.block_on(self.refresh_timeline())? {
|
||||
if !self.refresh_timeline_joined()? {
|
||||
return Ok(None);
|
||||
}
|
||||
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||
@@ -2640,6 +2640,36 @@ impl Core {
|
||||
|
||||
/// Refresh the timeline off the request path, so a stale page is still served
|
||||
/// at DB speed. At most one refresh runs at a time.
|
||||
/// Refresh the timeline and wait for it, sharing a refresh already under
|
||||
/// way instead of starting a second one beside it.
|
||||
///
|
||||
/// The first request after an empty cache has nothing to serve until the
|
||||
/// fetch lands, so it waits. Two such requests (the gallery and its retry,
|
||||
/// or a background refresh started by an import) used to each fetch the
|
||||
/// whole library at once, sharing the session's rate limit and taking
|
||||
/// minutes. Returns whether the account has a photos library.
|
||||
pub(crate) fn refresh_timeline_joined(&self) -> CoreResult<bool> {
|
||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||
while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() {
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
}
|
||||
// A refresh that landed stamped the timeline fresh. One that failed
|
||||
// did not, and the caller gets this refresh's own error instead of
|
||||
// an empty timeline.
|
||||
if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL)
|
||||
|| self.timeline_refreshing.swap(true, Ordering::SeqCst)
|
||||
{
|
||||
return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0));
|
||||
}
|
||||
}
|
||||
let refreshed = self.rt.block_on(self.refresh_timeline());
|
||||
self.timeline_refreshing.store(false, Ordering::SeqCst);
|
||||
if refreshed.is_ok() {
|
||||
self.events.publish(&[Topic::Photos]);
|
||||
}
|
||||
refreshed
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_timeline_refresh(&self) {
|
||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||
return;
|
||||
|
||||
+894
-239
File diff suppressed because it is too large.
Load diff
@@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState};
|
||||
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
|
||||
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress,
|
||||
PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope,
|
||||
Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry,
|
||||
ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
|
||||
ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||
DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem,
|
||||
LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo,
|
||||
PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo,
|
||||
SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase,
|
||||
SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||
};
|
||||
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
@@ -668,9 +668,11 @@ fn build_window(app: &adw::Application) {
|
||||
progress_bar: takeout_widgets.progress_bar.clone(),
|
||||
summary_group: takeout_widgets.summary_group.clone(),
|
||||
summary_rows: RefCell::new(Vec::new()),
|
||||
resume_group: takeout_widgets.resume_group.clone(),
|
||||
resume_row: takeout_widgets.resume_row.clone(),
|
||||
resume_button: takeout_widgets.resume_button.clone(),
|
||||
inflight: Cell::new(false),
|
||||
running: Cell::new(false),
|
||||
dry_run: Cell::new(false),
|
||||
},
|
||||
update,
|
||||
});
|
||||
|
||||
@@ -1115,9 +1115,6 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
// count falling to zero is what "the sync finished" looks like from here.
|
||||
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
||||
// starts its upload job mid-flight, which is not a thing finishing.
|
||||
// The Import page reads its progress off the same snapshot rather than
|
||||
// polling the daemon a second time for numbers already on the wire.
|
||||
takeout_progress(ui, jobs);
|
||||
|
||||
let previous = ui.status.active_transfers.replace(items.len());
|
||||
if !items.is_empty() {
|
||||
@@ -1211,13 +1208,18 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
|
||||
let done = j.done.to_string();
|
||||
let total = j.total.to_string();
|
||||
let (title, detail) = if j.title == IMPORT_JOB {
|
||||
import_job_text(&j.detail)
|
||||
} else {
|
||||
(j.title.clone(), j.detail.clone())
|
||||
};
|
||||
let args: [(&str, &str); 4] = [
|
||||
("title", &j.title),
|
||||
("detail", &j.detail),
|
||||
("title", &title),
|
||||
("detail", &detail),
|
||||
("done", &done),
|
||||
("total", &total),
|
||||
];
|
||||
let text = match (j.total > 0, j.detail.is_empty()) {
|
||||
let text = match (j.total > 0, detail.is_empty()) {
|
||||
// Translators: a background job's progress. {title} is the job, such as
|
||||
// "Scanning Photos"; {done} and {total} are counts.
|
||||
(true, true) => gettext_f("{title} — {done} of {total}", &args),
|
||||
|
||||
@@ -15,17 +15,30 @@
|
||||
//! Import — staging is local, so a mis-drop costs nothing.
|
||||
//!
|
||||
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
||||
//! stages paths, launches, asks [`Request::ImportStatus`] for the final report,
|
||||
//! and reads live progress off the import's job in the transfer snapshot the
|
||||
//! daemon already pushes (see [`takeout_progress`]).
|
||||
//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the
|
||||
//! running counts and the final report. It asks again whenever the import's job
|
||||
//! moves in the transfer snapshot the daemon already pushes. A run the service
|
||||
//! did not get to finish can be picked up with [`Request::ResumeImport`].
|
||||
|
||||
use crate::*;
|
||||
|
||||
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
||||
/// Matching on it is how this page borrows the progress the daemon already
|
||||
/// pushes instead of asking for it separately.
|
||||
/// Seeing it move is the cue to ask for the import's status; the Activity list
|
||||
/// shows it under its translated name ([`import_job_text`]).
|
||||
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
|
||||
|
||||
/// The import job's title and detail in the user's language. The daemon names
|
||||
/// its steps in English; the file being uploaded passes through as it is.
|
||||
pub(crate) fn import_job_text(detail: &str) -> (String, String) {
|
||||
let detail = match detail {
|
||||
"reading archives" => gettext("Reading the archives"),
|
||||
"checking for photos you already have" => gettext("Checking for photos you already have"),
|
||||
"sorting photos into albums" => gettext("Sorting photos into albums"),
|
||||
other => other.to_string(),
|
||||
};
|
||||
(gettext("Importing from Google Photos"), detail)
|
||||
}
|
||||
|
||||
/// What the archive list says before anything is staged. `repaint_archives`
|
||||
/// swaps in the count and total once there is something to report, and swaps
|
||||
/// this back when the list is emptied again.
|
||||
@@ -95,18 +108,19 @@ pub(crate) struct TakeoutState {
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
/// Report group for the last finished run, hidden until there is one.
|
||||
/// The running counts, or the report for the last finished run; hidden
|
||||
/// until there is one.
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
||||
/// Offers to resume an import the service did not get to finish.
|
||||
pub(crate) resume_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_row: adw::ActionRow,
|
||||
pub(crate) resume_button: gtk4::Button,
|
||||
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
||||
/// can't stack worker threads on the 2s tick.
|
||||
pub(crate) inflight: Cell<bool>,
|
||||
/// Whether the daemon reported an import running on the last poll.
|
||||
pub(crate) running: Cell<bool>,
|
||||
/// Whether the run in flight was launched as a dry run, so the report can
|
||||
/// say "would upload" rather than claiming photos were sent. The daemon's
|
||||
/// summary carries no such flag — it is the same shape either way.
|
||||
pub(crate) dry_run: Cell<bool>,
|
||||
}
|
||||
|
||||
/// Widgets the Import page hands back for wiring.
|
||||
@@ -126,6 +140,9 @@ pub(crate) struct TakeoutWidgets {
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_row: adw::ActionRow,
|
||||
pub(crate) resume_button: gtk4::Button,
|
||||
}
|
||||
|
||||
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
@@ -261,7 +278,23 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
.visible(false)
|
||||
.build();
|
||||
|
||||
// An import cut short by a restart or a lost connection, offered back at the
|
||||
// top of the page: picking the archives again is exactly what a reboot in
|
||||
// the middle of a migration should not cost.
|
||||
let resume_button = gtk4::Button::builder()
|
||||
.label(pgettext("verb", "Resume"))
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
resume_button.add_css_class("suggested-action");
|
||||
let resume_row = adw::ActionRow::builder()
|
||||
.title(gettext("The last import didn't finish"))
|
||||
.build();
|
||||
resume_row.add_suffix(&resume_button);
|
||||
let resume_group = adw::PreferencesGroup::builder().visible(false).build();
|
||||
resume_group.add(&resume_row);
|
||||
|
||||
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
||||
groups.append(&resume_group);
|
||||
groups.append(&dropzone);
|
||||
groups.append(&list_group);
|
||||
groups.append(&actions);
|
||||
@@ -301,6 +334,9 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
progress_label: progress_label.clone(),
|
||||
progress_bar: progress_bar.clone(),
|
||||
summary_group: summary_group.clone(),
|
||||
resume_group,
|
||||
resume_row,
|
||||
resume_button,
|
||||
};
|
||||
(frame.upcast(), widgets)
|
||||
}
|
||||
@@ -427,6 +463,28 @@ pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
|
||||
});
|
||||
});
|
||||
|
||||
let ui_resume = ui.clone();
|
||||
widgets.resume_button.connect_clicked(move |_| {
|
||||
let ui = ui_resume.clone();
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport);
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => import_started(&ui, false),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
&gettext("Couldn't resume the import"),
|
||||
&gettext("The Proton Drive service didn't respond."),
|
||||
),
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let ui_back = ui.clone();
|
||||
widgets
|
||||
.back_button
|
||||
@@ -559,6 +617,7 @@ fn sync_takeout_actions(ui: &Rc<Ui>) {
|
||||
.import_button
|
||||
.set_sensitive(has_files && !running);
|
||||
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
||||
ui.takeout.resume_button.set_sensitive(!running);
|
||||
ui.takeout.cancel_button.set_visible(running);
|
||||
}
|
||||
|
||||
@@ -577,7 +636,6 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
return;
|
||||
}
|
||||
|
||||
ui.takeout.dry_run.set(dry_run);
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(
|
||||
ui.dirs.control_socket(),
|
||||
@@ -588,27 +646,7 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => {
|
||||
ui.takeout.running.set(true);
|
||||
ui.gallery.import_banner.set_revealed(true);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout.progress_label.set_text(&if dry_run {
|
||||
gettext("Reading archives…")
|
||||
} else {
|
||||
gettext("Starting import…")
|
||||
});
|
||||
sync_takeout_actions(&ui);
|
||||
toast(
|
||||
&ui,
|
||||
&if dry_run {
|
||||
gettext("Scanning the export…")
|
||||
} else {
|
||||
gettext("Import started — you can leave this page, it keeps running.")
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
|
||||
}
|
||||
@@ -621,9 +659,39 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
});
|
||||
}
|
||||
|
||||
/// Ask the daemon for the import's state, and paint the report once a run has
|
||||
/// finished. Asked on arrival and while import progress keeps arriving. Cheap
|
||||
/// and inflight-guarded.
|
||||
/// Switch the page over to reporting progress once the daemon took the run.
|
||||
fn import_started(ui: &Rc<Ui>, dry_run: bool) {
|
||||
ui.takeout.running.set(true);
|
||||
if !dry_run {
|
||||
ui.gallery.import_banner.set_revealed(true);
|
||||
}
|
||||
ui.takeout.resume_group.set_visible(false);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_group.set_title(&if dry_run {
|
||||
gettext("Scan in progress")
|
||||
} else {
|
||||
gettext("Import in progress")
|
||||
});
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&gettext("Reading the archives…"));
|
||||
sync_takeout_actions(ui);
|
||||
toast(
|
||||
ui,
|
||||
&if dry_run {
|
||||
gettext("Scanning the export…")
|
||||
} else {
|
||||
gettext("Import started — you can leave this page, it keeps running.")
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Ask the daemon for the import's state and paint it: the step and counts
|
||||
/// while it runs, the report once it has ended, and the offer to resume a run
|
||||
/// that did not finish. Asked on arrival and while import progress keeps
|
||||
/// arriving. Cheap and inflight-guarded.
|
||||
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
if ui.takeout.inflight.get() {
|
||||
return;
|
||||
@@ -634,124 +702,250 @@ pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.takeout.inflight.set(false);
|
||||
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
|
||||
let Ok(Ok(Response::ImportStatus {
|
||||
running,
|
||||
summary,
|
||||
progress,
|
||||
resumable,
|
||||
})) = result
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let was_running = ui.takeout.running.replace(running);
|
||||
ui.gallery.import_banner.set_revealed(running);
|
||||
let dry = summary.as_ref().is_some_and(|s| s.dry_run);
|
||||
ui.gallery.import_banner.set_revealed(running && !dry);
|
||||
ui.takeout.progress_group.set_visible(running);
|
||||
if running {
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
} else {
|
||||
ui.takeout.progress_group.set_visible(false);
|
||||
// An import runs for hours, so its ending is announced rather than
|
||||
// left to be discovered: a toast for whoever is looking at the
|
||||
// window, and a desktop notification for whoever is not.
|
||||
let failed = summary.as_ref().and_then(|s| s.error.clone());
|
||||
if was_running && let Some(error) = &failed {
|
||||
let headline = gettext("Couldn't import from Google Photos");
|
||||
toast_error(&ui, &headline, error);
|
||||
notify("takeout-import", &headline, error);
|
||||
} else if was_running {
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
let headline = if dry {
|
||||
gettext("Scan finished")
|
||||
} else {
|
||||
gettext("Import finished")
|
||||
};
|
||||
toast(&ui, &headline);
|
||||
if !dry {
|
||||
let body = match &summary {
|
||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||
Some(s) if s.failed > 0 => ngettext_f(
|
||||
"{n} photo added, {failed} failed.",
|
||||
"{n} photos added, {failed} failed.",
|
||||
s.uploaded as u64,
|
||||
&[("failed", &s.failed.to_string())],
|
||||
),
|
||||
Some(s) if s.cancelled => ngettext_f(
|
||||
"Stopped after {n} photo.",
|
||||
"Stopped after {n} photos.",
|
||||
s.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
Some(s) => ngettext_f(
|
||||
"{n} photo added to Proton Photos.",
|
||||
"{n} photos added to Proton Photos.",
|
||||
s.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
None => gettext("The Google Photos import has finished."),
|
||||
};
|
||||
notify("takeout-import", &headline, &body);
|
||||
}
|
||||
}
|
||||
ui.takeout.progress_group.set_title(&if dry {
|
||||
gettext("Scan in progress")
|
||||
} else {
|
||||
gettext("Import in progress")
|
||||
});
|
||||
paint_progress(&ui, progress.as_ref());
|
||||
} else if was_running && let Some(summary) = &summary {
|
||||
announce_end(&ui, summary);
|
||||
}
|
||||
if was_running != running {
|
||||
sync_takeout_actions(&ui);
|
||||
}
|
||||
match summary {
|
||||
Some(summary) => repaint_summary(&ui, &summary),
|
||||
match &summary {
|
||||
Some(summary) => repaint_summary(&ui, summary, running),
|
||||
None => ui.takeout.summary_group.set_visible(false),
|
||||
}
|
||||
paint_resume(&ui, running, summary.as_ref(), &resumable);
|
||||
});
|
||||
}
|
||||
|
||||
/// Paint the last run's report.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
/// Tell the user how a run ended. An import runs for hours, so its ending is
|
||||
/// announced rather than left to be discovered: a toast for whoever is looking
|
||||
/// at the window, and a desktop notification for whoever is not.
|
||||
fn announce_end(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
if summary.dry_run {
|
||||
toast(ui, &gettext("Scan finished"));
|
||||
return;
|
||||
}
|
||||
if let Some(error) = &summary.error {
|
||||
let kind = summary.error_kind.unwrap_or_default();
|
||||
let headline = gettext("Couldn't import from Google Photos");
|
||||
toast_failure(ui, &headline, error, kind);
|
||||
notify(
|
||||
"takeout-import",
|
||||
&headline,
|
||||
&gettext("You can resume it once the problem is solved."),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let (headline, body) = if summary.interrupted {
|
||||
(
|
||||
gettext("Import interrupted"),
|
||||
gettext("You can resume it where it stopped."),
|
||||
)
|
||||
} else if summary.cancelled {
|
||||
(
|
||||
gettext("Import stopped"),
|
||||
ngettext_f(
|
||||
"Stopped after {n} photo.",
|
||||
"Stopped after {n} photos.",
|
||||
summary.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
)
|
||||
} else if summary.failed > 0 {
|
||||
(
|
||||
gettext("Import finished"),
|
||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||
ngettext_f(
|
||||
"{n} photo added, {failed} failed.",
|
||||
"{n} photos added, {failed} failed.",
|
||||
summary.uploaded as u64,
|
||||
&[("failed", &summary.failed.to_string())],
|
||||
),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
gettext("Import finished"),
|
||||
ngettext_f(
|
||||
"{n} photo added to Proton Photos.",
|
||||
"{n} photos added to Proton Photos.",
|
||||
summary.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
)
|
||||
};
|
||||
toast(ui, &headline);
|
||||
notify("takeout-import", &headline, &body);
|
||||
|
||||
// A complete import is done with its archives; keeping them staged would
|
||||
// only invite running the same export again.
|
||||
if summary.complete() {
|
||||
ui.takeout.archives.borrow_mut().clear();
|
||||
repaint_archives(ui);
|
||||
}
|
||||
}
|
||||
|
||||
/// Paint where a running import is.
|
||||
fn paint_progress(ui: &Rc<Ui>, progress: Option<&ImportProgress>) {
|
||||
let Some(progress) = progress else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
return;
|
||||
};
|
||||
let done = progress.done.to_string();
|
||||
let total = progress.total.to_string();
|
||||
let current = progress.current.clone().unwrap_or_default();
|
||||
let counted = progress.total > 0;
|
||||
let args = [
|
||||
("done", done.as_str()),
|
||||
("total", total.as_str()),
|
||||
("name", current.as_str()),
|
||||
];
|
||||
let text = match (progress.phase, counted) {
|
||||
(ImportPhase::Scanning, _) => gettext("Reading the archives…"),
|
||||
(ImportPhase::Checking, false) => gettext("Checking for photos you already have…"),
|
||||
// Translators: {done} and {total} are counts of photos.
|
||||
(ImportPhase::Checking, true) => gettext_f(
|
||||
"Checking for photos you already have — {done} of {total}",
|
||||
&args,
|
||||
),
|
||||
(ImportPhase::Uploading, false) => gettext("Uploading…"),
|
||||
(ImportPhase::Uploading, true) if current.is_empty() => {
|
||||
// Translators: {done} and {total} are counts of photos.
|
||||
gettext_f("Uploading {done} of {total}", &args)
|
||||
}
|
||||
// Translators: {done} and {total} are counts of photos; {name} is the file being uploaded.
|
||||
(ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args),
|
||||
(ImportPhase::Albums, false) => gettext("Sorting photos into albums…"),
|
||||
// Translators: {done} and {total} are counts of albums.
|
||||
(ImportPhase::Albums, true) => {
|
||||
gettext_f("Sorting photos into albums — {done} of {total}", &args)
|
||||
}
|
||||
};
|
||||
ui.takeout.progress_label.set_text(&text);
|
||||
if counted {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((progress.done as f64 / progress.total as f64).min(1.0));
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
}
|
||||
}
|
||||
|
||||
/// Offer to resume the last run when the service says it can be.
|
||||
fn paint_resume(ui: &Rc<Ui>, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) {
|
||||
let offer = !running && !resumable.is_empty();
|
||||
ui.takeout.resume_group.set_visible(offer);
|
||||
if !offer {
|
||||
return;
|
||||
}
|
||||
let uploaded = summary.map_or(0, |s| s.uploaded) as u64;
|
||||
// Translators: {n} is the number of photos uploaded before the import stopped.
|
||||
ui.takeout.resume_row.set_subtitle(&ngettext_f(
|
||||
"{n} photo uploaded before it stopped. Resuming skips what is already there.",
|
||||
"{n} photos uploaded before it stopped. Resuming skips what is already there.",
|
||||
uploaded,
|
||||
&[],
|
||||
));
|
||||
}
|
||||
|
||||
/// Paint the counts of a running import, or the report of the last one.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
|
||||
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
||||
ui.takeout.summary_group.remove(&row);
|
||||
}
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
ui.takeout.summary_group.set_title(&if dry {
|
||||
let dry = summary.dry_run;
|
||||
ui.takeout.summary_group.set_title(&if running {
|
||||
gettext("So far")
|
||||
} else if dry {
|
||||
gettext("Scan result")
|
||||
} else {
|
||||
gettext("Last import")
|
||||
});
|
||||
ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error {
|
||||
// The reason is the service's own text, which is not translated. A
|
||||
// group's description is markup, and the reason may name a path.
|
||||
glib::markup_escape_text(&gettext_f(
|
||||
"{what}: {detail}",
|
||||
&[
|
||||
("what", &gettext("Couldn't import from Google Photos")),
|
||||
("detail", error),
|
||||
],
|
||||
))
|
||||
.to_string()
|
||||
let description = if running {
|
||||
None
|
||||
} else if let Some(error) = &summary.error {
|
||||
// The reason is the service's own text, which is not translated; the
|
||||
// headline in front of it is. A group's description is markup, and the
|
||||
// reason may name a path.
|
||||
let what = gettext("Couldn't import from Google Photos");
|
||||
Some(
|
||||
glib::markup_escape_text(&gettext_f(
|
||||
"{what}: {detail}",
|
||||
&[
|
||||
(
|
||||
"what",
|
||||
error_headline(summary.error_kind.unwrap_or_default(), &what),
|
||||
),
|
||||
("detail", error),
|
||||
],
|
||||
))
|
||||
.to_string(),
|
||||
)
|
||||
} else if dry {
|
||||
gettext("Nothing was uploaded — this is what an import would do.")
|
||||
Some(gettext(
|
||||
"Nothing was uploaded — this is what an import would do.",
|
||||
))
|
||||
} else if summary.interrupted {
|
||||
Some(gettext(
|
||||
"The Proton Drive service stopped while the import ran. Resume it to continue where it left off.",
|
||||
))
|
||||
} else if summary.cancelled {
|
||||
gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.")
|
||||
Some(gettext(
|
||||
"Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.",
|
||||
))
|
||||
} else if !summary.complete() {
|
||||
Some(gettext(
|
||||
"Finished with problems. Run it again to retry what didn't upload.",
|
||||
))
|
||||
} else {
|
||||
gettext("Finished.")
|
||||
}));
|
||||
Some(gettext("Finished."))
|
||||
};
|
||||
ui.takeout
|
||||
.summary_group
|
||||
.set_description(description.as_deref());
|
||||
|
||||
// Only lines that carry information: a run with no failures should not have
|
||||
// to show a "0 failed" row for the user to read past.
|
||||
let mut lines: Vec<(String, String)> = vec![
|
||||
(gettext("Photos in the export"), summary.found.to_string()),
|
||||
(
|
||||
if dry {
|
||||
gettext("Would upload")
|
||||
} else {
|
||||
gettext("Uploaded")
|
||||
},
|
||||
summary.uploaded.to_string(),
|
||||
),
|
||||
];
|
||||
let mut lines: Vec<(String, String)> =
|
||||
vec![(gettext("Photos in the export"), summary.found.to_string())];
|
||||
if dry {
|
||||
lines.push((gettext("Would upload"), summary.to_upload.to_string()));
|
||||
} else {
|
||||
lines.push((gettext("Uploaded"), summary.uploaded.to_string()));
|
||||
}
|
||||
if summary.duplicates > 0 {
|
||||
lines.push((
|
||||
gettext("Already in Proton Photos"),
|
||||
summary.duplicates.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.live_photos > 0 {
|
||||
lines.push((
|
||||
gettext("Live photos kept together"),
|
||||
summary.live_photos.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
gettext("Albums to create")
|
||||
} else {
|
||||
gettext("Albums created")
|
||||
},
|
||||
gettext("Albums created"),
|
||||
summary.albums_created.to_string(),
|
||||
));
|
||||
}
|
||||
@@ -767,25 +961,31 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
summary.skipped_trashed.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
if dry && summary.to_upload_bytes > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
gettext("Data to upload")
|
||||
} else {
|
||||
gettext("Data uploaded")
|
||||
},
|
||||
human_bytes(summary.bytes),
|
||||
gettext("Data to upload"),
|
||||
human_bytes(summary.to_upload_bytes),
|
||||
));
|
||||
} else if summary.bytes > 0 {
|
||||
lines.push((gettext("Data uploaded"), human_bytes(summary.bytes)));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
|
||||
}
|
||||
for path in &summary.unreadable_archives {
|
||||
let name = Path::new(path)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| path.clone());
|
||||
lines.push((gettext("Couldn't read this archive"), name));
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (title, value) in lines {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(title)
|
||||
.subtitle(value)
|
||||
.use_markup(false)
|
||||
.build();
|
||||
row.add_css_class("property");
|
||||
ui.takeout.summary_group.add(&row);
|
||||
@@ -794,35 +994,3 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
*ui.takeout.summary_rows.borrow_mut() = rows;
|
||||
ui.takeout.summary_group.set_visible(true);
|
||||
}
|
||||
|
||||
/// Paint the import's progress from the transfer snapshot the daemon pushed,
|
||||
/// so this page costs no extra request. A job with no known total
|
||||
/// pulses instead of showing a fraction, the same as the Activity list.
|
||||
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
|
||||
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
|
||||
return;
|
||||
};
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
if job.total > 0 {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
|
||||
let done = job.done.to_string();
|
||||
let total = job.total.to_string();
|
||||
// Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on.
|
||||
let text = gettext_f(
|
||||
"{done} of {total} — {detail}",
|
||||
&[("done", &done), ("total", &total), ("detail", &job.detail)],
|
||||
);
|
||||
ui.takeout.progress_label.set_text(&text);
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&if job.detail.is_empty() {
|
||||
gettext("Working…")
|
||||
} else {
|
||||
job.detail.clone()
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user