diff --git a/crates/pdfs-fuse/src/takeout.rs b/crates/pdfs-fuse/src/takeout.rs index 365dece..2595ee4 100644 --- a/crates/pdfs-fuse/src/takeout.rs +++ b/crates/pdfs-fuse/src/takeout.rs @@ -256,6 +256,62 @@ impl Reporter { } } +/// Photos uploaded at once. Each upload waits on Proton between its blocks, +/// so two keep the link busy where one leaves it idle; more would mostly +/// stage more copies on disk. +const IMPORT_UPLOADS: usize = 2; + +/// Where an upload reads the photo from. +enum Source<'a> { + /// A copy staged on disk, which a thumbnail can be made from. + Staged(&'a Path), + /// Straight out of the open archive. + Archive(&'a mut zip::ZipArchive>), +} + +/// A staged photo handed to an upload worker. +struct Upload { + index: usize, + path: PathBuf, + photo: TakeoutPhoto, + tags: Vec, + main_uid: Option, +} + +impl Upload { + fn landed(self, result: CoreResult) -> Landed { + Landed { + index: self.index, + live: self.main_uid.is_some(), + photo: self.photo, + result, + } + } + + fn streamed( + index: usize, + photo: TakeoutPhoto, + main_uid: Option, + result: CoreResult, + ) -> Landed { + Landed { + index, + live: main_uid.is_some(), + photo, + result, + } + } +} + +/// A finished upload, to be counted on the import thread. +struct Landed { + index: usize, + /// The video of a live pair, filed with its still. + live: bool, + photo: TakeoutPhoto, + result: CoreResult, +} + /// How a planned photo relates to a live or motion photo pair. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LiveRole { @@ -714,72 +770,133 @@ impl Core { } } - for (archive_index, indices) in by_archive { - if self.import_should_stop(cancel) { - stopped = true; - break; - } - let mut archive = match open_archive(archives, archive_index) { - Ok(archive) => archive, - Err(e) => { - // The archive was readable at scan time; whatever broke - // it since costs its own photos, not the whole run. - warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive"); - summary.failed += indices.len(); - for _ in &indices { - job.step(); - report.step(); + // Staged photos go to IMPORT_UPLOADS workers while this thread + // reads the next one out of its archive. A round ends only when + // every upload in it has landed, so a pair's video always finds + // its still's uid. + let (jobs, queue) = std::sync::mpsc::sync_channel::(0); + let queue = std::sync::Mutex::new(queue); + std::thread::scope(|scope| { + let (landed_tx, landed) = std::sync::mpsc::channel::(); + for _ in 0..IMPORT_UPLOADS { + let queue = &queue; + let landed_tx = landed_tx.clone(); + scope.spawn(move || { + loop { + let next = queue.lock().map(|queue| queue.recv()); + let Ok(Ok(upload)) = next else { break }; + let result = self.upload_takeout_photo( + Source::Staged(&upload.path), + &upload.photo, + upload.tags.clone(), + upload.main_uid.clone(), + ); + let _ = std::fs::remove_file(&upload.path); + if landed_tx.send(upload.landed(result)).is_err() { + break; + } } - report.counts(summary); - continue; - } - }; + }); + } + drop(landed_tx); - for index in indices { + 'archives: for (archive_index, indices) in by_archive { if self.import_should_stop(cancel) { stopped = true; break; } - let main_uid = match planned[index].live { - Some(LiveRole::Video(still)) => planned[still].uid.clone(), - _ => None, - }; - let photo = planned[index].photo.clone(); - job.detail(photo.name.clone()); - report.current(&photo.name); - - let uploaded = - self.upload_takeout_photo(&mut archive, &planned[index], main_uid.clone()); - job.step(); - report.step(); - - match uploaded { - Ok(uid) => { - self.note_self_change(&uid); - summary.uploaded += 1; - summary.bytes += photo.size; - if main_uid.is_some() { - summary.live_photos += 1; - } - if photo.favorite { - favorites.push(uid.clone()); - } - planned[index].uid = Some(uid); - if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) { - self.spawn_timeline_refresh(); - } - if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) { - report.counts(summary); - report.checkpoint(); - } - } + let mut archive = match open_archive(archives, archive_index) { + Ok(archive) => archive, Err(e) => { - summary.failed += 1; - warn!(name = photo.name, error = %e, "takeout photo upload failed"); + // The archive was readable at scan time; whatever + // broke it since costs its own photos, not the run. + warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive"); + summary.failed += indices.len(); + for _ in &indices { + job.step(); + report.step(); + } + report.counts(summary); + continue; + } + }; + + for index in indices { + for done in landed.try_iter() { + self.record_upload(planned, done, summary, &mut favorites, job, report); + } + if self.import_should_stop(cancel) { + stopped = true; + break 'archives; + } + let item = &planned[index]; + let main_uid = match item.live { + Some(LiveRole::Video(still)) => planned[still].uid.clone(), + _ => None, + }; + let tags = match item.live { + Some(LiveRole::Still(tag)) => vec![tag], + _ => Vec::new(), + }; + let photo = item.photo.clone(); + job.detail(photo.name.clone()); + report.current(&photo.name); + + // A thumbnail is made from a file, so a photo that can + // have one is copied out of the archive and uploaded + // from the copy by a worker. Without one, Proton's apps + // show a blank tile until they make their own. + let staged = can_make_thumbnail(&photo.name, photo.size) + .then(|| self.stage_takeout_entry(&mut archive, &photo.entry)) + .and_then(|staged| { + staged + .inspect_err(|e| { + warn!(error = %e, + "copying a takeout photo out failed; uploading it without a thumbnail") + }) + .ok() + }); + match staged { + Some(path) => { + let upload = Upload { + index, + path, + photo, + tags, + main_uid, + }; + if let Err(std::sync::mpsc::SendError(upload)) = jobs.send(upload) { + let _ = std::fs::remove_file(&upload.path); + } + } + None => { + let result = self.upload_takeout_photo( + Source::Archive(&mut archive), + &photo, + tags, + main_uid.clone(), + ); + let done = Upload::streamed(index, photo, main_uid, result); + self.record_upload( + planned, + done, + summary, + &mut favorites, + job, + report, + ); + } } } - report.counts(summary); } + + drop(jobs); + for done in landed.iter() { + self.record_upload(planned, done, summary, &mut favorites, job, report); + } + }); + if stopped { + break; } } @@ -787,14 +904,55 @@ impl Core { stopped } - /// Upload one planned photo out of its open archive. + /// Count one finished upload and file its uid in the plan. + fn record_upload( + &self, + planned: &mut [Planned], + done: Landed, + summary: &mut ImportSummary, + favorites: &mut Vec, + job: &crate::transfers::JobGuard, + report: &Reporter, + ) { + job.step(); + report.step(); + match done.result { + Ok(uid) => { + self.note_self_change(&uid); + summary.uploaded += 1; + summary.bytes += done.photo.size; + if done.live { + summary.live_photos += 1; + } + if done.photo.favorite { + favorites.push(uid.clone()); + } + planned[done.index].uid = Some(uid); + if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) { + self.spawn_timeline_refresh(); + } + if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) { + report.counts(summary); + report.checkpoint(); + } + } + Err(e) => { + summary.failed += 1; + warn!(error = %e, "takeout photo upload failed"); + } + } + report.counts(summary); + } + + /// Upload one planned photo, from its staged copy or straight out of its + /// open archive. fn upload_takeout_photo( &self, - archive: &mut zip::ZipArchive>, - item: &Planned, + source: Source<'_>, + photo: &TakeoutPhoto, + tags: Vec, main_uid: Option, ) -> CoreResult { - let photo = &item.photo; let photos = self.photos(); let guard = self.transfers.begin( photo.name.clone(), @@ -802,10 +960,6 @@ impl Core { TransferDirection::Upload, photo.size, ); - let tags = match item.live { - Some(LiveRole::Still(tag)) => vec![tag], - _ => Vec::new(), - }; let metadata = |capture_time: Option| PhotoUploadMetadata { capture_time, last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time), @@ -820,21 +974,8 @@ impl Core { replace_existing_draft: true, }; - // A thumbnail is made from a file, so a photo that can have one is - // copied out of the archive first and uploaded from the copy. Without - // one, Proton's apps show a blank tile until they make their own. - let staged = can_make_thumbnail(&photo.name, photo.size) - .then(|| self.stage_takeout_entry(archive, &photo.entry)) - .and_then(|staged| { - staged - .inspect_err(|e| { - warn!(name = photo.name, error = %e, - "copying a takeout photo out failed; uploading it without a thumbnail") - }) - .ok() - }); - let uploaded = match &staged { - Some(path) => { + match source { + Source::Staged(path) => { let thumbnails = self.upload_thumbnails(path, &photo.name); // No sidecar date and none in the name: the photo's own EXIF, // then when Google received it, before Drive files it under @@ -860,7 +1001,7 @@ impl Core { .map_err(|e| CoreError::from_api(&e, "upload")) }) } - None => read_entry(archive, &photo.entry, |reader| { + Source::Archive(archive) => read_entry(archive, &photo.entry, |reader| { let counting = CountingReader::new(reader, &guard); self.rt .block_on(photos.upload_photo_from( @@ -875,11 +1016,7 @@ impl Core { .map_err(|e| CoreError::from_api(&e, "upload")) }) .and_then(|result| result), - }; - if let Some(path) = staged { - let _ = std::fs::remove_file(path); } - uploaded } /// Mark photos as favorites. A separate call: the upload seal takes diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4520a47..d1f554f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,9 @@ scratch (user data in `staging/` and `recovery/` is never touched by this). ### Changed +- A Google Photos import uploads two photos at once instead of one, while the next + photo is read out of its archive. Live-photo videos still go after their stills, and + cancelling still lets the uploads in flight finish. - proton-sdk 0.12.4: thumbnails need fewer requests to Proton (0.12.3), and the SDK can list photos that have no thumbnail (0.12.4). - Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 3dd278b..157847f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -63,7 +63,7 @@ them along. | Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK | | Photo name search | Not started | Photos search by date and filters only | | Automatic photo upload from a folder | Not started | | -| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos | +| Faster Google Photos import | Partly done | Two uploads run at once. Trash folders of locales missing from `TRASH_FOLDERS` import as photos | | More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions | | Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |