Compare commits

...
Author SHA1 Message Date
nvrl aa26a8e1aa Merge branch 'main' into perf/takeout-parallel 2026-11-21 23:07:08 +01:00
nvrl 9027a1465f perf(fuse): upload two Takeout photos at once
The import uploaded one photo at a time, leaving the link idle while each
upload waited on Proton between blocks. Staged photos now go to two upload
workers while the import thread reads the next one out of its archive. A
round ends only when its uploads have landed, so a live pair's video still
finds its still's uid. Photos without a thumbnail still stream straight from
the archive on the import thread. Warnings no longer name the photo.
2026-11-21 23:04:28 +01:00
nvrl 66b634e4a1 Merge pull request #120 from narrrl/chore/sdk-0.12.4
chore(deps): proton-sdk 0.12.4
2026-11-21 22:51:29 +01:00
nvrl ae46e1e544 chore(deps): proton-sdk 0.12.4 2026-11-21 22:49:27 +01:00
5 changed files with 233 additions and 91 deletions

No files matched your search

Generated
+4 -4
View File
@@ -3952,9 +3952,9 @@ dependencies = [
[[package]] [[package]]
name = "proton-drive-rs" name = "proton-drive-rs"
version = "0.12.3" version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9f7e67c864033e2d13ff48f9e0738f22ae6401cc47131b266b90c23e0dcc381" checksum = "670bfcf0eeb15886c962c18c76dbac976236e4377bfe073eb2360dc1ce448da2"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64 0.23.1", "base64 0.23.1",
@@ -4060,9 +4060,9 @@ dependencies = [
[[package]] [[package]]
name = "proton-sdk" name = "proton-sdk"
version = "0.12.3" version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a443d600225687a92724b455c025a049b8668fe9da52fbf7cb46583a8e4970a7" checksum = "f578a9d920cef3f6c4da611793001fe6d2ed81face13c45f5e721274b1945f7d"
dependencies = [ dependencies = [
"aes-gcm 0.11.1", "aes-gcm 0.11.1",
"async-trait", "async-trait",
+2 -2
View File
@@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
categories = ["filesystem", "command-line-utilities"] categories = ["filesystem", "command-line-utilities"]
[workspace.dependencies] [workspace.dependencies]
proton-sdk = { version = "0.12.3" } proton-sdk = { version = "0.12.4" }
proton-drive-rs = { version = "0.12.3" } proton-drive-rs = { version = "0.12.4" }
proton-gallery-app-rs = "0.1.0" proton-gallery-app-rs = "0.1.0"
proton-gallery-rs = "0.1.5" proton-gallery-rs = "0.1.5"
chrono = { version = "0.4", default-features = false, features = ["std"] } chrono = { version = "0.4", default-features = false, features = ["std"] }
+221 -84
View File
@@ -256,6 +256,62 @@ impl Reporter {
} }
} }
/// Photos uploaded at once. Each upload waits on Proton between its blocks,
/// so two keep the link busy where one leaves it idle; more would mostly
/// stage more copies on disk.
const IMPORT_UPLOADS: usize = 2;
/// Where an upload reads the photo from.
enum Source<'a> {
/// A copy staged on disk, which a thumbnail can be made from.
Staged(&'a Path),
/// Straight out of the open archive.
Archive(&'a mut zip::ZipArchive<std::io::BufReader<std::fs::File>>),
}
/// A staged photo handed to an upload worker.
struct Upload {
index: usize,
path: PathBuf,
photo: TakeoutPhoto,
tags: Vec<PhotoTag>,
main_uid: Option<NodeUid>,
}
impl Upload {
fn landed(self, result: CoreResult<NodeUid>) -> Landed {
Landed {
index: self.index,
live: self.main_uid.is_some(),
photo: self.photo,
result,
}
}
fn streamed(
index: usize,
photo: TakeoutPhoto,
main_uid: Option<NodeUid>,
result: CoreResult<NodeUid>,
) -> Landed {
Landed {
index,
live: main_uid.is_some(),
photo,
result,
}
}
}
/// A finished upload, to be counted on the import thread.
struct Landed {
index: usize,
/// The video of a live pair, filed with its still.
live: bool,
photo: TakeoutPhoto,
result: CoreResult<NodeUid>,
}
/// How a planned photo relates to a live or motion photo pair. /// How a planned photo relates to a live or motion photo pair.
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LiveRole { enum LiveRole {
@@ -714,72 +770,133 @@ impl Core {
} }
} }
for (archive_index, indices) in by_archive { // Staged photos go to IMPORT_UPLOADS workers while this thread
if self.import_should_stop(cancel) { // reads the next one out of its archive. A round ends only when
stopped = true; // every upload in it has landed, so a pair's video always finds
break; // its still's uid.
} let (jobs, queue) = std::sync::mpsc::sync_channel::<Upload>(0);
let mut archive = match open_archive(archives, archive_index) { let queue = std::sync::Mutex::new(queue);
Ok(archive) => archive, std::thread::scope(|scope| {
Err(e) => { let (landed_tx, landed) = std::sync::mpsc::channel::<Landed>();
// The archive was readable at scan time; whatever broke for _ in 0..IMPORT_UPLOADS {
// it since costs its own photos, not the whole run. let queue = &queue;
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive"); let landed_tx = landed_tx.clone();
summary.failed += indices.len(); scope.spawn(move || {
for _ in &indices { loop {
job.step(); let next = queue.lock().map(|queue| queue.recv());
report.step(); let Ok(Ok(upload)) = next else { break };
let result = self.upload_takeout_photo(
Source::Staged(&upload.path),
&upload.photo,
upload.tags.clone(),
upload.main_uid.clone(),
);
let _ = std::fs::remove_file(&upload.path);
if landed_tx.send(upload.landed(result)).is_err() {
break;
}
} }
report.counts(summary); });
continue; }
} drop(landed_tx);
};
for index in indices { 'archives: for (archive_index, indices) in by_archive {
if self.import_should_stop(cancel) { if self.import_should_stop(cancel) {
stopped = true; stopped = true;
break; break;
} }
let main_uid = match planned[index].live { let mut archive = match open_archive(archives, archive_index) {
Some(LiveRole::Video(still)) => planned[still].uid.clone(), Ok(archive) => archive,
_ => None,
};
let photo = planned[index].photo.clone();
job.detail(photo.name.clone());
report.current(&photo.name);
let uploaded =
self.upload_takeout_photo(&mut archive, &planned[index], main_uid.clone());
job.step();
report.step();
match uploaded {
Ok(uid) => {
self.note_self_change(&uid);
summary.uploaded += 1;
summary.bytes += photo.size;
if main_uid.is_some() {
summary.live_photos += 1;
}
if photo.favorite {
favorites.push(uid.clone());
}
planned[index].uid = Some(uid);
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
self.spawn_timeline_refresh();
}
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
report.counts(summary);
report.checkpoint();
}
}
Err(e) => { Err(e) => {
summary.failed += 1; // The archive was readable at scan time; whatever
warn!(name = photo.name, error = %e, "takeout photo upload failed"); // broke it since costs its own photos, not the run.
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive");
summary.failed += indices.len();
for _ in &indices {
job.step();
report.step();
}
report.counts(summary);
continue;
}
};
for index in indices {
for done in landed.try_iter() {
self.record_upload(planned, done, summary, &mut favorites, job, report);
}
if self.import_should_stop(cancel) {
stopped = true;
break 'archives;
}
let item = &planned[index];
let main_uid = match item.live {
Some(LiveRole::Video(still)) => planned[still].uid.clone(),
_ => None,
};
let tags = match item.live {
Some(LiveRole::Still(tag)) => vec![tag],
_ => Vec::new(),
};
let photo = item.photo.clone();
job.detail(photo.name.clone());
report.current(&photo.name);
// A thumbnail is made from a file, so a photo that can
// have one is copied out of the archive and uploaded
// from the copy by a worker. Without one, Proton's apps
// show a blank tile until they make their own.
let staged = can_make_thumbnail(&photo.name, photo.size)
.then(|| self.stage_takeout_entry(&mut archive, &photo.entry))
.and_then(|staged| {
staged
.inspect_err(|e| {
warn!(error = %e,
"copying a takeout photo out failed; uploading it without a thumbnail")
})
.ok()
});
match staged {
Some(path) => {
let upload = Upload {
index,
path,
photo,
tags,
main_uid,
};
if let Err(std::sync::mpsc::SendError(upload)) = jobs.send(upload) {
let _ = std::fs::remove_file(&upload.path);
}
}
None => {
let result = self.upload_takeout_photo(
Source::Archive(&mut archive),
&photo,
tags,
main_uid.clone(),
);
let done = Upload::streamed(index, photo, main_uid, result);
self.record_upload(
planned,
done,
summary,
&mut favorites,
job,
report,
);
}
} }
} }
report.counts(summary);
} }
drop(jobs);
for done in landed.iter() {
self.record_upload(planned, done, summary, &mut favorites, job, report);
}
});
if stopped {
break;
} }
} }
@@ -787,14 +904,55 @@ impl Core {
stopped stopped
} }
/// Upload one planned photo out of its open archive. /// Count one finished upload and file its uid in the plan.
fn record_upload(
&self,
planned: &mut [Planned],
done: Landed,
summary: &mut ImportSummary,
favorites: &mut Vec<NodeUid>,
job: &crate::transfers::JobGuard,
report: &Reporter,
) {
job.step();
report.step();
match done.result {
Ok(uid) => {
self.note_self_change(&uid);
summary.uploaded += 1;
summary.bytes += done.photo.size;
if done.live {
summary.live_photos += 1;
}
if done.photo.favorite {
favorites.push(uid.clone());
}
planned[done.index].uid = Some(uid);
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
self.spawn_timeline_refresh();
}
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
report.counts(summary);
report.checkpoint();
}
}
Err(e) => {
summary.failed += 1;
warn!(error = %e, "takeout photo upload failed");
}
}
report.counts(summary);
}
/// Upload one planned photo, from its staged copy or straight out of its
/// open archive.
fn upload_takeout_photo( fn upload_takeout_photo(
&self, &self,
archive: &mut zip::ZipArchive<std::io::BufReader<std::fs::File>>, source: Source<'_>,
item: &Planned, photo: &TakeoutPhoto,
tags: Vec<PhotoTag>,
main_uid: Option<NodeUid>, main_uid: Option<NodeUid>,
) -> CoreResult<NodeUid> { ) -> CoreResult<NodeUid> {
let photo = &item.photo;
let photos = self.photos(); let photos = self.photos();
let guard = self.transfers.begin( let guard = self.transfers.begin(
photo.name.clone(), photo.name.clone(),
@@ -802,10 +960,6 @@ impl Core {
TransferDirection::Upload, TransferDirection::Upload,
photo.size, photo.size,
); );
let tags = match item.live {
Some(LiveRole::Still(tag)) => vec![tag],
_ => Vec::new(),
};
let metadata = |capture_time: Option<i64>| PhotoUploadMetadata { let metadata = |capture_time: Option<i64>| PhotoUploadMetadata {
capture_time, capture_time,
last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time), last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time),
@@ -820,21 +974,8 @@ impl Core {
replace_existing_draft: true, replace_existing_draft: true,
}; };
// A thumbnail is made from a file, so a photo that can have one is match source {
// copied out of the archive first and uploaded from the copy. Without Source::Staged(path) => {
// one, Proton's apps show a blank tile until they make their own.
let staged = can_make_thumbnail(&photo.name, photo.size)
.then(|| self.stage_takeout_entry(archive, &photo.entry))
.and_then(|staged| {
staged
.inspect_err(|e| {
warn!(name = photo.name, error = %e,
"copying a takeout photo out failed; uploading it without a thumbnail")
})
.ok()
});
let uploaded = match &staged {
Some(path) => {
let thumbnails = self.upload_thumbnails(path, &photo.name); let thumbnails = self.upload_thumbnails(path, &photo.name);
// No sidecar date and none in the name: the photo's own EXIF, // No sidecar date and none in the name: the photo's own EXIF,
// then when Google received it, before Drive files it under // then when Google received it, before Drive files it under
@@ -860,7 +1001,7 @@ impl Core {
.map_err(|e| CoreError::from_api(&e, "upload")) .map_err(|e| CoreError::from_api(&e, "upload"))
}) })
} }
None => read_entry(archive, &photo.entry, |reader| { Source::Archive(archive) => read_entry(archive, &photo.entry, |reader| {
let counting = CountingReader::new(reader, &guard); let counting = CountingReader::new(reader, &guard);
self.rt self.rt
.block_on(photos.upload_photo_from( .block_on(photos.upload_photo_from(
@@ -875,11 +1016,7 @@ impl Core {
.map_err(|e| CoreError::from_api(&e, "upload")) .map_err(|e| CoreError::from_api(&e, "upload"))
}) })
.and_then(|result| result), .and_then(|result| result),
};
if let Some(path) = staged {
let _ = std::fs::remove_file(path);
} }
uploaded
} }
/// Mark photos as favorites. A separate call: the upload seal takes /// Mark photos as favorites. A separate call: the upload seal takes
+5
View File
@@ -13,6 +13,11 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
### Changed ### Changed
- A Google Photos import uploads two photos at once instead of one, while the next
photo is read out of its archive. Live-photo videos still go after their stills, and
cancelling still lets the uploads in flight finish.
- proton-sdk 0.12.4: thumbnails need fewer requests to Proton (0.12.3), and the SDK
can list photos that have no thumbnail (0.12.4).
- Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton - Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton
Gallery and Proton Stream use too. The stored session, the lock/unlock choice for two-password Gallery and Proton Stream use too. The stored session, the lock/unlock choice for two-password
accounts, the three mailbox-password tries and the messages stay as they were. accounts, the three mailbox-password tries and the messages stay as they were.
+1 -1
View File
@@ -63,7 +63,7 @@ them along.
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK | | Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
| Photo name search | Not started | Photos search by date and filters only | | Photo name search | Not started | Photos search by date and filters only |
| Automatic photo upload from a folder | Not started | | | Automatic photo upload from a folder | Not started | |
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos | | Faster Google Photos import | Partly done | Two uploads run at once. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions | | More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content | | Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |