Compare commits

...
Author SHA1 Message Date
nvrl aa26a8e1aa Merge branch 'main' into perf/takeout-parallel 2026-11-21 23:07:08 +01:00
nvrl 9027a1465f perf(fuse): upload two Takeout photos at once
The import uploaded one photo at a time, leaving the link idle while each
upload waited on Proton between blocks. Staged photos now go to two upload
workers while the import thread reads the next one out of its archive. A
round ends only when its uploads have landed, so a live pair's video still
finds its still's uid. Photos without a thumbnail still stream straight from
the archive on the import thread. Warnings no longer name the photo.
2026-11-21 23:04:28 +01:00
nvrl 66b634e4a1 Merge pull request #120 from narrrl/chore/sdk-0.12.4
chore(deps): proton-sdk 0.12.4
2026-11-21 22:51:29 +01:00
nvrl ae46e1e544 chore(deps): proton-sdk 0.12.4 2026-11-21 22:49:27 +01:00
nvrl 2c4c95812f Merge pull request #119 from narrrl/chore/sdk-0.12.3
chore(deps): proton-sdk 0.12.3 and proton-gallery-rs 0.1.5
2026-11-21 17:12:11 +01:00
nvrl 2bf1622793 chore(deps): proton-sdk 0.12.3 and proton-gallery-rs 0.1.5
SDK 0.12.3 fetches a thumbnail with fewer round trips and caches the
thumbnail target per revision. proton-gallery-rs 0.1.5 advances the
Photos revision only when an album changed.
2026-11-21 17:08:41 +01:00
nvrl 2ea189f1d9 Merge pull request #118 from narrrl/chore/gallery-rs-0.1.3
perf(gui): cluster Places markers with proton-gallery-rs 0.1.3
2026-11-20 22:12:51 +01:00
nvrl 8af671d8ed perf(gui): cluster Places markers with proton-gallery-rs 0.1.3
The map kept its own copy of the marker clustering, identical to the one
in proton-gallery-rs. 0.1.3 makes that one fast (a grid of cells instead
of a scan of every bubble so far, same results), so the map now uses it.
2026-11-20 22:09:04 +01:00
nvrl 9047bdd741 Merge pull request #117 from narrrl/perf/photos-main-thread
perf(gui): keep Photos decoding and file access off the main thread
2026-11-20 21:48:52 +01:00
nvrl 61b992cb48 Merge pull request #115 from narrrl/fix/drive-gallery-header
fix(gui): give Drive galleries room under their header
2026-11-20 20:04:38 +01:00
6 changed files with 239 additions and 158 deletions

No files matched your search

Generated
+7 -6
View File
@@ -3952,9 +3952,9 @@ dependencies = [
[[package]] [[package]]
name = "proton-drive-rs" name = "proton-drive-rs"
version = "0.12.2" version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdeb7ee7495bc34198c78d7ec569f6371ad01a88f188b5eee418b013b082626a" checksum = "670bfcf0eeb15886c962c18c76dbac976236e4377bfe073eb2360dc1ce448da2"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64 0.23.1", "base64 0.23.1",
@@ -4022,15 +4022,16 @@ dependencies = [
[[package]] [[package]]
name = "proton-gallery-rs" name = "proton-gallery-rs"
version = "0.1.1" version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc9c9bbb76a5fba4071ee90de4aada71c292c0e408024ddfb04a12869291404a" checksum = "e374d2053de5df0e6b0797df65d0d024dbce35f7aedfefd4a656896a0495dc9b"
dependencies = [ dependencies = [
"aes-gcm 0.11.1", "aes-gcm 0.11.1",
"async-trait", "async-trait",
"getrandom 0.4.3", "getrandom 0.4.3",
"hex", "hex",
"kamadak-exif", "kamadak-exif",
"proton-base-rs",
"rusqlite", "rusqlite",
"serde", "serde",
"serde_json", "serde_json",
@@ -4059,9 +4060,9 @@ dependencies = [
[[package]] [[package]]
name = "proton-sdk" name = "proton-sdk"
version = "0.12.2" version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3189873dcdf0bc0a3d0fa5619cea80594085252d1d66a57459ec6adfd1f35ac1" checksum = "f578a9d920cef3f6c4da611793001fe6d2ed81face13c45f5e721274b1945f7d"
dependencies = [ dependencies = [
"aes-gcm 0.11.1", "aes-gcm 0.11.1",
"async-trait", "async-trait",
+3 -3
View File
@@ -15,10 +15,10 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
categories = ["filesystem", "command-line-utilities"] categories = ["filesystem", "command-line-utilities"]
[workspace.dependencies] [workspace.dependencies]
proton-sdk = { version = "0.12.2" } proton-sdk = { version = "0.12.4" }
proton-drive-rs = { version = "0.12.2" } proton-drive-rs = { version = "0.12.4" }
proton-gallery-app-rs = "0.1.0" proton-gallery-app-rs = "0.1.0"
proton-gallery-rs = "0.1.1" proton-gallery-rs = "0.1.5"
chrono = { version = "0.4", default-features = false, features = ["std"] } chrono = { version = "0.4", default-features = false, features = ["std"] }
# Internal crates. # Internal crates.
+221 -84
View File
@@ -256,6 +256,62 @@ impl Reporter {
} }
} }
/// Photos uploaded at once. Each upload waits on Proton between its blocks,
/// so two keep the link busy where one leaves it idle; more would mostly
/// stage more copies on disk.
const IMPORT_UPLOADS: usize = 2;
/// Where an upload reads the photo from.
enum Source<'a> {
/// A copy staged on disk, which a thumbnail can be made from.
Staged(&'a Path),
/// Straight out of the open archive.
Archive(&'a mut zip::ZipArchive<std::io::BufReader<std::fs::File>>),
}
/// A staged photo handed to an upload worker.
struct Upload {
index: usize,
path: PathBuf,
photo: TakeoutPhoto,
tags: Vec<PhotoTag>,
main_uid: Option<NodeUid>,
}
impl Upload {
fn landed(self, result: CoreResult<NodeUid>) -> Landed {
Landed {
index: self.index,
live: self.main_uid.is_some(),
photo: self.photo,
result,
}
}
fn streamed(
index: usize,
photo: TakeoutPhoto,
main_uid: Option<NodeUid>,
result: CoreResult<NodeUid>,
) -> Landed {
Landed {
index,
live: main_uid.is_some(),
photo,
result,
}
}
}
/// A finished upload, to be counted on the import thread.
struct Landed {
index: usize,
/// The video of a live pair, filed with its still.
live: bool,
photo: TakeoutPhoto,
result: CoreResult<NodeUid>,
}
/// How a planned photo relates to a live or motion photo pair. /// How a planned photo relates to a live or motion photo pair.
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LiveRole { enum LiveRole {
@@ -714,72 +770,133 @@ impl Core {
} }
} }
for (archive_index, indices) in by_archive { // Staged photos go to IMPORT_UPLOADS workers while this thread
if self.import_should_stop(cancel) { // reads the next one out of its archive. A round ends only when
stopped = true; // every upload in it has landed, so a pair's video always finds
break; // its still's uid.
} let (jobs, queue) = std::sync::mpsc::sync_channel::<Upload>(0);
let mut archive = match open_archive(archives, archive_index) { let queue = std::sync::Mutex::new(queue);
Ok(archive) => archive, std::thread::scope(|scope| {
Err(e) => { let (landed_tx, landed) = std::sync::mpsc::channel::<Landed>();
// The archive was readable at scan time; whatever broke for _ in 0..IMPORT_UPLOADS {
// it since costs its own photos, not the whole run. let queue = &queue;
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive"); let landed_tx = landed_tx.clone();
summary.failed += indices.len(); scope.spawn(move || {
for _ in &indices { loop {
job.step(); let next = queue.lock().map(|queue| queue.recv());
report.step(); let Ok(Ok(upload)) = next else { break };
let result = self.upload_takeout_photo(
Source::Staged(&upload.path),
&upload.photo,
upload.tags.clone(),
upload.main_uid.clone(),
);
let _ = std::fs::remove_file(&upload.path);
if landed_tx.send(upload.landed(result)).is_err() {
break;
}
} }
report.counts(summary); });
continue; }
} drop(landed_tx);
};
for index in indices { 'archives: for (archive_index, indices) in by_archive {
if self.import_should_stop(cancel) { if self.import_should_stop(cancel) {
stopped = true; stopped = true;
break; break;
} }
let main_uid = match planned[index].live { let mut archive = match open_archive(archives, archive_index) {
Some(LiveRole::Video(still)) => planned[still].uid.clone(), Ok(archive) => archive,
_ => None,
};
let photo = planned[index].photo.clone();
job.detail(photo.name.clone());
report.current(&photo.name);
let uploaded =
self.upload_takeout_photo(&mut archive, &planned[index], main_uid.clone());
job.step();
report.step();
match uploaded {
Ok(uid) => {
self.note_self_change(&uid);
summary.uploaded += 1;
summary.bytes += photo.size;
if main_uid.is_some() {
summary.live_photos += 1;
}
if photo.favorite {
favorites.push(uid.clone());
}
planned[index].uid = Some(uid);
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
self.spawn_timeline_refresh();
}
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
report.counts(summary);
report.checkpoint();
}
}
Err(e) => { Err(e) => {
summary.failed += 1; // The archive was readable at scan time; whatever
warn!(name = photo.name, error = %e, "takeout photo upload failed"); // broke it since costs its own photos, not the run.
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive");
summary.failed += indices.len();
for _ in &indices {
job.step();
report.step();
}
report.counts(summary);
continue;
}
};
for index in indices {
for done in landed.try_iter() {
self.record_upload(planned, done, summary, &mut favorites, job, report);
}
if self.import_should_stop(cancel) {
stopped = true;
break 'archives;
}
let item = &planned[index];
let main_uid = match item.live {
Some(LiveRole::Video(still)) => planned[still].uid.clone(),
_ => None,
};
let tags = match item.live {
Some(LiveRole::Still(tag)) => vec![tag],
_ => Vec::new(),
};
let photo = item.photo.clone();
job.detail(photo.name.clone());
report.current(&photo.name);
// A thumbnail is made from a file, so a photo that can
// have one is copied out of the archive and uploaded
// from the copy by a worker. Without one, Proton's apps
// show a blank tile until they make their own.
let staged = can_make_thumbnail(&photo.name, photo.size)
.then(|| self.stage_takeout_entry(&mut archive, &photo.entry))
.and_then(|staged| {
staged
.inspect_err(|e| {
warn!(error = %e,
"copying a takeout photo out failed; uploading it without a thumbnail")
})
.ok()
});
match staged {
Some(path) => {
let upload = Upload {
index,
path,
photo,
tags,
main_uid,
};
if let Err(std::sync::mpsc::SendError(upload)) = jobs.send(upload) {
let _ = std::fs::remove_file(&upload.path);
}
}
None => {
let result = self.upload_takeout_photo(
Source::Archive(&mut archive),
&photo,
tags,
main_uid.clone(),
);
let done = Upload::streamed(index, photo, main_uid, result);
self.record_upload(
planned,
done,
summary,
&mut favorites,
job,
report,
);
}
} }
} }
report.counts(summary);
} }
drop(jobs);
for done in landed.iter() {
self.record_upload(planned, done, summary, &mut favorites, job, report);
}
});
if stopped {
break;
} }
} }
@@ -787,14 +904,55 @@ impl Core {
stopped stopped
} }
/// Upload one planned photo out of its open archive. /// Count one finished upload and file its uid in the plan.
fn record_upload(
&self,
planned: &mut [Planned],
done: Landed,
summary: &mut ImportSummary,
favorites: &mut Vec<NodeUid>,
job: &crate::transfers::JobGuard,
report: &Reporter,
) {
job.step();
report.step();
match done.result {
Ok(uid) => {
self.note_self_change(&uid);
summary.uploaded += 1;
summary.bytes += done.photo.size;
if done.live {
summary.live_photos += 1;
}
if done.photo.favorite {
favorites.push(uid.clone());
}
planned[done.index].uid = Some(uid);
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
self.spawn_timeline_refresh();
}
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
report.counts(summary);
report.checkpoint();
}
}
Err(e) => {
summary.failed += 1;
warn!(error = %e, "takeout photo upload failed");
}
}
report.counts(summary);
}
/// Upload one planned photo, from its staged copy or straight out of its
/// open archive.
fn upload_takeout_photo( fn upload_takeout_photo(
&self, &self,
archive: &mut zip::ZipArchive<std::io::BufReader<std::fs::File>>, source: Source<'_>,
item: &Planned, photo: &TakeoutPhoto,
tags: Vec<PhotoTag>,
main_uid: Option<NodeUid>, main_uid: Option<NodeUid>,
) -> CoreResult<NodeUid> { ) -> CoreResult<NodeUid> {
let photo = &item.photo;
let photos = self.photos(); let photos = self.photos();
let guard = self.transfers.begin( let guard = self.transfers.begin(
photo.name.clone(), photo.name.clone(),
@@ -802,10 +960,6 @@ impl Core {
TransferDirection::Upload, TransferDirection::Upload,
photo.size, photo.size,
); );
let tags = match item.live {
Some(LiveRole::Still(tag)) => vec![tag],
_ => Vec::new(),
};
let metadata = |capture_time: Option<i64>| PhotoUploadMetadata { let metadata = |capture_time: Option<i64>| PhotoUploadMetadata {
capture_time, capture_time,
last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time), last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time),
@@ -820,21 +974,8 @@ impl Core {
replace_existing_draft: true, replace_existing_draft: true,
}; };
// A thumbnail is made from a file, so a photo that can have one is match source {
// copied out of the archive first and uploaded from the copy. Without Source::Staged(path) => {
// one, Proton's apps show a blank tile until they make their own.
let staged = can_make_thumbnail(&photo.name, photo.size)
.then(|| self.stage_takeout_entry(archive, &photo.entry))
.and_then(|staged| {
staged
.inspect_err(|e| {
warn!(name = photo.name, error = %e,
"copying a takeout photo out failed; uploading it without a thumbnail")
})
.ok()
});
let uploaded = match &staged {
Some(path) => {
let thumbnails = self.upload_thumbnails(path, &photo.name); let thumbnails = self.upload_thumbnails(path, &photo.name);
// No sidecar date and none in the name: the photo's own EXIF, // No sidecar date and none in the name: the photo's own EXIF,
// then when Google received it, before Drive files it under // then when Google received it, before Drive files it under
@@ -860,7 +1001,7 @@ impl Core {
.map_err(|e| CoreError::from_api(&e, "upload")) .map_err(|e| CoreError::from_api(&e, "upload"))
}) })
} }
None => read_entry(archive, &photo.entry, |reader| { Source::Archive(archive) => read_entry(archive, &photo.entry, |reader| {
let counting = CountingReader::new(reader, &guard); let counting = CountingReader::new(reader, &guard);
self.rt self.rt
.block_on(photos.upload_photo_from( .block_on(photos.upload_photo_from(
@@ -875,11 +1016,7 @@ impl Core {
.map_err(|e| CoreError::from_api(&e, "upload")) .map_err(|e| CoreError::from_api(&e, "upload"))
}) })
.and_then(|result| result), .and_then(|result| result),
};
if let Some(path) = staged {
let _ = std::fs::remove_file(path);
} }
uploaded
} }
/// Mark photos as favorites. A separate call: the upload seal takes /// Mark photos as favorites. A separate call: the upload seal takes
+2 -64
View File
@@ -17,6 +17,8 @@
//! a single photo opens it in its town, and photos taken on one spot, which //! a single photo opens it in its town, and photos taken on one spot, which
//! never part, are listed to pick from. //! never part, are listed to pick from.
// Markers merge the same way in every Proton gallery app.
use proton_gallery_rs::places::{Bubble, bubble_radius, cluster, compact_count};
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::OnceLock; use std::sync::OnceLock;
@@ -137,13 +139,6 @@ fn parse_ring(line: &str) -> Option<Ring> {
Some(Ring { points, min, max }) Some(Ring { points, min, max })
} }
/// Radius in px of a marker holding `photos` photos: big enough to make the
/// photo out, and growing with the count, but slowly, so a home town with
/// thousands doesn't cover a continent.
fn bubble_radius(photos: usize) -> f64 {
18.0 + 2.0 * (photos.max(1) as f64).ln()
}
/// What the map shows: which part of the world, at what size. /// What the map shows: which part of the world, at what size.
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug)]
struct View { struct View {
@@ -195,17 +190,6 @@ struct Marker {
target: Target, target: Target,
} }
/// One marker as last drawn, standing for one or more [`Marker`]s: where, how
/// big, and which (indices into [`MapState::markers`], the one with the most
/// photos first).
struct Bubble {
x: f64,
y: f64,
radius: f64,
photos: usize,
members: Vec<usize>,
}
/// A street-map tile ready to draw, and the names on it. /// A street-map tile ready to draw, and the names on it.
struct CachedTile { struct CachedTile {
surface: cairo::ImageSurface, surface: cairo::ImageSurface,
@@ -1648,52 +1632,6 @@ fn spread(points: &[World]) -> f64 {
span(&mut xs.into_iter()).max(span(&mut ys.into_iter())) span(&mut xs.into_iter()).max(span(&mut ys.into_iter()))
} }
/// Merge markers whose circles would overlap on screen, the one with the
/// most photos claiming its neighbours (the first of equals, when counts
/// tie). Markers off screen are left out.
fn cluster(screen: &[(f64, f64)], counts: &[usize], width: f64, height: f64) -> Vec<Bubble> {
let mut order: Vec<usize> = (0..screen.len()).collect();
order.sort_by(|a, b| counts[*b].cmp(&counts[*a]));
let mut bubbles: Vec<Bubble> = Vec::new();
for index in order {
let (x, y) = screen[index];
let reach = bubble_radius(counts[index]);
if x < -reach || y < -reach || x > width + reach || y > height + reach {
continue;
}
match bubbles
.iter_mut()
.find(|b| (b.x - x).hypot(b.y - y) < b.radius + reach * 0.5)
{
Some(bubble) => {
bubble.photos += counts[index];
bubble.radius = bubble_radius(bubble.photos);
bubble.members.push(index);
}
None => bubbles.push(Bubble {
x,
y,
radius: reach,
photos: counts[index],
members: vec![index],
}),
}
}
// Smallest last, so a small marker next to a big one stays clickable.
bubbles.sort_by(|a, b| b.radius.total_cmp(&a.radius));
bubbles
}
/// "7", "840", "1.2k", "35k": short enough to fit a marker. Digits and the
/// "k" read the same in every language the app ships.
fn compact_count(n: usize) -> String {
match n {
0..1_000 => n.to_string(),
1_000..10_000 => format!("{:.1}k", n as f64 / 1_000.0),
_ => format!("{}k", n / 1_000),
}
}
fn set_color(cr: &cairo::Context, color: &gtk4::gdk::RGBA, alpha: f32) { fn set_color(cr: &cairo::Context, color: &gtk4::gdk::RGBA, alpha: f32) {
cr.set_source_rgba( cr.set_source_rgba(
f64::from(color.red()), f64::from(color.red()),
+5
View File
@@ -13,6 +13,11 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
### Changed ### Changed
- A Google Photos import uploads two photos at once instead of one, while the next
photo is read out of its archive. Live-photo videos still go after their stills, and
cancelling still lets the uploads in flight finish.
- proton-sdk 0.12.4: thumbnails need fewer requests to Proton (0.12.3), and the SDK
can list photos that have no thumbnail (0.12.4).
- Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton - Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton
Gallery and Proton Stream use too. The stored session, the lock/unlock choice for two-password Gallery and Proton Stream use too. The stored session, the lock/unlock choice for two-password
accounts, the three mailbox-password tries and the messages stay as they were. accounts, the three mailbox-password tries and the messages stay as they were.
+1 -1
View File
@@ -63,7 +63,7 @@ them along.
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK | | Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
| Photo name search | Not started | Photos search by date and filters only | | Photo name search | Not started | Photos search by date and filters only |
| Automatic photo upload from a folder | Not started | | | Automatic photo upload from a folder | Not started | |
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos | | Faster Google Photos import | Partly done | Two uploads run at once. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions | | More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content | | Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |