Compare commits

..
Author SHA1 Message Date
nvrl 2ad0bad09c Merge pull request #116 from narrrl/perf/photos-main-thread
perf(gui): keep Photos decoding and file access off the main thread
2026-11-20 20:57:38 +01:00
6 changed files with 154 additions and 235 deletions

No files matched your search

Generated
+6 -7
View File
@@ -3952,9 +3952,9 @@ dependencies = [
[[package]]
name = "proton-drive-rs"
version = "0.12.4"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670bfcf0eeb15886c962c18c76dbac976236e4377bfe073eb2360dc1ce448da2"
checksum = "bdeb7ee7495bc34198c78d7ec569f6371ad01a88f188b5eee418b013b082626a"
dependencies = [
"async-trait",
"base64 0.23.1",
@@ -4022,16 +4022,15 @@ dependencies = [
[[package]]
name = "proton-gallery-rs"
version = "0.1.5"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e374d2053de5df0e6b0797df65d0d024dbce35f7aedfefd4a656896a0495dc9b"
checksum = "dc9c9bbb76a5fba4071ee90de4aada71c292c0e408024ddfb04a12869291404a"
dependencies = [
"aes-gcm 0.11.1",
"async-trait",
"getrandom 0.4.3",
"hex",
"kamadak-exif",
"proton-base-rs",
"rusqlite",
"serde",
"serde_json",
@@ -4060,9 +4059,9 @@ dependencies = [
[[package]]
name = "proton-sdk"
version = "0.12.4"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f578a9d920cef3f6c4da611793001fe6d2ed81face13c45f5e721274b1945f7d"
checksum = "3189873dcdf0bc0a3d0fa5619cea80594085252d1d66a57459ec6adfd1f35ac1"
dependencies = [
"aes-gcm 0.11.1",
"async-trait",
+3 -3
View File
@@ -15,10 +15,10 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
categories = ["filesystem", "command-line-utilities"]
[workspace.dependencies]
proton-sdk = { version = "0.12.4" }
proton-drive-rs = { version = "0.12.4" }
proton-sdk = { version = "0.12.2" }
proton-drive-rs = { version = "0.12.2" }
proton-gallery-app-rs = "0.1.0"
proton-gallery-rs = "0.1.5"
proton-gallery-rs = "0.1.1"
chrono = { version = "0.4", default-features = false, features = ["std"] }
# Internal crates.
+80 -217
View File
@@ -256,62 +256,6 @@ impl Reporter {
}
}
/// Photos uploaded at once. Each upload waits on Proton between its blocks,
/// so two keep the link busy where one leaves it idle; more would mostly
/// stage more copies on disk.
const IMPORT_UPLOADS: usize = 2;
/// Where an upload reads the photo from.
enum Source<'a> {
/// A copy staged on disk, which a thumbnail can be made from.
Staged(&'a Path),
/// Straight out of the open archive.
Archive(&'a mut zip::ZipArchive<std::io::BufReader<std::fs::File>>),
}
/// A staged photo handed to an upload worker.
struct Upload {
index: usize,
path: PathBuf,
photo: TakeoutPhoto,
tags: Vec<PhotoTag>,
main_uid: Option<NodeUid>,
}
impl Upload {
fn landed(self, result: CoreResult<NodeUid>) -> Landed {
Landed {
index: self.index,
live: self.main_uid.is_some(),
photo: self.photo,
result,
}
}
fn streamed(
index: usize,
photo: TakeoutPhoto,
main_uid: Option<NodeUid>,
result: CoreResult<NodeUid>,
) -> Landed {
Landed {
index,
live: main_uid.is_some(),
photo,
result,
}
}
}
/// A finished upload, to be counted on the import thread.
struct Landed {
index: usize,
/// The video of a live pair, filed with its still.
live: bool,
photo: TakeoutPhoto,
result: CoreResult<NodeUid>,
}
/// How a planned photo relates to a live or motion photo pair.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LiveRole {
@@ -770,133 +714,72 @@ impl Core {
}
}
// Staged photos go to IMPORT_UPLOADS workers while this thread
// reads the next one out of its archive. A round ends only when
// every upload in it has landed, so a pair's video always finds
// its still's uid.
let (jobs, queue) = std::sync::mpsc::sync_channel::<Upload>(0);
let queue = std::sync::Mutex::new(queue);
std::thread::scope(|scope| {
let (landed_tx, landed) = std::sync::mpsc::channel::<Landed>();
for _ in 0..IMPORT_UPLOADS {
let queue = &queue;
let landed_tx = landed_tx.clone();
scope.spawn(move || {
loop {
let next = queue.lock().map(|queue| queue.recv());
let Ok(Ok(upload)) = next else { break };
let result = self.upload_takeout_photo(
Source::Staged(&upload.path),
&upload.photo,
upload.tags.clone(),
upload.main_uid.clone(),
);
let _ = std::fs::remove_file(&upload.path);
if landed_tx.send(upload.landed(result)).is_err() {
break;
}
}
});
for (archive_index, indices) in by_archive {
if self.import_should_stop(cancel) {
stopped = true;
break;
}
drop(landed_tx);
let mut archive = match open_archive(archives, archive_index) {
Ok(archive) => archive,
Err(e) => {
// The archive was readable at scan time; whatever broke
// it since costs its own photos, not the whole run.
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive");
summary.failed += indices.len();
for _ in &indices {
job.step();
report.step();
}
report.counts(summary);
continue;
}
};
'archives: for (archive_index, indices) in by_archive {
for index in indices {
if self.import_should_stop(cancel) {
stopped = true;
break;
}
let mut archive = match open_archive(archives, archive_index) {
Ok(archive) => archive,
Err(e) => {
// The archive was readable at scan time; whatever
// broke it since costs its own photos, not the run.
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive");
summary.failed += indices.len();
for _ in &indices {
job.step();
report.step();
}
report.counts(summary);
continue;
}
let main_uid = match planned[index].live {
Some(LiveRole::Video(still)) => planned[still].uid.clone(),
_ => None,
};
let photo = planned[index].photo.clone();
job.detail(photo.name.clone());
report.current(&photo.name);
for index in indices {
for done in landed.try_iter() {
self.record_upload(planned, done, summary, &mut favorites, job, report);
}
if self.import_should_stop(cancel) {
stopped = true;
break 'archives;
}
let item = &planned[index];
let main_uid = match item.live {
Some(LiveRole::Video(still)) => planned[still].uid.clone(),
_ => None,
};
let tags = match item.live {
Some(LiveRole::Still(tag)) => vec![tag],
_ => Vec::new(),
};
let photo = item.photo.clone();
job.detail(photo.name.clone());
report.current(&photo.name);
let uploaded =
self.upload_takeout_photo(&mut archive, &planned[index], main_uid.clone());
job.step();
report.step();
// A thumbnail is made from a file, so a photo that can
// have one is copied out of the archive and uploaded
// from the copy by a worker. Without one, Proton's apps
// show a blank tile until they make their own.
let staged = can_make_thumbnail(&photo.name, photo.size)
.then(|| self.stage_takeout_entry(&mut archive, &photo.entry))
.and_then(|staged| {
staged
.inspect_err(|e| {
warn!(error = %e,
"copying a takeout photo out failed; uploading it without a thumbnail")
})
.ok()
});
match staged {
Some(path) => {
let upload = Upload {
index,
path,
photo,
tags,
main_uid,
};
if let Err(std::sync::mpsc::SendError(upload)) = jobs.send(upload) {
let _ = std::fs::remove_file(&upload.path);
}
match uploaded {
Ok(uid) => {
self.note_self_change(&uid);
summary.uploaded += 1;
summary.bytes += photo.size;
if main_uid.is_some() {
summary.live_photos += 1;
}
None => {
let result = self.upload_takeout_photo(
Source::Archive(&mut archive),
&photo,
tags,
main_uid.clone(),
);
let done = Upload::streamed(index, photo, main_uid, result);
self.record_upload(
planned,
done,
summary,
&mut favorites,
job,
report,
);
if photo.favorite {
favorites.push(uid.clone());
}
planned[index].uid = Some(uid);
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
self.spawn_timeline_refresh();
}
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
report.counts(summary);
report.checkpoint();
}
}
Err(e) => {
summary.failed += 1;
warn!(name = photo.name, error = %e, "takeout photo upload failed");
}
}
report.counts(summary);
}
drop(jobs);
for done in landed.iter() {
self.record_upload(planned, done, summary, &mut favorites, job, report);
}
});
if stopped {
break;
}
}
@@ -904,55 +787,14 @@ impl Core {
stopped
}
/// Count one finished upload and file its uid in the plan.
fn record_upload(
&self,
planned: &mut [Planned],
done: Landed,
summary: &mut ImportSummary,
favorites: &mut Vec<NodeUid>,
job: &crate::transfers::JobGuard,
report: &Reporter,
) {
job.step();
report.step();
match done.result {
Ok(uid) => {
self.note_self_change(&uid);
summary.uploaded += 1;
summary.bytes += done.photo.size;
if done.live {
summary.live_photos += 1;
}
if done.photo.favorite {
favorites.push(uid.clone());
}
planned[done.index].uid = Some(uid);
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
self.spawn_timeline_refresh();
}
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
report.counts(summary);
report.checkpoint();
}
}
Err(e) => {
summary.failed += 1;
warn!(error = %e, "takeout photo upload failed");
}
}
report.counts(summary);
}
/// Upload one planned photo, from its staged copy or straight out of its
/// open archive.
/// Upload one planned photo out of its open archive.
fn upload_takeout_photo(
&self,
source: Source<'_>,
photo: &TakeoutPhoto,
tags: Vec<PhotoTag>,
archive: &mut zip::ZipArchive<std::io::BufReader<std::fs::File>>,
item: &Planned,
main_uid: Option<NodeUid>,
) -> CoreResult<NodeUid> {
let photo = &item.photo;
let photos = self.photos();
let guard = self.transfers.begin(
photo.name.clone(),
@@ -960,6 +802,10 @@ impl Core {
TransferDirection::Upload,
photo.size,
);
let tags = match item.live {
Some(LiveRole::Still(tag)) => vec![tag],
_ => Vec::new(),
};
let metadata = |capture_time: Option<i64>| PhotoUploadMetadata {
capture_time,
last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time),
@@ -974,8 +820,21 @@ impl Core {
replace_existing_draft: true,
};
match source {
Source::Staged(path) => {
// A thumbnail is made from a file, so a photo that can have one is
// copied out of the archive first and uploaded from the copy. Without
// one, Proton's apps show a blank tile until they make their own.
let staged = can_make_thumbnail(&photo.name, photo.size)
.then(|| self.stage_takeout_entry(archive, &photo.entry))
.and_then(|staged| {
staged
.inspect_err(|e| {
warn!(name = photo.name, error = %e,
"copying a takeout photo out failed; uploading it without a thumbnail")
})
.ok()
});
let uploaded = match &staged {
Some(path) => {
let thumbnails = self.upload_thumbnails(path, &photo.name);
// No sidecar date and none in the name: the photo's own EXIF,
// then when Google received it, before Drive files it under
@@ -1001,7 +860,7 @@ impl Core {
.map_err(|e| CoreError::from_api(&e, "upload"))
})
}
Source::Archive(archive) => read_entry(archive, &photo.entry, |reader| {
None => read_entry(archive, &photo.entry, |reader| {
let counting = CountingReader::new(reader, &guard);
self.rt
.block_on(photos.upload_photo_from(
@@ -1016,7 +875,11 @@ impl Core {
.map_err(|e| CoreError::from_api(&e, "upload"))
})
.and_then(|result| result),
};
if let Some(path) = staged {
let _ = std::fs::remove_file(path);
}
uploaded
}
/// Mark photos as favorites. A separate call: the upload seal takes
+64 -2
View File
@@ -17,8 +17,6 @@
//! a single photo opens it in its town, and photos taken on one spot, which
//! never part, are listed to pick from.
// Markers merge the same way in every Proton gallery app.
use proton_gallery_rs::places::{Bubble, bubble_radius, cluster, compact_count};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::OnceLock;
@@ -139,6 +137,13 @@ fn parse_ring(line: &str) -> Option<Ring> {
Some(Ring { points, min, max })
}
/// Radius in px of a marker holding `photos` photos: big enough to make the
/// photo out, and growing with the count, but slowly, so a home town with
/// thousands doesn't cover a continent.
fn bubble_radius(photos: usize) -> f64 {
18.0 + 2.0 * (photos.max(1) as f64).ln()
}
/// What the map shows: which part of the world, at what size.
#[derive(Clone, Copy, Debug)]
struct View {
@@ -190,6 +195,17 @@ struct Marker {
target: Target,
}
/// One marker as last drawn, standing for one or more [`Marker`]s: where, how
/// big, and which (indices into [`MapState::markers`], the one with the most
/// photos first).
struct Bubble {
x: f64,
y: f64,
radius: f64,
photos: usize,
members: Vec<usize>,
}
/// A street-map tile ready to draw, and the names on it.
struct CachedTile {
surface: cairo::ImageSurface,
@@ -1632,6 +1648,52 @@ fn spread(points: &[World]) -> f64 {
span(&mut xs.into_iter()).max(span(&mut ys.into_iter()))
}
/// Merge markers whose circles would overlap on screen, the one with the
/// most photos claiming its neighbours (the first of equals, when counts
/// tie). Markers off screen are left out.
fn cluster(screen: &[(f64, f64)], counts: &[usize], width: f64, height: f64) -> Vec<Bubble> {
let mut order: Vec<usize> = (0..screen.len()).collect();
order.sort_by(|a, b| counts[*b].cmp(&counts[*a]));
let mut bubbles: Vec<Bubble> = Vec::new();
for index in order {
let (x, y) = screen[index];
let reach = bubble_radius(counts[index]);
if x < -reach || y < -reach || x > width + reach || y > height + reach {
continue;
}
match bubbles
.iter_mut()
.find(|b| (b.x - x).hypot(b.y - y) < b.radius + reach * 0.5)
{
Some(bubble) => {
bubble.photos += counts[index];
bubble.radius = bubble_radius(bubble.photos);
bubble.members.push(index);
}
None => bubbles.push(Bubble {
x,
y,
radius: reach,
photos: counts[index],
members: vec![index],
}),
}
}
// Smallest last, so a small marker next to a big one stays clickable.
bubbles.sort_by(|a, b| b.radius.total_cmp(&a.radius));
bubbles
}
/// "7", "840", "1.2k", "35k": short enough to fit a marker. Digits and the
/// "k" read the same in every language the app ships.
fn compact_count(n: usize) -> String {
match n {
0..1_000 => n.to_string(),
1_000..10_000 => format!("{:.1}k", n as f64 / 1_000.0),
_ => format!("{}k", n / 1_000),
}
}
fn set_color(cr: &cairo::Context, color: &gtk4::gdk::RGBA, alpha: f32) {
cr.set_source_rgba(
f64::from(color.red()),
-5
View File
@@ -13,11 +13,6 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
### Changed
- A Google Photos import uploads two photos at once instead of one, while the next
photo is read out of its archive. Live-photo videos still go after their stills, and
cancelling still lets the uploads in flight finish.
- proton-sdk 0.12.4: thumbnails need fewer requests to Proton (0.12.3), and the SDK
can list photos that have no thumbnail (0.12.4).
- Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton
Gallery and Proton Stream use too. The stored session, the lock/unlock choice for two-password
accounts, the three mailbox-password tries and the messages stay as they were.
+1 -1
View File
@@ -63,7 +63,7 @@ them along.
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
| Photo name search | Not started | Photos search by date and filters only |
| Automatic photo upload from a folder | Not started | |
| Faster Google Photos import | Partly done | Two uploads run at once. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |