mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-22 10:33:31 +01:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa26a8e1aa | ||
|
|
9027a1465f | ||
|
|
66b634e4a1 |
No files matched your search
+221
-84
@@ -256,6 +256,62 @@ impl Reporter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Photos uploaded at once. Each upload waits on Proton between its blocks,
|
||||
/// so two keep the link busy where one leaves it idle; more would mostly
|
||||
/// stage more copies on disk.
|
||||
const IMPORT_UPLOADS: usize = 2;
|
||||
|
||||
/// Where an upload reads the photo from.
|
||||
enum Source<'a> {
|
||||
/// A copy staged on disk, which a thumbnail can be made from.
|
||||
Staged(&'a Path),
|
||||
/// Straight out of the open archive.
|
||||
Archive(&'a mut zip::ZipArchive<std::io::BufReader<std::fs::File>>),
|
||||
}
|
||||
|
||||
/// A staged photo handed to an upload worker.
|
||||
struct Upload {
|
||||
index: usize,
|
||||
path: PathBuf,
|
||||
photo: TakeoutPhoto,
|
||||
tags: Vec<PhotoTag>,
|
||||
main_uid: Option<NodeUid>,
|
||||
}
|
||||
|
||||
impl Upload {
|
||||
fn landed(self, result: CoreResult<NodeUid>) -> Landed {
|
||||
Landed {
|
||||
index: self.index,
|
||||
live: self.main_uid.is_some(),
|
||||
photo: self.photo,
|
||||
result,
|
||||
}
|
||||
}
|
||||
|
||||
fn streamed(
|
||||
index: usize,
|
||||
photo: TakeoutPhoto,
|
||||
main_uid: Option<NodeUid>,
|
||||
result: CoreResult<NodeUid>,
|
||||
) -> Landed {
|
||||
Landed {
|
||||
index,
|
||||
live: main_uid.is_some(),
|
||||
photo,
|
||||
result,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A finished upload, to be counted on the import thread.
|
||||
struct Landed {
|
||||
index: usize,
|
||||
/// The video of a live pair, filed with its still.
|
||||
live: bool,
|
||||
photo: TakeoutPhoto,
|
||||
result: CoreResult<NodeUid>,
|
||||
}
|
||||
|
||||
/// How a planned photo relates to a live or motion photo pair.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LiveRole {
|
||||
@@ -714,72 +770,133 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
for (archive_index, indices) in by_archive {
|
||||
if self.import_should_stop(cancel) {
|
||||
stopped = true;
|
||||
break;
|
||||
}
|
||||
let mut archive = match open_archive(archives, archive_index) {
|
||||
Ok(archive) => archive,
|
||||
Err(e) => {
|
||||
// The archive was readable at scan time; whatever broke
|
||||
// it since costs its own photos, not the whole run.
|
||||
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive");
|
||||
summary.failed += indices.len();
|
||||
for _ in &indices {
|
||||
job.step();
|
||||
report.step();
|
||||
// Staged photos go to IMPORT_UPLOADS workers while this thread
|
||||
// reads the next one out of its archive. A round ends only when
|
||||
// every upload in it has landed, so a pair's video always finds
|
||||
// its still's uid.
|
||||
let (jobs, queue) = std::sync::mpsc::sync_channel::<Upload>(0);
|
||||
let queue = std::sync::Mutex::new(queue);
|
||||
std::thread::scope(|scope| {
|
||||
let (landed_tx, landed) = std::sync::mpsc::channel::<Landed>();
|
||||
for _ in 0..IMPORT_UPLOADS {
|
||||
let queue = &queue;
|
||||
let landed_tx = landed_tx.clone();
|
||||
scope.spawn(move || {
|
||||
loop {
|
||||
let next = queue.lock().map(|queue| queue.recv());
|
||||
let Ok(Ok(upload)) = next else { break };
|
||||
let result = self.upload_takeout_photo(
|
||||
Source::Staged(&upload.path),
|
||||
&upload.photo,
|
||||
upload.tags.clone(),
|
||||
upload.main_uid.clone(),
|
||||
);
|
||||
let _ = std::fs::remove_file(&upload.path);
|
||||
if landed_tx.send(upload.landed(result)).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
report.counts(summary);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
});
|
||||
}
|
||||
drop(landed_tx);
|
||||
|
||||
for index in indices {
|
||||
'archives: for (archive_index, indices) in by_archive {
|
||||
if self.import_should_stop(cancel) {
|
||||
stopped = true;
|
||||
break;
|
||||
}
|
||||
let main_uid = match planned[index].live {
|
||||
Some(LiveRole::Video(still)) => planned[still].uid.clone(),
|
||||
_ => None,
|
||||
};
|
||||
let photo = planned[index].photo.clone();
|
||||
job.detail(photo.name.clone());
|
||||
report.current(&photo.name);
|
||||
|
||||
let uploaded =
|
||||
self.upload_takeout_photo(&mut archive, &planned[index], main_uid.clone());
|
||||
job.step();
|
||||
report.step();
|
||||
|
||||
match uploaded {
|
||||
Ok(uid) => {
|
||||
self.note_self_change(&uid);
|
||||
summary.uploaded += 1;
|
||||
summary.bytes += photo.size;
|
||||
if main_uid.is_some() {
|
||||
summary.live_photos += 1;
|
||||
}
|
||||
if photo.favorite {
|
||||
favorites.push(uid.clone());
|
||||
}
|
||||
planned[index].uid = Some(uid);
|
||||
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
|
||||
self.spawn_timeline_refresh();
|
||||
}
|
||||
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
|
||||
report.counts(summary);
|
||||
report.checkpoint();
|
||||
}
|
||||
}
|
||||
let mut archive = match open_archive(archives, archive_index) {
|
||||
Ok(archive) => archive,
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
warn!(name = photo.name, error = %e, "takeout photo upload failed");
|
||||
// The archive was readable at scan time; whatever
|
||||
// broke it since costs its own photos, not the run.
|
||||
warn!(error = %e, photos = indices.len(), "cannot reopen a takeout archive");
|
||||
summary.failed += indices.len();
|
||||
for _ in &indices {
|
||||
job.step();
|
||||
report.step();
|
||||
}
|
||||
report.counts(summary);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for index in indices {
|
||||
for done in landed.try_iter() {
|
||||
self.record_upload(planned, done, summary, &mut favorites, job, report);
|
||||
}
|
||||
if self.import_should_stop(cancel) {
|
||||
stopped = true;
|
||||
break 'archives;
|
||||
}
|
||||
let item = &planned[index];
|
||||
let main_uid = match item.live {
|
||||
Some(LiveRole::Video(still)) => planned[still].uid.clone(),
|
||||
_ => None,
|
||||
};
|
||||
let tags = match item.live {
|
||||
Some(LiveRole::Still(tag)) => vec![tag],
|
||||
_ => Vec::new(),
|
||||
};
|
||||
let photo = item.photo.clone();
|
||||
job.detail(photo.name.clone());
|
||||
report.current(&photo.name);
|
||||
|
||||
// A thumbnail is made from a file, so a photo that can
|
||||
// have one is copied out of the archive and uploaded
|
||||
// from the copy by a worker. Without one, Proton's apps
|
||||
// show a blank tile until they make their own.
|
||||
let staged = can_make_thumbnail(&photo.name, photo.size)
|
||||
.then(|| self.stage_takeout_entry(&mut archive, &photo.entry))
|
||||
.and_then(|staged| {
|
||||
staged
|
||||
.inspect_err(|e| {
|
||||
warn!(error = %e,
|
||||
"copying a takeout photo out failed; uploading it without a thumbnail")
|
||||
})
|
||||
.ok()
|
||||
});
|
||||
match staged {
|
||||
Some(path) => {
|
||||
let upload = Upload {
|
||||
index,
|
||||
path,
|
||||
photo,
|
||||
tags,
|
||||
main_uid,
|
||||
};
|
||||
if let Err(std::sync::mpsc::SendError(upload)) = jobs.send(upload) {
|
||||
let _ = std::fs::remove_file(&upload.path);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let result = self.upload_takeout_photo(
|
||||
Source::Archive(&mut archive),
|
||||
&photo,
|
||||
tags,
|
||||
main_uid.clone(),
|
||||
);
|
||||
let done = Upload::streamed(index, photo, main_uid, result);
|
||||
self.record_upload(
|
||||
planned,
|
||||
done,
|
||||
summary,
|
||||
&mut favorites,
|
||||
job,
|
||||
report,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
report.counts(summary);
|
||||
}
|
||||
|
||||
drop(jobs);
|
||||
for done in landed.iter() {
|
||||
self.record_upload(planned, done, summary, &mut favorites, job, report);
|
||||
}
|
||||
});
|
||||
if stopped {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -787,14 +904,55 @@ impl Core {
|
||||
stopped
|
||||
}
|
||||
|
||||
/// Upload one planned photo out of its open archive.
|
||||
/// Count one finished upload and file its uid in the plan.
|
||||
fn record_upload(
|
||||
&self,
|
||||
planned: &mut [Planned],
|
||||
done: Landed,
|
||||
summary: &mut ImportSummary,
|
||||
favorites: &mut Vec<NodeUid>,
|
||||
job: &crate::transfers::JobGuard,
|
||||
report: &Reporter,
|
||||
) {
|
||||
job.step();
|
||||
report.step();
|
||||
match done.result {
|
||||
Ok(uid) => {
|
||||
self.note_self_change(&uid);
|
||||
summary.uploaded += 1;
|
||||
summary.bytes += done.photo.size;
|
||||
if done.live {
|
||||
summary.live_photos += 1;
|
||||
}
|
||||
if done.photo.favorite {
|
||||
favorites.push(uid.clone());
|
||||
}
|
||||
planned[done.index].uid = Some(uid);
|
||||
if summary.uploaded.is_multiple_of(TIMELINE_REFRESH_EVERY) {
|
||||
self.spawn_timeline_refresh();
|
||||
}
|
||||
if summary.uploaded.is_multiple_of(CHECKPOINT_EVERY) {
|
||||
report.counts(summary);
|
||||
report.checkpoint();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
warn!(error = %e, "takeout photo upload failed");
|
||||
}
|
||||
}
|
||||
report.counts(summary);
|
||||
}
|
||||
|
||||
/// Upload one planned photo, from its staged copy or straight out of its
|
||||
/// open archive.
|
||||
fn upload_takeout_photo(
|
||||
&self,
|
||||
archive: &mut zip::ZipArchive<std::io::BufReader<std::fs::File>>,
|
||||
item: &Planned,
|
||||
source: Source<'_>,
|
||||
photo: &TakeoutPhoto,
|
||||
tags: Vec<PhotoTag>,
|
||||
main_uid: Option<NodeUid>,
|
||||
) -> CoreResult<NodeUid> {
|
||||
let photo = &item.photo;
|
||||
let photos = self.photos();
|
||||
let guard = self.transfers.begin(
|
||||
photo.name.clone(),
|
||||
@@ -802,10 +960,6 @@ impl Core {
|
||||
TransferDirection::Upload,
|
||||
photo.size,
|
||||
);
|
||||
let tags = match item.live {
|
||||
Some(LiveRole::Still(tag)) => vec![tag],
|
||||
_ => Vec::new(),
|
||||
};
|
||||
let metadata = |capture_time: Option<i64>| PhotoUploadMetadata {
|
||||
capture_time,
|
||||
last_modification_time: capture_time.and_then(pdfs_core::timestamp::modification_time),
|
||||
@@ -820,21 +974,8 @@ impl Core {
|
||||
replace_existing_draft: true,
|
||||
};
|
||||
|
||||
// A thumbnail is made from a file, so a photo that can have one is
|
||||
// copied out of the archive first and uploaded from the copy. Without
|
||||
// one, Proton's apps show a blank tile until they make their own.
|
||||
let staged = can_make_thumbnail(&photo.name, photo.size)
|
||||
.then(|| self.stage_takeout_entry(archive, &photo.entry))
|
||||
.and_then(|staged| {
|
||||
staged
|
||||
.inspect_err(|e| {
|
||||
warn!(name = photo.name, error = %e,
|
||||
"copying a takeout photo out failed; uploading it without a thumbnail")
|
||||
})
|
||||
.ok()
|
||||
});
|
||||
let uploaded = match &staged {
|
||||
Some(path) => {
|
||||
match source {
|
||||
Source::Staged(path) => {
|
||||
let thumbnails = self.upload_thumbnails(path, &photo.name);
|
||||
// No sidecar date and none in the name: the photo's own EXIF,
|
||||
// then when Google received it, before Drive files it under
|
||||
@@ -860,7 +1001,7 @@ impl Core {
|
||||
.map_err(|e| CoreError::from_api(&e, "upload"))
|
||||
})
|
||||
}
|
||||
None => read_entry(archive, &photo.entry, |reader| {
|
||||
Source::Archive(archive) => read_entry(archive, &photo.entry, |reader| {
|
||||
let counting = CountingReader::new(reader, &guard);
|
||||
self.rt
|
||||
.block_on(photos.upload_photo_from(
|
||||
@@ -875,11 +1016,7 @@ impl Core {
|
||||
.map_err(|e| CoreError::from_api(&e, "upload"))
|
||||
})
|
||||
.and_then(|result| result),
|
||||
};
|
||||
if let Some(path) = staged {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
uploaded
|
||||
}
|
||||
|
||||
/// Mark photos as favorites. A separate call: the upload seal takes
|
||||
|
||||
@@ -13,6 +13,9 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
|
||||
|
||||
### Changed
|
||||
|
||||
- A Google Photos import uploads two photos at once instead of one, while the next
|
||||
photo is read out of its archive. Live-photo videos still go after their stills, and
|
||||
cancelling still lets the uploads in flight finish.
|
||||
- proton-sdk 0.12.4: thumbnails need fewer requests to Proton (0.12.3), and the SDK
|
||||
can list photos that have no thumbnail (0.12.4).
|
||||
- Signing in runs through `proton_sdk::login` (proton-sdk 0.12.2), the sequence Proton
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ them along.
|
||||
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
|
||||
| Photo name search | Not started | Photos search by date and filters only |
|
||||
| Automatic photo upload from a folder | Not started | |
|
||||
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
|
||||
| Faster Google Photos import | Partly done | Two uploads run at once. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
|
||||
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
|
||||
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |
|
||||
|
||||
|
||||
Reference in new issue
Block a user