mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-15 17:41:59 +01:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a05772aa13 | ||
|
|
2f2fe96cdf | ||
|
|
2ec3b7ff62 | ||
|
|
38bd282d3a | ||
|
|
d773ba15d3 |
No files matched your search
Generated
+5
-4
@@ -3640,6 +3640,7 @@ dependencies = [
|
||||
"parking_lot 0.13.0",
|
||||
"pdfs-core",
|
||||
"proton-drive-rs",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.11.0",
|
||||
"tokio",
|
||||
@@ -3890,9 +3891,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-drive-rs"
|
||||
version = "0.11.0"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14a22e621abbc1c61b4640728a7de969c16b66d5d983b59c638d501fda25a7b9"
|
||||
checksum = "4265c2cf3bdc59ee273d7363c8b6a7cb4e98be6102ffa18144c4d8ebda00737b"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -3918,9 +3919,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-sdk"
|
||||
version = "0.11.0"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd55a5e8b6600f3aa282ebbdf13ff366431ff9aae84d4eb20b0ef06d598919fe"
|
||||
checksum = "fb98ab3e336ed95f82368a50348dad00e5fc919e982d748e83e844afcadbd5d8"
|
||||
dependencies = [
|
||||
"aes-gcm 0.11.1",
|
||||
"async-trait",
|
||||
|
||||
+2
-2
@@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
|
||||
categories = ["filesystem", "command-line-utilities"]
|
||||
|
||||
[workspace.dependencies]
|
||||
proton-sdk = { version = "0.11.0" }
|
||||
proton-drive-rs = { version = "0.11.0" }
|
||||
proton-sdk = { version = "0.12.0" }
|
||||
proton-drive-rs = { version = "0.12.0" }
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
|
||||
# Internal crates.
|
||||
|
||||
+180
-38
@@ -228,14 +228,19 @@ enum Command {
|
||||
/// numbered parts, and a photo's metadata often sits in a different part
|
||||
/// than the photo. Albums in the export are recreated as Proton albums, and
|
||||
/// photos already on the account (same name and same content) are skipped,
|
||||
/// so an interrupted import can simply be run again.
|
||||
/// so an interrupted import can simply be run again, or resumed with
|
||||
/// `--resume` without naming the archives again.
|
||||
ImportGooglePhotos {
|
||||
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
||||
#[arg(required = true, value_hint = ValueHint::FilePath)]
|
||||
#[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)]
|
||||
archives: Vec<PathBuf>,
|
||||
/// Report what would be imported without uploading anything.
|
||||
#[arg(long)]
|
||||
#[arg(long, conflicts_with = "resume")]
|
||||
dry_run: bool,
|
||||
/// Start the last import again with the same archives, after it was
|
||||
/// interrupted, stopped or failed.
|
||||
#[arg(long, conflicts_with = "archives")]
|
||||
resume: bool,
|
||||
/// Stay attached and print progress until the import finishes.
|
||||
#[arg(long)]
|
||||
wait: bool,
|
||||
@@ -1030,8 +1035,9 @@ fn main() -> Result<()> {
|
||||
Command::ImportGooglePhotos {
|
||||
archives,
|
||||
dry_run,
|
||||
resume,
|
||||
wait,
|
||||
} => cmd_import_google_photos(archives, dry_run, wait),
|
||||
} => cmd_import_google_photos(archives, dry_run, resume, wait),
|
||||
Command::RedatePhotos {
|
||||
dry_run,
|
||||
limit,
|
||||
@@ -2916,24 +2922,34 @@ fn photo_count(n: usize) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it gets
|
||||
// absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
fn cmd_import_google_photos(
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
resume: bool,
|
||||
wait: bool,
|
||||
) -> Result<()> {
|
||||
let request = if resume {
|
||||
CtlRequest::ResumeImport
|
||||
} else {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it
|
||||
// gets absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
}
|
||||
};
|
||||
|
||||
match control_request(CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
})? {
|
||||
match control_request(request)? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -2945,20 +2961,34 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
||||
|
||||
// Poll rather than stream: the control protocol is request/response, and an
|
||||
// import runs for hours — a held-open socket would be the only long-lived
|
||||
// connection the daemon has.
|
||||
// connection the daemon has. A line is printed when the step changes and
|
||||
// every few percent of it, not on every poll.
|
||||
let mut last_line = String::new();
|
||||
let mut last_printed = std::time::Instant::now();
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus {
|
||||
running: true,
|
||||
summary: _,
|
||||
} => continue,
|
||||
CtlResponse::ImportStatus { summary, .. } => {
|
||||
print_import_summary(summary.as_ref());
|
||||
return finished_run(
|
||||
"the import",
|
||||
summary.map(|s| (s.failed, s.cancelled, s.error)),
|
||||
);
|
||||
progress,
|
||||
..
|
||||
} => {
|
||||
let line = progress.as_ref().map(progress_line).unwrap_or_default();
|
||||
let step_changed = line.split(':').next() != last_line.split(':').next();
|
||||
if !line.is_empty()
|
||||
&& line != last_line
|
||||
&& (step_changed || last_printed.elapsed().as_secs() >= 30)
|
||||
{
|
||||
println!("{line}");
|
||||
last_line = line;
|
||||
last_printed = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
CtlResponse::ImportStatus {
|
||||
summary, resumable, ..
|
||||
} => {
|
||||
print_import_summary(summary.as_ref(), &resumable);
|
||||
return import_outcome(summary.as_ref());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -2966,6 +2996,47 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
||||
}
|
||||
}
|
||||
|
||||
/// One line saying where a running import is.
|
||||
fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String {
|
||||
use pdfs_core::control::ImportPhase;
|
||||
let step = match progress.phase {
|
||||
ImportPhase::Scanning => "Reading archives",
|
||||
ImportPhase::Checking => "Checking for photos already on Proton",
|
||||
ImportPhase::Uploading => "Uploading",
|
||||
ImportPhase::Albums => "Filing photos into albums",
|
||||
};
|
||||
match (progress.total, &progress.current) {
|
||||
(0, _) => format!("{step}…"),
|
||||
(total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done),
|
||||
(total, None) => format!("{step}: {} of {total}", progress.done),
|
||||
}
|
||||
}
|
||||
|
||||
/// How an import `--wait` followed ended, as the exit status: an error unless
|
||||
/// it did everything it set out to do, so a script can tell.
|
||||
fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> {
|
||||
let Some(summary) = summary else {
|
||||
bail!("the import ended without a result; the service may have restarted");
|
||||
};
|
||||
if let Some(error) = &summary.error {
|
||||
bail!("the import failed: {error}");
|
||||
}
|
||||
if summary.interrupted {
|
||||
bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue");
|
||||
}
|
||||
if summary.cancelled {
|
||||
bail!("the import was cancelled");
|
||||
}
|
||||
if summary.failed > 0 || !summary.unreadable_archives.is_empty() {
|
||||
bail!(
|
||||
"the import finished, but {} photo(s) and {} archive(s) failed",
|
||||
summary.failed,
|
||||
summary.unreadable_archives.len()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_redate_photos(
|
||||
dry_run: bool,
|
||||
limit: Option<usize>,
|
||||
@@ -3100,11 +3171,19 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) {
|
||||
|
||||
fn cmd_import_status() -> Result<()> {
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus { running, summary } => {
|
||||
CtlResponse::ImportStatus {
|
||||
running,
|
||||
summary,
|
||||
progress,
|
||||
resumable,
|
||||
} => {
|
||||
if running {
|
||||
println!("Import running. `pdfs transfers` shows its progress.");
|
||||
match progress.as_ref() {
|
||||
Some(progress) => println!("Import running. {}", progress_line(progress)),
|
||||
None => println!("Import running."),
|
||||
}
|
||||
}
|
||||
print_import_summary(summary.as_ref());
|
||||
print_import_summary(summary.as_ref(), &resumable);
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -3121,18 +3200,28 @@ fn cmd_cancel_import() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) {
|
||||
let Some(summary) = summary else {
|
||||
println!("No import has run yet.");
|
||||
return;
|
||||
};
|
||||
if summary.dry_run {
|
||||
println!("Scan only; nothing was uploaded.");
|
||||
}
|
||||
if let Some(error) = &summary.error {
|
||||
println!("Failed : {error}");
|
||||
return;
|
||||
println!("Stopped by : {error}");
|
||||
}
|
||||
println!("Photos in export : {}", summary.found);
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
if summary.dry_run {
|
||||
println!("Would upload : {}", summary.to_upload);
|
||||
println!("Bytes to upload : {}", summary.to_upload_bytes);
|
||||
} else {
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
}
|
||||
println!("Already on Proton: {}", summary.duplicates);
|
||||
if summary.live_photos > 0 {
|
||||
println!("Live photos : {}", summary.live_photos);
|
||||
}
|
||||
if summary.skipped_trashed > 0 {
|
||||
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
||||
}
|
||||
@@ -3145,12 +3234,20 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
if summary.failed > 0 {
|
||||
println!("Failed : {}", summary.failed);
|
||||
}
|
||||
for archive in &summary.unreadable_archives {
|
||||
println!("Unreadable : {archive}");
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
println!("Uploaded bytes : {}", summary.bytes);
|
||||
}
|
||||
if summary.cancelled {
|
||||
if summary.interrupted {
|
||||
println!("The service stopped before the import finished.");
|
||||
} else if summary.cancelled {
|
||||
println!("Import was cancelled before it finished.");
|
||||
}
|
||||
if !resumable.is_empty() {
|
||||
println!("Run `pdfs import-google-photos --resume` to continue it.");
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
||||
@@ -4509,6 +4606,51 @@ mod login_wait_tests {
|
||||
assert!(finished_run("the import", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_waited_for_import_that_did_not_finish_exits_with_an_error() {
|
||||
use pdfs_core::control::ImportSummary;
|
||||
let done = ImportSummary {
|
||||
uploaded: 4,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(import_outcome(Some(&done)).is_ok());
|
||||
assert!(import_outcome(None).is_err());
|
||||
for broken in [
|
||||
ImportSummary {
|
||||
interrupted: true,
|
||||
..Default::default()
|
||||
},
|
||||
ImportSummary {
|
||||
unreadable_archives: vec!["/exports/takeout-002.zip".into()],
|
||||
..Default::default()
|
||||
},
|
||||
ImportSummary {
|
||||
error: Some("check for existing photos: offline".into()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}");
|
||||
}
|
||||
let interrupted = import_outcome(Some(&ImportSummary {
|
||||
interrupted: true,
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(interrupted.unwrap_err().to_string().contains("--resume"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_needs_no_archives_and_excludes_them() {
|
||||
let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap();
|
||||
assert!(matches!(
|
||||
cli.command,
|
||||
Command::ImportGooglePhotos { resume: true, .. }
|
||||
));
|
||||
assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err());
|
||||
assert!(
|
||||
Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiting_for_a_session_stops_hammering_the_keyring() {
|
||||
let mut checks_left = 20;
|
||||
|
||||
+214
-29
@@ -123,13 +123,84 @@ fn client_config() -> ProtonClientConfiguration {
|
||||
ProtonClientConfiguration::new(app_version).with_user_agent(user_agent)
|
||||
}
|
||||
|
||||
fn keyring_entry() -> Result<Entry> {
|
||||
// Installed lazily rather than once at startup so a session bus that was not
|
||||
// up yet on the first attempt is retried on the next one.
|
||||
if keyring_core::get_default_store().is_none() {
|
||||
keyring_core::set_default_store(dbus_secret_service_keyring_store::Store::new()?);
|
||||
/// Run `op` on the session's keyring entry.
|
||||
fn with_keyring<T>(op: impl Fn(&Entry) -> keyring_core::Result<T>) -> Result<T> {
|
||||
with_keyring_entry(KEYRING_SERVICE, KEYRING_USER, op)
|
||||
}
|
||||
|
||||
/// Run `op` on the entry `service`/`user`, reconnecting once if the Secret
|
||||
/// Service fails the call.
|
||||
///
|
||||
/// The store holds one D-Bus connection and one Secret Service session for the
|
||||
/// whole process. The session lives in the provider's memory: once the provider
|
||||
/// restarts (oo7's unit restarts it on failure, and both oo7 and gnome-keyring
|
||||
/// can be `--replace`d), every read and write on the old session fails with
|
||||
/// `NoSession`, for as long as this process runs. A daemon then keeps refreshed
|
||||
/// tokens only in memory, and a cold boot loses them (`docs/BUGS.md` B285). A
|
||||
/// platform failure therefore drops the store, and the call is made once more
|
||||
/// on a new one. A locked keyring, a dismissed prompt or a missing entry is
|
||||
/// not the connection's fault and comes back as it is.
|
||||
fn with_keyring_entry<T>(
|
||||
service: &str,
|
||||
user: &str,
|
||||
op: impl Fn(&Entry) -> keyring_core::Result<T>,
|
||||
) -> Result<T> {
|
||||
let store = keyring_store()?;
|
||||
match op(&Entry::new(service, user)?) {
|
||||
Err(keyring_core::Error::PlatformFailure(e)) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"the Secret Service failed this process's connection; reconnecting"
|
||||
);
|
||||
drop_keyring_store(&store);
|
||||
keyring_store()?;
|
||||
Ok(op(&Entry::new(service, user)?)?)
|
||||
}
|
||||
result => Ok(result?),
|
||||
}
|
||||
}
|
||||
|
||||
/// Held while the default store is looked at and replaced, so that threads
|
||||
/// failing at the same time drop the broken store once and share its
|
||||
/// replacement.
|
||||
static KEYRING_STORE: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
|
||||
/// The process's Secret Service store, connected now if there is none.
|
||||
///
|
||||
/// Connected lazily rather than once at startup so a session bus that was not
|
||||
/// up yet on the first attempt is tried again on the next one.
|
||||
fn keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||
let _swap = KEYRING_STORE.lock();
|
||||
if let Some(store) = keyring_core::get_default_store() {
|
||||
return Ok(store);
|
||||
}
|
||||
let store = connect_keyring_store()?;
|
||||
keyring_core::set_default_store(Arc::clone(&store));
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
/// Forget `failed`, unless another thread has already replaced it.
|
||||
fn drop_keyring_store(failed: &Arc<keyring_core::CredentialStore>) {
|
||||
let _swap = KEYRING_STORE.lock();
|
||||
if keyring_core::get_default_store().is_some_and(|store| Arc::ptr_eq(&store, failed)) {
|
||||
keyring_core::unset_default_store();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||
Ok(dbus_secret_service_keyring_store::Store::new()?)
|
||||
}
|
||||
|
||||
/// The tests' in-memory store when they set one, so a reconnect finds the
|
||||
/// same entries; the real Secret Service otherwise.
|
||||
#[cfg(test)]
|
||||
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||
tests::KEYRING_CONNECTS.fetch_add(1, Ordering::SeqCst);
|
||||
match tests::MOCK_STORE.lock().clone() {
|
||||
Some(store) => Ok(store),
|
||||
None => Ok(dbus_secret_service_keyring_store::Store::new()?),
|
||||
}
|
||||
Ok(Entry::new(KEYRING_SERVICE, KEYRING_USER)?)
|
||||
}
|
||||
|
||||
/// What the user answered when asked for the mailbox password.
|
||||
@@ -421,17 +492,15 @@ fn classify_verification_gate(e: ProtonError) -> Error {
|
||||
|
||||
fn write_stored(stored: &StoredSession) -> Result<()> {
|
||||
let json = Zeroizing::new(serde_json::to_string(stored)?);
|
||||
keyring_entry()?.set_password(&json)?;
|
||||
Ok(())
|
||||
with_keyring(|entry| entry.set_password(&json))
|
||||
}
|
||||
|
||||
/// Load the persisted session blob, or `Error::NotLoggedIn` if absent.
|
||||
pub fn load() -> Result<StoredSession> {
|
||||
let entry = keyring_entry()?;
|
||||
match entry.get_password() {
|
||||
match with_keyring(|entry| entry.get_password()) {
|
||||
Ok(json) => Ok(serde_json::from_str(&json)?),
|
||||
Err(keyring_core::Error::NoEntry) => Err(Error::NotLoggedIn),
|
||||
Err(e) => Err(e.into()),
|
||||
Err(Error::Keyring(keyring_core::Error::NoEntry)) => Err(Error::NotLoggedIn),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -589,9 +658,9 @@ pub fn logout() -> Result<bool> {
|
||||
tracing::warn!(error = %e, "clearing the kernel-keyring unlock on logout failed");
|
||||
}
|
||||
let _ = kernelkey::UNSAVED_TOKENS.clear();
|
||||
match keyring_entry()?.delete_credential() {
|
||||
Ok(()) | Err(keyring_core::Error::NoEntry) => Ok(ended),
|
||||
Err(e) => Err(e.into()),
|
||||
match with_keyring(|entry| entry.delete_credential()) {
|
||||
Ok(()) | Err(Error::Keyring(keyring_core::Error::NoEntry)) => Ok(ended),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1207,12 +1276,138 @@ mod tests {
|
||||
/// The keyring is one store per process, so the tests that use it,
|
||||
/// through `keyring-core`'s in-memory store, take turns.
|
||||
fn mock_keyring() -> parking_lot::MutexGuard<'static, ()> {
|
||||
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
let turn = KEYRING.lock();
|
||||
keyring_core::set_default_store(keyring_core::mock::Store::new().unwrap());
|
||||
let store: Arc<keyring_core::CredentialStore> = keyring_core::mock::Store::new().unwrap();
|
||||
*MOCK_STORE.lock() = Some(Arc::clone(&store));
|
||||
keyring_core::set_default_store(store);
|
||||
turn
|
||||
}
|
||||
|
||||
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
|
||||
/// The in-memory store a reconnect hands out, so it finds the same
|
||||
/// entries; `None` connects to the real Secret Service.
|
||||
pub(super) static MOCK_STORE: parking_lot::Mutex<Option<Arc<keyring_core::CredentialStore>>> =
|
||||
parking_lot::Mutex::new(None);
|
||||
|
||||
/// How many times a store was connected.
|
||||
pub(super) static KEYRING_CONNECTS: std::sync::atomic::AtomicUsize =
|
||||
std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
/// The session's entry, to make its next call fail.
|
||||
fn keyring_entry() -> Entry {
|
||||
Entry::new(KEYRING_SERVICE, KEYRING_USER).unwrap()
|
||||
}
|
||||
|
||||
/// Make the session entry's next call fail with `error`.
|
||||
fn fail_next(error: keyring_core::Error) {
|
||||
keyring_entry()
|
||||
.as_any()
|
||||
.downcast_ref::<keyring_core::mock::Cred>()
|
||||
.unwrap()
|
||||
.set_error(error);
|
||||
}
|
||||
|
||||
fn no_session() -> keyring_core::Error {
|
||||
keyring_core::Error::PlatformFailure(
|
||||
std::io::Error::other("The session does not exist (NoSession)").into(),
|
||||
)
|
||||
}
|
||||
|
||||
/// B285: a restarted Secret Service no longer knows the process's
|
||||
/// session, and every call on it failed until the process restarted.
|
||||
#[test]
|
||||
fn a_dropped_secret_service_session_is_reconnected_and_retried() {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||
|
||||
fail_next(no_session());
|
||||
assert_eq!(load().unwrap().refresh_token, "r");
|
||||
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 1);
|
||||
|
||||
fail_next(no_session());
|
||||
write_stored(&stored_with("r2")).unwrap();
|
||||
assert_eq!(load().unwrap().refresh_token, "r2");
|
||||
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rotation_lands_at_once_after_the_keyring_restarts() {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||
|
||||
fail_next(no_session());
|
||||
sink.rotated(tokens("a2", "r2")).unwrap();
|
||||
assert_eq!(load().unwrap().refresh_token, "r2");
|
||||
assert!(sink.state.lock().unsaved.is_none());
|
||||
}
|
||||
|
||||
/// A locked keyring or a dismissed prompt is no fault of the connection.
|
||||
#[test]
|
||||
fn a_locked_keyring_is_not_reconnected() {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||
|
||||
fail_next(keyring_core::Error::NoStorageAccess(
|
||||
std::io::Error::other("locked").into(),
|
||||
));
|
||||
assert!(matches!(
|
||||
load(),
|
||||
Err(Error::Keyring(keyring_core::Error::NoStorageAccess(_)))
|
||||
));
|
||||
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects);
|
||||
assert_eq!(load().unwrap().refresh_token, "r");
|
||||
}
|
||||
|
||||
/// The real Secret Service, restarted between two calls on one store.
|
||||
///
|
||||
/// Run inside a throwaway session bus, never against the desktop's
|
||||
/// keyring. With gnome-keyring:
|
||||
///
|
||||
/// ```sh
|
||||
/// echo -n pw | gnome-keyring-daemon --unlock --components=secrets
|
||||
/// PDFS_SECRET_SERVICE_RESTART_TEST=1 \
|
||||
/// PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets' \
|
||||
/// cargo test -p pdfs-core real_secret_service -- --ignored
|
||||
/// ```
|
||||
#[test]
|
||||
#[ignore = "needs a throwaway Secret Service; see the doc comment"]
|
||||
fn real_secret_service_survives_a_provider_restart() {
|
||||
if std::env::var_os("PDFS_SECRET_SERVICE_RESTART_TEST").is_none() {
|
||||
return;
|
||||
}
|
||||
let restart = std::env::var("PDFS_RESTART_PROVIDER").expect("PDFS_RESTART_PROVIDER");
|
||||
let _turn = KEYRING.lock();
|
||||
*MOCK_STORE.lock() = None;
|
||||
keyring_core::unset_default_store();
|
||||
let (service, user) = ("pdfs-test", "restart");
|
||||
|
||||
with_keyring_entry(service, user, |e| e.set_password("before")).unwrap();
|
||||
let status = std::process::Command::new("sh")
|
||||
.args(["-c", &restart])
|
||||
.status()
|
||||
.unwrap();
|
||||
assert!(status.success(), "restarting the provider failed");
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
|
||||
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
|
||||
assert_eq!(read, "before");
|
||||
assert_eq!(
|
||||
KEYRING_CONNECTS.load(Ordering::SeqCst),
|
||||
connects + 1,
|
||||
"the provider kept the old session, so nothing was tested"
|
||||
);
|
||||
with_keyring_entry(service, user, |e| e.set_password("after")).unwrap();
|
||||
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
|
||||
assert_eq!(read, "after");
|
||||
with_keyring_entry(service, user, |e| e.delete_credential()).unwrap();
|
||||
keyring_core::unset_default_store();
|
||||
}
|
||||
|
||||
fn stored_with(refresh: &str) -> StoredSession {
|
||||
serde_json::from_str(&blob("").replace(r#""r""#, &format!("{refresh:?}"))).unwrap()
|
||||
}
|
||||
@@ -1267,12 +1462,7 @@ mod tests {
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||
|
||||
let entry = keyring_entry().unwrap();
|
||||
let cred = entry
|
||||
.as_any()
|
||||
.downcast_ref::<keyring_core::mock::Cred>()
|
||||
.unwrap();
|
||||
cred.set_error(keyring_core::Error::NoStorageAccess(
|
||||
fail_next(keyring_core::Error::NoStorageAccess(
|
||||
std::io::Error::other("locked").into(),
|
||||
));
|
||||
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
||||
@@ -1307,12 +1497,7 @@ mod tests {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||
let entry = keyring_entry().unwrap();
|
||||
entry
|
||||
.as_any()
|
||||
.downcast_ref::<keyring_core::mock::Cred>()
|
||||
.unwrap()
|
||||
.set_error(keyring_core::Error::NoEntry);
|
||||
fail_next(keyring_core::Error::NoEntry);
|
||||
|
||||
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
||||
assert!(sink.state.lock().unsaved.is_some());
|
||||
|
||||
@@ -562,6 +562,12 @@ impl AppDirs {
|
||||
self.state_dir().join("control.sock")
|
||||
}
|
||||
|
||||
/// The Google Photos import's last run: its archives and how it ended, so
|
||||
/// an import cut short by a restart can be resumed.
|
||||
pub fn takeout_import_path(&self) -> PathBuf {
|
||||
self.state_dir().join("takeout-import.json")
|
||||
}
|
||||
|
||||
/// Unix domain socket the tray uses to ensure single instance.
|
||||
pub fn tray_socket(&self) -> PathBuf {
|
||||
self.state_dir().join("tray.sock")
|
||||
|
||||
@@ -279,6 +279,11 @@ pub enum Request {
|
||||
/// and files what it has uploaded into its albums, then reports
|
||||
/// `cancelled`. Replies with [`Response::Ok`].
|
||||
CancelImport,
|
||||
/// Start the last import again with the same archives, after it was
|
||||
/// interrupted, cancelled or failed. Photos it already uploaded are found
|
||||
/// on the account and skipped. Replies like [`Request::ImportTakeout`];
|
||||
/// [`ErrorKind::NotFound`] when there is nothing to resume.
|
||||
ResumeImport,
|
||||
/// Upload the photo at `source_path` under the given name and media type.
|
||||
///
|
||||
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
||||
@@ -1689,10 +1694,75 @@ pub struct ImportSummary {
|
||||
pub bytes: u64,
|
||||
/// True when the run stopped early because it was cancelled.
|
||||
pub cancelled: bool,
|
||||
/// Why the run as a whole failed, when it did, such as an archive that
|
||||
/// is not a zip. The counts are then empty.
|
||||
/// Why the run as a whole stopped early, when it did, such as the
|
||||
/// account being unreachable. The counts say how far it got.
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
/// What kind of failure [`error`](Self::error) is, so a front-end can word
|
||||
/// it in the user's language. The text itself is the daemon's English.
|
||||
#[serde(default)]
|
||||
pub error_kind: Option<ErrorKind>,
|
||||
/// True for a scan that uploads nothing ([`Request::ImportTakeout`] with
|
||||
/// `dry_run`). [`to_upload`](Self::to_upload) then says what an import
|
||||
/// would upload.
|
||||
#[serde(default)]
|
||||
pub dry_run: bool,
|
||||
/// Photos not yet on the account: what a dry run would upload, and what a
|
||||
/// real run set out to upload once the duplicates were known.
|
||||
#[serde(default)]
|
||||
pub to_upload: usize,
|
||||
/// Bytes of [`to_upload`](Self::to_upload).
|
||||
#[serde(default)]
|
||||
pub to_upload_bytes: u64,
|
||||
/// Live and motion photos whose video was filed with its still.
|
||||
#[serde(default)]
|
||||
pub live_photos: usize,
|
||||
/// True when the service stopped while the import ran (a shutdown or a
|
||||
/// restart). [`Request::ResumeImport`] picks it up again.
|
||||
#[serde(default)]
|
||||
pub interrupted: bool,
|
||||
/// Archives that could not be read and were left out, by path.
|
||||
#[serde(default)]
|
||||
pub unreadable_archives: Vec<String>,
|
||||
}
|
||||
|
||||
impl ImportSummary {
|
||||
/// Whether the run did all it set out to do.
|
||||
pub fn complete(&self) -> bool {
|
||||
self.error.is_none()
|
||||
&& !self.cancelled
|
||||
&& !self.interrupted
|
||||
&& self.failed == 0
|
||||
&& self.unreadable_archives.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Which step a running Takeout import is on.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportPhase {
|
||||
/// Reading the archives' directories and metadata.
|
||||
#[default]
|
||||
Scanning,
|
||||
/// Asking the account which photos it already has.
|
||||
Checking,
|
||||
/// Uploading the photos that are new.
|
||||
Uploading,
|
||||
/// Creating albums and filing photos into them.
|
||||
Albums,
|
||||
}
|
||||
|
||||
/// Where a running import is ([`Response::ImportStatus`]).
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ImportProgress {
|
||||
pub phase: ImportPhase,
|
||||
/// Steps done in this phase.
|
||||
pub done: u64,
|
||||
/// Steps in this phase; `0` while not yet known.
|
||||
pub total: u64,
|
||||
/// The photo being uploaded, during [`ImportPhase::Uploading`].
|
||||
#[serde(default)]
|
||||
pub current: Option<String>,
|
||||
}
|
||||
|
||||
/// One album in a [`Response::Albums`] listing.
|
||||
@@ -2208,8 +2278,17 @@ pub enum Response {
|
||||
/// report; `summary` is `None` when no import has run this session.
|
||||
ImportStatus {
|
||||
running: bool,
|
||||
/// The counts so far while running, the final report after.
|
||||
#[serde(default)]
|
||||
summary: Option<ImportSummary>,
|
||||
/// The current step, while running.
|
||||
#[serde(default)]
|
||||
progress: Option<ImportProgress>,
|
||||
/// The archives of the last run, when it can be resumed with
|
||||
/// [`Request::ResumeImport`]: it was a real import that did not
|
||||
/// finish. Survives a restart of the service.
|
||||
#[serde(default)]
|
||||
resumable: Vec<String>,
|
||||
},
|
||||
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
|
||||
/// `running` is false once it has finished, when `summary` is the final
|
||||
|
||||
+488
-70
@@ -25,7 +25,9 @@
|
||||
//! The service folder (`Google Photos`) is localized, as are the year folders
|
||||
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
||||
//! service name: the first component is dropped when it is the export root, the
|
||||
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
|
||||
//! second is the service, and the rest is `<folder>/<file>`. The Photos service
|
||||
//! is the one whose sidecars carry a `photoTakenTime`; other services in the same
|
||||
//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an
|
||||
//! **album** when it carries an album metadata sidecar naming it; anything else
|
||||
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
||||
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
||||
@@ -121,8 +123,14 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
|
||||
("m2ts", "video/mp2t"),
|
||||
("mpg", "video/mpeg"),
|
||||
("mpeg", "video/mpeg"),
|
||||
// The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`.
|
||||
("mp", "video/mp4"),
|
||||
];
|
||||
|
||||
/// Sidecars bigger than this are not read. A real one is a few kilobytes; a
|
||||
/// `.json` this large is some other service's data and not worth the memory.
|
||||
const SIDECAR_MAX_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
/// The media type for `name`'s extension, or `None` when it is not media this
|
||||
/// import handles.
|
||||
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
||||
@@ -147,6 +155,13 @@ pub enum Bucket {
|
||||
Archived,
|
||||
}
|
||||
|
||||
/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct GeoPoint {
|
||||
pub latitude: f64,
|
||||
pub longitude: f64,
|
||||
}
|
||||
|
||||
/// One media file found in the export.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutPhoto {
|
||||
@@ -162,14 +177,49 @@ pub struct TakeoutPhoto {
|
||||
pub size: u64,
|
||||
/// Media type from the extension.
|
||||
pub media_type: String,
|
||||
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
|
||||
/// `creationTime`). `None` when the photo has no sidecar — the upload then
|
||||
/// defaults it to now, as the SDK does.
|
||||
/// Capture time in epoch seconds, from the sidecar's `photoTakenTime`,
|
||||
/// else from a date in the file name. `None` when neither has one; the
|
||||
/// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time).
|
||||
pub capture_time: Option<i64>,
|
||||
/// The sidecar's `creationTime`: when the photo reached Google Photos, not
|
||||
/// when it was taken. Only a last resort for the capture time.
|
||||
#[serde(default)]
|
||||
pub creation_time: Option<i64>,
|
||||
/// Where the photo was taken, from the sidecar's `geoData` (or
|
||||
/// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped.
|
||||
#[serde(default)]
|
||||
pub location: Option<GeoPoint>,
|
||||
/// The sidecar's `favorited` flag.
|
||||
pub favorite: bool,
|
||||
/// Which bucket the file was found in.
|
||||
pub bucket: Bucket,
|
||||
/// Pairs the halves of a live or motion photo: a still and a video in the
|
||||
/// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||
/// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half.
|
||||
#[serde(default)]
|
||||
pub live_key: Option<String>,
|
||||
}
|
||||
|
||||
impl TakeoutPhoto {
|
||||
/// Whether this is the video half of a pair, as opposed to the still.
|
||||
pub fn is_video(&self) -> bool {
|
||||
self.media_type.starts_with("video/")
|
||||
}
|
||||
|
||||
/// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather
|
||||
/// than an Apple live photo.
|
||||
pub fn is_motion_photo(&self) -> bool {
|
||||
self.live_key
|
||||
.as_deref()
|
||||
.is_some_and(|key| key.ends_with(".mp"))
|
||||
}
|
||||
}
|
||||
|
||||
/// An archive the scan had to leave out, and why.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UnreadableArchive {
|
||||
pub path: PathBuf,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// Everything one scan found, ready to be turned into an import plan.
|
||||
@@ -183,8 +233,16 @@ pub struct TakeoutScan {
|
||||
/// Media files skipped because they sit in the trash bucket.
|
||||
pub skipped_trashed: usize,
|
||||
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
||||
/// activity `.csv`s, `.mp` motion-photo parts).
|
||||
/// activity `.csv`s), and media from other Google services in the same
|
||||
/// export (`Google Drive/`, `YouTube/`).
|
||||
pub skipped_other: usize,
|
||||
/// Archives that could not be opened or are not a zip. The rest of the
|
||||
/// export is still imported.
|
||||
#[serde(default)]
|
||||
pub unreadable_archives: Vec<UnreadableArchive>,
|
||||
/// Entries inside an otherwise readable archive that could not be read.
|
||||
#[serde(default)]
|
||||
pub unreadable_entries: usize,
|
||||
}
|
||||
|
||||
impl TakeoutScan {
|
||||
@@ -195,7 +253,7 @@ impl TakeoutScan {
|
||||
}
|
||||
|
||||
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
||||
/// file also carries geo data, people tags and view counts.
|
||||
/// file also carries people tags, a description and view counts.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct PhotoSidecar {
|
||||
#[serde(default)]
|
||||
@@ -205,19 +263,69 @@ struct PhotoSidecar {
|
||||
#[serde(rename = "creationTime", default)]
|
||||
creation_time: Option<SidecarTime>,
|
||||
#[serde(default)]
|
||||
favorited: bool,
|
||||
favorited: Option<bool>,
|
||||
/// The position as edited in Google Photos, else as the camera wrote it.
|
||||
#[serde(rename = "geoData", default)]
|
||||
geo_data: Option<SidecarGeo>,
|
||||
#[serde(rename = "geoDataExif", default)]
|
||||
geo_data_exif: Option<SidecarGeo>,
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, as a *string*.
|
||||
impl PhotoSidecar {
|
||||
fn location(&self) -> Option<GeoPoint> {
|
||||
self.geo_data
|
||||
.as_ref()
|
||||
.and_then(SidecarGeo::point)
|
||||
.or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point))
|
||||
}
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarTime {
|
||||
#[serde(default)]
|
||||
timestamp: Option<String>,
|
||||
timestamp: Option<SidecarNumber>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum SidecarNumber {
|
||||
Number(i64),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
impl SidecarTime {
|
||||
/// The timestamp, when it is a real moment. `0` and negative values are
|
||||
/// Google's "unknown" and would file the photo under 1970.
|
||||
fn epoch_seconds(&self) -> Option<i64> {
|
||||
self.timestamp.as_ref()?.parse().ok()
|
||||
let seconds = match self.timestamp.as_ref()? {
|
||||
SidecarNumber::Number(seconds) => *seconds,
|
||||
SidecarNumber::Text(text) => text.trim().parse().ok()?,
|
||||
};
|
||||
(seconds > 0).then_some(seconds)
|
||||
}
|
||||
}
|
||||
|
||||
/// `geoData` in a sidecar. All-zero means Google has no position.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarGeo {
|
||||
#[serde(default)]
|
||||
latitude: Option<f64>,
|
||||
#[serde(default)]
|
||||
longitude: Option<f64>,
|
||||
}
|
||||
|
||||
impl SidecarGeo {
|
||||
fn point(&self) -> Option<GeoPoint> {
|
||||
let (latitude, longitude) = (self.latitude?, self.longitude?);
|
||||
let on_globe = latitude.is_finite()
|
||||
&& longitude.is_finite()
|
||||
&& (-90.0..=90.0).contains(&latitude)
|
||||
&& (-180.0..=180.0).contains(&longitude);
|
||||
(on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint {
|
||||
latitude,
|
||||
longitude,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,21 +370,58 @@ impl RawEntry {
|
||||
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
||||
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
||||
/// this stays fast on a 200 GB export.
|
||||
///
|
||||
/// An archive that cannot be opened is left out and listed in
|
||||
/// [`TakeoutScan::unreadable_archives`]; the rest still imports.
|
||||
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<RawEntry> = Vec::new();
|
||||
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
scan_archives_until(archives, || false)
|
||||
}
|
||||
|
||||
/// [`scan_archives`] that gives up when `stop` returns true, checked between
|
||||
/// archives and every few hundred entries.
|
||||
pub fn scan_archives_until(
|
||||
archives: &[PathBuf],
|
||||
stop: impl Fn() -> bool,
|
||||
) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<(String, RawEntry)> = Vec::new();
|
||||
let mut sidecars: Vec<(String, String, Vec<u8>)> = Vec::new();
|
||||
let mut unreadable_archives = Vec::new();
|
||||
let mut unreadable_entries = 0;
|
||||
|
||||
for (index, path) in archives.iter().enumerate() {
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
|
||||
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
|
||||
})?;
|
||||
if stop() {
|
||||
return Err(CoreError::conflict("the import was cancelled"));
|
||||
}
|
||||
let opened = std::fs::File::open(path)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|file| {
|
||||
zip::ZipArchive::new(std::io::BufReader::new(file))
|
||||
.map_err(|e| format!("not a readable zip: {e}"))
|
||||
});
|
||||
let mut zip = match opened {
|
||||
Ok(zip) => zip,
|
||||
Err(reason) => {
|
||||
tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive");
|
||||
unreadable_archives.push(UnreadableArchive {
|
||||
path: path.clone(),
|
||||
reason,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for position in 0..zip.len() {
|
||||
let mut entry = zip.by_index(position).map_err(|e| {
|
||||
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
|
||||
})?;
|
||||
if position % 512 == 0 && stop() {
|
||||
return Err(CoreError::conflict("the import was cancelled"));
|
||||
}
|
||||
let mut entry = match zip.by_index(position) {
|
||||
Ok(entry) => entry,
|
||||
Err(e) => {
|
||||
tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry");
|
||||
unreadable_entries += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.is_dir() {
|
||||
continue;
|
||||
}
|
||||
@@ -286,32 +431,92 @@ pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
continue;
|
||||
};
|
||||
let name = name.to_string_lossy().replace('\\', "/");
|
||||
let Some(relative) = strip_export_prefix(&name) else {
|
||||
let Some((service, relative)) = strip_export_prefix(&name) else {
|
||||
continue;
|
||||
};
|
||||
let (service, relative) = (service.to_string(), relative.to_string());
|
||||
let size = entry.size();
|
||||
|
||||
// Sidecars are read now, while the entry is open: they are a few
|
||||
// hundred bytes each and reopening the archive per photo later would
|
||||
// be a second full pass.
|
||||
if relative.to_ascii_lowercase().ends_with(".json") {
|
||||
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.insert(relative.to_string(), body);
|
||||
if size <= SIDECAR_MAX_BYTES {
|
||||
let mut body = Vec::with_capacity(size as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.push((service, relative, body));
|
||||
} else {
|
||||
unreadable_entries += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push(RawEntry {
|
||||
archive: index,
|
||||
entry: name.clone(),
|
||||
relative: relative.to_string(),
|
||||
size,
|
||||
});
|
||||
entries.push((
|
||||
service,
|
||||
RawEntry {
|
||||
archive: index,
|
||||
entry: name,
|
||||
relative,
|
||||
size,
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(assemble(entries, &sidecars))
|
||||
let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars);
|
||||
let mut scan = assemble(entries, &sidecars);
|
||||
scan.skipped_other += other_services;
|
||||
scan.unreadable_archives = unreadable_archives;
|
||||
scan.unreadable_entries = unreadable_entries;
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
/// Drop what other Google services put into the same export.
|
||||
///
|
||||
/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to
|
||||
/// `Google Photos/`, and their media must not land in the photo timeline. The
|
||||
/// service folder's name is localized (`Google Fotos`), so the Photos service
|
||||
/// is recognized by its content instead: it is the folder whose sidecars carry
|
||||
/// a `photoTakenTime`. When no folder does — an export without sidecars, or
|
||||
/// one re-zipped without the service level — everything is kept, as before.
|
||||
///
|
||||
/// Returns the entries and sidecars to import, and how many media files were
|
||||
/// left out.
|
||||
fn keep_photos_services(
|
||||
entries: Vec<(String, RawEntry)>,
|
||||
sidecars: Vec<(String, String, Vec<u8>)>,
|
||||
) -> (Vec<RawEntry>, HashMap<String, Vec<u8>>, usize) {
|
||||
#[derive(Deserialize)]
|
||||
struct Marker {
|
||||
#[serde(rename = "photoTakenTime")]
|
||||
photo_taken_time: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new();
|
||||
for (service, _, body) in &sidecars {
|
||||
if photos_services.contains(service.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if serde_json::from_slice::<Marker>(body).is_ok_and(|m| m.photo_taken_time.is_some()) {
|
||||
photos_services.insert(service.as_str());
|
||||
}
|
||||
}
|
||||
let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service);
|
||||
|
||||
let mut skipped = 0;
|
||||
let mut kept_entries = Vec::with_capacity(entries.len());
|
||||
for (service, raw) in entries {
|
||||
if keep(&service) {
|
||||
kept_entries.push(raw);
|
||||
} else if media_type_of(raw.file_name()).is_some() {
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
let kept_sidecars = sidecars
|
||||
.iter()
|
||||
.filter(|(service, _, _)| keep(service))
|
||||
.map(|(_, relative, body)| (relative.clone(), body.clone()))
|
||||
.collect();
|
||||
(kept_entries, kept_sidecars, skipped)
|
||||
}
|
||||
|
||||
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
||||
@@ -403,19 +608,19 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
||||
.unwrap_or(&file_name)
|
||||
.to_string();
|
||||
|
||||
// Sidecar first, then the date stamped into the name.
|
||||
// Sidecar first, then the date stamped into the name. `creationTime`
|
||||
// is when the photo reached Google, which for a scan or an old photo
|
||||
// uploaded late is years off; the importer uses it only after EXIF.
|
||||
let capture_time = sidecar
|
||||
.photo_taken_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
.or_else(|| {
|
||||
sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
})
|
||||
.or_else(|| capture_time_from_name(&name))
|
||||
.or_else(|| capture_time_from_name(&file_name));
|
||||
let creation_time = sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds);
|
||||
|
||||
scan.photos.push(TakeoutPhoto {
|
||||
archive: raw.archive,
|
||||
@@ -424,31 +629,57 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
||||
size: raw.size,
|
||||
media_type: media_type.to_string(),
|
||||
capture_time,
|
||||
favorite: sidecar.favorited,
|
||||
creation_time,
|
||||
location: sidecar.location(),
|
||||
favorite: sidecar.favorited.unwrap_or(false),
|
||||
bucket,
|
||||
live_key: live_key(&folder, &file_name),
|
||||
});
|
||||
}
|
||||
|
||||
scan
|
||||
}
|
||||
|
||||
/// Drop the export root (`Takeout/`) and the localized service folder from a
|
||||
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
|
||||
/// Split a path inside the archive into the localized service folder and the
|
||||
/// rest, dropping the export root (`Takeout/`): `("Google Fotos",
|
||||
/// "<folder>/<file>")`.
|
||||
///
|
||||
/// `None` for a path with no service folder at all (the export's own
|
||||
/// `archive_browser.html`), which carries nothing to import.
|
||||
fn strip_export_prefix(path: &str) -> Option<&str> {
|
||||
fn strip_export_prefix(path: &str) -> Option<(&str, &str)> {
|
||||
let mut rest = path;
|
||||
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
|
||||
// the user may not have it — so the root is dropped only when a service
|
||||
// folder follows it, which the component count decides.
|
||||
// The root is `Takeout/`, but an export unzipped and re-zipped by the user
|
||||
// may not have it. `Google Photos/<album>/<file>` then has three parts like
|
||||
// `Takeout/Google Photos/<file>`, so the count alone cannot tell them
|
||||
// apart: a root of another name is only taken for one with four parts.
|
||||
let (first, after) = rest.split_once('/')?;
|
||||
let components = rest.split('/').count();
|
||||
if components >= 3 {
|
||||
rest = rest.split_once('/')?.1;
|
||||
if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 {
|
||||
rest = after;
|
||||
}
|
||||
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
|
||||
let (_service, tail) = rest.split_once('/')?;
|
||||
Some(tail)
|
||||
// The service folder (`Google Photos`, `Google Fotos`, …).
|
||||
rest.split_once('/')
|
||||
}
|
||||
|
||||
/// The key that pairs the still and the video of one live or motion photo, or
|
||||
/// `None` when `file_name` cannot be either half.
|
||||
///
|
||||
/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem.
|
||||
/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the
|
||||
/// video's whole name. The folder is part of the key so two cameras' `IMG_1`
|
||||
/// in different albums never pair.
|
||||
fn live_key(folder: &str, file_name: &str) -> Option<String> {
|
||||
let lower = file_name.to_lowercase();
|
||||
let (stem, ext) = lower.rsplit_once('.')?;
|
||||
let key = match ext {
|
||||
"heic" | "heif" | "jpg" | "jpeg" => stem,
|
||||
// Apple's video half. A plain `.mp4` beside a photo of the same name is
|
||||
// far more often an unrelated clip than half of a live photo.
|
||||
"mov" => stem,
|
||||
"mp" => lower.as_str(),
|
||||
_ => return None,
|
||||
};
|
||||
(!key.is_empty()).then(|| format!("{folder}/{key}"))
|
||||
}
|
||||
|
||||
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
||||
@@ -858,13 +1089,13 @@ mod tests {
|
||||
RawEntry {
|
||||
archive,
|
||||
entry: path.to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().1.to_string(),
|
||||
size,
|
||||
}
|
||||
}
|
||||
|
||||
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
||||
let relative = strip_export_prefix(path).unwrap().to_string();
|
||||
let relative = strip_export_prefix(path).unwrap().1.to_string();
|
||||
map.insert(relative, body.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
@@ -872,12 +1103,17 @@ mod tests {
|
||||
fn strips_export_root_and_localized_service_folder() {
|
||||
assert_eq!(
|
||||
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
||||
Some("Island 2019/IMG_1.jpg")
|
||||
Some(("Google Fotos", "Island 2019/IMG_1.jpg"))
|
||||
);
|
||||
// Already unwrapped by the user: no `Takeout/` root left.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
||||
Some("IMG_1.jpg")
|
||||
Some(("Google Photos", "IMG_1.jpg"))
|
||||
);
|
||||
// ... and an album keeps its folder.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"),
|
||||
Some(("Google Photos", "Iceland/IMG_1.jpg"))
|
||||
);
|
||||
// Nothing below a service folder is nothing to import.
|
||||
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
||||
@@ -957,23 +1193,144 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creation_time_is_the_fallback_capture_time() {
|
||||
fn creation_time_is_kept_apart_as_the_last_resort() {
|
||||
// `creationTime` is when the photo reached Google. It must not beat the
|
||||
// date in the name, and the importer tries EXIF before it.
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1700000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
|
||||
entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg",
|
||||
10,
|
||||
),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
|
||||
assert_eq!(scan.photos[0].capture_time, None);
|
||||
assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000));
|
||||
assert_eq!(
|
||||
scan.photos[1].capture_time,
|
||||
capture_time_from_name("IMG_20190612_101500.jpg")
|
||||
);
|
||||
assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_timestamps_tolerate_numbers_zero_and_null() {
|
||||
let mut sidecars = HashMap::new();
|
||||
for (name, body) in [
|
||||
("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#),
|
||||
(
|
||||
"B.jpg",
|
||||
r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#,
|
||||
),
|
||||
(
|
||||
"C.jpg",
|
||||
r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#,
|
||||
),
|
||||
] {
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
&format!("Takeout/Google Photos/Photos from 2019/{name}.json"),
|
||||
body,
|
||||
);
|
||||
}
|
||||
let scan = assemble(
|
||||
["A.jpg", "B.jpg", "C.jpg"]
|
||||
.iter()
|
||||
.map(|name| {
|
||||
entry(
|
||||
0,
|
||||
&format!("Takeout/Google Photos/Photos from 2019/{name}"),
|
||||
1,
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970");
|
||||
assert!(!scan.photos[1].favorite);
|
||||
assert_eq!(scan.photos[2].capture_time, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_comes_from_geo_data_and_zero_means_none() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_1.jpg.json",
|
||||
r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_2.jpg.json",
|
||||
r#"{"geoData":{"latitude":0.0,"longitude":0.0},
|
||||
"geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_3.jpg.json",
|
||||
r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
(1..=3)
|
||||
.map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1))
|
||||
.collect(),
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
scan.photos[0].location,
|
||||
Some(GeoPoint {
|
||||
latitude: 64.1466,
|
||||
longitude: -21.9426
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
scan.photos[1].location.map(|p| p.latitude),
|
||||
Some(63.4186),
|
||||
"geoDataExif fills in for an empty geoData"
|
||||
);
|
||||
assert_eq!(scan.photos[2].location, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_and_motion_photo_halves_share_a_key() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1),
|
||||
entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/scan.png", 1),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
let key = |i: usize| scan.photos[i].live_key.clone();
|
||||
|
||||
assert_eq!(key(0), key(1));
|
||||
assert!(!scan.photos[0].is_motion_photo());
|
||||
assert_eq!(key(2), key(3));
|
||||
assert!(scan.photos[3].is_motion_photo());
|
||||
assert!(scan.photos[3].is_video(), "the .MP half is a video");
|
||||
assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair");
|
||||
assert_eq!(key(5), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1387,16 +1744,77 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_a_zip_is_a_readable_error() {
|
||||
let path =
|
||||
fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() {
|
||||
let broken =
|
||||
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
||||
std::fs::write(&path, b"not a zip at all").unwrap();
|
||||
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
std::fs::write(&broken, b"not a zip at all").unwrap();
|
||||
let good = write_zip(
|
||||
"beside-broken",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
|
||||
let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap();
|
||||
let _ = std::fs::remove_file(&broken);
|
||||
let _ = std::fs::remove_file(&good);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].archive, 1);
|
||||
assert_eq!(scan.unreadable_archives.len(), 1);
|
||||
assert_eq!(scan.unreadable_archives[0].path, broken);
|
||||
assert!(
|
||||
error.message.contains("not a readable zip"),
|
||||
"unexpected message: {}",
|
||||
error.message
|
||||
scan.unreadable_archives[0]
|
||||
.reason
|
||||
.contains("not a readable zip"),
|
||||
"unexpected reason: {}",
|
||||
scan.unreadable_archives[0].reason
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_services_in_the_same_export_are_not_imported() {
|
||||
let zip = write_zip(
|
||||
"services",
|
||||
&[
|
||||
(
|
||||
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg",
|
||||
b"x".as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json",
|
||||
br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(),
|
||||
),
|
||||
("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()),
|
||||
("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()),
|
||||
],
|
||||
);
|
||||
|
||||
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
|
||||
assert_eq!(scan.skipped_other, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_export_without_sidecars_keeps_every_service() {
|
||||
let zip = write_zip(
|
||||
"no-sidecars",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_request_ends_the_scan() {
|
||||
let zip = write_zip(
|
||||
"stop",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
let result = scan_archives_until(std::slice::from_ref(&zip), || true);
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
fuser.workspace = true
|
||||
libc.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
base64.workspace = true
|
||||
image.workspace = true
|
||||
|
||||
@@ -877,35 +877,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
if archives.is_empty() {
|
||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||
} else {
|
||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)))
|
||||
} else {
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
let paths = archives.into_iter().map(PathBuf::from).collect();
|
||||
start_takeout_import(core, paths, dry_run)
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ImportStatus) => {
|
||||
let (running, summary) = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus { running, summary }
|
||||
let report = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus {
|
||||
running: report.running,
|
||||
summary: report.summary,
|
||||
progress: report.progress,
|
||||
resumable: report.resumable,
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
|
||||
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
|
||||
Some(paths) => start_takeout_import(core, paths, false),
|
||||
},
|
||||
Ok(CtlRequest::CancelImport) => {
|
||||
if crate::takeout::cancel_import() {
|
||||
CtlResponse::Ok {
|
||||
@@ -2009,6 +1997,31 @@ pub(crate) fn run_control_socket(
|
||||
}
|
||||
}
|
||||
|
||||
/// Check that every archive is there, then start the import in the background.
|
||||
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)));
|
||||
}
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_limit_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1193,7 +1193,7 @@ impl Core {
|
||||
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !self.rt.block_on(self.refresh_timeline())? {
|
||||
if !self.refresh_timeline_joined()? {
|
||||
return Ok(None);
|
||||
}
|
||||
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||
@@ -2640,6 +2640,36 @@ impl Core {
|
||||
|
||||
/// Refresh the timeline off the request path, so a stale page is still served
|
||||
/// at DB speed. At most one refresh runs at a time.
|
||||
/// Refresh the timeline and wait for it, sharing a refresh already under
|
||||
/// way instead of starting a second one beside it.
|
||||
///
|
||||
/// The first request after an empty cache has nothing to serve until the
|
||||
/// fetch lands, so it waits. Two such requests (the gallery and its retry,
|
||||
/// or a background refresh started by an import) used to each fetch the
|
||||
/// whole library at once, sharing the session's rate limit and taking
|
||||
/// minutes. Returns whether the account has a photos library.
|
||||
pub(crate) fn refresh_timeline_joined(&self) -> CoreResult<bool> {
|
||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||
while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() {
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
}
|
||||
// A refresh that landed stamped the timeline fresh. One that failed
|
||||
// did not, and the caller gets this refresh's own error instead of
|
||||
// an empty timeline.
|
||||
if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL)
|
||||
|| self.timeline_refreshing.swap(true, Ordering::SeqCst)
|
||||
{
|
||||
return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0));
|
||||
}
|
||||
}
|
||||
let refreshed = self.rt.block_on(self.refresh_timeline());
|
||||
self.timeline_refreshing.store(false, Ordering::SeqCst);
|
||||
if refreshed.is_ok() {
|
||||
self.events.publish(&[Topic::Photos]);
|
||||
}
|
||||
refreshed
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_timeline_refresh(&self) {
|
||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||
return;
|
||||
|
||||
+894
-239
File diff suppressed because it is too large.
Load diff
@@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState};
|
||||
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
|
||||
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress,
|
||||
PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope,
|
||||
Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry,
|
||||
ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
|
||||
ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||
DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem,
|
||||
LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo,
|
||||
PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo,
|
||||
SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase,
|
||||
SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||
};
|
||||
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
@@ -668,9 +668,11 @@ fn build_window(app: &adw::Application) {
|
||||
progress_bar: takeout_widgets.progress_bar.clone(),
|
||||
summary_group: takeout_widgets.summary_group.clone(),
|
||||
summary_rows: RefCell::new(Vec::new()),
|
||||
resume_group: takeout_widgets.resume_group.clone(),
|
||||
resume_row: takeout_widgets.resume_row.clone(),
|
||||
resume_button: takeout_widgets.resume_button.clone(),
|
||||
inflight: Cell::new(false),
|
||||
running: Cell::new(false),
|
||||
dry_run: Cell::new(false),
|
||||
},
|
||||
update,
|
||||
});
|
||||
|
||||
@@ -1115,9 +1115,6 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
// count falling to zero is what "the sync finished" looks like from here.
|
||||
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
||||
// starts its upload job mid-flight, which is not a thing finishing.
|
||||
// The Import page reads its progress off the same snapshot rather than
|
||||
// polling the daemon a second time for numbers already on the wire.
|
||||
takeout_progress(ui, jobs);
|
||||
|
||||
let previous = ui.status.active_transfers.replace(items.len());
|
||||
if !items.is_empty() {
|
||||
@@ -1211,13 +1208,18 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
|
||||
let done = j.done.to_string();
|
||||
let total = j.total.to_string();
|
||||
let (title, detail) = if j.title == IMPORT_JOB {
|
||||
import_job_text(&j.detail)
|
||||
} else {
|
||||
(j.title.clone(), j.detail.clone())
|
||||
};
|
||||
let args: [(&str, &str); 4] = [
|
||||
("title", &j.title),
|
||||
("detail", &j.detail),
|
||||
("title", &title),
|
||||
("detail", &detail),
|
||||
("done", &done),
|
||||
("total", &total),
|
||||
];
|
||||
let text = match (j.total > 0, j.detail.is_empty()) {
|
||||
let text = match (j.total > 0, detail.is_empty()) {
|
||||
// Translators: a background job's progress. {title} is the job, such as
|
||||
// "Scanning Photos"; {done} and {total} are counts.
|
||||
(true, true) => gettext_f("{title} — {done} of {total}", &args),
|
||||
|
||||
@@ -15,17 +15,30 @@
|
||||
//! Import — staging is local, so a mis-drop costs nothing.
|
||||
//!
|
||||
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
||||
//! stages paths, launches, asks [`Request::ImportStatus`] for the final report,
|
||||
//! and reads live progress off the import's job in the transfer snapshot the
|
||||
//! daemon already pushes (see [`takeout_progress`]).
|
||||
//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the
|
||||
//! running counts and the final report. It asks again whenever the import's job
|
||||
//! moves in the transfer snapshot the daemon already pushes. A run the service
|
||||
//! did not get to finish can be picked up with [`Request::ResumeImport`].
|
||||
|
||||
use crate::*;
|
||||
|
||||
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
||||
/// Matching on it is how this page borrows the progress the daemon already
|
||||
/// pushes instead of asking for it separately.
|
||||
/// Seeing it move is the cue to ask for the import's status; the Activity list
|
||||
/// shows it under its translated name ([`import_job_text`]).
|
||||
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
|
||||
|
||||
/// The import job's title and detail in the user's language. The daemon names
|
||||
/// its steps in English; the file being uploaded passes through as it is.
|
||||
pub(crate) fn import_job_text(detail: &str) -> (String, String) {
|
||||
let detail = match detail {
|
||||
"reading archives" => gettext("Reading the archives"),
|
||||
"checking for photos you already have" => gettext("Checking for photos you already have"),
|
||||
"sorting photos into albums" => gettext("Sorting photos into albums"),
|
||||
other => other.to_string(),
|
||||
};
|
||||
(gettext("Importing from Google Photos"), detail)
|
||||
}
|
||||
|
||||
/// What the archive list says before anything is staged. `repaint_archives`
|
||||
/// swaps in the count and total once there is something to report, and swaps
|
||||
/// this back when the list is emptied again.
|
||||
@@ -95,18 +108,19 @@ pub(crate) struct TakeoutState {
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
/// Report group for the last finished run, hidden until there is one.
|
||||
/// The running counts, or the report for the last finished run; hidden
|
||||
/// until there is one.
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
||||
/// Offers to resume an import the service did not get to finish.
|
||||
pub(crate) resume_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_row: adw::ActionRow,
|
||||
pub(crate) resume_button: gtk4::Button,
|
||||
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
||||
/// can't stack worker threads on the 2s tick.
|
||||
pub(crate) inflight: Cell<bool>,
|
||||
/// Whether the daemon reported an import running on the last poll.
|
||||
pub(crate) running: Cell<bool>,
|
||||
/// Whether the run in flight was launched as a dry run, so the report can
|
||||
/// say "would upload" rather than claiming photos were sent. The daemon's
|
||||
/// summary carries no such flag — it is the same shape either way.
|
||||
pub(crate) dry_run: Cell<bool>,
|
||||
}
|
||||
|
||||
/// Widgets the Import page hands back for wiring.
|
||||
@@ -126,6 +140,9 @@ pub(crate) struct TakeoutWidgets {
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_row: adw::ActionRow,
|
||||
pub(crate) resume_button: gtk4::Button,
|
||||
}
|
||||
|
||||
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
@@ -261,7 +278,23 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
.visible(false)
|
||||
.build();
|
||||
|
||||
// An import cut short by a restart or a lost connection, offered back at the
|
||||
// top of the page: picking the archives again is exactly what a reboot in
|
||||
// the middle of a migration should not cost.
|
||||
let resume_button = gtk4::Button::builder()
|
||||
.label(pgettext("verb", "Resume"))
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
resume_button.add_css_class("suggested-action");
|
||||
let resume_row = adw::ActionRow::builder()
|
||||
.title(gettext("The last import didn't finish"))
|
||||
.build();
|
||||
resume_row.add_suffix(&resume_button);
|
||||
let resume_group = adw::PreferencesGroup::builder().visible(false).build();
|
||||
resume_group.add(&resume_row);
|
||||
|
||||
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
||||
groups.append(&resume_group);
|
||||
groups.append(&dropzone);
|
||||
groups.append(&list_group);
|
||||
groups.append(&actions);
|
||||
@@ -301,6 +334,9 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
progress_label: progress_label.clone(),
|
||||
progress_bar: progress_bar.clone(),
|
||||
summary_group: summary_group.clone(),
|
||||
resume_group,
|
||||
resume_row,
|
||||
resume_button,
|
||||
};
|
||||
(frame.upcast(), widgets)
|
||||
}
|
||||
@@ -427,6 +463,28 @@ pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
|
||||
});
|
||||
});
|
||||
|
||||
let ui_resume = ui.clone();
|
||||
widgets.resume_button.connect_clicked(move |_| {
|
||||
let ui = ui_resume.clone();
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport);
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => import_started(&ui, false),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
&gettext("Couldn't resume the import"),
|
||||
&gettext("The Proton Drive service didn't respond."),
|
||||
),
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let ui_back = ui.clone();
|
||||
widgets
|
||||
.back_button
|
||||
@@ -559,6 +617,7 @@ fn sync_takeout_actions(ui: &Rc<Ui>) {
|
||||
.import_button
|
||||
.set_sensitive(has_files && !running);
|
||||
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
||||
ui.takeout.resume_button.set_sensitive(!running);
|
||||
ui.takeout.cancel_button.set_visible(running);
|
||||
}
|
||||
|
||||
@@ -577,7 +636,6 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
return;
|
||||
}
|
||||
|
||||
ui.takeout.dry_run.set(dry_run);
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(
|
||||
ui.dirs.control_socket(),
|
||||
@@ -588,27 +646,7 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => {
|
||||
ui.takeout.running.set(true);
|
||||
ui.gallery.import_banner.set_revealed(true);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout.progress_label.set_text(&if dry_run {
|
||||
gettext("Reading archives…")
|
||||
} else {
|
||||
gettext("Starting import…")
|
||||
});
|
||||
sync_takeout_actions(&ui);
|
||||
toast(
|
||||
&ui,
|
||||
&if dry_run {
|
||||
gettext("Scanning the export…")
|
||||
} else {
|
||||
gettext("Import started — you can leave this page, it keeps running.")
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
|
||||
}
|
||||
@@ -621,9 +659,39 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
});
|
||||
}
|
||||
|
||||
/// Ask the daemon for the import's state, and paint the report once a run has
|
||||
/// finished. Asked on arrival and while import progress keeps arriving. Cheap
|
||||
/// and inflight-guarded.
|
||||
/// Switch the page over to reporting progress once the daemon took the run.
|
||||
fn import_started(ui: &Rc<Ui>, dry_run: bool) {
|
||||
ui.takeout.running.set(true);
|
||||
if !dry_run {
|
||||
ui.gallery.import_banner.set_revealed(true);
|
||||
}
|
||||
ui.takeout.resume_group.set_visible(false);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_group.set_title(&if dry_run {
|
||||
gettext("Scan in progress")
|
||||
} else {
|
||||
gettext("Import in progress")
|
||||
});
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&gettext("Reading the archives…"));
|
||||
sync_takeout_actions(ui);
|
||||
toast(
|
||||
ui,
|
||||
&if dry_run {
|
||||
gettext("Scanning the export…")
|
||||
} else {
|
||||
gettext("Import started — you can leave this page, it keeps running.")
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Ask the daemon for the import's state and paint it: the step and counts
|
||||
/// while it runs, the report once it has ended, and the offer to resume a run
|
||||
/// that did not finish. Asked on arrival and while import progress keeps
|
||||
/// arriving. Cheap and inflight-guarded.
|
||||
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
if ui.takeout.inflight.get() {
|
||||
return;
|
||||
@@ -634,124 +702,250 @@ pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.takeout.inflight.set(false);
|
||||
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
|
||||
let Ok(Ok(Response::ImportStatus {
|
||||
running,
|
||||
summary,
|
||||
progress,
|
||||
resumable,
|
||||
})) = result
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let was_running = ui.takeout.running.replace(running);
|
||||
ui.gallery.import_banner.set_revealed(running);
|
||||
let dry = summary.as_ref().is_some_and(|s| s.dry_run);
|
||||
ui.gallery.import_banner.set_revealed(running && !dry);
|
||||
ui.takeout.progress_group.set_visible(running);
|
||||
if running {
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
} else {
|
||||
ui.takeout.progress_group.set_visible(false);
|
||||
// An import runs for hours, so its ending is announced rather than
|
||||
// left to be discovered: a toast for whoever is looking at the
|
||||
// window, and a desktop notification for whoever is not.
|
||||
let failed = summary.as_ref().and_then(|s| s.error.clone());
|
||||
if was_running && let Some(error) = &failed {
|
||||
let headline = gettext("Couldn't import from Google Photos");
|
||||
toast_error(&ui, &headline, error);
|
||||
notify("takeout-import", &headline, error);
|
||||
} else if was_running {
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
let headline = if dry {
|
||||
gettext("Scan finished")
|
||||
} else {
|
||||
gettext("Import finished")
|
||||
};
|
||||
toast(&ui, &headline);
|
||||
if !dry {
|
||||
let body = match &summary {
|
||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||
Some(s) if s.failed > 0 => ngettext_f(
|
||||
"{n} photo added, {failed} failed.",
|
||||
"{n} photos added, {failed} failed.",
|
||||
s.uploaded as u64,
|
||||
&[("failed", &s.failed.to_string())],
|
||||
),
|
||||
Some(s) if s.cancelled => ngettext_f(
|
||||
"Stopped after {n} photo.",
|
||||
"Stopped after {n} photos.",
|
||||
s.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
Some(s) => ngettext_f(
|
||||
"{n} photo added to Proton Photos.",
|
||||
"{n} photos added to Proton Photos.",
|
||||
s.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
None => gettext("The Google Photos import has finished."),
|
||||
};
|
||||
notify("takeout-import", &headline, &body);
|
||||
}
|
||||
}
|
||||
ui.takeout.progress_group.set_title(&if dry {
|
||||
gettext("Scan in progress")
|
||||
} else {
|
||||
gettext("Import in progress")
|
||||
});
|
||||
paint_progress(&ui, progress.as_ref());
|
||||
} else if was_running && let Some(summary) = &summary {
|
||||
announce_end(&ui, summary);
|
||||
}
|
||||
if was_running != running {
|
||||
sync_takeout_actions(&ui);
|
||||
}
|
||||
match summary {
|
||||
Some(summary) => repaint_summary(&ui, &summary),
|
||||
match &summary {
|
||||
Some(summary) => repaint_summary(&ui, summary, running),
|
||||
None => ui.takeout.summary_group.set_visible(false),
|
||||
}
|
||||
paint_resume(&ui, running, summary.as_ref(), &resumable);
|
||||
});
|
||||
}
|
||||
|
||||
/// Paint the last run's report.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
/// Tell the user how a run ended. An import runs for hours, so its ending is
|
||||
/// announced rather than left to be discovered: a toast for whoever is looking
|
||||
/// at the window, and a desktop notification for whoever is not.
|
||||
fn announce_end(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
if summary.dry_run {
|
||||
toast(ui, &gettext("Scan finished"));
|
||||
return;
|
||||
}
|
||||
if let Some(error) = &summary.error {
|
||||
let kind = summary.error_kind.unwrap_or_default();
|
||||
let headline = gettext("Couldn't import from Google Photos");
|
||||
toast_failure(ui, &headline, error, kind);
|
||||
notify(
|
||||
"takeout-import",
|
||||
&headline,
|
||||
&gettext("You can resume it once the problem is solved."),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let (headline, body) = if summary.interrupted {
|
||||
(
|
||||
gettext("Import interrupted"),
|
||||
gettext("You can resume it where it stopped."),
|
||||
)
|
||||
} else if summary.cancelled {
|
||||
(
|
||||
gettext("Import stopped"),
|
||||
ngettext_f(
|
||||
"Stopped after {n} photo.",
|
||||
"Stopped after {n} photos.",
|
||||
summary.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
)
|
||||
} else if summary.failed > 0 {
|
||||
(
|
||||
gettext("Import finished"),
|
||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||
ngettext_f(
|
||||
"{n} photo added, {failed} failed.",
|
||||
"{n} photos added, {failed} failed.",
|
||||
summary.uploaded as u64,
|
||||
&[("failed", &summary.failed.to_string())],
|
||||
),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
gettext("Import finished"),
|
||||
ngettext_f(
|
||||
"{n} photo added to Proton Photos.",
|
||||
"{n} photos added to Proton Photos.",
|
||||
summary.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
)
|
||||
};
|
||||
toast(ui, &headline);
|
||||
notify("takeout-import", &headline, &body);
|
||||
|
||||
// A complete import is done with its archives; keeping them staged would
|
||||
// only invite running the same export again.
|
||||
if summary.complete() {
|
||||
ui.takeout.archives.borrow_mut().clear();
|
||||
repaint_archives(ui);
|
||||
}
|
||||
}
|
||||
|
||||
/// Paint where a running import is.
|
||||
fn paint_progress(ui: &Rc<Ui>, progress: Option<&ImportProgress>) {
|
||||
let Some(progress) = progress else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
return;
|
||||
};
|
||||
let done = progress.done.to_string();
|
||||
let total = progress.total.to_string();
|
||||
let current = progress.current.clone().unwrap_or_default();
|
||||
let counted = progress.total > 0;
|
||||
let args = [
|
||||
("done", done.as_str()),
|
||||
("total", total.as_str()),
|
||||
("name", current.as_str()),
|
||||
];
|
||||
let text = match (progress.phase, counted) {
|
||||
(ImportPhase::Scanning, _) => gettext("Reading the archives…"),
|
||||
(ImportPhase::Checking, false) => gettext("Checking for photos you already have…"),
|
||||
// Translators: {done} and {total} are counts of photos.
|
||||
(ImportPhase::Checking, true) => gettext_f(
|
||||
"Checking for photos you already have — {done} of {total}",
|
||||
&args,
|
||||
),
|
||||
(ImportPhase::Uploading, false) => gettext("Uploading…"),
|
||||
(ImportPhase::Uploading, true) if current.is_empty() => {
|
||||
// Translators: {done} and {total} are counts of photos.
|
||||
gettext_f("Uploading {done} of {total}", &args)
|
||||
}
|
||||
// Translators: {done} and {total} are counts of photos; {name} is the file being uploaded.
|
||||
(ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args),
|
||||
(ImportPhase::Albums, false) => gettext("Sorting photos into albums…"),
|
||||
// Translators: {done} and {total} are counts of albums.
|
||||
(ImportPhase::Albums, true) => {
|
||||
gettext_f("Sorting photos into albums — {done} of {total}", &args)
|
||||
}
|
||||
};
|
||||
ui.takeout.progress_label.set_text(&text);
|
||||
if counted {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((progress.done as f64 / progress.total as f64).min(1.0));
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
}
|
||||
}
|
||||
|
||||
/// Offer to resume the last run when the service says it can be.
|
||||
fn paint_resume(ui: &Rc<Ui>, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) {
|
||||
let offer = !running && !resumable.is_empty();
|
||||
ui.takeout.resume_group.set_visible(offer);
|
||||
if !offer {
|
||||
return;
|
||||
}
|
||||
let uploaded = summary.map_or(0, |s| s.uploaded) as u64;
|
||||
// Translators: {n} is the number of photos uploaded before the import stopped.
|
||||
ui.takeout.resume_row.set_subtitle(&ngettext_f(
|
||||
"{n} photo uploaded before it stopped. Resuming skips what is already there.",
|
||||
"{n} photos uploaded before it stopped. Resuming skips what is already there.",
|
||||
uploaded,
|
||||
&[],
|
||||
));
|
||||
}
|
||||
|
||||
/// Paint the counts of a running import, or the report of the last one.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
|
||||
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
||||
ui.takeout.summary_group.remove(&row);
|
||||
}
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
ui.takeout.summary_group.set_title(&if dry {
|
||||
let dry = summary.dry_run;
|
||||
ui.takeout.summary_group.set_title(&if running {
|
||||
gettext("So far")
|
||||
} else if dry {
|
||||
gettext("Scan result")
|
||||
} else {
|
||||
gettext("Last import")
|
||||
});
|
||||
ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error {
|
||||
// The reason is the service's own text, which is not translated. A
|
||||
// group's description is markup, and the reason may name a path.
|
||||
glib::markup_escape_text(&gettext_f(
|
||||
"{what}: {detail}",
|
||||
&[
|
||||
("what", &gettext("Couldn't import from Google Photos")),
|
||||
("detail", error),
|
||||
],
|
||||
))
|
||||
.to_string()
|
||||
let description = if running {
|
||||
None
|
||||
} else if let Some(error) = &summary.error {
|
||||
// The reason is the service's own text, which is not translated; the
|
||||
// headline in front of it is. A group's description is markup, and the
|
||||
// reason may name a path.
|
||||
let what = gettext("Couldn't import from Google Photos");
|
||||
Some(
|
||||
glib::markup_escape_text(&gettext_f(
|
||||
"{what}: {detail}",
|
||||
&[
|
||||
(
|
||||
"what",
|
||||
error_headline(summary.error_kind.unwrap_or_default(), &what),
|
||||
),
|
||||
("detail", error),
|
||||
],
|
||||
))
|
||||
.to_string(),
|
||||
)
|
||||
} else if dry {
|
||||
gettext("Nothing was uploaded — this is what an import would do.")
|
||||
Some(gettext(
|
||||
"Nothing was uploaded — this is what an import would do.",
|
||||
))
|
||||
} else if summary.interrupted {
|
||||
Some(gettext(
|
||||
"The Proton Drive service stopped while the import ran. Resume it to continue where it left off.",
|
||||
))
|
||||
} else if summary.cancelled {
|
||||
gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.")
|
||||
Some(gettext(
|
||||
"Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.",
|
||||
))
|
||||
} else if !summary.complete() {
|
||||
Some(gettext(
|
||||
"Finished with problems. Run it again to retry what didn't upload.",
|
||||
))
|
||||
} else {
|
||||
gettext("Finished.")
|
||||
}));
|
||||
Some(gettext("Finished."))
|
||||
};
|
||||
ui.takeout
|
||||
.summary_group
|
||||
.set_description(description.as_deref());
|
||||
|
||||
// Only lines that carry information: a run with no failures should not have
|
||||
// to show a "0 failed" row for the user to read past.
|
||||
let mut lines: Vec<(String, String)> = vec![
|
||||
(gettext("Photos in the export"), summary.found.to_string()),
|
||||
(
|
||||
if dry {
|
||||
gettext("Would upload")
|
||||
} else {
|
||||
gettext("Uploaded")
|
||||
},
|
||||
summary.uploaded.to_string(),
|
||||
),
|
||||
];
|
||||
let mut lines: Vec<(String, String)> =
|
||||
vec![(gettext("Photos in the export"), summary.found.to_string())];
|
||||
if dry {
|
||||
lines.push((gettext("Would upload"), summary.to_upload.to_string()));
|
||||
} else {
|
||||
lines.push((gettext("Uploaded"), summary.uploaded.to_string()));
|
||||
}
|
||||
if summary.duplicates > 0 {
|
||||
lines.push((
|
||||
gettext("Already in Proton Photos"),
|
||||
summary.duplicates.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.live_photos > 0 {
|
||||
lines.push((
|
||||
gettext("Live photos kept together"),
|
||||
summary.live_photos.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
gettext("Albums to create")
|
||||
} else {
|
||||
gettext("Albums created")
|
||||
},
|
||||
gettext("Albums created"),
|
||||
summary.albums_created.to_string(),
|
||||
));
|
||||
}
|
||||
@@ -767,25 +961,31 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
summary.skipped_trashed.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
if dry && summary.to_upload_bytes > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
gettext("Data to upload")
|
||||
} else {
|
||||
gettext("Data uploaded")
|
||||
},
|
||||
human_bytes(summary.bytes),
|
||||
gettext("Data to upload"),
|
||||
human_bytes(summary.to_upload_bytes),
|
||||
));
|
||||
} else if summary.bytes > 0 {
|
||||
lines.push((gettext("Data uploaded"), human_bytes(summary.bytes)));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
|
||||
}
|
||||
for path in &summary.unreadable_archives {
|
||||
let name = Path::new(path)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| path.clone());
|
||||
lines.push((gettext("Couldn't read this archive"), name));
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (title, value) in lines {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(title)
|
||||
.subtitle(value)
|
||||
.use_markup(false)
|
||||
.build();
|
||||
row.add_css_class("property");
|
||||
ui.takeout.summary_group.add(&row);
|
||||
@@ -794,35 +994,3 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
*ui.takeout.summary_rows.borrow_mut() = rows;
|
||||
ui.takeout.summary_group.set_visible(true);
|
||||
}
|
||||
|
||||
/// Paint the import's progress from the transfer snapshot the daemon pushed,
|
||||
/// so this page costs no extra request. A job with no known total
|
||||
/// pulses instead of showing a fraction, the same as the Activity list.
|
||||
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
|
||||
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
|
||||
return;
|
||||
};
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
if job.total > 0 {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
|
||||
let done = job.done.to_string();
|
||||
let total = job.total.to_string();
|
||||
// Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on.
|
||||
let text = gettext_f(
|
||||
"{done} of {total} — {detail}",
|
||||
&[("done", &done), ("total", &total), ("detail", &job.detail)],
|
||||
);
|
||||
ui.takeout.progress_label.set_text(&text);
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&if job.detail.is_empty() {
|
||||
gettext("Working…")
|
||||
} else {
|
||||
job.detail.clone()
|
||||
});
|
||||
}
|
||||
}
|
||||
+161
@@ -109,6 +109,11 @@ a live account since.
|
||||
|
||||
| Entry | Title |
|
||||
|---|---|
|
||||
| [B289](#b289--the-takeout-import-mixes-in-other-services-and-drops-location-and-live-photos) | The Takeout import mixes in other services and drops location and live photos |
|
||||
| [B288](#b288--an-interrupted-takeout-import-reads-as-finished-and-one-error-ends-it) | An interrupted Takeout import reads as finished, and one error ends it |
|
||||
| [B287](#b287--a-photo-draft-left-by-an-interrupted-upload-blocks-that-name) | A photo draft left by an interrupted upload blocks that name |
|
||||
| [B286](#b286--resuming-a-takeout-import-fails-on-a-draft-with-no-content-hash) | Resuming a Takeout import fails on a draft with no content hash |
|
||||
| [B285](#b285--a-restarted-secret-service-leaves-the-process-on-a-dead-session) | A restarted Secret Service leaves the process on a dead session |
|
||||
| [B284](#b284--a-keyring-entry-reported-missing-discards-refreshed-session-tokens) | A keyring entry reported missing discards refreshed session tokens |
|
||||
| [B283](#b283--videos-have-no-thumbnail-outside-photos-and-a-ts-recording-looks-like-source-code) | Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code |
|
||||
| [B282](#b282--a-file-kept-offline-is-downloaded-again-after-a-change-made-elsewhere) | A file kept offline is downloaded again after a change made elsewhere |
|
||||
@@ -387,6 +392,162 @@ a live account since.
|
||||
|
||||
Newest first.
|
||||
|
||||
### B289 — The Takeout import mixes in other services and drops location and live photos
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, auditing the Google Photos import alongside
|
||||
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||
**Where:** `crates/pdfs-core/src/takeout.rs`; `plan` and `upload_takeout_photo` in
|
||||
`crates/pdfs-fuse/src/takeout.rs`
|
||||
|
||||
**Repro:** Import a Takeout export that also holds `Drive/` or `YouTube/`, with Pixel motion
|
||||
photos, iPhone live photos and photos that have a place in Google Photos.
|
||||
|
||||
**Cause.** Five gaps in what the import carried over:
|
||||
- The service folder was dropped without being checked, so media from other Google services in
|
||||
the same export landed in the photo timeline.
|
||||
- `geoData` was never read, although `docs/PHOTOS.md` said the import takes locations from the
|
||||
export, and the SDK had no way to upload a location.
|
||||
- The two halves of a live or motion photo uploaded as unrelated photos, the `.MOV` half usually
|
||||
without a date, and the `.MP` half of a Pixel motion photo was skipped as unknown.
|
||||
- `creationTime`, the day the photo reached Google, outranked the date in the file name and the
|
||||
photo's own EXIF.
|
||||
- A sidecar with `null` in `favorited`, a numeric `timestamp` or a `0` timestamp lost its other
|
||||
fields or filed the photo under 1970.
|
||||
|
||||
**Fix:** The Photos service is the folder whose sidecars carry `photoTakenTime`, and only it is
|
||||
imported (everything, when no folder does). `geoData`, else `geoDataExif`, becomes the photo's
|
||||
`Location` attribute through SDK 0.12.0's `PhotoUploadMetadata::location`; `0, 0` is no position.
|
||||
Stills and videos with the same stem in one folder (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||
`PXL_1.MP.jpg` + `PXL_1.MP`) are paired unless their dates are minutes apart: the still is
|
||||
tagged as a live or motion photo, the video is uploaded after it with `main_photo_uid` and takes
|
||||
its date and place. The capture time is `photoTakenTime`, then the name, then EXIF, then
|
||||
`creationTime`. Favorites are also set on photos that were already on the account.
|
||||
|
||||
**Verified:** `takeout::tests` in `pdfs-core` (services, locations, timestamps, live keys) and
|
||||
`pdfs-fuse` (pairing, skew, folded copies). Not run against a real export and account.
|
||||
|
||||
---
|
||||
|
||||
### B288 — An interrupted Takeout import reads as finished, and one error ends it
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, auditing the Google Photos import alongside
|
||||
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||
**Where:** `crates/pdfs-fuse/src/takeout.rs`; `refresh_takeout` in
|
||||
`crates/pdfs-gui/src/app/pages/takeout.rs`; `cmd_import_google_photos` in
|
||||
`crates/pdfs-cli/src/main.rs`
|
||||
|
||||
**Repro:** Start an import of several archives, one of them damaged, and restart the service
|
||||
while it uploads.
|
||||
|
||||
**Cause.**
|
||||
- The import state lived only in memory. After a restart `ImportStatus` had no summary, and the
|
||||
app announced "Import finished". The archives had to be chosen again to continue.
|
||||
- A daemon shutdown broke the upload loop but still filed albums and logged a success.
|
||||
- One unreadable archive failed the scan, and a failed request in the middle of the run
|
||||
discarded every count and skipped the albums for what had uploaded.
|
||||
- The progress total counted photos already on the account, so the bar stopped short of the
|
||||
end; a dry run reported "Would upload: 0"; the job's phases reached the app in English.
|
||||
|
||||
**Fix:** The last run's archives and summary are kept in `takeout-import.json` in the state
|
||||
directory. A run still marked running at start-up is `interrupted`, and `ImportStatus` lists
|
||||
its archives as `resumable`; `ResumeImport` (`pdfs import-google-photos --resume`, **Resume** in
|
||||
the app) starts it again, and the duplicate check skips what is there. An unreadable archive is
|
||||
listed in `unreadable_archives` and the rest imports. A failure keeps the counts and still files
|
||||
albums. `ImportStatus` carries the phase and its own progress, the summary says `dry_run`,
|
||||
`to_upload` and `error_kind`, and the app words them in the user's language. The inline
|
||||
timeline refresh of an empty gallery waits for a refresh already running instead of starting a
|
||||
second one (`refresh_timeline_joined`).
|
||||
|
||||
**Verified:** `takeout::tests` in `pdfs-fuse` (saved state, resumable runs) and `pdfs-core`
|
||||
(unreadable archive), and the CLI tests for `--resume` and the exit status. Not run live.
|
||||
|
||||
---
|
||||
|
||||
### B287 — A photo draft left by an interrupted upload blocks that name
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, reading the SDK upload path for
|
||||
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||
**Where:** `upload_photo_from`, `upload_file_from_inner` and `abandon_file_draft` in
|
||||
`proton-drive-rs` (SDK 0.12.0)
|
||||
|
||||
**Repro:** Interrupt a photo upload after its draft is created, sign in again, and upload a
|
||||
photo with the same name.
|
||||
|
||||
**Cause.** Photo uploads replaced a leftover draft only when its `ClientUID` was the current
|
||||
session's. A new login is a new session, so the next upload of that name failed with 2500. A
|
||||
failed upload also never deleted its own draft, which the C# and TypeScript SDKs do.
|
||||
|
||||
**Fix:** SDK 0.12.0 adds `PhotoUploadMetadata::replace_existing_draft`, which the Takeout import
|
||||
sets, and deletes the draft of a new file or photo whose upload fails. After a failed seal it
|
||||
only does so on a definite rejection, because a seal lost in transit may have been applied.
|
||||
|
||||
**Verified:** SDK unit test `draft_cleanup_only_after_a_definite_seal_rejection`. Not run live.
|
||||
|
||||
---
|
||||
|
||||
### B286 — Resuming a Takeout import fails on a draft with no content hash
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, [issue #94](https://github.com/narrrl/proton-drive-linux/issues/94)
|
||||
(3.7.0, Fedora). An import stopped by a restart failed every re-run at once with
|
||||
`check for existing photos: serialization error: invalid type: null, expected a string at line
|
||||
1 column 2063`.
|
||||
**Where:** `FoundPhotoDuplicateDto` in `proton-drive-rs` (SDK 0.11.0)
|
||||
|
||||
**Repro:** Interrupt a photo upload after its draft is created, then run the import again.
|
||||
|
||||
**Cause.** The duplicates endpoint sends `ContentHash` (and can send `Hash`) as `null` for a
|
||||
draft, as its API schema says. The SDK declared both as `String` with `#[serde(default)]`,
|
||||
which covers a missing field but not `null`, so one draft failed the whole batch of 150 names
|
||||
and the import ended before uploading anything.
|
||||
|
||||
**Fix:** SDK 0.12.0 makes both fields optional and skips drafts, trashed links and candidates
|
||||
without hashes, like the TypeScript SDK. Every other defaulted DTO field reads `null` as its
|
||||
default too, and an undecodable response is logged with its path.
|
||||
|
||||
**Verified:** SDK unit tests `duplicate_response_tolerates_null_fields_and_skips_drafts` and
|
||||
`duplicate_response_tolerates_a_null_list`. Not run against the reporter's account.
|
||||
|
||||
---
|
||||
|
||||
### B285 — A restarted Secret Service leaves the process on a dead session
|
||||
|
||||
**Status:** Fixed (unverified)
|
||||
**Found:** 2026-11-14, following up [issue #78](https://github.com/narrrl/proton-drive-linux/issues/78)
|
||||
(signed out after every cold boot on Fedora 45, which replaced gnome-keyring with oo7). See
|
||||
[the investigation](ISSUE-78.md#a-restarted-secret-service).
|
||||
**Where:** `crates/pdfs-core/src/auth.rs`, `with_keyring_entry`
|
||||
|
||||
**Repro:** Write the session blob, restart the Secret Service provider
|
||||
(`gnome-keyring-daemon --replace`, or `oo7-daemon --replace`), then read or write it again from
|
||||
the same process. `dbus-secret-service-keyring-store` keeps one D-Bus connection and one Secret
|
||||
Service session for the life of the process, and pdfs installed it once. The session exists only
|
||||
in the provider's memory, so the new provider answers every `GetSecret` and `SetSecret` with
|
||||
`org.freedesktop.Secret.Error.NoSession`, for as long as the process runs. A fresh store in the
|
||||
same process reads the same entry fine. oo7's systemd unit restarts the daemon on failure, and
|
||||
both providers can be replaced, so the daemon could meet this mid-session. A token refresh after
|
||||
that point reached only memory and the kernel keyring: B61's retry kept asking on the dead
|
||||
session, and the next cold boot read the spent refresh token (`InvalidRefreshToken`).
|
||||
|
||||
**Fix:** Every keyring call goes through `with_keyring_entry`. A platform failure (which is what
|
||||
`NoSession` and other D-Bus errors become) logs "the Secret Service failed this process's
|
||||
connection; reconnecting", drops the store unless another thread already replaced it, and makes
|
||||
the call once more on a new connection and session. A locked keyring, a dismissed prompt and a
|
||||
missing entry come back unchanged. Affects the daemon, the app and the tray alike.
|
||||
|
||||
**Verified:** `auth::tests::a_dropped_secret_service_session_is_reconnected_and_retried`,
|
||||
`a_rotation_lands_at_once_after_the_keyring_restarts` and `a_locked_keyring_is_not_reconnected`
|
||||
on the in-memory store. The ignored `auth::tests::real_secret_service_survives_a_provider_restart`
|
||||
passes against gnome-keyring 51.1 and oo7 0.7.1 (built from its tag), each in a throwaway session
|
||||
bus and restarted with `--replace`; it checks that a reconnect actually happened. A standalone
|
||||
program holding one store, as before the fix, gets `NoSession` on read and write after the same
|
||||
restart under both providers. Whether the reporter's oo7 restarts during a session is not known.
|
||||
|
||||
---
|
||||
|
||||
### B284 — A keyring entry reported missing discards refreshed session tokens
|
||||
|
||||
**Status:** Fixed (unverified)
|
||||
|
||||
+45
-2
@@ -112,18 +112,61 @@ Existing folders stay device backups. Older builds cannot open the upgraded data
|
||||
carry their cover as their thumbnail, so they show it at once on other devices too. A cover
|
||||
image added to a folder later is picked up once the song changes or the service restarts.
|
||||
|
||||
- **Resume a Google Photos import** (B288). The archives and the outcome of the last import are
|
||||
kept, so an import cut short by a restart or a reboot no longer has to be set up again. The
|
||||
Import page offers **Resume**, and `pdfs import-google-photos --resume` does the same. Photos
|
||||
that are already uploaded are skipped as before.
|
||||
|
||||
### Changed
|
||||
|
||||
- **`proton-sdk` and `proton-drive-rs` 0.11.0.** Timestamps are converted where they enter
|
||||
- **`proton-sdk` and `proton-drive-rs` 0.12.0.** Timestamps are converted where they enter
|
||||
from the SDK, and a photo keeps its own modification time now that it no longer follows the
|
||||
capture time. [The SDK review](SDK-0.11.0.md) lists the new features and breaking changes.
|
||||
capture time. 0.12.0 fixes the photo duplicate check and failed-upload drafts below.
|
||||
[The 0.11.0 review](SDK-0.11.0.md) and [the 0.12.0 review](SDK-0.12.0.md) list the new
|
||||
features and breaking changes.
|
||||
|
||||
- **The Google Photos import brings more of the export along** (B289).
|
||||
- Places from Google Photos are kept, so imported photos show up under **Places**.
|
||||
- Live photos and Pixel motion photos stay one photo: the video is filed with its still and
|
||||
takes its date. The `.MP` half of a motion photo is no longer skipped.
|
||||
- Only the Google Photos part of an export is imported. Files from Google Drive, YouTube and
|
||||
other services in the same Takeout stay out of the timeline.
|
||||
- A photo without a date of its own is dated by its file name and then by its EXIF before
|
||||
the day it reached Google Photos.
|
||||
- Favorites are also set on photos that were already in Proton Photos.
|
||||
|
||||
- **The import reports what it is doing** (B288). The Import page shows the current step
|
||||
(reading, checking for photos you already have, uploading, sorting into albums), the counts
|
||||
so far, and a progress bar that reaches the end. A scan says how many photos and how much
|
||||
data an import would upload. `pdfs import-google-photos --wait` prints its progress, and
|
||||
`pdfs import-status` shows the step.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Resuming a Google Photos import no longer fails at once** (#94, B286, B287). A photo upload
|
||||
that was cut short left a draft behind, and every later run stopped with
|
||||
`check for existing photos: serialization error: invalid type: null`. Such drafts are now
|
||||
skipped by the duplicate check and replaced by the next upload of that photo, also after
|
||||
signing in again. A failed upload removes its draft.
|
||||
|
||||
- **One bad archive or error no longer ends the whole import** (B288). An archive that cannot be
|
||||
read is left out and named in the report, and the rest of the export imports. When an import
|
||||
stops early, it keeps the counts of what it did and still files the uploaded photos into their
|
||||
albums. A restart of the service while importing is reported as an interruption, not as a
|
||||
finished import.
|
||||
|
||||
- **Photos zoomed all the way out shows the whole timeline.** The first 200 photos fit on one
|
||||
screen there, and with nothing to scroll no more were loaded. More now load whenever the photos
|
||||
don't fill the window, also after a zoom or a resize.
|
||||
|
||||
- **Signed out after a cold boot on Fedora 45** (issue #78, B284, B285). Fedora 45 replaced the
|
||||
GNOME keyring with oo7, and two things there could keep a refreshed sign-in from being saved:
|
||||
oo7 reports the entries of a locked keyring as missing, which the client took for a sign-out,
|
||||
and a restarted oo7 no longer knew the client's connection, which then failed every save until
|
||||
the service restarted. The refreshed sign-in lived only in memory, and the next boot found the
|
||||
spent one. The client now keeps such a sign-in and saves it once the keyring can be read, and
|
||||
reconnects to a keyring service that restarted.
|
||||
|
||||
- **Thumbnails appear in the folder you are looking at as soon as they are made.** During
|
||||
**Build thumbnails**, the folder on screen kept its file icons until you left it and opened it
|
||||
again. Its tiles now fill in while the build runs, and the same goes for any tile whose
|
||||
|
||||
@@ -196,6 +196,7 @@ commands are for scripts and for cases where the mount is not available.
|
||||
| `pdfs add-to-album UID PHOTOS...` | Add photos to an album. |
|
||||
| `pdfs remove-from-album UID PHOTOS...` | Take photos out of an album. |
|
||||
| `pdfs import-google-photos ARCHIVES... [--dry-run] [--wait]` | Import a Google Takeout export. Pass every `.zip` of the export. |
|
||||
| `pdfs import-google-photos --resume [--wait]` | Start the last import again with the same archives after it was interrupted, stopped or failed. |
|
||||
| `pdfs import-status` / `pdfs cancel-import` | Follow or stop a running import. |
|
||||
| `pdfs redate-photos [--dry-run] [--from DATE] [--to DATE] [--limit N] [--wait]` | Fix photos filed under their upload day when the file name carries the real date. |
|
||||
| `pdfs redate-status` / `pdfs cancel-redate` | Follow or stop a running re-date. |
|
||||
|
||||
@@ -58,6 +58,44 @@ can resolve the default alias to a locked collection. Both have fixes in
|
||||
after the reporter's November 7 retest. Neither can be assigned as this report's cause without
|
||||
matching runtime evidence.
|
||||
|
||||
## A restarted Secret Service
|
||||
|
||||
A second path to the same symptom, independent of locking (B285). The keyring store pdfs used
|
||||
holds one D-Bus connection and one Secret Service session for the whole process, and was
|
||||
installed once. The session lives only in the provider's memory. When the provider restarts,
|
||||
the new one answers every `GetSecret` and `SetSecret` on the old session with
|
||||
`org.freedesktop.Secret.Error.NoSession`, until the pdfs process itself restarts. Reproduced
|
||||
with both gnome-keyring 51.1 and oo7 0.7.1 using `--replace`.
|
||||
|
||||
gnome-keyring practically never restarts within a desktop session. oo7 can:
|
||||
`oo7-daemon.service` has `Restart=on-failure`, the daemon allows itself to be replaced, and it
|
||||
exits on a wrong credential password and is D-Bus-activated again. A daemon that met a restarted
|
||||
provider kept every later token refresh only in memory and the kernel keyring, and a cold boot
|
||||
then left the spent refresh token in the saved blob.
|
||||
|
||||
Now a platform failure drops the store and the call is retried once on a new connection. The
|
||||
daemon log shows `the Secret Service failed this process's connection; reconnecting` when that
|
||||
happens. To check whether a machine meets this, look for oo7 restarts in the same boot:
|
||||
|
||||
```sh
|
||||
journalctl --user -b -1 -u oo7-daemon.service | grep -E 'Started|Stopped|Starting oo7|Main process exited'
|
||||
journalctl --user -b -1 -u proton-drive.service | grep -iE 'keyring|reconnecting|refresh'
|
||||
```
|
||||
|
||||
Reproduce in a throwaway session bus (never against the desktop's keyring; the runtime directory
|
||||
must be short, as it holds a Unix socket):
|
||||
|
||||
```sh
|
||||
T=$(mktemp -d /tmp/ss.XXXX); mkdir -p $T/data $T/run; chmod 700 $T/run
|
||||
XDG_DATA_HOME=$T/data XDG_RUNTIME_DIR=$T/run PDFS_SECRET_SERVICE_RESTART_TEST=1 \
|
||||
PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets >/dev/null' \
|
||||
dbus-run-session -- sh -c 'echo -n pw | gnome-keyring-daemon --unlock --components=secrets >/dev/null; sleep 1;
|
||||
cargo test -p pdfs-core real_secret_service -- --ignored'
|
||||
```
|
||||
|
||||
For oo7, start `oo7-daemon --login` with the password on stdin instead, and restart it with
|
||||
`--login --replace` in the background.
|
||||
|
||||
## Reproduce without a Proton account or desktop keyring
|
||||
|
||||
The oo7 fixture uses fake tokens, a temporary disk keyring and a private peer-to-peer D-Bus
|
||||
|
||||
+17
-2
@@ -55,6 +55,7 @@ server learns which areas you look at.
|
||||
2. In the app, open the **Upload** menu on the Photos page and choose **Import from Google
|
||||
Photos…**. Select all the `.zip` files at once, because a photo and its details are often in
|
||||
different parts of the export.
|
||||
3. Choose **Scan** to see what an import would do without uploading anything, then **Import**.
|
||||
|
||||
From a terminal:
|
||||
|
||||
@@ -63,8 +64,22 @@ pdfs import-google-photos --dry-run ~/Downloads/takeout-*.zip # show what woul
|
||||
pdfs import-google-photos --wait ~/Downloads/takeout-*.zip
|
||||
```
|
||||
|
||||
The import recreates your albums and takes capture dates and locations from the export. Photos
|
||||
already in your library are skipped, so an interrupted import can simply be run again.
|
||||
What the import takes from the export:
|
||||
|
||||
- **Albums** are recreated. A photo that is in several albums is uploaded once and added to each.
|
||||
- **Dates** come from the export's details, else from a date in the file name, else from the
|
||||
photo's EXIF.
|
||||
- **Places** come from the export, so imported photos appear under [Places](#places).
|
||||
- **Favorites** are kept.
|
||||
- **Live photos and motion photos** stay one photo: the video is filed with its still.
|
||||
|
||||
What it leaves out: photos in Google's trash, and files from other Google services (Drive,
|
||||
YouTube) that are in the same export. A `.zip` that cannot be read is left out and named in the
|
||||
report; the rest still imports.
|
||||
|
||||
Photos already in your library are skipped, so an interrupted import can simply be run again.
|
||||
If the import was cut short, for example by a restart, the Import page offers **Resume**, and
|
||||
`pdfs import-google-photos --resume` does the same without naming the archives again.
|
||||
`pdfs import-status` shows how far an import is, and `pdfs cancel-import` stops it.
|
||||
|
||||
## Fixing photos filed under the wrong date
|
||||
|
||||
+2
-1
@@ -63,7 +63,8 @@ them along.
|
||||
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
|
||||
| Photo name search | Not started | Photos search by date and filters only |
|
||||
| Automatic photo upload from a folder | Not started | |
|
||||
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos; motion-photo `.MP` parts are skipped |
|
||||
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
|
||||
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
|
||||
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |
|
||||
|
||||
## Performance and correctness
|
||||
|
||||
Whitespace-only changes.
+332
-224
File diff suppressed because it is too large.
Load diff
+313
-220
File diff suppressed because it is too large.
Load diff
+327
-219
File diff suppressed because it is too large.
Load diff
@@ -91,7 +91,7 @@ SECTIONS = [
|
||||
]
|
||||
|
||||
# Too long or too internal for the wiki; links to them go to GitHub.
|
||||
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md"}
|
||||
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md", "SDK-0.12.0.md"}
|
||||
|
||||
# Files outside docs/ whose links are checked but not published.
|
||||
CHECKED = ["README.md", "CONTRIBUTING.md", "SECURITY.md"]
|
||||
|
||||
Reference in new issue
Block a user