Compare commits

..
Author SHA1 Message Date
nvrl 16281ce5c6 feat(gui): play videos and music in the app
Opening a video or a song in My files, by a double-click or Open, now
plays it in an in-app player window, streamed from the mount as before.
Open With… is the way to another player, and the separate Play item is
gone. A file GStreamer can't play shows the error with Open With….

Both that player and the Photos lightbox share the new widgets/player.rs:
a fullscreen button, fullscreen on a double-click, Repeat in the ⋮ menu,
K to play and pause, J/L or Shift+arrows to jump 10 seconds, M to mute,
and a volume that carries over between videos. The lightbox's thumbnail
strip stays off a video, where it covered the seek bar.
2026-11-14 15:45:44 +01:00
37 changed files with 2573 additions and 6528 deletions

No files matched your search

Generated
+4 -47
View File
@@ -1114,12 +1114,6 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "data-encoding"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
[[package]]
name = "dbl"
version = "0.3.2"
@@ -2908,32 +2902,6 @@ dependencies = [
"scopeguard",
]
[[package]]
name = "lofty"
version = "0.25.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35d45c446382e04663705aa8e786157f64ef1fa85755d410121ffbf08f54b0db"
dependencies = [
"byteorder",
"data-encoding",
"flate2",
"lofty_attr",
"log",
"ogg_pager",
"paste",
]
[[package]]
name = "lofty_attr"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdbca1b60ba7cea959ffc84ac27f3dbdbacd6dfda168c7eabbc00e48654bb9fd"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.9",
]
[[package]]
name = "log"
version = "0.4.35"
@@ -3355,15 +3323,6 @@ dependencies = [
"subtle",
]
[[package]]
name = "ogg_pager"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "166611b491f939446e64c6be930843a31eb8e6798204ec3e9f57e084c18e7446"
dependencies = [
"byteorder",
]
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -3635,12 +3594,10 @@ dependencies = [
"futures",
"image",
"libc",
"lofty",
"notify",
"parking_lot 0.13.0",
"pdfs-core",
"proton-drive-rs",
"serde",
"serde_json",
"sha1 0.11.0",
"tokio",
@@ -3891,9 +3848,9 @@ dependencies = [
[[package]]
name = "proton-drive-rs"
version = "0.12.0"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4265c2cf3bdc59ee273d7363c8b6a7cb4e98be6102ffa18144c4d8ebda00737b"
checksum = "14a22e621abbc1c61b4640728a7de969c16b66d5d983b59c638d501fda25a7b9"
dependencies = [
"async-trait",
"base64 0.23.1",
@@ -3919,9 +3876,9 @@ dependencies = [
[[package]]
name = "proton-sdk"
version = "0.12.0"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb98ab3e336ed95f82368a50348dad00e5fc919e982d748e83e844afcadbd5d8"
checksum = "bd55a5e8b6600f3aa282ebbdf13ff366431ff9aae84d4eb20b0ef06d598919fe"
dependencies = [
"aes-gcm 0.11.1",
"async-trait",
+2 -6
View File
@@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
categories = ["filesystem", "command-line-utilities"]
[workspace.dependencies]
proton-sdk = { version = "0.12.0" }
proton-drive-rs = { version = "0.12.0" }
proton-sdk = { version = "0.11.0" }
proton-drive-rs = { version = "0.11.0" }
chrono = { version = "0.4", default-features = false, features = ["std"] }
# Internal crates.
@@ -50,10 +50,6 @@ sha1 = "0.11"
# formats a camera or phone actually produces — the rest of the crate's codecs
# would be dead weight in the daemon.
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "tiff", "bmp", "rayon"] }
# Cover pictures of songs, read from their tags (ID3, FLAC, Ogg, MP4, APE) for
# the song's thumbnail. Its default feature is required: 0.25 does not build
# without it.
lofty = "0.25"
rusqlite = { version = "0.41", features = ["bundled"] }
# keyring 4 split into `keyring-core` plus per-platform stores. The libdbus
# Secret Service store is the one keyring 3's `sync-secret-service` wrapped, and
+38 -180
View File
@@ -228,19 +228,14 @@ enum Command {
/// numbered parts, and a photo's metadata often sits in a different part
/// than the photo. Albums in the export are recreated as Proton albums, and
/// photos already on the account (same name and same content) are skipped,
/// so an interrupted import can simply be run again, or resumed with
/// `--resume` without naming the archives again.
/// so an interrupted import can simply be run again.
ImportGooglePhotos {
/// The export's `.zip` files (e.g. `takeout-*.zip`).
#[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)]
#[arg(required = true, value_hint = ValueHint::FilePath)]
archives: Vec<PathBuf>,
/// Report what would be imported without uploading anything.
#[arg(long, conflicts_with = "resume")]
#[arg(long)]
dry_run: bool,
/// Start the last import again with the same archives, after it was
/// interrupted, stopped or failed.
#[arg(long, conflicts_with = "archives")]
resume: bool,
/// Stay attached and print progress until the import finishes.
#[arg(long)]
wait: bool,
@@ -1035,9 +1030,8 @@ fn main() -> Result<()> {
Command::ImportGooglePhotos {
archives,
dry_run,
resume,
wait,
} => cmd_import_google_photos(archives, dry_run, resume, wait),
} => cmd_import_google_photos(archives, dry_run, wait),
Command::RedatePhotos {
dry_run,
limit,
@@ -2922,34 +2916,24 @@ fn photo_count(n: usize) -> String {
}
}
fn cmd_import_google_photos(
archives: Vec<PathBuf>,
dry_run: bool,
resume: bool,
wait: bool,
) -> Result<()> {
let request = if resume {
CtlRequest::ResumeImport
} else {
// The daemon opens the archives itself off the shared filesystem, so it
// gets absolute paths — the same arrangement `pdfs upload` uses.
let mut absolute = Vec::with_capacity(archives.len());
for archive in &archives {
let path = std::fs::canonicalize(archive)
.with_context(|| format!("resolve {}", archive.display()))?;
absolute.push(
path.to_str()
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
.to_owned(),
);
}
CtlRequest::ImportTakeout {
archives: absolute,
dry_run,
}
};
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
// The daemon opens the archives itself off the shared filesystem, so it gets
// absolute paths — the same arrangement `pdfs upload` uses.
let mut absolute = Vec::with_capacity(archives.len());
for archive in &archives {
let path = std::fs::canonicalize(archive)
.with_context(|| format!("resolve {}", archive.display()))?;
absolute.push(
path.to_str()
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
.to_owned(),
);
}
match control_request(request)? {
match control_request(CtlRequest::ImportTakeout {
archives: absolute,
dry_run,
})? {
CtlResponse::Ok { message } => println!("{message}"),
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
other => bail!("unexpected response: {other:?}"),
@@ -2961,34 +2945,20 @@ fn cmd_import_google_photos(
// Poll rather than stream: the control protocol is request/response, and an
// import runs for hours — a held-open socket would be the only long-lived
// connection the daemon has. A line is printed when the step changes and
// every few percent of it, not on every poll.
let mut last_line = String::new();
let mut last_printed = std::time::Instant::now();
// connection the daemon has.
loop {
std::thread::sleep(std::time::Duration::from_secs(2));
match control_request(CtlRequest::ImportStatus)? {
CtlResponse::ImportStatus {
running: true,
progress,
..
} => {
let line = progress.as_ref().map(progress_line).unwrap_or_default();
let step_changed = line.split(':').next() != last_line.split(':').next();
if !line.is_empty()
&& line != last_line
&& (step_changed || last_printed.elapsed().as_secs() >= 30)
{
println!("{line}");
last_line = line;
last_printed = std::time::Instant::now();
}
}
CtlResponse::ImportStatus {
summary, resumable, ..
} => {
print_import_summary(summary.as_ref(), &resumable);
return import_outcome(summary.as_ref());
summary: _,
} => continue,
CtlResponse::ImportStatus { summary, .. } => {
print_import_summary(summary.as_ref());
return finished_run(
"the import",
summary.map(|s| (s.failed, s.cancelled, s.error)),
);
}
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
other => bail!("unexpected response: {other:?}"),
@@ -2996,47 +2966,6 @@ fn cmd_import_google_photos(
}
}
/// One line saying where a running import is.
fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String {
use pdfs_core::control::ImportPhase;
let step = match progress.phase {
ImportPhase::Scanning => "Reading archives",
ImportPhase::Checking => "Checking for photos already on Proton",
ImportPhase::Uploading => "Uploading",
ImportPhase::Albums => "Filing photos into albums",
};
match (progress.total, &progress.current) {
(0, _) => format!("{step}…"),
(total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done),
(total, None) => format!("{step}: {} of {total}", progress.done),
}
}
/// How an import `--wait` followed ended, as the exit status: an error unless
/// it did everything it set out to do, so a script can tell.
fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> {
let Some(summary) = summary else {
bail!("the import ended without a result; the service may have restarted");
};
if let Some(error) = &summary.error {
bail!("the import failed: {error}");
}
if summary.interrupted {
bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue");
}
if summary.cancelled {
bail!("the import was cancelled");
}
if summary.failed > 0 || !summary.unreadable_archives.is_empty() {
bail!(
"the import finished, but {} photo(s) and {} archive(s) failed",
summary.failed,
summary.unreadable_archives.len()
);
}
Ok(())
}
fn cmd_redate_photos(
dry_run: bool,
limit: Option<usize>,
@@ -3171,19 +3100,11 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) {
fn cmd_import_status() -> Result<()> {
match control_request(CtlRequest::ImportStatus)? {
CtlResponse::ImportStatus {
running,
summary,
progress,
resumable,
} => {
CtlResponse::ImportStatus { running, summary } => {
if running {
match progress.as_ref() {
Some(progress) => println!("Import running. {}", progress_line(progress)),
None => println!("Import running."),
}
println!("Import running. `pdfs transfers` shows its progress.");
}
print_import_summary(summary.as_ref(), &resumable);
print_import_summary(summary.as_ref());
}
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
other => bail!("unexpected response: {other:?}"),
@@ -3200,28 +3121,18 @@ fn cmd_cancel_import() -> Result<()> {
Ok(())
}
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) {
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
let Some(summary) = summary else {
println!("No import has run yet.");
return;
};
if summary.dry_run {
println!("Scan only; nothing was uploaded.");
}
if let Some(error) = &summary.error {
println!("Stopped by : {error}");
println!("Failed : {error}");
return;
}
println!("Photos in export : {}", summary.found);
if summary.dry_run {
println!("Would upload : {}", summary.to_upload);
println!("Bytes to upload : {}", summary.to_upload_bytes);
} else {
println!("Uploaded : {}", summary.uploaded);
}
println!("Uploaded : {}", summary.uploaded);
println!("Already on Proton: {}", summary.duplicates);
if summary.live_photos > 0 {
println!("Live photos : {}", summary.live_photos);
}
if summary.skipped_trashed > 0 {
println!("Skipped (trash) : {}", summary.skipped_trashed);
}
@@ -3234,20 +3145,12 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, res
if summary.failed > 0 {
println!("Failed : {}", summary.failed);
}
for archive in &summary.unreadable_archives {
println!("Unreadable : {archive}");
}
if summary.bytes > 0 {
println!("Uploaded bytes : {}", summary.bytes);
}
if summary.interrupted {
println!("The service stopped before the import finished.");
} else if summary.cancelled {
if summary.cancelled {
println!("Import was cancelled before it finished.");
}
if !resumable.is_empty() {
println!("Run `pdfs import-google-photos --resume` to continue it.");
}
}
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
@@ -4606,51 +4509,6 @@ mod login_wait_tests {
assert!(finished_run("the import", None).is_err());
}
#[test]
fn a_waited_for_import_that_did_not_finish_exits_with_an_error() {
use pdfs_core::control::ImportSummary;
let done = ImportSummary {
uploaded: 4,
..Default::default()
};
assert!(import_outcome(Some(&done)).is_ok());
assert!(import_outcome(None).is_err());
for broken in [
ImportSummary {
interrupted: true,
..Default::default()
},
ImportSummary {
unreadable_archives: vec!["/exports/takeout-002.zip".into()],
..Default::default()
},
ImportSummary {
error: Some("check for existing photos: offline".into()),
..Default::default()
},
] {
assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}");
}
let interrupted = import_outcome(Some(&ImportSummary {
interrupted: true,
..Default::default()
}));
assert!(interrupted.unwrap_err().to_string().contains("--resume"));
}
#[test]
fn resume_needs_no_archives_and_excludes_them() {
let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap();
assert!(matches!(
cli.command,
Command::ImportGooglePhotos { resume: true, .. }
));
assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err());
assert!(
Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err()
);
}
#[test]
fn waiting_for_a_session_stops_hammering_the_keyring() {
let mut checks_left = 20;
+33 -253
View File
@@ -123,84 +123,13 @@ fn client_config() -> ProtonClientConfiguration {
ProtonClientConfiguration::new(app_version).with_user_agent(user_agent)
}
/// Run `op` on the session's keyring entry.
fn with_keyring<T>(op: impl Fn(&Entry) -> keyring_core::Result<T>) -> Result<T> {
with_keyring_entry(KEYRING_SERVICE, KEYRING_USER, op)
}
/// Run `op` on the entry `service`/`user`, reconnecting once if the Secret
/// Service fails the call.
///
/// The store holds one D-Bus connection and one Secret Service session for the
/// whole process. The session lives in the provider's memory: once the provider
/// restarts (oo7's unit restarts it on failure, and both oo7 and gnome-keyring
/// can be `--replace`d), every read and write on the old session fails with
/// `NoSession`, for as long as this process runs. A daemon then keeps refreshed
/// tokens only in memory, and a cold boot loses them (`docs/BUGS.md` B285). A
/// platform failure therefore drops the store, and the call is made once more
/// on a new one. A locked keyring, a dismissed prompt or a missing entry is
/// not the connection's fault and comes back as it is.
fn with_keyring_entry<T>(
service: &str,
user: &str,
op: impl Fn(&Entry) -> keyring_core::Result<T>,
) -> Result<T> {
let store = keyring_store()?;
match op(&Entry::new(service, user)?) {
Err(keyring_core::Error::PlatformFailure(e)) => {
tracing::warn!(
error = %e,
"the Secret Service failed this process's connection; reconnecting"
);
drop_keyring_store(&store);
keyring_store()?;
Ok(op(&Entry::new(service, user)?)?)
}
result => Ok(result?),
}
}
/// Held while the default store is looked at and replaced, so that threads
/// failing at the same time drop the broken store once and share its
/// replacement.
static KEYRING_STORE: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
/// The process's Secret Service store, connected now if there is none.
///
/// Connected lazily rather than once at startup so a session bus that was not
/// up yet on the first attempt is tried again on the next one.
fn keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
let _swap = KEYRING_STORE.lock();
if let Some(store) = keyring_core::get_default_store() {
return Ok(store);
}
let store = connect_keyring_store()?;
keyring_core::set_default_store(Arc::clone(&store));
Ok(store)
}
/// Forget `failed`, unless another thread has already replaced it.
fn drop_keyring_store(failed: &Arc<keyring_core::CredentialStore>) {
let _swap = KEYRING_STORE.lock();
if keyring_core::get_default_store().is_some_and(|store| Arc::ptr_eq(&store, failed)) {
keyring_core::unset_default_store();
}
}
#[cfg(not(test))]
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
Ok(dbus_secret_service_keyring_store::Store::new()?)
}
/// The tests' in-memory store when they set one, so a reconnect finds the
/// same entries; the real Secret Service otherwise.
#[cfg(test)]
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
tests::KEYRING_CONNECTS.fetch_add(1, Ordering::SeqCst);
match tests::MOCK_STORE.lock().clone() {
Some(store) => Ok(store),
None => Ok(dbus_secret_service_keyring_store::Store::new()?),
fn keyring_entry() -> Result<Entry> {
// Installed lazily rather than once at startup so a session bus that was not
// up yet on the first attempt is retried on the next one.
if keyring_core::get_default_store().is_none() {
keyring_core::set_default_store(dbus_secret_service_keyring_store::Store::new()?);
}
Ok(Entry::new(KEYRING_SERVICE, KEYRING_USER)?)
}
/// What the user answered when asked for the mailbox password.
@@ -492,15 +421,17 @@ fn classify_verification_gate(e: ProtonError) -> Error {
fn write_stored(stored: &StoredSession) -> Result<()> {
let json = Zeroizing::new(serde_json::to_string(stored)?);
with_keyring(|entry| entry.set_password(&json))
keyring_entry()?.set_password(&json)?;
Ok(())
}
/// Load the persisted session blob, or `Error::NotLoggedIn` if absent.
pub fn load() -> Result<StoredSession> {
match with_keyring(|entry| entry.get_password()) {
let entry = keyring_entry()?;
match entry.get_password() {
Ok(json) => Ok(serde_json::from_str(&json)?),
Err(Error::Keyring(keyring_core::Error::NoEntry)) => Err(Error::NotLoggedIn),
Err(e) => Err(e),
Err(keyring_core::Error::NoEntry) => Err(Error::NotLoggedIn),
Err(e) => Err(e.into()),
}
}
@@ -658,9 +589,9 @@ pub fn logout() -> Result<bool> {
tracing::warn!(error = %e, "clearing the kernel-keyring unlock on logout failed");
}
let _ = kernelkey::UNSAVED_TOKENS.clear();
match with_keyring(|entry| entry.delete_credential()) {
Ok(()) | Err(Error::Keyring(keyring_core::Error::NoEntry)) => Ok(ended),
Err(e) => Err(e),
match keyring_entry()?.delete_credential() {
Ok(()) | Err(keyring_core::Error::NoEntry) => Ok(ended),
Err(e) => Err(e.into()),
}
}
@@ -1065,15 +996,21 @@ impl TokenSink {
}
}
/// Write the unsaved rotation, if there is one. A blob that moved on drops
/// it. A missing blob is retried: some Secret Service providers also report
/// locked or temporarily hidden items as absent. Never recreate it here.
/// Write the unsaved rotation, if there is one. A blob that moved on, or
/// is gone with a logout, drops it: there is nothing it may still go into.
fn flush(&self) -> Result<()> {
let mut state = self.state.lock();
let Some(tokens) = state.unsaved.clone() else {
return Ok(());
};
let mut stored = load()?;
let mut stored = match load() {
Ok(stored) => stored,
Err(Error::NotLoggedIn) => {
state.settle();
return Ok(());
}
Err(e) => return Err(e),
};
match take_rotation(
&mut stored,
&self.session_id,
@@ -1276,138 +1213,12 @@ mod tests {
/// The keyring is one store per process, so the tests that use it,
/// through `keyring-core`'s in-memory store, take turns.
fn mock_keyring() -> parking_lot::MutexGuard<'static, ()> {
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
let turn = KEYRING.lock();
let store: Arc<keyring_core::CredentialStore> = keyring_core::mock::Store::new().unwrap();
*MOCK_STORE.lock() = Some(Arc::clone(&store));
keyring_core::set_default_store(store);
keyring_core::set_default_store(keyring_core::mock::Store::new().unwrap());
turn
}
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
/// The in-memory store a reconnect hands out, so it finds the same
/// entries; `None` connects to the real Secret Service.
pub(super) static MOCK_STORE: parking_lot::Mutex<Option<Arc<keyring_core::CredentialStore>>> =
parking_lot::Mutex::new(None);
/// How many times a store was connected.
pub(super) static KEYRING_CONNECTS: std::sync::atomic::AtomicUsize =
std::sync::atomic::AtomicUsize::new(0);
/// The session's entry, to make its next call fail.
fn keyring_entry() -> Entry {
Entry::new(KEYRING_SERVICE, KEYRING_USER).unwrap()
}
/// Make the session entry's next call fail with `error`.
fn fail_next(error: keyring_core::Error) {
keyring_entry()
.as_any()
.downcast_ref::<keyring_core::mock::Cred>()
.unwrap()
.set_error(error);
}
fn no_session() -> keyring_core::Error {
keyring_core::Error::PlatformFailure(
std::io::Error::other("The session does not exist (NoSession)").into(),
)
}
/// B285: a restarted Secret Service no longer knows the process's
/// session, and every call on it failed until the process restarted.
#[test]
fn a_dropped_secret_service_session_is_reconnected_and_retried() {
let _turn = mock_keyring();
write_stored(&stored_with("r")).unwrap();
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
fail_next(no_session());
assert_eq!(load().unwrap().refresh_token, "r");
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 1);
fail_next(no_session());
write_stored(&stored_with("r2")).unwrap();
assert_eq!(load().unwrap().refresh_token, "r2");
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 2);
}
#[test]
fn a_rotation_lands_at_once_after_the_keyring_restarts() {
let _turn = mock_keyring();
write_stored(&stored_with("r")).unwrap();
let sink = Arc::new(TokenSink::new("s", "r".into()));
fail_next(no_session());
sink.rotated(tokens("a2", "r2")).unwrap();
assert_eq!(load().unwrap().refresh_token, "r2");
assert!(sink.state.lock().unsaved.is_none());
}
/// A locked keyring or a dismissed prompt is no fault of the connection.
#[test]
fn a_locked_keyring_is_not_reconnected() {
let _turn = mock_keyring();
write_stored(&stored_with("r")).unwrap();
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
fail_next(keyring_core::Error::NoStorageAccess(
std::io::Error::other("locked").into(),
));
assert!(matches!(
load(),
Err(Error::Keyring(keyring_core::Error::NoStorageAccess(_)))
));
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects);
assert_eq!(load().unwrap().refresh_token, "r");
}
/// The real Secret Service, restarted between two calls on one store.
///
/// Run inside a throwaway session bus, never against the desktop's
/// keyring. With gnome-keyring:
///
/// ```sh
/// echo -n pw | gnome-keyring-daemon --unlock --components=secrets
/// PDFS_SECRET_SERVICE_RESTART_TEST=1 \
/// PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets' \
/// cargo test -p pdfs-core real_secret_service -- --ignored
/// ```
#[test]
#[ignore = "needs a throwaway Secret Service; see the doc comment"]
fn real_secret_service_survives_a_provider_restart() {
if std::env::var_os("PDFS_SECRET_SERVICE_RESTART_TEST").is_none() {
return;
}
let restart = std::env::var("PDFS_RESTART_PROVIDER").expect("PDFS_RESTART_PROVIDER");
let _turn = KEYRING.lock();
*MOCK_STORE.lock() = None;
keyring_core::unset_default_store();
let (service, user) = ("pdfs-test", "restart");
with_keyring_entry(service, user, |e| e.set_password("before")).unwrap();
let status = std::process::Command::new("sh")
.args(["-c", &restart])
.status()
.unwrap();
assert!(status.success(), "restarting the provider failed");
std::thread::sleep(Duration::from_secs(1));
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
assert_eq!(read, "before");
assert_eq!(
KEYRING_CONNECTS.load(Ordering::SeqCst),
connects + 1,
"the provider kept the old session, so nothing was tested"
);
with_keyring_entry(service, user, |e| e.set_password("after")).unwrap();
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
assert_eq!(read, "after");
with_keyring_entry(service, user, |e| e.delete_credential()).unwrap();
keyring_core::unset_default_store();
}
fn stored_with(refresh: &str) -> StoredSession {
serde_json::from_str(&blob("").replace(r#""r""#, &format!("{refresh:?}"))).unwrap()
}
@@ -1462,7 +1273,12 @@ mod tests {
write_stored(&stored_with("r")).unwrap();
let sink = Arc::new(TokenSink::new("s", "r".into()));
fail_next(keyring_core::Error::NoStorageAccess(
let entry = keyring_entry().unwrap();
let cred = entry
.as_any()
.downcast_ref::<keyring_core::mock::Cred>()
.unwrap();
cred.set_error(keyring_core::Error::NoStorageAccess(
std::io::Error::other("locked").into(),
));
assert!(sink.rotated(tokens("a2", "r2")).is_err());
@@ -1490,42 +1306,6 @@ mod tests {
}
}
/// oo7 0.7.0 reports a locked collection's items as absent (upstream #614).
/// Treating that as logout loses the only live tokens (B284, issue #78).
#[test]
fn a_temporarily_missing_session_does_not_discard_a_rotation() {
let _turn = mock_keyring();
write_stored(&stored_with("r")).unwrap();
let sink = Arc::new(TokenSink::new("s", "r".into()));
fail_next(keyring_core::Error::NoEntry);
assert!(sink.rotated(tokens("a2", "r2")).is_err());
assert!(sink.state.lock().unsaved.is_some());
assert_eq!(load().unwrap().refresh_token, "r");
sink.flush().unwrap();
assert_eq!(load().unwrap().refresh_token, "r2");
assert!(sink.state.lock().unsaved.is_none());
}
#[test]
fn retrying_a_missing_session_never_recreates_it_or_overwrites_a_new_login() {
let _turn = mock_keyring();
let sink = TokenSink::new("s", "r".into());
sink.state.lock().unsaved = Some(tokens("a2", "r2"));
assert!(matches!(sink.flush(), Err(Error::NotLoggedIn)));
assert!(matches!(load(), Err(Error::NotLoggedIn)));
assert!(sink.state.lock().unsaved.is_some());
let mut login = stored_with("fresh");
login.session_id = "new".into();
write_stored(&login).unwrap();
sink.flush().unwrap();
assert!(sink.state.lock().unsaved.is_none());
assert_eq!(load().unwrap().refresh_token, "fresh");
}
#[test]
fn an_unsaved_rotation_goes_only_into_the_blob_it_was_refused_for() {
let unsaved = UnsavedRotation {
-6
View File
@@ -562,12 +562,6 @@ impl AppDirs {
self.state_dir().join("control.sock")
}
/// The Google Photos import's last run: its archives and how it ended, so
/// an import cut short by a restart can be resumed.
pub fn takeout_import_path(&self) -> PathBuf {
self.state_dir().join("takeout-import.json")
}
/// Unix domain socket the tray uses to ensure single instance.
pub fn tray_socket(&self) -> PathBuf {
self.state_dir().join("tray.sock")
+11 -130
View File
@@ -279,11 +279,6 @@ pub enum Request {
/// and files what it has uploaded into its albums, then reports
/// `cancelled`. Replies with [`Response::Ok`].
CancelImport,
/// Start the last import again with the same archives, after it was
/// interrupted, cancelled or failed. Photos it already uploaded are found
/// on the account and skipped. Replies like [`Request::ImportTakeout`];
/// [`ErrorKind::NotFound`] when there is nothing to resume.
ResumeImport,
/// Upload the photo at `source_path` under the given name and media type.
///
/// A path, not the bytes: this protocol is line-delimited JSON, and
@@ -773,9 +768,6 @@ pub enum Topic {
Conflicts,
/// The activity log gained entries.
Activity,
/// File thumbnails were made or fetched. A front-end asks again for the
/// tiles it still shows blank; the ones it already painted stay as they are.
Thumbnails,
/// A topic from a newer daemon, which this front-end cannot act on.
#[serde(other)]
Unknown,
@@ -785,7 +777,7 @@ impl Topic {
/// Every topic a front-end can act on. A new subscription is told all of
/// them have changed, so it catches up on whatever happened while it was
/// not connected.
pub const ALL: [Topic; 10] = [
pub const ALL: [Topic; 9] = [
Topic::Files,
Topic::Photos,
Topic::Trash,
@@ -795,7 +787,6 @@ impl Topic {
Topic::Queue,
Topic::Conflicts,
Topic::Activity,
Topic::Thumbnails,
];
}
@@ -1694,75 +1685,10 @@ pub struct ImportSummary {
pub bytes: u64,
/// True when the run stopped early because it was cancelled.
pub cancelled: bool,
/// Why the run as a whole stopped early, when it did, such as the
/// account being unreachable. The counts say how far it got.
/// Why the run as a whole failed, when it did, such as an archive that
/// is not a zip. The counts are then empty.
#[serde(default)]
pub error: Option<String>,
/// What kind of failure [`error`](Self::error) is, so a front-end can word
/// it in the user's language. The text itself is the daemon's English.
#[serde(default)]
pub error_kind: Option<ErrorKind>,
/// True for a scan that uploads nothing ([`Request::ImportTakeout`] with
/// `dry_run`). [`to_upload`](Self::to_upload) then says what an import
/// would upload.
#[serde(default)]
pub dry_run: bool,
/// Photos not yet on the account: what a dry run would upload, and what a
/// real run set out to upload once the duplicates were known.
#[serde(default)]
pub to_upload: usize,
/// Bytes of [`to_upload`](Self::to_upload).
#[serde(default)]
pub to_upload_bytes: u64,
/// Live and motion photos whose video was filed with its still.
#[serde(default)]
pub live_photos: usize,
/// True when the service stopped while the import ran (a shutdown or a
/// restart). [`Request::ResumeImport`] picks it up again.
#[serde(default)]
pub interrupted: bool,
/// Archives that could not be read and were left out, by path.
#[serde(default)]
pub unreadable_archives: Vec<String>,
}
impl ImportSummary {
/// Whether the run did all it set out to do.
pub fn complete(&self) -> bool {
self.error.is_none()
&& !self.cancelled
&& !self.interrupted
&& self.failed == 0
&& self.unreadable_archives.is_empty()
}
}
/// Which step a running Takeout import is on.
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ImportPhase {
/// Reading the archives' directories and metadata.
#[default]
Scanning,
/// Asking the account which photos it already has.
Checking,
/// Uploading the photos that are new.
Uploading,
/// Creating albums and filing photos into them.
Albums,
}
/// Where a running import is ([`Response::ImportStatus`]).
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
pub struct ImportProgress {
pub phase: ImportPhase,
/// Steps done in this phase.
pub done: u64,
/// Steps in this phase; `0` while not yet known.
pub total: u64,
/// The photo being uploaded, during [`ImportPhase::Uploading`].
#[serde(default)]
pub current: Option<String>,
}
/// One album in a [`Response::Albums`] listing.
@@ -1951,26 +1877,11 @@ pub fn is_video_file_name(name: &str, size: Option<u64>) -> bool {
!transport_stream || size.is_none_or(|size| size >= TS_VIDEO_MIN_BYTES)
}
/// Whether `name` is a song or other recording in a format that can carry
/// its own cover picture: ID3 (MP3, AAC, WAV, AIFF), FLAC and Ogg pictures,
/// MP4 `covr` (M4A, M4B), or APE tags (WavPack, Monkey's Audio).
pub fn is_audio_file_name(name: &str) -> bool {
let Some((_, extension)) = name.rsplit_once('.') else {
return false;
};
AUDIO_COVER_EXTENSIONS.contains(&extension.to_ascii_lowercase().as_str())
}
const AUDIO_COVER_EXTENSIONS: &[&str] = &[
"aac", "aif", "aiff", "ape", "flac", "m4a", "m4b", "mp3", "oga", "ogg", "opus", "wav", "wv",
];
/// Whether file listings show a thumbnail for a file named `name`, of `size`
/// bytes when known: an image [`is_thumbnail_image_name`] accepts, a video,
/// whose thumbnail is a frame the daemon takes with ffmpeg, or a song, whose
/// thumbnail is its cover: the picture in its tags, or a cover image beside it.
/// bytes when known: an image [`is_thumbnail_image_name`] accepts, or a video,
/// whose thumbnail is a frame the daemon takes with ffmpeg.
pub fn is_thumbnail_media_name(name: &str, size: Option<u64>) -> bool {
is_thumbnail_image_name(name) || is_video_file_name(name, size) || is_audio_file_name(name)
is_thumbnail_image_name(name) || is_video_file_name(name, size)
}
/// Whether `name` is one of the concrete RAW formats handled by the exiftool
@@ -2278,17 +2189,8 @@ pub enum Response {
/// report; `summary` is `None` when no import has run this session.
ImportStatus {
running: bool,
/// The counts so far while running, the final report after.
#[serde(default)]
summary: Option<ImportSummary>,
/// The current step, while running.
#[serde(default)]
progress: Option<ImportProgress>,
/// The archives of the last run, when it can be resumed with
/// [`Request::ResumeImport`]: it was a real import that did not
/// finish. Survives a restart of the service.
#[serde(default)]
resumable: Vec<String>,
},
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
/// `running` is false once it has finished, when `summary` is the final
@@ -2761,16 +2663,6 @@ mod tests {
assert!(end.is_none(), "{end:?}");
}
#[test]
fn a_thumbnail_change_travels_as_its_own_topic() {
let line = serde_json::to_string(&Event::Changed {
topics: vec![Topic::Thumbnails],
})
.unwrap();
assert_eq!(line, r#"{"Changed":{"topics":["thumbnails"]}}"#);
assert!(Topic::ALL.contains(&Topic::Thumbnails));
}
#[test]
fn a_daemon_without_subscriptions_is_told_apart_from_a_dead_one() {
let socket = fake_daemon("legacy", "{\"Error\":{\"message\":\"unknown request\"}}\n");
@@ -2835,7 +2727,7 @@ mod tests {
}
#[test]
fn listings_thumbnail_images_videos_and_songs() {
fn listings_thumbnail_images_and_videos() {
assert!(is_thumbnail_media_name("photo.JPG", Some(10)));
assert!(is_thumbnail_media_name("camera.nef", None));
assert!(is_thumbnail_media_name("clip.mp4", Some(10)));
@@ -2844,24 +2736,13 @@ mod tests {
Some(64 * 1024 * 1024)
));
assert!(!is_thumbnail_media_name("app.ts", Some(2048)));
assert!(is_thumbnail_media_name("song.flac", Some(64 * 1024 * 1024)));
assert!(is_thumbnail_media_name("Track 01.MP3", None));
assert!(!is_thumbnail_media_name(
"song.flac",
Some(64 * 1024 * 1024)
));
assert!(!is_thumbnail_media_name("notes.txt", None));
}
#[test]
fn songs_are_told_by_their_extension() {
for name in [
"a.mp3", "b.FLAC", "c.ogg", "d.oga", "e.opus", "f.m4a", "g.m4b", "h.aac", "i.wav",
"j.aiff", "k.aif", "l.wv", "m.ape",
] {
assert!(is_audio_file_name(name), "{name}");
}
for name in ["mp3", "song.mp3.zip", "clip.mp4", "notes.txt", "cover.jpg"] {
assert!(!is_audio_file_name(name), "{name}");
}
}
/// Keep every extension accepted by the shared classifier covered. Drive
/// activation uses this classification before it knows whether opening a
/// file should stream through FUSE or materialise a complete cache entry;
+70 -488
View File
@@ -25,9 +25,7 @@
//! The service folder (`Google Photos`) is localized, as are the year folders
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
//! service name: the first component is dropped when it is the export root, the
//! second is the service, and the rest is `<folder>/<file>`. The Photos service
//! is the one whose sidecars carry a `photoTakenTime`; other services in the same
//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
//! **album** when it carries an album metadata sidecar naming it; anything else
//! is a plain bucket that imports into the timeline only. Trash is matched by
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
@@ -123,14 +121,8 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
("m2ts", "video/mp2t"),
("mpg", "video/mpeg"),
("mpeg", "video/mpeg"),
// The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`.
("mp", "video/mp4"),
];
/// Sidecars bigger than this are not read. A real one is a few kilobytes; a
/// `.json` this large is some other service's data and not worth the memory.
const SIDECAR_MAX_BYTES: u64 = 1024 * 1024;
/// The media type for `name`'s extension, or `None` when it is not media this
/// import handles.
pub fn media_type_of(name: &str) -> Option<&'static str> {
@@ -155,13 +147,6 @@ pub enum Bucket {
Archived,
}
/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct GeoPoint {
pub latitude: f64,
pub longitude: f64,
}
/// One media file found in the export.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TakeoutPhoto {
@@ -177,49 +162,14 @@ pub struct TakeoutPhoto {
pub size: u64,
/// Media type from the extension.
pub media_type: String,
/// Capture time in epoch seconds, from the sidecar's `photoTakenTime`,
/// else from a date in the file name. `None` when neither has one; the
/// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time).
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
/// `creationTime`). `None` when the photo has no sidecar — the upload then
/// defaults it to now, as the SDK does.
pub capture_time: Option<i64>,
/// The sidecar's `creationTime`: when the photo reached Google Photos, not
/// when it was taken. Only a last resort for the capture time.
#[serde(default)]
pub creation_time: Option<i64>,
/// Where the photo was taken, from the sidecar's `geoData` (or
/// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped.
#[serde(default)]
pub location: Option<GeoPoint>,
/// The sidecar's `favorited` flag.
pub favorite: bool,
/// Which bucket the file was found in.
pub bucket: Bucket,
/// Pairs the halves of a live or motion photo: a still and a video in the
/// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`,
/// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half.
#[serde(default)]
pub live_key: Option<String>,
}
impl TakeoutPhoto {
/// Whether this is the video half of a pair, as opposed to the still.
pub fn is_video(&self) -> bool {
self.media_type.starts_with("video/")
}
/// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather
/// than an Apple live photo.
pub fn is_motion_photo(&self) -> bool {
self.live_key
.as_deref()
.is_some_and(|key| key.ends_with(".mp"))
}
}
/// An archive the scan had to leave out, and why.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UnreadableArchive {
pub path: PathBuf,
pub reason: String,
}
/// Everything one scan found, ready to be turned into an import plan.
@@ -233,16 +183,8 @@ pub struct TakeoutScan {
/// Media files skipped because they sit in the trash bucket.
pub skipped_trashed: usize,
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
/// activity `.csv`s), and media from other Google services in the same
/// export (`Google Drive/`, `YouTube/`).
/// activity `.csv`s, `.mp` motion-photo parts).
pub skipped_other: usize,
/// Archives that could not be opened or are not a zip. The rest of the
/// export is still imported.
#[serde(default)]
pub unreadable_archives: Vec<UnreadableArchive>,
/// Entries inside an otherwise readable archive that could not be read.
#[serde(default)]
pub unreadable_entries: usize,
}
impl TakeoutScan {
@@ -253,7 +195,7 @@ impl TakeoutScan {
}
/// Google's per-photo sidecar. Only the fields the import uses are read; the
/// file also carries people tags, a description and view counts.
/// file also carries geo data, people tags and view counts.
#[derive(Debug, Default, Deserialize)]
struct PhotoSidecar {
#[serde(default)]
@@ -263,69 +205,19 @@ struct PhotoSidecar {
#[serde(rename = "creationTime", default)]
creation_time: Option<SidecarTime>,
#[serde(default)]
favorited: Option<bool>,
/// The position as edited in Google Photos, else as the camera wrote it.
#[serde(rename = "geoData", default)]
geo_data: Option<SidecarGeo>,
#[serde(rename = "geoDataExif", default)]
geo_data_exif: Option<SidecarGeo>,
favorited: bool,
}
impl PhotoSidecar {
fn location(&self) -> Option<GeoPoint> {
self.geo_data
.as_ref()
.and_then(SidecarGeo::point)
.or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point))
}
}
/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number.
/// A sidecar timestamp: epoch seconds, as a *string*.
#[derive(Debug, Deserialize)]
struct SidecarTime {
#[serde(default)]
timestamp: Option<SidecarNumber>,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum SidecarNumber {
Number(i64),
Text(String),
timestamp: Option<String>,
}
impl SidecarTime {
/// The timestamp, when it is a real moment. `0` and negative values are
/// Google's "unknown" and would file the photo under 1970.
fn epoch_seconds(&self) -> Option<i64> {
let seconds = match self.timestamp.as_ref()? {
SidecarNumber::Number(seconds) => *seconds,
SidecarNumber::Text(text) => text.trim().parse().ok()?,
};
(seconds > 0).then_some(seconds)
}
}
/// `geoData` in a sidecar. All-zero means Google has no position.
#[derive(Debug, Deserialize)]
struct SidecarGeo {
#[serde(default)]
latitude: Option<f64>,
#[serde(default)]
longitude: Option<f64>,
}
impl SidecarGeo {
fn point(&self) -> Option<GeoPoint> {
let (latitude, longitude) = (self.latitude?, self.longitude?);
let on_globe = latitude.is_finite()
&& longitude.is_finite()
&& (-90.0..=90.0).contains(&latitude)
&& (-180.0..=180.0).contains(&longitude);
(on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint {
latitude,
longitude,
})
self.timestamp.as_ref()?.parse().ok()
}
}
@@ -370,58 +262,21 @@ impl RawEntry {
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
/// Reads only the central directory and the JSON sidecars — no media bytes, so
/// this stays fast on a 200 GB export.
///
/// An archive that cannot be opened is left out and listed in
/// [`TakeoutScan::unreadable_archives`]; the rest still imports.
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
scan_archives_until(archives, || false)
}
/// [`scan_archives`] that gives up when `stop` returns true, checked between
/// archives and every few hundred entries.
pub fn scan_archives_until(
archives: &[PathBuf],
stop: impl Fn() -> bool,
) -> CoreResult<TakeoutScan> {
let mut entries: Vec<(String, RawEntry)> = Vec::new();
let mut sidecars: Vec<(String, String, Vec<u8>)> = Vec::new();
let mut unreadable_archives = Vec::new();
let mut unreadable_entries = 0;
let mut entries: Vec<RawEntry> = Vec::new();
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
for (index, path) in archives.iter().enumerate() {
if stop() {
return Err(CoreError::conflict("the import was cancelled"));
}
let opened = std::fs::File::open(path)
.map_err(|e| e.to_string())
.and_then(|file| {
zip::ZipArchive::new(std::io::BufReader::new(file))
.map_err(|e| format!("not a readable zip: {e}"))
});
let mut zip = match opened {
Ok(zip) => zip,
Err(reason) => {
tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive");
unreadable_archives.push(UnreadableArchive {
path: path.clone(),
reason,
});
continue;
}
};
let file = std::fs::File::open(path)
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
})?;
for position in 0..zip.len() {
if position % 512 == 0 && stop() {
return Err(CoreError::conflict("the import was cancelled"));
}
let mut entry = match zip.by_index(position) {
Ok(entry) => entry,
Err(e) => {
tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry");
unreadable_entries += 1;
continue;
}
};
let mut entry = zip.by_index(position).map_err(|e| {
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
})?;
if entry.is_dir() {
continue;
}
@@ -431,92 +286,32 @@ pub fn scan_archives_until(
continue;
};
let name = name.to_string_lossy().replace('\\', "/");
let Some((service, relative)) = strip_export_prefix(&name) else {
let Some(relative) = strip_export_prefix(&name) else {
continue;
};
let (service, relative) = (service.to_string(), relative.to_string());
let size = entry.size();
// Sidecars are read now, while the entry is open: they are a few
// hundred bytes each and reopening the archive per photo later would
// be a second full pass.
if relative.to_ascii_lowercase().ends_with(".json") {
if size <= SIDECAR_MAX_BYTES {
let mut body = Vec::with_capacity(size as usize);
if entry.read_to_end(&mut body).is_ok() {
sidecars.push((service, relative, body));
} else {
unreadable_entries += 1;
}
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
if entry.read_to_end(&mut body).is_ok() {
sidecars.insert(relative.to_string(), body);
}
continue;
}
entries.push((
service,
RawEntry {
archive: index,
entry: name,
relative,
size,
},
));
entries.push(RawEntry {
archive: index,
entry: name.clone(),
relative: relative.to_string(),
size,
});
}
}
let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars);
let mut scan = assemble(entries, &sidecars);
scan.skipped_other += other_services;
scan.unreadable_archives = unreadable_archives;
scan.unreadable_entries = unreadable_entries;
Ok(scan)
}
/// Drop what other Google services put into the same export.
///
/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to
/// `Google Photos/`, and their media must not land in the photo timeline. The
/// service folder's name is localized (`Google Fotos`), so the Photos service
/// is recognized by its content instead: it is the folder whose sidecars carry
/// a `photoTakenTime`. When no folder does — an export without sidecars, or
/// one re-zipped without the service level — everything is kept, as before.
///
/// Returns the entries and sidecars to import, and how many media files were
/// left out.
fn keep_photos_services(
entries: Vec<(String, RawEntry)>,
sidecars: Vec<(String, String, Vec<u8>)>,
) -> (Vec<RawEntry>, HashMap<String, Vec<u8>>, usize) {
#[derive(Deserialize)]
struct Marker {
#[serde(rename = "photoTakenTime")]
photo_taken_time: Option<serde::de::IgnoredAny>,
}
let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new();
for (service, _, body) in &sidecars {
if photos_services.contains(service.as_str()) {
continue;
}
if serde_json::from_slice::<Marker>(body).is_ok_and(|m| m.photo_taken_time.is_some()) {
photos_services.insert(service.as_str());
}
}
let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service);
let mut skipped = 0;
let mut kept_entries = Vec::with_capacity(entries.len());
for (service, raw) in entries {
if keep(&service) {
kept_entries.push(raw);
} else if media_type_of(raw.file_name()).is_some() {
skipped += 1;
}
}
let kept_sidecars = sidecars
.iter()
.filter(|(service, _, _)| keep(service))
.map(|(_, relative, body)| (relative.clone(), body.clone()))
.collect();
(kept_entries, kept_sidecars, skipped)
Ok(assemble(entries, &sidecars))
}
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
@@ -608,19 +403,19 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
.unwrap_or(&file_name)
.to_string();
// Sidecar first, then the date stamped into the name. `creationTime`
// is when the photo reached Google, which for a scan or an old photo
// uploaded late is years off; the importer uses it only after EXIF.
// Sidecar first, then the date stamped into the name.
let capture_time = sidecar
.photo_taken_time
.as_ref()
.and_then(SidecarTime::epoch_seconds)
.or_else(|| {
sidecar
.creation_time
.as_ref()
.and_then(SidecarTime::epoch_seconds)
})
.or_else(|| capture_time_from_name(&name))
.or_else(|| capture_time_from_name(&file_name));
let creation_time = sidecar
.creation_time
.as_ref()
.and_then(SidecarTime::epoch_seconds);
scan.photos.push(TakeoutPhoto {
archive: raw.archive,
@@ -629,57 +424,31 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
size: raw.size,
media_type: media_type.to_string(),
capture_time,
creation_time,
location: sidecar.location(),
favorite: sidecar.favorited.unwrap_or(false),
favorite: sidecar.favorited,
bucket,
live_key: live_key(&folder, &file_name),
});
}
scan
}
/// Split a path inside the archive into the localized service folder and the
/// rest, dropping the export root (`Takeout/`): `("Google Fotos",
/// "<folder>/<file>")`.
/// Drop the export root (`Takeout/`) and the localized service folder from a
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
///
/// `None` for a path with no service folder at all (the export's own
/// `archive_browser.html`), which carries nothing to import.
fn strip_export_prefix(path: &str) -> Option<(&str, &str)> {
fn strip_export_prefix(path: &str) -> Option<&str> {
let mut rest = path;
// The root is `Takeout/`, but an export unzipped and re-zipped by the user
// may not have it. `Google Photos/<album>/<file>` then has three parts like
// `Takeout/Google Photos/<file>`, so the count alone cannot tell them
// apart: a root of another name is only taken for one with four parts.
let (first, after) = rest.split_once('/')?;
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
// the user may not have it — so the root is dropped only when a service
// folder follows it, which the component count decides.
let components = rest.split('/').count();
if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 {
rest = after;
if components >= 3 {
rest = rest.split_once('/')?.1;
}
// The service folder (`Google Photos`, `Google Fotos`, …).
rest.split_once('/')
}
/// The key that pairs the still and the video of one live or motion photo, or
/// `None` when `file_name` cannot be either half.
///
/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem.
/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the
/// video's whole name. The folder is part of the key so two cameras' `IMG_1`
/// in different albums never pair.
fn live_key(folder: &str, file_name: &str) -> Option<String> {
let lower = file_name.to_lowercase();
let (stem, ext) = lower.rsplit_once('.')?;
let key = match ext {
"heic" | "heif" | "jpg" | "jpeg" => stem,
// Apple's video half. A plain `.mp4` beside a photo of the same name is
// far more often an unrelated clip than half of a live photo.
"mov" => stem,
"mp" => lower.as_str(),
_ => return None,
};
(!key.is_empty()).then(|| format!("{folder}/{key}"))
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
let (_service, tail) = rest.split_once('/')?;
Some(tail)
}
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
@@ -1089,13 +858,13 @@ mod tests {
RawEntry {
archive,
entry: path.to_string(),
relative: strip_export_prefix(path).unwrap().1.to_string(),
relative: strip_export_prefix(path).unwrap().to_string(),
size,
}
}
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
let relative = strip_export_prefix(path).unwrap().1.to_string();
let relative = strip_export_prefix(path).unwrap().to_string();
map.insert(relative, body.as_bytes().to_vec());
}
@@ -1103,17 +872,12 @@ mod tests {
fn strips_export_root_and_localized_service_folder() {
assert_eq!(
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
Some(("Google Fotos", "Island 2019/IMG_1.jpg"))
Some("Island 2019/IMG_1.jpg")
);
// Already unwrapped by the user: no `Takeout/` root left.
assert_eq!(
strip_export_prefix("Google Photos/IMG_1.jpg"),
Some(("Google Photos", "IMG_1.jpg"))
);
// ... and an album keeps its folder.
assert_eq!(
strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"),
Some(("Google Photos", "Iceland/IMG_1.jpg"))
Some("IMG_1.jpg")
);
// Nothing below a service folder is nothing to import.
assert_eq!(strip_export_prefix("archive_browser.html"), None);
@@ -1193,144 +957,23 @@ mod tests {
}
#[test]
fn creation_time_is_kept_apart_as_the_last_resort() {
// `creationTime` is when the photo reached Google. It must not beat the
// date in the name, and the importer tries EXIF before it.
fn creation_time_is_the_fallback_capture_time() {
let mut sidecars = HashMap::new();
sidecar(
&mut sidecars,
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
);
sidecar(
&mut sidecars,
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json",
r#"{"creationTime":{"timestamp":"1700000000"}}"#,
);
let scan = assemble(
vec![
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
entry(
0,
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg",
10,
),
],
vec![entry(
0,
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
10,
)],
&sidecars,
);
assert_eq!(scan.photos[0].capture_time, None);
assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000));
assert_eq!(
scan.photos[1].capture_time,
capture_time_from_name("IMG_20190612_101500.jpg")
);
assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000));
}
#[test]
fn sidecar_timestamps_tolerate_numbers_zero_and_null() {
let mut sidecars = HashMap::new();
for (name, body) in [
("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#),
(
"B.jpg",
r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#,
),
(
"C.jpg",
r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#,
),
] {
sidecar(
&mut sidecars,
&format!("Takeout/Google Photos/Photos from 2019/{name}.json"),
body,
);
}
let scan = assemble(
["A.jpg", "B.jpg", "C.jpg"]
.iter()
.map(|name| {
entry(
0,
&format!("Takeout/Google Photos/Photos from 2019/{name}"),
1,
)
})
.collect(),
&sidecars,
);
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970");
assert!(!scan.photos[1].favorite);
assert_eq!(scan.photos[2].capture_time, None);
}
#[test]
fn location_comes_from_geo_data_and_zero_means_none() {
let mut sidecars = HashMap::new();
sidecar(
&mut sidecars,
"Takeout/Google Photos/Iceland/IMG_1.jpg.json",
r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#,
);
sidecar(
&mut sidecars,
"Takeout/Google Photos/Iceland/IMG_2.jpg.json",
r#"{"geoData":{"latitude":0.0,"longitude":0.0},
"geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#,
);
sidecar(
&mut sidecars,
"Takeout/Google Photos/Iceland/IMG_3.jpg.json",
r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#,
);
let scan = assemble(
(1..=3)
.map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1))
.collect(),
&sidecars,
);
assert_eq!(
scan.photos[0].location,
Some(GeoPoint {
latitude: 64.1466,
longitude: -21.9426
})
);
assert_eq!(
scan.photos[1].location.map(|p| p.latitude),
Some(63.4186),
"geoDataExif fills in for an empty geoData"
);
assert_eq!(scan.photos[2].location, None);
}
#[test]
fn live_and_motion_photo_halves_share_a_key() {
let scan = assemble(
vec![
entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1),
entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1),
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1),
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1),
entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1),
entry(0, "Takeout/Google Photos/Trip/scan.png", 1),
],
&HashMap::new(),
);
let key = |i: usize| scan.photos[i].live_key.clone();
assert_eq!(key(0), key(1));
assert!(!scan.photos[0].is_motion_photo());
assert_eq!(key(2), key(3));
assert!(scan.photos[3].is_motion_photo());
assert!(scan.photos[3].is_video(), "the .MP half is a video");
assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair");
assert_eq!(key(5), None);
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
}
#[test]
@@ -1744,77 +1387,16 @@ mod tests {
}
#[test]
fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() {
let broken =
fn a_file_that_is_not_a_zip_is_a_readable_error() {
let path =
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
std::fs::write(&broken, b"not a zip at all").unwrap();
let good = write_zip(
"beside-broken",
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
);
let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap();
let _ = std::fs::remove_file(&broken);
let _ = std::fs::remove_file(&good);
assert_eq!(scan.photos.len(), 1);
assert_eq!(scan.photos[0].archive, 1);
assert_eq!(scan.unreadable_archives.len(), 1);
assert_eq!(scan.unreadable_archives[0].path, broken);
std::fs::write(&path, b"not a zip at all").unwrap();
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
let _ = std::fs::remove_file(&path);
assert!(
scan.unreadable_archives[0]
.reason
.contains("not a readable zip"),
"unexpected reason: {}",
scan.unreadable_archives[0].reason
error.message.contains("not a readable zip"),
"unexpected message: {}",
error.message
);
}
#[test]
fn other_services_in_the_same_export_are_not_imported() {
let zip = write_zip(
"services",
&[
(
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg",
b"x".as_slice(),
),
(
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json",
br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(),
),
("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()),
("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()),
],
);
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
let _ = std::fs::remove_file(&zip);
assert_eq!(scan.photos.len(), 1);
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
assert_eq!(scan.skipped_other, 2);
}
#[test]
fn an_export_without_sidecars_keeps_every_service() {
let zip = write_zip(
"no-sidecars",
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
);
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
let _ = std::fs::remove_file(&zip);
assert_eq!(scan.photos.len(), 1);
}
#[test]
fn a_stop_request_ends_the_scan() {
let zip = write_zip(
"stop",
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
);
let result = scan_archives_until(std::slice::from_ref(&zip), || true);
let _ = std::fs::remove_file(&zip);
assert!(result.is_err());
}
}
-2
View File
@@ -20,11 +20,9 @@ tokio.workspace = true
tracing.workspace = true
fuser.workspace = true
libc.workspace = true
serde.workspace = true
serde_json.workspace = true
base64.workspace = true
image.workspace = true
lofty.workspace = true
notify = "8"
parking_lot.workspace = true
+25 -38
View File
@@ -877,23 +877,35 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
if archives.is_empty() {
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
} else {
let paths = archives.into_iter().map(PathBuf::from).collect();
start_takeout_import(core, paths, dry_run)
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
let missing: Vec<String> = paths
.iter()
.filter(|path| !path.is_file())
.map(|path| path.display().to_string())
.collect();
if !missing.is_empty() {
CtlResponse::error(CoreError::not_found(format!(
"cannot read: {}",
missing.join(", ")
)))
} else {
let count = paths.len();
match core.spawn_takeout_import(paths, dry_run) {
Ok(()) => CtlResponse::Ok {
message: format!(
"{} {count} archive(s)",
if dry_run { "scanning" } else { "importing" }
),
},
Err(e) => CtlResponse::error(e),
}
}
}
}
Ok(CtlRequest::ImportStatus) => {
let report = crate::takeout::import_status();
CtlResponse::ImportStatus {
running: report.running,
summary: report.summary,
progress: report.progress,
resumable: report.resumable,
}
let (running, summary) = crate::takeout::import_status();
CtlResponse::ImportStatus { running, summary }
}
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
Some(paths) => start_takeout_import(core, paths, false),
},
Ok(CtlRequest::CancelImport) => {
if crate::takeout::cancel_import() {
CtlResponse::Ok {
@@ -1997,31 +2009,6 @@ pub(crate) fn run_control_socket(
}
}
/// Check that every archive is there, then start the import in the background.
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
let missing: Vec<String> = paths
.iter()
.filter(|path| !path.is_file())
.map(|path| path.display().to_string())
.collect();
if !missing.is_empty() {
return CtlResponse::error(CoreError::not_found(format!(
"cannot read: {}",
missing.join(", ")
)));
}
let count = paths.len();
match core.spawn_takeout_import(paths, dry_run) {
Ok(()) => CtlResponse::Ok {
message: format!(
"{} {count} archive(s)",
if dry_run { "scanning" } else { "importing" }
),
},
Err(e) => CtlResponse::error(e),
}
}
#[cfg(test)]
mod request_limit_tests {
use super::*;
+5 -560
View File
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
use base64::Engine as _;
use pdfs_core::control::{
FileThumbRequest, MappingProgress, PhotoItem, PhotoKind, PhotoLocation, PhotoThumb, PlaceInfo,
ThumbnailBuildStatus, Topic, is_audio_file_name, is_raw_image_name, is_thumbnail_image_name,
ThumbnailBuildStatus, Topic, is_raw_image_name, is_thumbnail_image_name,
is_thumbnail_media_name,
};
use pdfs_core::db::{self, StoredPhoto};
@@ -76,36 +76,6 @@ const VIDEO_BOX_WALK_MAX: usize = 32;
/// A single frame is seconds of work even for 4K; this only stops a wedged
/// ffmpeg from holding a thumbnail permit.
const FFMPEG_TIMEOUT: Duration = Duration::from_secs(30);
/// A song no larger than this is downloaded whole for its cover.
const AUDIO_WHOLE_MAX_BYTES: u64 = 4 * 1024 * 1024;
/// How much of a song's start is read first, to learn where its tags end.
const AUDIO_PROBE_BYTES: u64 = 64 * 1024;
/// How much of a song's start is read when its first bytes don't say where its
/// tags end: an Ogg comment packet, which holds the cover, has no size up front.
const AUDIO_HEAD_BYTES: u64 = 4 * 1024 * 1024;
/// Read past the tags at a song's start, so the first audio frames are there
/// for a reader that looks for them.
const AUDIO_HEAD_SLACK: u64 = 64 * 1024;
/// Largest start of a song read for its tags. A cover bigger than this is not
/// worth the download for a 512-pixel thumbnail.
const AUDIO_TAG_MAX_BYTES: u64 = 16 * 1024 * 1024;
/// How much of a song's end is read in formats that keep their tags there
/// (APE tags in WavPack and Monkey's Audio, `id3 ` chunks in WAV and AIFF).
const AUDIO_TAIL_BYTES: u64 = 4 * 1024 * 1024;
/// Names of a folder's cover image, best first, matched without case and
/// extension. A song without a picture in its tags gets this image's.
const FOLDER_COVER_STEMS: [&str; 6] = [
"cover",
"folder",
"front",
"album",
"albumart",
"albumartsmall",
];
/// Image types a folder cover is taken in.
const FOLDER_COVER_EXTENSIONS: [&str; 4] = ["jpg", "jpeg", "png", "webp"];
/// Largest folder cover image downloaded whole for a song's thumbnail.
const FOLDER_COVER_MAX_BYTES: u64 = 32 * 1024 * 1024;
/// Largest thumbnail attached to an upload. Proton's own clients hold theirs to
/// this, stepping the JPEG quality down until it fits.
const UPLOAD_THUMB_MAX_BYTES: usize = 60 * 1024;
@@ -660,7 +630,6 @@ enum Fetched {
/// A whole image.
Whole(Vec<u8>),
Video(VideoSample),
Audio(AudioSample),
}
/// The parts of a video its thumbnail needs, downloaded: the start of the
@@ -797,181 +766,13 @@ fn video_sample_thumbnail(
video_file_thumbnail(&file.0, ffmpeg, timeout)
}
/// The parts of a song its cover needs, downloaded: the start, where most
/// formats keep their tags, an M4A's index, and for formats that keep their
/// tags at the end, the end.
struct AudioSample {
/// The whole file's length.
size: u64,
/// Each part and the offset it came from.
parts: Vec<(u64, Vec<u8>)>,
}
/// Where the tags at the start of a song end, read from its first `probe`
/// bytes: past an ID3v2 tag, and past FLAC's metadata blocks, which hold its
/// picture. `None` when the start says nothing, as for Ogg. A FLAC block
/// header past `probe` stops the walk with a guess of [`AUDIO_HEAD_BYTES`]
/// more.
fn audio_tags_end(probe: &[u8]) -> Option<u64> {
let mut offset = 0u64;
let mut known = false;
if probe.get(0..3) == Some(b"ID3") {
let size = probe.get(6..10)?;
if size.iter().any(|byte| byte & 0x80 != 0) {
return None;
}
let size = size
.iter()
.fold(0u64, |acc, byte| (acc << 7) | u64::from(*byte));
let footer = probe.get(5).is_some_and(|flags| flags & 0x10 != 0);
offset = 10 + size + if footer { 10 } else { 0 };
known = true;
}
let start = usize::try_from(offset).ok()?;
if probe.get(start..start + 4) == Some(b"fLaC") {
offset += 4;
loop {
let at = usize::try_from(offset).ok()?;
let Some(header) = probe.get(at..at + 4) else {
return Some(offset + AUDIO_HEAD_BYTES);
};
let len = u64::from_be_bytes([0, 0, 0, 0, 0, header[1], header[2], header[3]]);
offset += 4 + len;
if header[0] & 0x80 != 0 {
return Some(offset);
}
}
}
known.then_some(offset)
}
/// Whether a song in format `name` may keep its tags at its end.
fn audio_tags_at_end(name: &str) -> bool {
let extension = name
.rsplit_once('.')
.map(|(_, extension)| extension.to_ascii_lowercase())
.unwrap_or_default();
matches!(extension.as_str(), "wv" | "ape" | "wav" | "aif" | "aiff")
}
/// Download what the cover of the song `reader` reads, named `name`, needs. A
/// song of up to [`AUDIO_WHOLE_MAX_BYTES`] comes whole; a larger one brings its
/// tags, never its audio.
async fn sample_audio(reader: &dyn RevisionRead, name: &str) -> DriveResult<AudioSample> {
let size = reader.size();
if size <= AUDIO_WHOLE_MAX_BYTES {
let whole = reader.read_at(0, size).await?;
return Ok(AudioSample {
size,
parts: vec![(0, whole)],
});
}
let probe = reader.read_at(0, AUDIO_PROBE_BYTES.min(size)).await?;
// An M4A keeps its cover in its index, fetched below wherever it sits.
let iso_media = probe.get(4..8) == Some(b"ftyp");
let want = match audio_tags_end(&probe) {
_ if iso_media => 0,
Some(end) => end.saturating_add(AUDIO_HEAD_SLACK),
None => AUDIO_HEAD_BYTES,
}
.min(AUDIO_TAG_MAX_BYTES)
.min(size);
let head = if want > probe.len() as u64 {
reader.read_at(0, want).await?
} else {
probe
};
let mut parts = Vec::new();
match iso_media_index(reader, &head, size).await? {
Some((offset, len)) if offset.saturating_add(len) > head.len() as u64 => {
parts.push((offset, reader.read_at(offset, len).await?));
}
_ => {}
}
let head_len = head.len() as u64;
parts.push((0, head));
if audio_tags_at_end(name) {
let start = size.saturating_sub(AUDIO_TAIL_BYTES).max(head_len);
if start < size {
parts.push((start, reader.read_at(start, size - start).await?));
}
}
Ok(AudioSample { size, parts })
}
/// The picture a song's tags carry: the front cover, or else the first one.
/// `None` for a song without one, or one lofty can't read.
fn embedded_cover(path: &Path) -> Option<Vec<u8>> {
use lofty::config::{ParseOptions, ParsingMode};
use lofty::file::TaggedFileExt as _;
use lofty::picture::PictureType;
let tagged = lofty::probe::Probe::open(path)
.ok()?
.options(
ParseOptions::new()
.read_properties(false)
.parsing_mode(ParsingMode::Relaxed),
)
.guess_file_type()
.ok()?
.read()
.ok()?;
let pictures = || tagged.tags().iter().flat_map(|tag| tag.pictures());
pictures()
.find(|picture| picture.pic_type() == PictureType::CoverFront)
.or_else(|| pictures().next())
.map(|picture| picture.data().to_vec())
.filter(|data| !data.is_empty())
}
/// A thumbnail from the cover in a downloaded [`AudioSample`]'s tags. The
/// sample is laid out in a staging file at the offsets it came from, so the
/// tags are where the file's own headers say they are; the audio between
/// holds no data and costs no disk. A song without a cover, or with one the
/// image decoder can't read, is [`ScaleAttempt::Undecodable`].
fn audio_cover_thumbnail(sample: &AudioSample, name: &str, staging_dir: &Path) -> ScaleAttempt {
let Some(file) = RawTempFile::create_in(staging_dir, &[], name) else {
return ScaleAttempt::Unavailable;
};
let laid_out = OpenOptions::new()
.write(true)
.open(&file.0)
.and_then(|staged| {
for (offset, bytes) in &sample.parts {
staged.write_all_at(bytes, *offset)?;
}
staged.set_len(sample.size)
});
if laid_out.is_err() {
return ScaleAttempt::Unavailable;
}
match embedded_cover(&file.0) {
Some(picture) => scale_thumbnail(&picture, "cover", staging_dir),
None => ScaleAttempt::Undecodable,
}
}
/// How good a folder cover `name` is: its place in [`FOLDER_COVER_STEMS`],
/// lowest best, or `None` for a file that is no cover image.
fn cover_rank(name: &str) -> Option<usize> {
let (stem, extension) = name.rsplit_once('.')?;
let extension = extension.to_ascii_lowercase();
if !FOLDER_COVER_EXTENSIONS.contains(&extension.as_str()) {
return None;
}
let stem = stem.to_ascii_lowercase();
FOLDER_COVER_STEMS.iter().position(|known| *known == stem)
}
/// The thumbnail to attach to an upload of the local file at `path`, which is
/// named `name` in Drive. Without one, Drive's own apps show a generic icon and
/// the gallery has to download the whole file to make its own.
///
/// Best effort: a file that isn't an image, a video or a song with a cover in
/// its tags, is too large to read, or can't be decoded uploads without a
/// thumbnail, as it always did. Blocking and CPU-bound, so async callers run it
/// on the blocking pool.
/// Best effort: a file that isn't an image or a video, is too large to read, or
/// can't be decoded uploads without a thumbnail, as it always did. Blocking and
/// CPU-bound, so async callers run it on the blocking pool.
pub(crate) fn upload_thumbnails(path: &Path, name: &str, staging_dir: &Path) -> Vec<Thumbnail> {
if is_video_name(name) && !ffmpeg_available() {
return Vec::new();
@@ -1005,14 +806,6 @@ fn upload_thumbnails_with(
return Vec::new();
};
scale_thumbnail_unbudgeted(&bytes, name, staging_dir)
} else if is_audio_file_name(name) {
// Only the song's own picture: a cover image beside it is not known
// here, where `path` may be a staging copy rather than the song's
// folder. The daemon finds that one when a listing asks.
match embedded_cover(path) {
Some(picture) => scale_thumbnail(&picture, "cover", staging_dir),
None => return Vec::new(),
}
} else {
return Vec::new();
};
@@ -1193,7 +986,7 @@ impl Core {
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
return Ok(None);
}
if !self.refresh_timeline_joined()? {
if !self.rt.block_on(self.refresh_timeline())? {
return Ok(None);
}
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
@@ -1910,7 +1703,6 @@ impl Core {
return (uid, ThumbAttempt::Cancelled);
}
let video = is_video_name(&name);
let audio = is_audio_file_name(&name);
// Without ffmpeg a video can still have its uploader's
// thumbnail, but Photos has asked the server already.
let can_sample = !video || ffmpeg_available();
@@ -1965,7 +1757,6 @@ impl Core {
let download_uid = uid.clone();
let drive = core.drive.clone();
let audio_name = name.clone();
let download = async move {
// Photos asked the server in one batch already. A
// file view's or a build's file may carry a
@@ -1985,11 +1776,6 @@ impl Core {
if video {
let reader = drive.open_revision(&download_uid).await?;
sample_video(reader.as_ref()).await.map(Fetched::Video)
} else if audio {
let reader = drive.open_revision(&download_uid).await?;
sample_audio(reader.as_ref(), &audio_name)
.await
.map(Fetched::Audio)
} else {
client
.download_file(&download_uid)
@@ -2047,11 +1833,7 @@ impl Core {
// Decoding + scaling a 20 MP JPEG is CPU-bound and would
// stall the runtime's worker; hand it to the blocking pool.
let staging_dir = core.cache.raw_thumbnail_staging_dir().to_path_buf();
let staging = staging_dir.clone();
let made = tokio::task::spawn_blocking(move || match fetched {
Fetched::Audio(sample) => {
audio_cover_thumbnail(&sample, &name, &staging_dir)
}
Fetched::Video(sample) => video_sample_thumbnail(
&sample,
&name,
@@ -2068,14 +1850,6 @@ impl Core {
})
.await
.unwrap_or(ScaleAttempt::Unavailable);
// A song without a picture of its own shows its
// folder's cover.
let made = match made {
ScaleAttempt::Undecodable if audio && core.thumb_job_current(job) => {
core.folder_cover_thumbnail(&uid, staging).await
}
made => made,
};
if !core.thumb_job_current(job) {
return (uid, ThumbAttempt::Cancelled);
}
@@ -2136,85 +1910,9 @@ impl Core {
ThumbAttempt::Cancelled => summary.cancelled += 1,
}
}
// A folder on screen whose tiles were given up on, as a recursive
// build's start does to them, fills in without being opened again.
if summary.made > 0 {
self.events.publish(&[Topic::Thumbnails]);
}
summary
}
/// The cover image in the folder of the song `uid`: the best-named one of
/// [`FOLDER_COVER_STEMS`] among its siblings the database knows.
fn folder_cover(&self, uid: &NodeUid) -> Option<proton_drive_rs::Node> {
let node = self.db.node_by_uid(&uid.to_string()).ok()??;
let parent = node.parent_uid?;
let siblings = match self.db.children_if_listed(&parent) {
Ok(Some(children)) => children,
_ => self.db.known_children(&parent).ok()?,
};
siblings
.into_iter()
.filter(|sibling| !sibling.is_folder() && !sibling.trashed)
.filter_map(|sibling| cover_rank(&sibling.name).map(|rank| (rank, sibling)))
.min_by(|(a, x), (b, y)| a.cmp(b).then_with(|| x.name.cmp(&y.name)))
.map(|(_, cover)| cover)
}
/// A thumbnail of the song `uid` made from its folder's cover image: the
/// cover's own cached thumbnail when it has one, else the thumbnail its
/// uploader attached, else the image itself, up to
/// [`FOLDER_COVER_MAX_BYTES`].
///
/// The result is stored under the song, so a cover added to the folder or
/// replaced later shows only once the song changes or the daemon restarts
/// and forgets the songs it found no cover for.
async fn folder_cover_thumbnail(&self, uid: &NodeUid, staging_dir: PathBuf) -> ScaleAttempt {
let Some(cover) = self.folder_cover(uid) else {
return ScaleAttempt::Undecodable;
};
let ttype = ThumbnailType::Thumbnail;
let cached = self
.cache
.cached_thumbnail_path(&cover.uid, ttype.as_i32(), cover.modification_time)
.and_then(|path| std::fs::read(path).ok());
let ready = match cached {
Some(bytes) => Some(bytes),
None => match self.drive.download_thumbnail(&cover.uid, ttype).await {
Ok(bytes) => bytes.filter(|bytes| !bytes.is_empty()),
Err(e) => {
warn!(cover = %cover.uid, error = %e, "folder cover thumbnail download failed");
return ScaleAttempt::Unavailable;
}
},
};
if let Some(bytes) = ready {
return match ratio_of(&bytes) {
Some(ratio) => ScaleAttempt::Made(GeneratedThumb { bytes, ratio }),
None => ScaleAttempt::Undecodable,
};
}
let read = async {
let reader = self.drive.open_revision(&cover.uid).await?;
if reader.size() > FOLDER_COVER_MAX_BYTES {
return Ok(None);
}
reader.read_at(0, reader.size()).await.map(Some)
};
let bytes = match read.await {
Ok(Some(bytes)) => bytes,
Ok(None) => return ScaleAttempt::Undecodable,
Err(e) => {
warn!(cover = %cover.uid, error = %e, "folder cover download failed");
return ScaleAttempt::Unavailable;
}
};
let name = cover.name;
tokio::task::spawn_blocking(move || scale_thumbnail(&bytes, &name, &staging_dir))
.await
.unwrap_or(ScaleAttempt::Unavailable)
}
pub(crate) fn thumbnail_build_status(&self) -> ThumbnailBuildStatus {
self.thumbnail_build.lock().clone()
}
@@ -2640,36 +2338,6 @@ impl Core {
/// Refresh the timeline off the request path, so a stale page is still served
/// at DB speed. At most one refresh runs at a time.
/// Refresh the timeline and wait for it, sharing a refresh already under
/// way instead of starting a second one beside it.
///
/// The first request after an empty cache has nothing to serve until the
/// fetch lands, so it waits. Two such requests (the gallery and its retry,
/// or a background refresh started by an import) used to each fetch the
/// whole library at once, sharing the session's rate limit and taking
/// minutes. Returns whether the account has a photos library.
pub(crate) fn refresh_timeline_joined(&self) -> CoreResult<bool> {
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() {
std::thread::sleep(std::time::Duration::from_millis(200));
}
// A refresh that landed stamped the timeline fresh. One that failed
// did not, and the caller gets this refresh's own error instead of
// an empty timeline.
if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL)
|| self.timeline_refreshing.swap(true, Ordering::SeqCst)
{
return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0));
}
}
let refreshed = self.rt.block_on(self.refresh_timeline());
self.timeline_refreshing.store(false, Ordering::SeqCst);
if refreshed.is_ok() {
self.events.publish(&[Topic::Photos]);
}
refreshed
}
pub(crate) fn spawn_timeline_refresh(&self) {
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
return;
@@ -2786,10 +2454,6 @@ mod decode_budget_tests {
#[cfg(test)]
mod thumb_tests {
use super::{
AUDIO_HEAD_BYTES, AUDIO_TAIL_BYTES, AudioSample, audio_cover_thumbnail, audio_tags_end,
cover_rank, sample_audio,
};
use super::{
RawTempFile, ScaleAttempt, THUMB_EDGE, UPLOAD_IMAGE_MAX_BYTES, UPLOAD_THUMB_MAX_BYTES,
VIDEO_HEAD_BYTES, VIDEO_WHOLE_MAX_BYTES, VideoSample, can_make_thumbnail, exiftool_binary,
@@ -3301,28 +2965,6 @@ mod thumb_tests {
assert!(thumb.ratio > 0.0);
}
/// Optional check against real songs, through the same partial download a
/// Drive song gets. `PDFS_AUDIO_FIXTURE` lists files separated by `:`.
#[test]
#[ignore = "set PDFS_AUDIO_FIXTURE to songs with a cover in their tags"]
fn real_audio_fixture_yields_its_cover() {
let paths = std::env::var_os("PDFS_AUDIO_FIXTURE").expect("PDFS_AUDIO_FIXTURE");
for path in std::env::split_paths(&paths) {
let name = path.file_name().unwrap().to_string_lossy().into_owned();
let bytes = std::fs::read(&path).unwrap();
let revision = FakeRevision::new(bytes.len() as u64).with(0, bytes);
let sample = block_on(sample_audio(&revision, &name)).unwrap();
let read = revision.read.load(Ordering::Relaxed);
let attempt = audio_cover_thumbnail(&sample, &name, &std::env::temp_dir());
assert!(
matches!(attempt, ScaleAttempt::Made(_)),
"{name}: no cover, read {read} of {} bytes",
revision.size
);
eprintln!("{name}: read {read} of {} bytes", revision.size);
}
}
#[test]
fn videos_are_told_apart_by_name() {
for name in [
@@ -3479,201 +3121,4 @@ mod thumb_tests {
"a small one is untouched"
);
}
/// A `width`×`height` PNG, the cover a tag carries.
fn png(width: u32, height: u32) -> Vec<u8> {
let mut bytes = Vec::new();
image::DynamicImage::ImageRgb8(image::RgbImage::new(width, height))
.write_to(
&mut std::io::Cursor::new(&mut bytes),
image::ImageFormat::Png,
)
.unwrap();
bytes
}
/// An ID3v2.3 tag holding `picture` as its front cover.
fn id3_with_cover(picture: &[u8]) -> Vec<u8> {
let mut body = vec![0u8];
body.extend_from_slice(b"image/png\0");
body.push(3);
body.push(0);
body.extend_from_slice(picture);
let mut frame = b"APIC".to_vec();
frame.extend_from_slice(&(body.len() as u32).to_be_bytes());
frame.extend_from_slice(&[0, 0]);
frame.extend_from_slice(&body);
let len = frame.len() as u32;
let mut tag = b"ID3\x03\x00\x00".to_vec();
tag.extend((0..4).rev().map(|i| ((len >> (7 * i)) & 0x7f) as u8));
tag.extend_from_slice(&frame);
tag
}
/// Ten silent MPEG-1 Layer III frames at 128 kbit/s and 44.1 kHz.
fn mpeg_frames() -> Vec<u8> {
let mut frame = vec![0u8; 417];
frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x64]);
frame.repeat(10)
}
/// A FLAC stream's metadata: STREAMINFO, then `picture` as its front cover.
fn flac_with_cover(picture: &[u8]) -> Vec<u8> {
let mut info = vec![0x10, 0x00, 0x10, 0x00, 0, 0, 0, 0, 0, 0];
let packed: u64 = (44_100 << 44) | (1 << 41) | (15 << 36);
info.extend_from_slice(&packed.to_be_bytes());
info.extend_from_slice(&[0; 16]);
let mut block = Vec::new();
block.extend_from_slice(&3u32.to_be_bytes());
block.extend_from_slice(&9u32.to_be_bytes());
block.extend_from_slice(b"image/png");
block.extend_from_slice(&0u32.to_be_bytes());
for value in [16u32, 8, 24, 0, picture.len() as u32] {
block.extend_from_slice(&value.to_be_bytes());
}
block.extend_from_slice(picture);
let mut out = b"fLaC".to_vec();
out.push(0);
out.extend_from_slice(&(info.len() as u32).to_be_bytes()[1..]);
out.extend_from_slice(&info);
out.push(0x80 | 6);
out.extend_from_slice(&(block.len() as u32).to_be_bytes()[1..]);
out.extend_from_slice(&block);
out
}
fn whole_song(bytes: Vec<u8>) -> AudioSample {
AudioSample {
size: bytes.len() as u64,
parts: vec![(0, bytes)],
}
}
#[test]
fn a_song_shows_the_cover_in_its_tags() {
let tmp = scratch("song-cover");
std::fs::create_dir_all(&tmp).unwrap();
let mut mp3 = id3_with_cover(&png(16, 8));
mp3.extend(mpeg_frames());
let ScaleAttempt::Made(thumb) = audio_cover_thumbnail(&whole_song(mp3), "song.mp3", &tmp)
else {
panic!("an ID3 cover scales");
};
assert!((thumb.ratio - 2.0).abs() < 1e-6);
let ScaleAttempt::Made(thumb) =
audio_cover_thumbnail(&whole_song(flac_with_cover(&png(8, 16))), "song.flac", &tmp)
else {
panic!("a FLAC picture scales");
};
assert!((thumb.ratio - 0.5).abs() < 1e-6);
std::fs::remove_dir_all(&tmp).unwrap();
}
#[test]
fn a_song_without_a_cover_is_no_thumbnail_of_its_own() {
let tmp = scratch("song-bare");
std::fs::create_dir_all(&tmp).unwrap();
let attempt = audio_cover_thumbnail(&whole_song(mpeg_frames()), "song.mp3", &tmp);
assert!(matches!(attempt, ScaleAttempt::Undecodable));
std::fs::remove_dir_all(&tmp).unwrap();
}
#[test]
fn the_start_of_a_song_says_where_its_tags_end() {
let tag = id3_with_cover(&png(4, 4));
assert_eq!(audio_tags_end(&tag), Some(tag.len() as u64));
let flac = flac_with_cover(&png(4, 4));
assert_eq!(audio_tags_end(&flac), Some(flac.len() as u64));
// FLAC after an ID3 tag, as some taggers write it.
let both = [tag.clone(), flac.clone()].concat();
assert_eq!(audio_tags_end(&both), Some(both.len() as u64));
// The picture's block header cut off by the end of the probe, just
// past STREAMINFO: a guess past it.
let cut = &flac[..4 + 4 + 34 + 2];
assert!(audio_tags_end(cut).is_some_and(|end| end > cut.len() as u64));
assert_eq!(audio_tags_end(b"OggS\0\0\0\0"), None);
assert_eq!(
audio_tags_end(b"ID3\x03\x00\x00\x80\0\0\0"),
None,
"not synchsafe"
);
}
#[test]
fn a_large_song_downloads_its_tags_not_its_audio() {
const SIZE: u64 = 300 << 20;
let mut start = id3_with_cover(&png(16, 8));
start.extend(mpeg_frames());
let revision = FakeRevision::new(SIZE).with(0, start);
let sample = block_on(sample_audio(&revision, "long mix.mp3")).unwrap();
let read = revision.read.load(Ordering::Relaxed);
assert!(read < 1 << 20, "read {read} bytes");
let tmp = scratch("song-sparse");
std::fs::create_dir_all(&tmp).unwrap();
let ScaleAttempt::Made(thumb) = audio_cover_thumbnail(&sample, "long mix.mp3", &tmp) else {
panic!("the cover comes from the start alone");
};
assert!((thumb.ratio - 2.0).abs() < 1e-6);
std::fs::remove_dir_all(&tmp).unwrap();
// Ogg says nothing up front: a bounded head. WavPack keeps its tags at
// the end: the tail too.
let revision = FakeRevision::new(SIZE);
block_on(sample_audio(&revision, "live.ogg")).unwrap();
assert!(revision.read.load(Ordering::Relaxed) <= AUDIO_HEAD_BYTES + (64 << 10));
let revision = FakeRevision::new(SIZE);
let sample = block_on(sample_audio(&revision, "live.wv")).unwrap();
assert!(
sample
.parts
.iter()
.any(|(offset, bytes)| *offset == SIZE - AUDIO_TAIL_BYTES
&& bytes.len() as u64 == AUDIO_TAIL_BYTES)
);
}
#[test]
fn an_uploaded_song_carries_its_cover() {
let tmp = scratch("upload-song");
std::fs::create_dir_all(&tmp).unwrap();
// A staging blob has no extension: the format comes from the bytes.
let blob = tmp.join("blob");
let mut mp3 = id3_with_cover(&png(64, 64));
mp3.extend(mpeg_frames());
std::fs::write(&blob, mp3).unwrap();
let ffmpeg = fake_ffmpeg(&tmp, "exit 1");
let thumbs = upload_thumbnails_with(&blob, "song.mp3", &tmp, ffmpeg.as_os_str());
let [thumbnail] = thumbs.as_slice() else {
panic!("one thumbnail, got {}", thumbs.len());
};
assert!(matches!(thumbnail.thumbnail_type, ThumbnailType::Thumbnail));
assert_eq!(ratio_of(&thumbnail.content), Some(1.0));
std::fs::write(&blob, mpeg_frames()).unwrap();
assert!(
upload_thumbnails_with(&blob, "song.mp3", &tmp, ffmpeg.as_os_str()).is_empty(),
"a song without a cover uploads bare"
);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn a_folder_cover_is_found_by_its_usual_names() {
assert_eq!(cover_rank("cover.jpg"), Some(0));
assert_eq!(cover_rank("Cover.JPG"), Some(0));
assert_eq!(cover_rank("folder.png"), Some(1));
assert_eq!(cover_rank("AlbumArtSmall.jpg"), Some(5));
assert!(cover_rank("cover.jpg") < cover_rank("front.webp"));
for name in [
"cover.txt",
"my cover.jpg",
"cover",
"back.jpg",
"cover.jpg.part",
] {
assert_eq!(cover_rank(name), None, "{name}");
}
}
}
-82
View File
@@ -3732,88 +3732,6 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
/// A song with no picture in its tags shows the cover image in its folder,
/// and the daemon tells a following front-end that a thumbnail landed, so
/// a folder on screen fills in without being opened again.
#[test]
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
fn a_song_without_a_cover_shows_its_folders_and_says_so() {
let dir = scratch("song-cover");
std::fs::create_dir_all(&dir).unwrap();
// Silent MPEG-1 Layer III frames, without any tag.
let mut frame = vec![0u8; 417];
frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x64]);
let song = frame.repeat(64);
let mut cover = Vec::new();
image::codecs::jpeg::JpegEncoder::new(&mut cover)
.encode_image(&image::RgbImage::new(300, 300))
.unwrap();
let drive = FakeDrive::new();
drive.device().write("track one.mp3", &song);
drive.device().write("Cover.jpg", &cover);
let client = drive.client(1, Faults::lan());
let daemon = Daemon::start(&dir.join("daemon"), client).unwrap();
let landed = Arc::new(std::sync::atomic::AtomicBool::new(false));
let mut subscription = control::subscribe(&daemon.socket).unwrap();
let flag = landed.clone();
std::thread::spawn(move || {
// The first change names every topic: a new subscription catches up.
let mut caught_up = false;
while let Ok(Some(event)) = subscription.next_event() {
if let control::Event::Changed { topics } = event {
if caught_up && topics.contains(&control::Topic::Thumbnails) {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
return;
}
caught_up = true;
}
}
});
let Ok(Response::Entries { entries }) = daemon.request(&Request::ListDir {
path: String::new(),
}) else {
panic!("listing");
};
let song = entries
.iter()
.find(|entry| entry.name == "track one.mp3")
.expect("the song is listed");
let items = vec![control::FileThumbRequest {
uid: song.uid.clone(),
modified: song.modified,
name: song.name.clone(),
}];
let Ok(Response::FileThumbGeneration { generation }) =
daemon.request(&Request::ReserveFileThumbGeneration)
else {
panic!("generation");
};
let mut path = None;
let settled = wait_until(Duration::from_secs(60), || {
let Ok(Response::Thumbs { items: replies }) = daemon.request(&Request::FileThumbs {
items: items.clone(),
generation,
}) else {
return false;
};
path = replies.first().and_then(|reply| reply.path.clone());
path.is_some()
});
assert!(settled, "the song got no thumbnail");
let thumb = image::load_from_memory(&std::fs::read(path.unwrap()).unwrap()).unwrap();
assert_eq!(thumb.width(), thumb.height(), "the square cover");
assert!(
wait_until(Duration::from_secs(15), || landed
.load(std::sync::atomic::Ordering::SeqCst)),
"no thumbnails change was published"
);
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
let _ = std::fs::remove_dir_all(&dir);
}
/// A video in My files gets a thumbnail from a frame of its own, but a
/// recording can be gigabytes: only its start, and an MP4's index, are
/// downloaded for it. A video with a thumbnail from its uploader is not
File diff suppressed because it is too large. Load diff
-1
View File
@@ -63,7 +63,6 @@ scale.browser-status-meter trough { min-width: 140px; min-height: 6px; }
.photo-viewer-window { background-color: #111014; }
.media-player-window video { background-color: black; }
.media-player-note { color: rgba(255, 255, 255, 0.5); }
.media-player-note .file-thumbnail { border-radius: 12px; }
.viewer-top-bar { background: linear-gradient(to bottom, rgba(0, 0, 0, 0.85), rgba(0, 0, 0, 0.45) 60%, rgba(0, 0, 0, 0)); padding: 10px 16px 40px 20px; color: white; }
.viewer-title { font-weight: 700; font-size: 1.05rem; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); color: white; }
.viewer-counter { font-size: 0.8rem; color: rgba(255, 255, 255, 0.72); text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); }
-4
View File
@@ -128,10 +128,6 @@ fn follow_changes(ui: &Rc<Ui>, state: &FeedState, topics: &[Topic]) {
if changed(Topic::Shares) {
refresh_invitations_badge(ui, Duration::ZERO);
}
// On any page: a grid, a list or a player may show a tile still blank.
if changed(Topic::Thumbnails) {
refresh_file_thumbnails(ui);
}
let page = ui.stack.visible_child_name();
// A timeline out of sight is read again when the user comes back to it;
// one on screen is left alone (below).
+6 -8
View File
@@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState};
use pdfs_core::control::{
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem,
LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo,
PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo,
SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase,
SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress,
PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope,
Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry,
ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
};
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
@@ -668,11 +668,9 @@ fn build_window(app: &adw::Application) {
progress_bar: takeout_widgets.progress_bar.clone(),
summary_group: takeout_widgets.summary_group.clone(),
summary_rows: RefCell::new(Vec::new()),
resume_group: takeout_widgets.resume_group.clone(),
resume_row: takeout_widgets.resume_row.clone(),
resume_button: takeout_widgets.resume_button.clone(),
inflight: Cell::new(false),
running: Cell::new(false),
dry_run: Cell::new(false),
},
update,
});
+6 -8
View File
@@ -1115,6 +1115,9 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
// count falling to zero is what "the sync finished" looks like from here.
// Jobs are deliberately not counted — a bulk upload retires its scan job and
// starts its upload job mid-flight, which is not a thing finishing.
// The Import page reads its progress off the same snapshot rather than
// polling the daemon a second time for numbers already on the wire.
takeout_progress(ui, jobs);
let previous = ui.status.active_transfers.replace(items.len());
if !items.is_empty() {
@@ -1208,18 +1211,13 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
let done = j.done.to_string();
let total = j.total.to_string();
let (title, detail) = if j.title == IMPORT_JOB {
import_job_text(&j.detail)
} else {
(j.title.clone(), j.detail.clone())
};
let args: [(&str, &str); 4] = [
("title", &title),
("detail", &detail),
("title", &j.title),
("detail", &j.detail),
("done", &done),
("total", &total),
];
let text = match (j.total > 0, detail.is_empty()) {
let text = match (j.total > 0, j.detail.is_empty()) {
// Translators: a background job's progress. {title} is the job, such as
// "Scanning Photos"; {done} and {total} are counts.
(true, true) => gettext_f("{title} — {done} of {total}", &args),
+158 -326
View File
@@ -15,30 +15,17 @@
//! Import — staging is local, so a mis-drop costs nothing.
//!
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the
//! running counts and the final report. It asks again whenever the import's job
//! moves in the transfer snapshot the daemon already pushes. A run the service
//! did not get to finish can be picked up with [`Request::ResumeImport`].
//! stages paths, launches, asks [`Request::ImportStatus`] for the final report,
//! and reads live progress off the import's job in the transfer snapshot the
//! daemon already pushes (see [`takeout_progress`]).
use crate::*;
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
/// Seeing it move is the cue to ask for the import's status; the Activity list
/// shows it under its translated name ([`import_job_text`]).
/// Matching on it is how this page borrows the progress the daemon already
/// pushes instead of asking for it separately.
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
/// The import job's title and detail in the user's language. The daemon names
/// its steps in English; the file being uploaded passes through as it is.
pub(crate) fn import_job_text(detail: &str) -> (String, String) {
let detail = match detail {
"reading archives" => gettext("Reading the archives"),
"checking for photos you already have" => gettext("Checking for photos you already have"),
"sorting photos into albums" => gettext("Sorting photos into albums"),
other => other.to_string(),
};
(gettext("Importing from Google Photos"), detail)
}
/// What the archive list says before anything is staged. `repaint_archives`
/// swaps in the count and total once there is something to report, and swaps
/// this back when the list is emptied again.
@@ -108,19 +95,18 @@ pub(crate) struct TakeoutState {
pub(crate) progress_group: adw::PreferencesGroup,
pub(crate) progress_label: gtk4::Label,
pub(crate) progress_bar: gtk4::ProgressBar,
/// The running counts, or the report for the last finished run; hidden
/// until there is one.
/// Report group for the last finished run, hidden until there is one.
pub(crate) summary_group: adw::PreferencesGroup,
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
/// Offers to resume an import the service did not get to finish.
pub(crate) resume_group: adw::PreferencesGroup,
pub(crate) resume_row: adw::ActionRow,
pub(crate) resume_button: gtk4::Button,
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
/// can't stack worker threads on the 2s tick.
pub(crate) inflight: Cell<bool>,
/// Whether the daemon reported an import running on the last poll.
pub(crate) running: Cell<bool>,
/// Whether the run in flight was launched as a dry run, so the report can
/// say "would upload" rather than claiming photos were sent. The daemon's
/// summary carries no such flag — it is the same shape either way.
pub(crate) dry_run: Cell<bool>,
}
/// Widgets the Import page hands back for wiring.
@@ -140,9 +126,6 @@ pub(crate) struct TakeoutWidgets {
pub(crate) progress_label: gtk4::Label,
pub(crate) progress_bar: gtk4::ProgressBar,
pub(crate) summary_group: adw::PreferencesGroup,
pub(crate) resume_group: adw::PreferencesGroup,
pub(crate) resume_row: adw::ActionRow,
pub(crate) resume_button: gtk4::Button,
}
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
@@ -278,23 +261,7 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
.visible(false)
.build();
// An import cut short by a restart or a lost connection, offered back at the
// top of the page: picking the archives again is exactly what a reboot in
// the middle of a migration should not cost.
let resume_button = gtk4::Button::builder()
.label(pgettext("verb", "Resume"))
.valign(gtk4::Align::Center)
.build();
resume_button.add_css_class("suggested-action");
let resume_row = adw::ActionRow::builder()
.title(gettext("The last import didn't finish"))
.build();
resume_row.add_suffix(&resume_button);
let resume_group = adw::PreferencesGroup::builder().visible(false).build();
resume_group.add(&resume_row);
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
groups.append(&resume_group);
groups.append(&dropzone);
groups.append(&list_group);
groups.append(&actions);
@@ -334,9 +301,6 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
progress_label: progress_label.clone(),
progress_bar: progress_bar.clone(),
summary_group: summary_group.clone(),
resume_group,
resume_row,
resume_button,
};
(frame.upcast(), widgets)
}
@@ -463,28 +427,6 @@ pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
});
});
let ui_resume = ui.clone();
widgets.resume_button.connect_clicked(move |_| {
let ui = ui_resume.clone();
ui.busy_begin();
let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport);
glib::spawn_future_local(async move {
let result = rx.recv().await;
ui.busy_end();
match result {
Ok(Ok(Response::Ok { .. })) => import_started(&ui, false),
Ok(Ok(Response::Error { message, kind })) => {
toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind);
}
_ => toast_error(
&ui,
&gettext("Couldn't resume the import"),
&gettext("The Proton Drive service didn't respond."),
),
}
});
});
let ui_back = ui.clone();
widgets
.back_button
@@ -617,7 +559,6 @@ fn sync_takeout_actions(ui: &Rc<Ui>) {
.import_button
.set_sensitive(has_files && !running);
ui.takeout.clear_button.set_sensitive(has_files && !running);
ui.takeout.resume_button.set_sensitive(!running);
ui.takeout.cancel_button.set_visible(running);
}
@@ -636,6 +577,7 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
return;
}
ui.takeout.dry_run.set(dry_run);
ui.busy_begin();
let rx = spawn_request(
ui.dirs.control_socket(),
@@ -646,7 +588,27 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
let result = rx.recv().await;
ui.busy_end();
match result {
Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run),
Ok(Ok(Response::Ok { .. })) => {
ui.takeout.running.set(true);
ui.gallery.import_banner.set_revealed(true);
ui.takeout.summary_group.set_visible(false);
ui.takeout.progress_group.set_visible(true);
ui.takeout.progress_bar.set_fraction(0.0);
ui.takeout.progress_label.set_text(&if dry_run {
gettext("Reading archives…")
} else {
gettext("Starting import…")
});
sync_takeout_actions(&ui);
toast(
&ui,
&if dry_run {
gettext("Scanning the export…")
} else {
gettext("Import started — you can leave this page, it keeps running.")
},
);
}
Ok(Ok(Response::Error { message, kind })) => {
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
}
@@ -659,39 +621,9 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
});
}
/// Switch the page over to reporting progress once the daemon took the run.
fn import_started(ui: &Rc<Ui>, dry_run: bool) {
ui.takeout.running.set(true);
if !dry_run {
ui.gallery.import_banner.set_revealed(true);
}
ui.takeout.resume_group.set_visible(false);
ui.takeout.summary_group.set_visible(false);
ui.takeout.progress_group.set_visible(true);
ui.takeout.progress_group.set_title(&if dry_run {
gettext("Scan in progress")
} else {
gettext("Import in progress")
});
ui.takeout.progress_bar.set_fraction(0.0);
ui.takeout
.progress_label
.set_text(&gettext("Reading the archives…"));
sync_takeout_actions(ui);
toast(
ui,
&if dry_run {
gettext("Scanning the export…")
} else {
gettext("Import started — you can leave this page, it keeps running.")
},
);
}
/// Ask the daemon for the import's state and paint it: the step and counts
/// while it runs, the report once it has ended, and the offer to resume a run
/// that did not finish. Asked on arrival and while import progress keeps
/// arriving. Cheap and inflight-guarded.
/// Ask the daemon for the import's state, and paint the report once a run has
/// finished. Asked on arrival and while import progress keeps arriving. Cheap
/// and inflight-guarded.
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
if ui.takeout.inflight.get() {
return;
@@ -702,250 +634,124 @@ pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
glib::spawn_future_local(async move {
let result = rx.recv().await;
ui.takeout.inflight.set(false);
let Ok(Ok(Response::ImportStatus {
running,
summary,
progress,
resumable,
})) = result
else {
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
return;
};
let was_running = ui.takeout.running.replace(running);
let dry = summary.as_ref().is_some_and(|s| s.dry_run);
ui.gallery.import_banner.set_revealed(running && !dry);
ui.takeout.progress_group.set_visible(running);
ui.gallery.import_banner.set_revealed(running);
if running {
ui.takeout.progress_group.set_title(&if dry {
gettext("Scan in progress")
} else {
gettext("Import in progress")
});
paint_progress(&ui, progress.as_ref());
} else if was_running && let Some(summary) = &summary {
announce_end(&ui, summary);
ui.takeout.progress_group.set_visible(true);
} else {
ui.takeout.progress_group.set_visible(false);
// An import runs for hours, so its ending is announced rather than
// left to be discovered: a toast for whoever is looking at the
// window, and a desktop notification for whoever is not.
let failed = summary.as_ref().and_then(|s| s.error.clone());
if was_running && let Some(error) = &failed {
let headline = gettext("Couldn't import from Google Photos");
toast_error(&ui, &headline, error);
notify("takeout-import", &headline, error);
} else if was_running {
let dry = ui.takeout.dry_run.get();
let headline = if dry {
gettext("Scan finished")
} else {
gettext("Import finished")
};
toast(&ui, &headline);
if !dry {
let body = match &summary {
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
Some(s) if s.failed > 0 => ngettext_f(
"{n} photo added, {failed} failed.",
"{n} photos added, {failed} failed.",
s.uploaded as u64,
&[("failed", &s.failed.to_string())],
),
Some(s) if s.cancelled => ngettext_f(
"Stopped after {n} photo.",
"Stopped after {n} photos.",
s.uploaded as u64,
&[],
),
Some(s) => ngettext_f(
"{n} photo added to Proton Photos.",
"{n} photos added to Proton Photos.",
s.uploaded as u64,
&[],
),
None => gettext("The Google Photos import has finished."),
};
notify("takeout-import", &headline, &body);
}
}
}
if was_running != running {
sync_takeout_actions(&ui);
}
match &summary {
Some(summary) => repaint_summary(&ui, summary, running),
match summary {
Some(summary) => repaint_summary(&ui, &summary),
None => ui.takeout.summary_group.set_visible(false),
}
paint_resume(&ui, running, summary.as_ref(), &resumable);
});
}
/// Tell the user how a run ended. An import runs for hours, so its ending is
/// announced rather than left to be discovered: a toast for whoever is looking
/// at the window, and a desktop notification for whoever is not.
fn announce_end(ui: &Rc<Ui>, summary: &ImportSummary) {
if summary.dry_run {
toast(ui, &gettext("Scan finished"));
return;
}
if let Some(error) = &summary.error {
let kind = summary.error_kind.unwrap_or_default();
let headline = gettext("Couldn't import from Google Photos");
toast_failure(ui, &headline, error, kind);
notify(
"takeout-import",
&headline,
&gettext("You can resume it once the problem is solved."),
);
return;
}
let (headline, body) = if summary.interrupted {
(
gettext("Import interrupted"),
gettext("You can resume it where it stopped."),
)
} else if summary.cancelled {
(
gettext("Import stopped"),
ngettext_f(
"Stopped after {n} photo.",
"Stopped after {n} photos.",
summary.uploaded as u64,
&[],
),
)
} else if summary.failed > 0 {
(
gettext("Import finished"),
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
ngettext_f(
"{n} photo added, {failed} failed.",
"{n} photos added, {failed} failed.",
summary.uploaded as u64,
&[("failed", &summary.failed.to_string())],
),
)
} else {
(
gettext("Import finished"),
ngettext_f(
"{n} photo added to Proton Photos.",
"{n} photos added to Proton Photos.",
summary.uploaded as u64,
&[],
),
)
};
toast(ui, &headline);
notify("takeout-import", &headline, &body);
// A complete import is done with its archives; keeping them staged would
// only invite running the same export again.
if summary.complete() {
ui.takeout.archives.borrow_mut().clear();
repaint_archives(ui);
}
}
/// Paint where a running import is.
fn paint_progress(ui: &Rc<Ui>, progress: Option<&ImportProgress>) {
let Some(progress) = progress else {
ui.takeout.progress_bar.pulse();
return;
};
let done = progress.done.to_string();
let total = progress.total.to_string();
let current = progress.current.clone().unwrap_or_default();
let counted = progress.total > 0;
let args = [
("done", done.as_str()),
("total", total.as_str()),
("name", current.as_str()),
];
let text = match (progress.phase, counted) {
(ImportPhase::Scanning, _) => gettext("Reading the archives…"),
(ImportPhase::Checking, false) => gettext("Checking for photos you already have…"),
// Translators: {done} and {total} are counts of photos.
(ImportPhase::Checking, true) => gettext_f(
"Checking for photos you already have — {done} of {total}",
&args,
),
(ImportPhase::Uploading, false) => gettext("Uploading…"),
(ImportPhase::Uploading, true) if current.is_empty() => {
// Translators: {done} and {total} are counts of photos.
gettext_f("Uploading {done} of {total}", &args)
}
// Translators: {done} and {total} are counts of photos; {name} is the file being uploaded.
(ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args),
(ImportPhase::Albums, false) => gettext("Sorting photos into albums…"),
// Translators: {done} and {total} are counts of albums.
(ImportPhase::Albums, true) => {
gettext_f("Sorting photos into albums — {done} of {total}", &args)
}
};
ui.takeout.progress_label.set_text(&text);
if counted {
ui.takeout
.progress_bar
.set_fraction((progress.done as f64 / progress.total as f64).min(1.0));
} else {
ui.takeout.progress_bar.pulse();
}
}
/// Offer to resume the last run when the service says it can be.
fn paint_resume(ui: &Rc<Ui>, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) {
let offer = !running && !resumable.is_empty();
ui.takeout.resume_group.set_visible(offer);
if !offer {
return;
}
let uploaded = summary.map_or(0, |s| s.uploaded) as u64;
// Translators: {n} is the number of photos uploaded before the import stopped.
ui.takeout.resume_row.set_subtitle(&ngettext_f(
"{n} photo uploaded before it stopped. Resuming skips what is already there.",
"{n} photos uploaded before it stopped. Resuming skips what is already there.",
uploaded,
&[],
));
}
/// Paint the counts of a running import, or the report of the last one.
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
/// Paint the last run's report.
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
ui.takeout.summary_group.remove(&row);
}
let dry = summary.dry_run;
ui.takeout.summary_group.set_title(&if running {
gettext("So far")
} else if dry {
let dry = ui.takeout.dry_run.get();
ui.takeout.summary_group.set_title(&if dry {
gettext("Scan result")
} else {
gettext("Last import")
});
let description = if running {
None
} else if let Some(error) = &summary.error {
// The reason is the service's own text, which is not translated; the
// headline in front of it is. A group's description is markup, and the
// reason may name a path.
let what = gettext("Couldn't import from Google Photos");
Some(
glib::markup_escape_text(&gettext_f(
"{what}: {detail}",
&[
(
"what",
error_headline(summary.error_kind.unwrap_or_default(), &what),
),
("detail", error),
],
))
.to_string(),
)
ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error {
// The reason is the service's own text, which is not translated. A
// group's description is markup, and the reason may name a path.
glib::markup_escape_text(&gettext_f(
"{what}: {detail}",
&[
("what", &gettext("Couldn't import from Google Photos")),
("detail", error),
],
))
.to_string()
} else if dry {
Some(gettext(
"Nothing was uploaded — this is what an import would do.",
))
} else if summary.interrupted {
Some(gettext(
"The Proton Drive service stopped while the import ran. Resume it to continue where it left off.",
))
gettext("Nothing was uploaded — this is what an import would do.")
} else if summary.cancelled {
Some(gettext(
"Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.",
))
} else if !summary.complete() {
Some(gettext(
"Finished with problems. Run it again to retry what didn't upload.",
))
gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.")
} else {
Some(gettext("Finished."))
};
ui.takeout
.summary_group
.set_description(description.as_deref());
gettext("Finished.")
}));
// Only lines that carry information: a run with no failures should not have
// to show a "0 failed" row for the user to read past.
let mut lines: Vec<(String, String)> =
vec![(gettext("Photos in the export"), summary.found.to_string())];
if dry {
lines.push((gettext("Would upload"), summary.to_upload.to_string()));
} else {
lines.push((gettext("Uploaded"), summary.uploaded.to_string()));
}
let mut lines: Vec<(String, String)> = vec![
(gettext("Photos in the export"), summary.found.to_string()),
(
if dry {
gettext("Would upload")
} else {
gettext("Uploaded")
},
summary.uploaded.to_string(),
),
];
if summary.duplicates > 0 {
lines.push((
gettext("Already in Proton Photos"),
summary.duplicates.to_string(),
));
}
if summary.live_photos > 0 {
lines.push((
gettext("Live photos kept together"),
summary.live_photos.to_string(),
));
}
if summary.albums_created > 0 {
lines.push((
gettext("Albums created"),
if dry {
gettext("Albums to create")
} else {
gettext("Albums created")
},
summary.albums_created.to_string(),
));
}
@@ -961,31 +767,25 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
summary.skipped_trashed.to_string(),
));
}
if dry && summary.to_upload_bytes > 0 {
if summary.bytes > 0 {
lines.push((
gettext("Data to upload"),
human_bytes(summary.to_upload_bytes),
if dry {
gettext("Data to upload")
} else {
gettext("Data uploaded")
},
human_bytes(summary.bytes),
));
} else if summary.bytes > 0 {
lines.push((gettext("Data uploaded"), human_bytes(summary.bytes)));
}
if summary.failed > 0 {
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
}
for path in &summary.unreadable_archives {
let name = Path::new(path)
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| path.clone());
lines.push((gettext("Couldn't read this archive"), name));
}
let mut rows = Vec::new();
for (title, value) in lines {
let row = adw::ActionRow::builder()
.title(title)
.subtitle(value)
.use_markup(false)
.build();
row.add_css_class("property");
ui.takeout.summary_group.add(&row);
@@ -994,3 +794,35 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
*ui.takeout.summary_rows.borrow_mut() = rows;
ui.takeout.summary_group.set_visible(true);
}
/// Paint the import's progress from the transfer snapshot the daemon pushed,
/// so this page costs no extra request. A job with no known total
/// pulses instead of showing a fraction, the same as the Activity list.
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
return;
};
ui.takeout.progress_group.set_visible(true);
if job.total > 0 {
ui.takeout
.progress_bar
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
let done = job.done.to_string();
let total = job.total.to_string();
// Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on.
let text = gettext_f(
"{done} of {total} — {detail}",
&[("done", &done), ("total", &total), ("detail", &job.detail)],
);
ui.takeout.progress_label.set_text(&text);
} else {
ui.takeout.progress_bar.pulse();
ui.takeout
.progress_label
.set_text(&if job.detail.is_empty() {
gettext("Working…")
} else {
job.detail.clone()
});
}
}
+9 -17
View File
@@ -19,9 +19,6 @@ const SEEK_SHORT: i64 = 5;
/// How much ↑ and ↓ change the volume in the My files player.
const VOLUME_STEP: f64 = 0.1;
/// The edge of a song's cover in the My files player, in logical pixels.
const COVER_SIZE: i32 = 320;
thread_local! {
/// The volume and mute the user last set, applied to the next stream so
/// muting one video doesn't leave the next one blaring. Kept for the run
@@ -205,21 +202,16 @@ fn build_media_player(ui: &Rc<Ui>, entry: &DirEntry, path: &Path) {
wire_player(&video, &gtk_window);
// A song has no picture, which would leave a black window above the
// controls: its cover stands in for it, the one its listing tile shows,
// and a note while there is none.
let note = file_thumbnail_widget(COVER_SIZE, 128);
note.set_can_target(false);
note.set_visible(false);
// controls: a note stands in for it.
let note = gtk4::Image::builder()
.icon_name("audio-x-generic-symbolic")
.pixel_size(128)
.halign(gtk4::Align::Center)
.valign(gtk4::Align::Center)
.can_target(false)
.visible(false)
.build();
note.add_css_class("media-player-note");
if let Some(cover) = note
.child()
.and_then(|fallback| fallback.next_sibling())
.and_downcast::<gtk4::Picture>()
{
// The whole cover, not a square cut out of it.
cover.set_content_fit(gtk4::ContentFit::Contain);
}
bind_file_thumbnail(ui, &note, entry, true);
let overlay = gtk4::Overlay::new();
overlay.set_child(Some(&video));
overlay.add_overlay(&note);
+6 -114
View File
@@ -11,7 +11,6 @@ use pdfs_core::control::{FileThumbRequest, is_thumbnail_media_name, is_video_fil
const THUMB_BATCH: usize = 32;
const THUMB_CACHE_MAX: usize = 256;
const THUMB_MISSING_MAX: usize = 2048;
const THUMB_BLANK_MAX: usize = 1024;
const THUMB_DEBOUNCE: Duration = Duration::from_millis(40);
const THUMB_RETRY: Duration = Duration::from_secs(2);
@@ -21,9 +20,6 @@ pub(crate) struct FileThumbnailState {
missing: RefCell<HashSet<String>>,
missing_order: RefCell<VecDeque<String>>,
wanted: RefCell<HashMap<String, Vec<gtk4::Picture>>>,
/// Tiles on screen still showing their icon, by key, kept across a
/// cancelled request so [`refresh_file_thumbnails`] can ask for them again.
blank: RefCell<HashMap<String, BlankTile>>,
queue: RefCell<VecDeque<FileThumbRequest>>,
inflight: Cell<bool>,
source: RefCell<Option<glib::SourceId>>,
@@ -31,18 +27,6 @@ pub(crate) struct FileThumbnailState {
generation_loading: Cell<bool>,
}
/// What a tile left blank needs to be asked for again.
struct BlankTile {
request: FileThumbRequest,
pictures: Vec<glib::WeakRef<gtk4::Picture>>,
}
/// Whether a picture registered as blank under `key` still is: bound to that
/// key, not painted, and in a window rather than in a list's spare pool.
fn tile_still_blank(key: &str, bound: &str, painted: bool, attached: bool) -> bool {
bound == key && !painted && attached
}
#[derive(Debug, PartialEq, Eq)]
enum ReplyDisposition {
Ignore,
@@ -71,7 +55,6 @@ impl FileThumbnailState {
missing: RefCell::new(HashSet::new()),
missing_order: RefCell::new(VecDeque::new()),
wanted: RefCell::new(HashMap::new()),
blank: RefCell::new(HashMap::new()),
queue: RefCell::new(VecDeque::new()),
inflight: Cell::new(false),
source: RefCell::new(None),
@@ -105,85 +88,6 @@ impl FileThumbnailState {
}
}
}
fn remember_blank(&self, key: &str, request: &FileThumbRequest, picture: &gtk4::Picture) {
let mut blank = self.blank.borrow_mut();
if blank.len() >= THUMB_BLANK_MAX {
prune_blank(&mut blank);
}
let tile = blank.entry(key.to_string()).or_insert_with(|| BlankTile {
request: request.clone(),
pictures: Vec::new(),
});
if !tile
.pictures
.iter()
.any(|weak| weak.upgrade().as_ref() == Some(picture))
{
tile.pictures.push(picture.downgrade());
}
}
}
/// Drop the tiles that were painted, rebound to another file, or freed.
fn prune_blank(blank: &mut HashMap<String, BlankTile>) {
blank.retain(|key, tile| {
tile.pictures.retain(|weak| {
weak.upgrade().is_some_and(|picture| {
tile_still_blank(
key,
&picture.widget_name(),
picture.is_visible(),
picture.root().is_some(),
)
})
});
!tile.pictures.is_empty()
});
}
/// Ask again for every tile on screen still showing its icon. Sent when the
/// daemon says thumbnails landed, so a folder fills in while a build runs in
/// it, or after its own requests were cancelled, without being opened again.
pub(crate) fn refresh_file_thumbnails(ui: &Rc<Ui>) {
let tiles: Vec<(String, FileThumbRequest, Vec<gtk4::Picture>)> = {
let mut blank = ui.file_thumbs.blank.borrow_mut();
prune_blank(&mut blank);
blank
.iter()
.map(|(key, tile)| {
let pictures = tile.pictures.iter().filter_map(|w| w.upgrade()).collect();
(key.clone(), tile.request.clone(), pictures)
})
.collect()
};
if tiles.is_empty() {
return;
}
for (key, request, pictures) in tiles {
let cached = ui.file_thumbs.textures.borrow().get(&key).cloned();
if let Some(texture) = cached {
for picture in &pictures {
paint(picture, &key, &texture);
}
ui.file_thumbs.blank.borrow_mut().remove(&key);
continue;
}
// A miss from before the thumbnail landed is no verdict any more.
ui.file_thumbs.missing.borrow_mut().remove(&key);
let mut wanted = ui.file_thumbs.wanted.borrow_mut();
let waiters = wanted.entry(key).or_default();
for picture in pictures {
if !waiters.contains(&picture) {
waiters.push(picture);
}
}
drop(wanted);
let mut queue = ui.file_thumbs.queue.borrow_mut();
queue.retain(|item| item.uid != request.uid);
queue.push_back(request);
}
schedule_file_thumbs(ui);
}
/// Abandon every opportunistic ordinary-file thumbnail associated with the
@@ -326,12 +230,6 @@ pub(crate) fn bind_file_thumbnail(
paint(&picture, &key, texture);
return;
}
let request = FileThumbRequest {
uid: entry.uid.clone(),
modified: entry.modified,
name: entry.name.clone(),
};
ui.file_thumbs.remember_blank(&key, &request, &picture);
if ui.file_thumbs.missing.borrow().contains(&key) {
return;
}
@@ -344,7 +242,11 @@ pub(crate) fn bind_file_thumbnail(
.push(picture);
let mut queue = ui.file_thumbs.queue.borrow_mut();
queue.retain(|item| item.uid != entry.uid);
queue.push_back(request);
queue.push_back(FileThumbRequest {
uid: entry.uid.clone(),
modified: entry.modified,
name: entry.name.clone(),
});
drop(queue);
schedule_file_thumbs(ui);
}
@@ -485,7 +387,6 @@ fn flush_file_thumbs(ui: &Rc<Ui>) {
match gtk4::gdk::Texture::from_filename(&path) {
Ok(texture) => {
ui.file_thumbs.store_texture(&key, texture.clone());
ui.file_thumbs.blank.borrow_mut().remove(&key);
if let Some(pictures) =
ui.file_thumbs.wanted.borrow_mut().remove(&key)
{
@@ -546,16 +447,7 @@ fn retry_file_thumbs(ui: &Rc<Ui>, items: Vec<FileThumbRequest>) {
#[cfg(test)]
mod tests {
use super::{ReplyDisposition, reply_disposition, tile_still_blank};
#[test]
fn only_a_tile_still_showing_its_icon_is_asked_for_again() {
assert!(tile_still_blank("a:1", "a:1", false, true));
assert!(!tile_still_blank("a:1", "a:1", true, true), "painted");
assert!(!tile_still_blank("a:1", "b:1", false, true), "rebound");
assert!(!tile_still_blank("a:1", "a:2", false, true), "file changed");
assert!(!tile_still_blank("a:1", "a:1", false, false), "spare row");
}
use super::{ReplyDisposition, reply_disposition};
fn is_image_name(name: &str) -> bool {
pdfs_core::control::is_thumbnail_image_name(name)
-190
View File
@@ -109,12 +109,6 @@ a live account since.
| Entry | Title |
|---|---|
| [B289](#b289--the-takeout-import-mixes-in-other-services-and-drops-location-and-live-photos) | The Takeout import mixes in other services and drops location and live photos |
| [B288](#b288--an-interrupted-takeout-import-reads-as-finished-and-one-error-ends-it) | An interrupted Takeout import reads as finished, and one error ends it |
| [B287](#b287--a-photo-draft-left-by-an-interrupted-upload-blocks-that-name) | A photo draft left by an interrupted upload blocks that name |
| [B286](#b286--resuming-a-takeout-import-fails-on-a-draft-with-no-content-hash) | Resuming a Takeout import fails on a draft with no content hash |
| [B285](#b285--a-restarted-secret-service-leaves-the-process-on-a-dead-session) | A restarted Secret Service leaves the process on a dead session |
| [B284](#b284--a-keyring-entry-reported-missing-discards-refreshed-session-tokens) | A keyring entry reported missing discards refreshed session tokens |
| [B283](#b283--videos-have-no-thumbnail-outside-photos-and-a-ts-recording-looks-like-source-code) | Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code |
| [B282](#b282--a-file-kept-offline-is-downloaded-again-after-a-change-made-elsewhere) | A file kept offline is downloaded again after a change made elsewhere |
| [B281](#b281--a-file-in-a-synced-folder-cannot-be-kept-offline) | A file in a synced folder cannot be kept offline |
@@ -392,190 +386,6 @@ a live account since.
Newest first.
### B289 — The Takeout import mixes in other services and drops location and live photos
**Status:** Fixed, not verified live
**Found:** 2026-11-15, auditing the Google Photos import alongside
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
**Where:** `crates/pdfs-core/src/takeout.rs`; `plan` and `upload_takeout_photo` in
`crates/pdfs-fuse/src/takeout.rs`
**Repro:** Import a Takeout export that also holds `Drive/` or `YouTube/`, with Pixel motion
photos, iPhone live photos and photos that have a place in Google Photos.
**Cause.** Five gaps in what the import carried over:
- The service folder was dropped without being checked, so media from other Google services in
the same export landed in the photo timeline.
- `geoData` was never read, although `docs/PHOTOS.md` said the import takes locations from the
export, and the SDK had no way to upload a location.
- The two halves of a live or motion photo uploaded as unrelated photos, the `.MOV` half usually
without a date, and the `.MP` half of a Pixel motion photo was skipped as unknown.
- `creationTime`, the day the photo reached Google, outranked the date in the file name and the
photo's own EXIF.
- A sidecar with `null` in `favorited`, a numeric `timestamp` or a `0` timestamp lost its other
fields or filed the photo under 1970.
**Fix:** The Photos service is the folder whose sidecars carry `photoTakenTime`, and only it is
imported (everything, when no folder does). `geoData`, else `geoDataExif`, becomes the photo's
`Location` attribute through SDK 0.12.0's `PhotoUploadMetadata::location`; `0, 0` is no position.
Stills and videos with the same stem in one folder (`IMG_1.HEIC` + `IMG_1.MOV`,
`PXL_1.MP.jpg` + `PXL_1.MP`) are paired unless their dates are minutes apart: the still is
tagged as a live or motion photo, the video is uploaded after it with `main_photo_uid` and takes
its date and place. The capture time is `photoTakenTime`, then the name, then EXIF, then
`creationTime`. Favorites are also set on photos that were already on the account.
**Verified:** `takeout::tests` in `pdfs-core` (services, locations, timestamps, live keys) and
`pdfs-fuse` (pairing, skew, folded copies). Not run against a real export and account.
---
### B288 — An interrupted Takeout import reads as finished, and one error ends it
**Status:** Fixed, not verified live
**Found:** 2026-11-15, auditing the Google Photos import alongside
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
**Where:** `crates/pdfs-fuse/src/takeout.rs`; `refresh_takeout` in
`crates/pdfs-gui/src/app/pages/takeout.rs`; `cmd_import_google_photos` in
`crates/pdfs-cli/src/main.rs`
**Repro:** Start an import of several archives, one of them damaged, and restart the service
while it uploads.
**Cause.**
- The import state lived only in memory. After a restart `ImportStatus` had no summary, and the
app announced "Import finished". The archives had to be chosen again to continue.
- A daemon shutdown broke the upload loop but still filed albums and logged a success.
- One unreadable archive failed the scan, and a failed request in the middle of the run
discarded every count and skipped the albums for what had uploaded.
- The progress total counted photos already on the account, so the bar stopped short of the
end; a dry run reported "Would upload: 0"; the job's phases reached the app in English.
**Fix:** The last run's archives and summary are kept in `takeout-import.json` in the state
directory. A run still marked running at start-up is `interrupted`, and `ImportStatus` lists
its archives as `resumable`; `ResumeImport` (`pdfs import-google-photos --resume`, **Resume** in
the app) starts it again, and the duplicate check skips what is there. An unreadable archive is
listed in `unreadable_archives` and the rest imports. A failure keeps the counts and still files
albums. `ImportStatus` carries the phase and its own progress, the summary says `dry_run`,
`to_upload` and `error_kind`, and the app words them in the user's language. The inline
timeline refresh of an empty gallery waits for a refresh already running instead of starting a
second one (`refresh_timeline_joined`).
**Verified:** `takeout::tests` in `pdfs-fuse` (saved state, resumable runs) and `pdfs-core`
(unreadable archive), and the CLI tests for `--resume` and the exit status. Not run live.
---
### B287 — A photo draft left by an interrupted upload blocks that name
**Status:** Fixed, not verified live
**Found:** 2026-11-15, reading the SDK upload path for
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
**Where:** `upload_photo_from`, `upload_file_from_inner` and `abandon_file_draft` in
`proton-drive-rs` (SDK 0.12.0)
**Repro:** Interrupt a photo upload after its draft is created, sign in again, and upload a
photo with the same name.
**Cause.** Photo uploads replaced a leftover draft only when its `ClientUID` was the current
session's. A new login is a new session, so the next upload of that name failed with 2500. A
failed upload also never deleted its own draft, which the C# and TypeScript SDKs do.
**Fix:** SDK 0.12.0 adds `PhotoUploadMetadata::replace_existing_draft`, which the Takeout import
sets, and deletes the draft of a new file or photo whose upload fails. After a failed seal it
only does so on a definite rejection, because a seal lost in transit may have been applied.
**Verified:** SDK unit test `draft_cleanup_only_after_a_definite_seal_rejection`. Not run live.
---
### B286 — Resuming a Takeout import fails on a draft with no content hash
**Status:** Fixed, not verified live
**Found:** 2026-11-15, [issue #94](https://github.com/narrrl/proton-drive-linux/issues/94)
(3.7.0, Fedora). An import stopped by a restart failed every re-run at once with
`check for existing photos: serialization error: invalid type: null, expected a string at line
1 column 2063`.
**Where:** `FoundPhotoDuplicateDto` in `proton-drive-rs` (SDK 0.11.0)
**Repro:** Interrupt a photo upload after its draft is created, then run the import again.
**Cause.** The duplicates endpoint sends `ContentHash` (and can send `Hash`) as `null` for a
draft, as its API schema says. The SDK declared both as `String` with `#[serde(default)]`,
which covers a missing field but not `null`, so one draft failed the whole batch of 150 names
and the import ended before uploading anything.
**Fix:** SDK 0.12.0 makes both fields optional and skips drafts, trashed links and candidates
without hashes, like the TypeScript SDK. Every other defaulted DTO field reads `null` as its
default too, and an undecodable response is logged with its path.
**Verified:** SDK unit tests `duplicate_response_tolerates_null_fields_and_skips_drafts` and
`duplicate_response_tolerates_a_null_list`. Not run against the reporter's account.
---
### B285 — A restarted Secret Service leaves the process on a dead session
**Status:** Fixed (unverified)
**Found:** 2026-11-14, following up [issue #78](https://github.com/narrrl/proton-drive-linux/issues/78)
(signed out after every cold boot on Fedora 45, which replaced gnome-keyring with oo7). See
[the investigation](ISSUE-78.md#a-restarted-secret-service).
**Where:** `crates/pdfs-core/src/auth.rs`, `with_keyring_entry`
**Repro:** Write the session blob, restart the Secret Service provider
(`gnome-keyring-daemon --replace`, or `oo7-daemon --replace`), then read or write it again from
the same process. `dbus-secret-service-keyring-store` keeps one D-Bus connection and one Secret
Service session for the life of the process, and pdfs installed it once. The session exists only
in the provider's memory, so the new provider answers every `GetSecret` and `SetSecret` with
`org.freedesktop.Secret.Error.NoSession`, for as long as the process runs. A fresh store in the
same process reads the same entry fine. oo7's systemd unit restarts the daemon on failure, and
both providers can be replaced, so the daemon could meet this mid-session. A token refresh after
that point reached only memory and the kernel keyring: B61's retry kept asking on the dead
session, and the next cold boot read the spent refresh token (`InvalidRefreshToken`).
**Fix:** Every keyring call goes through `with_keyring_entry`. A platform failure (which is what
`NoSession` and other D-Bus errors become) logs "the Secret Service failed this process's
connection; reconnecting", drops the store unless another thread already replaced it, and makes
the call once more on a new connection and session. A locked keyring, a dismissed prompt and a
missing entry come back unchanged. Affects the daemon, the app and the tray alike.
**Verified:** `auth::tests::a_dropped_secret_service_session_is_reconnected_and_retried`,
`a_rotation_lands_at_once_after_the_keyring_restarts` and `a_locked_keyring_is_not_reconnected`
on the in-memory store. The ignored `auth::tests::real_secret_service_survives_a_provider_restart`
passes against gnome-keyring 51.1 and oo7 0.7.1 (built from its tag), each in a throwaway session
bus and restarted with `--replace`; it checks that a reconnect actually happened. A standalone
program holding one store, as before the fix, gets `NoSession` on read and write after the same
restart under both providers. Whether the reporter's oo7 restarts during a session is not known.
---
### B284 — A keyring entry reported missing discards refreshed session tokens
**Status:** Fixed (unverified)
**Found:** 2026-11-14, investigating [issue #78](https://github.com/narrrl/proton-drive-linux/issues/78)
after the reporter's oo7 0.7.0 retest ruled out the original persistence fix as a sufficient
explanation. See [the investigation and isolated reproducer](ISSUE-78.md).
**Where:** `crates/pdfs-core/src/auth.rs`, `TokenSink::flush`
**Repro:** Refresh a session while the keyring lookup reports `NoEntry` for its existing blob.
oo7 0.7.0 does this for a locked collection (upstream #614), reproduced with fake tokens on a
private D-Bus connection. pdfs dropped the pending rotation and returned success, leaving the
spent refresh token in the saved blob without triggering B61's retry or kernel recovery paths.
The running session can still work, but restarting it exposes the stale saved credentials.
**Fix:** Propagate the missing-session error so the rotation stays pending and is retried.
Persistence still requires the matching existing blob; a retry never recreates a deleted
session or overwrites a new login. This does not make a locked keyring writable, and pending
tokens still need to reach it before power-off to survive a cold boot.
**Verified:** The two regression tests
`auth::tests::a_temporarily_missing_session_does_not_discard_a_rotation` and
`auth::tests::retrying_a_missing_session_never_recreates_it_or_overwrites_a_new_login`
failed before the fix and pass afterward. The oo7 fixture confirms its locked lookup behavior
and repeated binary `SetSecret` persistence. The reporter's complete Fedora cold-boot sequence
has not been reproduced; issue #78's precise trigger remains unconfirmed.
---
### B283 — Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code
**Status:** Fixed, not verified live
+2 -58
View File
@@ -104,74 +104,18 @@ Existing folders stay device backups. Older builds cannot open the upgraded data
10 seconds, and `M` to mute. The volume carries over from one video to the next, and the
thumbnail strip in Photos stays off a video, where it covered the seek bar.
- **Album covers for songs.** A song in My files, in search and in Shared shows its cover as its
thumbnail, and the player shows it above the controls instead of a music note. The cover is the
picture in the song's tags (MP3, FLAC, Ogg, Opus, M4A, WAV, AIFF, WavPack, Monkey's Audio), or
else a `cover`, `folder`, `front` or `album` image (`.jpg`, `.png`, `.webp`) in the same folder.
Only the song's tags are downloaded for it, not the whole song. Songs uploaded from this computer
carry their cover as their thumbnail, so they show it at once on other devices too. A cover
image added to a folder later is picked up once the song changes or the service restarts.
- **Resume a Google Photos import** (B288). The archives and the outcome of the last import are
kept, so an import cut short by a restart or a reboot no longer has to be set up again. The
Import page offers **Resume**, and `pdfs import-google-photos --resume` does the same. Photos
that are already uploaded are skipped as before.
### Changed
- **`proton-sdk` and `proton-drive-rs` 0.12.0.** Timestamps are converted where they enter
- **`proton-sdk` and `proton-drive-rs` 0.11.0.** Timestamps are converted where they enter
from the SDK, and a photo keeps its own modification time now that it no longer follows the
capture time. 0.12.0 fixes the photo duplicate check and failed-upload drafts below.
[The 0.11.0 review](SDK-0.11.0.md) and [the 0.12.0 review](SDK-0.12.0.md) list the new
features and breaking changes.
- **The Google Photos import brings more of the export along** (B289).
- Places from Google Photos are kept, so imported photos show up under **Places**.
- Live photos and Pixel motion photos stay one photo: the video is filed with its still and
takes its date. The `.MP` half of a motion photo is no longer skipped.
- Only the Google Photos part of an export is imported. Files from Google Drive, YouTube and
other services in the same Takeout stay out of the timeline.
- A photo without a date of its own is dated by its file name and then by its EXIF before
the day it reached Google Photos.
- Favorites are also set on photos that were already in Proton Photos.
- **The import reports what it is doing** (B288). The Import page shows the current step
(reading, checking for photos you already have, uploading, sorting into albums), the counts
so far, and a progress bar that reaches the end. A scan says how many photos and how much
data an import would upload. `pdfs import-google-photos --wait` prints its progress, and
`pdfs import-status` shows the step.
capture time. [The SDK review](SDK-0.11.0.md) lists the new features and breaking changes.
### Fixed
- **Resuming a Google Photos import no longer fails at once** (#94, B286, B287). A photo upload
that was cut short left a draft behind, and every later run stopped with
`check for existing photos: serialization error: invalid type: null`. Such drafts are now
skipped by the duplicate check and replaced by the next upload of that photo, also after
signing in again. A failed upload removes its draft.
- **One bad archive or error no longer ends the whole import** (B288). An archive that cannot be
read is left out and named in the report, and the rest of the export imports. When an import
stops early, it keeps the counts of what it did and still files the uploaded photos into their
albums. A restart of the service while importing is reported as an interruption, not as a
finished import.
- **Photos zoomed all the way out shows the whole timeline.** The first 200 photos fit on one
screen there, and with nothing to scroll no more were loaded. More now load whenever the photos
don't fill the window, also after a zoom or a resize.
- **Signed out after a cold boot on Fedora 45** (issue #78, B284, B285). Fedora 45 replaced the
GNOME keyring with oo7, and two things there could keep a refreshed sign-in from being saved:
oo7 reports the entries of a locked keyring as missing, which the client took for a sign-out,
and a restarted oo7 no longer knew the client's connection, which then failed every save until
the service restarted. The refreshed sign-in lived only in memory, and the next boot found the
spent one. The client now keeps such a sign-in and saves it once the keyring can be read, and
reconnects to a keyring service that restarted.
- **Thumbnails appear in the folder you are looking at as soon as they are made.** During
**Build thumbnails**, the folder on screen kept its file icons until you left it and opened it
again. Its tiles now fill in while the build runs, and the same goes for any tile whose
thumbnail is made while it is on screen.
- **My files comes back after the service was killed for hanging** (B279). A killed service left
its folders behind as dead mounts, and every restart then failed with `File exists` and retried
forever without opening anything, until the mounts were cleared by hand. The service now clears
-1
View File
@@ -196,7 +196,6 @@ commands are for scripts and for cases where the mount is not available.
| `pdfs add-to-album UID PHOTOS...` | Add photos to an album. |
| `pdfs remove-from-album UID PHOTOS...` | Take photos out of an album. |
| `pdfs import-google-photos ARCHIVES... [--dry-run] [--wait]` | Import a Google Takeout export. Pass every `.zip` of the export. |
| `pdfs import-google-photos --resume [--wait]` | Start the last import again with the same archives after it was interrupted, stopped or failed. |
| `pdfs import-status` / `pdfs cancel-import` | Follow or stop a running import. |
| `pdfs redate-photos [--dry-run] [--from DATE] [--to DATE] [--limit N] [--wait]` | Fix photos filed under their upload day when the file name carries the real date. |
| `pdfs redate-status` / `pdfs cancel-redate` | Follow or stop a running re-date. |
-129
View File
@@ -1,129 +0,0 @@
# Issue #78: session loss on Fedora 45
Investigation on 2026-11-14 of [issue #78](https://github.com/narrrl/proton-drive-linux/issues/78),
using its report and comments through `gh`, pdfs 3.7.0/current source, and the oo7 0.7.0 tag.
## What is established
The reporter still saw the problem with oo7 0.7.0 on November 7. The original
[SetSecret persistence fix](https://github.com/linux-credentials/oo7/commit/7fb11dd111c4be8fec01996e3caa31c44fa962b3)
is present in that release. In an isolated test, three successive binary `SetSecret` updates
survived reopening the keyring file. Repeating the original persistence explanation is therefore
insufficient.
A different, reproducible failure exists at the boundary between oo7 and pdfs:
1. oo7 0.7.0's `Collection::search_inner_items` returns an empty list for a locked collection.
`SearchItems` consequently returns **both** the unlocked and locked lists empty even when the
matching item is on disk. The isolated test reproduces this with pdfs's exact lookup attributes.
This is upstream [oo7 #614](https://github.com/linux-credentials/oo7/issues/614).
2. `dbus-secret-service-keyring-store` 1.0.1 maps an empty search to `keyring_core::Error::NoEntry`;
pdfs maps that to `Error::NotLoggedIn`. The client cannot offer an unlock when no locked item
is reported.
3. In pdfs 3.7.0 and the code before this change, `TokenSink::flush` treated `NotLoggedIn` as a
logout, cleared the pending rotation and returned success. It skipped the error path that
retains the new tokens in the kernel keyring and retries their persistence. The code was
unchanged between v3.7.0 and this investigation.
4. If a token refresh encounters that reply, the saved blob retains a spent refresh token. The
running process can keep working with the new tokens in memory; a restart then exposes the
stale saved credentials. A regression test failed before the fix and passes afterward.
The same unsafe assumption also applies to transient empty lookups such as the race reported in
[oo7 #652](https://github.com/linux-credentials/oo7/issues/652), fixed in 0.7.1. That issue's
replacement race was not independently reproduced here; pdfs normally updates existing items
with `SetSecret`, so it must not be assumed to encounter the same-item `CreateItem` race.
## Change and limits
`TokenSink::flush` now propagates a missing-session error instead of discarding the rotation.
The existing retry, kernel-keyring recovery and unsaved-token status paths can then run. Once
the collection is unlocked and readable, the pending rotation is saved. A genuinely deleted
session is never recreated: flushing still requires a readable blob with the matching session
ID and expected refresh token. A new login causes the old rotation to be discarded as before.
This is a confirmed pdfs defect (B284), **not a confirmed complete explanation of the reporter's
cold-boot failure**. The issue has no oo7 logs or record showing that its collection was locked
when a token rotated. A locked keyring only at startup can hide a login, but does not by itself
explain `InvalidRefreshToken`. The token-loss path requires a successful refresh while the saved
entry is unreadable.
The fix does not make a locked keyring writable. If it stays unreadable until power-off, tokens
retained in process/kernel memory still cannot survive a cold boot. No new disk credential store
or credential format is introduced.
Other upstream Fedora-upgrade issues were checked: [oo7 #651](https://github.com/linux-credentials/oo7/issues/651)
creates duplicate placeholder/migrated collections and refuses writes; [oo7 #662](https://github.com/linux-credentials/oo7/issues/662)
can resolve the default alias to a locked collection. Both have fixes in
[0.7.1](https://github.com/linux-credentials/oo7/releases/tag/0.7.1), released November 10,
after the reporter's November 7 retest. Neither can be assigned as this report's cause without
matching runtime evidence.
## A restarted Secret Service
A second path to the same symptom, independent of locking (B285). The keyring store pdfs used
holds one D-Bus connection and one Secret Service session for the whole process, and was
installed once. The session lives only in the provider's memory. When the provider restarts,
the new one answers every `GetSecret` and `SetSecret` on the old session with
`org.freedesktop.Secret.Error.NoSession`, until the pdfs process itself restarts. Reproduced
with both gnome-keyring 51.1 and oo7 0.7.1 using `--replace`.
gnome-keyring practically never restarts within a desktop session. oo7 can:
`oo7-daemon.service` has `Restart=on-failure`, the daemon allows itself to be replaced, and it
exits on a wrong credential password and is D-Bus-activated again. A daemon that met a restarted
provider kept every later token refresh only in memory and the kernel keyring, and a cold boot
then left the spent refresh token in the saved blob.
Now a platform failure drops the store and the call is retried once on a new connection. The
daemon log shows `the Secret Service failed this process's connection; reconnecting` when that
happens. To check whether a machine meets this, look for oo7 restarts in the same boot:
```sh
journalctl --user -b -1 -u oo7-daemon.service | grep -E 'Started|Stopped|Starting oo7|Main process exited'
journalctl --user -b -1 -u proton-drive.service | grep -iE 'keyring|reconnecting|refresh'
```
Reproduce in a throwaway session bus (never against the desktop's keyring; the runtime directory
must be short, as it holds a Unix socket):
```sh
T=$(mktemp -d /tmp/ss.XXXX); mkdir -p $T/data $T/run; chmod 700 $T/run
XDG_DATA_HOME=$T/data XDG_RUNTIME_DIR=$T/run PDFS_SECRET_SERVICE_RESTART_TEST=1 \
PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets >/dev/null' \
dbus-run-session -- sh -c 'echo -n pw | gnome-keyring-daemon --unlock --components=secrets >/dev/null; sleep 1;
cargo test -p pdfs-core real_secret_service -- --ignored'
```
For oo7, start `oo7-daemon --login` with the password on stdin instead, and restart it with
`--login --replace` in the background.
## Reproduce without a Proton account or desktop keyring
The oo7 fixture uses fake tokens, a temporary disk keyring and a private peer-to-peer D-Bus
connection. It asserts the observed 0.7.0 behavior, including the incorrect empty locked lookup;
passing means that behavior was reproduced. It does not simulate Proton refresh or a full boot.
From the pdfs repository, choose an unused temporary checkout path:
```sh
git clone --depth 1 --branch 0.7.0 https://github.com/linux-credentials/oo7.git /tmp/pdfs-oo7-repro
git -C /tmp/pdfs-oo7-repro apply "$PWD/docs/repro/oo7-0.7.0-issue78.patch"
cargo test --manifest-path /tmp/pdfs-oo7-repro/Cargo.toml \
-p oo7-daemon --lib --features test-util issue78 -- --nocapture
```
Observed output:
```text
oo7 0.7.0: locked session is on disk but SearchItems returns (0, 0)
oo7 0.7.0: all three binary SetSecret updates survived reopening the disk keyring
test item::tests::issue78_locked_lookup_hides_saved_tokens ... ok
```
The pdfs side is covered by `cargo test -p pdfs-core auth::tests`, specifically
`a_temporarily_missing_session_does_not_discard_a_rotation` and
`retrying_a_missing_session_never_recreates_it_or_overwrites_a_new_login`.
Remaining live validation: reproduce on a Fedora 45 GNOME session across logout/shutdown,
correlating oo7 lock/write errors with pdfs refresh and retry logs, and checking the running
daemon's package version rather than just the CLI version. The investigation did not use a real
Proton account, alter a desktop keyring, or post to either GitHub issue.
+2 -17
View File
@@ -55,7 +55,6 @@ server learns which areas you look at.
2. In the app, open the **Upload** menu on the Photos page and choose **Import from Google
Photos…**. Select all the `.zip` files at once, because a photo and its details are often in
different parts of the export.
3. Choose **Scan** to see what an import would do without uploading anything, then **Import**.
From a terminal:
@@ -64,22 +63,8 @@ pdfs import-google-photos --dry-run ~/Downloads/takeout-*.zip # show what woul
pdfs import-google-photos --wait ~/Downloads/takeout-*.zip
```
What the import takes from the export:
- **Albums** are recreated. A photo that is in several albums is uploaded once and added to each.
- **Dates** come from the export's details, else from a date in the file name, else from the
photo's EXIF.
- **Places** come from the export, so imported photos appear under [Places](#places).
- **Favorites** are kept.
- **Live photos and motion photos** stay one photo: the video is filed with its still.
What it leaves out: photos in Google's trash, and files from other Google services (Drive,
YouTube) that are in the same export. A `.zip` that cannot be read is left out and named in the
report; the rest still imports.
Photos already in your library are skipped, so an interrupted import can simply be run again.
If the import was cut short, for example by a restart, the Import page offers **Resume**, and
`pdfs import-google-photos --resume` does the same without naming the archives again.
The import recreates your albums and takes capture dates and locations from the export. Photos
already in your library are skipped, so an interrupted import can simply be run again.
`pdfs import-status` shows how far an import is, and `pdfs cancel-import` stops it.
## Fixing photos filed under the wrong date
+1 -2
View File
@@ -63,8 +63,7 @@ them along.
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
| Photo name search | Not started | Photos search by date and filters only |
| Automatic photo upload from a folder | Not started | |
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos; motion-photo `.MP` parts are skipped |
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |
## Performance and correctness
View File
Whitespace-only changes.
-53
View File
@@ -1,53 +0,0 @@
diff --git a/server/src/item/tests.rs b/server/src/item/tests.rs
index 99c25dc..2ba705d 100644
--- a/server/src/item/tests.rs
+++ b/server/src/item/tests.rs
@@ -5,6 +5,48 @@ use tokio_stream::StreamExt;
use crate::{service::PrompterType, tests::TestServiceSetup};
+// Investigation for proton-drive-linux #78. Fake tokens and a private p2p bus;
+// no desktop keyring or Proton account is used.
+#[tokio::test]
+async fn issue78_locked_lookup_hides_saved_tokens() -> Result<(), Box<dyn std::error::Error>> {
+ let setup =
+ TestServiceSetup::with_login_keyring(oo7::Secret::from("test-password-long-enough"))
+ .await?;
+ let collection = setup.default_collection().await?;
+ let attributes = [("service", "proton-drive-linux"), ("username", "session")];
+ let secret = setup.create_dbus_secret(oo7::Secret::blob(b"fake-refresh-r1"))?;
+ collection
+ .create_item("pdfs fake session", &attributes, &secret, false, None)
+ .await?;
+
+ let (unlocked, locked) = setup.service_api.search_items(&attributes).await?;
+ assert_eq!((unlocked.len(), locked.len()), (1, 0));
+ setup.lock_collection(collection).await?;
+ assert!(collection.is_locked().await?);
+
+ let (unlocked, locked) = setup.service_api.search_items(&attributes).await?;
+ // This is the problematic reply, not the Secret Service behavior expected
+ // by keyring-store: the saved item is neither unlocked nor reported locked.
+ assert_eq!((unlocked.len(), locked.len()), (0, 0));
+ assert_eq!(setup.login_keyring().await?.items().await?.len(), 1);
+ eprintln!("oo7 0.7.0: locked session is on disk but SearchItems returns (0, 0)");
+
+ setup.unlock_collection(collection).await?;
+ let (unlocked, locked) = setup.service_api.search_items(&attributes).await?;
+ assert_eq!((unlocked.len(), locked.len()), (1, 0));
+
+ // Exercise the binary SetSecret path used by pdfs's keyring-store. Reopen
+ // the disk keyring after each update, rather than trusting a memory read.
+ for refresh in [b"fake-refresh-r2", b"fake-refresh-r3", b"fake-refresh-r4"] {
+ let secret = setup.create_dbus_secret(oo7::Secret::blob(refresh))?;
+ unlocked[0].set_secret(&secret).await?;
+ let items = setup.login_keyring().await?.items().await?;
+ assert_eq!(items[0].secret().as_bytes(), refresh);
+ }
+ eprintln!("oo7 0.7.0: all three binary SetSecret updates survived reopening the disk keyring");
+ Ok(())
+}
+
#[tokio::test]
async fn label_property() -> Result<(), Box<dyn std::error::Error>> {
let setup = TestServiceSetup::plain_session(true).await?;
+218 -326
View File
File diff suppressed because it is too large. Load diff
+218 -326
View File
File diff suppressed because it is too large. Load diff
+218 -326
View File
File diff suppressed because it is too large. Load diff
+218 -326
View File
File diff suppressed because it is too large. Load diff
+218 -326
View File
File diff suppressed because it is too large. Load diff
+218 -326
View File
File diff suppressed because it is too large. Load diff
+213 -306
View File
File diff suppressed because it is too large. Load diff
+219 -328
View File
File diff suppressed because it is too large. Load diff
+218 -326
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -91,7 +91,7 @@ SECTIONS = [
]
# Too long or too internal for the wiki; links to them go to GitHub.
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md", "SDK-0.12.0.md"}
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "SDK-0.11.0.md"}
# Files outside docs/ whose links are checked but not published.
CHECKED = ["README.md", "CONTRIBUTING.md", "SECURITY.md"]