mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-15 17:41:59 +01:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1dcf7c84ca | ||
|
|
a05772aa13 | ||
|
|
2f2fe96cdf | ||
|
|
2ec3b7ff62 | ||
|
|
38bd282d3a | ||
|
|
d773ba15d3 | ||
|
|
d93982baa9 | ||
|
|
3f5001b2d2 | ||
|
|
2308021aa2 |
No files matched your search
Generated
+47
-4
@@ -1114,6 +1114,12 @@ dependencies = [
|
|||||||
"syn 2.0.119",
|
"syn 2.0.119",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "data-encoding"
|
||||||
|
version = "2.11.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dbl"
|
name = "dbl"
|
||||||
version = "0.3.2"
|
version = "0.3.2"
|
||||||
@@ -2902,6 +2908,32 @@ dependencies = [
|
|||||||
"scopeguard",
|
"scopeguard",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lofty"
|
||||||
|
version = "0.25.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "35d45c446382e04663705aa8e786157f64ef1fa85755d410121ffbf08f54b0db"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"data-encoding",
|
||||||
|
"flate2",
|
||||||
|
"lofty_attr",
|
||||||
|
"log",
|
||||||
|
"ogg_pager",
|
||||||
|
"paste",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lofty_attr"
|
||||||
|
version = "0.13.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fdbca1b60ba7cea959ffc84ac27f3dbdbacd6dfda168c7eabbc00e48654bb9fd"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 3.0.9",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "log"
|
name = "log"
|
||||||
version = "0.4.35"
|
version = "0.4.35"
|
||||||
@@ -3323,6 +3355,15 @@ dependencies = [
|
|||||||
"subtle",
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ogg_pager"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "166611b491f939446e64c6be930843a31eb8e6798204ec3e9f57e084c18e7446"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.21.4"
|
version = "1.21.4"
|
||||||
@@ -3594,10 +3635,12 @@ dependencies = [
|
|||||||
"futures",
|
"futures",
|
||||||
"image",
|
"image",
|
||||||
"libc",
|
"libc",
|
||||||
|
"lofty",
|
||||||
"notify",
|
"notify",
|
||||||
"parking_lot 0.13.0",
|
"parking_lot 0.13.0",
|
||||||
"pdfs-core",
|
"pdfs-core",
|
||||||
"proton-drive-rs",
|
"proton-drive-rs",
|
||||||
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha1 0.11.0",
|
"sha1 0.11.0",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -3848,9 +3891,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proton-drive-rs"
|
name = "proton-drive-rs"
|
||||||
version = "0.11.0"
|
version = "0.12.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "14a22e621abbc1c61b4640728a7de969c16b66d5d983b59c638d501fda25a7b9"
|
checksum = "4265c2cf3bdc59ee273d7363c8b6a7cb4e98be6102ffa18144c4d8ebda00737b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64 0.23.1",
|
"base64 0.23.1",
|
||||||
@@ -3876,9 +3919,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proton-sdk"
|
name = "proton-sdk"
|
||||||
version = "0.11.0"
|
version = "0.12.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bd55a5e8b6600f3aa282ebbdf13ff366431ff9aae84d4eb20b0ef06d598919fe"
|
checksum = "fb98ab3e336ed95f82368a50348dad00e5fc919e982d748e83e844afcadbd5d8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm 0.11.1",
|
"aes-gcm 0.11.1",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
|||||||
+6
-2
@@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
|
|||||||
categories = ["filesystem", "command-line-utilities"]
|
categories = ["filesystem", "command-line-utilities"]
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
proton-sdk = { version = "0.11.0" }
|
proton-sdk = { version = "0.12.0" }
|
||||||
proton-drive-rs = { version = "0.11.0" }
|
proton-drive-rs = { version = "0.12.0" }
|
||||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||||
|
|
||||||
# Internal crates.
|
# Internal crates.
|
||||||
@@ -50,6 +50,10 @@ sha1 = "0.11"
|
|||||||
# formats a camera or phone actually produces — the rest of the crate's codecs
|
# formats a camera or phone actually produces — the rest of the crate's codecs
|
||||||
# would be dead weight in the daemon.
|
# would be dead weight in the daemon.
|
||||||
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "tiff", "bmp", "rayon"] }
|
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "tiff", "bmp", "rayon"] }
|
||||||
|
# Cover pictures of songs, read from their tags (ID3, FLAC, Ogg, MP4, APE) for
|
||||||
|
# the song's thumbnail. Its default feature is required: 0.25 does not build
|
||||||
|
# without it.
|
||||||
|
lofty = "0.25"
|
||||||
rusqlite = { version = "0.41", features = ["bundled"] }
|
rusqlite = { version = "0.41", features = ["bundled"] }
|
||||||
# keyring 4 split into `keyring-core` plus per-platform stores. The libdbus
|
# keyring 4 split into `keyring-core` plus per-platform stores. The libdbus
|
||||||
# Secret Service store is the one keyring 3's `sync-secret-service` wrapped, and
|
# Secret Service store is the one keyring 3's `sync-secret-service` wrapped, and
|
||||||
|
|||||||
+180
-38
@@ -228,14 +228,19 @@ enum Command {
|
|||||||
/// numbered parts, and a photo's metadata often sits in a different part
|
/// numbered parts, and a photo's metadata often sits in a different part
|
||||||
/// than the photo. Albums in the export are recreated as Proton albums, and
|
/// than the photo. Albums in the export are recreated as Proton albums, and
|
||||||
/// photos already on the account (same name and same content) are skipped,
|
/// photos already on the account (same name and same content) are skipped,
|
||||||
/// so an interrupted import can simply be run again.
|
/// so an interrupted import can simply be run again, or resumed with
|
||||||
|
/// `--resume` without naming the archives again.
|
||||||
ImportGooglePhotos {
|
ImportGooglePhotos {
|
||||||
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
||||||
#[arg(required = true, value_hint = ValueHint::FilePath)]
|
#[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)]
|
||||||
archives: Vec<PathBuf>,
|
archives: Vec<PathBuf>,
|
||||||
/// Report what would be imported without uploading anything.
|
/// Report what would be imported without uploading anything.
|
||||||
#[arg(long)]
|
#[arg(long, conflicts_with = "resume")]
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
|
/// Start the last import again with the same archives, after it was
|
||||||
|
/// interrupted, stopped or failed.
|
||||||
|
#[arg(long, conflicts_with = "archives")]
|
||||||
|
resume: bool,
|
||||||
/// Stay attached and print progress until the import finishes.
|
/// Stay attached and print progress until the import finishes.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
wait: bool,
|
wait: bool,
|
||||||
@@ -1030,8 +1035,9 @@ fn main() -> Result<()> {
|
|||||||
Command::ImportGooglePhotos {
|
Command::ImportGooglePhotos {
|
||||||
archives,
|
archives,
|
||||||
dry_run,
|
dry_run,
|
||||||
|
resume,
|
||||||
wait,
|
wait,
|
||||||
} => cmd_import_google_photos(archives, dry_run, wait),
|
} => cmd_import_google_photos(archives, dry_run, resume, wait),
|
||||||
Command::RedatePhotos {
|
Command::RedatePhotos {
|
||||||
dry_run,
|
dry_run,
|
||||||
limit,
|
limit,
|
||||||
@@ -2916,24 +2922,34 @@ fn photo_count(n: usize) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
|
fn cmd_import_google_photos(
|
||||||
// The daemon opens the archives itself off the shared filesystem, so it gets
|
archives: Vec<PathBuf>,
|
||||||
// absolute paths — the same arrangement `pdfs upload` uses.
|
dry_run: bool,
|
||||||
let mut absolute = Vec::with_capacity(archives.len());
|
resume: bool,
|
||||||
for archive in &archives {
|
wait: bool,
|
||||||
let path = std::fs::canonicalize(archive)
|
) -> Result<()> {
|
||||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
let request = if resume {
|
||||||
absolute.push(
|
CtlRequest::ResumeImport
|
||||||
path.to_str()
|
} else {
|
||||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
// The daemon opens the archives itself off the shared filesystem, so it
|
||||||
.to_owned(),
|
// gets absolute paths — the same arrangement `pdfs upload` uses.
|
||||||
);
|
let mut absolute = Vec::with_capacity(archives.len());
|
||||||
}
|
for archive in &archives {
|
||||||
|
let path = std::fs::canonicalize(archive)
|
||||||
|
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||||
|
absolute.push(
|
||||||
|
path.to_str()
|
||||||
|
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||||
|
.to_owned(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
CtlRequest::ImportTakeout {
|
||||||
|
archives: absolute,
|
||||||
|
dry_run,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
match control_request(CtlRequest::ImportTakeout {
|
match control_request(request)? {
|
||||||
archives: absolute,
|
|
||||||
dry_run,
|
|
||||||
})? {
|
|
||||||
CtlResponse::Ok { message } => println!("{message}"),
|
CtlResponse::Ok { message } => println!("{message}"),
|
||||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||||
other => bail!("unexpected response: {other:?}"),
|
other => bail!("unexpected response: {other:?}"),
|
||||||
@@ -2945,20 +2961,34 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
|||||||
|
|
||||||
// Poll rather than stream: the control protocol is request/response, and an
|
// Poll rather than stream: the control protocol is request/response, and an
|
||||||
// import runs for hours — a held-open socket would be the only long-lived
|
// import runs for hours — a held-open socket would be the only long-lived
|
||||||
// connection the daemon has.
|
// connection the daemon has. A line is printed when the step changes and
|
||||||
|
// every few percent of it, not on every poll.
|
||||||
|
let mut last_line = String::new();
|
||||||
|
let mut last_printed = std::time::Instant::now();
|
||||||
loop {
|
loop {
|
||||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||||
match control_request(CtlRequest::ImportStatus)? {
|
match control_request(CtlRequest::ImportStatus)? {
|
||||||
CtlResponse::ImportStatus {
|
CtlResponse::ImportStatus {
|
||||||
running: true,
|
running: true,
|
||||||
summary: _,
|
progress,
|
||||||
} => continue,
|
..
|
||||||
CtlResponse::ImportStatus { summary, .. } => {
|
} => {
|
||||||
print_import_summary(summary.as_ref());
|
let line = progress.as_ref().map(progress_line).unwrap_or_default();
|
||||||
return finished_run(
|
let step_changed = line.split(':').next() != last_line.split(':').next();
|
||||||
"the import",
|
if !line.is_empty()
|
||||||
summary.map(|s| (s.failed, s.cancelled, s.error)),
|
&& line != last_line
|
||||||
);
|
&& (step_changed || last_printed.elapsed().as_secs() >= 30)
|
||||||
|
{
|
||||||
|
println!("{line}");
|
||||||
|
last_line = line;
|
||||||
|
last_printed = std::time::Instant::now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CtlResponse::ImportStatus {
|
||||||
|
summary, resumable, ..
|
||||||
|
} => {
|
||||||
|
print_import_summary(summary.as_ref(), &resumable);
|
||||||
|
return import_outcome(summary.as_ref());
|
||||||
}
|
}
|
||||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||||
other => bail!("unexpected response: {other:?}"),
|
other => bail!("unexpected response: {other:?}"),
|
||||||
@@ -2966,6 +2996,47 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One line saying where a running import is.
|
||||||
|
fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String {
|
||||||
|
use pdfs_core::control::ImportPhase;
|
||||||
|
let step = match progress.phase {
|
||||||
|
ImportPhase::Scanning => "Reading archives",
|
||||||
|
ImportPhase::Checking => "Checking for photos already on Proton",
|
||||||
|
ImportPhase::Uploading => "Uploading",
|
||||||
|
ImportPhase::Albums => "Filing photos into albums",
|
||||||
|
};
|
||||||
|
match (progress.total, &progress.current) {
|
||||||
|
(0, _) => format!("{step}…"),
|
||||||
|
(total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done),
|
||||||
|
(total, None) => format!("{step}: {} of {total}", progress.done),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How an import `--wait` followed ended, as the exit status: an error unless
|
||||||
|
/// it did everything it set out to do, so a script can tell.
|
||||||
|
fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> {
|
||||||
|
let Some(summary) = summary else {
|
||||||
|
bail!("the import ended without a result; the service may have restarted");
|
||||||
|
};
|
||||||
|
if let Some(error) = &summary.error {
|
||||||
|
bail!("the import failed: {error}");
|
||||||
|
}
|
||||||
|
if summary.interrupted {
|
||||||
|
bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue");
|
||||||
|
}
|
||||||
|
if summary.cancelled {
|
||||||
|
bail!("the import was cancelled");
|
||||||
|
}
|
||||||
|
if summary.failed > 0 || !summary.unreadable_archives.is_empty() {
|
||||||
|
bail!(
|
||||||
|
"the import finished, but {} photo(s) and {} archive(s) failed",
|
||||||
|
summary.failed,
|
||||||
|
summary.unreadable_archives.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn cmd_redate_photos(
|
fn cmd_redate_photos(
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
limit: Option<usize>,
|
limit: Option<usize>,
|
||||||
@@ -3100,11 +3171,19 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) {
|
|||||||
|
|
||||||
fn cmd_import_status() -> Result<()> {
|
fn cmd_import_status() -> Result<()> {
|
||||||
match control_request(CtlRequest::ImportStatus)? {
|
match control_request(CtlRequest::ImportStatus)? {
|
||||||
CtlResponse::ImportStatus { running, summary } => {
|
CtlResponse::ImportStatus {
|
||||||
|
running,
|
||||||
|
summary,
|
||||||
|
progress,
|
||||||
|
resumable,
|
||||||
|
} => {
|
||||||
if running {
|
if running {
|
||||||
println!("Import running. `pdfs transfers` shows its progress.");
|
match progress.as_ref() {
|
||||||
|
Some(progress) => println!("Import running. {}", progress_line(progress)),
|
||||||
|
None => println!("Import running."),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
print_import_summary(summary.as_ref());
|
print_import_summary(summary.as_ref(), &resumable);
|
||||||
}
|
}
|
||||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||||
other => bail!("unexpected response: {other:?}"),
|
other => bail!("unexpected response: {other:?}"),
|
||||||
@@ -3121,18 +3200,28 @@ fn cmd_cancel_import() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) {
|
||||||
let Some(summary) = summary else {
|
let Some(summary) = summary else {
|
||||||
println!("No import has run yet.");
|
println!("No import has run yet.");
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
if summary.dry_run {
|
||||||
|
println!("Scan only; nothing was uploaded.");
|
||||||
|
}
|
||||||
if let Some(error) = &summary.error {
|
if let Some(error) = &summary.error {
|
||||||
println!("Failed : {error}");
|
println!("Stopped by : {error}");
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
println!("Photos in export : {}", summary.found);
|
println!("Photos in export : {}", summary.found);
|
||||||
println!("Uploaded : {}", summary.uploaded);
|
if summary.dry_run {
|
||||||
|
println!("Would upload : {}", summary.to_upload);
|
||||||
|
println!("Bytes to upload : {}", summary.to_upload_bytes);
|
||||||
|
} else {
|
||||||
|
println!("Uploaded : {}", summary.uploaded);
|
||||||
|
}
|
||||||
println!("Already on Proton: {}", summary.duplicates);
|
println!("Already on Proton: {}", summary.duplicates);
|
||||||
|
if summary.live_photos > 0 {
|
||||||
|
println!("Live photos : {}", summary.live_photos);
|
||||||
|
}
|
||||||
if summary.skipped_trashed > 0 {
|
if summary.skipped_trashed > 0 {
|
||||||
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
||||||
}
|
}
|
||||||
@@ -3145,12 +3234,20 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
|||||||
if summary.failed > 0 {
|
if summary.failed > 0 {
|
||||||
println!("Failed : {}", summary.failed);
|
println!("Failed : {}", summary.failed);
|
||||||
}
|
}
|
||||||
|
for archive in &summary.unreadable_archives {
|
||||||
|
println!("Unreadable : {archive}");
|
||||||
|
}
|
||||||
if summary.bytes > 0 {
|
if summary.bytes > 0 {
|
||||||
println!("Uploaded bytes : {}", summary.bytes);
|
println!("Uploaded bytes : {}", summary.bytes);
|
||||||
}
|
}
|
||||||
if summary.cancelled {
|
if summary.interrupted {
|
||||||
|
println!("The service stopped before the import finished.");
|
||||||
|
} else if summary.cancelled {
|
||||||
println!("Import was cancelled before it finished.");
|
println!("Import was cancelled before it finished.");
|
||||||
}
|
}
|
||||||
|
if !resumable.is_empty() {
|
||||||
|
println!("Run `pdfs import-google-photos --resume` to continue it.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
||||||
@@ -4509,6 +4606,51 @@ mod login_wait_tests {
|
|||||||
assert!(finished_run("the import", None).is_err());
|
assert!(finished_run("the import", None).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_waited_for_import_that_did_not_finish_exits_with_an_error() {
|
||||||
|
use pdfs_core::control::ImportSummary;
|
||||||
|
let done = ImportSummary {
|
||||||
|
uploaded: 4,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(import_outcome(Some(&done)).is_ok());
|
||||||
|
assert!(import_outcome(None).is_err());
|
||||||
|
for broken in [
|
||||||
|
ImportSummary {
|
||||||
|
interrupted: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
ImportSummary {
|
||||||
|
unreadable_archives: vec!["/exports/takeout-002.zip".into()],
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
ImportSummary {
|
||||||
|
error: Some("check for existing photos: offline".into()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}");
|
||||||
|
}
|
||||||
|
let interrupted = import_outcome(Some(&ImportSummary {
|
||||||
|
interrupted: true,
|
||||||
|
..Default::default()
|
||||||
|
}));
|
||||||
|
assert!(interrupted.unwrap_err().to_string().contains("--resume"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resume_needs_no_archives_and_excludes_them() {
|
||||||
|
let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
cli.command,
|
||||||
|
Command::ImportGooglePhotos { resume: true, .. }
|
||||||
|
));
|
||||||
|
assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err());
|
||||||
|
assert!(
|
||||||
|
Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn waiting_for_a_session_stops_hammering_the_keyring() {
|
fn waiting_for_a_session_stops_hammering_the_keyring() {
|
||||||
let mut checks_left = 20;
|
let mut checks_left = 20;
|
||||||
|
|||||||
+214
-29
@@ -123,13 +123,84 @@ fn client_config() -> ProtonClientConfiguration {
|
|||||||
ProtonClientConfiguration::new(app_version).with_user_agent(user_agent)
|
ProtonClientConfiguration::new(app_version).with_user_agent(user_agent)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn keyring_entry() -> Result<Entry> {
|
/// Run `op` on the session's keyring entry.
|
||||||
// Installed lazily rather than once at startup so a session bus that was not
|
fn with_keyring<T>(op: impl Fn(&Entry) -> keyring_core::Result<T>) -> Result<T> {
|
||||||
// up yet on the first attempt is retried on the next one.
|
with_keyring_entry(KEYRING_SERVICE, KEYRING_USER, op)
|
||||||
if keyring_core::get_default_store().is_none() {
|
}
|
||||||
keyring_core::set_default_store(dbus_secret_service_keyring_store::Store::new()?);
|
|
||||||
|
/// Run `op` on the entry `service`/`user`, reconnecting once if the Secret
|
||||||
|
/// Service fails the call.
|
||||||
|
///
|
||||||
|
/// The store holds one D-Bus connection and one Secret Service session for the
|
||||||
|
/// whole process. The session lives in the provider's memory: once the provider
|
||||||
|
/// restarts (oo7's unit restarts it on failure, and both oo7 and gnome-keyring
|
||||||
|
/// can be `--replace`d), every read and write on the old session fails with
|
||||||
|
/// `NoSession`, for as long as this process runs. A daemon then keeps refreshed
|
||||||
|
/// tokens only in memory, and a cold boot loses them (`docs/BUGS.md` B285). A
|
||||||
|
/// platform failure therefore drops the store, and the call is made once more
|
||||||
|
/// on a new one. A locked keyring, a dismissed prompt or a missing entry is
|
||||||
|
/// not the connection's fault and comes back as it is.
|
||||||
|
fn with_keyring_entry<T>(
|
||||||
|
service: &str,
|
||||||
|
user: &str,
|
||||||
|
op: impl Fn(&Entry) -> keyring_core::Result<T>,
|
||||||
|
) -> Result<T> {
|
||||||
|
let store = keyring_store()?;
|
||||||
|
match op(&Entry::new(service, user)?) {
|
||||||
|
Err(keyring_core::Error::PlatformFailure(e)) => {
|
||||||
|
tracing::warn!(
|
||||||
|
error = %e,
|
||||||
|
"the Secret Service failed this process's connection; reconnecting"
|
||||||
|
);
|
||||||
|
drop_keyring_store(&store);
|
||||||
|
keyring_store()?;
|
||||||
|
Ok(op(&Entry::new(service, user)?)?)
|
||||||
|
}
|
||||||
|
result => Ok(result?),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Held while the default store is looked at and replaced, so that threads
|
||||||
|
/// failing at the same time drop the broken store once and share its
|
||||||
|
/// replacement.
|
||||||
|
static KEYRING_STORE: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||||
|
|
||||||
|
/// The process's Secret Service store, connected now if there is none.
|
||||||
|
///
|
||||||
|
/// Connected lazily rather than once at startup so a session bus that was not
|
||||||
|
/// up yet on the first attempt is tried again on the next one.
|
||||||
|
fn keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||||
|
let _swap = KEYRING_STORE.lock();
|
||||||
|
if let Some(store) = keyring_core::get_default_store() {
|
||||||
|
return Ok(store);
|
||||||
|
}
|
||||||
|
let store = connect_keyring_store()?;
|
||||||
|
keyring_core::set_default_store(Arc::clone(&store));
|
||||||
|
Ok(store)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Forget `failed`, unless another thread has already replaced it.
|
||||||
|
fn drop_keyring_store(failed: &Arc<keyring_core::CredentialStore>) {
|
||||||
|
let _swap = KEYRING_STORE.lock();
|
||||||
|
if keyring_core::get_default_store().is_some_and(|store| Arc::ptr_eq(&store, failed)) {
|
||||||
|
keyring_core::unset_default_store();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||||
|
Ok(dbus_secret_service_keyring_store::Store::new()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The tests' in-memory store when they set one, so a reconnect finds the
|
||||||
|
/// same entries; the real Secret Service otherwise.
|
||||||
|
#[cfg(test)]
|
||||||
|
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||||
|
tests::KEYRING_CONNECTS.fetch_add(1, Ordering::SeqCst);
|
||||||
|
match tests::MOCK_STORE.lock().clone() {
|
||||||
|
Some(store) => Ok(store),
|
||||||
|
None => Ok(dbus_secret_service_keyring_store::Store::new()?),
|
||||||
}
|
}
|
||||||
Ok(Entry::new(KEYRING_SERVICE, KEYRING_USER)?)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What the user answered when asked for the mailbox password.
|
/// What the user answered when asked for the mailbox password.
|
||||||
@@ -421,17 +492,15 @@ fn classify_verification_gate(e: ProtonError) -> Error {
|
|||||||
|
|
||||||
fn write_stored(stored: &StoredSession) -> Result<()> {
|
fn write_stored(stored: &StoredSession) -> Result<()> {
|
||||||
let json = Zeroizing::new(serde_json::to_string(stored)?);
|
let json = Zeroizing::new(serde_json::to_string(stored)?);
|
||||||
keyring_entry()?.set_password(&json)?;
|
with_keyring(|entry| entry.set_password(&json))
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load the persisted session blob, or `Error::NotLoggedIn` if absent.
|
/// Load the persisted session blob, or `Error::NotLoggedIn` if absent.
|
||||||
pub fn load() -> Result<StoredSession> {
|
pub fn load() -> Result<StoredSession> {
|
||||||
let entry = keyring_entry()?;
|
match with_keyring(|entry| entry.get_password()) {
|
||||||
match entry.get_password() {
|
|
||||||
Ok(json) => Ok(serde_json::from_str(&json)?),
|
Ok(json) => Ok(serde_json::from_str(&json)?),
|
||||||
Err(keyring_core::Error::NoEntry) => Err(Error::NotLoggedIn),
|
Err(Error::Keyring(keyring_core::Error::NoEntry)) => Err(Error::NotLoggedIn),
|
||||||
Err(e) => Err(e.into()),
|
Err(e) => Err(e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -589,9 +658,9 @@ pub fn logout() -> Result<bool> {
|
|||||||
tracing::warn!(error = %e, "clearing the kernel-keyring unlock on logout failed");
|
tracing::warn!(error = %e, "clearing the kernel-keyring unlock on logout failed");
|
||||||
}
|
}
|
||||||
let _ = kernelkey::UNSAVED_TOKENS.clear();
|
let _ = kernelkey::UNSAVED_TOKENS.clear();
|
||||||
match keyring_entry()?.delete_credential() {
|
match with_keyring(|entry| entry.delete_credential()) {
|
||||||
Ok(()) | Err(keyring_core::Error::NoEntry) => Ok(ended),
|
Ok(()) | Err(Error::Keyring(keyring_core::Error::NoEntry)) => Ok(ended),
|
||||||
Err(e) => Err(e.into()),
|
Err(e) => Err(e),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1207,12 +1276,138 @@ mod tests {
|
|||||||
/// The keyring is one store per process, so the tests that use it,
|
/// The keyring is one store per process, so the tests that use it,
|
||||||
/// through `keyring-core`'s in-memory store, take turns.
|
/// through `keyring-core`'s in-memory store, take turns.
|
||||||
fn mock_keyring() -> parking_lot::MutexGuard<'static, ()> {
|
fn mock_keyring() -> parking_lot::MutexGuard<'static, ()> {
|
||||||
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
|
||||||
let turn = KEYRING.lock();
|
let turn = KEYRING.lock();
|
||||||
keyring_core::set_default_store(keyring_core::mock::Store::new().unwrap());
|
let store: Arc<keyring_core::CredentialStore> = keyring_core::mock::Store::new().unwrap();
|
||||||
|
*MOCK_STORE.lock() = Some(Arc::clone(&store));
|
||||||
|
keyring_core::set_default_store(store);
|
||||||
turn
|
turn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||||
|
|
||||||
|
/// The in-memory store a reconnect hands out, so it finds the same
|
||||||
|
/// entries; `None` connects to the real Secret Service.
|
||||||
|
pub(super) static MOCK_STORE: parking_lot::Mutex<Option<Arc<keyring_core::CredentialStore>>> =
|
||||||
|
parking_lot::Mutex::new(None);
|
||||||
|
|
||||||
|
/// How many times a store was connected.
|
||||||
|
pub(super) static KEYRING_CONNECTS: std::sync::atomic::AtomicUsize =
|
||||||
|
std::sync::atomic::AtomicUsize::new(0);
|
||||||
|
|
||||||
|
/// The session's entry, to make its next call fail.
|
||||||
|
fn keyring_entry() -> Entry {
|
||||||
|
Entry::new(KEYRING_SERVICE, KEYRING_USER).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Make the session entry's next call fail with `error`.
|
||||||
|
fn fail_next(error: keyring_core::Error) {
|
||||||
|
keyring_entry()
|
||||||
|
.as_any()
|
||||||
|
.downcast_ref::<keyring_core::mock::Cred>()
|
||||||
|
.unwrap()
|
||||||
|
.set_error(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn no_session() -> keyring_core::Error {
|
||||||
|
keyring_core::Error::PlatformFailure(
|
||||||
|
std::io::Error::other("The session does not exist (NoSession)").into(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// B285: a restarted Secret Service no longer knows the process's
|
||||||
|
/// session, and every call on it failed until the process restarted.
|
||||||
|
#[test]
|
||||||
|
fn a_dropped_secret_service_session_is_reconnected_and_retried() {
|
||||||
|
let _turn = mock_keyring();
|
||||||
|
write_stored(&stored_with("r")).unwrap();
|
||||||
|
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||||
|
|
||||||
|
fail_next(no_session());
|
||||||
|
assert_eq!(load().unwrap().refresh_token, "r");
|
||||||
|
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 1);
|
||||||
|
|
||||||
|
fail_next(no_session());
|
||||||
|
write_stored(&stored_with("r2")).unwrap();
|
||||||
|
assert_eq!(load().unwrap().refresh_token, "r2");
|
||||||
|
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_rotation_lands_at_once_after_the_keyring_restarts() {
|
||||||
|
let _turn = mock_keyring();
|
||||||
|
write_stored(&stored_with("r")).unwrap();
|
||||||
|
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||||
|
|
||||||
|
fail_next(no_session());
|
||||||
|
sink.rotated(tokens("a2", "r2")).unwrap();
|
||||||
|
assert_eq!(load().unwrap().refresh_token, "r2");
|
||||||
|
assert!(sink.state.lock().unsaved.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A locked keyring or a dismissed prompt is no fault of the connection.
|
||||||
|
#[test]
|
||||||
|
fn a_locked_keyring_is_not_reconnected() {
|
||||||
|
let _turn = mock_keyring();
|
||||||
|
write_stored(&stored_with("r")).unwrap();
|
||||||
|
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||||
|
|
||||||
|
fail_next(keyring_core::Error::NoStorageAccess(
|
||||||
|
std::io::Error::other("locked").into(),
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
load(),
|
||||||
|
Err(Error::Keyring(keyring_core::Error::NoStorageAccess(_)))
|
||||||
|
));
|
||||||
|
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects);
|
||||||
|
assert_eq!(load().unwrap().refresh_token, "r");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The real Secret Service, restarted between two calls on one store.
|
||||||
|
///
|
||||||
|
/// Run inside a throwaway session bus, never against the desktop's
|
||||||
|
/// keyring. With gnome-keyring:
|
||||||
|
///
|
||||||
|
/// ```sh
|
||||||
|
/// echo -n pw | gnome-keyring-daemon --unlock --components=secrets
|
||||||
|
/// PDFS_SECRET_SERVICE_RESTART_TEST=1 \
|
||||||
|
/// PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets' \
|
||||||
|
/// cargo test -p pdfs-core real_secret_service -- --ignored
|
||||||
|
/// ```
|
||||||
|
#[test]
|
||||||
|
#[ignore = "needs a throwaway Secret Service; see the doc comment"]
|
||||||
|
fn real_secret_service_survives_a_provider_restart() {
|
||||||
|
if std::env::var_os("PDFS_SECRET_SERVICE_RESTART_TEST").is_none() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let restart = std::env::var("PDFS_RESTART_PROVIDER").expect("PDFS_RESTART_PROVIDER");
|
||||||
|
let _turn = KEYRING.lock();
|
||||||
|
*MOCK_STORE.lock() = None;
|
||||||
|
keyring_core::unset_default_store();
|
||||||
|
let (service, user) = ("pdfs-test", "restart");
|
||||||
|
|
||||||
|
with_keyring_entry(service, user, |e| e.set_password("before")).unwrap();
|
||||||
|
let status = std::process::Command::new("sh")
|
||||||
|
.args(["-c", &restart])
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert!(status.success(), "restarting the provider failed");
|
||||||
|
std::thread::sleep(Duration::from_secs(1));
|
||||||
|
|
||||||
|
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||||
|
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
|
||||||
|
assert_eq!(read, "before");
|
||||||
|
assert_eq!(
|
||||||
|
KEYRING_CONNECTS.load(Ordering::SeqCst),
|
||||||
|
connects + 1,
|
||||||
|
"the provider kept the old session, so nothing was tested"
|
||||||
|
);
|
||||||
|
with_keyring_entry(service, user, |e| e.set_password("after")).unwrap();
|
||||||
|
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
|
||||||
|
assert_eq!(read, "after");
|
||||||
|
with_keyring_entry(service, user, |e| e.delete_credential()).unwrap();
|
||||||
|
keyring_core::unset_default_store();
|
||||||
|
}
|
||||||
|
|
||||||
fn stored_with(refresh: &str) -> StoredSession {
|
fn stored_with(refresh: &str) -> StoredSession {
|
||||||
serde_json::from_str(&blob("").replace(r#""r""#, &format!("{refresh:?}"))).unwrap()
|
serde_json::from_str(&blob("").replace(r#""r""#, &format!("{refresh:?}"))).unwrap()
|
||||||
}
|
}
|
||||||
@@ -1267,12 +1462,7 @@ mod tests {
|
|||||||
write_stored(&stored_with("r")).unwrap();
|
write_stored(&stored_with("r")).unwrap();
|
||||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||||
|
|
||||||
let entry = keyring_entry().unwrap();
|
fail_next(keyring_core::Error::NoStorageAccess(
|
||||||
let cred = entry
|
|
||||||
.as_any()
|
|
||||||
.downcast_ref::<keyring_core::mock::Cred>()
|
|
||||||
.unwrap();
|
|
||||||
cred.set_error(keyring_core::Error::NoStorageAccess(
|
|
||||||
std::io::Error::other("locked").into(),
|
std::io::Error::other("locked").into(),
|
||||||
));
|
));
|
||||||
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
||||||
@@ -1307,12 +1497,7 @@ mod tests {
|
|||||||
let _turn = mock_keyring();
|
let _turn = mock_keyring();
|
||||||
write_stored(&stored_with("r")).unwrap();
|
write_stored(&stored_with("r")).unwrap();
|
||||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||||
let entry = keyring_entry().unwrap();
|
fail_next(keyring_core::Error::NoEntry);
|
||||||
entry
|
|
||||||
.as_any()
|
|
||||||
.downcast_ref::<keyring_core::mock::Cred>()
|
|
||||||
.unwrap()
|
|
||||||
.set_error(keyring_core::Error::NoEntry);
|
|
||||||
|
|
||||||
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
||||||
assert!(sink.state.lock().unsaved.is_some());
|
assert!(sink.state.lock().unsaved.is_some());
|
||||||
|
|||||||
@@ -562,6 +562,12 @@ impl AppDirs {
|
|||||||
self.state_dir().join("control.sock")
|
self.state_dir().join("control.sock")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The Google Photos import's last run: its archives and how it ended, so
|
||||||
|
/// an import cut short by a restart can be resumed.
|
||||||
|
pub fn takeout_import_path(&self) -> PathBuf {
|
||||||
|
self.state_dir().join("takeout-import.json")
|
||||||
|
}
|
||||||
|
|
||||||
/// Unix domain socket the tray uses to ensure single instance.
|
/// Unix domain socket the tray uses to ensure single instance.
|
||||||
pub fn tray_socket(&self) -> PathBuf {
|
pub fn tray_socket(&self) -> PathBuf {
|
||||||
self.state_dir().join("tray.sock")
|
self.state_dir().join("tray.sock")
|
||||||
|
|||||||
+130
-11
@@ -279,6 +279,11 @@ pub enum Request {
|
|||||||
/// and files what it has uploaded into its albums, then reports
|
/// and files what it has uploaded into its albums, then reports
|
||||||
/// `cancelled`. Replies with [`Response::Ok`].
|
/// `cancelled`. Replies with [`Response::Ok`].
|
||||||
CancelImport,
|
CancelImport,
|
||||||
|
/// Start the last import again with the same archives, after it was
|
||||||
|
/// interrupted, cancelled or failed. Photos it already uploaded are found
|
||||||
|
/// on the account and skipped. Replies like [`Request::ImportTakeout`];
|
||||||
|
/// [`ErrorKind::NotFound`] when there is nothing to resume.
|
||||||
|
ResumeImport,
|
||||||
/// Upload the photo at `source_path` under the given name and media type.
|
/// Upload the photo at `source_path` under the given name and media type.
|
||||||
///
|
///
|
||||||
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
||||||
@@ -768,6 +773,9 @@ pub enum Topic {
|
|||||||
Conflicts,
|
Conflicts,
|
||||||
/// The activity log gained entries.
|
/// The activity log gained entries.
|
||||||
Activity,
|
Activity,
|
||||||
|
/// File thumbnails were made or fetched. A front-end asks again for the
|
||||||
|
/// tiles it still shows blank; the ones it already painted stay as they are.
|
||||||
|
Thumbnails,
|
||||||
/// A topic from a newer daemon, which this front-end cannot act on.
|
/// A topic from a newer daemon, which this front-end cannot act on.
|
||||||
#[serde(other)]
|
#[serde(other)]
|
||||||
Unknown,
|
Unknown,
|
||||||
@@ -777,7 +785,7 @@ impl Topic {
|
|||||||
/// Every topic a front-end can act on. A new subscription is told all of
|
/// Every topic a front-end can act on. A new subscription is told all of
|
||||||
/// them have changed, so it catches up on whatever happened while it was
|
/// them have changed, so it catches up on whatever happened while it was
|
||||||
/// not connected.
|
/// not connected.
|
||||||
pub const ALL: [Topic; 9] = [
|
pub const ALL: [Topic; 10] = [
|
||||||
Topic::Files,
|
Topic::Files,
|
||||||
Topic::Photos,
|
Topic::Photos,
|
||||||
Topic::Trash,
|
Topic::Trash,
|
||||||
@@ -787,6 +795,7 @@ impl Topic {
|
|||||||
Topic::Queue,
|
Topic::Queue,
|
||||||
Topic::Conflicts,
|
Topic::Conflicts,
|
||||||
Topic::Activity,
|
Topic::Activity,
|
||||||
|
Topic::Thumbnails,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1685,10 +1694,75 @@ pub struct ImportSummary {
|
|||||||
pub bytes: u64,
|
pub bytes: u64,
|
||||||
/// True when the run stopped early because it was cancelled.
|
/// True when the run stopped early because it was cancelled.
|
||||||
pub cancelled: bool,
|
pub cancelled: bool,
|
||||||
/// Why the run as a whole failed, when it did, such as an archive that
|
/// Why the run as a whole stopped early, when it did, such as the
|
||||||
/// is not a zip. The counts are then empty.
|
/// account being unreachable. The counts say how far it got.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub error: Option<String>,
|
pub error: Option<String>,
|
||||||
|
/// What kind of failure [`error`](Self::error) is, so a front-end can word
|
||||||
|
/// it in the user's language. The text itself is the daemon's English.
|
||||||
|
#[serde(default)]
|
||||||
|
pub error_kind: Option<ErrorKind>,
|
||||||
|
/// True for a scan that uploads nothing ([`Request::ImportTakeout`] with
|
||||||
|
/// `dry_run`). [`to_upload`](Self::to_upload) then says what an import
|
||||||
|
/// would upload.
|
||||||
|
#[serde(default)]
|
||||||
|
pub dry_run: bool,
|
||||||
|
/// Photos not yet on the account: what a dry run would upload, and what a
|
||||||
|
/// real run set out to upload once the duplicates were known.
|
||||||
|
#[serde(default)]
|
||||||
|
pub to_upload: usize,
|
||||||
|
/// Bytes of [`to_upload`](Self::to_upload).
|
||||||
|
#[serde(default)]
|
||||||
|
pub to_upload_bytes: u64,
|
||||||
|
/// Live and motion photos whose video was filed with its still.
|
||||||
|
#[serde(default)]
|
||||||
|
pub live_photos: usize,
|
||||||
|
/// True when the service stopped while the import ran (a shutdown or a
|
||||||
|
/// restart). [`Request::ResumeImport`] picks it up again.
|
||||||
|
#[serde(default)]
|
||||||
|
pub interrupted: bool,
|
||||||
|
/// Archives that could not be read and were left out, by path.
|
||||||
|
#[serde(default)]
|
||||||
|
pub unreadable_archives: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ImportSummary {
|
||||||
|
/// Whether the run did all it set out to do.
|
||||||
|
pub fn complete(&self) -> bool {
|
||||||
|
self.error.is_none()
|
||||||
|
&& !self.cancelled
|
||||||
|
&& !self.interrupted
|
||||||
|
&& self.failed == 0
|
||||||
|
&& self.unreadable_archives.is_empty()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which step a running Takeout import is on.
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum ImportPhase {
|
||||||
|
/// Reading the archives' directories and metadata.
|
||||||
|
#[default]
|
||||||
|
Scanning,
|
||||||
|
/// Asking the account which photos it already has.
|
||||||
|
Checking,
|
||||||
|
/// Uploading the photos that are new.
|
||||||
|
Uploading,
|
||||||
|
/// Creating albums and filing photos into them.
|
||||||
|
Albums,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where a running import is ([`Response::ImportStatus`]).
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct ImportProgress {
|
||||||
|
pub phase: ImportPhase,
|
||||||
|
/// Steps done in this phase.
|
||||||
|
pub done: u64,
|
||||||
|
/// Steps in this phase; `0` while not yet known.
|
||||||
|
pub total: u64,
|
||||||
|
/// The photo being uploaded, during [`ImportPhase::Uploading`].
|
||||||
|
#[serde(default)]
|
||||||
|
pub current: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One album in a [`Response::Albums`] listing.
|
/// One album in a [`Response::Albums`] listing.
|
||||||
@@ -1877,11 +1951,26 @@ pub fn is_video_file_name(name: &str, size: Option<u64>) -> bool {
|
|||||||
!transport_stream || size.is_none_or(|size| size >= TS_VIDEO_MIN_BYTES)
|
!transport_stream || size.is_none_or(|size| size >= TS_VIDEO_MIN_BYTES)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether `name` is a song or other recording in a format that can carry
|
||||||
|
/// its own cover picture: ID3 (MP3, AAC, WAV, AIFF), FLAC and Ogg pictures,
|
||||||
|
/// MP4 `covr` (M4A, M4B), or APE tags (WavPack, Monkey's Audio).
|
||||||
|
pub fn is_audio_file_name(name: &str) -> bool {
|
||||||
|
let Some((_, extension)) = name.rsplit_once('.') else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
AUDIO_COVER_EXTENSIONS.contains(&extension.to_ascii_lowercase().as_str())
|
||||||
|
}
|
||||||
|
|
||||||
|
const AUDIO_COVER_EXTENSIONS: &[&str] = &[
|
||||||
|
"aac", "aif", "aiff", "ape", "flac", "m4a", "m4b", "mp3", "oga", "ogg", "opus", "wav", "wv",
|
||||||
|
];
|
||||||
|
|
||||||
/// Whether file listings show a thumbnail for a file named `name`, of `size`
|
/// Whether file listings show a thumbnail for a file named `name`, of `size`
|
||||||
/// bytes when known: an image [`is_thumbnail_image_name`] accepts, or a video,
|
/// bytes when known: an image [`is_thumbnail_image_name`] accepts, a video,
|
||||||
/// whose thumbnail is a frame the daemon takes with ffmpeg.
|
/// whose thumbnail is a frame the daemon takes with ffmpeg, or a song, whose
|
||||||
|
/// thumbnail is its cover: the picture in its tags, or a cover image beside it.
|
||||||
pub fn is_thumbnail_media_name(name: &str, size: Option<u64>) -> bool {
|
pub fn is_thumbnail_media_name(name: &str, size: Option<u64>) -> bool {
|
||||||
is_thumbnail_image_name(name) || is_video_file_name(name, size)
|
is_thumbnail_image_name(name) || is_video_file_name(name, size) || is_audio_file_name(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether `name` is one of the concrete RAW formats handled by the exiftool
|
/// Whether `name` is one of the concrete RAW formats handled by the exiftool
|
||||||
@@ -2189,8 +2278,17 @@ pub enum Response {
|
|||||||
/// report; `summary` is `None` when no import has run this session.
|
/// report; `summary` is `None` when no import has run this session.
|
||||||
ImportStatus {
|
ImportStatus {
|
||||||
running: bool,
|
running: bool,
|
||||||
|
/// The counts so far while running, the final report after.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
summary: Option<ImportSummary>,
|
summary: Option<ImportSummary>,
|
||||||
|
/// The current step, while running.
|
||||||
|
#[serde(default)]
|
||||||
|
progress: Option<ImportProgress>,
|
||||||
|
/// The archives of the last run, when it can be resumed with
|
||||||
|
/// [`Request::ResumeImport`]: it was a real import that did not
|
||||||
|
/// finish. Survives a restart of the service.
|
||||||
|
#[serde(default)]
|
||||||
|
resumable: Vec<String>,
|
||||||
},
|
},
|
||||||
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
|
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
|
||||||
/// `running` is false once it has finished, when `summary` is the final
|
/// `running` is false once it has finished, when `summary` is the final
|
||||||
@@ -2663,6 +2761,16 @@ mod tests {
|
|||||||
assert!(end.is_none(), "{end:?}");
|
assert!(end.is_none(), "{end:?}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_thumbnail_change_travels_as_its_own_topic() {
|
||||||
|
let line = serde_json::to_string(&Event::Changed {
|
||||||
|
topics: vec![Topic::Thumbnails],
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(line, r#"{"Changed":{"topics":["thumbnails"]}}"#);
|
||||||
|
assert!(Topic::ALL.contains(&Topic::Thumbnails));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_daemon_without_subscriptions_is_told_apart_from_a_dead_one() {
|
fn a_daemon_without_subscriptions_is_told_apart_from_a_dead_one() {
|
||||||
let socket = fake_daemon("legacy", "{\"Error\":{\"message\":\"unknown request\"}}\n");
|
let socket = fake_daemon("legacy", "{\"Error\":{\"message\":\"unknown request\"}}\n");
|
||||||
@@ -2727,7 +2835,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn listings_thumbnail_images_and_videos() {
|
fn listings_thumbnail_images_videos_and_songs() {
|
||||||
assert!(is_thumbnail_media_name("photo.JPG", Some(10)));
|
assert!(is_thumbnail_media_name("photo.JPG", Some(10)));
|
||||||
assert!(is_thumbnail_media_name("camera.nef", None));
|
assert!(is_thumbnail_media_name("camera.nef", None));
|
||||||
assert!(is_thumbnail_media_name("clip.mp4", Some(10)));
|
assert!(is_thumbnail_media_name("clip.mp4", Some(10)));
|
||||||
@@ -2736,13 +2844,24 @@ mod tests {
|
|||||||
Some(64 * 1024 * 1024)
|
Some(64 * 1024 * 1024)
|
||||||
));
|
));
|
||||||
assert!(!is_thumbnail_media_name("app.ts", Some(2048)));
|
assert!(!is_thumbnail_media_name("app.ts", Some(2048)));
|
||||||
assert!(!is_thumbnail_media_name(
|
assert!(is_thumbnail_media_name("song.flac", Some(64 * 1024 * 1024)));
|
||||||
"song.flac",
|
assert!(is_thumbnail_media_name("Track 01.MP3", None));
|
||||||
Some(64 * 1024 * 1024)
|
|
||||||
));
|
|
||||||
assert!(!is_thumbnail_media_name("notes.txt", None));
|
assert!(!is_thumbnail_media_name("notes.txt", None));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn songs_are_told_by_their_extension() {
|
||||||
|
for name in [
|
||||||
|
"a.mp3", "b.FLAC", "c.ogg", "d.oga", "e.opus", "f.m4a", "g.m4b", "h.aac", "i.wav",
|
||||||
|
"j.aiff", "k.aif", "l.wv", "m.ape",
|
||||||
|
] {
|
||||||
|
assert!(is_audio_file_name(name), "{name}");
|
||||||
|
}
|
||||||
|
for name in ["mp3", "song.mp3.zip", "clip.mp4", "notes.txt", "cover.jpg"] {
|
||||||
|
assert!(!is_audio_file_name(name), "{name}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Keep every extension accepted by the shared classifier covered. Drive
|
/// Keep every extension accepted by the shared classifier covered. Drive
|
||||||
/// activation uses this classification before it knows whether opening a
|
/// activation uses this classification before it knows whether opening a
|
||||||
/// file should stream through FUSE or materialise a complete cache entry;
|
/// file should stream through FUSE or materialise a complete cache entry;
|
||||||
|
|||||||
+488
-70
@@ -25,7 +25,9 @@
|
|||||||
//! The service folder (`Google Photos`) is localized, as are the year folders
|
//! The service folder (`Google Photos`) is localized, as are the year folders
|
||||||
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
||||||
//! service name: the first component is dropped when it is the export root, the
|
//! service name: the first component is dropped when it is the export root, the
|
||||||
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
|
//! second is the service, and the rest is `<folder>/<file>`. The Photos service
|
||||||
|
//! is the one whose sidecars carry a `photoTakenTime`; other services in the same
|
||||||
|
//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an
|
||||||
//! **album** when it carries an album metadata sidecar naming it; anything else
|
//! **album** when it carries an album metadata sidecar naming it; anything else
|
||||||
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
||||||
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
||||||
@@ -121,8 +123,14 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
|
|||||||
("m2ts", "video/mp2t"),
|
("m2ts", "video/mp2t"),
|
||||||
("mpg", "video/mpeg"),
|
("mpg", "video/mpeg"),
|
||||||
("mpeg", "video/mpeg"),
|
("mpeg", "video/mpeg"),
|
||||||
|
// The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`.
|
||||||
|
("mp", "video/mp4"),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
/// Sidecars bigger than this are not read. A real one is a few kilobytes; a
|
||||||
|
/// `.json` this large is some other service's data and not worth the memory.
|
||||||
|
const SIDECAR_MAX_BYTES: u64 = 1024 * 1024;
|
||||||
|
|
||||||
/// The media type for `name`'s extension, or `None` when it is not media this
|
/// The media type for `name`'s extension, or `None` when it is not media this
|
||||||
/// import handles.
|
/// import handles.
|
||||||
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
||||||
@@ -147,6 +155,13 @@ pub enum Bucket {
|
|||||||
Archived,
|
Archived,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct GeoPoint {
|
||||||
|
pub latitude: f64,
|
||||||
|
pub longitude: f64,
|
||||||
|
}
|
||||||
|
|
||||||
/// One media file found in the export.
|
/// One media file found in the export.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct TakeoutPhoto {
|
pub struct TakeoutPhoto {
|
||||||
@@ -162,14 +177,49 @@ pub struct TakeoutPhoto {
|
|||||||
pub size: u64,
|
pub size: u64,
|
||||||
/// Media type from the extension.
|
/// Media type from the extension.
|
||||||
pub media_type: String,
|
pub media_type: String,
|
||||||
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
|
/// Capture time in epoch seconds, from the sidecar's `photoTakenTime`,
|
||||||
/// `creationTime`). `None` when the photo has no sidecar — the upload then
|
/// else from a date in the file name. `None` when neither has one; the
|
||||||
/// defaults it to now, as the SDK does.
|
/// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time).
|
||||||
pub capture_time: Option<i64>,
|
pub capture_time: Option<i64>,
|
||||||
|
/// The sidecar's `creationTime`: when the photo reached Google Photos, not
|
||||||
|
/// when it was taken. Only a last resort for the capture time.
|
||||||
|
#[serde(default)]
|
||||||
|
pub creation_time: Option<i64>,
|
||||||
|
/// Where the photo was taken, from the sidecar's `geoData` (or
|
||||||
|
/// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped.
|
||||||
|
#[serde(default)]
|
||||||
|
pub location: Option<GeoPoint>,
|
||||||
/// The sidecar's `favorited` flag.
|
/// The sidecar's `favorited` flag.
|
||||||
pub favorite: bool,
|
pub favorite: bool,
|
||||||
/// Which bucket the file was found in.
|
/// Which bucket the file was found in.
|
||||||
pub bucket: Bucket,
|
pub bucket: Bucket,
|
||||||
|
/// Pairs the halves of a live or motion photo: a still and a video in the
|
||||||
|
/// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||||
|
/// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half.
|
||||||
|
#[serde(default)]
|
||||||
|
pub live_key: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TakeoutPhoto {
|
||||||
|
/// Whether this is the video half of a pair, as opposed to the still.
|
||||||
|
pub fn is_video(&self) -> bool {
|
||||||
|
self.media_type.starts_with("video/")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather
|
||||||
|
/// than an Apple live photo.
|
||||||
|
pub fn is_motion_photo(&self) -> bool {
|
||||||
|
self.live_key
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|key| key.ends_with(".mp"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An archive the scan had to leave out, and why.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct UnreadableArchive {
|
||||||
|
pub path: PathBuf,
|
||||||
|
pub reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Everything one scan found, ready to be turned into an import plan.
|
/// Everything one scan found, ready to be turned into an import plan.
|
||||||
@@ -183,8 +233,16 @@ pub struct TakeoutScan {
|
|||||||
/// Media files skipped because they sit in the trash bucket.
|
/// Media files skipped because they sit in the trash bucket.
|
||||||
pub skipped_trashed: usize,
|
pub skipped_trashed: usize,
|
||||||
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
||||||
/// activity `.csv`s, `.mp` motion-photo parts).
|
/// activity `.csv`s), and media from other Google services in the same
|
||||||
|
/// export (`Google Drive/`, `YouTube/`).
|
||||||
pub skipped_other: usize,
|
pub skipped_other: usize,
|
||||||
|
/// Archives that could not be opened or are not a zip. The rest of the
|
||||||
|
/// export is still imported.
|
||||||
|
#[serde(default)]
|
||||||
|
pub unreadable_archives: Vec<UnreadableArchive>,
|
||||||
|
/// Entries inside an otherwise readable archive that could not be read.
|
||||||
|
#[serde(default)]
|
||||||
|
pub unreadable_entries: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TakeoutScan {
|
impl TakeoutScan {
|
||||||
@@ -195,7 +253,7 @@ impl TakeoutScan {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
||||||
/// file also carries geo data, people tags and view counts.
|
/// file also carries people tags, a description and view counts.
|
||||||
#[derive(Debug, Default, Deserialize)]
|
#[derive(Debug, Default, Deserialize)]
|
||||||
struct PhotoSidecar {
|
struct PhotoSidecar {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -205,19 +263,69 @@ struct PhotoSidecar {
|
|||||||
#[serde(rename = "creationTime", default)]
|
#[serde(rename = "creationTime", default)]
|
||||||
creation_time: Option<SidecarTime>,
|
creation_time: Option<SidecarTime>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
favorited: bool,
|
favorited: Option<bool>,
|
||||||
|
/// The position as edited in Google Photos, else as the camera wrote it.
|
||||||
|
#[serde(rename = "geoData", default)]
|
||||||
|
geo_data: Option<SidecarGeo>,
|
||||||
|
#[serde(rename = "geoDataExif", default)]
|
||||||
|
geo_data_exif: Option<SidecarGeo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A sidecar timestamp: epoch seconds, as a *string*.
|
impl PhotoSidecar {
|
||||||
|
fn location(&self) -> Option<GeoPoint> {
|
||||||
|
self.geo_data
|
||||||
|
.as_ref()
|
||||||
|
.and_then(SidecarGeo::point)
|
||||||
|
.or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number.
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct SidecarTime {
|
struct SidecarTime {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
timestamp: Option<String>,
|
timestamp: Option<SidecarNumber>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum SidecarNumber {
|
||||||
|
Number(i64),
|
||||||
|
Text(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SidecarTime {
|
impl SidecarTime {
|
||||||
|
/// The timestamp, when it is a real moment. `0` and negative values are
|
||||||
|
/// Google's "unknown" and would file the photo under 1970.
|
||||||
fn epoch_seconds(&self) -> Option<i64> {
|
fn epoch_seconds(&self) -> Option<i64> {
|
||||||
self.timestamp.as_ref()?.parse().ok()
|
let seconds = match self.timestamp.as_ref()? {
|
||||||
|
SidecarNumber::Number(seconds) => *seconds,
|
||||||
|
SidecarNumber::Text(text) => text.trim().parse().ok()?,
|
||||||
|
};
|
||||||
|
(seconds > 0).then_some(seconds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `geoData` in a sidecar. All-zero means Google has no position.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SidecarGeo {
|
||||||
|
#[serde(default)]
|
||||||
|
latitude: Option<f64>,
|
||||||
|
#[serde(default)]
|
||||||
|
longitude: Option<f64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SidecarGeo {
|
||||||
|
fn point(&self) -> Option<GeoPoint> {
|
||||||
|
let (latitude, longitude) = (self.latitude?, self.longitude?);
|
||||||
|
let on_globe = latitude.is_finite()
|
||||||
|
&& longitude.is_finite()
|
||||||
|
&& (-90.0..=90.0).contains(&latitude)
|
||||||
|
&& (-180.0..=180.0).contains(&longitude);
|
||||||
|
(on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint {
|
||||||
|
latitude,
|
||||||
|
longitude,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,21 +370,58 @@ impl RawEntry {
|
|||||||
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
||||||
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
||||||
/// this stays fast on a 200 GB export.
|
/// this stays fast on a 200 GB export.
|
||||||
|
///
|
||||||
|
/// An archive that cannot be opened is left out and listed in
|
||||||
|
/// [`TakeoutScan::unreadable_archives`]; the rest still imports.
|
||||||
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||||
let mut entries: Vec<RawEntry> = Vec::new();
|
scan_archives_until(archives, || false)
|
||||||
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
|
}
|
||||||
|
|
||||||
|
/// [`scan_archives`] that gives up when `stop` returns true, checked between
|
||||||
|
/// archives and every few hundred entries.
|
||||||
|
pub fn scan_archives_until(
|
||||||
|
archives: &[PathBuf],
|
||||||
|
stop: impl Fn() -> bool,
|
||||||
|
) -> CoreResult<TakeoutScan> {
|
||||||
|
let mut entries: Vec<(String, RawEntry)> = Vec::new();
|
||||||
|
let mut sidecars: Vec<(String, String, Vec<u8>)> = Vec::new();
|
||||||
|
let mut unreadable_archives = Vec::new();
|
||||||
|
let mut unreadable_entries = 0;
|
||||||
|
|
||||||
for (index, path) in archives.iter().enumerate() {
|
for (index, path) in archives.iter().enumerate() {
|
||||||
let file = std::fs::File::open(path)
|
if stop() {
|
||||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
return Err(CoreError::conflict("the import was cancelled"));
|
||||||
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
|
}
|
||||||
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
|
let opened = std::fs::File::open(path)
|
||||||
})?;
|
.map_err(|e| e.to_string())
|
||||||
|
.and_then(|file| {
|
||||||
|
zip::ZipArchive::new(std::io::BufReader::new(file))
|
||||||
|
.map_err(|e| format!("not a readable zip: {e}"))
|
||||||
|
});
|
||||||
|
let mut zip = match opened {
|
||||||
|
Ok(zip) => zip,
|
||||||
|
Err(reason) => {
|
||||||
|
tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive");
|
||||||
|
unreadable_archives.push(UnreadableArchive {
|
||||||
|
path: path.clone(),
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
for position in 0..zip.len() {
|
for position in 0..zip.len() {
|
||||||
let mut entry = zip.by_index(position).map_err(|e| {
|
if position % 512 == 0 && stop() {
|
||||||
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
|
return Err(CoreError::conflict("the import was cancelled"));
|
||||||
})?;
|
}
|
||||||
|
let mut entry = match zip.by_index(position) {
|
||||||
|
Ok(entry) => entry,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry");
|
||||||
|
unreadable_entries += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
if entry.is_dir() {
|
if entry.is_dir() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -286,32 +431,92 @@ pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
|||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let name = name.to_string_lossy().replace('\\', "/");
|
let name = name.to_string_lossy().replace('\\', "/");
|
||||||
let Some(relative) = strip_export_prefix(&name) else {
|
let Some((service, relative)) = strip_export_prefix(&name) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
let (service, relative) = (service.to_string(), relative.to_string());
|
||||||
let size = entry.size();
|
let size = entry.size();
|
||||||
|
|
||||||
// Sidecars are read now, while the entry is open: they are a few
|
// Sidecars are read now, while the entry is open: they are a few
|
||||||
// hundred bytes each and reopening the archive per photo later would
|
// hundred bytes each and reopening the archive per photo later would
|
||||||
// be a second full pass.
|
// be a second full pass.
|
||||||
if relative.to_ascii_lowercase().ends_with(".json") {
|
if relative.to_ascii_lowercase().ends_with(".json") {
|
||||||
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
|
if size <= SIDECAR_MAX_BYTES {
|
||||||
if entry.read_to_end(&mut body).is_ok() {
|
let mut body = Vec::with_capacity(size as usize);
|
||||||
sidecars.insert(relative.to_string(), body);
|
if entry.read_to_end(&mut body).is_ok() {
|
||||||
|
sidecars.push((service, relative, body));
|
||||||
|
} else {
|
||||||
|
unreadable_entries += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
entries.push((
|
||||||
entries.push(RawEntry {
|
service,
|
||||||
archive: index,
|
RawEntry {
|
||||||
entry: name.clone(),
|
archive: index,
|
||||||
relative: relative.to_string(),
|
entry: name,
|
||||||
size,
|
relative,
|
||||||
});
|
size,
|
||||||
|
},
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(assemble(entries, &sidecars))
|
let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars);
|
||||||
|
let mut scan = assemble(entries, &sidecars);
|
||||||
|
scan.skipped_other += other_services;
|
||||||
|
scan.unreadable_archives = unreadable_archives;
|
||||||
|
scan.unreadable_entries = unreadable_entries;
|
||||||
|
Ok(scan)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop what other Google services put into the same export.
|
||||||
|
///
|
||||||
|
/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to
|
||||||
|
/// `Google Photos/`, and their media must not land in the photo timeline. The
|
||||||
|
/// service folder's name is localized (`Google Fotos`), so the Photos service
|
||||||
|
/// is recognized by its content instead: it is the folder whose sidecars carry
|
||||||
|
/// a `photoTakenTime`. When no folder does — an export without sidecars, or
|
||||||
|
/// one re-zipped without the service level — everything is kept, as before.
|
||||||
|
///
|
||||||
|
/// Returns the entries and sidecars to import, and how many media files were
|
||||||
|
/// left out.
|
||||||
|
fn keep_photos_services(
|
||||||
|
entries: Vec<(String, RawEntry)>,
|
||||||
|
sidecars: Vec<(String, String, Vec<u8>)>,
|
||||||
|
) -> (Vec<RawEntry>, HashMap<String, Vec<u8>>, usize) {
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct Marker {
|
||||||
|
#[serde(rename = "photoTakenTime")]
|
||||||
|
photo_taken_time: Option<serde::de::IgnoredAny>,
|
||||||
|
}
|
||||||
|
let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new();
|
||||||
|
for (service, _, body) in &sidecars {
|
||||||
|
if photos_services.contains(service.as_str()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if serde_json::from_slice::<Marker>(body).is_ok_and(|m| m.photo_taken_time.is_some()) {
|
||||||
|
photos_services.insert(service.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service);
|
||||||
|
|
||||||
|
let mut skipped = 0;
|
||||||
|
let mut kept_entries = Vec::with_capacity(entries.len());
|
||||||
|
for (service, raw) in entries {
|
||||||
|
if keep(&service) {
|
||||||
|
kept_entries.push(raw);
|
||||||
|
} else if media_type_of(raw.file_name()).is_some() {
|
||||||
|
skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let kept_sidecars = sidecars
|
||||||
|
.iter()
|
||||||
|
.filter(|(service, _, _)| keep(service))
|
||||||
|
.map(|(_, relative, body)| (relative.clone(), body.clone()))
|
||||||
|
.collect();
|
||||||
|
(kept_entries, kept_sidecars, skipped)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
||||||
@@ -403,19 +608,19 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
|||||||
.unwrap_or(&file_name)
|
.unwrap_or(&file_name)
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
// Sidecar first, then the date stamped into the name.
|
// Sidecar first, then the date stamped into the name. `creationTime`
|
||||||
|
// is when the photo reached Google, which for a scan or an old photo
|
||||||
|
// uploaded late is years off; the importer uses it only after EXIF.
|
||||||
let capture_time = sidecar
|
let capture_time = sidecar
|
||||||
.photo_taken_time
|
.photo_taken_time
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(SidecarTime::epoch_seconds)
|
.and_then(SidecarTime::epoch_seconds)
|
||||||
.or_else(|| {
|
|
||||||
sidecar
|
|
||||||
.creation_time
|
|
||||||
.as_ref()
|
|
||||||
.and_then(SidecarTime::epoch_seconds)
|
|
||||||
})
|
|
||||||
.or_else(|| capture_time_from_name(&name))
|
.or_else(|| capture_time_from_name(&name))
|
||||||
.or_else(|| capture_time_from_name(&file_name));
|
.or_else(|| capture_time_from_name(&file_name));
|
||||||
|
let creation_time = sidecar
|
||||||
|
.creation_time
|
||||||
|
.as_ref()
|
||||||
|
.and_then(SidecarTime::epoch_seconds);
|
||||||
|
|
||||||
scan.photos.push(TakeoutPhoto {
|
scan.photos.push(TakeoutPhoto {
|
||||||
archive: raw.archive,
|
archive: raw.archive,
|
||||||
@@ -424,31 +629,57 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
|||||||
size: raw.size,
|
size: raw.size,
|
||||||
media_type: media_type.to_string(),
|
media_type: media_type.to_string(),
|
||||||
capture_time,
|
capture_time,
|
||||||
favorite: sidecar.favorited,
|
creation_time,
|
||||||
|
location: sidecar.location(),
|
||||||
|
favorite: sidecar.favorited.unwrap_or(false),
|
||||||
bucket,
|
bucket,
|
||||||
|
live_key: live_key(&folder, &file_name),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
scan
|
scan
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drop the export root (`Takeout/`) and the localized service folder from a
|
/// Split a path inside the archive into the localized service folder and the
|
||||||
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
|
/// rest, dropping the export root (`Takeout/`): `("Google Fotos",
|
||||||
|
/// "<folder>/<file>")`.
|
||||||
///
|
///
|
||||||
/// `None` for a path with no service folder at all (the export's own
|
/// `None` for a path with no service folder at all (the export's own
|
||||||
/// `archive_browser.html`), which carries nothing to import.
|
/// `archive_browser.html`), which carries nothing to import.
|
||||||
fn strip_export_prefix(path: &str) -> Option<&str> {
|
fn strip_export_prefix(path: &str) -> Option<(&str, &str)> {
|
||||||
let mut rest = path;
|
let mut rest = path;
|
||||||
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
|
// The root is `Takeout/`, but an export unzipped and re-zipped by the user
|
||||||
// the user may not have it — so the root is dropped only when a service
|
// may not have it. `Google Photos/<album>/<file>` then has three parts like
|
||||||
// folder follows it, which the component count decides.
|
// `Takeout/Google Photos/<file>`, so the count alone cannot tell them
|
||||||
|
// apart: a root of another name is only taken for one with four parts.
|
||||||
|
let (first, after) = rest.split_once('/')?;
|
||||||
let components = rest.split('/').count();
|
let components = rest.split('/').count();
|
||||||
if components >= 3 {
|
if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 {
|
||||||
rest = rest.split_once('/')?.1;
|
rest = after;
|
||||||
}
|
}
|
||||||
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
|
// The service folder (`Google Photos`, `Google Fotos`, …).
|
||||||
let (_service, tail) = rest.split_once('/')?;
|
rest.split_once('/')
|
||||||
Some(tail)
|
}
|
||||||
|
|
||||||
|
/// The key that pairs the still and the video of one live or motion photo, or
|
||||||
|
/// `None` when `file_name` cannot be either half.
|
||||||
|
///
|
||||||
|
/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem.
|
||||||
|
/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the
|
||||||
|
/// video's whole name. The folder is part of the key so two cameras' `IMG_1`
|
||||||
|
/// in different albums never pair.
|
||||||
|
fn live_key(folder: &str, file_name: &str) -> Option<String> {
|
||||||
|
let lower = file_name.to_lowercase();
|
||||||
|
let (stem, ext) = lower.rsplit_once('.')?;
|
||||||
|
let key = match ext {
|
||||||
|
"heic" | "heif" | "jpg" | "jpeg" => stem,
|
||||||
|
// Apple's video half. A plain `.mp4` beside a photo of the same name is
|
||||||
|
// far more often an unrelated clip than half of a live photo.
|
||||||
|
"mov" => stem,
|
||||||
|
"mp" => lower.as_str(),
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
(!key.is_empty()).then(|| format!("{folder}/{key}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
||||||
@@ -858,13 +1089,13 @@ mod tests {
|
|||||||
RawEntry {
|
RawEntry {
|
||||||
archive,
|
archive,
|
||||||
entry: path.to_string(),
|
entry: path.to_string(),
|
||||||
relative: strip_export_prefix(path).unwrap().to_string(),
|
relative: strip_export_prefix(path).unwrap().1.to_string(),
|
||||||
size,
|
size,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
||||||
let relative = strip_export_prefix(path).unwrap().to_string();
|
let relative = strip_export_prefix(path).unwrap().1.to_string();
|
||||||
map.insert(relative, body.as_bytes().to_vec());
|
map.insert(relative, body.as_bytes().to_vec());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -872,12 +1103,17 @@ mod tests {
|
|||||||
fn strips_export_root_and_localized_service_folder() {
|
fn strips_export_root_and_localized_service_folder() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
||||||
Some("Island 2019/IMG_1.jpg")
|
Some(("Google Fotos", "Island 2019/IMG_1.jpg"))
|
||||||
);
|
);
|
||||||
// Already unwrapped by the user: no `Takeout/` root left.
|
// Already unwrapped by the user: no `Takeout/` root left.
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
||||||
Some("IMG_1.jpg")
|
Some(("Google Photos", "IMG_1.jpg"))
|
||||||
|
);
|
||||||
|
// ... and an album keeps its folder.
|
||||||
|
assert_eq!(
|
||||||
|
strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"),
|
||||||
|
Some(("Google Photos", "Iceland/IMG_1.jpg"))
|
||||||
);
|
);
|
||||||
// Nothing below a service folder is nothing to import.
|
// Nothing below a service folder is nothing to import.
|
||||||
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
||||||
@@ -957,23 +1193,144 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn creation_time_is_the_fallback_capture_time() {
|
fn creation_time_is_kept_apart_as_the_last_resort() {
|
||||||
|
// `creationTime` is when the photo reached Google. It must not beat the
|
||||||
|
// date in the name, and the importer tries EXIF before it.
|
||||||
let mut sidecars = HashMap::new();
|
let mut sidecars = HashMap::new();
|
||||||
sidecar(
|
sidecar(
|
||||||
&mut sidecars,
|
&mut sidecars,
|
||||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
||||||
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
||||||
);
|
);
|
||||||
|
sidecar(
|
||||||
|
&mut sidecars,
|
||||||
|
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json",
|
||||||
|
r#"{"creationTime":{"timestamp":"1700000000"}}"#,
|
||||||
|
);
|
||||||
let scan = assemble(
|
let scan = assemble(
|
||||||
vec![entry(
|
vec![
|
||||||
0,
|
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
|
||||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
entry(
|
||||||
10,
|
0,
|
||||||
)],
|
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg",
|
||||||
|
10,
|
||||||
|
),
|
||||||
|
],
|
||||||
&sidecars,
|
&sidecars,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
|
assert_eq!(scan.photos[0].capture_time, None);
|
||||||
|
assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000));
|
||||||
|
assert_eq!(
|
||||||
|
scan.photos[1].capture_time,
|
||||||
|
capture_time_from_name("IMG_20190612_101500.jpg")
|
||||||
|
);
|
||||||
|
assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sidecar_timestamps_tolerate_numbers_zero_and_null() {
|
||||||
|
let mut sidecars = HashMap::new();
|
||||||
|
for (name, body) in [
|
||||||
|
("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#),
|
||||||
|
(
|
||||||
|
"B.jpg",
|
||||||
|
r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"C.jpg",
|
||||||
|
r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#,
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
sidecar(
|
||||||
|
&mut sidecars,
|
||||||
|
&format!("Takeout/Google Photos/Photos from 2019/{name}.json"),
|
||||||
|
body,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let scan = assemble(
|
||||||
|
["A.jpg", "B.jpg", "C.jpg"]
|
||||||
|
.iter()
|
||||||
|
.map(|name| {
|
||||||
|
entry(
|
||||||
|
0,
|
||||||
|
&format!("Takeout/Google Photos/Photos from 2019/{name}"),
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
&sidecars,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||||
|
assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970");
|
||||||
|
assert!(!scan.photos[1].favorite);
|
||||||
|
assert_eq!(scan.photos[2].capture_time, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn location_comes_from_geo_data_and_zero_means_none() {
|
||||||
|
let mut sidecars = HashMap::new();
|
||||||
|
sidecar(
|
||||||
|
&mut sidecars,
|
||||||
|
"Takeout/Google Photos/Iceland/IMG_1.jpg.json",
|
||||||
|
r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#,
|
||||||
|
);
|
||||||
|
sidecar(
|
||||||
|
&mut sidecars,
|
||||||
|
"Takeout/Google Photos/Iceland/IMG_2.jpg.json",
|
||||||
|
r#"{"geoData":{"latitude":0.0,"longitude":0.0},
|
||||||
|
"geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#,
|
||||||
|
);
|
||||||
|
sidecar(
|
||||||
|
&mut sidecars,
|
||||||
|
"Takeout/Google Photos/Iceland/IMG_3.jpg.json",
|
||||||
|
r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#,
|
||||||
|
);
|
||||||
|
let scan = assemble(
|
||||||
|
(1..=3)
|
||||||
|
.map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1))
|
||||||
|
.collect(),
|
||||||
|
&sidecars,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
scan.photos[0].location,
|
||||||
|
Some(GeoPoint {
|
||||||
|
latitude: 64.1466,
|
||||||
|
longitude: -21.9426
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
scan.photos[1].location.map(|p| p.latitude),
|
||||||
|
Some(63.4186),
|
||||||
|
"geoDataExif fills in for an empty geoData"
|
||||||
|
);
|
||||||
|
assert_eq!(scan.photos[2].location, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn live_and_motion_photo_halves_share_a_key() {
|
||||||
|
let scan = assemble(
|
||||||
|
vec![
|
||||||
|
entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1),
|
||||||
|
entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1),
|
||||||
|
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1),
|
||||||
|
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1),
|
||||||
|
entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1),
|
||||||
|
entry(0, "Takeout/Google Photos/Trip/scan.png", 1),
|
||||||
|
],
|
||||||
|
&HashMap::new(),
|
||||||
|
);
|
||||||
|
let key = |i: usize| scan.photos[i].live_key.clone();
|
||||||
|
|
||||||
|
assert_eq!(key(0), key(1));
|
||||||
|
assert!(!scan.photos[0].is_motion_photo());
|
||||||
|
assert_eq!(key(2), key(3));
|
||||||
|
assert!(scan.photos[3].is_motion_photo());
|
||||||
|
assert!(scan.photos[3].is_video(), "the .MP half is a video");
|
||||||
|
assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair");
|
||||||
|
assert_eq!(key(5), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1387,16 +1744,77 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_file_that_is_not_a_zip_is_a_readable_error() {
|
fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() {
|
||||||
let path =
|
let broken =
|
||||||
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
||||||
std::fs::write(&path, b"not a zip at all").unwrap();
|
std::fs::write(&broken, b"not a zip at all").unwrap();
|
||||||
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
|
let good = write_zip(
|
||||||
let _ = std::fs::remove_file(&path);
|
"beside-broken",
|
||||||
|
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||||
|
);
|
||||||
|
|
||||||
|
let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap();
|
||||||
|
let _ = std::fs::remove_file(&broken);
|
||||||
|
let _ = std::fs::remove_file(&good);
|
||||||
|
|
||||||
|
assert_eq!(scan.photos.len(), 1);
|
||||||
|
assert_eq!(scan.photos[0].archive, 1);
|
||||||
|
assert_eq!(scan.unreadable_archives.len(), 1);
|
||||||
|
assert_eq!(scan.unreadable_archives[0].path, broken);
|
||||||
assert!(
|
assert!(
|
||||||
error.message.contains("not a readable zip"),
|
scan.unreadable_archives[0]
|
||||||
"unexpected message: {}",
|
.reason
|
||||||
error.message
|
.contains("not a readable zip"),
|
||||||
|
"unexpected reason: {}",
|
||||||
|
scan.unreadable_archives[0].reason
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn other_services_in_the_same_export_are_not_imported() {
|
||||||
|
let zip = write_zip(
|
||||||
|
"services",
|
||||||
|
&[
|
||||||
|
(
|
||||||
|
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg",
|
||||||
|
b"x".as_slice(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json",
|
||||||
|
br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(),
|
||||||
|
),
|
||||||
|
("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()),
|
||||||
|
("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||||
|
let _ = std::fs::remove_file(&zip);
|
||||||
|
|
||||||
|
assert_eq!(scan.photos.len(), 1);
|
||||||
|
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
|
||||||
|
assert_eq!(scan.skipped_other, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_export_without_sidecars_keeps_every_service() {
|
||||||
|
let zip = write_zip(
|
||||||
|
"no-sidecars",
|
||||||
|
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||||
|
);
|
||||||
|
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||||
|
let _ = std::fs::remove_file(&zip);
|
||||||
|
assert_eq!(scan.photos.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_stop_request_ends_the_scan() {
|
||||||
|
let zip = write_zip(
|
||||||
|
"stop",
|
||||||
|
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||||
|
);
|
||||||
|
let result = scan_archives_until(std::slice::from_ref(&zip), || true);
|
||||||
|
let _ = std::fs::remove_file(&zip);
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -20,9 +20,11 @@ tokio.workspace = true
|
|||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
fuser.workspace = true
|
fuser.workspace = true
|
||||||
libc.workspace = true
|
libc.workspace = true
|
||||||
|
serde.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
image.workspace = true
|
image.workspace = true
|
||||||
|
lofty.workspace = true
|
||||||
notify = "8"
|
notify = "8"
|
||||||
parking_lot.workspace = true
|
parking_lot.workspace = true
|
||||||
|
|
||||||
|
|||||||
@@ -877,35 +877,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
|||||||
if archives.is_empty() {
|
if archives.is_empty() {
|
||||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||||
} else {
|
} else {
|
||||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
let paths = archives.into_iter().map(PathBuf::from).collect();
|
||||||
let missing: Vec<String> = paths
|
start_takeout_import(core, paths, dry_run)
|
||||||
.iter()
|
|
||||||
.filter(|path| !path.is_file())
|
|
||||||
.map(|path| path.display().to_string())
|
|
||||||
.collect();
|
|
||||||
if !missing.is_empty() {
|
|
||||||
CtlResponse::error(CoreError::not_found(format!(
|
|
||||||
"cannot read: {}",
|
|
||||||
missing.join(", ")
|
|
||||||
)))
|
|
||||||
} else {
|
|
||||||
let count = paths.len();
|
|
||||||
match core.spawn_takeout_import(paths, dry_run) {
|
|
||||||
Ok(()) => CtlResponse::Ok {
|
|
||||||
message: format!(
|
|
||||||
"{} {count} archive(s)",
|
|
||||||
if dry_run { "scanning" } else { "importing" }
|
|
||||||
),
|
|
||||||
},
|
|
||||||
Err(e) => CtlResponse::error(e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(CtlRequest::ImportStatus) => {
|
Ok(CtlRequest::ImportStatus) => {
|
||||||
let (running, summary) = crate::takeout::import_status();
|
let report = crate::takeout::import_status();
|
||||||
CtlResponse::ImportStatus { running, summary }
|
CtlResponse::ImportStatus {
|
||||||
|
running: report.running,
|
||||||
|
summary: report.summary,
|
||||||
|
progress: report.progress,
|
||||||
|
resumable: report.resumable,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
|
||||||
|
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
|
||||||
|
Some(paths) => start_takeout_import(core, paths, false),
|
||||||
|
},
|
||||||
Ok(CtlRequest::CancelImport) => {
|
Ok(CtlRequest::CancelImport) => {
|
||||||
if crate::takeout::cancel_import() {
|
if crate::takeout::cancel_import() {
|
||||||
CtlResponse::Ok {
|
CtlResponse::Ok {
|
||||||
@@ -2009,6 +1997,31 @@ pub(crate) fn run_control_socket(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Check that every archive is there, then start the import in the background.
|
||||||
|
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
|
||||||
|
let missing: Vec<String> = paths
|
||||||
|
.iter()
|
||||||
|
.filter(|path| !path.is_file())
|
||||||
|
.map(|path| path.display().to_string())
|
||||||
|
.collect();
|
||||||
|
if !missing.is_empty() {
|
||||||
|
return CtlResponse::error(CoreError::not_found(format!(
|
||||||
|
"cannot read: {}",
|
||||||
|
missing.join(", ")
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let count = paths.len();
|
||||||
|
match core.spawn_takeout_import(paths, dry_run) {
|
||||||
|
Ok(()) => CtlResponse::Ok {
|
||||||
|
message: format!(
|
||||||
|
"{} {count} archive(s)",
|
||||||
|
if dry_run { "scanning" } else { "importing" }
|
||||||
|
),
|
||||||
|
},
|
||||||
|
Err(e) => CtlResponse::error(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod request_limit_tests {
|
mod request_limit_tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
|
|||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
use pdfs_core::control::{
|
use pdfs_core::control::{
|
||||||
FileThumbRequest, MappingProgress, PhotoItem, PhotoKind, PhotoLocation, PhotoThumb, PlaceInfo,
|
FileThumbRequest, MappingProgress, PhotoItem, PhotoKind, PhotoLocation, PhotoThumb, PlaceInfo,
|
||||||
ThumbnailBuildStatus, Topic, is_raw_image_name, is_thumbnail_image_name,
|
ThumbnailBuildStatus, Topic, is_audio_file_name, is_raw_image_name, is_thumbnail_image_name,
|
||||||
is_thumbnail_media_name,
|
is_thumbnail_media_name,
|
||||||
};
|
};
|
||||||
use pdfs_core::db::{self, StoredPhoto};
|
use pdfs_core::db::{self, StoredPhoto};
|
||||||
@@ -76,6 +76,36 @@ const VIDEO_BOX_WALK_MAX: usize = 32;
|
|||||||
/// A single frame is seconds of work even for 4K; this only stops a wedged
|
/// A single frame is seconds of work even for 4K; this only stops a wedged
|
||||||
/// ffmpeg from holding a thumbnail permit.
|
/// ffmpeg from holding a thumbnail permit.
|
||||||
const FFMPEG_TIMEOUT: Duration = Duration::from_secs(30);
|
const FFMPEG_TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
/// A song no larger than this is downloaded whole for its cover.
|
||||||
|
const AUDIO_WHOLE_MAX_BYTES: u64 = 4 * 1024 * 1024;
|
||||||
|
/// How much of a song's start is read first, to learn where its tags end.
|
||||||
|
const AUDIO_PROBE_BYTES: u64 = 64 * 1024;
|
||||||
|
/// How much of a song's start is read when its first bytes don't say where its
|
||||||
|
/// tags end: an Ogg comment packet, which holds the cover, has no size up front.
|
||||||
|
const AUDIO_HEAD_BYTES: u64 = 4 * 1024 * 1024;
|
||||||
|
/// Read past the tags at a song's start, so the first audio frames are there
|
||||||
|
/// for a reader that looks for them.
|
||||||
|
const AUDIO_HEAD_SLACK: u64 = 64 * 1024;
|
||||||
|
/// Largest start of a song read for its tags. A cover bigger than this is not
|
||||||
|
/// worth the download for a 512-pixel thumbnail.
|
||||||
|
const AUDIO_TAG_MAX_BYTES: u64 = 16 * 1024 * 1024;
|
||||||
|
/// How much of a song's end is read in formats that keep their tags there
|
||||||
|
/// (APE tags in WavPack and Monkey's Audio, `id3 ` chunks in WAV and AIFF).
|
||||||
|
const AUDIO_TAIL_BYTES: u64 = 4 * 1024 * 1024;
|
||||||
|
/// Names of a folder's cover image, best first, matched without case and
|
||||||
|
/// extension. A song without a picture in its tags gets this image's.
|
||||||
|
const FOLDER_COVER_STEMS: [&str; 6] = [
|
||||||
|
"cover",
|
||||||
|
"folder",
|
||||||
|
"front",
|
||||||
|
"album",
|
||||||
|
"albumart",
|
||||||
|
"albumartsmall",
|
||||||
|
];
|
||||||
|
/// Image types a folder cover is taken in.
|
||||||
|
const FOLDER_COVER_EXTENSIONS: [&str; 4] = ["jpg", "jpeg", "png", "webp"];
|
||||||
|
/// Largest folder cover image downloaded whole for a song's thumbnail.
|
||||||
|
const FOLDER_COVER_MAX_BYTES: u64 = 32 * 1024 * 1024;
|
||||||
/// Largest thumbnail attached to an upload. Proton's own clients hold theirs to
|
/// Largest thumbnail attached to an upload. Proton's own clients hold theirs to
|
||||||
/// this, stepping the JPEG quality down until it fits.
|
/// this, stepping the JPEG quality down until it fits.
|
||||||
const UPLOAD_THUMB_MAX_BYTES: usize = 60 * 1024;
|
const UPLOAD_THUMB_MAX_BYTES: usize = 60 * 1024;
|
||||||
@@ -630,6 +660,7 @@ enum Fetched {
|
|||||||
/// A whole image.
|
/// A whole image.
|
||||||
Whole(Vec<u8>),
|
Whole(Vec<u8>),
|
||||||
Video(VideoSample),
|
Video(VideoSample),
|
||||||
|
Audio(AudioSample),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The parts of a video its thumbnail needs, downloaded: the start of the
|
/// The parts of a video its thumbnail needs, downloaded: the start of the
|
||||||
@@ -766,13 +797,181 @@ fn video_sample_thumbnail(
|
|||||||
video_file_thumbnail(&file.0, ffmpeg, timeout)
|
video_file_thumbnail(&file.0, ffmpeg, timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The parts of a song its cover needs, downloaded: the start, where most
|
||||||
|
/// formats keep their tags, an M4A's index, and for formats that keep their
|
||||||
|
/// tags at the end, the end.
|
||||||
|
struct AudioSample {
|
||||||
|
/// The whole file's length.
|
||||||
|
size: u64,
|
||||||
|
/// Each part and the offset it came from.
|
||||||
|
parts: Vec<(u64, Vec<u8>)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where the tags at the start of a song end, read from its first `probe`
|
||||||
|
/// bytes: past an ID3v2 tag, and past FLAC's metadata blocks, which hold its
|
||||||
|
/// picture. `None` when the start says nothing, as for Ogg. A FLAC block
|
||||||
|
/// header past `probe` stops the walk with a guess of [`AUDIO_HEAD_BYTES`]
|
||||||
|
/// more.
|
||||||
|
fn audio_tags_end(probe: &[u8]) -> Option<u64> {
|
||||||
|
let mut offset = 0u64;
|
||||||
|
let mut known = false;
|
||||||
|
if probe.get(0..3) == Some(b"ID3") {
|
||||||
|
let size = probe.get(6..10)?;
|
||||||
|
if size.iter().any(|byte| byte & 0x80 != 0) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let size = size
|
||||||
|
.iter()
|
||||||
|
.fold(0u64, |acc, byte| (acc << 7) | u64::from(*byte));
|
||||||
|
let footer = probe.get(5).is_some_and(|flags| flags & 0x10 != 0);
|
||||||
|
offset = 10 + size + if footer { 10 } else { 0 };
|
||||||
|
known = true;
|
||||||
|
}
|
||||||
|
let start = usize::try_from(offset).ok()?;
|
||||||
|
if probe.get(start..start + 4) == Some(b"fLaC") {
|
||||||
|
offset += 4;
|
||||||
|
loop {
|
||||||
|
let at = usize::try_from(offset).ok()?;
|
||||||
|
let Some(header) = probe.get(at..at + 4) else {
|
||||||
|
return Some(offset + AUDIO_HEAD_BYTES);
|
||||||
|
};
|
||||||
|
let len = u64::from_be_bytes([0, 0, 0, 0, 0, header[1], header[2], header[3]]);
|
||||||
|
offset += 4 + len;
|
||||||
|
if header[0] & 0x80 != 0 {
|
||||||
|
return Some(offset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
known.then_some(offset)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a song in format `name` may keep its tags at its end.
|
||||||
|
fn audio_tags_at_end(name: &str) -> bool {
|
||||||
|
let extension = name
|
||||||
|
.rsplit_once('.')
|
||||||
|
.map(|(_, extension)| extension.to_ascii_lowercase())
|
||||||
|
.unwrap_or_default();
|
||||||
|
matches!(extension.as_str(), "wv" | "ape" | "wav" | "aif" | "aiff")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download what the cover of the song `reader` reads, named `name`, needs. A
|
||||||
|
/// song of up to [`AUDIO_WHOLE_MAX_BYTES`] comes whole; a larger one brings its
|
||||||
|
/// tags, never its audio.
|
||||||
|
async fn sample_audio(reader: &dyn RevisionRead, name: &str) -> DriveResult<AudioSample> {
|
||||||
|
let size = reader.size();
|
||||||
|
if size <= AUDIO_WHOLE_MAX_BYTES {
|
||||||
|
let whole = reader.read_at(0, size).await?;
|
||||||
|
return Ok(AudioSample {
|
||||||
|
size,
|
||||||
|
parts: vec![(0, whole)],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let probe = reader.read_at(0, AUDIO_PROBE_BYTES.min(size)).await?;
|
||||||
|
// An M4A keeps its cover in its index, fetched below wherever it sits.
|
||||||
|
let iso_media = probe.get(4..8) == Some(b"ftyp");
|
||||||
|
let want = match audio_tags_end(&probe) {
|
||||||
|
_ if iso_media => 0,
|
||||||
|
Some(end) => end.saturating_add(AUDIO_HEAD_SLACK),
|
||||||
|
None => AUDIO_HEAD_BYTES,
|
||||||
|
}
|
||||||
|
.min(AUDIO_TAG_MAX_BYTES)
|
||||||
|
.min(size);
|
||||||
|
let head = if want > probe.len() as u64 {
|
||||||
|
reader.read_at(0, want).await?
|
||||||
|
} else {
|
||||||
|
probe
|
||||||
|
};
|
||||||
|
let mut parts = Vec::new();
|
||||||
|
match iso_media_index(reader, &head, size).await? {
|
||||||
|
Some((offset, len)) if offset.saturating_add(len) > head.len() as u64 => {
|
||||||
|
parts.push((offset, reader.read_at(offset, len).await?));
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
let head_len = head.len() as u64;
|
||||||
|
parts.push((0, head));
|
||||||
|
if audio_tags_at_end(name) {
|
||||||
|
let start = size.saturating_sub(AUDIO_TAIL_BYTES).max(head_len);
|
||||||
|
if start < size {
|
||||||
|
parts.push((start, reader.read_at(start, size - start).await?));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(AudioSample { size, parts })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The picture a song's tags carry: the front cover, or else the first one.
|
||||||
|
/// `None` for a song without one, or one lofty can't read.
|
||||||
|
fn embedded_cover(path: &Path) -> Option<Vec<u8>> {
|
||||||
|
use lofty::config::{ParseOptions, ParsingMode};
|
||||||
|
use lofty::file::TaggedFileExt as _;
|
||||||
|
use lofty::picture::PictureType;
|
||||||
|
|
||||||
|
let tagged = lofty::probe::Probe::open(path)
|
||||||
|
.ok()?
|
||||||
|
.options(
|
||||||
|
ParseOptions::new()
|
||||||
|
.read_properties(false)
|
||||||
|
.parsing_mode(ParsingMode::Relaxed),
|
||||||
|
)
|
||||||
|
.guess_file_type()
|
||||||
|
.ok()?
|
||||||
|
.read()
|
||||||
|
.ok()?;
|
||||||
|
let pictures = || tagged.tags().iter().flat_map(|tag| tag.pictures());
|
||||||
|
pictures()
|
||||||
|
.find(|picture| picture.pic_type() == PictureType::CoverFront)
|
||||||
|
.or_else(|| pictures().next())
|
||||||
|
.map(|picture| picture.data().to_vec())
|
||||||
|
.filter(|data| !data.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A thumbnail from the cover in a downloaded [`AudioSample`]'s tags. The
|
||||||
|
/// sample is laid out in a staging file at the offsets it came from, so the
|
||||||
|
/// tags are where the file's own headers say they are; the audio between
|
||||||
|
/// holds no data and costs no disk. A song without a cover, or with one the
|
||||||
|
/// image decoder can't read, is [`ScaleAttempt::Undecodable`].
|
||||||
|
fn audio_cover_thumbnail(sample: &AudioSample, name: &str, staging_dir: &Path) -> ScaleAttempt {
|
||||||
|
let Some(file) = RawTempFile::create_in(staging_dir, &[], name) else {
|
||||||
|
return ScaleAttempt::Unavailable;
|
||||||
|
};
|
||||||
|
let laid_out = OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.open(&file.0)
|
||||||
|
.and_then(|staged| {
|
||||||
|
for (offset, bytes) in &sample.parts {
|
||||||
|
staged.write_all_at(bytes, *offset)?;
|
||||||
|
}
|
||||||
|
staged.set_len(sample.size)
|
||||||
|
});
|
||||||
|
if laid_out.is_err() {
|
||||||
|
return ScaleAttempt::Unavailable;
|
||||||
|
}
|
||||||
|
match embedded_cover(&file.0) {
|
||||||
|
Some(picture) => scale_thumbnail(&picture, "cover", staging_dir),
|
||||||
|
None => ScaleAttempt::Undecodable,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How good a folder cover `name` is: its place in [`FOLDER_COVER_STEMS`],
|
||||||
|
/// lowest best, or `None` for a file that is no cover image.
|
||||||
|
fn cover_rank(name: &str) -> Option<usize> {
|
||||||
|
let (stem, extension) = name.rsplit_once('.')?;
|
||||||
|
let extension = extension.to_ascii_lowercase();
|
||||||
|
if !FOLDER_COVER_EXTENSIONS.contains(&extension.as_str()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let stem = stem.to_ascii_lowercase();
|
||||||
|
FOLDER_COVER_STEMS.iter().position(|known| *known == stem)
|
||||||
|
}
|
||||||
|
|
||||||
/// The thumbnail to attach to an upload of the local file at `path`, which is
|
/// The thumbnail to attach to an upload of the local file at `path`, which is
|
||||||
/// named `name` in Drive. Without one, Drive's own apps show a generic icon and
|
/// named `name` in Drive. Without one, Drive's own apps show a generic icon and
|
||||||
/// the gallery has to download the whole file to make its own.
|
/// the gallery has to download the whole file to make its own.
|
||||||
///
|
///
|
||||||
/// Best effort: a file that isn't an image or a video, is too large to read, or
|
/// Best effort: a file that isn't an image, a video or a song with a cover in
|
||||||
/// can't be decoded uploads without a thumbnail, as it always did. Blocking and
|
/// its tags, is too large to read, or can't be decoded uploads without a
|
||||||
/// CPU-bound, so async callers run it on the blocking pool.
|
/// thumbnail, as it always did. Blocking and CPU-bound, so async callers run it
|
||||||
|
/// on the blocking pool.
|
||||||
pub(crate) fn upload_thumbnails(path: &Path, name: &str, staging_dir: &Path) -> Vec<Thumbnail> {
|
pub(crate) fn upload_thumbnails(path: &Path, name: &str, staging_dir: &Path) -> Vec<Thumbnail> {
|
||||||
if is_video_name(name) && !ffmpeg_available() {
|
if is_video_name(name) && !ffmpeg_available() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -806,6 +1005,14 @@ fn upload_thumbnails_with(
|
|||||||
return Vec::new();
|
return Vec::new();
|
||||||
};
|
};
|
||||||
scale_thumbnail_unbudgeted(&bytes, name, staging_dir)
|
scale_thumbnail_unbudgeted(&bytes, name, staging_dir)
|
||||||
|
} else if is_audio_file_name(name) {
|
||||||
|
// Only the song's own picture: a cover image beside it is not known
|
||||||
|
// here, where `path` may be a staging copy rather than the song's
|
||||||
|
// folder. The daemon finds that one when a listing asks.
|
||||||
|
match embedded_cover(path) {
|
||||||
|
Some(picture) => scale_thumbnail(&picture, "cover", staging_dir),
|
||||||
|
None => return Vec::new(),
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
};
|
};
|
||||||
@@ -986,7 +1193,7 @@ impl Core {
|
|||||||
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
if !self.rt.block_on(self.refresh_timeline())? {
|
if !self.refresh_timeline_joined()? {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||||
@@ -1703,6 +1910,7 @@ impl Core {
|
|||||||
return (uid, ThumbAttempt::Cancelled);
|
return (uid, ThumbAttempt::Cancelled);
|
||||||
}
|
}
|
||||||
let video = is_video_name(&name);
|
let video = is_video_name(&name);
|
||||||
|
let audio = is_audio_file_name(&name);
|
||||||
// Without ffmpeg a video can still have its uploader's
|
// Without ffmpeg a video can still have its uploader's
|
||||||
// thumbnail, but Photos has asked the server already.
|
// thumbnail, but Photos has asked the server already.
|
||||||
let can_sample = !video || ffmpeg_available();
|
let can_sample = !video || ffmpeg_available();
|
||||||
@@ -1757,6 +1965,7 @@ impl Core {
|
|||||||
|
|
||||||
let download_uid = uid.clone();
|
let download_uid = uid.clone();
|
||||||
let drive = core.drive.clone();
|
let drive = core.drive.clone();
|
||||||
|
let audio_name = name.clone();
|
||||||
let download = async move {
|
let download = async move {
|
||||||
// Photos asked the server in one batch already. A
|
// Photos asked the server in one batch already. A
|
||||||
// file view's or a build's file may carry a
|
// file view's or a build's file may carry a
|
||||||
@@ -1776,6 +1985,11 @@ impl Core {
|
|||||||
if video {
|
if video {
|
||||||
let reader = drive.open_revision(&download_uid).await?;
|
let reader = drive.open_revision(&download_uid).await?;
|
||||||
sample_video(reader.as_ref()).await.map(Fetched::Video)
|
sample_video(reader.as_ref()).await.map(Fetched::Video)
|
||||||
|
} else if audio {
|
||||||
|
let reader = drive.open_revision(&download_uid).await?;
|
||||||
|
sample_audio(reader.as_ref(), &audio_name)
|
||||||
|
.await
|
||||||
|
.map(Fetched::Audio)
|
||||||
} else {
|
} else {
|
||||||
client
|
client
|
||||||
.download_file(&download_uid)
|
.download_file(&download_uid)
|
||||||
@@ -1833,7 +2047,11 @@ impl Core {
|
|||||||
// Decoding + scaling a 20 MP JPEG is CPU-bound and would
|
// Decoding + scaling a 20 MP JPEG is CPU-bound and would
|
||||||
// stall the runtime's worker; hand it to the blocking pool.
|
// stall the runtime's worker; hand it to the blocking pool.
|
||||||
let staging_dir = core.cache.raw_thumbnail_staging_dir().to_path_buf();
|
let staging_dir = core.cache.raw_thumbnail_staging_dir().to_path_buf();
|
||||||
|
let staging = staging_dir.clone();
|
||||||
let made = tokio::task::spawn_blocking(move || match fetched {
|
let made = tokio::task::spawn_blocking(move || match fetched {
|
||||||
|
Fetched::Audio(sample) => {
|
||||||
|
audio_cover_thumbnail(&sample, &name, &staging_dir)
|
||||||
|
}
|
||||||
Fetched::Video(sample) => video_sample_thumbnail(
|
Fetched::Video(sample) => video_sample_thumbnail(
|
||||||
&sample,
|
&sample,
|
||||||
&name,
|
&name,
|
||||||
@@ -1850,6 +2068,14 @@ impl Core {
|
|||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.unwrap_or(ScaleAttempt::Unavailable);
|
.unwrap_or(ScaleAttempt::Unavailable);
|
||||||
|
// A song without a picture of its own shows its
|
||||||
|
// folder's cover.
|
||||||
|
let made = match made {
|
||||||
|
ScaleAttempt::Undecodable if audio && core.thumb_job_current(job) => {
|
||||||
|
core.folder_cover_thumbnail(&uid, staging).await
|
||||||
|
}
|
||||||
|
made => made,
|
||||||
|
};
|
||||||
if !core.thumb_job_current(job) {
|
if !core.thumb_job_current(job) {
|
||||||
return (uid, ThumbAttempt::Cancelled);
|
return (uid, ThumbAttempt::Cancelled);
|
||||||
}
|
}
|
||||||
@@ -1910,9 +2136,85 @@ impl Core {
|
|||||||
ThumbAttempt::Cancelled => summary.cancelled += 1,
|
ThumbAttempt::Cancelled => summary.cancelled += 1,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A folder on screen whose tiles were given up on, as a recursive
|
||||||
|
// build's start does to them, fills in without being opened again.
|
||||||
|
if summary.made > 0 {
|
||||||
|
self.events.publish(&[Topic::Thumbnails]);
|
||||||
|
}
|
||||||
summary
|
summary
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The cover image in the folder of the song `uid`: the best-named one of
|
||||||
|
/// [`FOLDER_COVER_STEMS`] among its siblings the database knows.
|
||||||
|
fn folder_cover(&self, uid: &NodeUid) -> Option<proton_drive_rs::Node> {
|
||||||
|
let node = self.db.node_by_uid(&uid.to_string()).ok()??;
|
||||||
|
let parent = node.parent_uid?;
|
||||||
|
let siblings = match self.db.children_if_listed(&parent) {
|
||||||
|
Ok(Some(children)) => children,
|
||||||
|
_ => self.db.known_children(&parent).ok()?,
|
||||||
|
};
|
||||||
|
siblings
|
||||||
|
.into_iter()
|
||||||
|
.filter(|sibling| !sibling.is_folder() && !sibling.trashed)
|
||||||
|
.filter_map(|sibling| cover_rank(&sibling.name).map(|rank| (rank, sibling)))
|
||||||
|
.min_by(|(a, x), (b, y)| a.cmp(b).then_with(|| x.name.cmp(&y.name)))
|
||||||
|
.map(|(_, cover)| cover)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A thumbnail of the song `uid` made from its folder's cover image: the
|
||||||
|
/// cover's own cached thumbnail when it has one, else the thumbnail its
|
||||||
|
/// uploader attached, else the image itself, up to
|
||||||
|
/// [`FOLDER_COVER_MAX_BYTES`].
|
||||||
|
///
|
||||||
|
/// The result is stored under the song, so a cover added to the folder or
|
||||||
|
/// replaced later shows only once the song changes or the daemon restarts
|
||||||
|
/// and forgets the songs it found no cover for.
|
||||||
|
async fn folder_cover_thumbnail(&self, uid: &NodeUid, staging_dir: PathBuf) -> ScaleAttempt {
|
||||||
|
let Some(cover) = self.folder_cover(uid) else {
|
||||||
|
return ScaleAttempt::Undecodable;
|
||||||
|
};
|
||||||
|
let ttype = ThumbnailType::Thumbnail;
|
||||||
|
let cached = self
|
||||||
|
.cache
|
||||||
|
.cached_thumbnail_path(&cover.uid, ttype.as_i32(), cover.modification_time)
|
||||||
|
.and_then(|path| std::fs::read(path).ok());
|
||||||
|
let ready = match cached {
|
||||||
|
Some(bytes) => Some(bytes),
|
||||||
|
None => match self.drive.download_thumbnail(&cover.uid, ttype).await {
|
||||||
|
Ok(bytes) => bytes.filter(|bytes| !bytes.is_empty()),
|
||||||
|
Err(e) => {
|
||||||
|
warn!(cover = %cover.uid, error = %e, "folder cover thumbnail download failed");
|
||||||
|
return ScaleAttempt::Unavailable;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if let Some(bytes) = ready {
|
||||||
|
return match ratio_of(&bytes) {
|
||||||
|
Some(ratio) => ScaleAttempt::Made(GeneratedThumb { bytes, ratio }),
|
||||||
|
None => ScaleAttempt::Undecodable,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let read = async {
|
||||||
|
let reader = self.drive.open_revision(&cover.uid).await?;
|
||||||
|
if reader.size() > FOLDER_COVER_MAX_BYTES {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
reader.read_at(0, reader.size()).await.map(Some)
|
||||||
|
};
|
||||||
|
let bytes = match read.await {
|
||||||
|
Ok(Some(bytes)) => bytes,
|
||||||
|
Ok(None) => return ScaleAttempt::Undecodable,
|
||||||
|
Err(e) => {
|
||||||
|
warn!(cover = %cover.uid, error = %e, "folder cover download failed");
|
||||||
|
return ScaleAttempt::Unavailable;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let name = cover.name;
|
||||||
|
tokio::task::spawn_blocking(move || scale_thumbnail(&bytes, &name, &staging_dir))
|
||||||
|
.await
|
||||||
|
.unwrap_or(ScaleAttempt::Unavailable)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn thumbnail_build_status(&self) -> ThumbnailBuildStatus {
|
pub(crate) fn thumbnail_build_status(&self) -> ThumbnailBuildStatus {
|
||||||
self.thumbnail_build.lock().clone()
|
self.thumbnail_build.lock().clone()
|
||||||
}
|
}
|
||||||
@@ -2338,6 +2640,36 @@ impl Core {
|
|||||||
|
|
||||||
/// Refresh the timeline off the request path, so a stale page is still served
|
/// Refresh the timeline off the request path, so a stale page is still served
|
||||||
/// at DB speed. At most one refresh runs at a time.
|
/// at DB speed. At most one refresh runs at a time.
|
||||||
|
/// Refresh the timeline and wait for it, sharing a refresh already under
|
||||||
|
/// way instead of starting a second one beside it.
|
||||||
|
///
|
||||||
|
/// The first request after an empty cache has nothing to serve until the
|
||||||
|
/// fetch lands, so it waits. Two such requests (the gallery and its retry,
|
||||||
|
/// or a background refresh started by an import) used to each fetch the
|
||||||
|
/// whole library at once, sharing the session's rate limit and taking
|
||||||
|
/// minutes. Returns whether the account has a photos library.
|
||||||
|
pub(crate) fn refresh_timeline_joined(&self) -> CoreResult<bool> {
|
||||||
|
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||||
|
while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() {
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||||
|
}
|
||||||
|
// A refresh that landed stamped the timeline fresh. One that failed
|
||||||
|
// did not, and the caller gets this refresh's own error instead of
|
||||||
|
// an empty timeline.
|
||||||
|
if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL)
|
||||||
|
|| self.timeline_refreshing.swap(true, Ordering::SeqCst)
|
||||||
|
{
|
||||||
|
return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let refreshed = self.rt.block_on(self.refresh_timeline());
|
||||||
|
self.timeline_refreshing.store(false, Ordering::SeqCst);
|
||||||
|
if refreshed.is_ok() {
|
||||||
|
self.events.publish(&[Topic::Photos]);
|
||||||
|
}
|
||||||
|
refreshed
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn spawn_timeline_refresh(&self) {
|
pub(crate) fn spawn_timeline_refresh(&self) {
|
||||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||||
return;
|
return;
|
||||||
@@ -2454,6 +2786,10 @@ mod decode_budget_tests {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod thumb_tests {
|
mod thumb_tests {
|
||||||
|
use super::{
|
||||||
|
AUDIO_HEAD_BYTES, AUDIO_TAIL_BYTES, AudioSample, audio_cover_thumbnail, audio_tags_end,
|
||||||
|
cover_rank, sample_audio,
|
||||||
|
};
|
||||||
use super::{
|
use super::{
|
||||||
RawTempFile, ScaleAttempt, THUMB_EDGE, UPLOAD_IMAGE_MAX_BYTES, UPLOAD_THUMB_MAX_BYTES,
|
RawTempFile, ScaleAttempt, THUMB_EDGE, UPLOAD_IMAGE_MAX_BYTES, UPLOAD_THUMB_MAX_BYTES,
|
||||||
VIDEO_HEAD_BYTES, VIDEO_WHOLE_MAX_BYTES, VideoSample, can_make_thumbnail, exiftool_binary,
|
VIDEO_HEAD_BYTES, VIDEO_WHOLE_MAX_BYTES, VideoSample, can_make_thumbnail, exiftool_binary,
|
||||||
@@ -2965,6 +3301,28 @@ mod thumb_tests {
|
|||||||
assert!(thumb.ratio > 0.0);
|
assert!(thumb.ratio > 0.0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Optional check against real songs, through the same partial download a
|
||||||
|
/// Drive song gets. `PDFS_AUDIO_FIXTURE` lists files separated by `:`.
|
||||||
|
#[test]
|
||||||
|
#[ignore = "set PDFS_AUDIO_FIXTURE to songs with a cover in their tags"]
|
||||||
|
fn real_audio_fixture_yields_its_cover() {
|
||||||
|
let paths = std::env::var_os("PDFS_AUDIO_FIXTURE").expect("PDFS_AUDIO_FIXTURE");
|
||||||
|
for path in std::env::split_paths(&paths) {
|
||||||
|
let name = path.file_name().unwrap().to_string_lossy().into_owned();
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
let revision = FakeRevision::new(bytes.len() as u64).with(0, bytes);
|
||||||
|
let sample = block_on(sample_audio(&revision, &name)).unwrap();
|
||||||
|
let read = revision.read.load(Ordering::Relaxed);
|
||||||
|
let attempt = audio_cover_thumbnail(&sample, &name, &std::env::temp_dir());
|
||||||
|
assert!(
|
||||||
|
matches!(attempt, ScaleAttempt::Made(_)),
|
||||||
|
"{name}: no cover, read {read} of {} bytes",
|
||||||
|
revision.size
|
||||||
|
);
|
||||||
|
eprintln!("{name}: read {read} of {} bytes", revision.size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn videos_are_told_apart_by_name() {
|
fn videos_are_told_apart_by_name() {
|
||||||
for name in [
|
for name in [
|
||||||
@@ -3121,4 +3479,201 @@ mod thumb_tests {
|
|||||||
"a small one is untouched"
|
"a small one is untouched"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A `width`×`height` PNG, the cover a tag carries.
|
||||||
|
fn png(width: u32, height: u32) -> Vec<u8> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
image::DynamicImage::ImageRgb8(image::RgbImage::new(width, height))
|
||||||
|
.write_to(
|
||||||
|
&mut std::io::Cursor::new(&mut bytes),
|
||||||
|
image::ImageFormat::Png,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An ID3v2.3 tag holding `picture` as its front cover.
|
||||||
|
fn id3_with_cover(picture: &[u8]) -> Vec<u8> {
|
||||||
|
let mut body = vec![0u8];
|
||||||
|
body.extend_from_slice(b"image/png\0");
|
||||||
|
body.push(3);
|
||||||
|
body.push(0);
|
||||||
|
body.extend_from_slice(picture);
|
||||||
|
let mut frame = b"APIC".to_vec();
|
||||||
|
frame.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||||
|
frame.extend_from_slice(&[0, 0]);
|
||||||
|
frame.extend_from_slice(&body);
|
||||||
|
let len = frame.len() as u32;
|
||||||
|
let mut tag = b"ID3\x03\x00\x00".to_vec();
|
||||||
|
tag.extend((0..4).rev().map(|i| ((len >> (7 * i)) & 0x7f) as u8));
|
||||||
|
tag.extend_from_slice(&frame);
|
||||||
|
tag
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ten silent MPEG-1 Layer III frames at 128 kbit/s and 44.1 kHz.
|
||||||
|
fn mpeg_frames() -> Vec<u8> {
|
||||||
|
let mut frame = vec![0u8; 417];
|
||||||
|
frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x64]);
|
||||||
|
frame.repeat(10)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A FLAC stream's metadata: STREAMINFO, then `picture` as its front cover.
|
||||||
|
fn flac_with_cover(picture: &[u8]) -> Vec<u8> {
|
||||||
|
let mut info = vec![0x10, 0x00, 0x10, 0x00, 0, 0, 0, 0, 0, 0];
|
||||||
|
let packed: u64 = (44_100 << 44) | (1 << 41) | (15 << 36);
|
||||||
|
info.extend_from_slice(&packed.to_be_bytes());
|
||||||
|
info.extend_from_slice(&[0; 16]);
|
||||||
|
let mut block = Vec::new();
|
||||||
|
block.extend_from_slice(&3u32.to_be_bytes());
|
||||||
|
block.extend_from_slice(&9u32.to_be_bytes());
|
||||||
|
block.extend_from_slice(b"image/png");
|
||||||
|
block.extend_from_slice(&0u32.to_be_bytes());
|
||||||
|
for value in [16u32, 8, 24, 0, picture.len() as u32] {
|
||||||
|
block.extend_from_slice(&value.to_be_bytes());
|
||||||
|
}
|
||||||
|
block.extend_from_slice(picture);
|
||||||
|
let mut out = b"fLaC".to_vec();
|
||||||
|
out.push(0);
|
||||||
|
out.extend_from_slice(&(info.len() as u32).to_be_bytes()[1..]);
|
||||||
|
out.extend_from_slice(&info);
|
||||||
|
out.push(0x80 | 6);
|
||||||
|
out.extend_from_slice(&(block.len() as u32).to_be_bytes()[1..]);
|
||||||
|
out.extend_from_slice(&block);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn whole_song(bytes: Vec<u8>) -> AudioSample {
|
||||||
|
AudioSample {
|
||||||
|
size: bytes.len() as u64,
|
||||||
|
parts: vec![(0, bytes)],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_song_shows_the_cover_in_its_tags() {
|
||||||
|
let tmp = scratch("song-cover");
|
||||||
|
std::fs::create_dir_all(&tmp).unwrap();
|
||||||
|
let mut mp3 = id3_with_cover(&png(16, 8));
|
||||||
|
mp3.extend(mpeg_frames());
|
||||||
|
let ScaleAttempt::Made(thumb) = audio_cover_thumbnail(&whole_song(mp3), "song.mp3", &tmp)
|
||||||
|
else {
|
||||||
|
panic!("an ID3 cover scales");
|
||||||
|
};
|
||||||
|
assert!((thumb.ratio - 2.0).abs() < 1e-6);
|
||||||
|
|
||||||
|
let ScaleAttempt::Made(thumb) =
|
||||||
|
audio_cover_thumbnail(&whole_song(flac_with_cover(&png(8, 16))), "song.flac", &tmp)
|
||||||
|
else {
|
||||||
|
panic!("a FLAC picture scales");
|
||||||
|
};
|
||||||
|
assert!((thumb.ratio - 0.5).abs() < 1e-6);
|
||||||
|
std::fs::remove_dir_all(&tmp).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_song_without_a_cover_is_no_thumbnail_of_its_own() {
|
||||||
|
let tmp = scratch("song-bare");
|
||||||
|
std::fs::create_dir_all(&tmp).unwrap();
|
||||||
|
let attempt = audio_cover_thumbnail(&whole_song(mpeg_frames()), "song.mp3", &tmp);
|
||||||
|
assert!(matches!(attempt, ScaleAttempt::Undecodable));
|
||||||
|
std::fs::remove_dir_all(&tmp).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_start_of_a_song_says_where_its_tags_end() {
|
||||||
|
let tag = id3_with_cover(&png(4, 4));
|
||||||
|
assert_eq!(audio_tags_end(&tag), Some(tag.len() as u64));
|
||||||
|
let flac = flac_with_cover(&png(4, 4));
|
||||||
|
assert_eq!(audio_tags_end(&flac), Some(flac.len() as u64));
|
||||||
|
// FLAC after an ID3 tag, as some taggers write it.
|
||||||
|
let both = [tag.clone(), flac.clone()].concat();
|
||||||
|
assert_eq!(audio_tags_end(&both), Some(both.len() as u64));
|
||||||
|
// The picture's block header cut off by the end of the probe, just
|
||||||
|
// past STREAMINFO: a guess past it.
|
||||||
|
let cut = &flac[..4 + 4 + 34 + 2];
|
||||||
|
assert!(audio_tags_end(cut).is_some_and(|end| end > cut.len() as u64));
|
||||||
|
assert_eq!(audio_tags_end(b"OggS\0\0\0\0"), None);
|
||||||
|
assert_eq!(
|
||||||
|
audio_tags_end(b"ID3\x03\x00\x00\x80\0\0\0"),
|
||||||
|
None,
|
||||||
|
"not synchsafe"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_large_song_downloads_its_tags_not_its_audio() {
|
||||||
|
const SIZE: u64 = 300 << 20;
|
||||||
|
let mut start = id3_with_cover(&png(16, 8));
|
||||||
|
start.extend(mpeg_frames());
|
||||||
|
let revision = FakeRevision::new(SIZE).with(0, start);
|
||||||
|
let sample = block_on(sample_audio(&revision, "long mix.mp3")).unwrap();
|
||||||
|
let read = revision.read.load(Ordering::Relaxed);
|
||||||
|
assert!(read < 1 << 20, "read {read} bytes");
|
||||||
|
|
||||||
|
let tmp = scratch("song-sparse");
|
||||||
|
std::fs::create_dir_all(&tmp).unwrap();
|
||||||
|
let ScaleAttempt::Made(thumb) = audio_cover_thumbnail(&sample, "long mix.mp3", &tmp) else {
|
||||||
|
panic!("the cover comes from the start alone");
|
||||||
|
};
|
||||||
|
assert!((thumb.ratio - 2.0).abs() < 1e-6);
|
||||||
|
std::fs::remove_dir_all(&tmp).unwrap();
|
||||||
|
|
||||||
|
// Ogg says nothing up front: a bounded head. WavPack keeps its tags at
|
||||||
|
// the end: the tail too.
|
||||||
|
let revision = FakeRevision::new(SIZE);
|
||||||
|
block_on(sample_audio(&revision, "live.ogg")).unwrap();
|
||||||
|
assert!(revision.read.load(Ordering::Relaxed) <= AUDIO_HEAD_BYTES + (64 << 10));
|
||||||
|
let revision = FakeRevision::new(SIZE);
|
||||||
|
let sample = block_on(sample_audio(&revision, "live.wv")).unwrap();
|
||||||
|
assert!(
|
||||||
|
sample
|
||||||
|
.parts
|
||||||
|
.iter()
|
||||||
|
.any(|(offset, bytes)| *offset == SIZE - AUDIO_TAIL_BYTES
|
||||||
|
&& bytes.len() as u64 == AUDIO_TAIL_BYTES)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_uploaded_song_carries_its_cover() {
|
||||||
|
let tmp = scratch("upload-song");
|
||||||
|
std::fs::create_dir_all(&tmp).unwrap();
|
||||||
|
// A staging blob has no extension: the format comes from the bytes.
|
||||||
|
let blob = tmp.join("blob");
|
||||||
|
let mut mp3 = id3_with_cover(&png(64, 64));
|
||||||
|
mp3.extend(mpeg_frames());
|
||||||
|
std::fs::write(&blob, mp3).unwrap();
|
||||||
|
let ffmpeg = fake_ffmpeg(&tmp, "exit 1");
|
||||||
|
let thumbs = upload_thumbnails_with(&blob, "song.mp3", &tmp, ffmpeg.as_os_str());
|
||||||
|
let [thumbnail] = thumbs.as_slice() else {
|
||||||
|
panic!("one thumbnail, got {}", thumbs.len());
|
||||||
|
};
|
||||||
|
assert!(matches!(thumbnail.thumbnail_type, ThumbnailType::Thumbnail));
|
||||||
|
assert_eq!(ratio_of(&thumbnail.content), Some(1.0));
|
||||||
|
|
||||||
|
std::fs::write(&blob, mpeg_frames()).unwrap();
|
||||||
|
assert!(
|
||||||
|
upload_thumbnails_with(&blob, "song.mp3", &tmp, ffmpeg.as_os_str()).is_empty(),
|
||||||
|
"a song without a cover uploads bare"
|
||||||
|
);
|
||||||
|
let _ = std::fs::remove_dir_all(&tmp);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_folder_cover_is_found_by_its_usual_names() {
|
||||||
|
assert_eq!(cover_rank("cover.jpg"), Some(0));
|
||||||
|
assert_eq!(cover_rank("Cover.JPG"), Some(0));
|
||||||
|
assert_eq!(cover_rank("folder.png"), Some(1));
|
||||||
|
assert_eq!(cover_rank("AlbumArtSmall.jpg"), Some(5));
|
||||||
|
assert!(cover_rank("cover.jpg") < cover_rank("front.webp"));
|
||||||
|
for name in [
|
||||||
|
"cover.txt",
|
||||||
|
"my cover.jpg",
|
||||||
|
"cover",
|
||||||
|
"back.jpg",
|
||||||
|
"cover.jpg.part",
|
||||||
|
] {
|
||||||
|
assert_eq!(cover_rank(name), None, "{name}");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -3732,6 +3732,88 @@ mod tests {
|
|||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A song with no picture in its tags shows the cover image in its folder,
|
||||||
|
/// and the daemon tells a following front-end that a thumbnail landed, so
|
||||||
|
/// a folder on screen fills in without being opened again.
|
||||||
|
#[test]
|
||||||
|
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
|
||||||
|
fn a_song_without_a_cover_shows_its_folders_and_says_so() {
|
||||||
|
let dir = scratch("song-cover");
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
// Silent MPEG-1 Layer III frames, without any tag.
|
||||||
|
let mut frame = vec![0u8; 417];
|
||||||
|
frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x64]);
|
||||||
|
let song = frame.repeat(64);
|
||||||
|
let mut cover = Vec::new();
|
||||||
|
image::codecs::jpeg::JpegEncoder::new(&mut cover)
|
||||||
|
.encode_image(&image::RgbImage::new(300, 300))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let drive = FakeDrive::new();
|
||||||
|
drive.device().write("track one.mp3", &song);
|
||||||
|
drive.device().write("Cover.jpg", &cover);
|
||||||
|
let client = drive.client(1, Faults::lan());
|
||||||
|
let daemon = Daemon::start(&dir.join("daemon"), client).unwrap();
|
||||||
|
|
||||||
|
let landed = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let mut subscription = control::subscribe(&daemon.socket).unwrap();
|
||||||
|
let flag = landed.clone();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
// The first change names every topic: a new subscription catches up.
|
||||||
|
let mut caught_up = false;
|
||||||
|
while let Ok(Some(event)) = subscription.next_event() {
|
||||||
|
if let control::Event::Changed { topics } = event {
|
||||||
|
if caught_up && topics.contains(&control::Topic::Thumbnails) {
|
||||||
|
flag.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
caught_up = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let Ok(Response::Entries { entries }) = daemon.request(&Request::ListDir {
|
||||||
|
path: String::new(),
|
||||||
|
}) else {
|
||||||
|
panic!("listing");
|
||||||
|
};
|
||||||
|
let song = entries
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.name == "track one.mp3")
|
||||||
|
.expect("the song is listed");
|
||||||
|
let items = vec![control::FileThumbRequest {
|
||||||
|
uid: song.uid.clone(),
|
||||||
|
modified: song.modified,
|
||||||
|
name: song.name.clone(),
|
||||||
|
}];
|
||||||
|
let Ok(Response::FileThumbGeneration { generation }) =
|
||||||
|
daemon.request(&Request::ReserveFileThumbGeneration)
|
||||||
|
else {
|
||||||
|
panic!("generation");
|
||||||
|
};
|
||||||
|
let mut path = None;
|
||||||
|
let settled = wait_until(Duration::from_secs(60), || {
|
||||||
|
let Ok(Response::Thumbs { items: replies }) = daemon.request(&Request::FileThumbs {
|
||||||
|
items: items.clone(),
|
||||||
|
generation,
|
||||||
|
}) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
path = replies.first().and_then(|reply| reply.path.clone());
|
||||||
|
path.is_some()
|
||||||
|
});
|
||||||
|
assert!(settled, "the song got no thumbnail");
|
||||||
|
let thumb = image::load_from_memory(&std::fs::read(path.unwrap()).unwrap()).unwrap();
|
||||||
|
assert_eq!(thumb.width(), thumb.height(), "the square cover");
|
||||||
|
assert!(
|
||||||
|
wait_until(Duration::from_secs(15), || landed
|
||||||
|
.load(std::sync::atomic::Ordering::SeqCst)),
|
||||||
|
"no thumbnails change was published"
|
||||||
|
);
|
||||||
|
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
}
|
||||||
|
|
||||||
/// A video in My files gets a thumbnail from a frame of its own, but a
|
/// A video in My files gets a thumbnail from a frame of its own, but a
|
||||||
/// recording can be gigabytes: only its start, and an MP4's index, are
|
/// recording can be gigabytes: only its start, and an MP4's index, are
|
||||||
/// downloaded for it. A video with a thumbnail from its uploader is not
|
/// downloaded for it. A video with a thumbnail from its uploader is not
|
||||||
|
|||||||
+894
-239
File diff suppressed because it is too large.
Load diff
@@ -63,6 +63,7 @@ scale.browser-status-meter trough { min-width: 140px; min-height: 6px; }
|
|||||||
.photo-viewer-window { background-color: #111014; }
|
.photo-viewer-window { background-color: #111014; }
|
||||||
.media-player-window video { background-color: black; }
|
.media-player-window video { background-color: black; }
|
||||||
.media-player-note { color: rgba(255, 255, 255, 0.5); }
|
.media-player-note { color: rgba(255, 255, 255, 0.5); }
|
||||||
|
.media-player-note .file-thumbnail { border-radius: 12px; }
|
||||||
.viewer-top-bar { background: linear-gradient(to bottom, rgba(0, 0, 0, 0.85), rgba(0, 0, 0, 0.45) 60%, rgba(0, 0, 0, 0)); padding: 10px 16px 40px 20px; color: white; }
|
.viewer-top-bar { background: linear-gradient(to bottom, rgba(0, 0, 0, 0.85), rgba(0, 0, 0, 0.45) 60%, rgba(0, 0, 0, 0)); padding: 10px 16px 40px 20px; color: white; }
|
||||||
.viewer-title { font-weight: 700; font-size: 1.05rem; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); color: white; }
|
.viewer-title { font-weight: 700; font-size: 1.05rem; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); color: white; }
|
||||||
.viewer-counter { font-size: 0.8rem; color: rgba(255, 255, 255, 0.72); text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); }
|
.viewer-counter { font-size: 0.8rem; color: rgba(255, 255, 255, 0.72); text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); }
|
||||||
|
|||||||
@@ -128,6 +128,10 @@ fn follow_changes(ui: &Rc<Ui>, state: &FeedState, topics: &[Topic]) {
|
|||||||
if changed(Topic::Shares) {
|
if changed(Topic::Shares) {
|
||||||
refresh_invitations_badge(ui, Duration::ZERO);
|
refresh_invitations_badge(ui, Duration::ZERO);
|
||||||
}
|
}
|
||||||
|
// On any page: a grid, a list or a player may show a tile still blank.
|
||||||
|
if changed(Topic::Thumbnails) {
|
||||||
|
refresh_file_thumbnails(ui);
|
||||||
|
}
|
||||||
let page = ui.stack.visible_child_name();
|
let page = ui.stack.visible_child_name();
|
||||||
// A timeline out of sight is read again when the user comes back to it;
|
// A timeline out of sight is read again when the user comes back to it;
|
||||||
// one on screen is left alone (below).
|
// one on screen is left alone (below).
|
||||||
|
|||||||
@@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState};
|
|||||||
|
|
||||||
use pdfs_core::control::{
|
use pdfs_core::control::{
|
||||||
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
|
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
|
||||||
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress,
|
DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem,
|
||||||
PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope,
|
LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo,
|
||||||
Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry,
|
PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo,
|
||||||
ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
|
SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase,
|
||||||
ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||||
};
|
};
|
||||||
|
|
||||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||||
@@ -668,9 +668,11 @@ fn build_window(app: &adw::Application) {
|
|||||||
progress_bar: takeout_widgets.progress_bar.clone(),
|
progress_bar: takeout_widgets.progress_bar.clone(),
|
||||||
summary_group: takeout_widgets.summary_group.clone(),
|
summary_group: takeout_widgets.summary_group.clone(),
|
||||||
summary_rows: RefCell::new(Vec::new()),
|
summary_rows: RefCell::new(Vec::new()),
|
||||||
|
resume_group: takeout_widgets.resume_group.clone(),
|
||||||
|
resume_row: takeout_widgets.resume_row.clone(),
|
||||||
|
resume_button: takeout_widgets.resume_button.clone(),
|
||||||
inflight: Cell::new(false),
|
inflight: Cell::new(false),
|
||||||
running: Cell::new(false),
|
running: Cell::new(false),
|
||||||
dry_run: Cell::new(false),
|
|
||||||
},
|
},
|
||||||
update,
|
update,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1115,9 +1115,6 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
|||||||
// count falling to zero is what "the sync finished" looks like from here.
|
// count falling to zero is what "the sync finished" looks like from here.
|
||||||
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
||||||
// starts its upload job mid-flight, which is not a thing finishing.
|
// starts its upload job mid-flight, which is not a thing finishing.
|
||||||
// The Import page reads its progress off the same snapshot rather than
|
|
||||||
// polling the daemon a second time for numbers already on the wire.
|
|
||||||
takeout_progress(ui, jobs);
|
|
||||||
|
|
||||||
let previous = ui.status.active_transfers.replace(items.len());
|
let previous = ui.status.active_transfers.replace(items.len());
|
||||||
if !items.is_empty() {
|
if !items.is_empty() {
|
||||||
@@ -1211,13 +1208,18 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
|||||||
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
|
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
|
||||||
let done = j.done.to_string();
|
let done = j.done.to_string();
|
||||||
let total = j.total.to_string();
|
let total = j.total.to_string();
|
||||||
|
let (title, detail) = if j.title == IMPORT_JOB {
|
||||||
|
import_job_text(&j.detail)
|
||||||
|
} else {
|
||||||
|
(j.title.clone(), j.detail.clone())
|
||||||
|
};
|
||||||
let args: [(&str, &str); 4] = [
|
let args: [(&str, &str); 4] = [
|
||||||
("title", &j.title),
|
("title", &title),
|
||||||
("detail", &j.detail),
|
("detail", &detail),
|
||||||
("done", &done),
|
("done", &done),
|
||||||
("total", &total),
|
("total", &total),
|
||||||
];
|
];
|
||||||
let text = match (j.total > 0, j.detail.is_empty()) {
|
let text = match (j.total > 0, detail.is_empty()) {
|
||||||
// Translators: a background job's progress. {title} is the job, such as
|
// Translators: a background job's progress. {title} is the job, such as
|
||||||
// "Scanning Photos"; {done} and {total} are counts.
|
// "Scanning Photos"; {done} and {total} are counts.
|
||||||
(true, true) => gettext_f("{title} — {done} of {total}", &args),
|
(true, true) => gettext_f("{title} — {done} of {total}", &args),
|
||||||
|
|||||||
@@ -15,17 +15,30 @@
|
|||||||
//! Import — staging is local, so a mis-drop costs nothing.
|
//! Import — staging is local, so a mis-drop costs nothing.
|
||||||
//!
|
//!
|
||||||
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
||||||
//! stages paths, launches, asks [`Request::ImportStatus`] for the final report,
|
//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the
|
||||||
//! and reads live progress off the import's job in the transfer snapshot the
|
//! running counts and the final report. It asks again whenever the import's job
|
||||||
//! daemon already pushes (see [`takeout_progress`]).
|
//! moves in the transfer snapshot the daemon already pushes. A run the service
|
||||||
|
//! did not get to finish can be picked up with [`Request::ResumeImport`].
|
||||||
|
|
||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
||||||
/// Matching on it is how this page borrows the progress the daemon already
|
/// Seeing it move is the cue to ask for the import's status; the Activity list
|
||||||
/// pushes instead of asking for it separately.
|
/// shows it under its translated name ([`import_job_text`]).
|
||||||
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
|
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
|
||||||
|
|
||||||
|
/// The import job's title and detail in the user's language. The daemon names
|
||||||
|
/// its steps in English; the file being uploaded passes through as it is.
|
||||||
|
pub(crate) fn import_job_text(detail: &str) -> (String, String) {
|
||||||
|
let detail = match detail {
|
||||||
|
"reading archives" => gettext("Reading the archives"),
|
||||||
|
"checking for photos you already have" => gettext("Checking for photos you already have"),
|
||||||
|
"sorting photos into albums" => gettext("Sorting photos into albums"),
|
||||||
|
other => other.to_string(),
|
||||||
|
};
|
||||||
|
(gettext("Importing from Google Photos"), detail)
|
||||||
|
}
|
||||||
|
|
||||||
/// What the archive list says before anything is staged. `repaint_archives`
|
/// What the archive list says before anything is staged. `repaint_archives`
|
||||||
/// swaps in the count and total once there is something to report, and swaps
|
/// swaps in the count and total once there is something to report, and swaps
|
||||||
/// this back when the list is emptied again.
|
/// this back when the list is emptied again.
|
||||||
@@ -95,18 +108,19 @@ pub(crate) struct TakeoutState {
|
|||||||
pub(crate) progress_group: adw::PreferencesGroup,
|
pub(crate) progress_group: adw::PreferencesGroup,
|
||||||
pub(crate) progress_label: gtk4::Label,
|
pub(crate) progress_label: gtk4::Label,
|
||||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||||
/// Report group for the last finished run, hidden until there is one.
|
/// The running counts, or the report for the last finished run; hidden
|
||||||
|
/// until there is one.
|
||||||
pub(crate) summary_group: adw::PreferencesGroup,
|
pub(crate) summary_group: adw::PreferencesGroup,
|
||||||
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
||||||
|
/// Offers to resume an import the service did not get to finish.
|
||||||
|
pub(crate) resume_group: adw::PreferencesGroup,
|
||||||
|
pub(crate) resume_row: adw::ActionRow,
|
||||||
|
pub(crate) resume_button: gtk4::Button,
|
||||||
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
||||||
/// can't stack worker threads on the 2s tick.
|
/// can't stack worker threads on the 2s tick.
|
||||||
pub(crate) inflight: Cell<bool>,
|
pub(crate) inflight: Cell<bool>,
|
||||||
/// Whether the daemon reported an import running on the last poll.
|
/// Whether the daemon reported an import running on the last poll.
|
||||||
pub(crate) running: Cell<bool>,
|
pub(crate) running: Cell<bool>,
|
||||||
/// Whether the run in flight was launched as a dry run, so the report can
|
|
||||||
/// say "would upload" rather than claiming photos were sent. The daemon's
|
|
||||||
/// summary carries no such flag — it is the same shape either way.
|
|
||||||
pub(crate) dry_run: Cell<bool>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Widgets the Import page hands back for wiring.
|
/// Widgets the Import page hands back for wiring.
|
||||||
@@ -126,6 +140,9 @@ pub(crate) struct TakeoutWidgets {
|
|||||||
pub(crate) progress_label: gtk4::Label,
|
pub(crate) progress_label: gtk4::Label,
|
||||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||||
pub(crate) summary_group: adw::PreferencesGroup,
|
pub(crate) summary_group: adw::PreferencesGroup,
|
||||||
|
pub(crate) resume_group: adw::PreferencesGroup,
|
||||||
|
pub(crate) resume_row: adw::ActionRow,
|
||||||
|
pub(crate) resume_button: gtk4::Button,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||||
@@ -261,7 +278,23 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
|||||||
.visible(false)
|
.visible(false)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
// An import cut short by a restart or a lost connection, offered back at the
|
||||||
|
// top of the page: picking the archives again is exactly what a reboot in
|
||||||
|
// the middle of a migration should not cost.
|
||||||
|
let resume_button = gtk4::Button::builder()
|
||||||
|
.label(pgettext("verb", "Resume"))
|
||||||
|
.valign(gtk4::Align::Center)
|
||||||
|
.build();
|
||||||
|
resume_button.add_css_class("suggested-action");
|
||||||
|
let resume_row = adw::ActionRow::builder()
|
||||||
|
.title(gettext("The last import didn't finish"))
|
||||||
|
.build();
|
||||||
|
resume_row.add_suffix(&resume_button);
|
||||||
|
let resume_group = adw::PreferencesGroup::builder().visible(false).build();
|
||||||
|
resume_group.add(&resume_row);
|
||||||
|
|
||||||
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
||||||
|
groups.append(&resume_group);
|
||||||
groups.append(&dropzone);
|
groups.append(&dropzone);
|
||||||
groups.append(&list_group);
|
groups.append(&list_group);
|
||||||
groups.append(&actions);
|
groups.append(&actions);
|
||||||
@@ -301,6 +334,9 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
|||||||
progress_label: progress_label.clone(),
|
progress_label: progress_label.clone(),
|
||||||
progress_bar: progress_bar.clone(),
|
progress_bar: progress_bar.clone(),
|
||||||
summary_group: summary_group.clone(),
|
summary_group: summary_group.clone(),
|
||||||
|
resume_group,
|
||||||
|
resume_row,
|
||||||
|
resume_button,
|
||||||
};
|
};
|
||||||
(frame.upcast(), widgets)
|
(frame.upcast(), widgets)
|
||||||
}
|
}
|
||||||
@@ -427,6 +463,28 @@ pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let ui_resume = ui.clone();
|
||||||
|
widgets.resume_button.connect_clicked(move |_| {
|
||||||
|
let ui = ui_resume.clone();
|
||||||
|
ui.busy_begin();
|
||||||
|
let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport);
|
||||||
|
glib::spawn_future_local(async move {
|
||||||
|
let result = rx.recv().await;
|
||||||
|
ui.busy_end();
|
||||||
|
match result {
|
||||||
|
Ok(Ok(Response::Ok { .. })) => import_started(&ui, false),
|
||||||
|
Ok(Ok(Response::Error { message, kind })) => {
|
||||||
|
toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind);
|
||||||
|
}
|
||||||
|
_ => toast_error(
|
||||||
|
&ui,
|
||||||
|
&gettext("Couldn't resume the import"),
|
||||||
|
&gettext("The Proton Drive service didn't respond."),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
let ui_back = ui.clone();
|
let ui_back = ui.clone();
|
||||||
widgets
|
widgets
|
||||||
.back_button
|
.back_button
|
||||||
@@ -559,6 +617,7 @@ fn sync_takeout_actions(ui: &Rc<Ui>) {
|
|||||||
.import_button
|
.import_button
|
||||||
.set_sensitive(has_files && !running);
|
.set_sensitive(has_files && !running);
|
||||||
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
||||||
|
ui.takeout.resume_button.set_sensitive(!running);
|
||||||
ui.takeout.cancel_button.set_visible(running);
|
ui.takeout.cancel_button.set_visible(running);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -577,7 +636,6 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
ui.takeout.dry_run.set(dry_run);
|
|
||||||
ui.busy_begin();
|
ui.busy_begin();
|
||||||
let rx = spawn_request(
|
let rx = spawn_request(
|
||||||
ui.dirs.control_socket(),
|
ui.dirs.control_socket(),
|
||||||
@@ -588,27 +646,7 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
|||||||
let result = rx.recv().await;
|
let result = rx.recv().await;
|
||||||
ui.busy_end();
|
ui.busy_end();
|
||||||
match result {
|
match result {
|
||||||
Ok(Ok(Response::Ok { .. })) => {
|
Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run),
|
||||||
ui.takeout.running.set(true);
|
|
||||||
ui.gallery.import_banner.set_revealed(true);
|
|
||||||
ui.takeout.summary_group.set_visible(false);
|
|
||||||
ui.takeout.progress_group.set_visible(true);
|
|
||||||
ui.takeout.progress_bar.set_fraction(0.0);
|
|
||||||
ui.takeout.progress_label.set_text(&if dry_run {
|
|
||||||
gettext("Reading archives…")
|
|
||||||
} else {
|
|
||||||
gettext("Starting import…")
|
|
||||||
});
|
|
||||||
sync_takeout_actions(&ui);
|
|
||||||
toast(
|
|
||||||
&ui,
|
|
||||||
&if dry_run {
|
|
||||||
gettext("Scanning the export…")
|
|
||||||
} else {
|
|
||||||
gettext("Import started — you can leave this page, it keeps running.")
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(Ok(Response::Error { message, kind })) => {
|
Ok(Ok(Response::Error { message, kind })) => {
|
||||||
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
|
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
|
||||||
}
|
}
|
||||||
@@ -621,9 +659,39 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ask the daemon for the import's state, and paint the report once a run has
|
/// Switch the page over to reporting progress once the daemon took the run.
|
||||||
/// finished. Asked on arrival and while import progress keeps arriving. Cheap
|
fn import_started(ui: &Rc<Ui>, dry_run: bool) {
|
||||||
/// and inflight-guarded.
|
ui.takeout.running.set(true);
|
||||||
|
if !dry_run {
|
||||||
|
ui.gallery.import_banner.set_revealed(true);
|
||||||
|
}
|
||||||
|
ui.takeout.resume_group.set_visible(false);
|
||||||
|
ui.takeout.summary_group.set_visible(false);
|
||||||
|
ui.takeout.progress_group.set_visible(true);
|
||||||
|
ui.takeout.progress_group.set_title(&if dry_run {
|
||||||
|
gettext("Scan in progress")
|
||||||
|
} else {
|
||||||
|
gettext("Import in progress")
|
||||||
|
});
|
||||||
|
ui.takeout.progress_bar.set_fraction(0.0);
|
||||||
|
ui.takeout
|
||||||
|
.progress_label
|
||||||
|
.set_text(&gettext("Reading the archives…"));
|
||||||
|
sync_takeout_actions(ui);
|
||||||
|
toast(
|
||||||
|
ui,
|
||||||
|
&if dry_run {
|
||||||
|
gettext("Scanning the export…")
|
||||||
|
} else {
|
||||||
|
gettext("Import started — you can leave this page, it keeps running.")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask the daemon for the import's state and paint it: the step and counts
|
||||||
|
/// while it runs, the report once it has ended, and the offer to resume a run
|
||||||
|
/// that did not finish. Asked on arrival and while import progress keeps
|
||||||
|
/// arriving. Cheap and inflight-guarded.
|
||||||
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||||
if ui.takeout.inflight.get() {
|
if ui.takeout.inflight.get() {
|
||||||
return;
|
return;
|
||||||
@@ -634,124 +702,250 @@ pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
|||||||
glib::spawn_future_local(async move {
|
glib::spawn_future_local(async move {
|
||||||
let result = rx.recv().await;
|
let result = rx.recv().await;
|
||||||
ui.takeout.inflight.set(false);
|
ui.takeout.inflight.set(false);
|
||||||
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
|
let Ok(Ok(Response::ImportStatus {
|
||||||
|
running,
|
||||||
|
summary,
|
||||||
|
progress,
|
||||||
|
resumable,
|
||||||
|
})) = result
|
||||||
|
else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let was_running = ui.takeout.running.replace(running);
|
let was_running = ui.takeout.running.replace(running);
|
||||||
ui.gallery.import_banner.set_revealed(running);
|
let dry = summary.as_ref().is_some_and(|s| s.dry_run);
|
||||||
|
ui.gallery.import_banner.set_revealed(running && !dry);
|
||||||
|
ui.takeout.progress_group.set_visible(running);
|
||||||
if running {
|
if running {
|
||||||
ui.takeout.progress_group.set_visible(true);
|
ui.takeout.progress_group.set_title(&if dry {
|
||||||
} else {
|
gettext("Scan in progress")
|
||||||
ui.takeout.progress_group.set_visible(false);
|
} else {
|
||||||
// An import runs for hours, so its ending is announced rather than
|
gettext("Import in progress")
|
||||||
// left to be discovered: a toast for whoever is looking at the
|
});
|
||||||
// window, and a desktop notification for whoever is not.
|
paint_progress(&ui, progress.as_ref());
|
||||||
let failed = summary.as_ref().and_then(|s| s.error.clone());
|
} else if was_running && let Some(summary) = &summary {
|
||||||
if was_running && let Some(error) = &failed {
|
announce_end(&ui, summary);
|
||||||
let headline = gettext("Couldn't import from Google Photos");
|
|
||||||
toast_error(&ui, &headline, error);
|
|
||||||
notify("takeout-import", &headline, error);
|
|
||||||
} else if was_running {
|
|
||||||
let dry = ui.takeout.dry_run.get();
|
|
||||||
let headline = if dry {
|
|
||||||
gettext("Scan finished")
|
|
||||||
} else {
|
|
||||||
gettext("Import finished")
|
|
||||||
};
|
|
||||||
toast(&ui, &headline);
|
|
||||||
if !dry {
|
|
||||||
let body = match &summary {
|
|
||||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
|
||||||
Some(s) if s.failed > 0 => ngettext_f(
|
|
||||||
"{n} photo added, {failed} failed.",
|
|
||||||
"{n} photos added, {failed} failed.",
|
|
||||||
s.uploaded as u64,
|
|
||||||
&[("failed", &s.failed.to_string())],
|
|
||||||
),
|
|
||||||
Some(s) if s.cancelled => ngettext_f(
|
|
||||||
"Stopped after {n} photo.",
|
|
||||||
"Stopped after {n} photos.",
|
|
||||||
s.uploaded as u64,
|
|
||||||
&[],
|
|
||||||
),
|
|
||||||
Some(s) => ngettext_f(
|
|
||||||
"{n} photo added to Proton Photos.",
|
|
||||||
"{n} photos added to Proton Photos.",
|
|
||||||
s.uploaded as u64,
|
|
||||||
&[],
|
|
||||||
),
|
|
||||||
None => gettext("The Google Photos import has finished."),
|
|
||||||
};
|
|
||||||
notify("takeout-import", &headline, &body);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if was_running != running {
|
if was_running != running {
|
||||||
sync_takeout_actions(&ui);
|
sync_takeout_actions(&ui);
|
||||||
}
|
}
|
||||||
match summary {
|
match &summary {
|
||||||
Some(summary) => repaint_summary(&ui, &summary),
|
Some(summary) => repaint_summary(&ui, summary, running),
|
||||||
None => ui.takeout.summary_group.set_visible(false),
|
None => ui.takeout.summary_group.set_visible(false),
|
||||||
}
|
}
|
||||||
|
paint_resume(&ui, running, summary.as_ref(), &resumable);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Paint the last run's report.
|
/// Tell the user how a run ended. An import runs for hours, so its ending is
|
||||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
/// announced rather than left to be discovered: a toast for whoever is looking
|
||||||
|
/// at the window, and a desktop notification for whoever is not.
|
||||||
|
fn announce_end(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||||
|
if summary.dry_run {
|
||||||
|
toast(ui, &gettext("Scan finished"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(error) = &summary.error {
|
||||||
|
let kind = summary.error_kind.unwrap_or_default();
|
||||||
|
let headline = gettext("Couldn't import from Google Photos");
|
||||||
|
toast_failure(ui, &headline, error, kind);
|
||||||
|
notify(
|
||||||
|
"takeout-import",
|
||||||
|
&headline,
|
||||||
|
&gettext("You can resume it once the problem is solved."),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let (headline, body) = if summary.interrupted {
|
||||||
|
(
|
||||||
|
gettext("Import interrupted"),
|
||||||
|
gettext("You can resume it where it stopped."),
|
||||||
|
)
|
||||||
|
} else if summary.cancelled {
|
||||||
|
(
|
||||||
|
gettext("Import stopped"),
|
||||||
|
ngettext_f(
|
||||||
|
"Stopped after {n} photo.",
|
||||||
|
"Stopped after {n} photos.",
|
||||||
|
summary.uploaded as u64,
|
||||||
|
&[],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
} else if summary.failed > 0 {
|
||||||
|
(
|
||||||
|
gettext("Import finished"),
|
||||||
|
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||||
|
ngettext_f(
|
||||||
|
"{n} photo added, {failed} failed.",
|
||||||
|
"{n} photos added, {failed} failed.",
|
||||||
|
summary.uploaded as u64,
|
||||||
|
&[("failed", &summary.failed.to_string())],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
gettext("Import finished"),
|
||||||
|
ngettext_f(
|
||||||
|
"{n} photo added to Proton Photos.",
|
||||||
|
"{n} photos added to Proton Photos.",
|
||||||
|
summary.uploaded as u64,
|
||||||
|
&[],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
toast(ui, &headline);
|
||||||
|
notify("takeout-import", &headline, &body);
|
||||||
|
|
||||||
|
// A complete import is done with its archives; keeping them staged would
|
||||||
|
// only invite running the same export again.
|
||||||
|
if summary.complete() {
|
||||||
|
ui.takeout.archives.borrow_mut().clear();
|
||||||
|
repaint_archives(ui);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paint where a running import is.
|
||||||
|
fn paint_progress(ui: &Rc<Ui>, progress: Option<&ImportProgress>) {
|
||||||
|
let Some(progress) = progress else {
|
||||||
|
ui.takeout.progress_bar.pulse();
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let done = progress.done.to_string();
|
||||||
|
let total = progress.total.to_string();
|
||||||
|
let current = progress.current.clone().unwrap_or_default();
|
||||||
|
let counted = progress.total > 0;
|
||||||
|
let args = [
|
||||||
|
("done", done.as_str()),
|
||||||
|
("total", total.as_str()),
|
||||||
|
("name", current.as_str()),
|
||||||
|
];
|
||||||
|
let text = match (progress.phase, counted) {
|
||||||
|
(ImportPhase::Scanning, _) => gettext("Reading the archives…"),
|
||||||
|
(ImportPhase::Checking, false) => gettext("Checking for photos you already have…"),
|
||||||
|
// Translators: {done} and {total} are counts of photos.
|
||||||
|
(ImportPhase::Checking, true) => gettext_f(
|
||||||
|
"Checking for photos you already have — {done} of {total}",
|
||||||
|
&args,
|
||||||
|
),
|
||||||
|
(ImportPhase::Uploading, false) => gettext("Uploading…"),
|
||||||
|
(ImportPhase::Uploading, true) if current.is_empty() => {
|
||||||
|
// Translators: {done} and {total} are counts of photos.
|
||||||
|
gettext_f("Uploading {done} of {total}", &args)
|
||||||
|
}
|
||||||
|
// Translators: {done} and {total} are counts of photos; {name} is the file being uploaded.
|
||||||
|
(ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args),
|
||||||
|
(ImportPhase::Albums, false) => gettext("Sorting photos into albums…"),
|
||||||
|
// Translators: {done} and {total} are counts of albums.
|
||||||
|
(ImportPhase::Albums, true) => {
|
||||||
|
gettext_f("Sorting photos into albums — {done} of {total}", &args)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ui.takeout.progress_label.set_text(&text);
|
||||||
|
if counted {
|
||||||
|
ui.takeout
|
||||||
|
.progress_bar
|
||||||
|
.set_fraction((progress.done as f64 / progress.total as f64).min(1.0));
|
||||||
|
} else {
|
||||||
|
ui.takeout.progress_bar.pulse();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Offer to resume the last run when the service says it can be.
|
||||||
|
fn paint_resume(ui: &Rc<Ui>, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) {
|
||||||
|
let offer = !running && !resumable.is_empty();
|
||||||
|
ui.takeout.resume_group.set_visible(offer);
|
||||||
|
if !offer {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let uploaded = summary.map_or(0, |s| s.uploaded) as u64;
|
||||||
|
// Translators: {n} is the number of photos uploaded before the import stopped.
|
||||||
|
ui.takeout.resume_row.set_subtitle(&ngettext_f(
|
||||||
|
"{n} photo uploaded before it stopped. Resuming skips what is already there.",
|
||||||
|
"{n} photos uploaded before it stopped. Resuming skips what is already there.",
|
||||||
|
uploaded,
|
||||||
|
&[],
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paint the counts of a running import, or the report of the last one.
|
||||||
|
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
|
||||||
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
||||||
ui.takeout.summary_group.remove(&row);
|
ui.takeout.summary_group.remove(&row);
|
||||||
}
|
}
|
||||||
let dry = ui.takeout.dry_run.get();
|
let dry = summary.dry_run;
|
||||||
ui.takeout.summary_group.set_title(&if dry {
|
ui.takeout.summary_group.set_title(&if running {
|
||||||
|
gettext("So far")
|
||||||
|
} else if dry {
|
||||||
gettext("Scan result")
|
gettext("Scan result")
|
||||||
} else {
|
} else {
|
||||||
gettext("Last import")
|
gettext("Last import")
|
||||||
});
|
});
|
||||||
ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error {
|
let description = if running {
|
||||||
// The reason is the service's own text, which is not translated. A
|
None
|
||||||
// group's description is markup, and the reason may name a path.
|
} else if let Some(error) = &summary.error {
|
||||||
glib::markup_escape_text(&gettext_f(
|
// The reason is the service's own text, which is not translated; the
|
||||||
"{what}: {detail}",
|
// headline in front of it is. A group's description is markup, and the
|
||||||
&[
|
// reason may name a path.
|
||||||
("what", &gettext("Couldn't import from Google Photos")),
|
let what = gettext("Couldn't import from Google Photos");
|
||||||
("detail", error),
|
Some(
|
||||||
],
|
glib::markup_escape_text(&gettext_f(
|
||||||
))
|
"{what}: {detail}",
|
||||||
.to_string()
|
&[
|
||||||
|
(
|
||||||
|
"what",
|
||||||
|
error_headline(summary.error_kind.unwrap_or_default(), &what),
|
||||||
|
),
|
||||||
|
("detail", error),
|
||||||
|
],
|
||||||
|
))
|
||||||
|
.to_string(),
|
||||||
|
)
|
||||||
} else if dry {
|
} else if dry {
|
||||||
gettext("Nothing was uploaded — this is what an import would do.")
|
Some(gettext(
|
||||||
|
"Nothing was uploaded — this is what an import would do.",
|
||||||
|
))
|
||||||
|
} else if summary.interrupted {
|
||||||
|
Some(gettext(
|
||||||
|
"The Proton Drive service stopped while the import ran. Resume it to continue where it left off.",
|
||||||
|
))
|
||||||
} else if summary.cancelled {
|
} else if summary.cancelled {
|
||||||
gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.")
|
Some(gettext(
|
||||||
|
"Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.",
|
||||||
|
))
|
||||||
|
} else if !summary.complete() {
|
||||||
|
Some(gettext(
|
||||||
|
"Finished with problems. Run it again to retry what didn't upload.",
|
||||||
|
))
|
||||||
} else {
|
} else {
|
||||||
gettext("Finished.")
|
Some(gettext("Finished."))
|
||||||
}));
|
};
|
||||||
|
ui.takeout
|
||||||
|
.summary_group
|
||||||
|
.set_description(description.as_deref());
|
||||||
|
|
||||||
// Only lines that carry information: a run with no failures should not have
|
// Only lines that carry information: a run with no failures should not have
|
||||||
// to show a "0 failed" row for the user to read past.
|
// to show a "0 failed" row for the user to read past.
|
||||||
let mut lines: Vec<(String, String)> = vec![
|
let mut lines: Vec<(String, String)> =
|
||||||
(gettext("Photos in the export"), summary.found.to_string()),
|
vec![(gettext("Photos in the export"), summary.found.to_string())];
|
||||||
(
|
if dry {
|
||||||
if dry {
|
lines.push((gettext("Would upload"), summary.to_upload.to_string()));
|
||||||
gettext("Would upload")
|
} else {
|
||||||
} else {
|
lines.push((gettext("Uploaded"), summary.uploaded.to_string()));
|
||||||
gettext("Uploaded")
|
}
|
||||||
},
|
|
||||||
summary.uploaded.to_string(),
|
|
||||||
),
|
|
||||||
];
|
|
||||||
if summary.duplicates > 0 {
|
if summary.duplicates > 0 {
|
||||||
lines.push((
|
lines.push((
|
||||||
gettext("Already in Proton Photos"),
|
gettext("Already in Proton Photos"),
|
||||||
summary.duplicates.to_string(),
|
summary.duplicates.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if summary.live_photos > 0 {
|
||||||
|
lines.push((
|
||||||
|
gettext("Live photos kept together"),
|
||||||
|
summary.live_photos.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
if summary.albums_created > 0 {
|
if summary.albums_created > 0 {
|
||||||
lines.push((
|
lines.push((
|
||||||
if dry {
|
gettext("Albums created"),
|
||||||
gettext("Albums to create")
|
|
||||||
} else {
|
|
||||||
gettext("Albums created")
|
|
||||||
},
|
|
||||||
summary.albums_created.to_string(),
|
summary.albums_created.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
@@ -767,25 +961,31 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
|||||||
summary.skipped_trashed.to_string(),
|
summary.skipped_trashed.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if summary.bytes > 0 {
|
if dry && summary.to_upload_bytes > 0 {
|
||||||
lines.push((
|
lines.push((
|
||||||
if dry {
|
gettext("Data to upload"),
|
||||||
gettext("Data to upload")
|
human_bytes(summary.to_upload_bytes),
|
||||||
} else {
|
|
||||||
gettext("Data uploaded")
|
|
||||||
},
|
|
||||||
human_bytes(summary.bytes),
|
|
||||||
));
|
));
|
||||||
|
} else if summary.bytes > 0 {
|
||||||
|
lines.push((gettext("Data uploaded"), human_bytes(summary.bytes)));
|
||||||
}
|
}
|
||||||
if summary.failed > 0 {
|
if summary.failed > 0 {
|
||||||
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
|
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
|
||||||
}
|
}
|
||||||
|
for path in &summary.unreadable_archives {
|
||||||
|
let name = Path::new(path)
|
||||||
|
.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| path.clone());
|
||||||
|
lines.push((gettext("Couldn't read this archive"), name));
|
||||||
|
}
|
||||||
|
|
||||||
let mut rows = Vec::new();
|
let mut rows = Vec::new();
|
||||||
for (title, value) in lines {
|
for (title, value) in lines {
|
||||||
let row = adw::ActionRow::builder()
|
let row = adw::ActionRow::builder()
|
||||||
.title(title)
|
.title(title)
|
||||||
.subtitle(value)
|
.subtitle(value)
|
||||||
|
.use_markup(false)
|
||||||
.build();
|
.build();
|
||||||
row.add_css_class("property");
|
row.add_css_class("property");
|
||||||
ui.takeout.summary_group.add(&row);
|
ui.takeout.summary_group.add(&row);
|
||||||
@@ -794,35 +994,3 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
|||||||
*ui.takeout.summary_rows.borrow_mut() = rows;
|
*ui.takeout.summary_rows.borrow_mut() = rows;
|
||||||
ui.takeout.summary_group.set_visible(true);
|
ui.takeout.summary_group.set_visible(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Paint the import's progress from the transfer snapshot the daemon pushed,
|
|
||||||
/// so this page costs no extra request. A job with no known total
|
|
||||||
/// pulses instead of showing a fraction, the same as the Activity list.
|
|
||||||
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
|
|
||||||
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
ui.takeout.progress_group.set_visible(true);
|
|
||||||
if job.total > 0 {
|
|
||||||
ui.takeout
|
|
||||||
.progress_bar
|
|
||||||
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
|
|
||||||
let done = job.done.to_string();
|
|
||||||
let total = job.total.to_string();
|
|
||||||
// Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on.
|
|
||||||
let text = gettext_f(
|
|
||||||
"{done} of {total} — {detail}",
|
|
||||||
&[("done", &done), ("total", &total), ("detail", &job.detail)],
|
|
||||||
);
|
|
||||||
ui.takeout.progress_label.set_text(&text);
|
|
||||||
} else {
|
|
||||||
ui.takeout.progress_bar.pulse();
|
|
||||||
ui.takeout
|
|
||||||
.progress_label
|
|
||||||
.set_text(&if job.detail.is_empty() {
|
|
||||||
gettext("Working…")
|
|
||||||
} else {
|
|
||||||
job.detail.clone()
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -19,6 +19,9 @@ const SEEK_SHORT: i64 = 5;
|
|||||||
/// How much ↑ and ↓ change the volume in the My files player.
|
/// How much ↑ and ↓ change the volume in the My files player.
|
||||||
const VOLUME_STEP: f64 = 0.1;
|
const VOLUME_STEP: f64 = 0.1;
|
||||||
|
|
||||||
|
/// The edge of a song's cover in the My files player, in logical pixels.
|
||||||
|
const COVER_SIZE: i32 = 320;
|
||||||
|
|
||||||
thread_local! {
|
thread_local! {
|
||||||
/// The volume and mute the user last set, applied to the next stream so
|
/// The volume and mute the user last set, applied to the next stream so
|
||||||
/// muting one video doesn't leave the next one blaring. Kept for the run
|
/// muting one video doesn't leave the next one blaring. Kept for the run
|
||||||
@@ -202,16 +205,21 @@ fn build_media_player(ui: &Rc<Ui>, entry: &DirEntry, path: &Path) {
|
|||||||
wire_player(&video, >k_window);
|
wire_player(&video, >k_window);
|
||||||
|
|
||||||
// A song has no picture, which would leave a black window above the
|
// A song has no picture, which would leave a black window above the
|
||||||
// controls: a note stands in for it.
|
// controls: its cover stands in for it, the one its listing tile shows,
|
||||||
let note = gtk4::Image::builder()
|
// and a note while there is none.
|
||||||
.icon_name("audio-x-generic-symbolic")
|
let note = file_thumbnail_widget(COVER_SIZE, 128);
|
||||||
.pixel_size(128)
|
note.set_can_target(false);
|
||||||
.halign(gtk4::Align::Center)
|
note.set_visible(false);
|
||||||
.valign(gtk4::Align::Center)
|
|
||||||
.can_target(false)
|
|
||||||
.visible(false)
|
|
||||||
.build();
|
|
||||||
note.add_css_class("media-player-note");
|
note.add_css_class("media-player-note");
|
||||||
|
if let Some(cover) = note
|
||||||
|
.child()
|
||||||
|
.and_then(|fallback| fallback.next_sibling())
|
||||||
|
.and_downcast::<gtk4::Picture>()
|
||||||
|
{
|
||||||
|
// The whole cover, not a square cut out of it.
|
||||||
|
cover.set_content_fit(gtk4::ContentFit::Contain);
|
||||||
|
}
|
||||||
|
bind_file_thumbnail(ui, ¬e, entry, true);
|
||||||
let overlay = gtk4::Overlay::new();
|
let overlay = gtk4::Overlay::new();
|
||||||
overlay.set_child(Some(&video));
|
overlay.set_child(Some(&video));
|
||||||
overlay.add_overlay(¬e);
|
overlay.add_overlay(¬e);
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ use pdfs_core::control::{FileThumbRequest, is_thumbnail_media_name, is_video_fil
|
|||||||
const THUMB_BATCH: usize = 32;
|
const THUMB_BATCH: usize = 32;
|
||||||
const THUMB_CACHE_MAX: usize = 256;
|
const THUMB_CACHE_MAX: usize = 256;
|
||||||
const THUMB_MISSING_MAX: usize = 2048;
|
const THUMB_MISSING_MAX: usize = 2048;
|
||||||
|
const THUMB_BLANK_MAX: usize = 1024;
|
||||||
const THUMB_DEBOUNCE: Duration = Duration::from_millis(40);
|
const THUMB_DEBOUNCE: Duration = Duration::from_millis(40);
|
||||||
const THUMB_RETRY: Duration = Duration::from_secs(2);
|
const THUMB_RETRY: Duration = Duration::from_secs(2);
|
||||||
|
|
||||||
@@ -20,6 +21,9 @@ pub(crate) struct FileThumbnailState {
|
|||||||
missing: RefCell<HashSet<String>>,
|
missing: RefCell<HashSet<String>>,
|
||||||
missing_order: RefCell<VecDeque<String>>,
|
missing_order: RefCell<VecDeque<String>>,
|
||||||
wanted: RefCell<HashMap<String, Vec<gtk4::Picture>>>,
|
wanted: RefCell<HashMap<String, Vec<gtk4::Picture>>>,
|
||||||
|
/// Tiles on screen still showing their icon, by key, kept across a
|
||||||
|
/// cancelled request so [`refresh_file_thumbnails`] can ask for them again.
|
||||||
|
blank: RefCell<HashMap<String, BlankTile>>,
|
||||||
queue: RefCell<VecDeque<FileThumbRequest>>,
|
queue: RefCell<VecDeque<FileThumbRequest>>,
|
||||||
inflight: Cell<bool>,
|
inflight: Cell<bool>,
|
||||||
source: RefCell<Option<glib::SourceId>>,
|
source: RefCell<Option<glib::SourceId>>,
|
||||||
@@ -27,6 +31,18 @@ pub(crate) struct FileThumbnailState {
|
|||||||
generation_loading: Cell<bool>,
|
generation_loading: Cell<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What a tile left blank needs to be asked for again.
|
||||||
|
struct BlankTile {
|
||||||
|
request: FileThumbRequest,
|
||||||
|
pictures: Vec<glib::WeakRef<gtk4::Picture>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a picture registered as blank under `key` still is: bound to that
|
||||||
|
/// key, not painted, and in a window rather than in a list's spare pool.
|
||||||
|
fn tile_still_blank(key: &str, bound: &str, painted: bool, attached: bool) -> bool {
|
||||||
|
bound == key && !painted && attached
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq)]
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
enum ReplyDisposition {
|
enum ReplyDisposition {
|
||||||
Ignore,
|
Ignore,
|
||||||
@@ -55,6 +71,7 @@ impl FileThumbnailState {
|
|||||||
missing: RefCell::new(HashSet::new()),
|
missing: RefCell::new(HashSet::new()),
|
||||||
missing_order: RefCell::new(VecDeque::new()),
|
missing_order: RefCell::new(VecDeque::new()),
|
||||||
wanted: RefCell::new(HashMap::new()),
|
wanted: RefCell::new(HashMap::new()),
|
||||||
|
blank: RefCell::new(HashMap::new()),
|
||||||
queue: RefCell::new(VecDeque::new()),
|
queue: RefCell::new(VecDeque::new()),
|
||||||
inflight: Cell::new(false),
|
inflight: Cell::new(false),
|
||||||
source: RefCell::new(None),
|
source: RefCell::new(None),
|
||||||
@@ -88,6 +105,85 @@ impl FileThumbnailState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn remember_blank(&self, key: &str, request: &FileThumbRequest, picture: >k4::Picture) {
|
||||||
|
let mut blank = self.blank.borrow_mut();
|
||||||
|
if blank.len() >= THUMB_BLANK_MAX {
|
||||||
|
prune_blank(&mut blank);
|
||||||
|
}
|
||||||
|
let tile = blank.entry(key.to_string()).or_insert_with(|| BlankTile {
|
||||||
|
request: request.clone(),
|
||||||
|
pictures: Vec::new(),
|
||||||
|
});
|
||||||
|
if !tile
|
||||||
|
.pictures
|
||||||
|
.iter()
|
||||||
|
.any(|weak| weak.upgrade().as_ref() == Some(picture))
|
||||||
|
{
|
||||||
|
tile.pictures.push(picture.downgrade());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop the tiles that were painted, rebound to another file, or freed.
|
||||||
|
fn prune_blank(blank: &mut HashMap<String, BlankTile>) {
|
||||||
|
blank.retain(|key, tile| {
|
||||||
|
tile.pictures.retain(|weak| {
|
||||||
|
weak.upgrade().is_some_and(|picture| {
|
||||||
|
tile_still_blank(
|
||||||
|
key,
|
||||||
|
&picture.widget_name(),
|
||||||
|
picture.is_visible(),
|
||||||
|
picture.root().is_some(),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
});
|
||||||
|
!tile.pictures.is_empty()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask again for every tile on screen still showing its icon. Sent when the
|
||||||
|
/// daemon says thumbnails landed, so a folder fills in while a build runs in
|
||||||
|
/// it, or after its own requests were cancelled, without being opened again.
|
||||||
|
pub(crate) fn refresh_file_thumbnails(ui: &Rc<Ui>) {
|
||||||
|
let tiles: Vec<(String, FileThumbRequest, Vec<gtk4::Picture>)> = {
|
||||||
|
let mut blank = ui.file_thumbs.blank.borrow_mut();
|
||||||
|
prune_blank(&mut blank);
|
||||||
|
blank
|
||||||
|
.iter()
|
||||||
|
.map(|(key, tile)| {
|
||||||
|
let pictures = tile.pictures.iter().filter_map(|w| w.upgrade()).collect();
|
||||||
|
(key.clone(), tile.request.clone(), pictures)
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
if tiles.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (key, request, pictures) in tiles {
|
||||||
|
let cached = ui.file_thumbs.textures.borrow().get(&key).cloned();
|
||||||
|
if let Some(texture) = cached {
|
||||||
|
for picture in &pictures {
|
||||||
|
paint(picture, &key, &texture);
|
||||||
|
}
|
||||||
|
ui.file_thumbs.blank.borrow_mut().remove(&key);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// A miss from before the thumbnail landed is no verdict any more.
|
||||||
|
ui.file_thumbs.missing.borrow_mut().remove(&key);
|
||||||
|
let mut wanted = ui.file_thumbs.wanted.borrow_mut();
|
||||||
|
let waiters = wanted.entry(key).or_default();
|
||||||
|
for picture in pictures {
|
||||||
|
if !waiters.contains(&picture) {
|
||||||
|
waiters.push(picture);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(wanted);
|
||||||
|
let mut queue = ui.file_thumbs.queue.borrow_mut();
|
||||||
|
queue.retain(|item| item.uid != request.uid);
|
||||||
|
queue.push_back(request);
|
||||||
|
}
|
||||||
|
schedule_file_thumbs(ui);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Abandon every opportunistic ordinary-file thumbnail associated with the
|
/// Abandon every opportunistic ordinary-file thumbnail associated with the
|
||||||
@@ -230,6 +326,12 @@ pub(crate) fn bind_file_thumbnail(
|
|||||||
paint(&picture, &key, texture);
|
paint(&picture, &key, texture);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
let request = FileThumbRequest {
|
||||||
|
uid: entry.uid.clone(),
|
||||||
|
modified: entry.modified,
|
||||||
|
name: entry.name.clone(),
|
||||||
|
};
|
||||||
|
ui.file_thumbs.remember_blank(&key, &request, &picture);
|
||||||
if ui.file_thumbs.missing.borrow().contains(&key) {
|
if ui.file_thumbs.missing.borrow().contains(&key) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -242,11 +344,7 @@ pub(crate) fn bind_file_thumbnail(
|
|||||||
.push(picture);
|
.push(picture);
|
||||||
let mut queue = ui.file_thumbs.queue.borrow_mut();
|
let mut queue = ui.file_thumbs.queue.borrow_mut();
|
||||||
queue.retain(|item| item.uid != entry.uid);
|
queue.retain(|item| item.uid != entry.uid);
|
||||||
queue.push_back(FileThumbRequest {
|
queue.push_back(request);
|
||||||
uid: entry.uid.clone(),
|
|
||||||
modified: entry.modified,
|
|
||||||
name: entry.name.clone(),
|
|
||||||
});
|
|
||||||
drop(queue);
|
drop(queue);
|
||||||
schedule_file_thumbs(ui);
|
schedule_file_thumbs(ui);
|
||||||
}
|
}
|
||||||
@@ -387,6 +485,7 @@ fn flush_file_thumbs(ui: &Rc<Ui>) {
|
|||||||
match gtk4::gdk::Texture::from_filename(&path) {
|
match gtk4::gdk::Texture::from_filename(&path) {
|
||||||
Ok(texture) => {
|
Ok(texture) => {
|
||||||
ui.file_thumbs.store_texture(&key, texture.clone());
|
ui.file_thumbs.store_texture(&key, texture.clone());
|
||||||
|
ui.file_thumbs.blank.borrow_mut().remove(&key);
|
||||||
if let Some(pictures) =
|
if let Some(pictures) =
|
||||||
ui.file_thumbs.wanted.borrow_mut().remove(&key)
|
ui.file_thumbs.wanted.borrow_mut().remove(&key)
|
||||||
{
|
{
|
||||||
@@ -447,7 +546,16 @@ fn retry_file_thumbs(ui: &Rc<Ui>, items: Vec<FileThumbRequest>) {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{ReplyDisposition, reply_disposition};
|
use super::{ReplyDisposition, reply_disposition, tile_still_blank};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_a_tile_still_showing_its_icon_is_asked_for_again() {
|
||||||
|
assert!(tile_still_blank("a:1", "a:1", false, true));
|
||||||
|
assert!(!tile_still_blank("a:1", "a:1", true, true), "painted");
|
||||||
|
assert!(!tile_still_blank("a:1", "b:1", false, true), "rebound");
|
||||||
|
assert!(!tile_still_blank("a:1", "a:2", false, true), "file changed");
|
||||||
|
assert!(!tile_still_blank("a:1", "a:1", false, false), "spare row");
|
||||||
|
}
|
||||||
|
|
||||||
fn is_image_name(name: &str) -> bool {
|
fn is_image_name(name: &str) -> bool {
|
||||||
pdfs_core::control::is_thumbnail_image_name(name)
|
pdfs_core::control::is_thumbnail_image_name(name)
|
||||||
|
|||||||
+161
@@ -109,6 +109,11 @@ a live account since.
|
|||||||
|
|
||||||
| Entry | Title |
|
| Entry | Title |
|
||||||
|---|---|
|
|---|---|
|
||||||
|
| [B289](#b289--the-takeout-import-mixes-in-other-services-and-drops-location-and-live-photos) | The Takeout import mixes in other services and drops location and live photos |
|
||||||
|
| [B288](#b288--an-interrupted-takeout-import-reads-as-finished-and-one-error-ends-it) | An interrupted Takeout import reads as finished, and one error ends it |
|
||||||
|
| [B287](#b287--a-photo-draft-left-by-an-interrupted-upload-blocks-that-name) | A photo draft left by an interrupted upload blocks that name |
|
||||||
|
| [B286](#b286--resuming-a-takeout-import-fails-on-a-draft-with-no-content-hash) | Resuming a Takeout import fails on a draft with no content hash |
|
||||||
|
| [B285](#b285--a-restarted-secret-service-leaves-the-process-on-a-dead-session) | A restarted Secret Service leaves the process on a dead session |
|
||||||
| [B284](#b284--a-keyring-entry-reported-missing-discards-refreshed-session-tokens) | A keyring entry reported missing discards refreshed session tokens |
|
| [B284](#b284--a-keyring-entry-reported-missing-discards-refreshed-session-tokens) | A keyring entry reported missing discards refreshed session tokens |
|
||||||
| [B283](#b283--videos-have-no-thumbnail-outside-photos-and-a-ts-recording-looks-like-source-code) | Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code |
|
| [B283](#b283--videos-have-no-thumbnail-outside-photos-and-a-ts-recording-looks-like-source-code) | Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code |
|
||||||
| [B282](#b282--a-file-kept-offline-is-downloaded-again-after-a-change-made-elsewhere) | A file kept offline is downloaded again after a change made elsewhere |
|
| [B282](#b282--a-file-kept-offline-is-downloaded-again-after-a-change-made-elsewhere) | A file kept offline is downloaded again after a change made elsewhere |
|
||||||
@@ -387,6 +392,162 @@ a live account since.
|
|||||||
|
|
||||||
Newest first.
|
Newest first.
|
||||||
|
|
||||||
|
### B289 — The Takeout import mixes in other services and drops location and live photos
|
||||||
|
|
||||||
|
**Status:** Fixed, not verified live
|
||||||
|
**Found:** 2026-11-15, auditing the Google Photos import alongside
|
||||||
|
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||||
|
**Where:** `crates/pdfs-core/src/takeout.rs`; `plan` and `upload_takeout_photo` in
|
||||||
|
`crates/pdfs-fuse/src/takeout.rs`
|
||||||
|
|
||||||
|
**Repro:** Import a Takeout export that also holds `Drive/` or `YouTube/`, with Pixel motion
|
||||||
|
photos, iPhone live photos and photos that have a place in Google Photos.
|
||||||
|
|
||||||
|
**Cause.** Five gaps in what the import carried over:
|
||||||
|
- The service folder was dropped without being checked, so media from other Google services in
|
||||||
|
the same export landed in the photo timeline.
|
||||||
|
- `geoData` was never read, although `docs/PHOTOS.md` said the import takes locations from the
|
||||||
|
export, and the SDK had no way to upload a location.
|
||||||
|
- The two halves of a live or motion photo uploaded as unrelated photos, the `.MOV` half usually
|
||||||
|
without a date, and the `.MP` half of a Pixel motion photo was skipped as unknown.
|
||||||
|
- `creationTime`, the day the photo reached Google, outranked the date in the file name and the
|
||||||
|
photo's own EXIF.
|
||||||
|
- A sidecar with `null` in `favorited`, a numeric `timestamp` or a `0` timestamp lost its other
|
||||||
|
fields or filed the photo under 1970.
|
||||||
|
|
||||||
|
**Fix:** The Photos service is the folder whose sidecars carry `photoTakenTime`, and only it is
|
||||||
|
imported (everything, when no folder does). `geoData`, else `geoDataExif`, becomes the photo's
|
||||||
|
`Location` attribute through SDK 0.12.0's `PhotoUploadMetadata::location`; `0, 0` is no position.
|
||||||
|
Stills and videos with the same stem in one folder (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||||
|
`PXL_1.MP.jpg` + `PXL_1.MP`) are paired unless their dates are minutes apart: the still is
|
||||||
|
tagged as a live or motion photo, the video is uploaded after it with `main_photo_uid` and takes
|
||||||
|
its date and place. The capture time is `photoTakenTime`, then the name, then EXIF, then
|
||||||
|
`creationTime`. Favorites are also set on photos that were already on the account.
|
||||||
|
|
||||||
|
**Verified:** `takeout::tests` in `pdfs-core` (services, locations, timestamps, live keys) and
|
||||||
|
`pdfs-fuse` (pairing, skew, folded copies). Not run against a real export and account.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B288 — An interrupted Takeout import reads as finished, and one error ends it
|
||||||
|
|
||||||
|
**Status:** Fixed, not verified live
|
||||||
|
**Found:** 2026-11-15, auditing the Google Photos import alongside
|
||||||
|
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||||
|
**Where:** `crates/pdfs-fuse/src/takeout.rs`; `refresh_takeout` in
|
||||||
|
`crates/pdfs-gui/src/app/pages/takeout.rs`; `cmd_import_google_photos` in
|
||||||
|
`crates/pdfs-cli/src/main.rs`
|
||||||
|
|
||||||
|
**Repro:** Start an import of several archives, one of them damaged, and restart the service
|
||||||
|
while it uploads.
|
||||||
|
|
||||||
|
**Cause.**
|
||||||
|
- The import state lived only in memory. After a restart `ImportStatus` had no summary, and the
|
||||||
|
app announced "Import finished". The archives had to be chosen again to continue.
|
||||||
|
- A daemon shutdown broke the upload loop but still filed albums and logged a success.
|
||||||
|
- One unreadable archive failed the scan, and a failed request in the middle of the run
|
||||||
|
discarded every count and skipped the albums for what had uploaded.
|
||||||
|
- The progress total counted photos already on the account, so the bar stopped short of the
|
||||||
|
end; a dry run reported "Would upload: 0"; the job's phases reached the app in English.
|
||||||
|
|
||||||
|
**Fix:** The last run's archives and summary are kept in `takeout-import.json` in the state
|
||||||
|
directory. A run still marked running at start-up is `interrupted`, and `ImportStatus` lists
|
||||||
|
its archives as `resumable`; `ResumeImport` (`pdfs import-google-photos --resume`, **Resume** in
|
||||||
|
the app) starts it again, and the duplicate check skips what is there. An unreadable archive is
|
||||||
|
listed in `unreadable_archives` and the rest imports. A failure keeps the counts and still files
|
||||||
|
albums. `ImportStatus` carries the phase and its own progress, the summary says `dry_run`,
|
||||||
|
`to_upload` and `error_kind`, and the app words them in the user's language. The inline
|
||||||
|
timeline refresh of an empty gallery waits for a refresh already running instead of starting a
|
||||||
|
second one (`refresh_timeline_joined`).
|
||||||
|
|
||||||
|
**Verified:** `takeout::tests` in `pdfs-fuse` (saved state, resumable runs) and `pdfs-core`
|
||||||
|
(unreadable archive), and the CLI tests for `--resume` and the exit status. Not run live.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B287 — A photo draft left by an interrupted upload blocks that name
|
||||||
|
|
||||||
|
**Status:** Fixed, not verified live
|
||||||
|
**Found:** 2026-11-15, reading the SDK upload path for
|
||||||
|
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||||
|
**Where:** `upload_photo_from`, `upload_file_from_inner` and `abandon_file_draft` in
|
||||||
|
`proton-drive-rs` (SDK 0.12.0)
|
||||||
|
|
||||||
|
**Repro:** Interrupt a photo upload after its draft is created, sign in again, and upload a
|
||||||
|
photo with the same name.
|
||||||
|
|
||||||
|
**Cause.** Photo uploads replaced a leftover draft only when its `ClientUID` was the current
|
||||||
|
session's. A new login is a new session, so the next upload of that name failed with 2500. A
|
||||||
|
failed upload also never deleted its own draft, which the C# and TypeScript SDKs do.
|
||||||
|
|
||||||
|
**Fix:** SDK 0.12.0 adds `PhotoUploadMetadata::replace_existing_draft`, which the Takeout import
|
||||||
|
sets, and deletes the draft of a new file or photo whose upload fails. After a failed seal it
|
||||||
|
only does so on a definite rejection, because a seal lost in transit may have been applied.
|
||||||
|
|
||||||
|
**Verified:** SDK unit test `draft_cleanup_only_after_a_definite_seal_rejection`. Not run live.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B286 — Resuming a Takeout import fails on a draft with no content hash
|
||||||
|
|
||||||
|
**Status:** Fixed, not verified live
|
||||||
|
**Found:** 2026-11-15, [issue #94](https://github.com/narrrl/proton-drive-linux/issues/94)
|
||||||
|
(3.7.0, Fedora). An import stopped by a restart failed every re-run at once with
|
||||||
|
`check for existing photos: serialization error: invalid type: null, expected a string at line
|
||||||
|
1 column 2063`.
|
||||||
|
**Where:** `FoundPhotoDuplicateDto` in `proton-drive-rs` (SDK 0.11.0)
|
||||||
|
|
||||||
|
**Repro:** Interrupt a photo upload after its draft is created, then run the import again.
|
||||||
|
|
||||||
|
**Cause.** The duplicates endpoint sends `ContentHash` (and can send `Hash`) as `null` for a
|
||||||
|
draft, as its API schema says. The SDK declared both as `String` with `#[serde(default)]`,
|
||||||
|
which covers a missing field but not `null`, so one draft failed the whole batch of 150 names
|
||||||
|
and the import ended before uploading anything.
|
||||||
|
|
||||||
|
**Fix:** SDK 0.12.0 makes both fields optional and skips drafts, trashed links and candidates
|
||||||
|
without hashes, like the TypeScript SDK. Every other defaulted DTO field reads `null` as its
|
||||||
|
default too, and an undecodable response is logged with its path.
|
||||||
|
|
||||||
|
**Verified:** SDK unit tests `duplicate_response_tolerates_null_fields_and_skips_drafts` and
|
||||||
|
`duplicate_response_tolerates_a_null_list`. Not run against the reporter's account.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B285 — A restarted Secret Service leaves the process on a dead session
|
||||||
|
|
||||||
|
**Status:** Fixed (unverified)
|
||||||
|
**Found:** 2026-11-14, following up [issue #78](https://github.com/narrrl/proton-drive-linux/issues/78)
|
||||||
|
(signed out after every cold boot on Fedora 45, which replaced gnome-keyring with oo7). See
|
||||||
|
[the investigation](ISSUE-78.md#a-restarted-secret-service).
|
||||||
|
**Where:** `crates/pdfs-core/src/auth.rs`, `with_keyring_entry`
|
||||||
|
|
||||||
|
**Repro:** Write the session blob, restart the Secret Service provider
|
||||||
|
(`gnome-keyring-daemon --replace`, or `oo7-daemon --replace`), then read or write it again from
|
||||||
|
the same process. `dbus-secret-service-keyring-store` keeps one D-Bus connection and one Secret
|
||||||
|
Service session for the life of the process, and pdfs installed it once. The session exists only
|
||||||
|
in the provider's memory, so the new provider answers every `GetSecret` and `SetSecret` with
|
||||||
|
`org.freedesktop.Secret.Error.NoSession`, for as long as the process runs. A fresh store in the
|
||||||
|
same process reads the same entry fine. oo7's systemd unit restarts the daemon on failure, and
|
||||||
|
both providers can be replaced, so the daemon could meet this mid-session. A token refresh after
|
||||||
|
that point reached only memory and the kernel keyring: B61's retry kept asking on the dead
|
||||||
|
session, and the next cold boot read the spent refresh token (`InvalidRefreshToken`).
|
||||||
|
|
||||||
|
**Fix:** Every keyring call goes through `with_keyring_entry`. A platform failure (which is what
|
||||||
|
`NoSession` and other D-Bus errors become) logs "the Secret Service failed this process's
|
||||||
|
connection; reconnecting", drops the store unless another thread already replaced it, and makes
|
||||||
|
the call once more on a new connection and session. A locked keyring, a dismissed prompt and a
|
||||||
|
missing entry come back unchanged. Affects the daemon, the app and the tray alike.
|
||||||
|
|
||||||
|
**Verified:** `auth::tests::a_dropped_secret_service_session_is_reconnected_and_retried`,
|
||||||
|
`a_rotation_lands_at_once_after_the_keyring_restarts` and `a_locked_keyring_is_not_reconnected`
|
||||||
|
on the in-memory store. The ignored `auth::tests::real_secret_service_survives_a_provider_restart`
|
||||||
|
passes against gnome-keyring 51.1 and oo7 0.7.1 (built from its tag), each in a throwaway session
|
||||||
|
bus and restarted with `--replace`; it checks that a reconnect actually happened. A standalone
|
||||||
|
program holding one store, as before the fix, gets `NoSession` on read and write after the same
|
||||||
|
restart under both providers. Whether the reporter's oo7 restarts during a session is not known.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### B284 — A keyring entry reported missing discards refreshed session tokens
|
### B284 — A keyring entry reported missing discards refreshed session tokens
|
||||||
|
|
||||||
**Status:** Fixed (unverified)
|
**Status:** Fixed (unverified)
|
||||||
|
|||||||
+58
-2
@@ -104,18 +104,74 @@ Existing folders stay device backups. Older builds cannot open the upgraded data
|
|||||||
10 seconds, and `M` to mute. The volume carries over from one video to the next, and the
|
10 seconds, and `M` to mute. The volume carries over from one video to the next, and the
|
||||||
thumbnail strip in Photos stays off a video, where it covered the seek bar.
|
thumbnail strip in Photos stays off a video, where it covered the seek bar.
|
||||||
|
|
||||||
|
- **Album covers for songs.** A song in My files, in search and in Shared shows its cover as its
|
||||||
|
thumbnail, and the player shows it above the controls instead of a music note. The cover is the
|
||||||
|
picture in the song's tags (MP3, FLAC, Ogg, Opus, M4A, WAV, AIFF, WavPack, Monkey's Audio), or
|
||||||
|
else a `cover`, `folder`, `front` or `album` image (`.jpg`, `.png`, `.webp`) in the same folder.
|
||||||
|
Only the song's tags are downloaded for it, not the whole song. Songs uploaded from this computer
|
||||||
|
carry their cover as their thumbnail, so they show it at once on other devices too. A cover
|
||||||
|
image added to a folder later is picked up once the song changes or the service restarts.
|
||||||
|
|
||||||
|
- **Resume a Google Photos import** (B288). The archives and the outcome of the last import are
|
||||||
|
kept, so an import cut short by a restart or a reboot no longer has to be set up again. The
|
||||||
|
Import page offers **Resume**, and `pdfs import-google-photos --resume` does the same. Photos
|
||||||
|
that are already uploaded are skipped as before.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- **`proton-sdk` and `proton-drive-rs` 0.11.0.** Timestamps are converted where they enter
|
- **`proton-sdk` and `proton-drive-rs` 0.12.0.** Timestamps are converted where they enter
|
||||||
from the SDK, and a photo keeps its own modification time now that it no longer follows the
|
from the SDK, and a photo keeps its own modification time now that it no longer follows the
|
||||||
capture time. [The SDK review](SDK-0.11.0.md) lists the new features and breaking changes.
|
capture time. 0.12.0 fixes the photo duplicate check and failed-upload drafts below.
|
||||||
|
[The 0.11.0 review](SDK-0.11.0.md) and [the 0.12.0 review](SDK-0.12.0.md) list the new
|
||||||
|
features and breaking changes.
|
||||||
|
|
||||||
|
- **The Google Photos import brings more of the export along** (B289).
|
||||||
|
- Places from Google Photos are kept, so imported photos show up under **Places**.
|
||||||
|
- Live photos and Pixel motion photos stay one photo: the video is filed with its still and
|
||||||
|
takes its date. The `.MP` half of a motion photo is no longer skipped.
|
||||||
|
- Only the Google Photos part of an export is imported. Files from Google Drive, YouTube and
|
||||||
|
other services in the same Takeout stay out of the timeline.
|
||||||
|
- A photo without a date of its own is dated by its file name and then by its EXIF before
|
||||||
|
the day it reached Google Photos.
|
||||||
|
- Favorites are also set on photos that were already in Proton Photos.
|
||||||
|
|
||||||
|
- **The import reports what it is doing** (B288). The Import page shows the current step
|
||||||
|
(reading, checking for photos you already have, uploading, sorting into albums), the counts
|
||||||
|
so far, and a progress bar that reaches the end. A scan says how many photos and how much
|
||||||
|
data an import would upload. `pdfs import-google-photos --wait` prints its progress, and
|
||||||
|
`pdfs import-status` shows the step.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **Resuming a Google Photos import no longer fails at once** (#94, B286, B287). A photo upload
|
||||||
|
that was cut short left a draft behind, and every later run stopped with
|
||||||
|
`check for existing photos: serialization error: invalid type: null`. Such drafts are now
|
||||||
|
skipped by the duplicate check and replaced by the next upload of that photo, also after
|
||||||
|
signing in again. A failed upload removes its draft.
|
||||||
|
|
||||||
|
- **One bad archive or error no longer ends the whole import** (B288). An archive that cannot be
|
||||||
|
read is left out and named in the report, and the rest of the export imports. When an import
|
||||||
|
stops early, it keeps the counts of what it did and still files the uploaded photos into their
|
||||||
|
albums. A restart of the service while importing is reported as an interruption, not as a
|
||||||
|
finished import.
|
||||||
|
|
||||||
- **Photos zoomed all the way out shows the whole timeline.** The first 200 photos fit on one
|
- **Photos zoomed all the way out shows the whole timeline.** The first 200 photos fit on one
|
||||||
screen there, and with nothing to scroll no more were loaded. More now load whenever the photos
|
screen there, and with nothing to scroll no more were loaded. More now load whenever the photos
|
||||||
don't fill the window, also after a zoom or a resize.
|
don't fill the window, also after a zoom or a resize.
|
||||||
|
|
||||||
|
- **Signed out after a cold boot on Fedora 45** (issue #78, B284, B285). Fedora 45 replaced the
|
||||||
|
GNOME keyring with oo7, and two things there could keep a refreshed sign-in from being saved:
|
||||||
|
oo7 reports the entries of a locked keyring as missing, which the client took for a sign-out,
|
||||||
|
and a restarted oo7 no longer knew the client's connection, which then failed every save until
|
||||||
|
the service restarted. The refreshed sign-in lived only in memory, and the next boot found the
|
||||||
|
spent one. The client now keeps such a sign-in and saves it once the keyring can be read, and
|
||||||
|
reconnects to a keyring service that restarted.
|
||||||
|
|
||||||
|
- **Thumbnails appear in the folder you are looking at as soon as they are made.** During
|
||||||
|
**Build thumbnails**, the folder on screen kept its file icons until you left it and opened it
|
||||||
|
again. Its tiles now fill in while the build runs, and the same goes for any tile whose
|
||||||
|
thumbnail is made while it is on screen.
|
||||||
|
|
||||||
- **My files comes back after the service was killed for hanging** (B279). A killed service left
|
- **My files comes back after the service was killed for hanging** (B279). A killed service left
|
||||||
its folders behind as dead mounts, and every restart then failed with `File exists` and retried
|
its folders behind as dead mounts, and every restart then failed with `File exists` and retried
|
||||||
forever without opening anything, until the mounts were cleared by hand. The service now clears
|
forever without opening anything, until the mounts were cleared by hand. The service now clears
|
||||||
|
|||||||
@@ -196,6 +196,7 @@ commands are for scripts and for cases where the mount is not available.
|
|||||||
| `pdfs add-to-album UID PHOTOS...` | Add photos to an album. |
|
| `pdfs add-to-album UID PHOTOS...` | Add photos to an album. |
|
||||||
| `pdfs remove-from-album UID PHOTOS...` | Take photos out of an album. |
|
| `pdfs remove-from-album UID PHOTOS...` | Take photos out of an album. |
|
||||||
| `pdfs import-google-photos ARCHIVES... [--dry-run] [--wait]` | Import a Google Takeout export. Pass every `.zip` of the export. |
|
| `pdfs import-google-photos ARCHIVES... [--dry-run] [--wait]` | Import a Google Takeout export. Pass every `.zip` of the export. |
|
||||||
|
| `pdfs import-google-photos --resume [--wait]` | Start the last import again with the same archives after it was interrupted, stopped or failed. |
|
||||||
| `pdfs import-status` / `pdfs cancel-import` | Follow or stop a running import. |
|
| `pdfs import-status` / `pdfs cancel-import` | Follow or stop a running import. |
|
||||||
| `pdfs redate-photos [--dry-run] [--from DATE] [--to DATE] [--limit N] [--wait]` | Fix photos filed under their upload day when the file name carries the real date. |
|
| `pdfs redate-photos [--dry-run] [--from DATE] [--to DATE] [--limit N] [--wait]` | Fix photos filed under their upload day when the file name carries the real date. |
|
||||||
| `pdfs redate-status` / `pdfs cancel-redate` | Follow or stop a running re-date. |
|
| `pdfs redate-status` / `pdfs cancel-redate` | Follow or stop a running re-date. |
|
||||||
|
|||||||
@@ -58,6 +58,44 @@ can resolve the default alias to a locked collection. Both have fixes in
|
|||||||
after the reporter's November 7 retest. Neither can be assigned as this report's cause without
|
after the reporter's November 7 retest. Neither can be assigned as this report's cause without
|
||||||
matching runtime evidence.
|
matching runtime evidence.
|
||||||
|
|
||||||
|
## A restarted Secret Service
|
||||||
|
|
||||||
|
A second path to the same symptom, independent of locking (B285). The keyring store pdfs used
|
||||||
|
holds one D-Bus connection and one Secret Service session for the whole process, and was
|
||||||
|
installed once. The session lives only in the provider's memory. When the provider restarts,
|
||||||
|
the new one answers every `GetSecret` and `SetSecret` on the old session with
|
||||||
|
`org.freedesktop.Secret.Error.NoSession`, until the pdfs process itself restarts. Reproduced
|
||||||
|
with both gnome-keyring 51.1 and oo7 0.7.1 using `--replace`.
|
||||||
|
|
||||||
|
gnome-keyring practically never restarts within a desktop session. oo7 can:
|
||||||
|
`oo7-daemon.service` has `Restart=on-failure`, the daemon allows itself to be replaced, and it
|
||||||
|
exits on a wrong credential password and is D-Bus-activated again. A daemon that met a restarted
|
||||||
|
provider kept every later token refresh only in memory and the kernel keyring, and a cold boot
|
||||||
|
then left the spent refresh token in the saved blob.
|
||||||
|
|
||||||
|
Now a platform failure drops the store and the call is retried once on a new connection. The
|
||||||
|
daemon log shows `the Secret Service failed this process's connection; reconnecting` when that
|
||||||
|
happens. To check whether a machine meets this, look for oo7 restarts in the same boot:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
journalctl --user -b -1 -u oo7-daemon.service | grep -E 'Started|Stopped|Starting oo7|Main process exited'
|
||||||
|
journalctl --user -b -1 -u proton-drive.service | grep -iE 'keyring|reconnecting|refresh'
|
||||||
|
```
|
||||||
|
|
||||||
|
Reproduce in a throwaway session bus (never against the desktop's keyring; the runtime directory
|
||||||
|
must be short, as it holds a Unix socket):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
T=$(mktemp -d /tmp/ss.XXXX); mkdir -p $T/data $T/run; chmod 700 $T/run
|
||||||
|
XDG_DATA_HOME=$T/data XDG_RUNTIME_DIR=$T/run PDFS_SECRET_SERVICE_RESTART_TEST=1 \
|
||||||
|
PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets >/dev/null' \
|
||||||
|
dbus-run-session -- sh -c 'echo -n pw | gnome-keyring-daemon --unlock --components=secrets >/dev/null; sleep 1;
|
||||||
|
cargo test -p pdfs-core real_secret_service -- --ignored'
|
||||||
|
```
|
||||||
|
|
||||||
|
For oo7, start `oo7-daemon --login` with the password on stdin instead, and restart it with
|
||||||
|
`--login --replace` in the background.
|
||||||
|
|
||||||
## Reproduce without a Proton account or desktop keyring
|
## Reproduce without a Proton account or desktop keyring
|
||||||
|
|
||||||
The oo7 fixture uses fake tokens, a temporary disk keyring and a private peer-to-peer D-Bus
|
The oo7 fixture uses fake tokens, a temporary disk keyring and a private peer-to-peer D-Bus
|
||||||
|
|||||||
+17
-2
@@ -55,6 +55,7 @@ server learns which areas you look at.
|
|||||||
2. In the app, open the **Upload** menu on the Photos page and choose **Import from Google
|
2. In the app, open the **Upload** menu on the Photos page and choose **Import from Google
|
||||||
Photos…**. Select all the `.zip` files at once, because a photo and its details are often in
|
Photos…**. Select all the `.zip` files at once, because a photo and its details are often in
|
||||||
different parts of the export.
|
different parts of the export.
|
||||||
|
3. Choose **Scan** to see what an import would do without uploading anything, then **Import**.
|
||||||
|
|
||||||
From a terminal:
|
From a terminal:
|
||||||
|
|
||||||
@@ -63,8 +64,22 @@ pdfs import-google-photos --dry-run ~/Downloads/takeout-*.zip # show what woul
|
|||||||
pdfs import-google-photos --wait ~/Downloads/takeout-*.zip
|
pdfs import-google-photos --wait ~/Downloads/takeout-*.zip
|
||||||
```
|
```
|
||||||
|
|
||||||
The import recreates your albums and takes capture dates and locations from the export. Photos
|
What the import takes from the export:
|
||||||
already in your library are skipped, so an interrupted import can simply be run again.
|
|
||||||
|
- **Albums** are recreated. A photo that is in several albums is uploaded once and added to each.
|
||||||
|
- **Dates** come from the export's details, else from a date in the file name, else from the
|
||||||
|
photo's EXIF.
|
||||||
|
- **Places** come from the export, so imported photos appear under [Places](#places).
|
||||||
|
- **Favorites** are kept.
|
||||||
|
- **Live photos and motion photos** stay one photo: the video is filed with its still.
|
||||||
|
|
||||||
|
What it leaves out: photos in Google's trash, and files from other Google services (Drive,
|
||||||
|
YouTube) that are in the same export. A `.zip` that cannot be read is left out and named in the
|
||||||
|
report; the rest still imports.
|
||||||
|
|
||||||
|
Photos already in your library are skipped, so an interrupted import can simply be run again.
|
||||||
|
If the import was cut short, for example by a restart, the Import page offers **Resume**, and
|
||||||
|
`pdfs import-google-photos --resume` does the same without naming the archives again.
|
||||||
`pdfs import-status` shows how far an import is, and `pdfs cancel-import` stops it.
|
`pdfs import-status` shows how far an import is, and `pdfs cancel-import` stops it.
|
||||||
|
|
||||||
## Fixing photos filed under the wrong date
|
## Fixing photos filed under the wrong date
|
||||||
|
|||||||
+2
-1
@@ -63,7 +63,8 @@ them along.
|
|||||||
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
|
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
|
||||||
| Photo name search | Not started | Photos search by date and filters only |
|
| Photo name search | Not started | Photos search by date and filters only |
|
||||||
| Automatic photo upload from a folder | Not started | |
|
| Automatic photo upload from a folder | Not started | |
|
||||||
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos; motion-photo `.MP` parts are skipped |
|
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
|
||||||
|
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
|
||||||
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |
|
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |
|
||||||
|
|
||||||
## Performance and correctness
|
## Performance and correctness
|
||||||
|
|||||||
Whitespace-only changes.
+332
-224
File diff suppressed because it is too large.
Load diff
+313
-220
File diff suppressed because it is too large.
Load diff
+327
-219
File diff suppressed because it is too large.
Load diff
@@ -91,7 +91,7 @@ SECTIONS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
# Too long or too internal for the wiki; links to them go to GitHub.
|
# Too long or too internal for the wiki; links to them go to GitHub.
|
||||||
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md"}
|
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md", "SDK-0.12.0.md"}
|
||||||
|
|
||||||
# Files outside docs/ whose links are checked but not published.
|
# Files outside docs/ whose links are checked but not published.
|
||||||
CHECKED = ["README.md", "CONTRIBUTING.md", "SECURITY.md"]
|
CHECKED = ["README.md", "CONTRIBUTING.md", "SECURITY.md"]
|
||||||
|
|||||||
Reference in new issue
Block a user