Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
34902390f3 | ||
|
|
1dcf7c84ca | ||
|
|
a05772aa13 | ||
|
|
2f2fe96cdf | ||
|
|
2ec3b7ff62 | ||
|
|
38bd282d3a |
No files matched your search
@@ -3640,6 +3640,7 @@ dependencies = [
|
||||
"parking_lot 0.13.0",
|
||||
"pdfs-core",
|
||||
"proton-drive-rs",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha1 0.11.0",
|
||||
"tokio",
|
||||
@@ -3890,9 +3891,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-drive-rs"
|
||||
version = "0.11.0"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14a22e621abbc1c61b4640728a7de969c16b66d5d983b59c638d501fda25a7b9"
|
||||
checksum = "4265c2cf3bdc59ee273d7363c8b6a7cb4e98be6102ffa18144c4d8ebda00737b"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -3918,9 +3919,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-sdk"
|
||||
version = "0.11.0"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd55a5e8b6600f3aa282ebbdf13ff366431ff9aae84d4eb20b0ef06d598919fe"
|
||||
checksum = "fb98ab3e336ed95f82368a50348dad00e5fc919e982d748e83e844afcadbd5d8"
|
||||
dependencies = [
|
||||
"aes-gcm 0.11.1",
|
||||
"async-trait",
|
||||
|
||||
@@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
|
||||
categories = ["filesystem", "command-line-utilities"]
|
||||
|
||||
[workspace.dependencies]
|
||||
proton-sdk = { version = "0.11.0" }
|
||||
proton-drive-rs = { version = "0.11.0" }
|
||||
proton-sdk = { version = "0.12.0" }
|
||||
proton-drive-rs = { version = "0.12.0" }
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
|
||||
# Internal crates.
|
||||
|
||||
@@ -228,14 +228,19 @@ enum Command {
|
||||
/// numbered parts, and a photo's metadata often sits in a different part
|
||||
/// than the photo. Albums in the export are recreated as Proton albums, and
|
||||
/// photos already on the account (same name and same content) are skipped,
|
||||
/// so an interrupted import can simply be run again.
|
||||
/// so an interrupted import can simply be run again, or resumed with
|
||||
/// `--resume` without naming the archives again.
|
||||
ImportGooglePhotos {
|
||||
/// The export's `.zip` files (e.g. `takeout-*.zip`).
|
||||
#[arg(required = true, value_hint = ValueHint::FilePath)]
|
||||
#[arg(required_unless_present = "resume", value_hint = ValueHint::FilePath)]
|
||||
archives: Vec<PathBuf>,
|
||||
/// Report what would be imported without uploading anything.
|
||||
#[arg(long)]
|
||||
#[arg(long, conflicts_with = "resume")]
|
||||
dry_run: bool,
|
||||
/// Start the last import again with the same archives, after it was
|
||||
/// interrupted, stopped or failed.
|
||||
#[arg(long, conflicts_with = "archives")]
|
||||
resume: bool,
|
||||
/// Stay attached and print progress until the import finishes.
|
||||
#[arg(long)]
|
||||
wait: bool,
|
||||
@@ -1030,8 +1035,9 @@ fn main() -> Result<()> {
|
||||
Command::ImportGooglePhotos {
|
||||
archives,
|
||||
dry_run,
|
||||
resume,
|
||||
wait,
|
||||
} => cmd_import_google_photos(archives, dry_run, wait),
|
||||
} => cmd_import_google_photos(archives, dry_run, resume, wait),
|
||||
Command::RedatePhotos {
|
||||
dry_run,
|
||||
limit,
|
||||
@@ -2916,24 +2922,34 @@ fn photo_count(n: usize) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -> Result<()> {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it gets
|
||||
// absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
fn cmd_import_google_photos(
|
||||
archives: Vec<PathBuf>,
|
||||
dry_run: bool,
|
||||
resume: bool,
|
||||
wait: bool,
|
||||
) -> Result<()> {
|
||||
let request = if resume {
|
||||
CtlRequest::ResumeImport
|
||||
} else {
|
||||
// The daemon opens the archives itself off the shared filesystem, so it
|
||||
// gets absolute paths — the same arrangement `pdfs upload` uses.
|
||||
let mut absolute = Vec::with_capacity(archives.len());
|
||||
for archive in &archives {
|
||||
let path = std::fs::canonicalize(archive)
|
||||
.with_context(|| format!("resolve {}", archive.display()))?;
|
||||
absolute.push(
|
||||
path.to_str()
|
||||
.ok_or_else(|| anyhow!("path is not valid UTF-8: {}", path.display()))?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
}
|
||||
};
|
||||
|
||||
match control_request(CtlRequest::ImportTakeout {
|
||||
archives: absolute,
|
||||
dry_run,
|
||||
})? {
|
||||
match control_request(request)? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -2945,20 +2961,34 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
||||
|
||||
// Poll rather than stream: the control protocol is request/response, and an
|
||||
// import runs for hours — a held-open socket would be the only long-lived
|
||||
// connection the daemon has.
|
||||
// connection the daemon has. A line is printed when the step changes and
|
||||
// every few percent of it, not on every poll.
|
||||
let mut last_line = String::new();
|
||||
let mut last_printed = std::time::Instant::now();
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus {
|
||||
running: true,
|
||||
summary: _,
|
||||
} => continue,
|
||||
CtlResponse::ImportStatus { summary, .. } => {
|
||||
print_import_summary(summary.as_ref());
|
||||
return finished_run(
|
||||
"the import",
|
||||
summary.map(|s| (s.failed, s.cancelled, s.error)),
|
||||
);
|
||||
progress,
|
||||
..
|
||||
} => {
|
||||
let line = progress.as_ref().map(progress_line).unwrap_or_default();
|
||||
let step_changed = line.split(':').next() != last_line.split(':').next();
|
||||
if !line.is_empty()
|
||||
&& line != last_line
|
||||
&& (step_changed || last_printed.elapsed().as_secs() >= 30)
|
||||
{
|
||||
println!("{line}");
|
||||
last_line = line;
|
||||
last_printed = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
CtlResponse::ImportStatus {
|
||||
summary, resumable, ..
|
||||
} => {
|
||||
print_import_summary(summary.as_ref(), &resumable);
|
||||
return import_outcome(summary.as_ref());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -2966,6 +2996,47 @@ fn cmd_import_google_photos(archives: Vec<PathBuf>, dry_run: bool, wait: bool) -
|
||||
}
|
||||
}
|
||||
|
||||
/// One line saying where a running import is.
|
||||
fn progress_line(progress: &pdfs_core::control::ImportProgress) -> String {
|
||||
use pdfs_core::control::ImportPhase;
|
||||
let step = match progress.phase {
|
||||
ImportPhase::Scanning => "Reading archives",
|
||||
ImportPhase::Checking => "Checking for photos already on Proton",
|
||||
ImportPhase::Uploading => "Uploading",
|
||||
ImportPhase::Albums => "Filing photos into albums",
|
||||
};
|
||||
match (progress.total, &progress.current) {
|
||||
(0, _) => format!("{step}…"),
|
||||
(total, Some(name)) => format!("{step}: {} of {total} ({name})", progress.done),
|
||||
(total, None) => format!("{step}: {} of {total}", progress.done),
|
||||
}
|
||||
}
|
||||
|
||||
/// How an import `--wait` followed ended, as the exit status: an error unless
|
||||
/// it did everything it set out to do, so a script can tell.
|
||||
fn import_outcome(summary: Option<&pdfs_core::control::ImportSummary>) -> Result<()> {
|
||||
let Some(summary) = summary else {
|
||||
bail!("the import ended without a result; the service may have restarted");
|
||||
};
|
||||
if let Some(error) = &summary.error {
|
||||
bail!("the import failed: {error}");
|
||||
}
|
||||
if summary.interrupted {
|
||||
bail!("the import was interrupted; run `pdfs import-google-photos --resume` to continue");
|
||||
}
|
||||
if summary.cancelled {
|
||||
bail!("the import was cancelled");
|
||||
}
|
||||
if summary.failed > 0 || !summary.unreadable_archives.is_empty() {
|
||||
bail!(
|
||||
"the import finished, but {} photo(s) and {} archive(s) failed",
|
||||
summary.failed,
|
||||
summary.unreadable_archives.len()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_redate_photos(
|
||||
dry_run: bool,
|
||||
limit: Option<usize>,
|
||||
@@ -3100,11 +3171,19 @@ fn print_redate_summary(summary: Option<&pdfs_core::control::RedateSummary>) {
|
||||
|
||||
fn cmd_import_status() -> Result<()> {
|
||||
match control_request(CtlRequest::ImportStatus)? {
|
||||
CtlResponse::ImportStatus { running, summary } => {
|
||||
CtlResponse::ImportStatus {
|
||||
running,
|
||||
summary,
|
||||
progress,
|
||||
resumable,
|
||||
} => {
|
||||
if running {
|
||||
println!("Import running. `pdfs transfers` shows its progress.");
|
||||
match progress.as_ref() {
|
||||
Some(progress) => println!("Import running. {}", progress_line(progress)),
|
||||
None => println!("Import running."),
|
||||
}
|
||||
}
|
||||
print_import_summary(summary.as_ref());
|
||||
print_import_summary(summary.as_ref(), &resumable);
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
@@ -3121,18 +3200,28 @@ fn cmd_cancel_import() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>, resumable: &[String]) {
|
||||
let Some(summary) = summary else {
|
||||
println!("No import has run yet.");
|
||||
return;
|
||||
};
|
||||
if summary.dry_run {
|
||||
println!("Scan only; nothing was uploaded.");
|
||||
}
|
||||
if let Some(error) = &summary.error {
|
||||
println!("Failed : {error}");
|
||||
return;
|
||||
println!("Stopped by : {error}");
|
||||
}
|
||||
println!("Photos in export : {}", summary.found);
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
if summary.dry_run {
|
||||
println!("Would upload : {}", summary.to_upload);
|
||||
println!("Bytes to upload : {}", summary.to_upload_bytes);
|
||||
} else {
|
||||
println!("Uploaded : {}", summary.uploaded);
|
||||
}
|
||||
println!("Already on Proton: {}", summary.duplicates);
|
||||
if summary.live_photos > 0 {
|
||||
println!("Live photos : {}", summary.live_photos);
|
||||
}
|
||||
if summary.skipped_trashed > 0 {
|
||||
println!("Skipped (trash) : {}", summary.skipped_trashed);
|
||||
}
|
||||
@@ -3145,12 +3234,20 @@ fn print_import_summary(summary: Option<&pdfs_core::control::ImportSummary>) {
|
||||
if summary.failed > 0 {
|
||||
println!("Failed : {}", summary.failed);
|
||||
}
|
||||
for archive in &summary.unreadable_archives {
|
||||
println!("Unreadable : {archive}");
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
println!("Uploaded bytes : {}", summary.bytes);
|
||||
}
|
||||
if summary.cancelled {
|
||||
if summary.interrupted {
|
||||
println!("The service stopped before the import finished.");
|
||||
} else if summary.cancelled {
|
||||
println!("Import was cancelled before it finished.");
|
||||
}
|
||||
if !resumable.is_empty() {
|
||||
println!("Run `pdfs import-google-photos --resume` to continue it.");
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_album(uid: String, limit: usize, offset: usize) -> Result<()> {
|
||||
@@ -4509,6 +4606,51 @@ mod login_wait_tests {
|
||||
assert!(finished_run("the import", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_waited_for_import_that_did_not_finish_exits_with_an_error() {
|
||||
use pdfs_core::control::ImportSummary;
|
||||
let done = ImportSummary {
|
||||
uploaded: 4,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(import_outcome(Some(&done)).is_ok());
|
||||
assert!(import_outcome(None).is_err());
|
||||
for broken in [
|
||||
ImportSummary {
|
||||
interrupted: true,
|
||||
..Default::default()
|
||||
},
|
||||
ImportSummary {
|
||||
unreadable_archives: vec!["/exports/takeout-002.zip".into()],
|
||||
..Default::default()
|
||||
},
|
||||
ImportSummary {
|
||||
error: Some("check for existing photos: offline".into()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(import_outcome(Some(&broken)).is_err(), "{broken:?}");
|
||||
}
|
||||
let interrupted = import_outcome(Some(&ImportSummary {
|
||||
interrupted: true,
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(interrupted.unwrap_err().to_string().contains("--resume"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_needs_no_archives_and_excludes_them() {
|
||||
let cli = Cli::try_parse_from(["pdfs", "import-google-photos", "--resume"]).unwrap();
|
||||
assert!(matches!(
|
||||
cli.command,
|
||||
Command::ImportGooglePhotos { resume: true, .. }
|
||||
));
|
||||
assert!(Cli::try_parse_from(["pdfs", "import-google-photos"]).is_err());
|
||||
assert!(
|
||||
Cli::try_parse_from(["pdfs", "import-google-photos", "--resume", "a.zip"]).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiting_for_a_session_stops_hammering_the_keyring() {
|
||||
let mut checks_left = 20;
|
||||
|
||||
@@ -0,0 +1,925 @@
|
||||
//! A stand-in daemon that serves made-up data, for the README screenshots.
|
||||
//!
|
||||
//! `scripts/screenshots.sh` runs it; see `docs/DEVELOPMENT.md`. It answers the
|
||||
//! control socket the way the real daemon does, for the requests the captured
|
||||
//! pages send, with an invented account: no Proton login, no FUSE mount and
|
||||
//! nothing private. Every name, place and number below is made up.
|
||||
//!
|
||||
//! ```text
|
||||
//! demo_daemon serve <photos-dir> answer the control socket until killed
|
||||
//! demo_daemon seed-keyring store a fake session, so the app signs in
|
||||
//! ```
|
||||
//!
|
||||
//! Both refuse to run outside a throwaway environment: `PDFS_DEMO_ROOT` must be
|
||||
//! set and every XDG directory must lie inside it, so the real daemon's socket
|
||||
//! is never touched (`HOME` may instead name a folder that does not exist, the
|
||||
//! demo user's, so paths read as theirs), and `seed-keyring` refuses the session bus the script was
|
||||
//! started on (`PDFS_DEMO_OUTER_BUS`), so the fake session only ever lands in a
|
||||
//! private keyring.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{Datelike, Local, TimeZone};
|
||||
use pdfs_core::cache::Pin;
|
||||
use pdfs_core::config::{AppDirs, KEYRING_SERVICE};
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, DeviceInfo, DirEntry, Event, LocalHit, PhotoItem,
|
||||
PhotoKind, PhotoLocation, PhotoMonth, PhotoThumb, PlaceInfo, Request, Response, SearchHit,
|
||||
SyncFolderInfo, Topic,
|
||||
};
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
|
||||
const USERNAME: &str = "alex.example";
|
||||
const MOUNTPOINT: &str = "/home/alex/ProtonDrive";
|
||||
const GIB: u64 = 1 << 30;
|
||||
const DAY: i64 = 86_400;
|
||||
|
||||
/// How many photos the timeline holds. The demo pictures repeat, but not
|
||||
/// within a screen.
|
||||
const PHOTOS: usize = 1_840;
|
||||
|
||||
fn main() {
|
||||
if let Err(e) = run() {
|
||||
eprintln!("demo_daemon: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<(), String> {
|
||||
check_sandbox()?;
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
match args.iter().map(String::as_str).collect::<Vec<_>>()[..] {
|
||||
["serve", photos] => serve(Path::new(photos)),
|
||||
["seed-keyring"] => seed_keyring(),
|
||||
_ => Err("usage: demo_daemon serve <photos-dir> | seed-keyring".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuse to run anywhere but inside the throwaway root the script made.
|
||||
fn check_sandbox() -> Result<(), String> {
|
||||
let root = std::env::var_os("PDFS_DEMO_ROOT")
|
||||
.map(PathBuf::from)
|
||||
.ok_or("PDFS_DEMO_ROOT is not set; run scripts/screenshots.sh")?;
|
||||
let home = std::env::var_os("HOME").map(PathBuf::from);
|
||||
if home.is_none_or(|home| home.exists() && !home.starts_with(&root)) {
|
||||
return Err("HOME must lie inside PDFS_DEMO_ROOT or not exist".into());
|
||||
}
|
||||
for var in [
|
||||
"XDG_CONFIG_HOME",
|
||||
"XDG_STATE_HOME",
|
||||
"XDG_CACHE_HOME",
|
||||
"XDG_DATA_HOME",
|
||||
] {
|
||||
let inside = std::env::var_os(var)
|
||||
.map(PathBuf::from)
|
||||
.is_some_and(|dir| dir.starts_with(&root));
|
||||
if !inside {
|
||||
return Err(format!("{var} must lie inside PDFS_DEMO_ROOT"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Store a made-up session where the app looks for one. Its keys "unlock"
|
||||
/// without a prompt; nothing ever uses them against Proton.
|
||||
fn seed_keyring() -> Result<(), String> {
|
||||
let bus = std::env::var("DBUS_SESSION_BUS_ADDRESS").unwrap_or_default();
|
||||
let outer =
|
||||
std::env::var("PDFS_DEMO_OUTER_BUS").map_err(|_| "PDFS_DEMO_OUTER_BUS is not set")?;
|
||||
if bus.is_empty() || bus == outer {
|
||||
return Err("refusing to write a fake session to your own keyring".into());
|
||||
}
|
||||
let session = serde_json::json!({
|
||||
"session_id": "demo-session",
|
||||
"username": USERNAME,
|
||||
"user_id": "demo-user",
|
||||
"access_token": "demo",
|
||||
"refresh_token": "demo",
|
||||
"scopes": ["full", "self", "drive"],
|
||||
"password_mode": 1,
|
||||
"key_passphrases": { "demo-key": "ZGVtbw==" },
|
||||
});
|
||||
let store: Arc<keyring_core::CredentialStore> =
|
||||
dbus_secret_service_keyring_store::Store::new().map_err(|e| e.to_string())?;
|
||||
keyring_core::set_default_store(store);
|
||||
keyring_core::Entry::new(KEYRING_SERVICE, "session")
|
||||
.and_then(|entry| entry.set_password(&session.to_string()))
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn serve(photos: &Path) -> Result<(), String> {
|
||||
let demo = Arc::new(Demo::new(photos)?);
|
||||
let dirs = AppDirs::new().map_err(|e| e.to_string())?;
|
||||
dirs.ensure().map_err(|e| e.to_string())?;
|
||||
let socket = dirs.control_socket();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let listener = UnixListener::bind(&socket).map_err(|e| format!("{}: {e}", socket.display()))?;
|
||||
std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o600))
|
||||
.map_err(|e| e.to_string())?;
|
||||
eprintln!("demo_daemon: serving {}", socket.display());
|
||||
for stream in listener.incoming().flatten() {
|
||||
let demo = Arc::clone(&demo);
|
||||
std::thread::spawn(move || {
|
||||
if let Err(e) = connection(&demo, stream) {
|
||||
eprintln!("demo_daemon: {e}");
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn connection(demo: &Demo, stream: UnixStream) -> std::io::Result<()> {
|
||||
let mut reader = BufReader::new(stream.try_clone()?);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line)?;
|
||||
let request: Request = match serde_json::from_str(line.trim()) {
|
||||
Ok(request) => request,
|
||||
Err(e) => {
|
||||
eprintln!("demo_daemon: unreadable request: {e}");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let mut out = stream;
|
||||
if matches!(request, Request::Subscribe) {
|
||||
return subscription(demo, &mut out);
|
||||
}
|
||||
send(&mut out, &demo.answer(&request))
|
||||
}
|
||||
|
||||
/// What the real daemon sends a new subscriber, then a heartbeat now and then
|
||||
/// so the window never counts the daemon as gone.
|
||||
fn subscription(demo: &Demo, out: &mut UnixStream) -> std::io::Result<()> {
|
||||
send(out, &Event::Status(Box::new(demo.status())))?;
|
||||
send(
|
||||
out,
|
||||
&Event::Transfers {
|
||||
items: Vec::new(),
|
||||
jobs: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
send(
|
||||
out,
|
||||
&Event::Changed {
|
||||
topics: Topic::ALL.to_vec(),
|
||||
},
|
||||
)?;
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_secs(10));
|
||||
send(out, &Event::Heartbeat)?;
|
||||
}
|
||||
}
|
||||
|
||||
fn send(out: &mut UnixStream, value: &impl serde::Serialize) -> std::io::Result<()> {
|
||||
let mut line = serde_json::to_vec(value)?;
|
||||
line.push(b'\n');
|
||||
out.write_all(&line)?;
|
||||
out.flush()
|
||||
}
|
||||
|
||||
/// One invented account.
|
||||
struct Demo {
|
||||
now: i64,
|
||||
pictures: Vec<PathBuf>,
|
||||
photos: Vec<PhotoItem>,
|
||||
/// Each photo's place, by index into [`PLACES`].
|
||||
place_of: Vec<Option<usize>>,
|
||||
}
|
||||
|
||||
/// Towns the photos were "taken" in: name, ISO country, latitude, longitude.
|
||||
const PLACES: &[(&str, &str, f64, f64)] = &[
|
||||
("Lisbon", "PT", 38.7223, -9.1393),
|
||||
("Porto", "PT", 41.1579, -8.6291),
|
||||
("Barcelona", "ES", 41.3874, 2.1686),
|
||||
("Paris", "FR", 48.8566, 2.3522),
|
||||
("Amsterdam", "NL", 52.3676, 4.9041),
|
||||
("Copenhagen", "DK", 55.6761, 12.5683),
|
||||
("Berlin", "DE", 52.5200, 13.4050),
|
||||
("Prague", "CZ", 50.0755, 14.4378),
|
||||
("Vienna", "AT", 48.2082, 16.3738),
|
||||
("Florence", "IT", 43.7696, 11.2558),
|
||||
("Rome", "IT", 41.9028, 12.4964),
|
||||
("Thira", "GR", 36.4166, 25.4318),
|
||||
];
|
||||
|
||||
const ALBUMS: &[&str] = &[
|
||||
"Summer in Portugal",
|
||||
"Family",
|
||||
"Hiking",
|
||||
"Food",
|
||||
"City trips",
|
||||
"Best of 2025",
|
||||
];
|
||||
|
||||
impl Demo {
|
||||
fn new(dir: &Path) -> Result<Self, String> {
|
||||
let mut pictures: Vec<PathBuf> = std::fs::read_dir(dir)
|
||||
.map_err(|e| format!("{}: {e}", dir.display()))?
|
||||
.flatten()
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.extension().is_some_and(|ext| ext == "jpg"))
|
||||
.collect();
|
||||
pictures.sort();
|
||||
if pictures.is_empty() {
|
||||
return Err(format!("no .jpg pictures in {}", dir.display()));
|
||||
}
|
||||
let now = Local::now().timestamp();
|
||||
let mut demo = Demo {
|
||||
now,
|
||||
pictures,
|
||||
photos: Vec::new(),
|
||||
place_of: Vec::new(),
|
||||
};
|
||||
demo.make_photos();
|
||||
Ok(demo)
|
||||
}
|
||||
|
||||
/// The timeline: newest first, a few a week back three years, plus a
|
||||
/// photo on this day in each of the last three years.
|
||||
fn make_photos(&mut self) {
|
||||
let mut seed = 0x5eed_u64;
|
||||
let mut next = move || {
|
||||
seed = seed
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
(seed >> 33) as i64
|
||||
};
|
||||
let today = Local::now().date_naive();
|
||||
let noon = |years_back: i32| {
|
||||
let day = today
|
||||
.with_year(today.year() - years_back)
|
||||
.unwrap_or(today - chrono::Days::new(365 * years_back as u64));
|
||||
Local
|
||||
.from_local_datetime(&day.and_hms_opt(11, 30, 0).unwrap_or_default())
|
||||
.single()
|
||||
.map_or(0, |t| t.timestamp())
|
||||
};
|
||||
// Three years, so a few a day.
|
||||
let step = 3 * 365 * DAY / PHOTOS as i64;
|
||||
let mut times: Vec<i64> = (0..PHOTOS as i64)
|
||||
.map(|i| self.now - 3 * 3600 - i * step - next() % step)
|
||||
.collect();
|
||||
times.extend([noon(1), noon(2), noon(3)]);
|
||||
times.sort_unstable_by(|a, b| b.cmp(a));
|
||||
for (i, time) in times.into_iter().enumerate() {
|
||||
let picture = &self.pictures[i % self.pictures.len()];
|
||||
let kind = match i % 23 {
|
||||
7 | 15 => PhotoKind::Video,
|
||||
19 => PhotoKind::Raw,
|
||||
_ => PhotoKind::Photo,
|
||||
};
|
||||
let place = (i % 5 != 4).then_some((i / 7) % PLACES.len());
|
||||
self.photos.push(PhotoItem {
|
||||
uid: format!("demo-photos~{i}"),
|
||||
capture_time: time,
|
||||
thumb_path: Some(picture.display().to_string()),
|
||||
name: Some(format!(
|
||||
"IMG_{}.{}",
|
||||
20_000 + i,
|
||||
if kind == PhotoKind::Video {
|
||||
"MP4"
|
||||
} else {
|
||||
"JPG"
|
||||
}
|
||||
)),
|
||||
ratio: Some(if picture.to_string_lossy().ends_with("-p.jpg") {
|
||||
2.0 / 3.0
|
||||
} else {
|
||||
1.5
|
||||
}),
|
||||
no_thumb: false,
|
||||
kind,
|
||||
favorite: i % 11 == 3,
|
||||
group_size: 0,
|
||||
has_raw: kind == PhotoKind::Raw,
|
||||
place_id: place.map(|p| p as u32 + 1),
|
||||
});
|
||||
self.place_of.push(place);
|
||||
}
|
||||
}
|
||||
|
||||
fn status(&self) -> Response {
|
||||
Response::Status {
|
||||
username: USERNAME.into(),
|
||||
mountpoint: MOUNTPOINT.into(),
|
||||
pinned: 3,
|
||||
used: 6 * GIB + 300 * (1 << 20),
|
||||
budget: 20 * GIB,
|
||||
pins: self.pins(),
|
||||
online: true,
|
||||
pending_uploads: 0,
|
||||
pending_changes: 0,
|
||||
parked_uploads: 0,
|
||||
failing_ops: 0,
|
||||
failing_error: None,
|
||||
staged_bytes: 0,
|
||||
staged_oldest_secs: 0,
|
||||
paused: false,
|
||||
paused_until: None,
|
||||
session_expired: false,
|
||||
tokens_unsaved: false,
|
||||
version: env!("CARGO_PKG_VERSION").into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn answer(&self, request: &Request) -> Response {
|
||||
match request {
|
||||
Request::Status => self.status(),
|
||||
Request::AccountQuota => Response::AccountQuota {
|
||||
max_space: 500 * GIB as i64,
|
||||
used_space: 137 * GIB as i64,
|
||||
},
|
||||
Request::ListDir { path } => Response::Entries {
|
||||
entries: self.dir(path),
|
||||
},
|
||||
Request::ReserveFileThumbGeneration => Response::FileThumbGeneration { generation: 1 },
|
||||
Request::FileThumbs { items, .. } => Response::Thumbs {
|
||||
items: items
|
||||
.iter()
|
||||
.map(|item| PhotoThumb {
|
||||
uid: item.uid.clone(),
|
||||
path: self.file_picture(&item.uid),
|
||||
pending: false,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
Request::PhotosTimeline {
|
||||
offset,
|
||||
limit,
|
||||
kind,
|
||||
favorites,
|
||||
..
|
||||
} => {
|
||||
let items: Vec<PhotoItem> = self
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| kind.is_none_or(|k| p.kind == k))
|
||||
.filter(|p| !favorites || p.favorite)
|
||||
.skip(*offset)
|
||||
.take(*limit)
|
||||
.cloned()
|
||||
.collect();
|
||||
Response::Photos {
|
||||
available: true,
|
||||
items,
|
||||
counts: Some(self.counts()),
|
||||
}
|
||||
}
|
||||
Request::PhotoMonths { kind } => Response::PhotoMonths {
|
||||
months: self.months(*kind),
|
||||
},
|
||||
Request::PhotosOnThisDay { .. } => {
|
||||
let today = Local::now();
|
||||
let items = self
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| {
|
||||
let at = Local.timestamp_opt(p.capture_time, 0).single();
|
||||
at.is_some_and(|at| {
|
||||
at.year() < today.year()
|
||||
&& at.month() == today.month()
|
||||
&& at.day() == today.day()
|
||||
})
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
Response::Photos {
|
||||
available: true,
|
||||
items,
|
||||
counts: None,
|
||||
}
|
||||
}
|
||||
Request::PhotoThumbs { uids } => Response::Thumbs {
|
||||
items: uids
|
||||
.iter()
|
||||
.map(|uid| PhotoThumb {
|
||||
uid: uid.clone(),
|
||||
path: self
|
||||
.photos
|
||||
.iter()
|
||||
.find(|p| &p.uid == uid)
|
||||
.and_then(|p| p.thumb_path.clone()),
|
||||
pending: false,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
Request::PhotoAlbums => Response::Albums {
|
||||
available: true,
|
||||
items: self.albums(),
|
||||
},
|
||||
Request::AlbumPhotos { uid, offset, limit } => {
|
||||
let index: usize = uid
|
||||
.rsplit('~')
|
||||
.next()
|
||||
.and_then(|n| n.parse().ok())
|
||||
.unwrap_or(0);
|
||||
Response::Photos {
|
||||
available: true,
|
||||
items: self
|
||||
.album_photos(index)
|
||||
.skip(*offset)
|
||||
.take(*limit)
|
||||
.cloned()
|
||||
.collect(),
|
||||
counts: None,
|
||||
}
|
||||
}
|
||||
Request::PhotoPlaces => Response::Places {
|
||||
items: self.places(),
|
||||
mapping: None,
|
||||
},
|
||||
Request::PlaceLocations { id } => Response::PhotoLocations {
|
||||
items: self.place_locations(*id),
|
||||
},
|
||||
Request::PlacePhotos { id, offset, limit } => Response::Photos {
|
||||
available: true,
|
||||
items: self
|
||||
.in_place(*id)
|
||||
.map(|(_, p)| p)
|
||||
.skip(*offset)
|
||||
.take(*limit)
|
||||
.cloned()
|
||||
.collect(),
|
||||
counts: None,
|
||||
},
|
||||
Request::ListDevices => Response::Devices {
|
||||
items: self.devices(),
|
||||
},
|
||||
Request::ListSyncFolders => Response::SyncFolders {
|
||||
items: self.sync_folders(),
|
||||
},
|
||||
Request::ListLocations => Response::Locations {
|
||||
items: self.locations(),
|
||||
},
|
||||
// Every conflict in the history is resolved: an open one would
|
||||
// badge every page.
|
||||
Request::ListConflicts => Response::Conflicts { items: Vec::new() },
|
||||
Request::ListPendingOps => Response::PendingOps { items: Vec::new() },
|
||||
Request::ListActivity { limit } => Response::Activity {
|
||||
items: self.activity().into_iter().take(*limit).collect(),
|
||||
},
|
||||
Request::ListInvitations => Response::Invitations { items: Vec::new() },
|
||||
Request::ListBookmarks => Response::Bookmarks { items: Vec::new() },
|
||||
Request::GetQueueStatus => Response::Transfers {
|
||||
items: Vec::new(),
|
||||
jobs: Vec::new(),
|
||||
},
|
||||
Request::ListPins => Response::Pins { pins: self.pins() },
|
||||
Request::SearchV2 { query, limit, .. } => Response::SearchResultsV2 {
|
||||
drive_hits: self.search(query, *limit),
|
||||
local_hits: self.local_search(query),
|
||||
local_indexing: false,
|
||||
},
|
||||
Request::Search { query, limit, .. } => Response::SearchResults {
|
||||
hits: self.search(query, *limit),
|
||||
},
|
||||
Request::ImportStatus => Response::ImportStatus {
|
||||
running: false,
|
||||
summary: None,
|
||||
progress: None,
|
||||
resumable: Vec::new(),
|
||||
},
|
||||
Request::PhotosRefreshStatus => Response::PhotosRefresh { running: false },
|
||||
other => {
|
||||
eprintln!("demo_daemon: no demo answer for {}", request_name(other));
|
||||
Response::Ok {
|
||||
message: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn counts(&self) -> (usize, usize, usize) {
|
||||
let of = |kind| self.photos.iter().filter(|p| p.kind == kind).count();
|
||||
(
|
||||
of(PhotoKind::Photo),
|
||||
of(PhotoKind::Video),
|
||||
of(PhotoKind::Raw),
|
||||
)
|
||||
}
|
||||
|
||||
fn months(&self, kind: Option<PhotoKind>) -> Vec<PhotoMonth> {
|
||||
let mut months: BTreeMap<(i32, i32), usize> = BTreeMap::new();
|
||||
for photo in self
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| kind.is_none_or(|k| p.kind == k))
|
||||
{
|
||||
if let Some(at) = Local.timestamp_opt(photo.capture_time, 0).single() {
|
||||
*months.entry((at.year(), at.month() as i32)).or_default() += 1;
|
||||
}
|
||||
}
|
||||
months
|
||||
.into_iter()
|
||||
.rev()
|
||||
.map(|((year, month), count)| PhotoMonth { year, month, count })
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn albums(&self) -> Vec<AlbumInfo> {
|
||||
ALBUMS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, name)| AlbumInfo {
|
||||
uid: format!("demo-album~{i}"),
|
||||
name: (*name).into(),
|
||||
photo_count: self.album_photos(i).count(),
|
||||
cover_uid: self.album_photos(i).next().map(|p| p.uid.clone()),
|
||||
last_activity: Some(self.now - (i as i64 * 9 + 2) * DAY),
|
||||
shared: i == 1,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn album_photos(&self, album: usize) -> impl Iterator<Item = &PhotoItem> {
|
||||
let stride = 3 + album;
|
||||
self.photos
|
||||
.iter()
|
||||
.skip(album * 2)
|
||||
.step_by(stride)
|
||||
.take(40 + album * 17)
|
||||
}
|
||||
|
||||
fn in_place(&self, id: u32) -> impl Iterator<Item = (usize, &PhotoItem)> {
|
||||
self.photos
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(move |(i, _)| self.place_of[*i].is_some_and(|p| p as u32 + 1 == id))
|
||||
}
|
||||
|
||||
fn places(&self) -> Vec<PlaceInfo> {
|
||||
PLACES
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, (name, country, latitude, longitude))| {
|
||||
let id = i as u32 + 1;
|
||||
let count = self.in_place(id).count();
|
||||
let cover = self.in_place(id).next()?.1.clone();
|
||||
Some(PlaceInfo {
|
||||
id,
|
||||
name: (*name).into(),
|
||||
country: (*country).into(),
|
||||
latitude: *latitude,
|
||||
longitude: *longitude,
|
||||
photo_count: count,
|
||||
cover,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn place_locations(&self, id: u32) -> Vec<PhotoLocation> {
|
||||
let Some((_, _, latitude, longitude)) = PLACES.get(id as usize - 1) else {
|
||||
return Vec::new();
|
||||
};
|
||||
self.in_place(id)
|
||||
.map(|(i, photo)| {
|
||||
// Spread around the centre, the same way on every run.
|
||||
let angle = i as f64 * 2.399;
|
||||
let reach = 0.004 + (i % 9) as f64 * 0.003;
|
||||
PhotoLocation {
|
||||
photo: photo.clone(),
|
||||
latitude: latitude + reach * angle.sin(),
|
||||
longitude: longitude + reach * angle.cos(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// My files: folders first, then a few files, some of them pictures with
|
||||
/// thumbnails. Any folder below the root holds the same few documents.
|
||||
fn dir(&self, path: &str) -> Vec<DirEntry> {
|
||||
let folder = |name: &str, days: i64| DirEntry {
|
||||
name: name.into(),
|
||||
is_dir: true,
|
||||
modified: self.now - days * DAY,
|
||||
uid: format!("demo-files~{path}/{name}"),
|
||||
path: join(path, name),
|
||||
..Default::default()
|
||||
};
|
||||
let file = |name: &str, size: u64, days: i64| DirEntry {
|
||||
name: name.into(),
|
||||
is_dir: false,
|
||||
size,
|
||||
modified: self.now - days * DAY - 4000,
|
||||
uid: format!("demo-files~{path}/{name}"),
|
||||
path: join(path, name),
|
||||
..Default::default()
|
||||
};
|
||||
if !path.trim_matches('/').is_empty() {
|
||||
return vec![
|
||||
file("Notes.md", 4_812, 1),
|
||||
file("Overview.pdf", 912_384, 6),
|
||||
file("Budget.ods", 38_112, 12),
|
||||
];
|
||||
}
|
||||
let mut entries = vec![
|
||||
folder("Documents", 1),
|
||||
folder("Music", 30),
|
||||
folder("Pictures", 2),
|
||||
folder("Projects", 4),
|
||||
folder("Recipes", 60),
|
||||
folder("Taxes", 120),
|
||||
folder("Videos", 14),
|
||||
folder("Work", 3),
|
||||
];
|
||||
let mut pinned = file("Lisbon sunset.jpg", 4_218_112, 20);
|
||||
pinned.pinned = true;
|
||||
pinned.cached = true;
|
||||
entries.extend([
|
||||
file("Apartment lease.pdf", 1_482_011, 40),
|
||||
file("Budget 2026.ods", 52_340, 2),
|
||||
file("Cliffs.jpg", 3_844_201, 33),
|
||||
file("Garden.jpg", 2_904_877, 9),
|
||||
pinned,
|
||||
file("Meeting notes.md", 6_120, 0),
|
||||
file("Mountain lake.jpg", 5_120_330, 61),
|
||||
file("Packing list.txt", 1_204, 5),
|
||||
file("Road trip.jpg", 3_211_008, 75),
|
||||
file("Thesis draft.odt", 812_448, 18),
|
||||
]);
|
||||
entries
|
||||
}
|
||||
|
||||
/// The picture behind a demo file, for its thumbnail.
|
||||
fn file_picture(&self, uid: &str) -> Option<String> {
|
||||
let name = uid.rsplit('/').next()?;
|
||||
if !name.ends_with(".jpg") {
|
||||
return None;
|
||||
}
|
||||
let index = name.bytes().map(usize::from).sum::<usize>() % self.pictures.len();
|
||||
Some(self.pictures[index].display().to_string())
|
||||
}
|
||||
|
||||
fn pins(&self) -> Vec<Pin> {
|
||||
[
|
||||
("Documents", true),
|
||||
("Work/Quarterly report.odt", false),
|
||||
("Lisbon sunset.jpg", false),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(path, dir)| Pin {
|
||||
uid: format!("demo-files~/{path}"),
|
||||
path: (*path).into(),
|
||||
recursive: dir,
|
||||
is_dir: Some(dir),
|
||||
cached: true,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn devices(&self) -> Vec<DeviceInfo> {
|
||||
vec![
|
||||
DeviceInfo {
|
||||
uid: "demo-devices~1".into(),
|
||||
name: "alex-laptop".into(),
|
||||
device_type: "Linux".into(),
|
||||
last_sync: Some(self.now - 240),
|
||||
this_device: true,
|
||||
adopted: true,
|
||||
},
|
||||
DeviceInfo {
|
||||
uid: "demo-devices~2".into(),
|
||||
name: "studio-desktop".into(),
|
||||
device_type: "Windows".into(),
|
||||
last_sync: Some(self.now - 2 * DAY - 3 * 3600),
|
||||
this_device: false,
|
||||
adopted: false,
|
||||
},
|
||||
DeviceInfo {
|
||||
uid: "demo-devices~3".into(),
|
||||
name: "old-macbook".into(),
|
||||
device_type: "MacOs".into(),
|
||||
last_sync: Some(self.now - 210 * DAY),
|
||||
this_device: false,
|
||||
adopted: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// This computer's synced folders: mirrors, and one kept online only.
|
||||
const FOLDERS: &[(&str, &str)] = &[
|
||||
("/home/alex/Documents", "mirror"),
|
||||
("/home/alex/Pictures/Camera", "mirror"),
|
||||
("/home/alex/Music", "ondemand"),
|
||||
("/home/alex/Projects", "mirror"),
|
||||
];
|
||||
|
||||
fn sync_folders(&self) -> Vec<SyncFolderInfo> {
|
||||
Self::FOLDERS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (path, mode))| SyncFolderInfo {
|
||||
id: i as i64 + 1,
|
||||
kind: "device".into(),
|
||||
local_path: (*path).into(),
|
||||
remote_uid: format!("demo-sync~{i}"),
|
||||
mode: (*mode).into(),
|
||||
pending_mode: None,
|
||||
state: "idle".into(),
|
||||
last_sync: self.now - 240 - i as i64 * 60,
|
||||
progress: None,
|
||||
paused: false,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn locations(&self) -> Vec<MountSpec> {
|
||||
let mut items = vec![MountSpec {
|
||||
id: 1,
|
||||
kind: MountKind::MyFiles,
|
||||
local_path: MOUNTPOINT.into(),
|
||||
root_uid: "demo-files~".into(),
|
||||
root_share_id: String::new(),
|
||||
mode: MountMode::OnDemand,
|
||||
access: MountAccess::Rw,
|
||||
state: "idle".into(),
|
||||
last_sync: self.now - 60,
|
||||
pending_mode: None,
|
||||
mounted: true,
|
||||
progress: None,
|
||||
paused: false,
|
||||
}];
|
||||
items.extend(
|
||||
Self::FOLDERS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (path, mode))| MountSpec {
|
||||
id: i as i64 + 2,
|
||||
kind: MountKind::Device {
|
||||
sync_folder_id: i as i64 + 1,
|
||||
},
|
||||
local_path: (*path).into(),
|
||||
root_uid: format!("demo-sync~{i}"),
|
||||
root_share_id: String::new(),
|
||||
mode: MountMode::from(*mode),
|
||||
access: MountAccess::Rw,
|
||||
state: "idle".into(),
|
||||
last_sync: self.now - 240 - i as i64 * 60,
|
||||
pending_mode: None,
|
||||
mounted: *mode == "ondemand",
|
||||
progress: None,
|
||||
paused: false,
|
||||
}),
|
||||
);
|
||||
items
|
||||
}
|
||||
|
||||
fn activity(&self) -> Vec<ActivityEntry> {
|
||||
let at = |minutes: i64| self.now - minutes * 60;
|
||||
let entry = |time, kind, target: &str, detail: &str, ok| ActivityEntry {
|
||||
time,
|
||||
kind,
|
||||
target: target.into(),
|
||||
detail: detail.into(),
|
||||
ok,
|
||||
};
|
||||
let conflict = |time, name: &str, ext: &str| {
|
||||
entry(
|
||||
time,
|
||||
ActivityKind::Conflict,
|
||||
&format!("{name} (sync-conflict {time}).{ext}"),
|
||||
&format!("differs from {name}.{ext}"),
|
||||
false,
|
||||
)
|
||||
};
|
||||
vec![
|
||||
entry(at(2), ActivityKind::Upload, "Meeting notes.md", "", true),
|
||||
entry(
|
||||
at(14),
|
||||
ActivityKind::Share,
|
||||
"Budget 2026.ods",
|
||||
"1 recipient(s) as editor",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(38),
|
||||
ActivityKind::Upload,
|
||||
"Pictures/Camera/IMG_20481.JPG",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(39),
|
||||
ActivityKind::Upload,
|
||||
"Pictures/Camera/IMG_20480.JPG",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(52),
|
||||
ActivityKind::Rename,
|
||||
"Thesis draft.odt",
|
||||
"was thesis-v3.odt",
|
||||
true,
|
||||
),
|
||||
conflict(at(180), "Quarterly report", "odt"),
|
||||
entry(
|
||||
at(240),
|
||||
ActivityKind::Move,
|
||||
"Taxes/2025",
|
||||
"from Documents",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(26 * 60),
|
||||
ActivityKind::PublicLink,
|
||||
"Road trip.jpg",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
conflict(at(26 * 60 + 5), "Packing list", "txt"),
|
||||
entry(at(27 * 60), ActivityKind::CreateFolder, "Recipes", "", true),
|
||||
conflict(at(28 * 60), "Garden plan", "svg"),
|
||||
entry(at(29 * 60), ActivityKind::Trash, "Old invoices", "", true),
|
||||
entry(
|
||||
at(30 * 60),
|
||||
ActivityKind::Restore,
|
||||
"Apartment lease.pdf",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(50 * 60),
|
||||
ActivityKind::Upload,
|
||||
"Videos/Harbour at dusk.mp4",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
/// Hits for the search launcher: every demo file whose name holds the
|
||||
/// query, or a few to show when nothing does.
|
||||
fn search(&self, query: &str, limit: usize) -> Vec<SearchHit> {
|
||||
let names = [
|
||||
("Work/Quarterly report.odt", false, 180_912),
|
||||
("Work/Quarterly report 2025.pdf", false, 1_204_334),
|
||||
("Work/Reports", true, 0),
|
||||
("Documents/Expense report March.ods", false, 44_201),
|
||||
("Documents/Lab report.pdf", false, 2_301_442),
|
||||
("Projects/Annual report draft.md", false, 18_400),
|
||||
("Taxes/2025/Tax report.pdf", false, 640_008),
|
||||
("Pictures/Report cover.jpg", false, 2_801_220),
|
||||
];
|
||||
let needle = query.to_lowercase();
|
||||
names
|
||||
.iter()
|
||||
.filter(|(path, _, _)| needle.is_empty() || path.to_lowercase().contains(&needle))
|
||||
.take(limit)
|
||||
.enumerate()
|
||||
.map(|(i, (path, is_dir, size))| SearchHit {
|
||||
name: path.rsplit('/').next().unwrap_or(path).into(),
|
||||
path: (*path).into(),
|
||||
is_dir: *is_dir,
|
||||
size: *size,
|
||||
modified: self.now - (i as i64 * 5 + 1) * DAY,
|
||||
pinned: i == 0,
|
||||
cached: i == 0,
|
||||
uid: format!("demo-files~/{path}"),
|
||||
mounted_path: Some(format!("{MOUNTPOINT}/{path}")),
|
||||
score: 100 - i as i64,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn local_search(&self, query: &str) -> Vec<LocalHit> {
|
||||
let needle = query.to_lowercase();
|
||||
[("/home/alex/Downloads/report-template.ott", 22_018)]
|
||||
.iter()
|
||||
.filter(|(path, _)| path.to_lowercase().contains(&needle))
|
||||
.map(|(path, size)| LocalHit {
|
||||
name: path.rsplit('/').next().unwrap_or(path).into(),
|
||||
path: (*path).into(),
|
||||
is_dir: false,
|
||||
size: *size,
|
||||
modified: self.now - 4 * DAY,
|
||||
score: 10,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn join(dir: &str, name: &str) -> String {
|
||||
let dir = dir.trim_matches('/');
|
||||
if dir.is_empty() {
|
||||
name.into()
|
||||
} else {
|
||||
format!("{dir}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
/// The variant name of `request`, for the log of requests the demo ignores.
|
||||
fn request_name(request: &Request) -> String {
|
||||
let json = serde_json::to_string(request).unwrap_or_default();
|
||||
json.trim_start_matches(['{', '"'])
|
||||
.split('"')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
@@ -123,13 +123,84 @@ fn client_config() -> ProtonClientConfiguration {
|
||||
ProtonClientConfiguration::new(app_version).with_user_agent(user_agent)
|
||||
}
|
||||
|
||||
fn keyring_entry() -> Result<Entry> {
|
||||
// Installed lazily rather than once at startup so a session bus that was not
|
||||
// up yet on the first attempt is retried on the next one.
|
||||
if keyring_core::get_default_store().is_none() {
|
||||
keyring_core::set_default_store(dbus_secret_service_keyring_store::Store::new()?);
|
||||
/// Run `op` on the session's keyring entry.
|
||||
fn with_keyring<T>(op: impl Fn(&Entry) -> keyring_core::Result<T>) -> Result<T> {
|
||||
with_keyring_entry(KEYRING_SERVICE, KEYRING_USER, op)
|
||||
}
|
||||
|
||||
/// Run `op` on the entry `service`/`user`, reconnecting once if the Secret
|
||||
/// Service fails the call.
|
||||
///
|
||||
/// The store holds one D-Bus connection and one Secret Service session for the
|
||||
/// whole process. The session lives in the provider's memory: once the provider
|
||||
/// restarts (oo7's unit restarts it on failure, and both oo7 and gnome-keyring
|
||||
/// can be `--replace`d), every read and write on the old session fails with
|
||||
/// `NoSession`, for as long as this process runs. A daemon then keeps refreshed
|
||||
/// tokens only in memory, and a cold boot loses them (`docs/BUGS.md` B285). A
|
||||
/// platform failure therefore drops the store, and the call is made once more
|
||||
/// on a new one. A locked keyring, a dismissed prompt or a missing entry is
|
||||
/// not the connection's fault and comes back as it is.
|
||||
fn with_keyring_entry<T>(
|
||||
service: &str,
|
||||
user: &str,
|
||||
op: impl Fn(&Entry) -> keyring_core::Result<T>,
|
||||
) -> Result<T> {
|
||||
let store = keyring_store()?;
|
||||
match op(&Entry::new(service, user)?) {
|
||||
Err(keyring_core::Error::PlatformFailure(e)) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"the Secret Service failed this process's connection; reconnecting"
|
||||
);
|
||||
drop_keyring_store(&store);
|
||||
keyring_store()?;
|
||||
Ok(op(&Entry::new(service, user)?)?)
|
||||
}
|
||||
result => Ok(result?),
|
||||
}
|
||||
}
|
||||
|
||||
/// Held while the default store is looked at and replaced, so that threads
|
||||
/// failing at the same time drop the broken store once and share its
|
||||
/// replacement.
|
||||
static KEYRING_STORE: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
|
||||
/// The process's Secret Service store, connected now if there is none.
|
||||
///
|
||||
/// Connected lazily rather than once at startup so a session bus that was not
|
||||
/// up yet on the first attempt is tried again on the next one.
|
||||
fn keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||
let _swap = KEYRING_STORE.lock();
|
||||
if let Some(store) = keyring_core::get_default_store() {
|
||||
return Ok(store);
|
||||
}
|
||||
let store = connect_keyring_store()?;
|
||||
keyring_core::set_default_store(Arc::clone(&store));
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
/// Forget `failed`, unless another thread has already replaced it.
|
||||
fn drop_keyring_store(failed: &Arc<keyring_core::CredentialStore>) {
|
||||
let _swap = KEYRING_STORE.lock();
|
||||
if keyring_core::get_default_store().is_some_and(|store| Arc::ptr_eq(&store, failed)) {
|
||||
keyring_core::unset_default_store();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||
Ok(dbus_secret_service_keyring_store::Store::new()?)
|
||||
}
|
||||
|
||||
/// The tests' in-memory store when they set one, so a reconnect finds the
|
||||
/// same entries; the real Secret Service otherwise.
|
||||
#[cfg(test)]
|
||||
fn connect_keyring_store() -> Result<Arc<keyring_core::CredentialStore>> {
|
||||
tests::KEYRING_CONNECTS.fetch_add(1, Ordering::SeqCst);
|
||||
match tests::MOCK_STORE.lock().clone() {
|
||||
Some(store) => Ok(store),
|
||||
None => Ok(dbus_secret_service_keyring_store::Store::new()?),
|
||||
}
|
||||
Ok(Entry::new(KEYRING_SERVICE, KEYRING_USER)?)
|
||||
}
|
||||
|
||||
/// What the user answered when asked for the mailbox password.
|
||||
@@ -421,17 +492,15 @@ fn classify_verification_gate(e: ProtonError) -> Error {
|
||||
|
||||
fn write_stored(stored: &StoredSession) -> Result<()> {
|
||||
let json = Zeroizing::new(serde_json::to_string(stored)?);
|
||||
keyring_entry()?.set_password(&json)?;
|
||||
Ok(())
|
||||
with_keyring(|entry| entry.set_password(&json))
|
||||
}
|
||||
|
||||
/// Load the persisted session blob, or `Error::NotLoggedIn` if absent.
|
||||
pub fn load() -> Result<StoredSession> {
|
||||
let entry = keyring_entry()?;
|
||||
match entry.get_password() {
|
||||
match with_keyring(|entry| entry.get_password()) {
|
||||
Ok(json) => Ok(serde_json::from_str(&json)?),
|
||||
Err(keyring_core::Error::NoEntry) => Err(Error::NotLoggedIn),
|
||||
Err(e) => Err(e.into()),
|
||||
Err(Error::Keyring(keyring_core::Error::NoEntry)) => Err(Error::NotLoggedIn),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -589,9 +658,9 @@ pub fn logout() -> Result<bool> {
|
||||
tracing::warn!(error = %e, "clearing the kernel-keyring unlock on logout failed");
|
||||
}
|
||||
let _ = kernelkey::UNSAVED_TOKENS.clear();
|
||||
match keyring_entry()?.delete_credential() {
|
||||
Ok(()) | Err(keyring_core::Error::NoEntry) => Ok(ended),
|
||||
Err(e) => Err(e.into()),
|
||||
match with_keyring(|entry| entry.delete_credential()) {
|
||||
Ok(()) | Err(Error::Keyring(keyring_core::Error::NoEntry)) => Ok(ended),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1207,12 +1276,138 @@ mod tests {
|
||||
/// The keyring is one store per process, so the tests that use it,
|
||||
/// through `keyring-core`'s in-memory store, take turns.
|
||||
fn mock_keyring() -> parking_lot::MutexGuard<'static, ()> {
|
||||
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
let turn = KEYRING.lock();
|
||||
keyring_core::set_default_store(keyring_core::mock::Store::new().unwrap());
|
||||
let store: Arc<keyring_core::CredentialStore> = keyring_core::mock::Store::new().unwrap();
|
||||
*MOCK_STORE.lock() = Some(Arc::clone(&store));
|
||||
keyring_core::set_default_store(store);
|
||||
turn
|
||||
}
|
||||
|
||||
static KEYRING: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
|
||||
/// The in-memory store a reconnect hands out, so it finds the same
|
||||
/// entries; `None` connects to the real Secret Service.
|
||||
pub(super) static MOCK_STORE: parking_lot::Mutex<Option<Arc<keyring_core::CredentialStore>>> =
|
||||
parking_lot::Mutex::new(None);
|
||||
|
||||
/// How many times a store was connected.
|
||||
pub(super) static KEYRING_CONNECTS: std::sync::atomic::AtomicUsize =
|
||||
std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
/// The session's entry, to make its next call fail.
|
||||
fn keyring_entry() -> Entry {
|
||||
Entry::new(KEYRING_SERVICE, KEYRING_USER).unwrap()
|
||||
}
|
||||
|
||||
/// Make the session entry's next call fail with `error`.
|
||||
fn fail_next(error: keyring_core::Error) {
|
||||
keyring_entry()
|
||||
.as_any()
|
||||
.downcast_ref::<keyring_core::mock::Cred>()
|
||||
.unwrap()
|
||||
.set_error(error);
|
||||
}
|
||||
|
||||
fn no_session() -> keyring_core::Error {
|
||||
keyring_core::Error::PlatformFailure(
|
||||
std::io::Error::other("The session does not exist (NoSession)").into(),
|
||||
)
|
||||
}
|
||||
|
||||
/// B285: a restarted Secret Service no longer knows the process's
|
||||
/// session, and every call on it failed until the process restarted.
|
||||
#[test]
|
||||
fn a_dropped_secret_service_session_is_reconnected_and_retried() {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||
|
||||
fail_next(no_session());
|
||||
assert_eq!(load().unwrap().refresh_token, "r");
|
||||
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 1);
|
||||
|
||||
fail_next(no_session());
|
||||
write_stored(&stored_with("r2")).unwrap();
|
||||
assert_eq!(load().unwrap().refresh_token, "r2");
|
||||
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rotation_lands_at_once_after_the_keyring_restarts() {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||
|
||||
fail_next(no_session());
|
||||
sink.rotated(tokens("a2", "r2")).unwrap();
|
||||
assert_eq!(load().unwrap().refresh_token, "r2");
|
||||
assert!(sink.state.lock().unsaved.is_none());
|
||||
}
|
||||
|
||||
/// A locked keyring or a dismissed prompt is no fault of the connection.
|
||||
#[test]
|
||||
fn a_locked_keyring_is_not_reconnected() {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||
|
||||
fail_next(keyring_core::Error::NoStorageAccess(
|
||||
std::io::Error::other("locked").into(),
|
||||
));
|
||||
assert!(matches!(
|
||||
load(),
|
||||
Err(Error::Keyring(keyring_core::Error::NoStorageAccess(_)))
|
||||
));
|
||||
assert_eq!(KEYRING_CONNECTS.load(Ordering::SeqCst), connects);
|
||||
assert_eq!(load().unwrap().refresh_token, "r");
|
||||
}
|
||||
|
||||
/// The real Secret Service, restarted between two calls on one store.
|
||||
///
|
||||
/// Run inside a throwaway session bus, never against the desktop's
|
||||
/// keyring. With gnome-keyring:
|
||||
///
|
||||
/// ```sh
|
||||
/// echo -n pw | gnome-keyring-daemon --unlock --components=secrets
|
||||
/// PDFS_SECRET_SERVICE_RESTART_TEST=1 \
|
||||
/// PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets' \
|
||||
/// cargo test -p pdfs-core real_secret_service -- --ignored
|
||||
/// ```
|
||||
#[test]
|
||||
#[ignore = "needs a throwaway Secret Service; see the doc comment"]
|
||||
fn real_secret_service_survives_a_provider_restart() {
|
||||
if std::env::var_os("PDFS_SECRET_SERVICE_RESTART_TEST").is_none() {
|
||||
return;
|
||||
}
|
||||
let restart = std::env::var("PDFS_RESTART_PROVIDER").expect("PDFS_RESTART_PROVIDER");
|
||||
let _turn = KEYRING.lock();
|
||||
*MOCK_STORE.lock() = None;
|
||||
keyring_core::unset_default_store();
|
||||
let (service, user) = ("pdfs-test", "restart");
|
||||
|
||||
with_keyring_entry(service, user, |e| e.set_password("before")).unwrap();
|
||||
let status = std::process::Command::new("sh")
|
||||
.args(["-c", &restart])
|
||||
.status()
|
||||
.unwrap();
|
||||
assert!(status.success(), "restarting the provider failed");
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
|
||||
let connects = KEYRING_CONNECTS.load(Ordering::SeqCst);
|
||||
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
|
||||
assert_eq!(read, "before");
|
||||
assert_eq!(
|
||||
KEYRING_CONNECTS.load(Ordering::SeqCst),
|
||||
connects + 1,
|
||||
"the provider kept the old session, so nothing was tested"
|
||||
);
|
||||
with_keyring_entry(service, user, |e| e.set_password("after")).unwrap();
|
||||
let read = with_keyring_entry(service, user, |e| e.get_password()).unwrap();
|
||||
assert_eq!(read, "after");
|
||||
with_keyring_entry(service, user, |e| e.delete_credential()).unwrap();
|
||||
keyring_core::unset_default_store();
|
||||
}
|
||||
|
||||
fn stored_with(refresh: &str) -> StoredSession {
|
||||
serde_json::from_str(&blob("").replace(r#""r""#, &format!("{refresh:?}"))).unwrap()
|
||||
}
|
||||
@@ -1267,12 +1462,7 @@ mod tests {
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||
|
||||
let entry = keyring_entry().unwrap();
|
||||
let cred = entry
|
||||
.as_any()
|
||||
.downcast_ref::<keyring_core::mock::Cred>()
|
||||
.unwrap();
|
||||
cred.set_error(keyring_core::Error::NoStorageAccess(
|
||||
fail_next(keyring_core::Error::NoStorageAccess(
|
||||
std::io::Error::other("locked").into(),
|
||||
));
|
||||
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
||||
@@ -1307,12 +1497,7 @@ mod tests {
|
||||
let _turn = mock_keyring();
|
||||
write_stored(&stored_with("r")).unwrap();
|
||||
let sink = Arc::new(TokenSink::new("s", "r".into()));
|
||||
let entry = keyring_entry().unwrap();
|
||||
entry
|
||||
.as_any()
|
||||
.downcast_ref::<keyring_core::mock::Cred>()
|
||||
.unwrap()
|
||||
.set_error(keyring_core::Error::NoEntry);
|
||||
fail_next(keyring_core::Error::NoEntry);
|
||||
|
||||
assert!(sink.rotated(tokens("a2", "r2")).is_err());
|
||||
assert!(sink.state.lock().unsaved.is_some());
|
||||
|
||||
@@ -562,6 +562,12 @@ impl AppDirs {
|
||||
self.state_dir().join("control.sock")
|
||||
}
|
||||
|
||||
/// The Google Photos import's last run: its archives and how it ended, so
|
||||
/// an import cut short by a restart can be resumed.
|
||||
pub fn takeout_import_path(&self) -> PathBuf {
|
||||
self.state_dir().join("takeout-import.json")
|
||||
}
|
||||
|
||||
/// Unix domain socket the tray uses to ensure single instance.
|
||||
pub fn tray_socket(&self) -> PathBuf {
|
||||
self.state_dir().join("tray.sock")
|
||||
|
||||
@@ -279,6 +279,11 @@ pub enum Request {
|
||||
/// and files what it has uploaded into its albums, then reports
|
||||
/// `cancelled`. Replies with [`Response::Ok`].
|
||||
CancelImport,
|
||||
/// Start the last import again with the same archives, after it was
|
||||
/// interrupted, cancelled or failed. Photos it already uploaded are found
|
||||
/// on the account and skipped. Replies like [`Request::ImportTakeout`];
|
||||
/// [`ErrorKind::NotFound`] when there is nothing to resume.
|
||||
ResumeImport,
|
||||
/// Upload the photo at `source_path` under the given name and media type.
|
||||
///
|
||||
/// A path, not the bytes: this protocol is line-delimited JSON, and
|
||||
@@ -1689,10 +1694,75 @@ pub struct ImportSummary {
|
||||
pub bytes: u64,
|
||||
/// True when the run stopped early because it was cancelled.
|
||||
pub cancelled: bool,
|
||||
/// Why the run as a whole failed, when it did, such as an archive that
|
||||
/// is not a zip. The counts are then empty.
|
||||
/// Why the run as a whole stopped early, when it did, such as the
|
||||
/// account being unreachable. The counts say how far it got.
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
/// What kind of failure [`error`](Self::error) is, so a front-end can word
|
||||
/// it in the user's language. The text itself is the daemon's English.
|
||||
#[serde(default)]
|
||||
pub error_kind: Option<ErrorKind>,
|
||||
/// True for a scan that uploads nothing ([`Request::ImportTakeout`] with
|
||||
/// `dry_run`). [`to_upload`](Self::to_upload) then says what an import
|
||||
/// would upload.
|
||||
#[serde(default)]
|
||||
pub dry_run: bool,
|
||||
/// Photos not yet on the account: what a dry run would upload, and what a
|
||||
/// real run set out to upload once the duplicates were known.
|
||||
#[serde(default)]
|
||||
pub to_upload: usize,
|
||||
/// Bytes of [`to_upload`](Self::to_upload).
|
||||
#[serde(default)]
|
||||
pub to_upload_bytes: u64,
|
||||
/// Live and motion photos whose video was filed with its still.
|
||||
#[serde(default)]
|
||||
pub live_photos: usize,
|
||||
/// True when the service stopped while the import ran (a shutdown or a
|
||||
/// restart). [`Request::ResumeImport`] picks it up again.
|
||||
#[serde(default)]
|
||||
pub interrupted: bool,
|
||||
/// Archives that could not be read and were left out, by path.
|
||||
#[serde(default)]
|
||||
pub unreadable_archives: Vec<String>,
|
||||
}
|
||||
|
||||
impl ImportSummary {
|
||||
/// Whether the run did all it set out to do.
|
||||
pub fn complete(&self) -> bool {
|
||||
self.error.is_none()
|
||||
&& !self.cancelled
|
||||
&& !self.interrupted
|
||||
&& self.failed == 0
|
||||
&& self.unreadable_archives.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Which step a running Takeout import is on.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ImportPhase {
|
||||
/// Reading the archives' directories and metadata.
|
||||
#[default]
|
||||
Scanning,
|
||||
/// Asking the account which photos it already has.
|
||||
Checking,
|
||||
/// Uploading the photos that are new.
|
||||
Uploading,
|
||||
/// Creating albums and filing photos into them.
|
||||
Albums,
|
||||
}
|
||||
|
||||
/// Where a running import is ([`Response::ImportStatus`]).
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ImportProgress {
|
||||
pub phase: ImportPhase,
|
||||
/// Steps done in this phase.
|
||||
pub done: u64,
|
||||
/// Steps in this phase; `0` while not yet known.
|
||||
pub total: u64,
|
||||
/// The photo being uploaded, during [`ImportPhase::Uploading`].
|
||||
#[serde(default)]
|
||||
pub current: Option<String>,
|
||||
}
|
||||
|
||||
/// One album in a [`Response::Albums`] listing.
|
||||
@@ -2208,8 +2278,17 @@ pub enum Response {
|
||||
/// report; `summary` is `None` when no import has run this session.
|
||||
ImportStatus {
|
||||
running: bool,
|
||||
/// The counts so far while running, the final report after.
|
||||
#[serde(default)]
|
||||
summary: Option<ImportSummary>,
|
||||
/// The current step, while running.
|
||||
#[serde(default)]
|
||||
progress: Option<ImportProgress>,
|
||||
/// The archives of the last run, when it can be resumed with
|
||||
/// [`Request::ResumeImport`]: it was a real import that did not
|
||||
/// finish. Survives a restart of the service.
|
||||
#[serde(default)]
|
||||
resumable: Vec<String>,
|
||||
},
|
||||
/// How the photo re-date is doing (reply to [`Request::RedateStatus`]).
|
||||
/// `running` is false once it has finished, when `summary` is the final
|
||||
|
||||
@@ -25,7 +25,9 @@
|
||||
//! The service folder (`Google Photos`) is localized, as are the year folders
|
||||
//! (`Photos from 2019`) and the trash folder, so nothing here matches on the
|
||||
//! service name: the first component is dropped when it is the export root, the
|
||||
//! second is the service, and the rest is `<folder>/<file>`. A folder is an
|
||||
//! second is the service, and the rest is `<folder>/<file>`. The Photos service
|
||||
//! is the one whose sidecars carry a `photoTakenTime`; other services in the same
|
||||
//! export (`Google Drive/`, `YouTube/`) are left out. A folder is an
|
||||
//! **album** when it carries an album metadata sidecar naming it; anything else
|
||||
//! is a plain bucket that imports into the timeline only. Trash is matched by
|
||||
//! name against [`TRASH_FOLDERS`] — the one place a localized name is
|
||||
@@ -121,8 +123,14 @@ const MEDIA_EXTENSIONS: &[(&str, &str)] = &[
|
||||
("m2ts", "video/mp2t"),
|
||||
("mpg", "video/mpeg"),
|
||||
("mpeg", "video/mpeg"),
|
||||
// The video half of a Pixel motion photo, `PXL_….MP` beside `PXL_….MP.jpg`.
|
||||
("mp", "video/mp4"),
|
||||
];
|
||||
|
||||
/// Sidecars bigger than this are not read. A real one is a few kilobytes; a
|
||||
/// `.json` this large is some other service's data and not worth the memory.
|
||||
const SIDECAR_MAX_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
/// The media type for `name`'s extension, or `None` when it is not media this
|
||||
/// import handles.
|
||||
pub fn media_type_of(name: &str) -> Option<&'static str> {
|
||||
@@ -147,6 +155,13 @@ pub enum Bucket {
|
||||
Archived,
|
||||
}
|
||||
|
||||
/// A position in WGS 84 decimal degrees, from a sidecar's `geoData`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct GeoPoint {
|
||||
pub latitude: f64,
|
||||
pub longitude: f64,
|
||||
}
|
||||
|
||||
/// One media file found in the export.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TakeoutPhoto {
|
||||
@@ -162,14 +177,49 @@ pub struct TakeoutPhoto {
|
||||
pub size: u64,
|
||||
/// Media type from the extension.
|
||||
pub media_type: String,
|
||||
/// Capture time in epoch seconds, from `photoTakenTime` (falling back to
|
||||
/// `creationTime`). `None` when the photo has no sidecar — the upload then
|
||||
/// defaults it to now, as the SDK does.
|
||||
/// Capture time in epoch seconds, from the sidecar's `photoTakenTime`,
|
||||
/// else from a date in the file name. `None` when neither has one; the
|
||||
/// importer then tries the file's EXIF, then [`creation_time`](Self::creation_time).
|
||||
pub capture_time: Option<i64>,
|
||||
/// The sidecar's `creationTime`: when the photo reached Google Photos, not
|
||||
/// when it was taken. Only a last resort for the capture time.
|
||||
#[serde(default)]
|
||||
pub creation_time: Option<i64>,
|
||||
/// Where the photo was taken, from the sidecar's `geoData` (or
|
||||
/// `geoDataExif`). Google writes `0, 0` for "unknown", which is dropped.
|
||||
#[serde(default)]
|
||||
pub location: Option<GeoPoint>,
|
||||
/// The sidecar's `favorited` flag.
|
||||
pub favorite: bool,
|
||||
/// Which bucket the file was found in.
|
||||
pub bucket: Bucket,
|
||||
/// Pairs the halves of a live or motion photo: a still and a video in the
|
||||
/// same folder carry the same key (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||
/// `PXL_1.MP.jpg` + `PXL_1.MP`). `None` for media that cannot be either half.
|
||||
#[serde(default)]
|
||||
pub live_key: Option<String>,
|
||||
}
|
||||
|
||||
impl TakeoutPhoto {
|
||||
/// Whether this is the video half of a pair, as opposed to the still.
|
||||
pub fn is_video(&self) -> bool {
|
||||
self.media_type.starts_with("video/")
|
||||
}
|
||||
|
||||
/// Whether the pair this belongs to is a Pixel motion photo (`.MP`) rather
|
||||
/// than an Apple live photo.
|
||||
pub fn is_motion_photo(&self) -> bool {
|
||||
self.live_key
|
||||
.as_deref()
|
||||
.is_some_and(|key| key.ends_with(".mp"))
|
||||
}
|
||||
}
|
||||
|
||||
/// An archive the scan had to leave out, and why.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct UnreadableArchive {
|
||||
pub path: PathBuf,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// Everything one scan found, ready to be turned into an import plan.
|
||||
@@ -183,8 +233,16 @@ pub struct TakeoutScan {
|
||||
/// Media files skipped because they sit in the trash bucket.
|
||||
pub skipped_trashed: usize,
|
||||
/// Entries that were neither media nor a sidecar (`archive_browser.html`,
|
||||
/// activity `.csv`s, `.mp` motion-photo parts).
|
||||
/// activity `.csv`s), and media from other Google services in the same
|
||||
/// export (`Google Drive/`, `YouTube/`).
|
||||
pub skipped_other: usize,
|
||||
/// Archives that could not be opened or are not a zip. The rest of the
|
||||
/// export is still imported.
|
||||
#[serde(default)]
|
||||
pub unreadable_archives: Vec<UnreadableArchive>,
|
||||
/// Entries inside an otherwise readable archive that could not be read.
|
||||
#[serde(default)]
|
||||
pub unreadable_entries: usize,
|
||||
}
|
||||
|
||||
impl TakeoutScan {
|
||||
@@ -195,7 +253,7 @@ impl TakeoutScan {
|
||||
}
|
||||
|
||||
/// Google's per-photo sidecar. Only the fields the import uses are read; the
|
||||
/// file also carries geo data, people tags and view counts.
|
||||
/// file also carries people tags, a description and view counts.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct PhotoSidecar {
|
||||
#[serde(default)]
|
||||
@@ -205,19 +263,69 @@ struct PhotoSidecar {
|
||||
#[serde(rename = "creationTime", default)]
|
||||
creation_time: Option<SidecarTime>,
|
||||
#[serde(default)]
|
||||
favorited: bool,
|
||||
favorited: Option<bool>,
|
||||
/// The position as edited in Google Photos, else as the camera wrote it.
|
||||
#[serde(rename = "geoData", default)]
|
||||
geo_data: Option<SidecarGeo>,
|
||||
#[serde(rename = "geoDataExif", default)]
|
||||
geo_data_exif: Option<SidecarGeo>,
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, as a *string*.
|
||||
impl PhotoSidecar {
|
||||
fn location(&self) -> Option<GeoPoint> {
|
||||
self.geo_data
|
||||
.as_ref()
|
||||
.and_then(SidecarGeo::point)
|
||||
.or_else(|| self.geo_data_exif.as_ref().and_then(SidecarGeo::point))
|
||||
}
|
||||
}
|
||||
|
||||
/// A sidecar timestamp: epoch seconds, written as a string, sometimes a number.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarTime {
|
||||
#[serde(default)]
|
||||
timestamp: Option<String>,
|
||||
timestamp: Option<SidecarNumber>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum SidecarNumber {
|
||||
Number(i64),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
impl SidecarTime {
|
||||
/// The timestamp, when it is a real moment. `0` and negative values are
|
||||
/// Google's "unknown" and would file the photo under 1970.
|
||||
fn epoch_seconds(&self) -> Option<i64> {
|
||||
self.timestamp.as_ref()?.parse().ok()
|
||||
let seconds = match self.timestamp.as_ref()? {
|
||||
SidecarNumber::Number(seconds) => *seconds,
|
||||
SidecarNumber::Text(text) => text.trim().parse().ok()?,
|
||||
};
|
||||
(seconds > 0).then_some(seconds)
|
||||
}
|
||||
}
|
||||
|
||||
/// `geoData` in a sidecar. All-zero means Google has no position.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SidecarGeo {
|
||||
#[serde(default)]
|
||||
latitude: Option<f64>,
|
||||
#[serde(default)]
|
||||
longitude: Option<f64>,
|
||||
}
|
||||
|
||||
impl SidecarGeo {
|
||||
fn point(&self) -> Option<GeoPoint> {
|
||||
let (latitude, longitude) = (self.latitude?, self.longitude?);
|
||||
let on_globe = latitude.is_finite()
|
||||
&& longitude.is_finite()
|
||||
&& (-90.0..=90.0).contains(&latitude)
|
||||
&& (-180.0..=180.0).contains(&longitude);
|
||||
(on_globe && (latitude != 0.0 || longitude != 0.0)).then_some(GeoPoint {
|
||||
latitude,
|
||||
longitude,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,21 +370,58 @@ impl RawEntry {
|
||||
/// is one album, and a sidecar in part 3 is matched to its photo in part 1.
|
||||
/// Reads only the central directory and the JSON sidecars — no media bytes, so
|
||||
/// this stays fast on a 200 GB export.
|
||||
///
|
||||
/// An archive that cannot be opened is left out and listed in
|
||||
/// [`TakeoutScan::unreadable_archives`]; the rest still imports.
|
||||
pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<RawEntry> = Vec::new();
|
||||
let mut sidecars: HashMap<String, Vec<u8>> = HashMap::new();
|
||||
scan_archives_until(archives, || false)
|
||||
}
|
||||
|
||||
/// [`scan_archives`] that gives up when `stop` returns true, checked between
|
||||
/// archives and every few hundred entries.
|
||||
pub fn scan_archives_until(
|
||||
archives: &[PathBuf],
|
||||
stop: impl Fn() -> bool,
|
||||
) -> CoreResult<TakeoutScan> {
|
||||
let mut entries: Vec<(String, RawEntry)> = Vec::new();
|
||||
let mut sidecars: Vec<(String, String, Vec<u8>)> = Vec::new();
|
||||
let mut unreadable_archives = Vec::new();
|
||||
let mut unreadable_entries = 0;
|
||||
|
||||
for (index, path) in archives.iter().enumerate() {
|
||||
let file = std::fs::File::open(path)
|
||||
.map_err(|e| CoreError::invalid(format!("cannot open {}: {e}", path.display())))?;
|
||||
let mut zip = zip::ZipArchive::new(std::io::BufReader::new(file)).map_err(|e| {
|
||||
CoreError::invalid(format!("{} is not a readable zip: {e}", path.display()))
|
||||
})?;
|
||||
if stop() {
|
||||
return Err(CoreError::conflict("the import was cancelled"));
|
||||
}
|
||||
let opened = std::fs::File::open(path)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|file| {
|
||||
zip::ZipArchive::new(std::io::BufReader::new(file))
|
||||
.map_err(|e| format!("not a readable zip: {e}"))
|
||||
});
|
||||
let mut zip = match opened {
|
||||
Ok(zip) => zip,
|
||||
Err(reason) => {
|
||||
tracing::warn!(archive = %path.display(), %reason, "skipping an unreadable takeout archive");
|
||||
unreadable_archives.push(UnreadableArchive {
|
||||
path: path.clone(),
|
||||
reason,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for position in 0..zip.len() {
|
||||
let mut entry = zip.by_index(position).map_err(|e| {
|
||||
CoreError::invalid(format!("{}: unreadable entry: {e}", path.display()))
|
||||
})?;
|
||||
if position % 512 == 0 && stop() {
|
||||
return Err(CoreError::conflict("the import was cancelled"));
|
||||
}
|
||||
let mut entry = match zip.by_index(position) {
|
||||
Ok(entry) => entry,
|
||||
Err(e) => {
|
||||
tracing::debug!(archive = %path.display(), position, error = %e, "unreadable takeout entry");
|
||||
unreadable_entries += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if entry.is_dir() {
|
||||
continue;
|
||||
}
|
||||
@@ -286,32 +431,92 @@ pub fn scan_archives(archives: &[PathBuf]) -> CoreResult<TakeoutScan> {
|
||||
continue;
|
||||
};
|
||||
let name = name.to_string_lossy().replace('\\', "/");
|
||||
let Some(relative) = strip_export_prefix(&name) else {
|
||||
let Some((service, relative)) = strip_export_prefix(&name) else {
|
||||
continue;
|
||||
};
|
||||
let (service, relative) = (service.to_string(), relative.to_string());
|
||||
let size = entry.size();
|
||||
|
||||
// Sidecars are read now, while the entry is open: they are a few
|
||||
// hundred bytes each and reopening the archive per photo later would
|
||||
// be a second full pass.
|
||||
if relative.to_ascii_lowercase().ends_with(".json") {
|
||||
let mut body = Vec::with_capacity(size.min(64 * 1024) as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.insert(relative.to_string(), body);
|
||||
if size <= SIDECAR_MAX_BYTES {
|
||||
let mut body = Vec::with_capacity(size as usize);
|
||||
if entry.read_to_end(&mut body).is_ok() {
|
||||
sidecars.push((service, relative, body));
|
||||
} else {
|
||||
unreadable_entries += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
entries.push(RawEntry {
|
||||
archive: index,
|
||||
entry: name.clone(),
|
||||
relative: relative.to_string(),
|
||||
size,
|
||||
});
|
||||
entries.push((
|
||||
service,
|
||||
RawEntry {
|
||||
archive: index,
|
||||
entry: name,
|
||||
relative,
|
||||
size,
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(assemble(entries, &sidecars))
|
||||
let (entries, sidecars, other_services) = keep_photos_services(entries, sidecars);
|
||||
let mut scan = assemble(entries, &sidecars);
|
||||
scan.skipped_other += other_services;
|
||||
scan.unreadable_archives = unreadable_archives;
|
||||
scan.unreadable_entries = unreadable_entries;
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
/// Drop what other Google services put into the same export.
|
||||
///
|
||||
/// A Takeout can hold `Google Drive/`, `YouTube/` and others next to
|
||||
/// `Google Photos/`, and their media must not land in the photo timeline. The
|
||||
/// service folder's name is localized (`Google Fotos`), so the Photos service
|
||||
/// is recognized by its content instead: it is the folder whose sidecars carry
|
||||
/// a `photoTakenTime`. When no folder does — an export without sidecars, or
|
||||
/// one re-zipped without the service level — everything is kept, as before.
|
||||
///
|
||||
/// Returns the entries and sidecars to import, and how many media files were
|
||||
/// left out.
|
||||
fn keep_photos_services(
|
||||
entries: Vec<(String, RawEntry)>,
|
||||
sidecars: Vec<(String, String, Vec<u8>)>,
|
||||
) -> (Vec<RawEntry>, HashMap<String, Vec<u8>>, usize) {
|
||||
#[derive(Deserialize)]
|
||||
struct Marker {
|
||||
#[serde(rename = "photoTakenTime")]
|
||||
photo_taken_time: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
let mut photos_services: std::collections::HashSet<&str> = std::collections::HashSet::new();
|
||||
for (service, _, body) in &sidecars {
|
||||
if photos_services.contains(service.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if serde_json::from_slice::<Marker>(body).is_ok_and(|m| m.photo_taken_time.is_some()) {
|
||||
photos_services.insert(service.as_str());
|
||||
}
|
||||
}
|
||||
let keep = |service: &str| photos_services.is_empty() || photos_services.contains(service);
|
||||
|
||||
let mut skipped = 0;
|
||||
let mut kept_entries = Vec::with_capacity(entries.len());
|
||||
for (service, raw) in entries {
|
||||
if keep(&service) {
|
||||
kept_entries.push(raw);
|
||||
} else if media_type_of(raw.file_name()).is_some() {
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
let kept_sidecars = sidecars
|
||||
.iter()
|
||||
.filter(|(service, _, _)| keep(service))
|
||||
.map(|(_, relative, body)| (relative.clone(), body.clone()))
|
||||
.collect();
|
||||
(kept_entries, kept_sidecars, skipped)
|
||||
}
|
||||
|
||||
/// Build the scan from raw entries plus the sidecar bodies keyed by their path
|
||||
@@ -403,19 +608,19 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
||||
.unwrap_or(&file_name)
|
||||
.to_string();
|
||||
|
||||
// Sidecar first, then the date stamped into the name.
|
||||
// Sidecar first, then the date stamped into the name. `creationTime`
|
||||
// is when the photo reached Google, which for a scan or an old photo
|
||||
// uploaded late is years off; the importer uses it only after EXIF.
|
||||
let capture_time = sidecar
|
||||
.photo_taken_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
.or_else(|| {
|
||||
sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds)
|
||||
})
|
||||
.or_else(|| capture_time_from_name(&name))
|
||||
.or_else(|| capture_time_from_name(&file_name));
|
||||
let creation_time = sidecar
|
||||
.creation_time
|
||||
.as_ref()
|
||||
.and_then(SidecarTime::epoch_seconds);
|
||||
|
||||
scan.photos.push(TakeoutPhoto {
|
||||
archive: raw.archive,
|
||||
@@ -424,31 +629,57 @@ fn assemble(entries: Vec<RawEntry>, sidecars: &HashMap<String, Vec<u8>>) -> Take
|
||||
size: raw.size,
|
||||
media_type: media_type.to_string(),
|
||||
capture_time,
|
||||
favorite: sidecar.favorited,
|
||||
creation_time,
|
||||
location: sidecar.location(),
|
||||
favorite: sidecar.favorited.unwrap_or(false),
|
||||
bucket,
|
||||
live_key: live_key(&folder, &file_name),
|
||||
});
|
||||
}
|
||||
|
||||
scan
|
||||
}
|
||||
|
||||
/// Drop the export root (`Takeout/`) and the localized service folder from a
|
||||
/// path inside the archive, leaving `<folder>/<file>` or `<file>`.
|
||||
/// Split a path inside the archive into the localized service folder and the
|
||||
/// rest, dropping the export root (`Takeout/`): `("Google Fotos",
|
||||
/// "<folder>/<file>")`.
|
||||
///
|
||||
/// `None` for a path with no service folder at all (the export's own
|
||||
/// `archive_browser.html`), which carries nothing to import.
|
||||
fn strip_export_prefix(path: &str) -> Option<&str> {
|
||||
fn strip_export_prefix(path: &str) -> Option<(&str, &str)> {
|
||||
let mut rest = path;
|
||||
// The root is usually `Takeout/`, but an export unzipped and re-zipped by
|
||||
// the user may not have it — so the root is dropped only when a service
|
||||
// folder follows it, which the component count decides.
|
||||
// The root is `Takeout/`, but an export unzipped and re-zipped by the user
|
||||
// may not have it. `Google Photos/<album>/<file>` then has three parts like
|
||||
// `Takeout/Google Photos/<file>`, so the count alone cannot tell them
|
||||
// apart: a root of another name is only taken for one with four parts.
|
||||
let (first, after) = rest.split_once('/')?;
|
||||
let components = rest.split('/').count();
|
||||
if components >= 3 {
|
||||
rest = rest.split_once('/')?.1;
|
||||
if (first.eq_ignore_ascii_case("takeout") && components >= 3) || components >= 4 {
|
||||
rest = after;
|
||||
}
|
||||
// Drop the service folder (`Google Photos`, `Google Fotos`, …).
|
||||
let (_service, tail) = rest.split_once('/')?;
|
||||
Some(tail)
|
||||
// The service folder (`Google Photos`, `Google Fotos`, …).
|
||||
rest.split_once('/')
|
||||
}
|
||||
|
||||
/// The key that pairs the still and the video of one live or motion photo, or
|
||||
/// `None` when `file_name` cannot be either half.
|
||||
///
|
||||
/// Apple writes `IMG_1.HEIC` (or `.JPG`) and `IMG_1.MOV`, which share the stem.
|
||||
/// A Pixel writes `PXL_1.MP.jpg` and `PXL_1.MP`; there the still's stem is the
|
||||
/// video's whole name. The folder is part of the key so two cameras' `IMG_1`
|
||||
/// in different albums never pair.
|
||||
fn live_key(folder: &str, file_name: &str) -> Option<String> {
|
||||
let lower = file_name.to_lowercase();
|
||||
let (stem, ext) = lower.rsplit_once('.')?;
|
||||
let key = match ext {
|
||||
"heic" | "heif" | "jpg" | "jpeg" => stem,
|
||||
// Apple's video half. A plain `.mp4` beside a photo of the same name is
|
||||
// far more often an unrelated clip than half of a live photo.
|
||||
"mov" => stem,
|
||||
"mp" => lower.as_str(),
|
||||
_ => return None,
|
||||
};
|
||||
(!key.is_empty()).then(|| format!("{folder}/{key}"))
|
||||
}
|
||||
|
||||
/// Split `<folder>/<file>` into its parts; the folder is `""` at the root.
|
||||
@@ -858,13 +1089,13 @@ mod tests {
|
||||
RawEntry {
|
||||
archive,
|
||||
entry: path.to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().to_string(),
|
||||
relative: strip_export_prefix(path).unwrap().1.to_string(),
|
||||
size,
|
||||
}
|
||||
}
|
||||
|
||||
fn sidecar(map: &mut HashMap<String, Vec<u8>>, path: &str, body: &str) {
|
||||
let relative = strip_export_prefix(path).unwrap().to_string();
|
||||
let relative = strip_export_prefix(path).unwrap().1.to_string();
|
||||
map.insert(relative, body.as_bytes().to_vec());
|
||||
}
|
||||
|
||||
@@ -872,12 +1103,17 @@ mod tests {
|
||||
fn strips_export_root_and_localized_service_folder() {
|
||||
assert_eq!(
|
||||
strip_export_prefix("Takeout/Google Fotos/Island 2019/IMG_1.jpg"),
|
||||
Some("Island 2019/IMG_1.jpg")
|
||||
Some(("Google Fotos", "Island 2019/IMG_1.jpg"))
|
||||
);
|
||||
// Already unwrapped by the user: no `Takeout/` root left.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/IMG_1.jpg"),
|
||||
Some("IMG_1.jpg")
|
||||
Some(("Google Photos", "IMG_1.jpg"))
|
||||
);
|
||||
// ... and an album keeps its folder.
|
||||
assert_eq!(
|
||||
strip_export_prefix("Google Photos/Iceland/IMG_1.jpg"),
|
||||
Some(("Google Photos", "Iceland/IMG_1.jpg"))
|
||||
);
|
||||
// Nothing below a service folder is nothing to import.
|
||||
assert_eq!(strip_export_prefix("archive_browser.html"), None);
|
||||
@@ -957,23 +1193,144 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn creation_time_is_the_fallback_capture_time() {
|
||||
fn creation_time_is_kept_apart_as_the_last_resort() {
|
||||
// `creationTime` is when the photo reached Google. It must not beat the
|
||||
// date in the name, and the importer tries EXIF before it.
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1500000000"}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg.json",
|
||||
r#"{"creationTime":{"timestamp":"1700000000"}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
vec![entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_1.jpg",
|
||||
10,
|
||||
)],
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Photos from 2019/IMG_1.jpg", 10),
|
||||
entry(
|
||||
0,
|
||||
"Takeout/Google Photos/Photos from 2019/IMG_20190612_101500.jpg",
|
||||
10,
|
||||
),
|
||||
],
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_500_000_000));
|
||||
assert_eq!(scan.photos[0].capture_time, None);
|
||||
assert_eq!(scan.photos[0].creation_time, Some(1_500_000_000));
|
||||
assert_eq!(
|
||||
scan.photos[1].capture_time,
|
||||
capture_time_from_name("IMG_20190612_101500.jpg")
|
||||
);
|
||||
assert_eq!(scan.photos[1].creation_time, Some(1_700_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_timestamps_tolerate_numbers_zero_and_null() {
|
||||
let mut sidecars = HashMap::new();
|
||||
for (name, body) in [
|
||||
("A.jpg", r#"{"photoTakenTime":{"timestamp":1560000000}}"#),
|
||||
(
|
||||
"B.jpg",
|
||||
r#"{"photoTakenTime":{"timestamp":"0"},"favorited":null}"#,
|
||||
),
|
||||
(
|
||||
"C.jpg",
|
||||
r#"{"photoTakenTime":{"timestamp":null},"geoData":null}"#,
|
||||
),
|
||||
] {
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
&format!("Takeout/Google Photos/Photos from 2019/{name}.json"),
|
||||
body,
|
||||
);
|
||||
}
|
||||
let scan = assemble(
|
||||
["A.jpg", "B.jpg", "C.jpg"]
|
||||
.iter()
|
||||
.map(|name| {
|
||||
entry(
|
||||
0,
|
||||
&format!("Takeout/Google Photos/Photos from 2019/{name}"),
|
||||
1,
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(scan.photos[0].capture_time, Some(1_560_000_000));
|
||||
assert_eq!(scan.photos[1].capture_time, None, "0 is not 1970");
|
||||
assert!(!scan.photos[1].favorite);
|
||||
assert_eq!(scan.photos[2].capture_time, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn location_comes_from_geo_data_and_zero_means_none() {
|
||||
let mut sidecars = HashMap::new();
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_1.jpg.json",
|
||||
r#"{"geoData":{"latitude":64.1466,"longitude":-21.9426,"altitude":0.0}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_2.jpg.json",
|
||||
r#"{"geoData":{"latitude":0.0,"longitude":0.0},
|
||||
"geoDataExif":{"latitude":63.4186,"longitude":-19.0060}}"#,
|
||||
);
|
||||
sidecar(
|
||||
&mut sidecars,
|
||||
"Takeout/Google Photos/Iceland/IMG_3.jpg.json",
|
||||
r#"{"geoData":{"latitude":0.0,"longitude":0.0}}"#,
|
||||
);
|
||||
let scan = assemble(
|
||||
(1..=3)
|
||||
.map(|n| entry(0, &format!("Takeout/Google Photos/Iceland/IMG_{n}.jpg"), 1))
|
||||
.collect(),
|
||||
&sidecars,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
scan.photos[0].location,
|
||||
Some(GeoPoint {
|
||||
latitude: 64.1466,
|
||||
longitude: -21.9426
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
scan.photos[1].location.map(|p| p.latitude),
|
||||
Some(63.4186),
|
||||
"geoDataExif fills in for an empty geoData"
|
||||
);
|
||||
assert_eq!(scan.photos[2].location, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_and_motion_photo_halves_share_a_key() {
|
||||
let scan = assemble(
|
||||
vec![
|
||||
entry(0, "Takeout/Google Photos/Trip/IMG_1.HEIC", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/IMG_1.MOV", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP.jpg", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/PXL_2.MP", 1),
|
||||
entry(0, "Takeout/Google Photos/Other/IMG_1.MOV", 1),
|
||||
entry(0, "Takeout/Google Photos/Trip/scan.png", 1),
|
||||
],
|
||||
&HashMap::new(),
|
||||
);
|
||||
let key = |i: usize| scan.photos[i].live_key.clone();
|
||||
|
||||
assert_eq!(key(0), key(1));
|
||||
assert!(!scan.photos[0].is_motion_photo());
|
||||
assert_eq!(key(2), key(3));
|
||||
assert!(scan.photos[3].is_motion_photo());
|
||||
assert!(scan.photos[3].is_video(), "the .MP half is a video");
|
||||
assert_ne!(key(1), key(4), "another folder's IMG_1 does not pair");
|
||||
assert_eq!(key(5), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1387,16 +1744,77 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_a_zip_is_a_readable_error() {
|
||||
let path =
|
||||
fn a_file_that_is_not_a_zip_is_left_out_and_the_rest_imports() {
|
||||
let broken =
|
||||
std::env::temp_dir().join(format!("pdfs-takeout-not-a-zip-{}", std::process::id()));
|
||||
std::fs::write(&path, b"not a zip at all").unwrap();
|
||||
let error = scan_archives(std::slice::from_ref(&path)).unwrap_err();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
std::fs::write(&broken, b"not a zip at all").unwrap();
|
||||
let good = write_zip(
|
||||
"beside-broken",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
|
||||
let scan = scan_archives(&[broken.clone(), good.clone()]).unwrap();
|
||||
let _ = std::fs::remove_file(&broken);
|
||||
let _ = std::fs::remove_file(&good);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].archive, 1);
|
||||
assert_eq!(scan.unreadable_archives.len(), 1);
|
||||
assert_eq!(scan.unreadable_archives[0].path, broken);
|
||||
assert!(
|
||||
error.message.contains("not a readable zip"),
|
||||
"unexpected message: {}",
|
||||
error.message
|
||||
scan.unreadable_archives[0]
|
||||
.reason
|
||||
.contains("not a readable zip"),
|
||||
"unexpected reason: {}",
|
||||
scan.unreadable_archives[0].reason
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_services_in_the_same_export_are_not_imported() {
|
||||
let zip = write_zip(
|
||||
"services",
|
||||
&[
|
||||
(
|
||||
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg",
|
||||
b"x".as_slice(),
|
||||
),
|
||||
(
|
||||
"Takeout/Google Fotos/Fotos von 2019/IMG_1.jpg.json",
|
||||
br#"{"photoTakenTime":{"timestamp":"1560000000"}}"#.as_slice(),
|
||||
),
|
||||
("Takeout/Drive/My Drive/holiday.jpg", b"y".as_slice()),
|
||||
("Takeout/YouTube/videos/clip.mp4", b"z".as_slice()),
|
||||
],
|
||||
);
|
||||
|
||||
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
assert_eq!(scan.photos[0].name, "IMG_1.jpg");
|
||||
assert_eq!(scan.skipped_other, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_export_without_sidecars_keeps_every_service() {
|
||||
let zip = write_zip(
|
||||
"no-sidecars",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
let scan = scan_archives(std::slice::from_ref(&zip)).unwrap();
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
assert_eq!(scan.photos.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stop_request_ends_the_scan() {
|
||||
let zip = write_zip(
|
||||
"stop",
|
||||
&[("Takeout/Google Photos/Photos from 2019/IMG_1.jpg", b"x")],
|
||||
);
|
||||
let result = scan_archives_until(std::slice::from_ref(&zip), || true);
|
||||
let _ = std::fs::remove_file(&zip);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
fuser.workspace = true
|
||||
libc.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
base64.workspace = true
|
||||
image.workspace = true
|
||||
|
||||
@@ -877,35 +877,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
if archives.is_empty() {
|
||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||
} else {
|
||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)))
|
||||
} else {
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
let paths = archives.into_iter().map(PathBuf::from).collect();
|
||||
start_takeout_import(core, paths, dry_run)
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ImportStatus) => {
|
||||
let (running, summary) = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus { running, summary }
|
||||
let report = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus {
|
||||
running: report.running,
|
||||
summary: report.summary,
|
||||
progress: report.progress,
|
||||
resumable: report.resumable,
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
|
||||
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
|
||||
Some(paths) => start_takeout_import(core, paths, false),
|
||||
},
|
||||
Ok(CtlRequest::CancelImport) => {
|
||||
if crate::takeout::cancel_import() {
|
||||
CtlResponse::Ok {
|
||||
@@ -2009,6 +1997,31 @@ pub(crate) fn run_control_socket(
|
||||
}
|
||||
}
|
||||
|
||||
/// Check that every archive is there, then start the import in the background.
|
||||
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)));
|
||||
}
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_limit_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1193,7 +1193,7 @@ impl Core {
|
||||
if known_empty && !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !self.rt.block_on(self.refresh_timeline())? {
|
||||
if !self.refresh_timeline_joined()? {
|
||||
return Ok(None);
|
||||
}
|
||||
} else if self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL) {
|
||||
@@ -2640,6 +2640,36 @@ impl Core {
|
||||
|
||||
/// Refresh the timeline off the request path, so a stale page is still served
|
||||
/// at DB speed. At most one refresh runs at a time.
|
||||
/// Refresh the timeline and wait for it, sharing a refresh already under
|
||||
/// way instead of starting a second one beside it.
|
||||
///
|
||||
/// The first request after an empty cache has nothing to serve until the
|
||||
/// fetch lands, so it waits. Two such requests (the gallery and its retry,
|
||||
/// or a background refresh started by an import) used to each fetch the
|
||||
/// whole library at once, sharing the session's rate limit and taking
|
||||
/// minutes. Returns whether the account has a photos library.
|
||||
pub(crate) fn refresh_timeline_joined(&self) -> CoreResult<bool> {
|
||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||
while self.timeline_refreshing.load(Ordering::SeqCst) && !self.shutdown.is_stopping() {
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
}
|
||||
// A refresh that landed stamped the timeline fresh. One that failed
|
||||
// did not, and the caller gets this refresh's own error instead of
|
||||
// an empty timeline.
|
||||
if !self.listing_stale(PHOTOS_SYNCED_MS, TIMELINE_TTL)
|
||||
|| self.timeline_refreshing.swap(true, Ordering::SeqCst)
|
||||
{
|
||||
return Ok(self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() != Some(0));
|
||||
}
|
||||
}
|
||||
let refreshed = self.rt.block_on(self.refresh_timeline());
|
||||
self.timeline_refreshing.store(false, Ordering::SeqCst);
|
||||
if refreshed.is_ok() {
|
||||
self.events.publish(&[Topic::Photos]);
|
||||
}
|
||||
refreshed
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_timeline_refresh(&self) {
|
||||
if self.timeline_refreshing.swap(true, Ordering::SeqCst) {
|
||||
return;
|
||||
|
||||
@@ -77,11 +77,11 @@ use pdfs_core::config::{AppDirs, FileSort, FilesView, PhotosView, WindowState};
|
||||
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, BookmarkInfo, ConflictInfo, ConflictKeep, DeviceInfo,
|
||||
DirEntry, ErrorKind, ImportSummary, InvitationInfo, JobItem, LinkChange, MappingProgress,
|
||||
PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo, PublicLinkInfo, RefreshScope,
|
||||
Request, Response, RestorableFolder, RestoreItem, RevisionInfo, SearchHit, ShareEntry,
|
||||
ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase, SyncProgress,
|
||||
ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||
DirEntry, ErrorKind, ImportPhase, ImportProgress, ImportSummary, InvitationInfo, JobItem,
|
||||
LinkChange, MappingProgress, PendingOpInfo, PhotoItem, PhotoKind, PhotoMonth, PlaceInfo,
|
||||
PublicLinkInfo, RefreshScope, Request, Response, RestorableFolder, RestoreItem, RevisionInfo,
|
||||
SearchHit, ShareEntry, ShareEntryKind, SharedItem, SyncFolderInfo, SyncIssue, SyncPhase,
|
||||
SyncProgress, ThumbnailBuildStatus, Topic, TransferDirection, TransferItem, send,
|
||||
};
|
||||
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
@@ -297,6 +297,27 @@ fn parse_launch(args: &[String]) -> Launch {
|
||||
Launch::Window
|
||||
}
|
||||
|
||||
/// The Sync view a `--page` name opens: "activity" lives on as its History
|
||||
/// view, and "sync-folders" is its Folders view.
|
||||
fn sync_view(page: &str) -> Option<&'static str> {
|
||||
match page {
|
||||
"activity" => Some("history"),
|
||||
"sync-folders" => Some("folders"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The Photos view a `--page` name opens, and whether Places shows its map:
|
||||
/// "albums", "places" and "places-map". Other names are stack pages.
|
||||
fn photos_view(page: &str) -> Option<(u32, bool)> {
|
||||
match page {
|
||||
"albums" => Some((VIEW_ALBUMS, false)),
|
||||
"places" => Some((VIEW_PLACES, false)),
|
||||
"places-map" => Some((VIEW_PLACES, true)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Raise the window if there is one, else build it. A second launch reaches
|
||||
/// the running instance, which must not open a second window.
|
||||
fn present_window(app: &adw::Application) {
|
||||
@@ -668,9 +689,11 @@ fn build_window(app: &adw::Application) {
|
||||
progress_bar: takeout_widgets.progress_bar.clone(),
|
||||
summary_group: takeout_widgets.summary_group.clone(),
|
||||
summary_rows: RefCell::new(Vec::new()),
|
||||
resume_group: takeout_widgets.resume_group.clone(),
|
||||
resume_row: takeout_widgets.resume_row.clone(),
|
||||
resume_button: takeout_widgets.resume_button.clone(),
|
||||
inflight: Cell::new(false),
|
||||
running: Cell::new(false),
|
||||
dry_run: Cell::new(false),
|
||||
},
|
||||
update,
|
||||
});
|
||||
@@ -1448,10 +1471,18 @@ fn install_launch_actions(ui: &Rc<Ui>, app: &adw::Application, window: &adw::App
|
||||
show_login(&ui_page);
|
||||
return;
|
||||
}
|
||||
// Activity lives on as the Sync page's History view.
|
||||
if page == "activity" {
|
||||
ui_page.locations.views.set_visible_child_name("history");
|
||||
if let Some(view) = sync_view(page) {
|
||||
ui_page.locations.views.set_visible_child_name(view);
|
||||
ui_page.stack.set_visible_child_name("locations");
|
||||
} else if let Some((view, map)) = photos_view(page) {
|
||||
ui_page.stack.set_visible_child_name("gallery");
|
||||
if view == VIEW_PLACES {
|
||||
ui_page
|
||||
.gallery
|
||||
.places_layout
|
||||
.set_active(if map { PLACES_MAP } else { 0 });
|
||||
}
|
||||
ui_page.gallery.view_switch.set_active(view);
|
||||
} else if ui_page.stack.child_by_name(page).is_some() {
|
||||
ui_page.stack.set_visible_child_name(page);
|
||||
}
|
||||
@@ -2012,6 +2043,18 @@ mod tests {
|
||||
assert_eq!(parse_launch(&args(&["--page"])), Launch::Window);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_name_can_open_a_photos_or_sync_view() {
|
||||
assert_eq!(photos_view("albums"), Some((VIEW_ALBUMS, false)));
|
||||
assert_eq!(photos_view("places"), Some((VIEW_PLACES, false)));
|
||||
assert_eq!(photos_view("places-map"), Some((VIEW_PLACES, true)));
|
||||
// The timeline is the Photos stack page itself.
|
||||
assert_eq!(photos_view("gallery"), None);
|
||||
assert_eq!(sync_view("activity"), Some("history"));
|
||||
assert_eq!(sync_view("sync-folders"), Some("folders"));
|
||||
assert_eq!(sync_view("locations"), None);
|
||||
}
|
||||
|
||||
/// One file of a shot, for the group-switch tests.
|
||||
fn member(uid: &str, name: Option<&str>, kind: PhotoKind) -> PhotoItem {
|
||||
PhotoItem {
|
||||
|
||||
@@ -1115,9 +1115,6 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
// count falling to zero is what "the sync finished" looks like from here.
|
||||
// Jobs are deliberately not counted — a bulk upload retires its scan job and
|
||||
// starts its upload job mid-flight, which is not a thing finishing.
|
||||
// The Import page reads its progress off the same snapshot rather than
|
||||
// polling the daemon a second time for numbers already on the wire.
|
||||
takeout_progress(ui, jobs);
|
||||
|
||||
let previous = ui.status.active_transfers.replace(items.len());
|
||||
if !items.is_empty() {
|
||||
@@ -1211,13 +1208,18 @@ pub(crate) fn repaint_transfers(ui: &Rc<Ui>, items: &[TransferItem], jobs: &[Job
|
||||
pub(crate) fn job_line(j: &JobItem) -> ActivityLine {
|
||||
let done = j.done.to_string();
|
||||
let total = j.total.to_string();
|
||||
let (title, detail) = if j.title == IMPORT_JOB {
|
||||
import_job_text(&j.detail)
|
||||
} else {
|
||||
(j.title.clone(), j.detail.clone())
|
||||
};
|
||||
let args: [(&str, &str); 4] = [
|
||||
("title", &j.title),
|
||||
("detail", &j.detail),
|
||||
("title", &title),
|
||||
("detail", &detail),
|
||||
("done", &done),
|
||||
("total", &total),
|
||||
];
|
||||
let text = match (j.total > 0, j.detail.is_empty()) {
|
||||
let text = match (j.total > 0, detail.is_empty()) {
|
||||
// Translators: a background job's progress. {title} is the job, such as
|
||||
// "Scanning Photos"; {done} and {total} are counts.
|
||||
(true, true) => gettext_f("{title} — {done} of {total}", &args),
|
||||
|
||||
@@ -15,17 +15,30 @@
|
||||
//! Import — staging is local, so a mis-drop costs nothing.
|
||||
//!
|
||||
//! The daemon owns the run itself ([`Request::ImportTakeout`]): this page only
|
||||
//! stages paths, launches, asks [`Request::ImportStatus`] for the final report,
|
||||
//! and reads live progress off the import's job in the transfer snapshot the
|
||||
//! daemon already pushes (see [`takeout_progress`]).
|
||||
//! stages paths, launches, and asks [`Request::ImportStatus`] for the step, the
|
||||
//! running counts and the final report. It asks again whenever the import's job
|
||||
//! moves in the transfer snapshot the daemon already pushes. A run the service
|
||||
//! did not get to finish can be picked up with [`Request::ResumeImport`].
|
||||
|
||||
use crate::*;
|
||||
|
||||
/// The title the daemon gives the import's job in [`Request::GetQueueStatus`].
|
||||
/// Matching on it is how this page borrows the progress the daemon already
|
||||
/// pushes instead of asking for it separately.
|
||||
/// Seeing it move is the cue to ask for the import's status; the Activity list
|
||||
/// shows it under its translated name ([`import_job_text`]).
|
||||
pub(crate) const IMPORT_JOB: &str = "Importing Google Photos";
|
||||
|
||||
/// The import job's title and detail in the user's language. The daemon names
|
||||
/// its steps in English; the file being uploaded passes through as it is.
|
||||
pub(crate) fn import_job_text(detail: &str) -> (String, String) {
|
||||
let detail = match detail {
|
||||
"reading archives" => gettext("Reading the archives"),
|
||||
"checking for photos you already have" => gettext("Checking for photos you already have"),
|
||||
"sorting photos into albums" => gettext("Sorting photos into albums"),
|
||||
other => other.to_string(),
|
||||
};
|
||||
(gettext("Importing from Google Photos"), detail)
|
||||
}
|
||||
|
||||
/// What the archive list says before anything is staged. `repaint_archives`
|
||||
/// swaps in the count and total once there is something to report, and swaps
|
||||
/// this back when the list is emptied again.
|
||||
@@ -95,18 +108,19 @@ pub(crate) struct TakeoutState {
|
||||
pub(crate) progress_group: adw::PreferencesGroup,
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
/// Report group for the last finished run, hidden until there is one.
|
||||
/// The running counts, or the report for the last finished run; hidden
|
||||
/// until there is one.
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) summary_rows: RefCell<Vec<adw::ActionRow>>,
|
||||
/// Offers to resume an import the service did not get to finish.
|
||||
pub(crate) resume_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_row: adw::ActionRow,
|
||||
pub(crate) resume_button: gtk4::Button,
|
||||
/// One [`Request::ImportStatus`] in flight at a time, so a wedged daemon
|
||||
/// can't stack worker threads on the 2s tick.
|
||||
pub(crate) inflight: Cell<bool>,
|
||||
/// Whether the daemon reported an import running on the last poll.
|
||||
pub(crate) running: Cell<bool>,
|
||||
/// Whether the run in flight was launched as a dry run, so the report can
|
||||
/// say "would upload" rather than claiming photos were sent. The daemon's
|
||||
/// summary carries no such flag — it is the same shape either way.
|
||||
pub(crate) dry_run: Cell<bool>,
|
||||
}
|
||||
|
||||
/// Widgets the Import page hands back for wiring.
|
||||
@@ -126,6 +140,9 @@ pub(crate) struct TakeoutWidgets {
|
||||
pub(crate) progress_label: gtk4::Label,
|
||||
pub(crate) progress_bar: gtk4::ProgressBar,
|
||||
pub(crate) summary_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_group: adw::PreferencesGroup,
|
||||
pub(crate) resume_row: adw::ActionRow,
|
||||
pub(crate) resume_button: gtk4::Button,
|
||||
}
|
||||
|
||||
pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
@@ -261,7 +278,23 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
.visible(false)
|
||||
.build();
|
||||
|
||||
// An import cut short by a restart or a lost connection, offered back at the
|
||||
// top of the page: picking the archives again is exactly what a reboot in
|
||||
// the middle of a migration should not cost.
|
||||
let resume_button = gtk4::Button::builder()
|
||||
.label(pgettext("verb", "Resume"))
|
||||
.valign(gtk4::Align::Center)
|
||||
.build();
|
||||
resume_button.add_css_class("suggested-action");
|
||||
let resume_row = adw::ActionRow::builder()
|
||||
.title(gettext("The last import didn't finish"))
|
||||
.build();
|
||||
resume_row.add_suffix(&resume_button);
|
||||
let resume_group = adw::PreferencesGroup::builder().visible(false).build();
|
||||
resume_group.add(&resume_row);
|
||||
|
||||
let groups = gtk4::Box::new(gtk4::Orientation::Vertical, 18);
|
||||
groups.append(&resume_group);
|
||||
groups.append(&dropzone);
|
||||
groups.append(&list_group);
|
||||
groups.append(&actions);
|
||||
@@ -301,6 +334,9 @@ pub(crate) fn build_takeout_page() -> (gtk4::Widget, TakeoutWidgets) {
|
||||
progress_label: progress_label.clone(),
|
||||
progress_bar: progress_bar.clone(),
|
||||
summary_group: summary_group.clone(),
|
||||
resume_group,
|
||||
resume_row,
|
||||
resume_button,
|
||||
};
|
||||
(frame.upcast(), widgets)
|
||||
}
|
||||
@@ -427,6 +463,28 @@ pub(crate) fn wire_takeout(ui: &Rc<Ui>, widgets: &TakeoutWidgets) {
|
||||
});
|
||||
});
|
||||
|
||||
let ui_resume = ui.clone();
|
||||
widgets.resume_button.connect_clicked(move |_| {
|
||||
let ui = ui_resume.clone();
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::ResumeImport);
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => import_started(&ui, false),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, &gettext("Couldn't resume the import"), &message, kind);
|
||||
}
|
||||
_ => toast_error(
|
||||
&ui,
|
||||
&gettext("Couldn't resume the import"),
|
||||
&gettext("The Proton Drive service didn't respond."),
|
||||
),
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let ui_back = ui.clone();
|
||||
widgets
|
||||
.back_button
|
||||
@@ -559,6 +617,7 @@ fn sync_takeout_actions(ui: &Rc<Ui>) {
|
||||
.import_button
|
||||
.set_sensitive(has_files && !running);
|
||||
ui.takeout.clear_button.set_sensitive(has_files && !running);
|
||||
ui.takeout.resume_button.set_sensitive(!running);
|
||||
ui.takeout.cancel_button.set_visible(running);
|
||||
}
|
||||
|
||||
@@ -577,7 +636,6 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
return;
|
||||
}
|
||||
|
||||
ui.takeout.dry_run.set(dry_run);
|
||||
ui.busy_begin();
|
||||
let rx = spawn_request(
|
||||
ui.dirs.control_socket(),
|
||||
@@ -588,27 +646,7 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
let result = rx.recv().await;
|
||||
ui.busy_end();
|
||||
match result {
|
||||
Ok(Ok(Response::Ok { .. })) => {
|
||||
ui.takeout.running.set(true);
|
||||
ui.gallery.import_banner.set_revealed(true);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout.progress_label.set_text(&if dry_run {
|
||||
gettext("Reading archives…")
|
||||
} else {
|
||||
gettext("Starting import…")
|
||||
});
|
||||
sync_takeout_actions(&ui);
|
||||
toast(
|
||||
&ui,
|
||||
&if dry_run {
|
||||
gettext("Scanning the export…")
|
||||
} else {
|
||||
gettext("Import started — you can leave this page, it keeps running.")
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(Ok(Response::Ok { .. })) => import_started(&ui, dry_run),
|
||||
Ok(Ok(Response::Error { message, kind })) => {
|
||||
toast_failure(&ui, &gettext("Couldn't start the import"), &message, kind);
|
||||
}
|
||||
@@ -621,9 +659,39 @@ fn start_import(ui: &Rc<Ui>, dry_run: bool) {
|
||||
});
|
||||
}
|
||||
|
||||
/// Ask the daemon for the import's state, and paint the report once a run has
|
||||
/// finished. Asked on arrival and while import progress keeps arriving. Cheap
|
||||
/// and inflight-guarded.
|
||||
/// Switch the page over to reporting progress once the daemon took the run.
|
||||
fn import_started(ui: &Rc<Ui>, dry_run: bool) {
|
||||
ui.takeout.running.set(true);
|
||||
if !dry_run {
|
||||
ui.gallery.import_banner.set_revealed(true);
|
||||
}
|
||||
ui.takeout.resume_group.set_visible(false);
|
||||
ui.takeout.summary_group.set_visible(false);
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
ui.takeout.progress_group.set_title(&if dry_run {
|
||||
gettext("Scan in progress")
|
||||
} else {
|
||||
gettext("Import in progress")
|
||||
});
|
||||
ui.takeout.progress_bar.set_fraction(0.0);
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&gettext("Reading the archives…"));
|
||||
sync_takeout_actions(ui);
|
||||
toast(
|
||||
ui,
|
||||
&if dry_run {
|
||||
gettext("Scanning the export…")
|
||||
} else {
|
||||
gettext("Import started — you can leave this page, it keeps running.")
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Ask the daemon for the import's state and paint it: the step and counts
|
||||
/// while it runs, the report once it has ended, and the offer to resume a run
|
||||
/// that did not finish. Asked on arrival and while import progress keeps
|
||||
/// arriving. Cheap and inflight-guarded.
|
||||
pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
if ui.takeout.inflight.get() {
|
||||
return;
|
||||
@@ -634,124 +702,250 @@ pub(crate) fn refresh_takeout(ui: &Rc<Ui>) {
|
||||
glib::spawn_future_local(async move {
|
||||
let result = rx.recv().await;
|
||||
ui.takeout.inflight.set(false);
|
||||
let Ok(Ok(Response::ImportStatus { running, summary })) = result else {
|
||||
let Ok(Ok(Response::ImportStatus {
|
||||
running,
|
||||
summary,
|
||||
progress,
|
||||
resumable,
|
||||
})) = result
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let was_running = ui.takeout.running.replace(running);
|
||||
ui.gallery.import_banner.set_revealed(running);
|
||||
let dry = summary.as_ref().is_some_and(|s| s.dry_run);
|
||||
ui.gallery.import_banner.set_revealed(running && !dry);
|
||||
ui.takeout.progress_group.set_visible(running);
|
||||
if running {
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
} else {
|
||||
ui.takeout.progress_group.set_visible(false);
|
||||
// An import runs for hours, so its ending is announced rather than
|
||||
// left to be discovered: a toast for whoever is looking at the
|
||||
// window, and a desktop notification for whoever is not.
|
||||
let failed = summary.as_ref().and_then(|s| s.error.clone());
|
||||
if was_running && let Some(error) = &failed {
|
||||
let headline = gettext("Couldn't import from Google Photos");
|
||||
toast_error(&ui, &headline, error);
|
||||
notify("takeout-import", &headline, error);
|
||||
} else if was_running {
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
let headline = if dry {
|
||||
gettext("Scan finished")
|
||||
} else {
|
||||
gettext("Import finished")
|
||||
};
|
||||
toast(&ui, &headline);
|
||||
if !dry {
|
||||
let body = match &summary {
|
||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||
Some(s) if s.failed > 0 => ngettext_f(
|
||||
"{n} photo added, {failed} failed.",
|
||||
"{n} photos added, {failed} failed.",
|
||||
s.uploaded as u64,
|
||||
&[("failed", &s.failed.to_string())],
|
||||
),
|
||||
Some(s) if s.cancelled => ngettext_f(
|
||||
"Stopped after {n} photo.",
|
||||
"Stopped after {n} photos.",
|
||||
s.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
Some(s) => ngettext_f(
|
||||
"{n} photo added to Proton Photos.",
|
||||
"{n} photos added to Proton Photos.",
|
||||
s.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
None => gettext("The Google Photos import has finished."),
|
||||
};
|
||||
notify("takeout-import", &headline, &body);
|
||||
}
|
||||
}
|
||||
ui.takeout.progress_group.set_title(&if dry {
|
||||
gettext("Scan in progress")
|
||||
} else {
|
||||
gettext("Import in progress")
|
||||
});
|
||||
paint_progress(&ui, progress.as_ref());
|
||||
} else if was_running && let Some(summary) = &summary {
|
||||
announce_end(&ui, summary);
|
||||
}
|
||||
if was_running != running {
|
||||
sync_takeout_actions(&ui);
|
||||
}
|
||||
match summary {
|
||||
Some(summary) => repaint_summary(&ui, &summary),
|
||||
match &summary {
|
||||
Some(summary) => repaint_summary(&ui, summary, running),
|
||||
None => ui.takeout.summary_group.set_visible(false),
|
||||
}
|
||||
paint_resume(&ui, running, summary.as_ref(), &resumable);
|
||||
});
|
||||
}
|
||||
|
||||
/// Paint the last run's report.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
/// Tell the user how a run ended. An import runs for hours, so its ending is
|
||||
/// announced rather than left to be discovered: a toast for whoever is looking
|
||||
/// at the window, and a desktop notification for whoever is not.
|
||||
fn announce_end(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
if summary.dry_run {
|
||||
toast(ui, &gettext("Scan finished"));
|
||||
return;
|
||||
}
|
||||
if let Some(error) = &summary.error {
|
||||
let kind = summary.error_kind.unwrap_or_default();
|
||||
let headline = gettext("Couldn't import from Google Photos");
|
||||
toast_failure(ui, &headline, error, kind);
|
||||
notify(
|
||||
"takeout-import",
|
||||
&headline,
|
||||
&gettext("You can resume it once the problem is solved."),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let (headline, body) = if summary.interrupted {
|
||||
(
|
||||
gettext("Import interrupted"),
|
||||
gettext("You can resume it where it stopped."),
|
||||
)
|
||||
} else if summary.cancelled {
|
||||
(
|
||||
gettext("Import stopped"),
|
||||
ngettext_f(
|
||||
"Stopped after {n} photo.",
|
||||
"Stopped after {n} photos.",
|
||||
summary.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
)
|
||||
} else if summary.failed > 0 {
|
||||
(
|
||||
gettext("Import finished"),
|
||||
// Translators: {n} is the number of photos uploaded, {failed} the number that failed.
|
||||
ngettext_f(
|
||||
"{n} photo added, {failed} failed.",
|
||||
"{n} photos added, {failed} failed.",
|
||||
summary.uploaded as u64,
|
||||
&[("failed", &summary.failed.to_string())],
|
||||
),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
gettext("Import finished"),
|
||||
ngettext_f(
|
||||
"{n} photo added to Proton Photos.",
|
||||
"{n} photos added to Proton Photos.",
|
||||
summary.uploaded as u64,
|
||||
&[],
|
||||
),
|
||||
)
|
||||
};
|
||||
toast(ui, &headline);
|
||||
notify("takeout-import", &headline, &body);
|
||||
|
||||
// A complete import is done with its archives; keeping them staged would
|
||||
// only invite running the same export again.
|
||||
if summary.complete() {
|
||||
ui.takeout.archives.borrow_mut().clear();
|
||||
repaint_archives(ui);
|
||||
}
|
||||
}
|
||||
|
||||
/// Paint where a running import is.
|
||||
fn paint_progress(ui: &Rc<Ui>, progress: Option<&ImportProgress>) {
|
||||
let Some(progress) = progress else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
return;
|
||||
};
|
||||
let done = progress.done.to_string();
|
||||
let total = progress.total.to_string();
|
||||
let current = progress.current.clone().unwrap_or_default();
|
||||
let counted = progress.total > 0;
|
||||
let args = [
|
||||
("done", done.as_str()),
|
||||
("total", total.as_str()),
|
||||
("name", current.as_str()),
|
||||
];
|
||||
let text = match (progress.phase, counted) {
|
||||
(ImportPhase::Scanning, _) => gettext("Reading the archives…"),
|
||||
(ImportPhase::Checking, false) => gettext("Checking for photos you already have…"),
|
||||
// Translators: {done} and {total} are counts of photos.
|
||||
(ImportPhase::Checking, true) => gettext_f(
|
||||
"Checking for photos you already have — {done} of {total}",
|
||||
&args,
|
||||
),
|
||||
(ImportPhase::Uploading, false) => gettext("Uploading…"),
|
||||
(ImportPhase::Uploading, true) if current.is_empty() => {
|
||||
// Translators: {done} and {total} are counts of photos.
|
||||
gettext_f("Uploading {done} of {total}", &args)
|
||||
}
|
||||
// Translators: {done} and {total} are counts of photos; {name} is the file being uploaded.
|
||||
(ImportPhase::Uploading, true) => gettext_f("Uploading {done} of {total} — {name}", &args),
|
||||
(ImportPhase::Albums, false) => gettext("Sorting photos into albums…"),
|
||||
// Translators: {done} and {total} are counts of albums.
|
||||
(ImportPhase::Albums, true) => {
|
||||
gettext_f("Sorting photos into albums — {done} of {total}", &args)
|
||||
}
|
||||
};
|
||||
ui.takeout.progress_label.set_text(&text);
|
||||
if counted {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((progress.done as f64 / progress.total as f64).min(1.0));
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
}
|
||||
}
|
||||
|
||||
/// Offer to resume the last run when the service says it can be.
|
||||
fn paint_resume(ui: &Rc<Ui>, running: bool, summary: Option<&ImportSummary>, resumable: &[String]) {
|
||||
let offer = !running && !resumable.is_empty();
|
||||
ui.takeout.resume_group.set_visible(offer);
|
||||
if !offer {
|
||||
return;
|
||||
}
|
||||
let uploaded = summary.map_or(0, |s| s.uploaded) as u64;
|
||||
// Translators: {n} is the number of photos uploaded before the import stopped.
|
||||
ui.takeout.resume_row.set_subtitle(&ngettext_f(
|
||||
"{n} photo uploaded before it stopped. Resuming skips what is already there.",
|
||||
"{n} photos uploaded before it stopped. Resuming skips what is already there.",
|
||||
uploaded,
|
||||
&[],
|
||||
));
|
||||
}
|
||||
|
||||
/// Paint the counts of a running import, or the report of the last one.
|
||||
fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary, running: bool) {
|
||||
for row in ui.takeout.summary_rows.borrow_mut().drain(..) {
|
||||
ui.takeout.summary_group.remove(&row);
|
||||
}
|
||||
let dry = ui.takeout.dry_run.get();
|
||||
ui.takeout.summary_group.set_title(&if dry {
|
||||
let dry = summary.dry_run;
|
||||
ui.takeout.summary_group.set_title(&if running {
|
||||
gettext("So far")
|
||||
} else if dry {
|
||||
gettext("Scan result")
|
||||
} else {
|
||||
gettext("Last import")
|
||||
});
|
||||
ui.takeout.summary_group.set_description(Some(&if let Some(error) = &summary.error {
|
||||
// The reason is the service's own text, which is not translated. A
|
||||
// group's description is markup, and the reason may name a path.
|
||||
glib::markup_escape_text(&gettext_f(
|
||||
"{what}: {detail}",
|
||||
&[
|
||||
("what", &gettext("Couldn't import from Google Photos")),
|
||||
("detail", error),
|
||||
],
|
||||
))
|
||||
.to_string()
|
||||
let description = if running {
|
||||
None
|
||||
} else if let Some(error) = &summary.error {
|
||||
// The reason is the service's own text, which is not translated; the
|
||||
// headline in front of it is. A group's description is markup, and the
|
||||
// reason may name a path.
|
||||
let what = gettext("Couldn't import from Google Photos");
|
||||
Some(
|
||||
glib::markup_escape_text(&gettext_f(
|
||||
"{what}: {detail}",
|
||||
&[
|
||||
(
|
||||
"what",
|
||||
error_headline(summary.error_kind.unwrap_or_default(), &what),
|
||||
),
|
||||
("detail", error),
|
||||
],
|
||||
))
|
||||
.to_string(),
|
||||
)
|
||||
} else if dry {
|
||||
gettext("Nothing was uploaded — this is what an import would do.")
|
||||
Some(gettext(
|
||||
"Nothing was uploaded — this is what an import would do.",
|
||||
))
|
||||
} else if summary.interrupted {
|
||||
Some(gettext(
|
||||
"The Proton Drive service stopped while the import ran. Resume it to continue where it left off.",
|
||||
))
|
||||
} else if summary.cancelled {
|
||||
gettext("Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.")
|
||||
Some(gettext(
|
||||
"Stopped early. Everything uploaded so far is in your timeline; run it again to continue where it left off.",
|
||||
))
|
||||
} else if !summary.complete() {
|
||||
Some(gettext(
|
||||
"Finished with problems. Run it again to retry what didn't upload.",
|
||||
))
|
||||
} else {
|
||||
gettext("Finished.")
|
||||
}));
|
||||
Some(gettext("Finished."))
|
||||
};
|
||||
ui.takeout
|
||||
.summary_group
|
||||
.set_description(description.as_deref());
|
||||
|
||||
// Only lines that carry information: a run with no failures should not have
|
||||
// to show a "0 failed" row for the user to read past.
|
||||
let mut lines: Vec<(String, String)> = vec![
|
||||
(gettext("Photos in the export"), summary.found.to_string()),
|
||||
(
|
||||
if dry {
|
||||
gettext("Would upload")
|
||||
} else {
|
||||
gettext("Uploaded")
|
||||
},
|
||||
summary.uploaded.to_string(),
|
||||
),
|
||||
];
|
||||
let mut lines: Vec<(String, String)> =
|
||||
vec![(gettext("Photos in the export"), summary.found.to_string())];
|
||||
if dry {
|
||||
lines.push((gettext("Would upload"), summary.to_upload.to_string()));
|
||||
} else {
|
||||
lines.push((gettext("Uploaded"), summary.uploaded.to_string()));
|
||||
}
|
||||
if summary.duplicates > 0 {
|
||||
lines.push((
|
||||
gettext("Already in Proton Photos"),
|
||||
summary.duplicates.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.live_photos > 0 {
|
||||
lines.push((
|
||||
gettext("Live photos kept together"),
|
||||
summary.live_photos.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.albums_created > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
gettext("Albums to create")
|
||||
} else {
|
||||
gettext("Albums created")
|
||||
},
|
||||
gettext("Albums created"),
|
||||
summary.albums_created.to_string(),
|
||||
));
|
||||
}
|
||||
@@ -767,25 +961,31 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
summary.skipped_trashed.to_string(),
|
||||
));
|
||||
}
|
||||
if summary.bytes > 0 {
|
||||
if dry && summary.to_upload_bytes > 0 {
|
||||
lines.push((
|
||||
if dry {
|
||||
gettext("Data to upload")
|
||||
} else {
|
||||
gettext("Data uploaded")
|
||||
},
|
||||
human_bytes(summary.bytes),
|
||||
gettext("Data to upload"),
|
||||
human_bytes(summary.to_upload_bytes),
|
||||
));
|
||||
} else if summary.bytes > 0 {
|
||||
lines.push((gettext("Data uploaded"), human_bytes(summary.bytes)));
|
||||
}
|
||||
if summary.failed > 0 {
|
||||
lines.push((pgettext("state", "Failed"), summary.failed.to_string()));
|
||||
}
|
||||
for path in &summary.unreadable_archives {
|
||||
let name = Path::new(path)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| path.clone());
|
||||
lines.push((gettext("Couldn't read this archive"), name));
|
||||
}
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for (title, value) in lines {
|
||||
let row = adw::ActionRow::builder()
|
||||
.title(title)
|
||||
.subtitle(value)
|
||||
.use_markup(false)
|
||||
.build();
|
||||
row.add_css_class("property");
|
||||
ui.takeout.summary_group.add(&row);
|
||||
@@ -794,35 +994,3 @@ fn repaint_summary(ui: &Rc<Ui>, summary: &ImportSummary) {
|
||||
*ui.takeout.summary_rows.borrow_mut() = rows;
|
||||
ui.takeout.summary_group.set_visible(true);
|
||||
}
|
||||
|
||||
/// Paint the import's progress from the transfer snapshot the daemon pushed,
|
||||
/// so this page costs no extra request. A job with no known total
|
||||
/// pulses instead of showing a fraction, the same as the Activity list.
|
||||
pub(crate) fn takeout_progress(ui: &Rc<Ui>, jobs: &[JobItem]) {
|
||||
let Some(job) = jobs.iter().find(|job| job.title == IMPORT_JOB) else {
|
||||
return;
|
||||
};
|
||||
ui.takeout.progress_group.set_visible(true);
|
||||
if job.total > 0 {
|
||||
ui.takeout
|
||||
.progress_bar
|
||||
.set_fraction((job.done as f64 / job.total as f64).min(1.0));
|
||||
let done = job.done.to_string();
|
||||
let total = job.total.to_string();
|
||||
// Translators: progress such as "120 of 4000 — IMG_0001.jpg"; {detail} is what is being worked on.
|
||||
let text = gettext_f(
|
||||
"{done} of {total} — {detail}",
|
||||
&[("done", &done), ("total", &total), ("detail", &job.detail)],
|
||||
);
|
||||
ui.takeout.progress_label.set_text(&text);
|
||||
} else {
|
||||
ui.takeout.progress_bar.pulse();
|
||||
ui.takeout
|
||||
.progress_label
|
||||
.set_text(&if job.detail.is_empty() {
|
||||
gettext("Working…")
|
||||
} else {
|
||||
job.detail.clone()
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -109,6 +109,11 @@ a live account since.
|
||||
|
||||
| Entry | Title |
|
||||
|---|---|
|
||||
| [B289](#b289--the-takeout-import-mixes-in-other-services-and-drops-location-and-live-photos) | The Takeout import mixes in other services and drops location and live photos |
|
||||
| [B288](#b288--an-interrupted-takeout-import-reads-as-finished-and-one-error-ends-it) | An interrupted Takeout import reads as finished, and one error ends it |
|
||||
| [B287](#b287--a-photo-draft-left-by-an-interrupted-upload-blocks-that-name) | A photo draft left by an interrupted upload blocks that name |
|
||||
| [B286](#b286--resuming-a-takeout-import-fails-on-a-draft-with-no-content-hash) | Resuming a Takeout import fails on a draft with no content hash |
|
||||
| [B285](#b285--a-restarted-secret-service-leaves-the-process-on-a-dead-session) | A restarted Secret Service leaves the process on a dead session |
|
||||
| [B284](#b284--a-keyring-entry-reported-missing-discards-refreshed-session-tokens) | A keyring entry reported missing discards refreshed session tokens |
|
||||
| [B283](#b283--videos-have-no-thumbnail-outside-photos-and-a-ts-recording-looks-like-source-code) | Videos have no thumbnail outside Photos, and a `.ts` recording looks like source code |
|
||||
| [B282](#b282--a-file-kept-offline-is-downloaded-again-after-a-change-made-elsewhere) | A file kept offline is downloaded again after a change made elsewhere |
|
||||
@@ -387,6 +392,162 @@ a live account since.
|
||||
|
||||
Newest first.
|
||||
|
||||
### B289 — The Takeout import mixes in other services and drops location and live photos
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, auditing the Google Photos import alongside
|
||||
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||
**Where:** `crates/pdfs-core/src/takeout.rs`; `plan` and `upload_takeout_photo` in
|
||||
`crates/pdfs-fuse/src/takeout.rs`
|
||||
|
||||
**Repro:** Import a Takeout export that also holds `Drive/` or `YouTube/`, with Pixel motion
|
||||
photos, iPhone live photos and photos that have a place in Google Photos.
|
||||
|
||||
**Cause.** Five gaps in what the import carried over:
|
||||
- The service folder was dropped without being checked, so media from other Google services in
|
||||
the same export landed in the photo timeline.
|
||||
- `geoData` was never read, although `docs/PHOTOS.md` said the import takes locations from the
|
||||
export, and the SDK had no way to upload a location.
|
||||
- The two halves of a live or motion photo uploaded as unrelated photos, the `.MOV` half usually
|
||||
without a date, and the `.MP` half of a Pixel motion photo was skipped as unknown.
|
||||
- `creationTime`, the day the photo reached Google, outranked the date in the file name and the
|
||||
photo's own EXIF.
|
||||
- A sidecar with `null` in `favorited`, a numeric `timestamp` or a `0` timestamp lost its other
|
||||
fields or filed the photo under 1970.
|
||||
|
||||
**Fix:** The Photos service is the folder whose sidecars carry `photoTakenTime`, and only it is
|
||||
imported (everything, when no folder does). `geoData`, else `geoDataExif`, becomes the photo's
|
||||
`Location` attribute through SDK 0.12.0's `PhotoUploadMetadata::location`; `0, 0` is no position.
|
||||
Stills and videos with the same stem in one folder (`IMG_1.HEIC` + `IMG_1.MOV`,
|
||||
`PXL_1.MP.jpg` + `PXL_1.MP`) are paired unless their dates are minutes apart: the still is
|
||||
tagged as a live or motion photo, the video is uploaded after it with `main_photo_uid` and takes
|
||||
its date and place. The capture time is `photoTakenTime`, then the name, then EXIF, then
|
||||
`creationTime`. Favorites are also set on photos that were already on the account.
|
||||
|
||||
**Verified:** `takeout::tests` in `pdfs-core` (services, locations, timestamps, live keys) and
|
||||
`pdfs-fuse` (pairing, skew, folded copies). Not run against a real export and account.
|
||||
|
||||
---
|
||||
|
||||
### B288 — An interrupted Takeout import reads as finished, and one error ends it
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, auditing the Google Photos import alongside
|
||||
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||
**Where:** `crates/pdfs-fuse/src/takeout.rs`; `refresh_takeout` in
|
||||
`crates/pdfs-gui/src/app/pages/takeout.rs`; `cmd_import_google_photos` in
|
||||
`crates/pdfs-cli/src/main.rs`
|
||||
|
||||
**Repro:** Start an import of several archives, one of them damaged, and restart the service
|
||||
while it uploads.
|
||||
|
||||
**Cause.**
|
||||
- The import state lived only in memory. After a restart `ImportStatus` had no summary, and the
|
||||
app announced "Import finished". The archives had to be chosen again to continue.
|
||||
- A daemon shutdown broke the upload loop but still filed albums and logged a success.
|
||||
- One unreadable archive failed the scan, and a failed request in the middle of the run
|
||||
discarded every count and skipped the albums for what had uploaded.
|
||||
- The progress total counted photos already on the account, so the bar stopped short of the
|
||||
end; a dry run reported "Would upload: 0"; the job's phases reached the app in English.
|
||||
|
||||
**Fix:** The last run's archives and summary are kept in `takeout-import.json` in the state
|
||||
directory. A run still marked running at start-up is `interrupted`, and `ImportStatus` lists
|
||||
its archives as `resumable`; `ResumeImport` (`pdfs import-google-photos --resume`, **Resume** in
|
||||
the app) starts it again, and the duplicate check skips what is there. An unreadable archive is
|
||||
listed in `unreadable_archives` and the rest imports. A failure keeps the counts and still files
|
||||
albums. `ImportStatus` carries the phase and its own progress, the summary says `dry_run`,
|
||||
`to_upload` and `error_kind`, and the app words them in the user's language. The inline
|
||||
timeline refresh of an empty gallery waits for a refresh already running instead of starting a
|
||||
second one (`refresh_timeline_joined`).
|
||||
|
||||
**Verified:** `takeout::tests` in `pdfs-fuse` (saved state, resumable runs) and `pdfs-core`
|
||||
(unreadable archive), and the CLI tests for `--resume` and the exit status. Not run live.
|
||||
|
||||
---
|
||||
|
||||
### B287 — A photo draft left by an interrupted upload blocks that name
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, reading the SDK upload path for
|
||||
[issue #94](https://github.com/narrrl/proton-drive-linux/issues/94).
|
||||
**Where:** `upload_photo_from`, `upload_file_from_inner` and `abandon_file_draft` in
|
||||
`proton-drive-rs` (SDK 0.12.0)
|
||||
|
||||
**Repro:** Interrupt a photo upload after its draft is created, sign in again, and upload a
|
||||
photo with the same name.
|
||||
|
||||
**Cause.** Photo uploads replaced a leftover draft only when its `ClientUID` was the current
|
||||
session's. A new login is a new session, so the next upload of that name failed with 2500. A
|
||||
failed upload also never deleted its own draft, which the C# and TypeScript SDKs do.
|
||||
|
||||
**Fix:** SDK 0.12.0 adds `PhotoUploadMetadata::replace_existing_draft`, which the Takeout import
|
||||
sets, and deletes the draft of a new file or photo whose upload fails. After a failed seal it
|
||||
only does so on a definite rejection, because a seal lost in transit may have been applied.
|
||||
|
||||
**Verified:** SDK unit test `draft_cleanup_only_after_a_definite_seal_rejection`. Not run live.
|
||||
|
||||
---
|
||||
|
||||
### B286 — Resuming a Takeout import fails on a draft with no content hash
|
||||
|
||||
**Status:** Fixed, not verified live
|
||||
**Found:** 2026-11-15, [issue #94](https://github.com/narrrl/proton-drive-linux/issues/94)
|
||||
(3.7.0, Fedora). An import stopped by a restart failed every re-run at once with
|
||||
`check for existing photos: serialization error: invalid type: null, expected a string at line
|
||||
1 column 2063`.
|
||||
**Where:** `FoundPhotoDuplicateDto` in `proton-drive-rs` (SDK 0.11.0)
|
||||
|
||||
**Repro:** Interrupt a photo upload after its draft is created, then run the import again.
|
||||
|
||||
**Cause.** The duplicates endpoint sends `ContentHash` (and can send `Hash`) as `null` for a
|
||||
draft, as its API schema says. The SDK declared both as `String` with `#[serde(default)]`,
|
||||
which covers a missing field but not `null`, so one draft failed the whole batch of 150 names
|
||||
and the import ended before uploading anything.
|
||||
|
||||
**Fix:** SDK 0.12.0 makes both fields optional and skips drafts, trashed links and candidates
|
||||
without hashes, like the TypeScript SDK. Every other defaulted DTO field reads `null` as its
|
||||
default too, and an undecodable response is logged with its path.
|
||||
|
||||
**Verified:** SDK unit tests `duplicate_response_tolerates_null_fields_and_skips_drafts` and
|
||||
`duplicate_response_tolerates_a_null_list`. Not run against the reporter's account.
|
||||
|
||||
---
|
||||
|
||||
### B285 — A restarted Secret Service leaves the process on a dead session
|
||||
|
||||
**Status:** Fixed (unverified)
|
||||
**Found:** 2026-11-14, following up [issue #78](https://github.com/narrrl/proton-drive-linux/issues/78)
|
||||
(signed out after every cold boot on Fedora 45, which replaced gnome-keyring with oo7). See
|
||||
[the investigation](ISSUE-78.md#a-restarted-secret-service).
|
||||
**Where:** `crates/pdfs-core/src/auth.rs`, `with_keyring_entry`
|
||||
|
||||
**Repro:** Write the session blob, restart the Secret Service provider
|
||||
(`gnome-keyring-daemon --replace`, or `oo7-daemon --replace`), then read or write it again from
|
||||
the same process. `dbus-secret-service-keyring-store` keeps one D-Bus connection and one Secret
|
||||
Service session for the life of the process, and pdfs installed it once. The session exists only
|
||||
in the provider's memory, so the new provider answers every `GetSecret` and `SetSecret` with
|
||||
`org.freedesktop.Secret.Error.NoSession`, for as long as the process runs. A fresh store in the
|
||||
same process reads the same entry fine. oo7's systemd unit restarts the daemon on failure, and
|
||||
both providers can be replaced, so the daemon could meet this mid-session. A token refresh after
|
||||
that point reached only memory and the kernel keyring: B61's retry kept asking on the dead
|
||||
session, and the next cold boot read the spent refresh token (`InvalidRefreshToken`).
|
||||
|
||||
**Fix:** Every keyring call goes through `with_keyring_entry`. A platform failure (which is what
|
||||
`NoSession` and other D-Bus errors become) logs "the Secret Service failed this process's
|
||||
connection; reconnecting", drops the store unless another thread already replaced it, and makes
|
||||
the call once more on a new connection and session. A locked keyring, a dismissed prompt and a
|
||||
missing entry come back unchanged. Affects the daemon, the app and the tray alike.
|
||||
|
||||
**Verified:** `auth::tests::a_dropped_secret_service_session_is_reconnected_and_retried`,
|
||||
`a_rotation_lands_at_once_after_the_keyring_restarts` and `a_locked_keyring_is_not_reconnected`
|
||||
on the in-memory store. The ignored `auth::tests::real_secret_service_survives_a_provider_restart`
|
||||
passes against gnome-keyring 51.1 and oo7 0.7.1 (built from its tag), each in a throwaway session
|
||||
bus and restarted with `--replace`; it checks that a reconnect actually happened. A standalone
|
||||
program holding one store, as before the fix, gets `NoSession` on read and write after the same
|
||||
restart under both providers. Whether the reporter's oo7 restarts during a session is not known.
|
||||
|
||||
---
|
||||
|
||||
### B284 — A keyring entry reported missing discards refreshed session tokens
|
||||
|
||||
**Status:** Fixed (unverified)
|
||||
|
||||
@@ -112,18 +112,64 @@ Existing folders stay device backups. Older builds cannot open the upgraded data
|
||||
carry their cover as their thumbnail, so they show it at once on other devices too. A cover
|
||||
image added to a folder later is picked up once the song changes or the service restarts.
|
||||
|
||||
- **Resume a Google Photos import** (B288). The archives and the outcome of the last import are
|
||||
kept, so an import cut short by a restart or a reboot no longer has to be set up again. The
|
||||
Import page offers **Resume**, and `pdfs import-google-photos --resume` does the same. Photos
|
||||
that are already uploaded are skipped as before.
|
||||
|
||||
- **More pages to open the app on.** `pdfs-app --page` also takes `albums`, `places`,
|
||||
`places-map` and `sync-folders`.
|
||||
|
||||
### Changed
|
||||
|
||||
- **`proton-sdk` and `proton-drive-rs` 0.11.0.** Timestamps are converted where they enter
|
||||
- **`proton-sdk` and `proton-drive-rs` 0.12.0.** Timestamps are converted where they enter
|
||||
from the SDK, and a photo keeps its own modification time now that it no longer follows the
|
||||
capture time. [The SDK review](SDK-0.11.0.md) lists the new features and breaking changes.
|
||||
capture time. 0.12.0 fixes the photo duplicate check and failed-upload drafts below.
|
||||
[The 0.11.0 review](SDK-0.11.0.md) and [the 0.12.0 review](SDK-0.12.0.md) list the new
|
||||
features and breaking changes.
|
||||
|
||||
- **The Google Photos import brings more of the export along** (B289).
|
||||
- Places from Google Photos are kept, so imported photos show up under **Places**.
|
||||
- Live photos and Pixel motion photos stay one photo: the video is filed with its still and
|
||||
takes its date. The `.MP` half of a motion photo is no longer skipped.
|
||||
- Only the Google Photos part of an export is imported. Files from Google Drive, YouTube and
|
||||
other services in the same Takeout stay out of the timeline.
|
||||
- A photo without a date of its own is dated by its file name and then by its EXIF before
|
||||
the day it reached Google Photos.
|
||||
- Favorites are also set on photos that were already in Proton Photos.
|
||||
|
||||
- **The import reports what it is doing** (B288). The Import page shows the current step
|
||||
(reading, checking for photos you already have, uploading, sorting into albums), the counts
|
||||
so far, and a progress bar that reaches the end. A scan says how many photos and how much
|
||||
data an import would upload. `pdfs import-google-photos --wait` prints its progress, and
|
||||
`pdfs import-status` shows the step.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Resuming a Google Photos import no longer fails at once** (#94, B286, B287). A photo upload
|
||||
that was cut short left a draft behind, and every later run stopped with
|
||||
`check for existing photos: serialization error: invalid type: null`. Such drafts are now
|
||||
skipped by the duplicate check and replaced by the next upload of that photo, also after
|
||||
signing in again. A failed upload removes its draft.
|
||||
|
||||
- **One bad archive or error no longer ends the whole import** (B288). An archive that cannot be
|
||||
read is left out and named in the report, and the rest of the export imports. When an import
|
||||
stops early, it keeps the counts of what it did and still files the uploaded photos into their
|
||||
albums. A restart of the service while importing is reported as an interruption, not as a
|
||||
finished import.
|
||||
|
||||
- **Photos zoomed all the way out shows the whole timeline.** The first 200 photos fit on one
|
||||
screen there, and with nothing to scroll no more were loaded. More now load whenever the photos
|
||||
don't fill the window, also after a zoom or a resize.
|
||||
|
||||
- **Signed out after a cold boot on Fedora 45** (issue #78, B284, B285). Fedora 45 replaced the
|
||||
GNOME keyring with oo7, and two things there could keep a refreshed sign-in from being saved:
|
||||
oo7 reports the entries of a locked keyring as missing, which the client took for a sign-out,
|
||||
and a restarted oo7 no longer knew the client's connection, which then failed every save until
|
||||
the service restarted. The refreshed sign-in lived only in memory, and the next boot found the
|
||||
spent one. The client now keeps such a sign-in and saves it once the keyring can be read, and
|
||||
reconnects to a keyring service that restarted.
|
||||
|
||||
- **Thumbnails appear in the folder you are looking at as soon as they are made.** During
|
||||
**Build thumbnails**, the folder on screen kept its file icons until you left it and opened it
|
||||
again. Its tiles now fill in while the build runs, and the same goes for any tile whose
|
||||
|
||||
@@ -196,6 +196,7 @@ commands are for scripts and for cases where the mount is not available.
|
||||
| `pdfs add-to-album UID PHOTOS...` | Add photos to an album. |
|
||||
| `pdfs remove-from-album UID PHOTOS...` | Take photos out of an album. |
|
||||
| `pdfs import-google-photos ARCHIVES... [--dry-run] [--wait]` | Import a Google Takeout export. Pass every `.zip` of the export. |
|
||||
| `pdfs import-google-photos --resume [--wait]` | Start the last import again with the same archives after it was interrupted, stopped or failed. |
|
||||
| `pdfs import-status` / `pdfs cancel-import` | Follow or stop a running import. |
|
||||
| `pdfs redate-photos [--dry-run] [--from DATE] [--to DATE] [--limit N] [--wait]` | Fix photos filed under their upload day when the file name carries the real date. |
|
||||
| `pdfs redate-status` / `pdfs cancel-redate` | Follow or stop a running re-date. |
|
||||
|
||||
@@ -161,6 +161,29 @@ links, copies the images and writes the sidebar. `docs/README.md` becomes the wi
|
||||
|
||||
Never edit the wiki in the browser: the next publish replaces every page.
|
||||
|
||||
**Screenshots.** The pictures in `images/` (except the cover) are taken by a script, from an
|
||||
invented account, so they never show anyone's files:
|
||||
|
||||
```bash
|
||||
scripts/screenshots.sh # all of them, into images/
|
||||
scripts/screenshots.sh --only photos,files # just these
|
||||
scripts/screenshots.sh --website ../proton-drive-linux-website # and copy them to the website
|
||||
```
|
||||
|
||||
It builds the app and runs it against `crates/pdfs-core/examples/demo_daemon.rs`, a stand-in
|
||||
daemon that answers the control socket with made-up files, photos, places, computers and history.
|
||||
The app runs in a headless `cage` compositor with its own HOME, XDG directories, session bus and
|
||||
keyring, so nothing touches your account, your keyring or a running service. It needs `cage`,
|
||||
`grim`, `wlr-randr`, ImageMagick, `dbus-run-session` and `gnome-keyring-daemon`, plus `wtype` for
|
||||
the launcher's query, and `fzf` and `alacritty` for its fzf variant. The demo photos come from
|
||||
[picsum.photos](https://picsum.photos) (Unsplash licence) on the first run and are cached in
|
||||
`~/.cache/pdfs-screenshots`. Dates are relative to the day you run it.
|
||||
|
||||
Retake them before a release, and look at each one before committing. A page that grew new data
|
||||
needs a matching answer in the demo daemon; the requests it has no answer for are logged in
|
||||
`demo_daemon.log` (keep it with `--keep`). The demo daemon is built with every `cargo test` and
|
||||
`cargo clippy --all-targets`, so a change to the control protocol that breaks it fails CI.
|
||||
|
||||
## Releasing
|
||||
|
||||
Releases are cut from `main` by pushing a `v*` tag. `.github/workflows/release.yml` then:
|
||||
|
||||
@@ -58,6 +58,44 @@ can resolve the default alias to a locked collection. Both have fixes in
|
||||
after the reporter's November 7 retest. Neither can be assigned as this report's cause without
|
||||
matching runtime evidence.
|
||||
|
||||
## A restarted Secret Service
|
||||
|
||||
A second path to the same symptom, independent of locking (B285). The keyring store pdfs used
|
||||
holds one D-Bus connection and one Secret Service session for the whole process, and was
|
||||
installed once. The session lives only in the provider's memory. When the provider restarts,
|
||||
the new one answers every `GetSecret` and `SetSecret` on the old session with
|
||||
`org.freedesktop.Secret.Error.NoSession`, until the pdfs process itself restarts. Reproduced
|
||||
with both gnome-keyring 51.1 and oo7 0.7.1 using `--replace`.
|
||||
|
||||
gnome-keyring practically never restarts within a desktop session. oo7 can:
|
||||
`oo7-daemon.service` has `Restart=on-failure`, the daemon allows itself to be replaced, and it
|
||||
exits on a wrong credential password and is D-Bus-activated again. A daemon that met a restarted
|
||||
provider kept every later token refresh only in memory and the kernel keyring, and a cold boot
|
||||
then left the spent refresh token in the saved blob.
|
||||
|
||||
Now a platform failure drops the store and the call is retried once on a new connection. The
|
||||
daemon log shows `the Secret Service failed this process's connection; reconnecting` when that
|
||||
happens. To check whether a machine meets this, look for oo7 restarts in the same boot:
|
||||
|
||||
```sh
|
||||
journalctl --user -b -1 -u oo7-daemon.service | grep -E 'Started|Stopped|Starting oo7|Main process exited'
|
||||
journalctl --user -b -1 -u proton-drive.service | grep -iE 'keyring|reconnecting|refresh'
|
||||
```
|
||||
|
||||
Reproduce in a throwaway session bus (never against the desktop's keyring; the runtime directory
|
||||
must be short, as it holds a Unix socket):
|
||||
|
||||
```sh
|
||||
T=$(mktemp -d /tmp/ss.XXXX); mkdir -p $T/data $T/run; chmod 700 $T/run
|
||||
XDG_DATA_HOME=$T/data XDG_RUNTIME_DIR=$T/run PDFS_SECRET_SERVICE_RESTART_TEST=1 \
|
||||
PDFS_RESTART_PROVIDER='echo -n pw | gnome-keyring-daemon --replace --unlock --components=secrets >/dev/null' \
|
||||
dbus-run-session -- sh -c 'echo -n pw | gnome-keyring-daemon --unlock --components=secrets >/dev/null; sleep 1;
|
||||
cargo test -p pdfs-core real_secret_service -- --ignored'
|
||||
```
|
||||
|
||||
For oo7, start `oo7-daemon --login` with the password on stdin instead, and restart it with
|
||||
`--login --replace` in the background.
|
||||
|
||||
## Reproduce without a Proton account or desktop keyring
|
||||
|
||||
The oo7 fixture uses fake tokens, a temporary disk keyring and a private peer-to-peer D-Bus
|
||||
|
||||
@@ -55,6 +55,7 @@ server learns which areas you look at.
|
||||
2. In the app, open the **Upload** menu on the Photos page and choose **Import from Google
|
||||
Photos…**. Select all the `.zip` files at once, because a photo and its details are often in
|
||||
different parts of the export.
|
||||
3. Choose **Scan** to see what an import would do without uploading anything, then **Import**.
|
||||
|
||||
From a terminal:
|
||||
|
||||
@@ -63,8 +64,22 @@ pdfs import-google-photos --dry-run ~/Downloads/takeout-*.zip # show what woul
|
||||
pdfs import-google-photos --wait ~/Downloads/takeout-*.zip
|
||||
```
|
||||
|
||||
The import recreates your albums and takes capture dates and locations from the export. Photos
|
||||
already in your library are skipped, so an interrupted import can simply be run again.
|
||||
What the import takes from the export:
|
||||
|
||||
- **Albums** are recreated. A photo that is in several albums is uploaded once and added to each.
|
||||
- **Dates** come from the export's details, else from a date in the file name, else from the
|
||||
photo's EXIF.
|
||||
- **Places** come from the export, so imported photos appear under [Places](#places).
|
||||
- **Favorites** are kept.
|
||||
- **Live photos and motion photos** stay one photo: the video is filed with its still.
|
||||
|
||||
What it leaves out: photos in Google's trash, and files from other Google services (Drive,
|
||||
YouTube) that are in the same export. A `.zip` that cannot be read is left out and named in the
|
||||
report; the rest still imports.
|
||||
|
||||
Photos already in your library are skipped, so an interrupted import can simply be run again.
|
||||
If the import was cut short, for example by a restart, the Import page offers **Resume**, and
|
||||
`pdfs import-google-photos --resume` does the same without naming the archives again.
|
||||
`pdfs import-status` shows how far an import is, and `pdfs cancel-import` stops it.
|
||||
|
||||
## Fixing photos filed under the wrong date
|
||||
|
||||
@@ -63,7 +63,8 @@ them along.
|
||||
| Add someone else's photo to your album | Blocked | Needs `copyPhoto` in the SDK |
|
||||
| Photo name search | Not started | Photos search by date and filters only |
|
||||
| Automatic photo upload from a folder | Not started | |
|
||||
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos; motion-photo `.MP` parts are skipped |
|
||||
| Faster Google Photos import | Not started | Uploads run one at a time. Trash folders of locales missing from `TRASH_FOLDERS` import as photos |
|
||||
| More of a Google Photos export | Not started | `.tgz` exports and extracted folders; shared albums told apart from own ones; album descriptions and covers; photo descriptions |
|
||||
| Content-addressed block cache | Idea | Store blocks by hash to deduplicate identical content |
|
||||
|
||||
## Performance and correctness
|
||||
|
||||
|
Before Width: | Height: | Size: 43 KiB After Width: | Height: | Size: 51 KiB |
|
Before Width: | Height: | Size: 124 KiB After Width: | Height: | Size: 137 KiB |
|
Before Width: | Height: | Size: 52 KiB After Width: | Height: | Size: 115 KiB |
|
Before Width: | Height: | Size: 32 KiB After Width: | Height: | Size: 32 KiB |
|
Before Width: | Height: | Size: 1.6 MiB After Width: | Height: | Size: 1.6 MiB |
|
Before Width: | Height: | Size: 1.6 MiB After Width: | Height: | Size: 283 KiB |
|
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 36 KiB |
|
Before Width: | Height: | Size: 114 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 71 KiB After Width: | Height: | Size: 69 KiB |
@@ -91,7 +91,7 @@ SECTIONS = [
|
||||
]
|
||||
|
||||
# Too long or too internal for the wiki; links to them go to GitHub.
|
||||
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md"}
|
||||
EXCLUDED = {"BUGS.md", "CHANGELOG.md", "ISSUE-78.md", "SDK-0.11.0.md", "SDK-0.12.0.md"}
|
||||
|
||||
# Files outside docs/ whose links are checked but not published.
|
||||
CHECKED = ["README.md", "CONTRIBUTING.md", "SECURITY.md"]
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env bash
|
||||
# Retakes the README and website screenshots from made-up data.
|
||||
set -Eeuo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
usage: scripts/screenshots.sh [OPTION ...]
|
||||
|
||||
Builds the app, starts it against a stand-in daemon that serves an invented
|
||||
account (crates/pdfs-core/examples/demo_daemon.rs), and captures every page in
|
||||
a headless Wayland compositor. No Proton account, no FUSE mount, nothing from
|
||||
your own files: HOME, every XDG directory, the session bus and the keyring
|
||||
are throwaway ones under a temporary directory.
|
||||
|
||||
Options:
|
||||
--out DIR where the PNGs go (default: images/)
|
||||
--only LIST capture only these, comma-separated (default: all of
|
||||
login,files,photos,photos_map,computers,sync,conflicts,
|
||||
prompt_gtk,prompt_fzf)
|
||||
--website DIR afterwards, copy them into the website checkout at DIR
|
||||
with its scripts/sync-screenshots.sh
|
||||
--settle SECS how long a page gets to load before its capture (default 3)
|
||||
--debug use a debug build instead of a release build
|
||||
--keep keep the temporary directory, for a look at the logs
|
||||
|
||||
Needs cage, grim, wlr-randr, ImageMagick, dbus-run-session and
|
||||
gnome-keyring-daemon; prompt_gtk types its query with wtype when there is one,
|
||||
and prompt_fzf needs fzf and alacritty. Demo photos come from picsum.photos
|
||||
(Unsplash licence) on the first run and are cached in
|
||||
~/.cache/pdfs-screenshots.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
repo="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
all=login,files,photos,photos_map,computers,sync,conflicts,prompt_gtk,prompt_fzf
|
||||
|
||||
# The capture itself, run by cage inside the throwaway session (see below).
|
||||
if [[ "${1:-}" == --capture ]]; then
|
||||
shots=",$SHOTS,"
|
||||
wants() { [[ "$shots" == *",$1,"* ]]; }
|
||||
output() { wlr-randr --output "$(wlr-randr | awk 'NR==1 {print $1}')" --custom-mode "$1"; }
|
||||
shoot() {
|
||||
sleep "$SETTLE"
|
||||
grim "$OUT/$1.png"
|
||||
echo "captured $1"
|
||||
}
|
||||
# Cut the compositor's black background off a window smaller than it.
|
||||
trim() {
|
||||
magick "$OUT/$1.png" -bordercolor black -border 1 -trim +repage "$OUT/$1.png"
|
||||
}
|
||||
quit() {
|
||||
kill "$@" 2>/dev/null || true
|
||||
wait "$@" 2>/dev/null || true
|
||||
}
|
||||
|
||||
output 1836x1006
|
||||
if wants login; then
|
||||
"$BIN/pdfs-app" &
|
||||
app=$!
|
||||
shoot login
|
||||
quit "$app"
|
||||
fi
|
||||
"$BIN/demo_daemon" seed-keyring
|
||||
"$BIN/pdfs-app" &
|
||||
app=$!
|
||||
sleep "$SETTLE"
|
||||
for shot in files:browser photos:gallery photos_map:places-map computers:devices \
|
||||
sync:sync-folders conflicts:activity; do
|
||||
name=${shot%%:*}
|
||||
wants "$name" || continue
|
||||
"$BIN/pdfs-app" --page "${shot#*:}"
|
||||
shoot "$name"
|
||||
done
|
||||
quit "$app"
|
||||
|
||||
if wants prompt_gtk; then
|
||||
output 860x723
|
||||
"$BIN/pdfs-prompt" --gtk &
|
||||
prompt=$!
|
||||
sleep 1
|
||||
# The window has no --query; type one, as a person would.
|
||||
# The first key only sets up the keymap.
|
||||
command -v wtype >/dev/null && wtype -k Shift_L -s 200 report
|
||||
shoot prompt_gtk
|
||||
quit "$prompt"
|
||||
trim prompt_gtk
|
||||
fi
|
||||
if wants prompt_fzf; then
|
||||
if command -v fzf >/dev/null && command -v alacritty >/dev/null; then
|
||||
output 800x600
|
||||
alacritty -o font.size=11 -e "$BIN/pdfs-prompt" --fzf --inner --query report &
|
||||
term=$!
|
||||
shoot prompt_fzf
|
||||
quit "$term"
|
||||
trim prompt_fzf
|
||||
else
|
||||
echo "skipped prompt_fzf: needs fzf and alacritty" >&2
|
||||
fi
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Inside the private session bus: an unlocked, empty keyring, the demo daemon,
|
||||
# then the compositor running the capture above.
|
||||
if [[ "${1:-}" == --session ]]; then
|
||||
# A password it never asks for again: an empty one leaves no keyring.
|
||||
printf demo | gnome-keyring-daemon --unlock --components=secrets >/dev/null
|
||||
"$BIN/demo_daemon" serve "$PHOTOS" 2>"$PDFS_DEMO_ROOT/demo_daemon.log" &
|
||||
daemon=$!
|
||||
trap 'kill "$daemon" 2>/dev/null || true' EXIT
|
||||
for _ in $(seq 50); do
|
||||
[[ -S "$XDG_STATE_HOME/proton-drive-linux/control.sock" ]] && break
|
||||
sleep 0.1
|
||||
done
|
||||
WLR_BACKENDS=headless WLR_LIBINPUT_NO_DEVICES=1 \
|
||||
cage -- "$repo/scripts/screenshots.sh" --capture 2>"$PDFS_DEMO_ROOT/cage.log"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
out="$repo/images"
|
||||
shots=$all
|
||||
website=
|
||||
settle=3
|
||||
profile=release
|
||||
keep=
|
||||
while (($#)); do
|
||||
case "$1" in
|
||||
--out) out=$(realpath -m -- "${2:?}"); shift ;;
|
||||
--only) shots=${2:?}; shift ;;
|
||||
--website) website=$(realpath -- "${2:?}"); shift ;;
|
||||
--settle) settle=${2:?}; shift ;;
|
||||
--debug) profile=debug ;;
|
||||
--keep) keep=1 ;;
|
||||
-h | --help) usage ;;
|
||||
*) echo "unknown option: $1" >&2; usage ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
for shot in ${shots//,/ }; do
|
||||
[[ ",$all," == *",$shot,"* ]] || { echo "no screenshot called $shot" >&2; exit 2; }
|
||||
done
|
||||
for tool in cage grim wlr-randr magick dbus-run-session gnome-keyring-daemon cargo curl; do
|
||||
command -v "$tool" >/dev/null || { echo "$tool is required" >&2; exit 2; }
|
||||
done
|
||||
|
||||
# Demo photos, fetched once. Fixed picsum ids keep every run the same; a
|
||||
# trailing -p marks a portrait one.
|
||||
photos="${XDG_CACHE_HOME:-$HOME/.cache}/pdfs-screenshots"
|
||||
mkdir -p "$photos"
|
||||
for id in 10 11 13 15 16 17 28 29 37 42 43 49 54 57 58 76 82 83 84 96 101 103 110 \
|
||||
111 116 119 122 128 129 135 142 145 152 164 167 174 176 180 184 188 192 200 \
|
||||
210 218 219 225 231 237; do
|
||||
size=600/400 file="$photos/$id.jpg"
|
||||
if ((id % 7 == 0)); then size=400/600 file="$photos/$id-p.jpg"; fi
|
||||
[[ -s "$file" ]] || curl -fsSL "https://picsum.photos/id/$id/$size.jpg" -o "$file" ||
|
||||
{ rm -f "$file"; echo "could not fetch demo photo $id; skipped" >&2; }
|
||||
done
|
||||
|
||||
cargo_flags=(--manifest-path "$repo/Cargo.toml" --locked)
|
||||
[[ "$profile" == release ]] && cargo_flags+=(--release)
|
||||
cargo build "${cargo_flags[@]}" -p pdfs-gui --bin pdfs-app --bin pdfs-prompt
|
||||
cargo build "${cargo_flags[@]}" -p pdfs-core --example demo_daemon
|
||||
target=$(cargo metadata --manifest-path "$repo/Cargo.toml" --format-version 1 --no-deps |
|
||||
python3 -c 'import json, sys; print(json.load(sys.stdin)["target_directory"])')
|
||||
bin="$(mktemp -d)"
|
||||
ln -s "$target/$profile/pdfs-app" "$target/$profile/pdfs-prompt" \
|
||||
"$target/$profile/examples/demo_daemon" "$bin/"
|
||||
|
||||
root=$(mktemp -d -t pdfs-screenshots.XXXXXX)
|
||||
cleanup() {
|
||||
rm -rf "$bin"
|
||||
if [[ -n "$keep" ]]; then echo "kept $root" >&2; else rm -rf "$root"; fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$root"/{home,config,state,cache,data} "$root/run" "$out"
|
||||
chmod 700 "$root/run"
|
||||
# The user's own fonts, so the shots look like the app does here.
|
||||
mkdir -p "$root/data/fonts"
|
||||
[[ -d "$HOME/.local/share/fonts" ]] && cp -rs "$HOME/.local/share/fonts/." "$root/data/fonts/"
|
||||
mkdir -p "$root/config/proton-drive-linux"
|
||||
cat >"$root/config/proton-drive-linux/config.json" <<'EOF'
|
||||
{
|
||||
"proton_theme": true,
|
||||
"tray_hidden": true,
|
||||
"update_check": false,
|
||||
"online_map": false,
|
||||
"files_view": { "list": false }
|
||||
}
|
||||
EOF
|
||||
|
||||
# Only a close button, as GNOME shows it.
|
||||
mkdir -p "$root/config/gtk-4.0"
|
||||
printf '[Settings]\ngtk-decoration-layout=:close\n' >"$root/config/gtk-4.0/settings.ini"
|
||||
|
||||
# The demo user's home, so paths read "Home / …" for them, not for you. A
|
||||
# folder that does not exist takes no writes; everything goes to XDG_*.
|
||||
home=/home/alex
|
||||
[[ -e "$home" ]] && home="$root/home"
|
||||
|
||||
outer_bus=${DBUS_SESSION_BUS_ADDRESS:-unset}
|
||||
env -i \
|
||||
PATH="$PATH" LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 TZ="${TZ:-Europe/Berlin}" \
|
||||
HOME="$home" \
|
||||
XDG_CONFIG_HOME="$root/config" XDG_STATE_HOME="$root/state" \
|
||||
XDG_CACHE_HOME="$root/cache" XDG_DATA_HOME="$root/data" \
|
||||
XDG_RUNTIME_DIR="$root/run" \
|
||||
PDFS_DEMO_ROOT="$root" PDFS_DEMO_OUTER_BUS="$outer_bus" \
|
||||
GSETTINGS_BACKEND=memory ADW_DEBUG_COLOR_SCHEME=prefer-dark GTK_A11Y=none GDK_DEBUG=no-portals \
|
||||
BIN="$bin" PHOTOS="$photos" OUT="$out" SHOTS="$shots" SETTLE="$settle" \
|
||||
dbus-run-session -- "$repo/scripts/screenshots.sh" --session 2>"$root/session.log"
|
||||
|
||||
for shot in ${shots//,/ }; do
|
||||
[[ -s "$out/$shot.png" ]] || echo "missing: $out/$shot.png (logs: --keep)" >&2
|
||||
done
|
||||
if [[ -n "$website" ]]; then
|
||||
"$website/scripts/sync-screenshots.sh" "$repo"
|
||||
fi
|
||||