mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-03 16:41:26 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
860d29cfe8 | ||
|
|
6cc8f72bab | ||
|
|
376f3edb28 | ||
|
|
09c0684a5e | ||
|
|
c1df3b6cc2 | ||
|
|
afd71713d5 | ||
|
|
c6bdba024f | ||
|
|
5a8e73657c | ||
|
|
cf0d495f8c | ||
|
|
bee15ee7cf | ||
|
|
f24ac8a429 | ||
|
|
f78f663740 | ||
|
|
470948c191 | ||
|
|
b095b9f6b0 |
No files matched your search
Generated
+8
-8
@@ -3372,7 +3372,7 @@ checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-cli"
|
||||
version = "1.10.1"
|
||||
version = "1.11.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
@@ -3387,7 +3387,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-core"
|
||||
version = "1.10.1"
|
||||
version = "1.11.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"dbus-secret-service-keyring-store",
|
||||
@@ -3411,7 +3411,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-fuse"
|
||||
version = "1.10.1"
|
||||
version = "1.11.1"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"fuser",
|
||||
@@ -3431,7 +3431,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pdfs-gui"
|
||||
version = "1.10.1"
|
||||
version = "1.11.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-channel",
|
||||
@@ -3668,9 +3668,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-drive-rs"
|
||||
version = "0.6.4"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae1ebbfe1696ae497807d87699a55d860612ceac250b9d60825bae0baccd622a"
|
||||
checksum = "46cbe05623d9b73bd48b1473f42d43180142b2ee676b1cb9e9a06b562c60ba26"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -3695,9 +3695,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proton-sdk"
|
||||
version = "0.6.4"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de5426ca4b5682b6dc0399e7359f343c0146da6d8465196ee49bb8825f1a90ee"
|
||||
checksum = "7310d93bed440dfe4fd60306923f6a491901ee0d618b65ace4e5e22ff0387c1f"
|
||||
dependencies = [
|
||||
"aes-gcm 0.11.1",
|
||||
"async-trait",
|
||||
|
||||
+3
-3
@@ -3,7 +3,7 @@ resolver = "2"
|
||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.10.1"
|
||||
version = "1.11.1"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||
@@ -15,8 +15,8 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
|
||||
categories = ["filesystem", "command-line-utilities"]
|
||||
|
||||
[workspace.dependencies]
|
||||
proton-sdk = { version = "0.6.4" }
|
||||
proton-drive-rs = { version = "0.6.4" }
|
||||
proton-sdk = { version = "0.6.5" }
|
||||
proton-drive-rs = { version = "0.6.5" }
|
||||
|
||||
# Internal crates.
|
||||
pdfs-core = { path = "crates/pdfs-core" }
|
||||
|
||||
+103
-3
@@ -281,6 +281,10 @@ enum Command {
|
||||
},
|
||||
/// Show the daemon's in-flight transfers (active uploads/downloads).
|
||||
Transfers,
|
||||
/// Show what the daemon is doing right now: worker threads, queue depths,
|
||||
/// in-flight control requests and memory. For a daemon that has stopped
|
||||
/// answering rather than one that is merely busy.
|
||||
Diagnostics,
|
||||
/// List what is in the account's trash, with the uids the restore and
|
||||
/// delete-forever commands take.
|
||||
Trash,
|
||||
@@ -306,6 +310,11 @@ enum Command {
|
||||
/// What to refresh: a folder path (inside the mountpoint or relative to
|
||||
/// it), `trash`, or `photos`. The mount root if omitted.
|
||||
target: Option<String>,
|
||||
/// With `photos`: read every photo's metadata again instead of only what
|
||||
/// changed. Minutes on a large library; for a gallery that disagrees with
|
||||
/// the account.
|
||||
#[arg(long)]
|
||||
full: bool,
|
||||
},
|
||||
|
||||
/// Manage the account's registered devices.
|
||||
@@ -674,11 +683,12 @@ fn main() -> Result<()> {
|
||||
Command::Mkdir { parent, name } => cmd_mkdir(parent, name),
|
||||
Command::Upload { sources, parent } => cmd_upload(sources, parent),
|
||||
Command::Transfers => cmd_transfers(),
|
||||
Command::Diagnostics => cmd_diagnostics(),
|
||||
Command::Trash => cmd_trash(),
|
||||
Command::Restore { uids } => cmd_restore(uids),
|
||||
Command::DeleteForever { uids } => cmd_delete_forever(uids),
|
||||
Command::EmptyTrash => cmd_empty_trash(),
|
||||
Command::Refresh { target } => cmd_refresh(target),
|
||||
Command::Refresh { target, full } => cmd_refresh(target, full),
|
||||
Command::Devices { action } => cmd_devices(action),
|
||||
Command::Sync { action } => cmd_sync(action),
|
||||
Command::Share {
|
||||
@@ -2153,6 +2163,67 @@ fn cmd_rm(path: PathBuf) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Seconds as something a human reads at a glance: the ages here run from
|
||||
/// "still starting" to "wedged since yesterday".
|
||||
fn human_secs(secs: u64) -> String {
|
||||
match secs {
|
||||
s if s < 60 => format!("{s}s"),
|
||||
s if s < 3600 => format!("{}m{}s", s / 60, s % 60),
|
||||
s if s < 86400 => format!("{}h{}m", s / 3600, (s % 3600) / 60),
|
||||
s => format!("{}d{}h", s / 86400, (s % 86400) / 3600),
|
||||
}
|
||||
}
|
||||
|
||||
fn cmd_diagnostics() -> Result<()> {
|
||||
let response = control_request(CtlRequest::Diagnostics)?;
|
||||
if emit_json(&response)? {
|
||||
return Ok(());
|
||||
}
|
||||
match response {
|
||||
CtlResponse::Diagnostics { diagnostics: d } => {
|
||||
println!(
|
||||
"uptime {} rss {} pending ops {}",
|
||||
human_secs(d.uptime_secs),
|
||||
human_bytes(d.rss_bytes),
|
||||
d.pending_ops,
|
||||
);
|
||||
println!(
|
||||
"control handlers {}/{}",
|
||||
d.control_handlers, d.control_handler_limit,
|
||||
);
|
||||
println!(
|
||||
"queued: meta {} (done {}), transfer {} (done {})",
|
||||
d.meta_queued, d.meta_completed, d.transfer_queued, d.transfer_completed,
|
||||
);
|
||||
println!("workers:");
|
||||
for worker in &d.workers {
|
||||
let what = if worker.busy {
|
||||
worker.label.as_str()
|
||||
} else {
|
||||
"idle"
|
||||
};
|
||||
println!(
|
||||
" {:<20} {:<12} {}",
|
||||
worker.name,
|
||||
what,
|
||||
human_secs(worker.age_secs)
|
||||
);
|
||||
}
|
||||
if d.inflight.is_empty() {
|
||||
println!("in-flight requests: none");
|
||||
} else {
|
||||
println!("in-flight requests (oldest first):");
|
||||
for request in &d.inflight {
|
||||
println!(" {:<20} {}", request.kind, human_secs(request.age_secs));
|
||||
}
|
||||
}
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_trash() -> Result<()> {
|
||||
match control_request(CtlRequest::ListTrash)? {
|
||||
CtlResponse::Entries { entries } if entries.is_empty() => println!("(trash is empty)"),
|
||||
@@ -2203,22 +2274,51 @@ fn cmd_empty_trash() -> Result<()> {
|
||||
/// Drop a cached listing. `trash` and `photos` name those two listings; anything
|
||||
/// else is read as a folder path, so a folder actually called "trash" is still
|
||||
/// reachable as `./trash`.
|
||||
fn cmd_refresh(target: Option<String>) -> Result<()> {
|
||||
fn cmd_refresh(target: Option<String>, full: bool) -> Result<()> {
|
||||
let scope = match target.as_deref() {
|
||||
Some("trash") => RefreshScope::Trash,
|
||||
Some("photos") => RefreshScope::Photos,
|
||||
Some("photos") => RefreshScope::Photos { full },
|
||||
path => RefreshScope::Dir {
|
||||
path: path.unwrap_or("").to_string(),
|
||||
},
|
||||
};
|
||||
let photos = matches!(scope, RefreshScope::Photos { .. });
|
||||
match control_request(CtlRequest::Refresh { scope })? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
if photos {
|
||||
wait_for_photos_refresh()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// How often [`wait_for_photos_refresh`] asks the daemon whether the timeline
|
||||
/// refresh it started is done.
|
||||
const PHOTOS_REFRESH_POLL: std::time::Duration = std::time::Duration::from_secs(2);
|
||||
|
||||
/// Block until the daemon's timeline refresh has finished.
|
||||
///
|
||||
/// The daemon answers `Refresh { Photos }` as soon as the refresh is running,
|
||||
/// because re-reading a large library takes minutes and a control reply may not.
|
||||
/// `pdfs refresh photos` means "the timeline is up to date when I return", so
|
||||
/// the waiting happens here, where no socket read bound applies.
|
||||
fn wait_for_photos_refresh() -> Result<()> {
|
||||
loop {
|
||||
std::thread::sleep(PHOTOS_REFRESH_POLL);
|
||||
match control_request(CtlRequest::PhotosRefreshStatus)? {
|
||||
CtlResponse::PhotosRefresh { running: true } => continue,
|
||||
CtlResponse::PhotosRefresh { running: false } => {
|
||||
println!("refreshed");
|
||||
return Ok(());
|
||||
}
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask for a typed `yes` before an irreversible destroy. Errors (aborting the
|
||||
/// command) on anything else, including a non-interactive stdin: a piped or
|
||||
/// scripted invocation must not silently wipe the trash.
|
||||
|
||||
@@ -68,6 +68,11 @@ pub enum Request {
|
||||
#[serde(default)]
|
||||
favorites: bool,
|
||||
},
|
||||
/// Every photo of one photo's group — the JPEG and the RAW of one shot, a
|
||||
/// live photo and its clip. Replies with [`Response::Photos`], holding one
|
||||
/// item per file in server order; a photo that stands alone comes back as a
|
||||
/// single item. The lightbox uses it to switch between a shot's files.
|
||||
PhotoGroup { uid: String },
|
||||
/// The months the timeline spans (newest first, with per-month counts) so a
|
||||
/// front-end can build a date scrubber without paging the whole library.
|
||||
/// `kind` scopes the counts to one tab when set.
|
||||
@@ -93,6 +98,17 @@ pub enum Request {
|
||||
/// already cached (one batched round-trip) and replying with their on-disk
|
||||
/// paths. Keep the batch small — it is served on demand, as tiles scroll in.
|
||||
PhotoThumbs { uids: Vec<String> },
|
||||
/// Trash photos by node uid, answered with [`Response::Trashed`].
|
||||
///
|
||||
/// Uid-addressed rather than path-addressed because the photos volume is not
|
||||
/// part of the FUSE mount: the gallery knows a photo only by its uid, and
|
||||
/// [`Request::Delete`] has no path to take for it. The nodes land in Proton
|
||||
/// trash, so this stays recoverable from the Trash page.
|
||||
///
|
||||
/// A photo that belongs to a group takes the whole group with it: trashing
|
||||
/// the JPEG of a shot and leaving its RAW behind is not what anyone means by
|
||||
/// deleting that photo.
|
||||
TrashNodes { uids: Vec<String> },
|
||||
/// Fetch thumbnails for ordinary Drive image files shown outside the Photos
|
||||
/// timeline. The modification time is the cache validity tag, so replacing
|
||||
/// an image can never reuse its previous revision's thumbnail.
|
||||
@@ -119,6 +135,12 @@ pub enum Request {
|
||||
CancelThumbnailBuild,
|
||||
/// Read progress for the recursive ordinary-file thumbnail build.
|
||||
ThumbnailBuildStatus,
|
||||
/// Whether a photos-timeline refresh is running right now. Replies with
|
||||
/// [`Response::PhotosRefresh`].
|
||||
///
|
||||
/// [`RefreshScope::Photos`] starts the refresh and returns at once, so this
|
||||
/// is how a front-end knows when the refreshed timeline is worth re-reading.
|
||||
PhotosRefreshStatus,
|
||||
/// Download a photo's full content into the cache; replies with its path.
|
||||
OpenPhoto { uid: String },
|
||||
/// Add or remove Proton's `Favorite` tag on a photo. Replies with
|
||||
@@ -299,6 +321,13 @@ pub enum Request {
|
||||
/// [`Response::Transfers`]. Cheap to poll: the daemon keeps the registry in
|
||||
/// memory, so a front-end can render a live progress widget.
|
||||
GetQueueStatus,
|
||||
/// Report what the daemon is doing *right now*, for debugging a daemon that
|
||||
/// stopped answering rather than one that is merely busy. Replies
|
||||
/// [`Response::Diagnostics`].
|
||||
///
|
||||
/// Answered from lock-free state on purpose: it has to come back while the
|
||||
/// daemon is wedged, which is the only moment anyone asks for it.
|
||||
Diagnostics,
|
||||
/// List what is in the account's trash. Replies with [`Response::Entries`];
|
||||
/// a trashed node has no path inside the mount, so each entry carries only
|
||||
/// its `uid` — the handle for [`Request::Restore`] and
|
||||
@@ -559,8 +588,19 @@ pub enum RefreshScope {
|
||||
Dir { path: String },
|
||||
/// The trash listing.
|
||||
Trash,
|
||||
/// The photos timeline.
|
||||
Photos,
|
||||
/// The photos timeline. The reply comes back as soon as the refresh has
|
||||
/// *started*, not when it has finished: re-reading a large library takes
|
||||
/// minutes, which is far longer than a control reply may take. Poll
|
||||
/// [`Request::PhotosRefreshStatus`] to know when the new timeline is there.
|
||||
///
|
||||
/// A refresh normally only reads the photos the daemon has never read and
|
||||
/// the ones a remote event marked changed. `full` drops that knowledge and
|
||||
/// reads every photo's node again — the repair for a timeline that somehow
|
||||
/// disagrees with the account, and minutes of work on a large library.
|
||||
Photos {
|
||||
#[serde(default)]
|
||||
full: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// A registered device in a [`Response::Devices`] listing.
|
||||
@@ -1273,6 +1313,15 @@ pub struct PhotoItem {
|
||||
/// tags it has not learned yet.
|
||||
#[serde(default)]
|
||||
pub favorite: bool,
|
||||
/// How many files this entry stands for: a shot stored as a RAW and a JPEG
|
||||
/// is one entry and two files. Older daemons omit it, and `0` reads the same
|
||||
/// as `1` — one file, nothing to indicate.
|
||||
#[serde(default)]
|
||||
pub group_size: u32,
|
||||
/// Whether one of those files is a camera raw. The tile badges it, because
|
||||
/// the JPEG it shows is not the whole of what is stored.
|
||||
#[serde(default)]
|
||||
pub has_raw: bool,
|
||||
}
|
||||
|
||||
/// A daemon too old to classify a timeline entry is assumed to have served a
|
||||
@@ -1281,6 +1330,13 @@ fn default_photo_kind() -> PhotoKind {
|
||||
PhotoKind::Photo
|
||||
}
|
||||
|
||||
/// One node a [`Request::TrashNodes`] batch could not trash, and why.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct TrashFailure {
|
||||
pub uid: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// One thumbnail in a [`Response::Thumbs`] batch.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct PhotoThumb {
|
||||
@@ -1400,6 +1456,55 @@ pub fn pending_summary(uploads: u64, changes: u64) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// One worker thread in a [`Diagnostics`] report.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct WorkerState {
|
||||
/// Thread name (`pdfs-fuse-meta-0`, `pdfs-fuse-3`).
|
||||
pub name: String,
|
||||
pub busy: bool,
|
||||
/// The job in hand (`read`, `lookup`), empty when idle.
|
||||
pub label: String,
|
||||
/// How long it has been in that state. A busy worker with a large age is
|
||||
/// the shape of a hang.
|
||||
pub age_secs: u64,
|
||||
}
|
||||
|
||||
/// One control request the daemon is still serving, in a [`Diagnostics`] report.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct InflightRequest {
|
||||
/// The request's variant name (`OpenFile`, `PhotosTimeline`).
|
||||
pub kind: String,
|
||||
pub age_secs: u64,
|
||||
}
|
||||
|
||||
/// What the daemon is doing right now (reply to [`Request::Diagnostics`]).
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||
pub struct Diagnostics {
|
||||
/// How long this daemon process has been running.
|
||||
pub uptime_secs: u64,
|
||||
/// Resident set size, or `0` where it could not be read. The 2026-09-18 hang
|
||||
/// peaked at 5.2 GiB, so this is the number that says whether the daemon is
|
||||
/// growing again.
|
||||
pub rss_bytes: u64,
|
||||
pub workers: Vec<WorkerState>,
|
||||
/// Jobs waiting for a worker, per lane.
|
||||
pub meta_queued: usize,
|
||||
pub transfer_queued: usize,
|
||||
/// Jobs finished per lane since start. Queue depth that does not move while
|
||||
/// these do not either is a stalled lane, not a loaded one.
|
||||
pub meta_completed: u64,
|
||||
pub transfer_completed: u64,
|
||||
/// Control requests still being served, oldest first.
|
||||
pub inflight: Vec<InflightRequest>,
|
||||
/// Control handler threads in use, and the ceiling. At the ceiling the
|
||||
/// daemon refuses new connections, which looks exactly like a dead daemon
|
||||
/// from the CLI.
|
||||
pub control_handlers: usize,
|
||||
pub control_handler_limit: usize,
|
||||
/// Queued mutations not yet drained.
|
||||
pub pending_ops: u64,
|
||||
}
|
||||
|
||||
/// The daemon's reply to a [`Request`].
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub enum Response {
|
||||
@@ -1508,6 +1613,8 @@ pub enum Response {
|
||||
available: bool,
|
||||
items: Vec<AlbumInfo>,
|
||||
},
|
||||
/// What the daemon is doing right now (reply to [`Request::Diagnostics`]).
|
||||
Diagnostics { diagnostics: Diagnostics },
|
||||
/// How the Takeout import is doing (reply to [`Request::ImportStatus`]).
|
||||
/// `running` is false once it has finished, when `summary` is the final
|
||||
/// report; `summary` is `None` when no import has run this session.
|
||||
@@ -1529,6 +1636,14 @@ pub enum Response {
|
||||
PhotoMonths { months: Vec<PhotoMonth> },
|
||||
/// Thumbnails for a [`Request::PhotoThumbs`] batch.
|
||||
Thumbs { items: Vec<PhotoThumb> },
|
||||
/// Per-node outcome of a [`Request::TrashNodes`] batch. Reported per uid
|
||||
/// rather than as one verdict: the server answers each node separately, and
|
||||
/// a front-end that removed all of them on a partial success would show a
|
||||
/// photo as gone that is still there.
|
||||
Trashed {
|
||||
trashed: Vec<String>,
|
||||
failed: Vec<TrashFailure>,
|
||||
},
|
||||
/// A [`Request::FileThumbs`] generation is no longer current. This is
|
||||
/// retryable after reserving a fresh generation and must never be interpreted
|
||||
/// as a permanent "no thumbnail" verdict.
|
||||
@@ -1538,6 +1653,9 @@ pub enum Response {
|
||||
FileThumbGeneration { generation: u64 },
|
||||
/// Current recursive thumbnail-build progress.
|
||||
ThumbnailBuild { status: ThumbnailBuildStatus },
|
||||
/// Whether a photos-timeline refresh is still running (reply to
|
||||
/// [`Request::PhotosRefreshStatus`]).
|
||||
PhotosRefresh { running: bool },
|
||||
/// An on-disk path the front-end can open (e.g. a downloaded photo).
|
||||
FilePath { path: String },
|
||||
/// Full-text search results (reply to [`Request::Search`]).
|
||||
@@ -1867,6 +1985,62 @@ mod tests {
|
||||
assert!(items[0].name.is_empty());
|
||||
}
|
||||
|
||||
/// The lightbox asks for one shot's files by the uid of the tile that was
|
||||
/// clicked, and gets them back in the shape the gallery already paints.
|
||||
#[test]
|
||||
fn photo_group_requests_roundtrip() {
|
||||
let request = Request::PhotoGroup {
|
||||
uid: "volume~first".into(),
|
||||
};
|
||||
let line = serde_json::to_string(&request).unwrap();
|
||||
assert_eq!(line, r#"{"PhotoGroup":{"uid":"volume~first"}}"#);
|
||||
let decoded: Request = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(line, serde_json::to_string(&decoded).unwrap());
|
||||
}
|
||||
|
||||
/// The group fields are new, so a page from a daemon that predates them has
|
||||
/// to keep painting: one file, nothing to badge.
|
||||
#[test]
|
||||
fn an_older_daemons_photo_has_no_group() {
|
||||
let item: PhotoItem = serde_json::from_str(
|
||||
r#"{"uid":"volume~first","capture_time":1700000000,"thumb_path":null,
|
||||
"name":"a.jpg","ratio":1.5,"no_thumb":false}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(item.group_size, 0, "absent reads the same as one file");
|
||||
assert!(!item.has_raw);
|
||||
}
|
||||
|
||||
/// The gallery trashes by uid, and reports per-uid outcomes back: a partial
|
||||
/// failure has to survive the wire so the GUI can put those tiles back.
|
||||
#[test]
|
||||
fn trash_requests_and_outcomes_roundtrip() {
|
||||
let request = Request::TrashNodes {
|
||||
uids: vec!["volume~first".into(), "volume~second".into()],
|
||||
};
|
||||
let line = serde_json::to_string(&request).unwrap();
|
||||
assert!(!line.contains('\n'), "wire form must be a single line");
|
||||
let decoded: Request = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(line, serde_json::to_string(&decoded).unwrap());
|
||||
|
||||
let response = Response::Trashed {
|
||||
trashed: vec!["volume~first".into()],
|
||||
failed: vec![TrashFailure {
|
||||
uid: "volume~second".into(),
|
||||
message: "permission denied".into(),
|
||||
}],
|
||||
};
|
||||
let line = serde_json::to_string(&response).unwrap();
|
||||
let decoded: Response = serde_json::from_str(&line).unwrap();
|
||||
let Response::Trashed { trashed, failed } = decoded else {
|
||||
panic!("decoded the wrong response variant");
|
||||
};
|
||||
assert_eq!(trashed, ["volume~first"]);
|
||||
assert_eq!(failed.len(), 1);
|
||||
assert_eq!(failed[0].uid, "volume~second");
|
||||
assert_eq!(failed[0].message, "permission denied");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thumbnail_build_requests_and_progress_roundtrip() {
|
||||
let requests = [
|
||||
@@ -1911,6 +2085,31 @@ mod tests {
|
||||
assert_eq!(line, serde_json::to_string(&decoded).unwrap());
|
||||
}
|
||||
|
||||
/// `full` defaults, so a front-end built against the older wire shape still
|
||||
/// asks for the ordinary refresh rather than failing to parse.
|
||||
#[test]
|
||||
fn a_photos_refresh_scope_defaults_to_not_full() {
|
||||
let scope: RefreshScope = serde_json::from_str(r#"{"Photos":{}}"#).unwrap();
|
||||
assert_eq!(scope, RefreshScope::Photos { full: false });
|
||||
let line = serde_json::to_string(&RefreshScope::Photos { full: true }).unwrap();
|
||||
assert_eq!(line, r#"{"Photos":{"full":true}}"#);
|
||||
}
|
||||
|
||||
/// A photos refresh is followed, not awaited, so the request that reports
|
||||
/// its progress is part of the wire contract.
|
||||
#[test]
|
||||
fn photos_refresh_status_roundtrips() {
|
||||
let line = serde_json::to_string(&Request::PhotosRefreshStatus).unwrap();
|
||||
assert_eq!(line, r#""PhotosRefreshStatus""#);
|
||||
let decoded: Request = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(line, serde_json::to_string(&decoded).unwrap());
|
||||
|
||||
let line = serde_json::to_string(&Response::PhotosRefresh { running: true }).unwrap();
|
||||
assert_eq!(line, r#"{"PhotosRefresh":{"running":true}}"#);
|
||||
let decoded: Response = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(line, serde_json::to_string(&decoded).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unified_search_request_and_response_roundtrip() {
|
||||
let request = Request::SearchV2 {
|
||||
@@ -2081,7 +2280,10 @@ mod tests {
|
||||
scope: RefreshScope::Trash,
|
||||
},
|
||||
Request::Refresh {
|
||||
scope: RefreshScope::Photos,
|
||||
scope: RefreshScope::Photos { full: false },
|
||||
},
|
||||
Request::Refresh {
|
||||
scope: RefreshScope::Photos { full: true },
|
||||
},
|
||||
];
|
||||
for req in reqs {
|
||||
|
||||
@@ -182,6 +182,10 @@ impl Db {
|
||||
// photo on someone else's volume is not in our timeline, and
|
||||
// favouriting one is not supported yet anyway.
|
||||
favorite: false,
|
||||
// Nor are album rows grouped: an album is a list the user
|
||||
// made, and it shows exactly what they put in it.
|
||||
group_size: 1,
|
||||
has_raw: false,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
@@ -239,6 +243,10 @@ impl Db {
|
||||
// photo on someone else's volume is not in our timeline, and
|
||||
// favouriting one is not supported yet anyway.
|
||||
favorite: false,
|
||||
// Nor are album rows grouped: an album is a list the user
|
||||
// made, and it shows exactly what they put in it.
|
||||
group_size: 1,
|
||||
has_raw: false,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
/// Current schema version. Bump on every forward migration added below.
|
||||
pub(super) const SCHEMA_VERSION: i64 = 28;
|
||||
pub(super) const SCHEMA_VERSION: i64 = 31;
|
||||
|
||||
impl Db {
|
||||
pub(super) fn migrate(&self) -> Result<()> {
|
||||
@@ -218,6 +218,58 @@ impl Db {
|
||||
tx.execute_batch(MIGRATION_V28)?;
|
||||
}
|
||||
}
|
||||
if current < 29 {
|
||||
// Same guards as V26/V27/V28: an old fixture may not have `trash`,
|
||||
// and a fixture rewound from the current schema already has the
|
||||
// column.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('trash') WHERE name = 'parent_uid'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_trash: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'trash'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_trash && !has_column {
|
||||
tx.execute_batch(MIGRATION_V29)?;
|
||||
}
|
||||
}
|
||||
if current < 30 {
|
||||
// Same guards as V26-V29: the column may already be there on a
|
||||
// fixture rewound from the current schema, and a very old fixture
|
||||
// may not have `photos` at all.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('photos') WHERE name = 'group_key'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_photos: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'photos'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_photos && !has_column {
|
||||
tx.execute_batch(MIGRATION_V30)?;
|
||||
}
|
||||
}
|
||||
if current < 31 {
|
||||
// Same guards as V26-V30.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('photos') WHERE name = 'resolved_at'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_photos: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'photos'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_photos && !has_column {
|
||||
tx.execute_batch(MIGRATION_V31)?;
|
||||
}
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -846,3 +898,57 @@ ALTER TABLE pending_op ADD COLUMN access_deferred_since INTEGER NOT NULL DEFAULT
|
||||
const MIGRATION_V28: &str = "
|
||||
ALTER TABLE sync_entry ADD COLUMN local_mtime_ns INTEGER;
|
||||
";
|
||||
|
||||
/// Schema v29: the trash listing remembers each trashed node's parent, so a
|
||||
/// restore can reason about the shape of what it is putting back.
|
||||
///
|
||||
/// Trash is a flat list of uids on the wire, but the user deletes a *tree*: a
|
||||
/// folder and, often, items inside it that were trashed separately. Restoring
|
||||
/// one of those in isolation is what the parent link fixes — a child restored
|
||||
/// under a still-trashed parent lands somewhere invisible, and a folder restored
|
||||
/// without its separately-trashed contents comes back empty.
|
||||
///
|
||||
/// `NULL` means "this row predates the column" (or the node is a volume root);
|
||||
/// the next trash refresh fills it in, and until then the restore behaves as it
|
||||
/// did before.
|
||||
const MIGRATION_V29: &str = "
|
||||
ALTER TABLE trash ADD COLUMN parent_uid TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_trash_parent ON trash(parent_uid);
|
||||
";
|
||||
|
||||
/// Schema v30: a RAW and the JPEG of the same shot are one photo to the person
|
||||
/// who took it, so the timeline records what ties them together.
|
||||
///
|
||||
/// `content_hash` and `main_uid` come from the server's own photo properties —
|
||||
/// the duplicate-detection hash, and the main photo a related one belongs to.
|
||||
/// `group_key` is what the gallery pages by: every member of one shot carries
|
||||
/// the same key, and the member the grid shows is the one whose `main_uid` is
|
||||
/// its own uid.
|
||||
///
|
||||
/// All three are `NULL` until the next timeline refresh resolves the photo's
|
||||
/// node, and a row without a `group_key` is its own group — which is exactly how
|
||||
/// the gallery behaved before this column existed.
|
||||
const MIGRATION_V30: &str = "
|
||||
ALTER TABLE photos ADD COLUMN content_hash TEXT;
|
||||
ALTER TABLE photos ADD COLUMN main_uid TEXT;
|
||||
ALTER TABLE photos ADD COLUMN group_key TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_photos_group ON photos(group_key);
|
||||
";
|
||||
|
||||
/// Schema v31: which photos a refresh still has to read from the server.
|
||||
///
|
||||
/// A refresh used to resolve every photo's node to learn its name, media type,
|
||||
/// favourite tag, content hash and photo relation — minutes of round-trips for a
|
||||
/// library of any size, repeated every time the timeline went stale. `resolved_at`
|
||||
/// records that the work was already done, so a refresh only reads the photos it
|
||||
/// has never read and the ones a remote event marked stale (`resolved_at = NULL`).
|
||||
///
|
||||
/// The backfill treats an existing timeline as resolved, so an upgrade does not
|
||||
/// re-read the whole library once. `WHERE name IS NOT NULL` keeps the photos
|
||||
/// whose earlier resolve had failed eligible: a nameless row is one nothing was
|
||||
/// ever learned about.
|
||||
const MIGRATION_V31: &str = "
|
||||
ALTER TABLE photos ADD COLUMN resolved_at INTEGER;
|
||||
CREATE INDEX IF NOT EXISTS idx_photos_unresolved ON photos(resolved_at);
|
||||
UPDATE photos SET resolved_at = strftime('%s','now') WHERE name IS NOT NULL;
|
||||
";
|
||||
@@ -56,8 +56,8 @@ pub use local::LocalFileHit;
|
||||
pub use maintenance::{DbStats, VacuumOutcome};
|
||||
pub use nodes::{PublishedSharedRoot, SearchHit, StoredNode};
|
||||
pub use ops::{
|
||||
AttachedBlob, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_UNTIL,
|
||||
PendingCounts, PendingOp, RenameMeta, op_supersedes,
|
||||
AttachedBlob, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH,
|
||||
PARK_EXPIRY_MS, PARK_UNTIL, PendingCounts, PendingOp, RenameMeta, op_supersedes,
|
||||
};
|
||||
pub use photos::{StoredPhoto, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN, TimelineRow};
|
||||
pub use pins::PinRow;
|
||||
|
||||
@@ -84,6 +84,25 @@ pub const LOCAL_VOLUME: &str = "local";
|
||||
/// stall+resume, fork a `(sync-conflict)` copy (docs/BUGS.md B70).
|
||||
pub const PARK_UNTIL: i64 = 8_000_000_000_000;
|
||||
|
||||
/// How long a create may stay parked before the park is treated as permanent
|
||||
/// and the bytes are let through anyway (ms).
|
||||
///
|
||||
/// The park has exactly one exit: a rename from the transient name to the
|
||||
/// finished one. Every writer that reaches that rename does so in seconds — an
|
||||
/// atomic-write scratch file immediately, a download when it completes. A park
|
||||
/// still standing an hour later belongs to a writer that will never perform
|
||||
/// that rename: an editor that deletes its swap file instead, a download the
|
||||
/// user abandoned, or a file that simply *is* named `report.tmp` and is never
|
||||
/// going to be called anything else.
|
||||
///
|
||||
/// Before this existed such a row sat at [`PARK_UNTIL`] for the life of the
|
||||
/// database, holding the only copy of the file's bytes in `staging/` and
|
||||
/// counting against the queue while promising progress that could not come. The
|
||||
/// sweep un-parks it instead of dropping it: bytes the user can see in the mount
|
||||
/// are bytes the user expects on Drive, and invariant 1 of the drain (a staged
|
||||
/// blob is dropped only once its op has landed) admits no other answer.
|
||||
pub const PARK_EXPIRY_MS: i64 = 60 * 60 * 1000;
|
||||
|
||||
/// A mutation that has been accepted locally but not yet performed against the
|
||||
/// API — the durable half of the write-back queue (offline.md Phase 3).
|
||||
///
|
||||
@@ -661,6 +680,40 @@ impl Db {
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Every parked create/mkdir, oldest first, for the park sweep.
|
||||
///
|
||||
/// Parked rows are invisible to [`next_due_op`](Self::next_due_op) by
|
||||
/// construction, so nothing else in the drain ever looks at them. This is
|
||||
/// the one query that does.
|
||||
pub fn parked_create_ops(&self) -> Result<Vec<PendingOp>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, kind, uid, parent_uid, name, blob_path, meta_json, created_at,
|
||||
attempts, last_error, next_attempt_at
|
||||
FROM pending_op
|
||||
WHERE next_attempt_at >= ?1 AND kind IN (?2, ?3)
|
||||
ORDER BY created_at",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map(params![PARK_UNTIL, OP_CREATE, OP_MKDIR], |r| {
|
||||
Ok(PendingOp {
|
||||
id: r.get(0)?,
|
||||
kind: r.get(1)?,
|
||||
uid: r.get(2)?,
|
||||
parent_uid: r.get(3)?,
|
||||
name: r.get(4)?,
|
||||
blob_path: r.get(5)?,
|
||||
meta_json: r.get(6)?,
|
||||
created_at: r.get(7)?,
|
||||
attempts: r.get(8)?,
|
||||
last_error: r.get(9)?,
|
||||
next_attempt_at: r.get(10)?,
|
||||
})
|
||||
})?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// How many queued ops there are of each kind, for `Response::Status`.
|
||||
///
|
||||
/// Split because "3 uploads queued" has to mean three files whose bytes are
|
||||
|
||||
@@ -34,67 +34,347 @@ pub struct StoredPhoto {
|
||||
pub kind: crate::control::PhotoKind,
|
||||
/// Whether the photo carries Proton's `Favorite` tag.
|
||||
pub favorite: bool,
|
||||
/// How many photos this photo's group holds — one shot taken as RAW+JPEG is
|
||||
/// two files and one group. `1` for a photo that stands on its own.
|
||||
pub group_size: usize,
|
||||
/// Whether any member of the group is a camera raw file. The grid badges
|
||||
/// that, because the JPEG it shows is not the whole of what is stored.
|
||||
pub has_raw: bool,
|
||||
}
|
||||
|
||||
/// One row of a timeline replacement: `(uid, capture_time, name, media_type,
|
||||
/// favorite)`. The two trailing options are "what this refresh learned" — a
|
||||
/// `None` keeps whatever is already stored rather than clearing it.
|
||||
pub type TimelineRow = (String, i64, Option<String>, Option<String>, Option<bool>);
|
||||
/// One row of a timeline replacement. Everything but the uid and the capture
|
||||
/// time is "what this refresh learned": a `None` keeps whatever is already
|
||||
/// stored rather than clearing it, because the timeline DTO carries only the uid
|
||||
/// and the capture time and resolving the node can fail.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct TimelineRow {
|
||||
pub uid: String,
|
||||
pub capture_time: i64,
|
||||
pub name: Option<String>,
|
||||
pub media_type: Option<String>,
|
||||
pub favorite: Option<bool>,
|
||||
/// The server's duplicate-detection hash for the plaintext.
|
||||
pub content_hash: Option<String>,
|
||||
/// The main photo this one is *related* to (a live-photo video, a burst
|
||||
/// sibling), when the server says it belongs to one.
|
||||
pub main_uid: Option<String>,
|
||||
/// When this refresh resolved the photo's node, if it did at all. `Some`
|
||||
/// makes every other field authoritative — including a `None` that means
|
||||
/// "the server no longer reports one" — while `None` means the refresh
|
||||
/// skipped this photo and whatever is stored must be kept.
|
||||
pub resolved_at: Option<i64>,
|
||||
}
|
||||
|
||||
impl TimelineRow {
|
||||
/// A row carrying only what the timeline listing itself gives.
|
||||
pub fn new(uid: impl Into<String>, capture_time: i64) -> Self {
|
||||
Self {
|
||||
uid: uid.into(),
|
||||
capture_time,
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a previous refresh learned about a photo and a new one must not lose.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct Learned {
|
||||
name: Option<String>,
|
||||
ratio: Option<f64>,
|
||||
thumb_state: i64,
|
||||
media_type: Option<String>,
|
||||
favorite: bool,
|
||||
content_hash: Option<String>,
|
||||
main_uid: Option<String>,
|
||||
/// When this photo's node was last resolved, kept so a refresh that skips it
|
||||
/// does not make it look unresolved again.
|
||||
resolved_at: Option<i64>,
|
||||
}
|
||||
|
||||
/// The local calendar day a capture time falls on, as a `YYYY-MM-DD` string.
|
||||
/// Grouping by day rather than by exact second is deliberate: a camera writes
|
||||
/// the RAW and the JPEG of one shot a moment apart, and two shots of the same
|
||||
/// subject on different days are not one photo.
|
||||
fn capture_day(conn: &rusqlite::Connection, capture_time: i64) -> Option<String> {
|
||||
conn.query_row(
|
||||
"SELECT strftime('%Y-%m-%d', ?1, 'unixepoch', 'localtime')",
|
||||
params![capture_time],
|
||||
|r| r.get::<_, Option<String>>(0),
|
||||
)
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// The file name without its extension, lowercased — `IMG_1234.CR2` and
|
||||
/// `IMG_1234.JPG` share `img_1234` — with the camera's own part-of-a-shot
|
||||
/// marker removed, so the two files a raw capture writes share a stem too.
|
||||
fn name_stem(name: Option<&str>) -> Option<String> {
|
||||
let name = name?;
|
||||
let stem = match name.rsplit_once('.') {
|
||||
Some((stem, _)) if !stem.is_empty() => stem,
|
||||
_ => name,
|
||||
};
|
||||
let stem = stem.to_ascii_lowercase();
|
||||
Some(strip_shot_marker(&stem).to_string())
|
||||
}
|
||||
|
||||
/// Drop a `.RAW-NN.ROLE` marker from a stem.
|
||||
///
|
||||
/// Pixel stores one raw capture as `PXL_20260919_000625670.RAW-01.COVER.jpg`
|
||||
/// and `PXL_20260919_000625670.RAW-02.ORIGINAL.dng`: one shot, two files, two
|
||||
/// stems that differ in exactly the part naming which file of the shot this is.
|
||||
/// Cutting the marker off is what lets rule 2 see the pair. The check is narrow
|
||||
/// on purpose — digits, then a single role word — so an ordinary name that
|
||||
/// happens to contain `.raw-` keeps its stem.
|
||||
fn strip_shot_marker(stem: &str) -> &str {
|
||||
let Some(at) = stem.rfind(".raw-") else {
|
||||
return stem;
|
||||
};
|
||||
let rest = &stem[at + ".raw-".len()..];
|
||||
let Some((index, role)) = rest.split_once('.') else {
|
||||
return stem;
|
||||
};
|
||||
let marked = !index.is_empty()
|
||||
&& index.bytes().all(|b| b.is_ascii_digit())
|
||||
&& !role.is_empty()
|
||||
&& role.bytes().all(|b| b.is_ascii_alphabetic());
|
||||
if marked { &stem[..at] } else { stem }
|
||||
}
|
||||
|
||||
/// One entry of the grouping pass: what rule 2 needs, plus what choosing a
|
||||
/// representative needs.
|
||||
struct Grouped {
|
||||
uid: String,
|
||||
kind: crate::control::PhotoKind,
|
||||
main_uid: Option<String>,
|
||||
day: Option<String>,
|
||||
stem: Option<String>,
|
||||
}
|
||||
|
||||
/// Which group each photo belongs to, and which member of it the gallery shows.
|
||||
///
|
||||
/// Precedence, as agreed:
|
||||
///
|
||||
/// 1. The server's own relation — a photo with a `main_photo_uid` joins that
|
||||
/// photo's group. The server knows about live photos and bursts; we do not
|
||||
/// have to guess at them.
|
||||
/// 2. Otherwise, same capture day **and** same file-name stem **and** one of the
|
||||
/// two is a camera raw while the other is not. That is the RAW+JPEG pair a
|
||||
/// camera writes, and the differing-kind clause keeps two JPEGs that happen
|
||||
/// to share a name apart.
|
||||
/// 3. Otherwise the photo is its own group.
|
||||
///
|
||||
/// The representative is the non-raw member when the group has one — a JPEG
|
||||
/// decodes in milliseconds and is what the person expects to see — and the
|
||||
/// earliest member in server order otherwise.
|
||||
fn group_photos(entries: &[Grouped]) -> Vec<String> {
|
||||
let mut parent: Vec<usize> = (0..entries.len()).collect();
|
||||
fn find(parent: &mut [usize], mut i: usize) -> usize {
|
||||
while parent[i] != i {
|
||||
parent[i] = parent[parent[i]];
|
||||
i = parent[i];
|
||||
}
|
||||
i
|
||||
}
|
||||
let union = |parent: &mut [usize], a: usize, b: usize| {
|
||||
let (a, b) = (find(parent, a), find(parent, b));
|
||||
if a != b {
|
||||
parent[b] = a;
|
||||
}
|
||||
};
|
||||
|
||||
let index: HashMap<&str, usize> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, e)| (e.uid.as_str(), i))
|
||||
.collect();
|
||||
for (i, entry) in entries.iter().enumerate() {
|
||||
if let Some(main) = entry.main_uid.as_deref()
|
||||
&& let Some(&j) = index.get(main)
|
||||
{
|
||||
union(&mut parent, j, i);
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 2 needs only one pass: everything sharing a day and a stem lands in
|
||||
// the same bucket, and the raw / non-raw split is checked pairwise there.
|
||||
let mut by_name: HashMap<(&str, &str), Vec<usize>> = HashMap::new();
|
||||
for (i, entry) in entries.iter().enumerate() {
|
||||
if let (Some(day), Some(stem)) = (entry.day.as_deref(), entry.stem.as_deref()) {
|
||||
by_name.entry((day, stem)).or_default().push(i);
|
||||
}
|
||||
}
|
||||
let is_raw = |i: usize| entries[i].kind == crate::control::PhotoKind::Raw;
|
||||
for bucket in by_name.values() {
|
||||
for (n, &i) in bucket.iter().enumerate() {
|
||||
for &j in &bucket[n + 1..] {
|
||||
if is_raw(i) != is_raw(j) {
|
||||
union(&mut parent, i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Representative per root, then the key every member carries.
|
||||
let mut representative: HashMap<usize, usize> = HashMap::new();
|
||||
for i in 0..entries.len() {
|
||||
let root = find(&mut parent, i);
|
||||
match representative.get(&root).copied() {
|
||||
Some(current) if !is_raw(current) || is_raw(i) => {}
|
||||
_ => {
|
||||
representative.insert(root, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
(0..entries.len())
|
||||
.map(|i| {
|
||||
let root = find(&mut parent, i);
|
||||
entries[representative[&root]].uid.clone()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The columns every photo read selects, with the group aggregates joined in.
|
||||
/// `g.n` is how many photos the group holds and `g.raw` whether any of them is a
|
||||
/// camera raw file — one query rather than a second lookup per tile.
|
||||
///
|
||||
/// `COALESCE(group_key, uid)` is what makes a row written before schema v30 its
|
||||
/// own group: the column is `NULL` until the next refresh computes it, and a
|
||||
/// `NULL` would otherwise match nothing and drop the photo off the page.
|
||||
const PHOTO_SELECT: &str = "SELECT p.uid, p.capture_time, p.name, p.ratio, p.thumb_state, \
|
||||
p.kind, p.favorite, g.n, g.raw \
|
||||
FROM photos p JOIN (SELECT COALESCE(group_key, uid) AS k, COUNT(*) AS n, \
|
||||
MAX(kind = 2) AS raw FROM photos GROUP BY k) g ON g.k = COALESCE(p.group_key, p.uid)";
|
||||
|
||||
/// Read one row of [`PHOTO_SELECT`].
|
||||
fn stored_photo(r: &rusqlite::Row<'_>) -> rusqlite::Result<StoredPhoto> {
|
||||
Ok(StoredPhoto {
|
||||
uid: r.get(0)?,
|
||||
capture_time: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
ratio: r.get(3)?,
|
||||
thumb_state: r.get(4)?,
|
||||
kind: crate::control::PhotoKind::from_i64(r.get(5)?),
|
||||
favorite: r.get::<_, i64>(6)? != 0,
|
||||
group_size: r.get::<_, i64>(7)?.max(1) as usize,
|
||||
has_raw: r.get::<_, i64>(8)? != 0,
|
||||
})
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Replace the timeline wholesale. Each item is
|
||||
/// `(uid, capture_time, name, media_type, favorite)`; `favorite` is `None`
|
||||
/// when the refresh could not resolve that photo's node, in which case the
|
||||
/// flag already stored is kept rather than silently cleared.
|
||||
/// Replace the timeline wholesale. A row whose `resolved_at` is `Some` was
|
||||
/// read from the server in this pass, so its fields are taken as they are —
|
||||
/// a `None` there means the server no longer reports that value. A row whose
|
||||
/// `resolved_at` is `None` was skipped or failed to resolve, and keeps
|
||||
/// whatever is already stored rather than having it silently cleared.
|
||||
pub fn photos_replace(&self, items: &[TimelineRow]) -> Result<()> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
// `media_type` is learned-and-kept like the ratio and thumb verdict: the
|
||||
// timeline DTO carries only the uid and capture time, so the daemon may
|
||||
// not know a photo's media type yet when it replaces the timeline. Keep
|
||||
// any previously learned value so the Photos/Videos/Raw split survives a
|
||||
// refresh instead of collapsing back to name-extension guesses.
|
||||
let learned: HashMap<String, (Option<f64>, i64, Option<String>, bool)> = {
|
||||
let mut stmt =
|
||||
tx.prepare("SELECT uid, ratio, thumb_state, media_type, favorite FROM photos")?;
|
||||
// `name` and `media_type` are learned-and-kept like the ratio and thumb
|
||||
// verdict: the timeline DTO carries only the uid and capture time, so the
|
||||
// daemon may not know them when it replaces the timeline — because the
|
||||
// refresh skipped an already-resolved photo, or because resolving it
|
||||
// failed. Keep any previously learned value so the tab split and the
|
||||
// RAW+JPEG grouping survive a refresh instead of collapsing to nothing:
|
||||
// both are derived from the name. The photo relation is kept for the same
|
||||
// reason — losing it would break a group up until the next resolve.
|
||||
let learned: HashMap<String, Learned> = {
|
||||
let mut stmt = tx.prepare(
|
||||
"SELECT uid, name, ratio, thumb_state, media_type, favorite, content_hash, \
|
||||
main_uid, resolved_at FROM photos",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
(r.get(1)?, r.get(2)?, r.get(3)?, r.get::<_, i64>(4)? != 0),
|
||||
Learned {
|
||||
name: r.get(1)?,
|
||||
ratio: r.get(2)?,
|
||||
thumb_state: r.get(3)?,
|
||||
media_type: r.get(4)?,
|
||||
favorite: r.get::<_, i64>(5)? != 0,
|
||||
content_hash: r.get(6)?,
|
||||
main_uid: r.get(7)?,
|
||||
resolved_at: r.get(8)?,
|
||||
},
|
||||
))
|
||||
})?;
|
||||
rows.collect::<rusqlite::Result<_>>()?
|
||||
};
|
||||
|
||||
// Everything the insert and the grouping need, resolved once per photo.
|
||||
let resolved: Vec<(&TimelineRow, Learned, crate::control::PhotoKind)> = items
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let mut carried = learned.get(&row.uid).cloned().unwrap_or(Learned {
|
||||
thumb_state: THUMB_UNKNOWN,
|
||||
..Learned::default()
|
||||
});
|
||||
if row.resolved_at.is_some() {
|
||||
// This pass read the node itself, so what it says is the
|
||||
// truth — including what it no longer says. A photo unlinked
|
||||
// from its main on another device loses its `main_uid` here,
|
||||
// which a carry could never express.
|
||||
carried.name = row.name.clone();
|
||||
carried.media_type = row.media_type.clone();
|
||||
carried.favorite = row.favorite.unwrap_or(false);
|
||||
carried.content_hash = row.content_hash.clone();
|
||||
carried.main_uid = row.main_uid.clone();
|
||||
} else {
|
||||
carried.name = row.name.clone().or(carried.name);
|
||||
carried.media_type = row.media_type.clone().or(carried.media_type);
|
||||
carried.favorite = row.favorite.unwrap_or(carried.favorite);
|
||||
carried.content_hash = row.content_hash.clone().or(carried.content_hash);
|
||||
carried.main_uid = row.main_uid.clone().or(carried.main_uid);
|
||||
}
|
||||
carried.resolved_at = row.resolved_at.or(carried.resolved_at);
|
||||
// The tab this photo lands in is derived here, once, so a page or
|
||||
// count query is a plain indexed `WHERE kind = ?` rather than a
|
||||
// reclassification of every row.
|
||||
let kind = crate::control::PhotoKind::classify(
|
||||
carried.name.as_deref(),
|
||||
carried.media_type.as_deref(),
|
||||
);
|
||||
(row, carried, kind)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let entries: Vec<Grouped> = resolved
|
||||
.iter()
|
||||
.map(|(row, carried, kind)| Grouped {
|
||||
uid: row.uid.clone(),
|
||||
kind: *kind,
|
||||
main_uid: carried.main_uid.clone(),
|
||||
day: capture_day(&tx, row.capture_time),
|
||||
stem: name_stem(carried.name.as_deref()),
|
||||
})
|
||||
.collect();
|
||||
let group_keys = group_photos(&entries);
|
||||
|
||||
tx.execute("DELETE FROM photos", [])?;
|
||||
{
|
||||
let mut stmt = tx.prepare(
|
||||
"INSERT INTO photos
|
||||
(uid, capture_time, name, ratio, thumb_state, seq, media_type, kind, favorite)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
(uid, capture_time, name, ratio, thumb_state, seq, media_type, kind, favorite,
|
||||
content_hash, main_uid, group_key, resolved_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
|
||||
)?;
|
||||
for (seq, (uid, capture_time, name, media_type, favorite)) in items.iter().enumerate() {
|
||||
let (ratio, thumb_state, learned_media, learned_favorite) = learned
|
||||
.get(uid)
|
||||
.cloned()
|
||||
.unwrap_or((None, THUMB_UNKNOWN, None, false));
|
||||
let media_type = media_type.clone().or(learned_media);
|
||||
let favorite = favorite.unwrap_or(learned_favorite);
|
||||
// The tab this photo lands in is derived here, once, so a page or
|
||||
// count query is a plain indexed `WHERE kind = ?` rather than a
|
||||
// reclassification of every row.
|
||||
let kind =
|
||||
crate::control::PhotoKind::classify(name.as_deref(), media_type.as_deref());
|
||||
for (seq, ((row, carried, kind), group_key)) in
|
||||
resolved.iter().zip(group_keys.iter()).enumerate()
|
||||
{
|
||||
stmt.execute(params![
|
||||
uid,
|
||||
capture_time,
|
||||
name,
|
||||
ratio,
|
||||
thumb_state,
|
||||
row.uid,
|
||||
row.capture_time,
|
||||
carried.name,
|
||||
carried.ratio,
|
||||
carried.thumb_state,
|
||||
seq as i64,
|
||||
media_type,
|
||||
carried.media_type,
|
||||
kind.as_i64(),
|
||||
favorite as i64,
|
||||
carried.favorite as i64,
|
||||
carried.content_hash,
|
||||
carried.main_uid,
|
||||
group_key,
|
||||
carried.resolved_at,
|
||||
])?;
|
||||
}
|
||||
}
|
||||
@@ -102,11 +382,84 @@ impl Db {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The uids whose node metadata is already resolved, so a refresh can skip
|
||||
/// reading their nodes again. Everything else — a photo never seen before, or
|
||||
/// one a remote event marked stale — is resolved. That is what keeps a
|
||||
/// 12k-photo library off 63 round-trips of `enumerate_nodes` every refresh.
|
||||
pub fn photos_resolved(&self) -> Result<Vec<String>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare("SELECT uid FROM photos WHERE resolved_at IS NOT NULL")?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
|
||||
let mut uids = Vec::new();
|
||||
for row in rows {
|
||||
uids.push(row?);
|
||||
}
|
||||
Ok(uids)
|
||||
}
|
||||
|
||||
/// Mark photos as needing a fresh metadata resolve, and report how many rows
|
||||
/// that matched. This is how a remote change is recorded: the event feed says
|
||||
/// *which* node changed but never *what* changed about it, so the uid is
|
||||
/// written down and the next refresh reads the node.
|
||||
pub fn photos_mark_unresolved(&self, uids: &[String]) -> Result<usize> {
|
||||
if uids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let mut marked = 0;
|
||||
{
|
||||
let mut stmt = tx.prepare("UPDATE photos SET resolved_at = NULL WHERE uid = ?1")?;
|
||||
for uid in uids {
|
||||
marked += stmt.execute(params![uid])?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(marked)
|
||||
}
|
||||
|
||||
/// The same for the whole timeline: every photo is resolved again on the next
|
||||
/// refresh. For a continuity loss, where the daemon cannot know what it
|
||||
/// missed, and for an explicit `refresh photos --full`.
|
||||
pub fn photos_mark_all_unresolved(&self) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
Ok(conn.execute("UPDATE photos SET resolved_at = NULL", [])?)
|
||||
}
|
||||
|
||||
/// Forget photos that have been trashed, so the gallery does not show them
|
||||
/// until the next timeline refresh would have caught up.
|
||||
///
|
||||
/// Album membership goes with them: a trashed photo is gone from every album
|
||||
/// that held it, and leaving the rows behind would keep it on the album
|
||||
/// pages and in the album covers.
|
||||
pub fn photos_delete(&self, uids: &[String]) -> Result<usize> {
|
||||
if uids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let mut removed = 0;
|
||||
{
|
||||
let mut photos = tx.prepare("DELETE FROM photos WHERE uid = ?1")?;
|
||||
let mut albums = tx.prepare("DELETE FROM album_photos WHERE uid = ?1")?;
|
||||
for uid in uids {
|
||||
removed += photos.execute([uid])?;
|
||||
albums.execute([uid])?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
/// One page of the persisted timeline, newest first. `kind`, when set,
|
||||
/// restricts the page to one tab (Photos / Videos / Raw); `range`, when set,
|
||||
/// restricts it to a `[from, to)` capture-time window (epoch seconds) — the
|
||||
/// date scrubber's jump. `offset` is relative to whatever the filters leave.
|
||||
/// `favorites` restricts the page to favourited photos.
|
||||
///
|
||||
/// The page holds one row per *group*: the RAW and the JPEG of one shot are
|
||||
/// one tile. The Raw tab is the exception and lists every raw file, because
|
||||
/// that tab exists to answer "which raws do I have".
|
||||
pub fn photos_page(
|
||||
&self,
|
||||
offset: usize,
|
||||
@@ -118,23 +471,24 @@ impl Db {
|
||||
let conn = self.read();
|
||||
// Built up so any combination of the optional filters is one indexed
|
||||
// query rather than a statement per case.
|
||||
let mut sql = String::from(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind, favorite FROM photos",
|
||||
);
|
||||
let mut sql = String::from(PHOTO_SELECT);
|
||||
let mut binds: Vec<i64> = Vec::new();
|
||||
let mut conds: Vec<String> = Vec::new();
|
||||
if kind != Some(crate::control::PhotoKind::Raw) {
|
||||
conds.push("p.uid = COALESCE(p.group_key, p.uid)".to_string());
|
||||
}
|
||||
if let Some(k) = kind {
|
||||
binds.push(k.as_i64());
|
||||
conds.push(format!("kind = ?{}", binds.len()));
|
||||
conds.push(format!("p.kind = ?{}", binds.len()));
|
||||
}
|
||||
if let Some((from, to)) = range {
|
||||
binds.push(from);
|
||||
conds.push(format!("capture_time >= ?{}", binds.len()));
|
||||
conds.push(format!("p.capture_time >= ?{}", binds.len()));
|
||||
binds.push(to);
|
||||
conds.push(format!("capture_time < ?{}", binds.len()));
|
||||
conds.push(format!("p.capture_time < ?{}", binds.len()));
|
||||
}
|
||||
if favorites {
|
||||
conds.push("favorite = 1".to_string());
|
||||
conds.push("p.favorite = 1".to_string());
|
||||
}
|
||||
if !conds.is_empty() {
|
||||
sql.push_str(" WHERE ");
|
||||
@@ -145,47 +499,41 @@ impl Db {
|
||||
binds.push(offset as i64);
|
||||
let offset_pos = binds.len();
|
||||
sql.push_str(&format!(
|
||||
" ORDER BY seq LIMIT ?{limit_pos} OFFSET ?{offset_pos}"
|
||||
" ORDER BY p.seq LIMIT ?{limit_pos} OFFSET ?{offset_pos}"
|
||||
));
|
||||
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params_from_iter(binds), |r| {
|
||||
Ok(StoredPhoto {
|
||||
uid: r.get(0)?,
|
||||
capture_time: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
ratio: r.get(3)?,
|
||||
thumb_state: r.get(4)?,
|
||||
kind: crate::control::PhotoKind::from_i64(r.get(5)?),
|
||||
favorite: r.get::<_, i64>(6)? != 0,
|
||||
})
|
||||
})?
|
||||
.query_map(rusqlite::params_from_iter(binds), stored_photo)?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// The whole persisted timeline, in stored order. Only for whole-library
|
||||
/// passes (the photo re-date) — the gallery pages with
|
||||
/// [`Db::photos_page`].
|
||||
/// Every photo of one photo's group, in server order — the RAW and the JPEG
|
||||
/// of a shot, or just the photo itself when it stands alone. The lightbox
|
||||
/// switches between these, and deleting a tile trashes all of them.
|
||||
pub fn photos_group(&self, uid: &str) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.read();
|
||||
let sql = format!(
|
||||
"{PHOTO_SELECT} WHERE COALESCE(p.group_key, p.uid) = \
|
||||
(SELECT COALESCE(group_key, uid) FROM photos WHERE uid = ?1) ORDER BY p.seq"
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(params![uid], stored_photo)?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// The whole persisted timeline, in stored order — every file, not one row
|
||||
/// per group. Only for whole-library passes (the photo re-date); the gallery
|
||||
/// pages with [`Db::photos_page`].
|
||||
pub fn photos_all(&self) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind, favorite \
|
||||
FROM photos ORDER BY seq",
|
||||
)?;
|
||||
let sql = format!("{PHOTO_SELECT} ORDER BY p.seq");
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| {
|
||||
Ok(StoredPhoto {
|
||||
uid: r.get(0)?,
|
||||
capture_time: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
ratio: r.get(3)?,
|
||||
thumb_state: r.get(4)?,
|
||||
kind: crate::control::PhotoKind::from_i64(r.get(5)?),
|
||||
favorite: r.get::<_, i64>(6)? != 0,
|
||||
})
|
||||
})?
|
||||
.query_map([], stored_photo)?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
@@ -200,9 +548,15 @@ impl Db {
|
||||
kind: Option<crate::control::PhotoKind>,
|
||||
) -> Result<Vec<(i32, i32, usize)>> {
|
||||
let conn = self.read();
|
||||
// The scrubber has to agree with the page it scrubs, so it counts what
|
||||
// the page shows: groups everywhere except on the Raw tab.
|
||||
let (filter, binds): (&str, Vec<i64>) = match kind {
|
||||
Some(k) => (" WHERE kind = ?1", vec![k.as_i64()]),
|
||||
None => ("", Vec::new()),
|
||||
Some(crate::control::PhotoKind::Raw) => (" WHERE kind = 2", Vec::new()),
|
||||
Some(k) => (
|
||||
" WHERE uid = COALESCE(group_key, uid) AND kind = ?1",
|
||||
vec![k.as_i64()],
|
||||
),
|
||||
None => (" WHERE uid = COALESCE(group_key, uid)", Vec::new()),
|
||||
};
|
||||
let sql = format!(
|
||||
"SELECT CAST(strftime('%Y', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS y, \
|
||||
@@ -227,7 +581,13 @@ impl Db {
|
||||
pub fn photos_counts(&self) -> Result<(usize, usize, usize)> {
|
||||
use crate::control::PhotoKind;
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare("SELECT kind, COUNT(*) FROM photos GROUP BY kind")?;
|
||||
// Photos and Videos count groups — one shot is one entry, however many
|
||||
// files it was stored as. Raw counts files, because the Raw tab lists
|
||||
// them individually.
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT kind, COUNT(*) FROM photos \
|
||||
WHERE uid = COALESCE(group_key, uid) OR kind = 2 GROUP BY kind",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?)))?;
|
||||
let (mut photos, mut videos, mut raw) = (0usize, 0usize, 0usize);
|
||||
for row in rows {
|
||||
@@ -250,21 +610,9 @@ impl Db {
|
||||
}
|
||||
let conn = self.read();
|
||||
let placeholders = vec!["?"; uids.len()].join(",");
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind, favorite FROM photos
|
||||
WHERE uid IN ({placeholders})"
|
||||
))?;
|
||||
let rows = stmt.query_map(rusqlite::params_from_iter(uids), |r| {
|
||||
Ok(StoredPhoto {
|
||||
uid: r.get(0)?,
|
||||
capture_time: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
ratio: r.get(3)?,
|
||||
thumb_state: r.get(4)?,
|
||||
kind: crate::control::PhotoKind::from_i64(r.get(5)?),
|
||||
favorite: r.get::<_, i64>(6)? != 0,
|
||||
})
|
||||
})?;
|
||||
let sql = format!("{PHOTO_SELECT} WHERE p.uid IN ({placeholders})");
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt.query_map(rusqlite::params_from_iter(uids), stored_photo)?;
|
||||
let mut photos = Vec::new();
|
||||
for row in rows {
|
||||
photos.push(row?);
|
||||
|
||||
@@ -113,9 +113,13 @@ fn node_from(parent: serde_json::Value, link: &str, name: &str, kind: serde_json
|
||||
fn photos_replace_keeps_what_was_learned_and_drops_what_left() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
("p1".into(), 300, None, Some("video/mp4".into()), Some(true)),
|
||||
("p2".into(), 200, None, None, None),
|
||||
("p3".into(), 100, None, None, None),
|
||||
TimelineRow {
|
||||
media_type: Some("video/mp4".into()),
|
||||
favorite: Some(true),
|
||||
..TimelineRow::new("p1", 300)
|
||||
},
|
||||
TimelineRow::new("p2", 200),
|
||||
TimelineRow::new("p3", 100),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -126,9 +130,13 @@ fn photos_replace_keeps_what_was_learned_and_drops_what_left() {
|
||||
// The next refresh brings a new photo, keeps p1 and p2, and loses p3.
|
||||
// p1's media type arrives as `None` this time and must not be forgotten.
|
||||
db.photos_replace(&[
|
||||
("p0".into(), 400, Some("new.jpg".into()), None, Some(false)),
|
||||
("p1".into(), 300, None, None, None),
|
||||
("p2".into(), 200, None, None, None),
|
||||
TimelineRow {
|
||||
name: Some("new.jpg".into()),
|
||||
favorite: Some(false),
|
||||
..TimelineRow::new("p0", 400)
|
||||
},
|
||||
TimelineRow::new("p1", 300),
|
||||
TimelineRow::new("p2", 200),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -200,8 +208,14 @@ fn photos_replace_keeps_what_was_learned_and_drops_what_left() {
|
||||
fn favorites_are_remembered_across_refreshes_and_filter_a_page() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
("p1".into(), 300, None, None, Some(true)),
|
||||
("p2".into(), 200, None, None, Some(false)),
|
||||
TimelineRow {
|
||||
favorite: Some(true),
|
||||
..TimelineRow::new("p1", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
favorite: Some(false),
|
||||
..TimelineRow::new("p2", 200)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
let favorites = db.photos_page(0, 10, None, None, true).unwrap();
|
||||
@@ -215,8 +229,11 @@ fn favorites_are_remembered_across_refreshes_and_filter_a_page() {
|
||||
// (`None`), and loses to one that could.
|
||||
db.photos_set_favorite("p2", true).unwrap();
|
||||
db.photos_replace(&[
|
||||
("p1".into(), 300, None, None, Some(false)),
|
||||
("p2".into(), 200, None, None, None),
|
||||
TimelineRow {
|
||||
favorite: Some(false),
|
||||
..TimelineRow::new("p1", 300)
|
||||
},
|
||||
TimelineRow::new("p2", 200),
|
||||
])
|
||||
.unwrap();
|
||||
let favorites = db.photos_page(0, 10, None, None, true).unwrap();
|
||||
@@ -232,7 +249,7 @@ fn favorites_are_remembered_across_refreshes_and_filter_a_page() {
|
||||
fn photos_page_slices_the_timeline_in_order() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let items: Vec<_> = (0..5)
|
||||
.map(|i| (format!("p{i}"), 500 - i as i64, None, None::<String>, None))
|
||||
.map(|i| TimelineRow::new(format!("p{i}"), 500 - i as i64))
|
||||
.collect();
|
||||
db.photos_replace(&items).unwrap();
|
||||
|
||||
@@ -254,6 +271,7 @@ fn trash_replace_lists_folders_first() {
|
||||
is_dir: false,
|
||||
size: 10,
|
||||
mtime: 1,
|
||||
parent_uid: Some("t2".into()),
|
||||
},
|
||||
StoredTrash {
|
||||
uid: "t2".into(),
|
||||
@@ -261,6 +279,7 @@ fn trash_replace_lists_folders_first() {
|
||||
is_dir: true,
|
||||
size: 0,
|
||||
mtime: 2,
|
||||
parent_uid: None,
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
@@ -271,6 +290,18 @@ fn trash_replace_lists_folders_first() {
|
||||
["Alpha", "zeta.txt"]
|
||||
);
|
||||
|
||||
// The parent links come back with the rows: a restore reads the tree shape
|
||||
// from them.
|
||||
let mut parents = db.trash_parents().unwrap();
|
||||
parents.sort();
|
||||
assert_eq!(
|
||||
parents,
|
||||
[
|
||||
("t1".to_string(), Some("t2".to_string())),
|
||||
("t2".to_string(), None),
|
||||
]
|
||||
);
|
||||
|
||||
// A replace is a replace: emptying the trash on the server empties it here.
|
||||
db.trash_replace(&[]).unwrap();
|
||||
assert!(db.trash_list().unwrap().is_empty());
|
||||
@@ -3018,6 +3049,57 @@ fn a_parked_transient_create_stays_off_the_drain_until_finalized() {
|
||||
assert_eq!(due.blob_path.as_deref(), Some("/staging/blob1"));
|
||||
}
|
||||
|
||||
/// The park sweep's query: parked creates are the rows nothing else in the drain
|
||||
/// can see, so it has to find them by the sentinel and report how long each has
|
||||
/// been parked. Ordinary queued work is not its business.
|
||||
#[test]
|
||||
fn parked_creates_are_listed_with_the_age_the_sweep_judges_them_by() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let parent = uid("root").to_string();
|
||||
let parked = uid("swap").to_string();
|
||||
let due = uid("ordinary").to_string();
|
||||
for (node, name, next_attempt_at, created_at) in [
|
||||
(&parked, ".notes.txt.swp", PARK_UNTIL, 1_000),
|
||||
(&due, "notes.txt", 0, 2_000),
|
||||
] {
|
||||
db.enqueue_op(&PendingOp {
|
||||
id: 0,
|
||||
kind: OP_CREATE.to_string(),
|
||||
uid: node.clone(),
|
||||
parent_uid: Some(parent.clone()),
|
||||
name: Some(name.to_string()),
|
||||
blob_path: Some(format!("/staging/{name}")),
|
||||
meta_json: None,
|
||||
created_at,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let rows = db.parked_create_ops().unwrap();
|
||||
assert_eq!(rows.len(), 1, "only the parked create is swept");
|
||||
assert_eq!(rows[0].uid, parked);
|
||||
assert_eq!(rows[0].created_at, 1_000, "the park's age is readable");
|
||||
assert_eq!(
|
||||
rows[0].blob_path.as_deref(),
|
||||
Some("/staging/.notes.txt.swp")
|
||||
);
|
||||
|
||||
// What the sweep does with an expired park: un-park it, and it drains like
|
||||
// any other create rather than waiting for a rename that is not coming.
|
||||
assert!(db.set_create_hold(&parked, false).unwrap());
|
||||
assert!(
|
||||
db.parked_create_ops().unwrap().is_empty(),
|
||||
"an un-parked create is no longer the sweep's business"
|
||||
);
|
||||
assert_eq!(
|
||||
db.next_due_op(1_000 + PARK_EXPIRY_MS).unwrap().unwrap().uid,
|
||||
parked
|
||||
);
|
||||
}
|
||||
|
||||
/// Backoff still gates: nothing is returned before an op is due.
|
||||
#[test]
|
||||
fn next_due_op_respects_backoff() {
|
||||
@@ -3905,3 +3987,496 @@ fn an_idle_worker_does_not_wait_on_an_op_another_worker_has() {
|
||||
"an op somebody else is draining is not work this worker is waiting for"
|
||||
);
|
||||
}
|
||||
|
||||
/// Trashing a photo from the gallery must not leave it behind on an album page:
|
||||
/// the row goes, its album membership goes with it, and nothing else moves.
|
||||
#[test]
|
||||
fn photos_delete_removes_the_photo_and_its_album_membership() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow::new("p1", 300),
|
||||
TimelineRow::new("p2", 200),
|
||||
TimelineRow::new("p3", 100),
|
||||
])
|
||||
.unwrap();
|
||||
db.albums_replace(&[StoredAlbum {
|
||||
uid: "a1".into(),
|
||||
name: "Trip".into(),
|
||||
photo_count: 3,
|
||||
cover_uid: Some("p1".into()),
|
||||
last_activity: Some(500),
|
||||
shared: false,
|
||||
}])
|
||||
.unwrap();
|
||||
db.album_photos_replace(
|
||||
"a1",
|
||||
&[
|
||||
("p1".into(), 300, None, None),
|
||||
("p2".into(), 200, None, None),
|
||||
("p3".into(), 100, None, None),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(db.photos_delete(&["p2".into()]).unwrap(), 1);
|
||||
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(
|
||||
page.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["p1", "p3"],
|
||||
"only the trashed photo leaves the timeline"
|
||||
);
|
||||
assert_eq!(db.album_photos_count("a1").unwrap(), 2);
|
||||
let album = db.album_photos_page("a1", 0, 10).unwrap();
|
||||
assert_eq!(
|
||||
album.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["p1", "p3"]
|
||||
);
|
||||
|
||||
// A uid we no longer hold is not an error: the event poller and the trash
|
||||
// reply can both report the same deletion.
|
||||
assert_eq!(db.photos_delete(&["p2".into()]).unwrap(), 0);
|
||||
assert_eq!(db.photos_delete(&[]).unwrap(), 0);
|
||||
}
|
||||
|
||||
/// A shot taken as RAW+JPEG is one photo to the person who took it: the grid
|
||||
/// shows the JPEG once, and the Raw tab still lists the raw file.
|
||||
#[test]
|
||||
fn a_raw_and_its_jpeg_are_one_entry_in_the_grid() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.JPG".into()),
|
||||
..TimelineRow::new("jpeg", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("img_1234.cr2".into()),
|
||||
..TimelineRow::new("raw", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("IMG_9999.JPG".into()),
|
||||
..TimelineRow::new("alone", 200)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(
|
||||
page.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["jpeg", "alone"],
|
||||
"the pair shows once, as the member that can be displayed"
|
||||
);
|
||||
assert_eq!(page[0].group_size, 2);
|
||||
assert!(page[0].has_raw, "the tile can say the raw is there");
|
||||
assert_eq!(page[1].group_size, 1);
|
||||
assert!(!page[1].has_raw);
|
||||
|
||||
// The Raw tab answers "which raws do I have", so it lists files.
|
||||
let raws = db
|
||||
.photos_page(0, 10, Some(crate::control::PhotoKind::Raw), None, false)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
raws.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["raw"]
|
||||
);
|
||||
assert_eq!(db.photos_counts().unwrap(), (2, 0, 1));
|
||||
|
||||
// Both members are reachable from either uid: the lightbox switches between
|
||||
// them, and deleting the tile has to trash both.
|
||||
let group = db.photos_group("raw").unwrap();
|
||||
assert_eq!(
|
||||
group.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["jpeg", "raw"]
|
||||
);
|
||||
assert_eq!(db.photos_group("alone").unwrap().len(), 1);
|
||||
}
|
||||
|
||||
/// Pixel names the two files of one raw capture after the role each plays, so
|
||||
/// their stems differ. The shot is still one shot, and the grid shows the cover
|
||||
/// JPEG.
|
||||
#[test]
|
||||
fn a_pixel_raw_capture_groups_with_its_cover_jpeg() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("PXL_20260919_000625670.RAW-01.COVER.jpg".into()),
|
||||
..TimelineRow::new("cover", 1_789_776_385)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("PXL_20260919_000625670.RAW-02.ORIGINAL.dng".into()),
|
||||
..TimelineRow::new("original", 1_789_776_385)
|
||||
},
|
||||
// A name that merely contains the marker's opening is not a marker.
|
||||
TimelineRow {
|
||||
name: Some("scan.raw-notes.jpg".into()),
|
||||
..TimelineRow::new("notes", 1_789_776_385)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("scan.raw-other.dng".into()),
|
||||
..TimelineRow::new("other", 1_789_776_385)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(
|
||||
page.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["cover", "notes", "other"],
|
||||
"the capture shows once, as the cover JPEG"
|
||||
);
|
||||
assert_eq!(page[0].group_size, 2);
|
||||
assert!(page[0].has_raw);
|
||||
assert_eq!(
|
||||
db.photos_group("original")
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|p| p.uid.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["cover", "original"]
|
||||
);
|
||||
}
|
||||
|
||||
/// Two files that merely share a name are not one shot: the same stem on
|
||||
/// different days, or two files of the same kind, stay apart.
|
||||
#[test]
|
||||
fn a_shared_name_alone_does_not_group_photos() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.JPG".into()),
|
||||
..TimelineRow::new("today", 1_700_000_000)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.CR2".into()),
|
||||
..TimelineRow::new("last-year", 1_600_000_000)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.PNG".into()),
|
||||
..TimelineRow::new("same-kind", 1_700_000_000)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(page.len(), 3, "three shots, three tiles");
|
||||
assert!(page.iter().all(|p| p.group_size == 1));
|
||||
}
|
||||
|
||||
/// The server knows about live photos and bursts, so its own relation groups
|
||||
/// files this client would never pair by name.
|
||||
#[test]
|
||||
fn the_servers_photo_relation_groups_a_live_photo() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_0001.HEIC".into()),
|
||||
..TimelineRow::new("main", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("clip.mov".into()),
|
||||
main_uid: Some("main".into()),
|
||||
..TimelineRow::new("related", 299)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(
|
||||
page.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["main"]
|
||||
);
|
||||
assert_eq!(page[0].group_size, 2);
|
||||
assert!(!page[0].has_raw, "a live photo's video is not a raw file");
|
||||
|
||||
// The relation is learned-and-kept: a refresh that could not resolve the
|
||||
// nodes must not break the group up.
|
||||
db.photos_replace(&[
|
||||
TimelineRow::new("main", 300),
|
||||
TimelineRow::new("related", 299),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(db.photos_group("related").unwrap().len(), 2);
|
||||
}
|
||||
|
||||
/// A database written before grouping existed opens, keeps its timeline, and
|
||||
/// shows every photo — a row with no `group_key` is its own group, which is how
|
||||
/// the gallery behaved before the column.
|
||||
#[test]
|
||||
fn migration_v30_leaves_a_v29_timeline_intact() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v29-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.JPG".into()),
|
||||
..TimelineRow::new("jpeg", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.CR2".into()),
|
||||
..TimelineRow::new("raw", 300)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
db.photo_set_thumb("jpeg", THUMB_HAVE, Some(1.5)).unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V29 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"DROP INDEX idx_photos_group;
|
||||
ALTER TABLE photos DROP COLUMN group_key;
|
||||
ALTER TABLE photos DROP COLUMN main_uid;
|
||||
ALTER TABLE photos DROP COLUMN content_hash;
|
||||
UPDATE sync_state SET value = '29' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(
|
||||
page.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
|
||||
["jpeg", "raw"],
|
||||
"no photo goes missing before the next refresh groups them"
|
||||
);
|
||||
assert!(page.iter().all(|p| p.group_size == 1));
|
||||
assert_eq!(page[0].ratio, Some(1.5), "what was learned is still there");
|
||||
|
||||
// The next refresh fills the column in and the pair collapses.
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.JPG".into()),
|
||||
..TimelineRow::new("jpeg", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.CR2".into()),
|
||||
..TimelineRow::new("raw", 300)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(page.len(), 1);
|
||||
assert_eq!(page[0].uid, "jpeg");
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// A refresh that skips an already-resolved photo sends a row carrying only the
|
||||
/// uid and capture time. The name must survive that, because the tab split and
|
||||
/// the grouping are both derived from it.
|
||||
#[test]
|
||||
fn photos_replace_keeps_a_skipped_photos_name() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.JPG".into()),
|
||||
media_type: Some("image/jpeg".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("jpeg", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.CR2".into()),
|
||||
media_type: Some("application/octet-stream".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("raw", 300)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(page.len(), 1, "one shot, two files");
|
||||
assert!(page[0].has_raw);
|
||||
|
||||
// The next refresh resolves neither, because both are already resolved.
|
||||
db.photos_replace(&[TimelineRow::new("jpeg", 300), TimelineRow::new("raw", 300)])
|
||||
.unwrap();
|
||||
let page = db.photos_page(0, 10, None, None, false).unwrap();
|
||||
assert_eq!(page.len(), 1, "the group survives a skipped refresh");
|
||||
assert_eq!(page[0].name.as_deref(), Some("IMG_1234.JPG"));
|
||||
assert!(page[0].has_raw, "the raw file is still a raw file");
|
||||
assert_eq!(page[0].group_size, 2);
|
||||
}
|
||||
|
||||
/// Only the photos a refresh actually read are marked resolved; the rest stay
|
||||
/// eligible, so the next refresh tries them again.
|
||||
#[test]
|
||||
fn photos_replace_marks_only_the_rows_it_resolved() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("ok.jpg".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("ok", 300)
|
||||
},
|
||||
// The chunk holding this one failed.
|
||||
TimelineRow::new("failed", 200),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(db.photos_resolved().unwrap(), ["ok"]);
|
||||
|
||||
// A resolved photo stays resolved across a refresh that skips it.
|
||||
db.photos_replace(&[
|
||||
TimelineRow::new("ok", 300),
|
||||
TimelineRow {
|
||||
name: Some("late.jpg".into()),
|
||||
resolved_at: Some(2),
|
||||
..TimelineRow::new("failed", 200)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
let mut resolved = db.photos_resolved().unwrap();
|
||||
resolved.sort();
|
||||
assert_eq!(resolved, ["failed", "ok"]);
|
||||
}
|
||||
|
||||
/// What a refresh read is the truth, including what it no longer reports: a
|
||||
/// photo unlinked from its main on another device leaves the group.
|
||||
#[test]
|
||||
fn a_resolved_replace_clears_a_relation_the_server_dropped() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_0001.HEIC".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("main", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("clip.mov".into()),
|
||||
main_uid: Some("main".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("related", 299)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(db.photos_group("related").unwrap().len(), 2);
|
||||
|
||||
// Both are read again and the server no longer reports the relation.
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_0001.HEIC".into()),
|
||||
resolved_at: Some(2),
|
||||
..TimelineRow::new("main", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("clip.mov".into()),
|
||||
resolved_at: Some(2),
|
||||
..TimelineRow::new("related", 299)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.photos_group("related").unwrap().len(),
|
||||
1,
|
||||
"an unlinked photo stands on its own"
|
||||
);
|
||||
}
|
||||
|
||||
/// A refresh that did *not* read the nodes must not break the group up, which is
|
||||
/// the older behaviour and still the one a failed resolve gets.
|
||||
#[test]
|
||||
fn an_unresolved_replace_keeps_a_relation() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_0001.HEIC".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("main", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("clip.mov".into()),
|
||||
main_uid: Some("main".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("related", 299)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
db.photos_replace(&[
|
||||
TimelineRow::new("main", 300),
|
||||
TimelineRow::new("related", 299),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(db.photos_group("related").unwrap().len(), 2);
|
||||
}
|
||||
|
||||
/// A remote event names the photo it changed, and that is all it takes to have
|
||||
/// the next refresh read that one node again.
|
||||
#[test]
|
||||
fn marking_a_photo_unresolved_makes_it_eligible_again() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("a.jpg".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("a", 300)
|
||||
},
|
||||
TimelineRow {
|
||||
name: Some("b.jpg".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("b", 200)
|
||||
},
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(db.photos_mark_unresolved(&["a".to_string()]).unwrap(), 1);
|
||||
assert_eq!(db.photos_resolved().unwrap(), ["b"]);
|
||||
// A uid the timeline does not hold changes nothing.
|
||||
assert_eq!(db.photos_mark_unresolved(&["gone".to_string()]).unwrap(), 0);
|
||||
|
||||
assert_eq!(db.photos_mark_all_unresolved().unwrap(), 2);
|
||||
assert!(db.photos_resolved().unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// An existing timeline opens as resolved, so upgrading does not re-read the
|
||||
/// whole library once — except for the photos nothing was ever learned about.
|
||||
#[test]
|
||||
fn migration_v31_treats_an_existing_timeline_as_resolved() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v30-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
db.photos_replace(&[
|
||||
TimelineRow {
|
||||
name: Some("IMG_1234.JPG".into()),
|
||||
resolved_at: Some(1),
|
||||
..TimelineRow::new("named", 300)
|
||||
},
|
||||
// A photo whose resolve had failed: nothing was ever learned.
|
||||
TimelineRow::new("nameless", 200),
|
||||
])
|
||||
.unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V30 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"DROP INDEX idx_photos_unresolved;
|
||||
ALTER TABLE photos DROP COLUMN resolved_at;
|
||||
UPDATE sync_state SET value = '30' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
db.photos_resolved().unwrap(),
|
||||
["named"],
|
||||
"the stored timeline is trusted, a nameless row is not"
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
@@ -11,6 +11,10 @@ pub struct StoredTrash {
|
||||
pub is_dir: bool,
|
||||
pub size: i64,
|
||||
pub mtime: i64,
|
||||
/// The folder this node was trashed from, when it is known. `None` for a
|
||||
/// row written before schema v29, and for a node whose parent the server
|
||||
/// did not report.
|
||||
pub parent_uid: Option<String>,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
@@ -20,7 +24,8 @@ impl Db {
|
||||
tx.execute("DELETE FROM trash", [])?;
|
||||
{
|
||||
let mut stmt = tx.prepare(
|
||||
"INSERT INTO trash (uid, name, is_dir, size, mtime) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
"INSERT INTO trash (uid, name, is_dir, size, mtime, parent_uid) \
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
|
||||
)?;
|
||||
for item in items {
|
||||
stmt.execute(params![
|
||||
@@ -28,7 +33,8 @@ impl Db {
|
||||
item.name,
|
||||
item.is_dir as i64,
|
||||
item.size,
|
||||
item.mtime
|
||||
item.mtime,
|
||||
item.parent_uid
|
||||
])?;
|
||||
}
|
||||
}
|
||||
@@ -41,7 +47,7 @@ impl Db {
|
||||
pub fn trash_list(&self) -> Result<Vec<StoredTrash>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid, name, is_dir, size, mtime FROM trash
|
||||
"SELECT uid, name, is_dir, size, mtime, parent_uid FROM trash
|
||||
ORDER BY is_dir DESC, name COLLATE NOCASE",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
@@ -51,6 +57,7 @@ impl Db {
|
||||
is_dir: r.get::<_, i64>(2)? != 0,
|
||||
size: r.get(3)?,
|
||||
mtime: r.get(4)?,
|
||||
parent_uid: r.get(5)?,
|
||||
})
|
||||
})?;
|
||||
let mut items = Vec::new();
|
||||
@@ -60,5 +67,19 @@ impl Db {
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
/// Every trashed node paired with the folder it was trashed from, for the
|
||||
/// restore's tree walk. Kept separate from [`Db::trash_list`] because the
|
||||
/// restore wants only the links, not the names, sizes or ordering.
|
||||
pub fn trash_parents(&self) -> Result<Vec<(String, Option<String>)>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare("SELECT uid, parent_uid FROM trash")?;
|
||||
let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?;
|
||||
let mut pairs = Vec::new();
|
||||
for row in rows {
|
||||
pairs.push(row?);
|
||||
}
|
||||
Ok(pairs)
|
||||
}
|
||||
|
||||
// ---- device sync (devices.md) -----------------------------------------
|
||||
}
|
||||
@@ -199,6 +199,9 @@ fn to_entry(entry: &ignore::DirEntry) -> Option<LocalEntry> {
|
||||
/// The paths a scan should never descend into: the Drive mountpoint (walking it
|
||||
/// would fault every remote node in through FUSE) plus our own state and cache
|
||||
/// dirs, which hold blobs no user searches for by name.
|
||||
///
|
||||
/// Callers should extend this with [`nested_mount_points`] for each scan root —
|
||||
/// see that function for why `same_file_system(true)` is not enough on its own.
|
||||
pub fn default_excludes(mountpoint: &Path, state_dir: &Path, cache_dir: &Path) -> Vec<PathBuf> {
|
||||
vec![
|
||||
mountpoint.to_path_buf(),
|
||||
@@ -207,6 +210,66 @@ pub fn default_excludes(mountpoint: &Path, state_dir: &Path, cache_dir: &Path) -
|
||||
]
|
||||
}
|
||||
|
||||
/// Every mountpoint strictly below `root`, read from `/proc/self/mounts`.
|
||||
///
|
||||
/// [`scan`] already asks the walker to stay on one filesystem, but that check is
|
||||
/// a *stat of the mountpoint* (`ignore`'s `is_same_file_system`), performed after
|
||||
/// the directory is queued — so the walker has to touch a foreign mount to learn
|
||||
/// it should skip it. On a dead network FUSE mount (an sshfs whose server is
|
||||
/// gone, an unresponsive rclone) that stat blocks in `request_wait_answer` with
|
||||
/// nobody left to answer it, and blocks *uninterruptibly*: the walker thread then
|
||||
/// ignores SIGKILL, so the daemon can never exit, systemd's stop times out, and
|
||||
/// the half-dead process keeps the cache.db lock that the next start needs
|
||||
/// (docs/BUGS.md B90).
|
||||
///
|
||||
/// Excluding the mountpoints by path avoids that stat entirely. It does not
|
||||
/// change what ends up indexed: a nested mount is a different device, so
|
||||
/// `same_file_system(true)` was already going to skip it.
|
||||
pub fn nested_mount_points(root: &Path) -> Vec<PathBuf> {
|
||||
let Ok(mounts) = std::fs::read_to_string("/proc/self/mounts") else {
|
||||
return Vec::new();
|
||||
};
|
||||
parse_nested_mount_points(&mounts, root)
|
||||
}
|
||||
|
||||
/// The mountpoint-field parse behind [`nested_mount_points`], split out so it can
|
||||
/// be tested without a real `/proc`.
|
||||
fn parse_nested_mount_points(mounts: &str, root: &Path) -> Vec<PathBuf> {
|
||||
mounts
|
||||
.lines()
|
||||
.filter_map(|line| line.split_whitespace().nth(1))
|
||||
.map(|field| PathBuf::from(unescape_mount_field(field)))
|
||||
// Strictly below: `root` itself being a mountpoint must not exclude the
|
||||
// whole scan.
|
||||
.filter(|path| path.starts_with(root) && path != root)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Undo the octal escaping `/proc/self/mounts` applies to space, tab, newline and
|
||||
/// backslash in path fields. An un-unescaped `\040` would simply fail to match a
|
||||
/// real path, silently putting a mount with a space in its name back in the walk.
|
||||
fn unescape_mount_field(field: &str) -> String {
|
||||
let mut out = String::with_capacity(field.len());
|
||||
let mut rest = field;
|
||||
while let Some(at) = rest.find('\\') {
|
||||
out.push_str(&rest[..at]);
|
||||
let escape = rest[at + 1..].get(..3);
|
||||
match escape.and_then(|digits| u8::from_str_radix(digits, 8).ok()) {
|
||||
Some(byte) => {
|
||||
out.push(byte as char);
|
||||
rest = &rest[at + 4..];
|
||||
}
|
||||
// Not an octal escape after all — keep the backslash verbatim.
|
||||
None => {
|
||||
out.push('\\');
|
||||
rest = &rest[at + 1..];
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -268,4 +331,42 @@ mod tests {
|
||||
|
||||
std::fs::remove_dir_all(&tmp).unwrap();
|
||||
}
|
||||
|
||||
/// A dead network FUSE mount under the scan root must be skipped by *path*,
|
||||
/// before anything stats it — see `nested_mount_points`. The scan root itself
|
||||
/// being a mountpoint must not exclude the entire walk.
|
||||
#[test]
|
||||
fn nested_mount_points_are_excluded_but_the_root_mount_is_not() {
|
||||
let mounts = "\
|
||||
/dev/sda2 /home ext4 rw 0 0
|
||||
narl@narl.io:/opt /home/narl/remote/narl.io fuse.sshfs rw 0 0
|
||||
google_drive: /home/narl/remote/gdrive fuse.rclone rw 0 0
|
||||
tmpfs /run/user/1000 tmpfs rw 0 0
|
||||
";
|
||||
let nested = parse_nested_mount_points(mounts, Path::new("/home/narl"));
|
||||
assert_eq!(
|
||||
nested,
|
||||
vec![
|
||||
PathBuf::from("/home/narl/remote/narl.io"),
|
||||
PathBuf::from("/home/narl/remote/gdrive"),
|
||||
]
|
||||
);
|
||||
|
||||
// `/home` is an ancestor, not a descendant: excluding it would empty the
|
||||
// scan. `/run/user/1000` is unrelated.
|
||||
assert!(parse_nested_mount_points(mounts, Path::new("/home")).len() == 2);
|
||||
}
|
||||
|
||||
/// `/proc/self/mounts` octal-escapes a space; leaving it escaped would put a
|
||||
/// mount whose path contains one back into the walk.
|
||||
#[test]
|
||||
fn mount_paths_are_unescaped() {
|
||||
let mounts = "srv:/x /home/narl/My\\040Cloud fuse.sshfs rw 0 0\n";
|
||||
assert_eq!(
|
||||
parse_nested_mount_points(mounts, Path::new("/home/narl")),
|
||||
vec![PathBuf::from("/home/narl/My Cloud")]
|
||||
);
|
||||
assert_eq!(unescape_mount_field("/a\\134b"), "/a\\b");
|
||||
assert_eq!(unescape_mount_field("/plain/path"), "/plain/path");
|
||||
}
|
||||
}
|
||||
@@ -45,18 +45,30 @@ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
|
||||
"Thumbs.db",
|
||||
];
|
||||
|
||||
/// Suffixes of files that are, by convention, incomplete or throwaway while they
|
||||
/// wear the name: a download in flight, an editor's swap or backup, an atomic
|
||||
/// write's scratch copy. On an on-demand mount these should never seal a remote
|
||||
/// revision — the app renames the finished file to its real name, and only that
|
||||
/// belongs on Drive. See [`is_transient_name`] and docs/BUGS.md B70.
|
||||
/// Suffixes that mean a file is incomplete *by construction*: only the program
|
||||
/// writing it gives it this name, only while the write is in flight, and the
|
||||
/// finished article always arrives under a different one. On an on-demand mount
|
||||
/// these should never seal a remote revision — the app renames the finished file
|
||||
/// to its real name, and only that belongs on Drive. See [`is_transient_name`]
|
||||
/// and docs/BUGS.md B70.
|
||||
///
|
||||
/// `.tmp`/`.temp` are deliberately **not** here, though they are in
|
||||
/// [`DEFAULT_IGNORE_PATTERNS`]. The two lists answer different questions. Skipping
|
||||
/// a `report.tmp` during a mirror reconcile costs nothing — the file stays on
|
||||
/// disk, and a later pass picks it up if the rules change. Parking one on the
|
||||
/// on-demand mount is irreversible in practice: the only exit is a rename to a
|
||||
/// finished name, and a file that is simply *called* `index.tmp` — a static-site
|
||||
/// template, a tool's output — never gets one. It then never reaches Drive at
|
||||
/// all, silently, while its bytes pin `staging/`. That is what a real Takeout
|
||||
/// tree of `index.tmp`/`preview.tmp` gallery templates did: 43 creates parked
|
||||
/// forever. The generic-scratch case they were added for renames within
|
||||
/// milliseconds, so it never needed the park; the download suffixes below, which
|
||||
/// stay in flight for minutes, are what B70 was actually about.
|
||||
const TRANSIENT_SUFFIXES: &[&str] = &[
|
||||
".crdownload", // Chromium/Brave/Edge
|
||||
".part", // Firefox, wget, aria2c
|
||||
".partial", // some downloaders / older IE
|
||||
".download", // Safari / macOS
|
||||
".tmp", // generic scratch
|
||||
".temp", // generic scratch
|
||||
".swp", // vim swap
|
||||
".swx", // vim swap (secondary)
|
||||
];
|
||||
@@ -319,8 +331,6 @@ mod tests {
|
||||
"movie.mkv.part",
|
||||
"big.iso.partial",
|
||||
"photo.jpg.download",
|
||||
"export.xml.tmp",
|
||||
"notes.TEMP",
|
||||
".notes.txt.swp",
|
||||
"doc.txt~",
|
||||
".~lock.report.odt#",
|
||||
@@ -340,6 +350,10 @@ mod tests {
|
||||
"archive.partly.zip", // ".partly" is not ".part"
|
||||
"template.dotx",
|
||||
"readme",
|
||||
// A file that is *named* `.tmp` and never renamed: parking it would
|
||||
// keep it off Drive for good.
|
||||
"index.tmp",
|
||||
"notes.TEMP",
|
||||
] {
|
||||
assert!(!is_transient_name(name), "{name} should not be transient");
|
||||
}
|
||||
|
||||
@@ -425,6 +425,165 @@ pub(super) async fn run_event_sync(
|
||||
}
|
||||
}
|
||||
|
||||
/// The state key holding the photos volume's event cursor. Separate from the
|
||||
/// Drive cursor because the two volumes have their own event streams, and one
|
||||
/// cursor cannot address both.
|
||||
const PHOTOS_EVENT_CURSOR: &str = "photos_event_cursor";
|
||||
|
||||
/// Poll the photos volume's event stream, so a photo deleted or added on another
|
||||
/// device shows up in the gallery within one poll instead of on the timeline's
|
||||
/// freshness window.
|
||||
///
|
||||
/// Deliberately much smaller than [`run_event_sync`]: the photos volume is not
|
||||
/// part of the FUSE mount, so there is no inode space to converge and nothing to
|
||||
/// invalidate in the kernel. A trash or a delete removes the rows behind the
|
||||
/// gallery; anything else only drops the timeline's freshness stamp, and the
|
||||
/// next read re-enumerates. Getting that wrong costs a stale tile, not a wrong
|
||||
/// file.
|
||||
pub(super) async fn run_photos_event_sync(core: Core) {
|
||||
let photos = core.photos();
|
||||
// The photos volume only exists once the user has used Photos at all, and
|
||||
// the daemon may well start before the network does.
|
||||
let scope = loop {
|
||||
match photos.get_photos_root().await {
|
||||
Ok(Some(root)) => break root.tree_event_scope_id(),
|
||||
Ok(None) => {
|
||||
debug!("no photos volume; photo event sync not started");
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(%error, "reading the photos root failed; retrying");
|
||||
tokio::time::sleep(ONLINE_PROBE_MAX).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut cursor: Option<DriveEventId> = core
|
||||
.db
|
||||
.state_str(PHOTOS_EVENT_CURSOR)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(DriveEventId::from);
|
||||
info!(?cursor, "photo event sync started");
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
let events = match photos.enumerate_events(&scope, cursor.as_ref()).await {
|
||||
Ok(events) => events,
|
||||
Err(error) => {
|
||||
warn!(%error, "photo event poll failed; retrying after interval");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(head) = events.last().map(|event| event.id().clone()) else {
|
||||
continue;
|
||||
};
|
||||
apply_photo_events(&core, &events);
|
||||
// One cursor write per batch, after the batch is applied: a crash in
|
||||
// between replays events whose effects are idempotent.
|
||||
if let Err(error) = core.db.set_state_str(PHOTOS_EVENT_CURSOR, head.as_str()) {
|
||||
warn!(%error, "persisting the photo event cursor failed");
|
||||
continue;
|
||||
}
|
||||
cursor = Some(head);
|
||||
}
|
||||
}
|
||||
|
||||
/// What one batch of photos-volume events asks the gallery's tables to do.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
pub(super) struct PhotoEventPlan {
|
||||
/// Photos that left the volume, by delete or by trash.
|
||||
pub(super) gone: Vec<String>,
|
||||
/// Photos whose node changed, so their stored metadata is no longer trusted.
|
||||
pub(super) touched: Vec<String>,
|
||||
/// Whether the whole timeline has to be read again.
|
||||
pub(super) full: bool,
|
||||
}
|
||||
|
||||
impl PhotoEventPlan {
|
||||
fn is_empty(&self) -> bool {
|
||||
self.gone.is_empty() && self.touched.is_empty() && !self.full
|
||||
}
|
||||
}
|
||||
|
||||
/// Read one batch of photos-volume events. Pure, so the classification is
|
||||
/// testable without a volume behind it.
|
||||
///
|
||||
/// `is_echo` claims an event as this daemon's own write. The feed replays our
|
||||
/// writes back at us and a `NodeUpdated` carries no revision id, so without this
|
||||
/// every favourite the user toggles here would mark its own photo stale and buy
|
||||
/// a resolve of a node we just wrote.
|
||||
fn classify_photo_events(
|
||||
events: &[DriveEvent],
|
||||
mut is_echo: impl FnMut(&NodeUid) -> bool,
|
||||
) -> PhotoEventPlan {
|
||||
let mut plan = PhotoEventPlan::default();
|
||||
for event in events {
|
||||
match event {
|
||||
DriveEvent::NodeDeleted { node_uid, .. } => plan.gone.push(node_uid.to_string()),
|
||||
DriveEvent::NodeUpdated {
|
||||
node_uid,
|
||||
is_trashed,
|
||||
..
|
||||
} => {
|
||||
if *is_trashed {
|
||||
plan.gone.push(node_uid.to_string());
|
||||
} else if !is_echo(node_uid) {
|
||||
// An untrash, a rename, a favourite toggled on the phone: the
|
||||
// event says which node changed but never what about it, so
|
||||
// the uid is marked stale and the next refresh reads it.
|
||||
plan.touched.push(node_uid.to_string());
|
||||
}
|
||||
}
|
||||
// The daemon cannot know what it missed here, so everything it
|
||||
// thinks it knows about every photo is dropped.
|
||||
DriveEvent::ContinuityLost { .. } | DriveEvent::ScopeAccessLost { .. } => {
|
||||
plan.full = true;
|
||||
}
|
||||
DriveEvent::CursorAdvanced { .. } | DriveEvent::SharedWithMeUpdated { .. } => {}
|
||||
}
|
||||
}
|
||||
plan
|
||||
}
|
||||
|
||||
/// Apply one batch of photos-volume events to the gallery's tables.
|
||||
fn apply_photo_events(core: &Core, events: &[DriveEvent]) {
|
||||
let plan = classify_photo_events(events, |uid| core.take_self_change(uid));
|
||||
let PhotoEventPlan {
|
||||
gone,
|
||||
touched,
|
||||
full,
|
||||
} = &plan;
|
||||
if !gone.is_empty() {
|
||||
match core.db.photos_delete(gone) {
|
||||
Ok(removed) if removed > 0 => {
|
||||
debug!(removed, "photos removed by a remote event");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(%error, "removing remotely deleted photos failed"),
|
||||
}
|
||||
}
|
||||
// A stale mark is what makes the next refresh cheap: it reads these photos'
|
||||
// nodes and carries every other photo's metadata as it stands.
|
||||
if !touched.is_empty() {
|
||||
match core.db.photos_mark_unresolved(touched) {
|
||||
Ok(marked) if marked > 0 => debug!(marked, "photos changed by a remote event"),
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(%error, "marking remotely changed photos failed"),
|
||||
}
|
||||
}
|
||||
if *full {
|
||||
match core.db.photos_mark_all_unresolved() {
|
||||
Ok(marked) => debug!(marked, "the photo event stream lost continuity"),
|
||||
Err(error) => warn!(%error, "marking the whole timeline unresolved failed"),
|
||||
}
|
||||
}
|
||||
if !plan.is_empty() {
|
||||
core.invalidate_photos();
|
||||
core.invalidate_albums();
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep the local-file index fresh for the launcher prompt's "This computer"
|
||||
/// results. Rebuilds the index whenever it is older than [`LOCAL_INDEX_TTL`],
|
||||
/// then sleeps; runs on its own thread for the life of the daemon.
|
||||
@@ -487,7 +646,12 @@ fn scan_local_once(
|
||||
}
|
||||
};
|
||||
|
||||
let excludes = localindex::default_excludes(mountpoint, &dirs.state_dir(), &dirs.cache_dir());
|
||||
let mut excludes =
|
||||
localindex::default_excludes(mountpoint, &dirs.state_dir(), &dirs.cache_dir());
|
||||
// Resolved per scan rather than once at startup: mounts come and go, and a
|
||||
// mount that appeared since the daemon started is exactly the one most
|
||||
// likely to be a half-alive network filesystem (see `nested_mount_points`).
|
||||
excludes.extend(localindex::nested_mount_points(&home));
|
||||
indexing.store(true, Ordering::Relaxed);
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -789,4 +953,96 @@ mod tests {
|
||||
assert!(acknowledge_applied_event(&db, &event, applied).is_err());
|
||||
assert!(db.get_event_cursor().unwrap().is_none());
|
||||
}
|
||||
|
||||
/// A photo trashed or deleted elsewhere leaves the gallery; one that only
|
||||
/// changed is marked stale, so the next refresh reads that node and no other.
|
||||
#[test]
|
||||
fn a_photo_event_names_what_it_changed() {
|
||||
let plan = classify_photo_events(
|
||||
&[
|
||||
DriveEvent::NodeDeleted {
|
||||
id: DriveEventId::from("e1"),
|
||||
node_uid: uid("deleted"),
|
||||
parent_node_uid: None,
|
||||
},
|
||||
DriveEvent::NodeUpdated {
|
||||
id: DriveEventId::from("e2"),
|
||||
node_uid: uid("trashed"),
|
||||
parent_node_uid: None,
|
||||
is_trashed: true,
|
||||
is_shared: false,
|
||||
},
|
||||
DriveEvent::NodeUpdated {
|
||||
id: DriveEventId::from("e3"),
|
||||
node_uid: uid("favourited"),
|
||||
parent_node_uid: None,
|
||||
is_trashed: false,
|
||||
is_shared: false,
|
||||
},
|
||||
],
|
||||
|_| false,
|
||||
);
|
||||
assert_eq!(
|
||||
plan,
|
||||
PhotoEventPlan {
|
||||
gone: vec![uid("deleted").to_string(), uid("trashed").to_string()],
|
||||
touched: vec![uid("favourited").to_string()],
|
||||
full: false,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/// The feed replays this daemon's own writes. Applying one would cost a
|
||||
/// resolve of a node whose changed field the daemon just wrote itself.
|
||||
#[test]
|
||||
fn our_own_photo_write_is_not_a_remote_change() {
|
||||
let plan = classify_photo_events(
|
||||
&[DriveEvent::NodeUpdated {
|
||||
id: DriveEventId::from("e1"),
|
||||
node_uid: uid("ours"),
|
||||
parent_node_uid: None,
|
||||
is_trashed: false,
|
||||
is_shared: false,
|
||||
}],
|
||||
|claimed| *claimed == uid("ours"),
|
||||
);
|
||||
assert_eq!(plan, PhotoEventPlan::default());
|
||||
}
|
||||
|
||||
/// A gap in the stream means the daemon cannot know what it missed, so it
|
||||
/// keeps nothing.
|
||||
#[test]
|
||||
fn a_lost_photo_event_stream_asks_for_everything() {
|
||||
for event in [
|
||||
DriveEvent::ContinuityLost {
|
||||
id: DriveEventId::from("e1"),
|
||||
},
|
||||
DriveEvent::ScopeAccessLost {
|
||||
id: DriveEventId::from("e1"),
|
||||
},
|
||||
] {
|
||||
let plan = classify_photo_events(std::slice::from_ref(&event), |_| false);
|
||||
assert!(plan.full, "{event:?}");
|
||||
assert!(plan.gone.is_empty());
|
||||
assert!(plan.touched.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Bookkeeping events change nothing about the gallery, and must not buy a
|
||||
/// refresh of it.
|
||||
#[test]
|
||||
fn photo_bookkeeping_events_ask_for_nothing() {
|
||||
let plan = classify_photo_events(
|
||||
&[
|
||||
DriveEvent::CursorAdvanced {
|
||||
id: DriveEventId::from("e1"),
|
||||
},
|
||||
DriveEvent::SharedWithMeUpdated {
|
||||
id: DriveEventId::from("e2"),
|
||||
},
|
||||
],
|
||||
|_| false,
|
||||
);
|
||||
assert_eq!(plan, PhotoEventPlan::default());
|
||||
}
|
||||
}
|
||||
+172
-23
@@ -18,11 +18,11 @@ use std::time::{Duration, Instant};
|
||||
use pdfs_core::config::AppDirs;
|
||||
use pdfs_core::control::{
|
||||
ActivityKind, PhotoMonth, RefreshScope, Request as CtlRequest, Response as CtlResponse,
|
||||
TransferDirection,
|
||||
TransferDirection, TrashFailure,
|
||||
};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use tracing::{info, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::transfers::CountingReader;
|
||||
use super::{Core, count_noun, human_bytes, human_duration, parse_uid};
|
||||
@@ -67,6 +67,16 @@ impl Drop for ControlHandlerPermit {
|
||||
}
|
||||
}
|
||||
|
||||
/// Handler threads in use, for the diagnostics. At the limit the daemon refuses
|
||||
/// connections, which a front-end cannot tell apart from a dead daemon.
|
||||
pub(crate) fn active_handlers() -> usize {
|
||||
ACTIVE_CONTROL_HANDLERS.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn handler_limit() -> usize {
|
||||
MAX_CONTROL_HANDLERS
|
||||
}
|
||||
|
||||
fn read_request_line(reader: &mut impl BufRead) -> std::io::Result<Option<String>> {
|
||||
let mut bytes = Vec::new();
|
||||
loop {
|
||||
@@ -154,6 +164,50 @@ fn rel_to_mount(mountpoint: &Path, path: &str) -> CoreResult<PathBuf> {
|
||||
|
||||
/// Handle one control-socket connection: read a single JSON request line,
|
||||
/// dispatch it against `core`, and write back a JSON response line.
|
||||
/// Refresh scopes being served right now, one entry per running refresh.
|
||||
///
|
||||
/// A `Vec` because there are three scopes and at most `MAX_CONTROL_HANDLERS`
|
||||
/// handlers, so a set would buy nothing.
|
||||
static REFRESHING: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
/// What makes two refreshes the same work: the scope, and for a directory the
|
||||
/// folder, since refreshing one folder says nothing about another. A full
|
||||
/// photos refresh is not the same work as an ordinary one — it forgets what the
|
||||
/// daemon knows first — so it keys apart.
|
||||
fn scope_key(scope: &RefreshScope) -> String {
|
||||
match scope {
|
||||
RefreshScope::Dir { path } => format!("dir:{path}"),
|
||||
RefreshScope::Trash => "trash".to_string(),
|
||||
RefreshScope::Photos { full: true } => "photos:full".to_string(),
|
||||
RefreshScope::Photos { full: false } => "photos".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds one refresh scope for as long as it is being served. `acquire`
|
||||
/// returns `None` when that scope is already in flight.
|
||||
struct RefreshGuard(String);
|
||||
|
||||
impl RefreshGuard {
|
||||
fn acquire(scope: &RefreshScope) -> Option<Self> {
|
||||
let key = scope_key(scope);
|
||||
let mut running = REFRESHING.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if running.contains(&key) {
|
||||
return None;
|
||||
}
|
||||
running.push(key.clone());
|
||||
Some(Self(key))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RefreshGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut running = REFRESHING.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(at) = running.iter().position(|key| *key == self.0) {
|
||||
running.swap_remove(at);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: UnixStream) {
|
||||
if let Err(e) = stream.set_read_timeout(Some(CONTROL_READ_TIMEOUT)) {
|
||||
warn!(error = %e, "control: could not set request timeout");
|
||||
@@ -178,6 +232,9 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
return;
|
||||
}
|
||||
};
|
||||
// Registered before the request is even parsed, so a handler that wedges
|
||||
// inside the daemon shows up by name in `pdfs diagnostics`.
|
||||
let _inflight = super::diagnostics::InflightGuard::new(super::diagnostics::request_kind(&line));
|
||||
let response = match serde_json::from_str::<CtlRequest>(line.trim()) {
|
||||
Ok(CtlRequest::Status) => {
|
||||
let pins = core.cache.list_pins();
|
||||
@@ -234,29 +291,69 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::Refresh { scope }) => {
|
||||
let result = match &scope {
|
||||
RefreshScope::Dir { path } => route_to_mount(core, mountpoint, path)
|
||||
.and_then(|(core, rel)| core.refresh_dir(&rel)),
|
||||
RefreshScope::Trash => {
|
||||
core.invalidate_trash();
|
||||
Ok(())
|
||||
Ok(CtlRequest::Refresh { scope }) => match RefreshGuard::acquire(&scope) {
|
||||
// The same scope is already being refreshed by another handler.
|
||||
// Running it again cannot make the answer newer, and the duplicate
|
||||
// would spend its whole run contending for the state and database
|
||||
// locks the first one holds — which is how five stacked photo
|
||||
// refreshes sat in flight for twenty-three minutes at mount time.
|
||||
None => {
|
||||
debug!(scope = %scope_key(&scope), "a refresh of this scope is already running");
|
||||
CtlResponse::Ok {
|
||||
message: "refreshing".to_string(),
|
||||
}
|
||||
RefreshScope::Photos => {
|
||||
core.invalidate_photos();
|
||||
// Whatever changed the timeline can just as easily have
|
||||
// changed an album; one scope covers the whole photos view.
|
||||
core.invalidate_albums();
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: "refreshed".to_string(),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Some(_refreshing) => {
|
||||
let result = match &scope {
|
||||
RefreshScope::Dir { path } => route_to_mount(core, mountpoint, path)
|
||||
.and_then(|(core, rel)| core.refresh_dir(&rel)),
|
||||
RefreshScope::Trash => {
|
||||
core.invalidate_trash();
|
||||
Ok(())
|
||||
}
|
||||
RefreshScope::Photos { full } if !full && core.timeline_refreshing() => {
|
||||
// A refresh is already running and will publish exactly the
|
||||
// timeline this request asks for. Doing the work again
|
||||
// cannot make it newer, and the invalidation plus the
|
||||
// album read contend with the running refresh for the
|
||||
// database lock — which is how a handful of stacked photo
|
||||
// refreshes at mount time left five control requests
|
||||
// in flight for twenty-three minutes.
|
||||
debug!("a photos refresh is already running; joining it");
|
||||
Ok(())
|
||||
}
|
||||
RefreshScope::Photos { full } => {
|
||||
core.invalidate_photos();
|
||||
// A full refresh is "forget what you know", not a second code
|
||||
// path: the ordinary refresh then finds every photo stale.
|
||||
if let Some(Err(e)) = full.then(|| core.db.photos_mark_all_unresolved()) {
|
||||
warn!(error = %e, "marking the whole timeline unresolved failed");
|
||||
}
|
||||
// Whatever changed the timeline can just as easily have
|
||||
// changed an album; one scope covers the whole photos view.
|
||||
core.invalidate_albums();
|
||||
// The refresh is started here, not awaited. A library of
|
||||
// twelve thousand photos is re-read in chunks of
|
||||
// `TIMELINE_ENRICH_CHUNK` nodes, which takes minutes —
|
||||
// longer than the client's read bound, so awaiting it
|
||||
// returned `Resource temporarily unavailable` to a caller
|
||||
// whose refresh was in fact running fine (B92). The front
|
||||
// end follows it with `PhotosRefreshStatus`.
|
||||
core.spawn_timeline_refresh();
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: match scope {
|
||||
RefreshScope::Photos { .. } => "refreshing".to_string(),
|
||||
_ => "refreshed".to_string(),
|
||||
},
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(CtlRequest::PhotosTimeline {
|
||||
offset,
|
||||
limit,
|
||||
@@ -276,6 +373,14 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::PhotoGroup { uid }) => match core.photo_group(&uid) {
|
||||
Ok(items) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
counts: None,
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::SetPhotoFavorite { uid, favorite }) => match parse_uid(&uid) {
|
||||
Some(node_uid) => match core.set_photo_favorite(&node_uid, favorite) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
@@ -329,6 +434,19 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
items: core.photo_thumbs(&parsed),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::TrashNodes { uids }) => match Core::parse_uids(&uids) {
|
||||
Ok(parsed) => match core.trash_photos(&parsed) {
|
||||
Ok((trashed, failed)) => CtlResponse::Trashed {
|
||||
trashed,
|
||||
failed: failed
|
||||
.into_iter()
|
||||
.map(|(uid, message)| TrashFailure { uid, message })
|
||||
.collect(),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::FileThumbs { items, generation }) => {
|
||||
match core.file_thumbs(&items, generation) {
|
||||
Some(items) => CtlResponse::Thumbs { items },
|
||||
@@ -357,6 +475,9 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Ok(CtlRequest::ThumbnailBuildStatus) => CtlResponse::ThumbnailBuild {
|
||||
status: core.thumbnail_build_status(),
|
||||
},
|
||||
Ok(CtlRequest::PhotosRefreshStatus) => CtlResponse::PhotosRefresh {
|
||||
running: core.timeline_refreshing(),
|
||||
},
|
||||
Ok(CtlRequest::OpenPhoto { uid }) => match parse_uid(&uid) {
|
||||
Some(u) => match core.open_photo(&u) {
|
||||
Ok(p) => CtlResponse::FilePath {
|
||||
@@ -657,6 +778,9 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::Diagnostics) => CtlResponse::Diagnostics {
|
||||
diagnostics: core.diagnostics(),
|
||||
},
|
||||
Ok(CtlRequest::ListTrash) => match core.list_trash() {
|
||||
Ok(entries) => CtlResponse::Entries { entries },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
@@ -1314,3 +1438,28 @@ mod request_limit_tests {
|
||||
assert!(rel_to_mount(mountpoint, "/mnt/elsewhere").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Two refreshes of the same scope are one piece of work; two refreshes of
|
||||
/// different scopes are not.
|
||||
#[test]
|
||||
fn a_scope_refreshes_once_at_a_time() {
|
||||
let photos = RefreshScope::Photos { full: false };
|
||||
let held = RefreshGuard::acquire(&photos).expect("first refresh runs");
|
||||
assert!(RefreshGuard::acquire(&photos).is_none());
|
||||
// A full refresh forgets what the daemon knows, so it is other work.
|
||||
assert!(RefreshGuard::acquire(&RefreshScope::Photos { full: true }).is_some());
|
||||
assert!(
|
||||
RefreshGuard::acquire(&RefreshScope::Dir {
|
||||
path: "a".to_string()
|
||||
})
|
||||
.is_some()
|
||||
);
|
||||
|
||||
drop(held);
|
||||
assert!(RefreshGuard::acquire(&photos).is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
//! What the daemon is doing right now, for the case where it has stopped doing
|
||||
//! anything at all.
|
||||
//!
|
||||
//! On 2026-09-18 the daemon hung: it consumed 43 minutes of CPU over 20 hours,
|
||||
//! peaked at 5.2 GiB resident, went silent at 17:10, and only came back after a
|
||||
//! manual `systemctl --user restart`. There was no panic, so nothing in the
|
||||
//! journal said what it had been waiting on — and nothing in the daemon could be
|
||||
//! asked, either. This module is the answer to "what was it stuck on?".
|
||||
//!
|
||||
//! Everything here is built to work *while* the daemon is wedged, which rules
|
||||
//! out the obvious implementations: no lock that a stuck job might hold, no
|
||||
//! database query on the calling thread, no waiting on the worker pool. A
|
||||
//! diagnostic that blocks on the hang it is diagnosing reports nothing at
|
||||
//! exactly the moment it matters.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use pdfs_core::control::{Diagnostics, InflightRequest, WorkerState};
|
||||
|
||||
use super::Core;
|
||||
|
||||
/// When this process started. Read at the first touch, which
|
||||
/// [`super::mount::mount`] makes happen during startup so the uptime is the
|
||||
/// daemon's, not the first diagnostics request's.
|
||||
static STARTED: LazyLock<Instant> = LazyLock::new(Instant::now);
|
||||
|
||||
/// Control requests currently being served, keyed by a monotonic ticket.
|
||||
///
|
||||
/// A `Vec` rather than a map because it holds at most `MAX_CONTROL_HANDLERS`
|
||||
/// entries, and the only operations are push, remove-by-ticket and read-all.
|
||||
static INFLIGHT: Mutex<Vec<(u64, String, Instant)>> = Mutex::new(Vec::new());
|
||||
static NEXT_TICKET: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Start the daemon's uptime clock. Called once, from the mount path.
|
||||
pub(crate) fn start_clock() {
|
||||
LazyLock::force(&STARTED);
|
||||
}
|
||||
|
||||
/// Registers one in-flight control request for as long as it is held.
|
||||
pub(crate) struct InflightGuard(u64);
|
||||
|
||||
impl InflightGuard {
|
||||
pub(crate) fn new(kind: String) -> Self {
|
||||
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
|
||||
INFLIGHT.lock().push((ticket, kind, Instant::now()));
|
||||
Self(ticket)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InflightGuard {
|
||||
fn drop(&mut self) {
|
||||
INFLIGHT.lock().retain(|(ticket, _, _)| *ticket != self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// The variant name of a serialised [`pdfs_core::control::Request`], without
|
||||
/// parsing it into one.
|
||||
///
|
||||
/// serde's external tagging makes this a property of the text: a unit variant is
|
||||
/// the bare string `"Status"`, anything else is an object whose single key is
|
||||
/// the variant name. Reading the name off the line rather than matching on 80
|
||||
/// variants keeps this cheap, and keeps it correct when a variant is added.
|
||||
pub(crate) fn request_kind(line: &str) -> String {
|
||||
let line = line.trim();
|
||||
let name = if let Some(rest) = line.strip_prefix('{') {
|
||||
rest.trim_start()
|
||||
.strip_prefix('"')
|
||||
.and_then(|rest| rest.split('"').next())
|
||||
} else {
|
||||
line.strip_prefix('"')
|
||||
.and_then(|rest| rest.split('"').next())
|
||||
};
|
||||
let name = name.unwrap_or("unknown");
|
||||
// A bare `Refresh` says nothing about which view wedged, and the three
|
||||
// scopes cost wildly different amounts of work: a directory refresh is two
|
||||
// cheap calls, a photos refresh can pull a whole library. The scope is one
|
||||
// more peek into the same line.
|
||||
if name == "Refresh"
|
||||
&& let Some(scope) = tagged_name_after(line, "\"scope\"")
|
||||
{
|
||||
return format!("Refresh({scope})");
|
||||
}
|
||||
name.to_string()
|
||||
}
|
||||
|
||||
/// The variant name serde wrote for the value of `key`, whether that value is a
|
||||
/// bare string (a unit variant) or an object whose single key is the name.
|
||||
fn tagged_name_after(line: &str, key: &str) -> Option<String> {
|
||||
let rest = line.split_once(key)?.1.trim_start();
|
||||
let rest = rest.strip_prefix(':')?.trim_start();
|
||||
let rest = rest.strip_prefix('{').map_or(rest, str::trim_start);
|
||||
rest.strip_prefix('"')?
|
||||
.split('"')
|
||||
.next()
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Resident set size in bytes, or `0` when `/proc` did not answer.
|
||||
///
|
||||
/// From `VmRSS` in `/proc/self/status`, which is already in kB — `statm` would
|
||||
/// need the page size, and this needs no dependency at all.
|
||||
pub(crate) fn rss_bytes() -> u64 {
|
||||
let Ok(status) = std::fs::read_to_string("/proc/self/status") else {
|
||||
return 0;
|
||||
};
|
||||
parse_vm_rss(&status)
|
||||
}
|
||||
|
||||
/// The `VmRSS` line of a `/proc/<pid>/status`, in bytes.
|
||||
fn parse_vm_rss(status: &str) -> u64 {
|
||||
status
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("VmRSS:"))
|
||||
.and_then(|rest| rest.split_whitespace().next())
|
||||
.and_then(|kb| kb.parse::<u64>().ok())
|
||||
.map_or(0, |kb| kb.saturating_mul(1024))
|
||||
}
|
||||
|
||||
impl Core {
|
||||
/// Assemble a [`Diagnostics`] report.
|
||||
///
|
||||
/// The pending-op count is the one field that needs the database, so it is
|
||||
/// read on a throwaway thread with a short deadline: a daemon wedged *on*
|
||||
/// the database must still be able to report everything else about itself.
|
||||
pub(crate) fn diagnostics(&self) -> Diagnostics {
|
||||
let pool = self.workers.snapshot();
|
||||
let now = Instant::now();
|
||||
let mut inflight: Vec<InflightRequest> = INFLIGHT
|
||||
.try_lock()
|
||||
.map(|entries| {
|
||||
entries
|
||||
.iter()
|
||||
.map(|(_, kind, started)| InflightRequest {
|
||||
kind: kind.clone(),
|
||||
age_secs: now.duration_since(*started).as_secs(),
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
inflight.sort_by_key(|request| std::cmp::Reverse(request.age_secs));
|
||||
|
||||
Diagnostics {
|
||||
uptime_secs: STARTED.elapsed().as_secs(),
|
||||
rss_bytes: rss_bytes(),
|
||||
workers: pool
|
||||
.workers
|
||||
.into_iter()
|
||||
.map(|worker| WorkerState {
|
||||
name: worker.name,
|
||||
busy: worker.busy,
|
||||
label: worker.label,
|
||||
age_secs: worker.age_ms / 1000,
|
||||
})
|
||||
.collect(),
|
||||
meta_queued: pool.meta_queued,
|
||||
transfer_queued: pool.transfer_queued,
|
||||
meta_completed: pool.meta_completed,
|
||||
transfer_completed: pool.transfer_completed,
|
||||
inflight,
|
||||
control_handlers: super::control::active_handlers(),
|
||||
control_handler_limit: super::control::handler_limit(),
|
||||
pending_ops: self.pending_ops_quickly(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queued mutations, or `0` if the database did not answer within
|
||||
/// [`PENDING_COUNT_DEADLINE`].
|
||||
fn pending_ops_quickly(&self) -> u64 {
|
||||
let db = self.db.clone();
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
if std::thread::Builder::new()
|
||||
.name("pdfs-diagnostics".into())
|
||||
.spawn(move || {
|
||||
let counts = db.pending_op_counts().unwrap_or_default();
|
||||
let total = counts.uploads.max(0) as u64 + counts.changes.max(0) as u64;
|
||||
let _ = tx.send(total);
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
rx.recv_timeout(PENDING_COUNT_DEADLINE).unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// How long the report waits for the database before reporting without it.
|
||||
const PENDING_COUNT_DEADLINE: std::time::Duration = std::time::Duration::from_millis(500);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The request name is read off the wire format, so it must match both
|
||||
/// shapes serde produces.
|
||||
#[test]
|
||||
fn the_request_kind_comes_off_both_wire_shapes() {
|
||||
assert_eq!(request_kind("\"Status\""), "Status");
|
||||
assert_eq!(request_kind("{\"Pin\":{\"path\":\"a\"}}"), "Pin");
|
||||
assert_eq!(request_kind(" { \"Restore\" : {} } "), "Restore");
|
||||
assert_eq!(request_kind("garbage"), "unknown");
|
||||
}
|
||||
|
||||
/// A stuck `Refresh` is only actionable if the report says which view it
|
||||
/// was refreshing, so the scope rides along with the name.
|
||||
#[test]
|
||||
fn a_refresh_reports_its_scope() {
|
||||
assert_eq!(
|
||||
request_kind("{\"Refresh\":{\"scope\":{\"Photos\":{\"full\":true}}}}"),
|
||||
"Refresh(Photos)"
|
||||
);
|
||||
assert_eq!(
|
||||
request_kind("{\"Refresh\":{\"scope\":{\"Dir\":{\"path\":\"a\"}}}}"),
|
||||
"Refresh(Dir)"
|
||||
);
|
||||
assert_eq!(
|
||||
request_kind("{\"Refresh\":{\"scope\":\"Trash\"}}"),
|
||||
"Refresh(Trash)"
|
||||
);
|
||||
// A shape this does not recognise still names the request.
|
||||
assert_eq!(request_kind("{\"Refresh\":{}}"), "Refresh");
|
||||
}
|
||||
|
||||
/// A guard registers for its lifetime and nothing longer: a handler that
|
||||
/// returns must not leave a permanent "in flight" entry behind.
|
||||
#[test]
|
||||
fn a_guard_registers_only_while_it_lives() {
|
||||
let before = INFLIGHT.lock().len();
|
||||
{
|
||||
let _guard = InflightGuard::new("OpenFile".into());
|
||||
assert_eq!(INFLIGHT.lock().len(), before + 1);
|
||||
}
|
||||
assert_eq!(INFLIGHT.lock().len(), before);
|
||||
}
|
||||
|
||||
/// Whatever else it reports, the resident size has to be a real reading.
|
||||
#[test]
|
||||
fn rss_is_readable() {
|
||||
assert!(rss_bytes() > 0);
|
||||
}
|
||||
|
||||
/// `VmRSS` is in kB, and a status without one must read as unknown rather
|
||||
/// than as a daemon using no memory.
|
||||
#[test]
|
||||
fn vm_rss_is_kilobytes_and_absence_is_zero() {
|
||||
assert_eq!(
|
||||
parse_vm_rss("VmPeak:\t 100 kB\nVmRSS:\t 2048 kB\n"),
|
||||
2048 * 1024
|
||||
);
|
||||
assert_eq!(parse_vm_rss("VmPeak:\t 100 kB\n"), 0);
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,9 @@ use futures::StreamExt;
|
||||
use pdfs_core::batch;
|
||||
use pdfs_core::cache::{Baseline, StagedWrite};
|
||||
use pdfs_core::control::{ActivityKind, TransferDirection};
|
||||
use pdfs_core::db::{OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingOp, RenameMeta};
|
||||
use pdfs_core::db::{
|
||||
OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_EXPIRY_MS, PendingOp, RenameMeta,
|
||||
};
|
||||
use proton_drive_rs::proton_sdk::error::ProtonError;
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use proton_drive_rs::{Node, NodeMoveItem};
|
||||
@@ -299,6 +301,10 @@ impl Core {
|
||||
// Cheap and a no-op when empty.
|
||||
self.cache.flush_touches();
|
||||
self.recover_fsynced_writes();
|
||||
// Parked rows are invisible to `claim_next_due_op`, so
|
||||
// an idle queue is exactly when a park that has outlived
|
||||
// its writer has to be noticed.
|
||||
self.sweep_parked_creates();
|
||||
}
|
||||
// A debounced or backed-off op may be waiting: sleep only
|
||||
// until it becomes due rather than the full idle-poll.
|
||||
@@ -1201,6 +1207,101 @@ impl Core {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether any mount still holds an open handle on `uid`.
|
||||
///
|
||||
/// A transient file that is still open is still being written, however long
|
||||
/// it has been parked, and the rename that finishes it has not happened yet.
|
||||
fn uid_is_open(&self, uid: &NodeUid) -> bool {
|
||||
let mut open = false;
|
||||
self.for_each_state(|st| {
|
||||
if open {
|
||||
return;
|
||||
}
|
||||
open = st
|
||||
.by_uid
|
||||
.get(uid)
|
||||
.and_then(|ino| st.entries.get(ino))
|
||||
.is_some_and(|entry| entry.open_count > 0);
|
||||
});
|
||||
open
|
||||
}
|
||||
|
||||
/// Retire parks that are never going to end (docs/BUGS.md B70).
|
||||
///
|
||||
/// A parked create waits for one event: the rename from the transient name
|
||||
/// to the finished one. Nothing else ever un-parks it, and plenty of writers
|
||||
/// never perform it — an editor unlinks its swap file, a download is
|
||||
/// abandoned, or the file was always going to be called `index.tmp`. Such a
|
||||
/// row stayed at [`PARK_UNTIL`] for the life of the database, pinning the
|
||||
/// only copy of its bytes in `staging/` and counting against the queue.
|
||||
///
|
||||
/// Two outcomes, and neither deletes bytes a user can still see:
|
||||
///
|
||||
/// * The node is gone from the tree — the file was deleted while its create
|
||||
/// was parked, so the create is no longer wanted. The op and its blob go,
|
||||
/// which is what [`Core::discard_queued_ops`] would have done had the
|
||||
/// delete path reached it.
|
||||
/// * The node is still there and has been parked past
|
||||
/// [`PARK_EXPIRY_MS`] with nothing holding it open — the park has outlived
|
||||
/// the writer it was protecting. Un-park it, and the ordinary drain
|
||||
/// uploads it like any other create.
|
||||
///
|
||||
/// An open node is left alone whatever its age: it is still being written.
|
||||
fn sweep_parked_creates(&self) {
|
||||
let parked = match self.db.parked_create_ops() {
|
||||
Ok(parked) if !parked.is_empty() => parked,
|
||||
Ok(_) => return,
|
||||
Err(error) => {
|
||||
warn!(%error, "reading parked creates failed; skipping the park sweep");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let now = now_millis();
|
||||
let (mut released, mut dropped) = (0usize, 0usize);
|
||||
for op in &parked {
|
||||
let Some(uid) = parse_node_uid(&op.uid) else {
|
||||
continue;
|
||||
};
|
||||
let name = op.name.as_deref().unwrap_or("?");
|
||||
let known = match self.db.node_by_uid(&op.uid) {
|
||||
Ok(known) => known.is_some(),
|
||||
// Never infer "deleted" from a failed read: that would discard
|
||||
// the bytes over a transient database error.
|
||||
Err(error) => {
|
||||
debug!(%uid, %error, "cannot tell whether a parked node still exists");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if !known {
|
||||
if let Err(error) = self.discard_queued_ops(&uid) {
|
||||
warn!(%uid, name, ?error, "dropping a deleted transient's parked create failed");
|
||||
continue;
|
||||
}
|
||||
dropped += 1;
|
||||
debug!(%uid, name, "dropped a parked create whose file was deleted");
|
||||
continue;
|
||||
}
|
||||
if park_verdict(op.created_at, now, self.uid_is_open(&uid)) != ParkVerdict::Release {
|
||||
continue;
|
||||
}
|
||||
match self.db.set_create_hold(&op.uid, false) {
|
||||
Ok(true) => {
|
||||
released += 1;
|
||||
info!(%uid, name, parked_ms = now - op.created_at,
|
||||
"un-parked a transient create that was never renamed; uploading it");
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(error) => warn!(%uid, name, %error, "un-parking an expired park failed"),
|
||||
}
|
||||
}
|
||||
if released > 0 || dropped > 0 {
|
||||
info!(released, dropped, "swept parked creates");
|
||||
}
|
||||
if released > 0 {
|
||||
self.wake_drain();
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop `uid`'s pending entry, but only while it still names `blob`.
|
||||
///
|
||||
/// A supersede can replace the entry while the op that owned it is in
|
||||
@@ -1573,6 +1674,24 @@ fn is_own_self_supersede(complete: bool, remote_rev: Option<&str>, own_rev: Opti
|
||||
complete && remote_rev.is_some() && remote_rev == own_rev
|
||||
}
|
||||
|
||||
/// What the park sweep does with one parked create whose node still exists.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum ParkVerdict {
|
||||
/// Leave it parked: the rename it waits for can still come.
|
||||
Keep,
|
||||
/// Un-park it: nothing will rename it now, so the bytes go up as they are.
|
||||
Release,
|
||||
}
|
||||
|
||||
/// Judge one parked create by age alone. An open node is kept whatever its age
|
||||
/// — it is still being written, and the rename is the writer's last step.
|
||||
fn park_verdict(created_at: i64, now: i64, open: bool) -> ParkVerdict {
|
||||
if open || now - created_at < PARK_EXPIRY_MS {
|
||||
return ParkVerdict::Keep;
|
||||
}
|
||||
ParkVerdict::Release
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1598,6 +1717,28 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The park's only ordinary exit is the transient-to-final rename. The
|
||||
/// sweep is what happens when that rename never comes.
|
||||
#[test]
|
||||
fn a_park_expires_only_once_it_is_old_and_nobody_holds_the_file() {
|
||||
let now = PARK_EXPIRY_MS * 10;
|
||||
// Fresh: the writer may still rename it.
|
||||
assert_eq!(
|
||||
park_verdict(now - PARK_EXPIRY_MS + 1, now, false),
|
||||
ParkVerdict::Keep
|
||||
);
|
||||
// Old, but still open: a slow write is not an abandoned one.
|
||||
assert_eq!(
|
||||
park_verdict(now - PARK_EXPIRY_MS, now, true),
|
||||
ParkVerdict::Keep
|
||||
);
|
||||
// Old and closed: no rename is coming.
|
||||
assert_eq!(
|
||||
park_verdict(now - PARK_EXPIRY_MS, now, false),
|
||||
ParkVerdict::Release
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drain_authorities_match_each_operation_kind() {
|
||||
for kind in [OP_REVISION, OP_TRASH] {
|
||||
|
||||
@@ -49,7 +49,7 @@ impl Filesystem for ProtonFs {
|
||||
/// ([`Core::account_quota_cached`] serves a warm one from memory).
|
||||
fn statfs(&self, _req: &Request, _ino: INodeNo, reply: ReplyStatfs) {
|
||||
let core = self.core.clone();
|
||||
self.core.workers.run(Lane::Meta, move || {
|
||||
self.core.workers.run(Lane::Meta, "statfs", move || {
|
||||
let (max_space, used_space) = core.account_quota_cached().unwrap_or((0, 0));
|
||||
let bsize = u64::from(REPORTED_BLKSIZE);
|
||||
let blocks = (max_space.max(0) as u64).div_ceil(bsize);
|
||||
@@ -89,7 +89,7 @@ impl Filesystem for ProtonFs {
|
||||
return;
|
||||
}
|
||||
let fs = self.clone();
|
||||
self.core.workers.run(Lane::Meta, move || {
|
||||
self.core.workers.run(Lane::Meta, "lookup", move || {
|
||||
fs.serve_lookup(parent, &name, reply, true)
|
||||
});
|
||||
}
|
||||
@@ -183,7 +183,7 @@ impl Filesystem for ProtonFs {
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Meta, move || fs.serve_opendir(ino, reply));
|
||||
.run(Lane::Meta, "opendir", move || fs.serve_opendir(ino, reply));
|
||||
}
|
||||
|
||||
fn readdir(
|
||||
@@ -210,9 +210,9 @@ impl Filesystem for ProtonFs {
|
||||
return;
|
||||
}
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Meta, move || fs.serve_readdir(ino, offset, reply));
|
||||
self.core.workers.run(Lane::Meta, "readdir", move || {
|
||||
fs.serve_readdir(ino, offset, reply)
|
||||
});
|
||||
}
|
||||
|
||||
fn releasedir(
|
||||
@@ -237,9 +237,9 @@ impl Filesystem for ProtonFs {
|
||||
return;
|
||||
}
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Transfer, move || fs.serve_open(ino, flags, reply));
|
||||
self.core.workers.run(Lane::Transfer, "open", move || {
|
||||
fs.serve_open(ino, flags, reply)
|
||||
});
|
||||
}
|
||||
|
||||
fn read(
|
||||
@@ -329,7 +329,7 @@ impl Filesystem for ProtonFs {
|
||||
// even read off the FUSE device. It only reads `state`, so moving it
|
||||
// races with nothing. FUSE does not require replies in request order.
|
||||
let core = self.core.clone();
|
||||
self.core.workers.run(Lane::Transfer, move || {
|
||||
self.core.workers.run(Lane::Transfer, "read", move || {
|
||||
match core.read_range(&uid, mtime, fsize, offset, size as u64, cache_blocks) {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Err(e) => reply.error(e),
|
||||
@@ -366,9 +366,9 @@ impl Filesystem for ProtonFs {
|
||||
// one create at a time, with `ls` on the same mount timing out for as
|
||||
// long as the burst lasted. See [`Workers`].
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Meta, move || fs.serve_create(parent, &name, reply));
|
||||
self.core.workers.run(Lane::Meta, "create", move || {
|
||||
fs.serve_create(parent, &name, reply)
|
||||
});
|
||||
}
|
||||
|
||||
fn write(
|
||||
@@ -506,7 +506,7 @@ impl Filesystem for ProtonFs {
|
||||
// the scratch file).
|
||||
let ino = ino.0;
|
||||
let fs = self.clone();
|
||||
self.core.workers.run(Lane::Transfer, move || {
|
||||
self.core.workers.run(Lane::Transfer, "truncate", move || {
|
||||
if let Err(e) = fs.core.queue_truncate(ino, size) {
|
||||
reply.error(e);
|
||||
return;
|
||||
@@ -620,7 +620,7 @@ impl Filesystem for ProtonFs {
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Transfer, move || fs.serve_fsync(fh, reply));
|
||||
.run(Lane::Transfer, "fsync", move || fs.serve_fsync(fh, reply));
|
||||
}
|
||||
|
||||
fn release(
|
||||
@@ -680,7 +680,7 @@ impl Filesystem for ProtonFs {
|
||||
};
|
||||
// `trash_child` is a remote call; see `serve_create`.
|
||||
let fs = self.clone();
|
||||
self.core.workers.run(Lane::Meta, move || {
|
||||
self.core.workers.run(Lane::Meta, "unlink", move || {
|
||||
fs.serve_unlink(parent, &name_str, reply)
|
||||
});
|
||||
}
|
||||
@@ -701,9 +701,9 @@ impl Filesystem for ProtonFs {
|
||||
// Enumerates the folder to apply rmdir's emptiness rule, then trashes
|
||||
// it: two remote calls. See `serve_create`.
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Meta, move || fs.serve_rmdir(parent, &name_str, reply));
|
||||
self.core.workers.run(Lane::Meta, "rmdir", move || {
|
||||
fs.serve_rmdir(parent, &name_str, reply)
|
||||
});
|
||||
}
|
||||
|
||||
fn mkdir(
|
||||
@@ -731,9 +731,9 @@ impl Filesystem for ProtonFs {
|
||||
// remote round trip, and inline it blocks every other request on the
|
||||
// mount until it answers.
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Meta, move || fs.serve_mkdir(parent, &name, reply));
|
||||
self.core.workers.run(Lane::Meta, "mkdir", move || {
|
||||
fs.serve_mkdir(parent, &name, reply)
|
||||
});
|
||||
}
|
||||
|
||||
fn rename(
|
||||
@@ -774,7 +774,7 @@ impl Filesystem for ProtonFs {
|
||||
// move), and the replaced-destination path adds a trash on top. See
|
||||
// `serve_create` for why none of that may run on the dispatch loop.
|
||||
let fs = self.clone();
|
||||
self.core.workers.run(Lane::Meta, move || {
|
||||
self.core.workers.run(Lane::Meta, "rename", move || {
|
||||
fs.serve_rename(parent, &name, newparent, &newname, flags, reply)
|
||||
});
|
||||
}
|
||||
@@ -799,7 +799,7 @@ impl Filesystem for ProtonFs {
|
||||
// network per call (B5).
|
||||
let fs = self.clone();
|
||||
let name = name.to_os_string();
|
||||
self.core.workers.run(Lane::Meta, move || {
|
||||
self.core.workers.run(Lane::Meta, "getxattr", move || {
|
||||
fs.serve_getxattr(ino, &name, size, reply)
|
||||
});
|
||||
}
|
||||
@@ -1099,7 +1099,7 @@ impl ProtonFs {
|
||||
if !off_loop {
|
||||
let fs = self.clone();
|
||||
let name = name.to_string();
|
||||
self.core.workers.run(Lane::Meta, move || {
|
||||
self.core.workers.run(Lane::Meta, "lookup", move || {
|
||||
fs.serve_lookup(parent, &name, reply, true)
|
||||
});
|
||||
return;
|
||||
|
||||
+369
-38
@@ -83,6 +83,7 @@ mod background;
|
||||
mod control;
|
||||
use control::run_control_socket;
|
||||
mod devices;
|
||||
mod diagnostics;
|
||||
mod drain;
|
||||
mod filesystem;
|
||||
pub use filesystem::ProtonFs;
|
||||
@@ -95,14 +96,16 @@ mod revisions;
|
||||
mod sharing;
|
||||
mod shutdown;
|
||||
mod state;
|
||||
mod supervisor;
|
||||
mod sweep;
|
||||
mod sync;
|
||||
mod systemd;
|
||||
mod takeout;
|
||||
mod transfers;
|
||||
mod upload;
|
||||
mod r#virtual;
|
||||
mod workers;
|
||||
use background::{run_event_sync, run_local_index};
|
||||
use background::{run_event_sync, run_local_index, run_photos_event_sync};
|
||||
pub(crate) use mount::is_stale_mount;
|
||||
pub use mount::{MountOptions, MountOutcome, mount};
|
||||
use mount::{
|
||||
@@ -194,19 +197,42 @@ const UPLOAD_TIME_MEMORY: usize = 512;
|
||||
/// for the SDK's own per-request concurrency.
|
||||
const DRAIN_WORKERS: usize = 3;
|
||||
|
||||
/// How long a staged blob that names no node is kept before it is retired.
|
||||
///
|
||||
/// Staging is exempt from the cache budget because it holds the only copy of
|
||||
/// writes that have not been uploaded, so nothing there is ever evicted to
|
||||
/// reclaim space. That is right for a blob that still describes something. It is
|
||||
/// wrong for one whose sidecar is missing or names a `local~` placeholder whose
|
||||
/// create is gone: no future run can address it, no UI accounts for it, and it
|
||||
/// occupies disk for the life of the machine. Real instance: 95 such blobs,
|
||||
/// ~445 MiB, the oldest two months old, re-announced on every mount.
|
||||
///
|
||||
/// A month is far longer than any transfer or any plausible "I will look at that
|
||||
/// later", and the retirement is announced per blob with its size, so the one
|
||||
/// case that matters — bytes a human still wants — has a month of warnings and a
|
||||
/// path to go and fetch them from.
|
||||
const STAGING_ORPHAN_RETAIN: Duration = Duration::from_secs(30 * 24 * 60 * 60);
|
||||
|
||||
const ONLINE_PROBE_MIN: Duration = Duration::from_secs(5);
|
||||
const ONLINE_PROBE_MAX: Duration = Duration::from_secs(300);
|
||||
/// How long the persisted photos timeline stays good before a page request
|
||||
/// revalidates it. The SDK hands back the whole timeline at once, so it is stored
|
||||
/// in the DB and every page is sliced from there; a stale one is still served
|
||||
/// immediately and refreshed in the background.
|
||||
const TIMELINE_TTL: Duration = Duration::from_secs(5 * 60);
|
||||
///
|
||||
/// Short, because the cost of a stale timeline is a photo the user deleted on
|
||||
/// their phone still sitting in the gallery, and the cost of the check is one
|
||||
/// listing call that only runs when somebody is actually looking at the page.
|
||||
const TIMELINE_TTL: Duration = Duration::from_secs(60);
|
||||
/// How many photo nodes are resolved per [`ProtonPhotosClient::enumerate_nodes`]
|
||||
/// call when enriching a refreshed timeline with names and media types (for the
|
||||
/// Photos / Videos / Raw split). Batched so a large library is a handful of
|
||||
/// round-trips rather than one request per photo, and bounded so a single call
|
||||
/// never asks the server to decrypt the whole library at once.
|
||||
const TIMELINE_ENRICH_CHUNK: usize = 200;
|
||||
///
|
||||
/// 150 because that is the SDK's own `MAX_BATCH_COUNT`: a larger chunk is split
|
||||
/// there anyway, into one full request and a ragged tail.
|
||||
const TIMELINE_ENRICH_CHUNK: usize = 150;
|
||||
/// The same, for the persisted trash listing. Shorter, because the trash is the
|
||||
/// one listing a user changes and then immediately looks at — though our own
|
||||
/// mutations also invalidate it outright, so this only covers other clients.
|
||||
@@ -1384,6 +1410,15 @@ fn release_must_retain_queued_trash(db: &Db, uid: &NodeUid) -> pdfs_core::Result
|
||||
db.has_pending_op(&uid.to_string(), OP_TRASH)
|
||||
}
|
||||
|
||||
/// The size of a staged blob that has outlived [`STAGING_ORPHAN_RETAIN`], or
|
||||
/// `None` while it is younger than that — or while its age cannot be read at
|
||||
/// all, which is never grounds for deleting bytes.
|
||||
fn orphan_past_retention(blob: &Path) -> Option<u64> {
|
||||
let meta = std::fs::metadata(blob).ok()?;
|
||||
let age = meta.modified().ok()?.elapsed().ok()?;
|
||||
(age >= STAGING_ORPHAN_RETAIN).then_some(meta.len())
|
||||
}
|
||||
|
||||
fn release_can_discard_unlinked(db: &Db, uid: &NodeUid) -> bool {
|
||||
match release_must_retain_queued_trash(db, uid) {
|
||||
Ok(retain) => !retain,
|
||||
@@ -1708,6 +1743,9 @@ impl Core {
|
||||
};
|
||||
let mut queued = 0usize;
|
||||
let mut stranded = 0usize;
|
||||
let mut retired = 0usize;
|
||||
let mut retired_bytes = 0u64;
|
||||
let mut example: Option<PathBuf> = None;
|
||||
for (blob, meta) in staged {
|
||||
if claimed.contains(&blob.to_string_lossy().into_owned()) {
|
||||
continue;
|
||||
@@ -1715,9 +1753,17 @@ impl Core {
|
||||
let addressable = meta.as_ref().and_then(|m| parse_node_uid(&m.uid));
|
||||
let Some((meta, uid)) = meta.zip(addressable).filter(|(_, uid)| !is_local_uid(uid))
|
||||
else {
|
||||
if let Some(bytes) = orphan_past_retention(&blob) {
|
||||
retired += 1;
|
||||
retired_bytes += bytes;
|
||||
warn!(blob = %blob.display(), bytes,
|
||||
age_days = STAGING_ORPHAN_RETAIN.as_secs() / 86_400,
|
||||
"retiring staged bytes that name no node and have outlived retention");
|
||||
self.cache.discard_staged(&blob);
|
||||
continue;
|
||||
}
|
||||
stranded += 1;
|
||||
warn!(blob = %blob.display(),
|
||||
"staged bytes belong to no queued upload and name no node; kept for recovery");
|
||||
example.get_or_insert_with(|| blob.clone());
|
||||
continue;
|
||||
};
|
||||
if self.pending.lock().contains_key(&uid) {
|
||||
@@ -1763,8 +1809,21 @@ impl Core {
|
||||
}
|
||||
}
|
||||
}
|
||||
if queued > 0 || stranded > 0 {
|
||||
info!(queued, stranded, "reconciled orphaned staged writes");
|
||||
if stranded > 0 {
|
||||
// One line, not one per blob. A tree of them says exactly as much as
|
||||
// a single count and an example does, and said it on every mount:
|
||||
// 86 identical warnings for the same 86 files, boot after boot.
|
||||
warn!(
|
||||
count = stranded,
|
||||
example = %example.map(|b| b.display().to_string()).unwrap_or_default(),
|
||||
"staged bytes belong to no queued upload and name no node; kept for recovery"
|
||||
);
|
||||
}
|
||||
if queued > 0 || stranded > 0 || retired > 0 {
|
||||
info!(
|
||||
queued,
|
||||
stranded, retired, retired_bytes, "reconciled orphaned staged writes"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4366,6 +4425,9 @@ impl Core {
|
||||
is_dir: node.is_folder(),
|
||||
size: node_size(&node) as i64,
|
||||
mtime: node.modification_time,
|
||||
// The folder the node will go back to, which is also what tells
|
||||
// a restore whether that folder is itself in the trash.
|
||||
parent_uid: node.parent_uid.as_ref().map(|uid| uid.to_string()),
|
||||
}));
|
||||
// Cumulative, so the table is always a prefix of the real trash
|
||||
// rather than a mix of this refresh and the last one.
|
||||
@@ -4471,15 +4533,48 @@ impl Core {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Restore trashed nodes to the folders they were trashed from. The parents
|
||||
/// are read *before* the restore — a restored node reappears in a listing the
|
||||
/// daemon may already have cached, so each destination folder is invalidated
|
||||
/// and re-enumerated on next access.
|
||||
/// Restore trashed nodes to the folders they were trashed from, together
|
||||
/// with the rest of the trashed tree they belong to.
|
||||
///
|
||||
/// The trash is a flat list, but a user deletes a shape: a folder, and often
|
||||
/// items inside it that were trashed separately beforehand. Restoring one of
|
||||
/// those in isolation is not what anyone means by it — a folder would come
|
||||
/// back without the contents that were deleted with it, and a file would come
|
||||
/// back inside a folder that is still in the trash, which is to say nowhere
|
||||
/// the user can see. So a restore takes the whole connected piece:
|
||||
/// every trashed descendant of what was asked for, and every trashed ancestor
|
||||
/// above it. See [`expand_restore`] for the exact rule.
|
||||
///
|
||||
/// The waves are restored shallowest first, because a node cannot be put back
|
||||
/// under a parent that is still trashed.
|
||||
///
|
||||
/// The destination parents are read *before* the restore — a restored node
|
||||
/// reappears in a listing the daemon may already have cached, so each one is
|
||||
/// invalidated and re-enumerated on next access.
|
||||
fn restore(&self, uids: &[String]) -> CoreResult<usize> {
|
||||
let parsed = Self::parse_uids(uids)?;
|
||||
let requested = Self::parse_uids(uids)?;
|
||||
// A listing we could not read is not a reason to refuse the restore: it
|
||||
// only costs the expansion, and the requested nodes still go back.
|
||||
let relations = self.db.trash_parents().unwrap_or_default();
|
||||
let waves = expand_restore(uids, &relations);
|
||||
let waves: Vec<Vec<NodeUid>> = if waves.is_empty() {
|
||||
vec![requested.clone()]
|
||||
} else {
|
||||
waves
|
||||
.iter()
|
||||
.map(|wave| {
|
||||
wave.iter()
|
||||
.filter_map(|uid| parse_uid(uid))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.filter(|wave: &Vec<NodeUid>| !wave.is_empty())
|
||||
.collect()
|
||||
};
|
||||
|
||||
let all: Vec<NodeUid> = waves.iter().flatten().cloned().collect();
|
||||
let parents: Vec<NodeUid> = self
|
||||
.rt
|
||||
.block_on(self.client.enumerate_nodes(&parsed))
|
||||
.block_on(self.client.enumerate_nodes(&all))
|
||||
.map_err(|e| CoreError::from_api(&e, "enumerate nodes"))?
|
||||
.into_iter()
|
||||
.filter_map(|n| n.parent_uid)
|
||||
@@ -4490,27 +4585,29 @@ impl Core {
|
||||
// half way through a large restore leaves local state describing what the
|
||||
// server actually did. Only a batch that restored nothing is an error the
|
||||
// caller can act on.
|
||||
let mut restored: Vec<NodeUid> = Vec::with_capacity(parsed.len());
|
||||
let mut restored: Vec<NodeUid> = Vec::with_capacity(all.len());
|
||||
let mut first_error: Option<ProtonError> = None;
|
||||
self.rt
|
||||
.block_on(async {
|
||||
let mut outcomes = std::pin::pin!(self.client.restore_nodes_streaming(&parsed));
|
||||
while let Some(item) = outcomes.next().await {
|
||||
let (uid, outcome) = item?;
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
self.hidden.lock().remove(&uid);
|
||||
restored.push(uid);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(%uid, error = %e, "restore failed for a node");
|
||||
first_error.get_or_insert(e);
|
||||
for wave in &waves {
|
||||
self.rt
|
||||
.block_on(async {
|
||||
let mut outcomes = std::pin::pin!(self.client.restore_nodes_streaming(wave));
|
||||
while let Some(item) = outcomes.next().await {
|
||||
let (uid, outcome) = item?;
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
self.hidden.lock().remove(&uid);
|
||||
restored.push(uid);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(%uid, error = %e, "restore failed for a node");
|
||||
first_error.get_or_insert(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok::<(), ProtonError>(())
|
||||
})
|
||||
.map_err(|e| CoreError::from_api(&e, "restore"))?;
|
||||
Ok::<(), ProtonError>(())
|
||||
})
|
||||
.map_err(|e| CoreError::from_api(&e, "restore"))?;
|
||||
}
|
||||
if restored.is_empty()
|
||||
&& let Some(error) = first_error
|
||||
{
|
||||
@@ -5285,6 +5382,96 @@ fn node_size(node: &Node) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Expand a restore request across the trashed tree, and order it so a parent is
|
||||
/// always restored before its children.
|
||||
///
|
||||
/// `relations` is the persisted trash listing as `(uid, parent_uid)` pairs; a
|
||||
/// parent that is not itself in the listing is a live folder, which ends the
|
||||
/// walk upward. The rule is:
|
||||
///
|
||||
/// - every trashed descendant of a requested node joins the restore, so a folder
|
||||
/// comes back with the items that were trashed inside it;
|
||||
/// - every trashed ancestor of a requested node joins it too, so a file never
|
||||
/// lands in a folder that is still in the trash;
|
||||
/// - a requested uid the listing does not know about is restored on its own,
|
||||
/// because a stale or half-materialised listing must not silently drop it.
|
||||
///
|
||||
/// The result is one wave per depth, shallowest first. Waves are disjoint and
|
||||
/// together contain each uid exactly once.
|
||||
fn expand_restore(
|
||||
requested: &[String],
|
||||
relations: &[(String, Option<String>)],
|
||||
) -> Vec<Vec<String>> {
|
||||
let parents: HashMap<&str, Option<&str>> = relations
|
||||
.iter()
|
||||
.map(|(uid, parent)| (uid.as_str(), parent.as_deref()))
|
||||
.collect();
|
||||
let mut children: HashMap<&str, Vec<&str>> = HashMap::new();
|
||||
for (uid, parent) in relations {
|
||||
// Only a trashed parent: a live one keeps its child as a root here.
|
||||
if let Some(parent) = parent.as_deref()
|
||||
&& parents.contains_key(parent)
|
||||
{
|
||||
children.entry(parent).or_default().push(uid.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
let mut selected: HashSet<&str> = HashSet::new();
|
||||
for uid in requested {
|
||||
let uid = uid.as_str();
|
||||
if !parents.contains_key(uid) {
|
||||
selected.insert(uid);
|
||||
continue;
|
||||
}
|
||||
// Upward: every trashed ancestor, so the destination folder exists again.
|
||||
let mut cursor = Some(uid);
|
||||
while let Some(current) = cursor {
|
||||
// A parent outside the listing is live: the walk stops under it.
|
||||
if !parents.contains_key(current) || !selected.insert(current) {
|
||||
break;
|
||||
}
|
||||
cursor = parents.get(current).copied().flatten();
|
||||
}
|
||||
// Downward: the whole trashed subtree, so the folder comes back full.
|
||||
let mut stack = vec![uid];
|
||||
while let Some(current) = stack.pop() {
|
||||
for child in children.get(current).into_iter().flatten() {
|
||||
if selected.insert(child) {
|
||||
stack.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Depth within the *trash*, not within the drive: a node whose parent is live
|
||||
// (or unknown) is at depth 0 and can be restored straight away.
|
||||
let mut waves: Vec<Vec<String>> = Vec::new();
|
||||
for uid in &selected {
|
||||
let mut depth = 0usize;
|
||||
let mut cursor = parents.get(uid).copied().flatten();
|
||||
// Only trashed ancestors count, and the walk is bounded by the listing's
|
||||
// size: server data that somehow describes a cycle must not spin here.
|
||||
while let Some(current) = cursor
|
||||
&& parents.contains_key(current)
|
||||
&& depth < parents.len()
|
||||
{
|
||||
depth += 1;
|
||||
cursor = parents.get(current).copied().flatten();
|
||||
}
|
||||
if waves.len() <= depth {
|
||||
waves.resize(depth + 1, Vec::new());
|
||||
}
|
||||
waves[depth].push((*uid).to_string());
|
||||
}
|
||||
// Stable output, so a restore of the same trash is the same sequence of calls
|
||||
// every time — the batches are what shows up in the log and in the tests.
|
||||
for wave in &mut waves {
|
||||
wave.sort();
|
||||
}
|
||||
waves.retain(|wave| !wave.is_empty());
|
||||
waves
|
||||
}
|
||||
|
||||
/// The server revision id of a node's active revision, if it is a file that has
|
||||
/// one. The stable identity the drain conflict-checks against (see [`Baseline`]).
|
||||
fn node_revision_id(node: &Node) -> Option<String> {
|
||||
@@ -5985,15 +6172,159 @@ mod tests {
|
||||
use super::{
|
||||
Access, AccessFlags, Errno, HashMap, Intervals, PendingRevision, RootListingSnapshot,
|
||||
SELF_CHANGE_TTL_MS, ShareId, SharedWithMeItem, StateRegistry, VirtualRootPlan,
|
||||
accepted_share_provenance, conflict_name, copy_pending_for_truncate, fuse_name,
|
||||
is_stale_mount, node_visible, note_self_change, parse_node_uid, prepare_shared_roots,
|
||||
preserve_on_access_denied, publish_virtual_root_in_listing,
|
||||
reconcile_virtual_root_in_listing, release_can_discard_unlinked,
|
||||
release_must_retain_queued_trash, release_unlinked_entry, rename_needs_queue,
|
||||
require_node_parent_access, require_rename_access, resolve_anywhere_with,
|
||||
shared_with_me_uid, take_self_change, uid_write_authority, virtual_node,
|
||||
accepted_share_provenance, conflict_name, copy_pending_for_truncate, expand_restore,
|
||||
fuse_name, is_stale_mount, node_visible, note_self_change, orphan_past_retention,
|
||||
parse_node_uid, prepare_shared_roots, preserve_on_access_denied,
|
||||
publish_virtual_root_in_listing, reconcile_virtual_root_in_listing,
|
||||
release_can_discard_unlinked, release_must_retain_queued_trash, release_unlinked_entry,
|
||||
rename_needs_queue, require_node_parent_access, require_rename_access,
|
||||
resolve_anywhere_with, shared_with_me_uid, take_self_change, uid_write_authority,
|
||||
virtual_node,
|
||||
};
|
||||
use super::{Db, WriteAuthority};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Staged bytes that name no node are kept for a month, because the only
|
||||
/// copy of a write must outlive the bug that stranded it. Past that, one
|
||||
/// boot retires them instead of re-announcing them every boot forever.
|
||||
#[test]
|
||||
fn a_stranded_blob_is_retired_only_once_it_outlives_retention() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-orphan-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let blob = dir.join("blob");
|
||||
std::fs::write(&blob, b"twelve bytes").unwrap();
|
||||
|
||||
// Written just now: still recoverable, still kept.
|
||||
assert_eq!(orphan_past_retention(&blob), None);
|
||||
|
||||
set_mtime_ago(
|
||||
&blob,
|
||||
super::STAGING_ORPHAN_RETAIN + Duration::from_secs(60),
|
||||
);
|
||||
assert_eq!(orphan_past_retention(&blob), Some(12));
|
||||
|
||||
// A blob that is not there is not a blob to retire.
|
||||
std::fs::remove_file(&blob).unwrap();
|
||||
assert_eq!(orphan_past_retention(&blob), None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Backdate a file's mtime, which is what the retention check reads.
|
||||
fn set_mtime_ago(path: &std::path::Path, ago: Duration) {
|
||||
let when = std::time::SystemTime::now() - ago;
|
||||
let secs = when
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as libc::time_t;
|
||||
let times = [
|
||||
libc::timeval {
|
||||
tv_sec: secs,
|
||||
tv_usec: 0,
|
||||
},
|
||||
libc::timeval {
|
||||
tv_sec: secs,
|
||||
tv_usec: 0,
|
||||
},
|
||||
];
|
||||
let c_path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap();
|
||||
// SAFETY: both pointers are valid for the call and nothing keeps them.
|
||||
let rc = unsafe { libc::utimes(c_path.as_ptr(), times.as_ptr()) };
|
||||
assert_eq!(rc, 0, "utimes: {}", std::io::Error::last_os_error());
|
||||
}
|
||||
|
||||
/// `(uid, parent)` pairs in the shape [`Db::trash_parents`] returns.
|
||||
fn relations(pairs: &[(&str, Option<&str>)]) -> Vec<(String, Option<String>)> {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(uid, parent)| (uid.to_string(), parent.map(str::to_string)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn uids(values: &[&str]) -> Vec<String> {
|
||||
values.iter().map(|v| v.to_string()).collect()
|
||||
}
|
||||
|
||||
/// Restoring a folder brings back what was trashed inside it, and the folder
|
||||
/// goes first: the server has nowhere to put a child whose parent is trashed.
|
||||
#[test]
|
||||
fn restoring_a_folder_restores_its_trashed_contents_parents_first() {
|
||||
let trash = relations(&[
|
||||
("dir", Some("live")),
|
||||
("file", Some("dir")),
|
||||
("deep", Some("sub")),
|
||||
("sub", Some("dir")),
|
||||
]);
|
||||
assert_eq!(
|
||||
expand_restore(&uids(&["dir"]), &trash),
|
||||
vec![
|
||||
vec!["dir".to_string()],
|
||||
vec!["file".to_string(), "sub".to_string()],
|
||||
vec!["deep".to_string()],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// Restoring a file inside a trashed folder restores the folder too —
|
||||
/// otherwise the file comes back somewhere the user cannot reach.
|
||||
#[test]
|
||||
fn restoring_content_restores_the_trashed_parents_above_it() {
|
||||
let trash = relations(&[
|
||||
("dir", Some("live")),
|
||||
("sub", Some("dir")),
|
||||
("file", Some("sub")),
|
||||
("other", Some("dir")),
|
||||
]);
|
||||
// `other` is a sibling of `sub`, not an ancestor of `file`: it stays put.
|
||||
assert_eq!(
|
||||
expand_restore(&uids(&["file"]), &trash),
|
||||
vec![
|
||||
vec!["dir".to_string()],
|
||||
vec!["sub".to_string()],
|
||||
vec!["file".to_string()],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// A node whose parent is not in the trash is already at the top: one wave,
|
||||
/// no ancestors invented.
|
||||
#[test]
|
||||
fn a_node_under_a_live_parent_restores_on_its_own() {
|
||||
let trash = relations(&[("file", Some("live")), ("elsewhere", None)]);
|
||||
assert_eq!(
|
||||
expand_restore(&uids(&["file"]), &trash),
|
||||
vec![vec!["file".to_string()]]
|
||||
);
|
||||
}
|
||||
|
||||
/// The listing is materialised in chunks and can be stale, so a uid it does
|
||||
/// not know about must still be restored rather than dropped.
|
||||
#[test]
|
||||
fn an_unlisted_uid_is_still_restored() {
|
||||
let trash = relations(&[("dir", None)]);
|
||||
assert_eq!(
|
||||
expand_restore(&uids(&["missing"]), &trash),
|
||||
vec![vec!["missing".to_string()]]
|
||||
);
|
||||
}
|
||||
|
||||
/// Asking for both a folder and something inside it restores each node once.
|
||||
#[test]
|
||||
fn overlapping_requests_do_not_duplicate_nodes() {
|
||||
let trash = relations(&[("dir", None), ("file", Some("dir"))]);
|
||||
assert_eq!(
|
||||
expand_restore(&uids(&["dir", "file"]), &trash),
|
||||
vec![vec!["dir".to_string()], vec!["file".to_string()]]
|
||||
);
|
||||
}
|
||||
|
||||
/// A parent chain that loops back on itself must terminate.
|
||||
#[test]
|
||||
fn a_cyclic_listing_terminates() {
|
||||
let trash = relations(&[("a", Some("b")), ("b", Some("a"))]);
|
||||
let waves = expand_restore(&uids(&["a"]), &trash);
|
||||
let total: usize = waves.iter().map(Vec::len).sum();
|
||||
assert_eq!(total, 2);
|
||||
}
|
||||
|
||||
/// The collapse [`Core::require_uid_writable`] performs, without a `Core`:
|
||||
/// every non-writable authority is one `EACCES` on the syscall path.
|
||||
|
||||
@@ -341,6 +341,9 @@ pub fn mount(
|
||||
username,
|
||||
sweep_mode,
|
||||
} = options;
|
||||
// Start the uptime clock here rather than at the first `pdfs diagnostics`,
|
||||
// so the age it reports is the daemon's own.
|
||||
super::diagnostics::start_clock();
|
||||
let (root, online) = fetch_or_recall_root(&client, &rt, &db)?;
|
||||
let scope = root.tree_event_scope_id();
|
||||
db.mount_upsert_my_files(&mountpoint.to_string_lossy(), &root.uid.to_string(), None)
|
||||
@@ -551,6 +554,26 @@ pub fn mount(
|
||||
}
|
||||
}
|
||||
}
|
||||
// The supervisor watches the pool and answers the systemd watchdog. Started
|
||||
// after the listener, because its liveness probe is a round trip over that
|
||||
// socket.
|
||||
{
|
||||
let supervisor_core = core.clone();
|
||||
let socket = control_socket.to_path_buf();
|
||||
match std::thread::Builder::new()
|
||||
.name("pdfs-supervisor".into())
|
||||
.spawn(move || supervisor::run(supervisor_core, socket))
|
||||
{
|
||||
Ok(handle) => workers.push(handle),
|
||||
// Not fatal: the mount works without a supervisor, it just goes back
|
||||
// to being undiagnosable if it wedges.
|
||||
Err(error) => warn!(error = %error, "could not start the daemon supervisor"),
|
||||
}
|
||||
}
|
||||
// The mount is live and the control socket answers: this is the moment the
|
||||
// unit is actually usable, which is what `Type=notify` reports.
|
||||
systemd::ready();
|
||||
|
||||
// Re-establish on-demand mounts only after the primary session is live.
|
||||
// Keep the worker so shutdown can close publication, wait for any in-flight
|
||||
// fetch to observe that closure, and then drain a stable registry.
|
||||
@@ -568,6 +591,9 @@ pub fn mount(
|
||||
}
|
||||
};
|
||||
rt.spawn(run_event_sync(client, scope, core.clone()));
|
||||
// The photos volume has its own event stream; without it the gallery only
|
||||
// learns about a phone-side deletion when the timeline goes stale.
|
||||
rt.spawn(run_photos_event_sync(core.clone()));
|
||||
if online {
|
||||
let repair_core = core.clone();
|
||||
let root_uid = core.primary_root_uid.clone();
|
||||
@@ -663,6 +689,11 @@ pub fn mount(
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// How long teardown waits for the FUSE worker pool before proceeding without
|
||||
/// it. Long enough for an in-flight block to finish, short enough that
|
||||
/// `systemctl --user stop` still returns inside `TimeoutStopSec`.
|
||||
const WORKER_JOIN_DEADLINE: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Signal every background loop, wake the ones that are blocked, and join them.
|
||||
///
|
||||
/// The mount used to return with its drain workers, sync engine, sweep, online
|
||||
@@ -677,6 +708,9 @@ pub fn mount(
|
||||
/// expected and correct: a worker part-way through an upload finishes it rather
|
||||
/// than abandoning the user's bytes mid-flight.
|
||||
fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
||||
// Say so before the joins: a teardown that takes a while must not be read as
|
||||
// a missed watchdog ping and turned into a kill.
|
||||
systemd::stopping();
|
||||
core.shutdown.stop();
|
||||
core.wake_drain();
|
||||
let _ = core.sync_tx.send(sync::SyncMsg::Stop);
|
||||
@@ -691,6 +725,12 @@ fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::Jo
|
||||
warn!("a background worker panicked before shutdown");
|
||||
}
|
||||
}
|
||||
// The FUSE worker pool last, and *before* the caller drops the tokio
|
||||
// runtime: a worker still inside a job when the runtime goes away panics on
|
||||
// its next timer with "A Tokio 1.x context was found, but it is being
|
||||
// shutdown" (seen 2026-09-18 09:58:35). Bounded, so a worker wedged on the
|
||||
// network cannot hold the stop the user asked for.
|
||||
core.workers.stop_and_join(WORKER_JOIN_DEADLINE);
|
||||
debug!("background workers stopped");
|
||||
}
|
||||
|
||||
|
||||
+296
-23
@@ -29,8 +29,8 @@ use proton_drive_rs::{NodeKind, PhotoTag, PhotoTagsUpdate, ThumbnailType};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::{
|
||||
Core, PHOTOS_AVAILABLE, PHOTOS_SYNCED_MS, TIMELINE_ENRICH_CHUNK, TIMELINE_TTL, node_size,
|
||||
now_ms, parse_uid,
|
||||
Core, PHOTOS_AVAILABLE, PHOTOS_SYNCED_MS, TIMELINE_ENRICH_CHUNK, TIMELINE_TTL, WriteAuthority,
|
||||
node_size, now_ms, parse_uid,
|
||||
};
|
||||
|
||||
/// Longest edge, in px, of a thumbnail generated locally for a photo the server
|
||||
@@ -402,7 +402,21 @@ fn extract_raw_preview(
|
||||
.map(|preview| (preview, orientation)))
|
||||
}
|
||||
|
||||
/// What a [`Core::trash_photos`] batch did: the uids that left the server, and
|
||||
/// the ones that did not with the reason each was refused.
|
||||
pub(crate) type TrashOutcome = (Vec<String>, Vec<(String, String)>);
|
||||
|
||||
/// Formats supported by the local thumbnail decoder and exposed by the GUI.
|
||||
/// What a timeline enrichment pass resolves for one photo, beyond what the
|
||||
/// timeline listing already carries.
|
||||
#[derive(Clone, Debug)]
|
||||
struct PhotoMeta {
|
||||
name: Option<String>,
|
||||
media_type: Option<String>,
|
||||
favorite: bool,
|
||||
content_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl Core {
|
||||
pub(crate) fn photos_timeline(
|
||||
&self,
|
||||
@@ -454,6 +468,8 @@ impl Core {
|
||||
no_thumb: photo.thumb_state == db::THUMB_NONE,
|
||||
kind: photo.kind,
|
||||
favorite: photo.favorite,
|
||||
group_size: photo.group_size as u32,
|
||||
has_raw: photo.has_raw,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -483,12 +499,135 @@ impl Core {
|
||||
.map_err(|e| CoreError::from_api(&e, "update photo tags"))?;
|
||||
pdfs_core::batch::into_unit(outcomes)
|
||||
.map_err(|e| CoreError::from_api(&e, "update photo tags"))?;
|
||||
// The photo event feed replays this write back at us as a bare
|
||||
// `NodeUpdated`, which is indistinguishable from the same photo being
|
||||
// favourited on a phone. Claim the echo, or the gallery pays for a
|
||||
// resolve of a node whose one changed field is already stored below.
|
||||
self.note_self_change(uid);
|
||||
self.db
|
||||
.photos_set_favorite(&uid.to_string(), favorite)
|
||||
.map_err(CoreError::from)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Every file of one photo's group, in server order, as wire items. A photo
|
||||
/// the timeline does not hold comes back empty, which the lightbox reads as
|
||||
/// "nothing to switch between".
|
||||
pub(crate) fn photo_group(&self, uid: &str) -> CoreResult<Vec<PhotoItem>> {
|
||||
let members = self.db.photos_group(uid).map_err(CoreError::from)?;
|
||||
Ok(members
|
||||
.into_iter()
|
||||
.map(|photo| self.photo_item(photo))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Every uid asked for, plus the other files of any group they belong to,
|
||||
/// each once. A uid the timeline does not hold (an album photo on someone
|
||||
/// else's volume) is passed through unchanged.
|
||||
fn expand_photo_groups(&self, uids: &[NodeUid]) -> Vec<NodeUid> {
|
||||
let mut out: Vec<NodeUid> = Vec::with_capacity(uids.len());
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
for uid in uids {
|
||||
let members = self.db.photos_group(&uid.to_string()).unwrap_or_default();
|
||||
let mut group: Vec<NodeUid> = members
|
||||
.iter()
|
||||
.filter_map(|photo| parse_uid(&photo.uid))
|
||||
.collect();
|
||||
if group.is_empty() {
|
||||
group.push(uid.clone());
|
||||
}
|
||||
for member in group {
|
||||
if seen.insert(member.to_string()) {
|
||||
out.push(member);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Mutation admission for a photo, which [`Core::require_uid_writable`]
|
||||
/// cannot answer on its own.
|
||||
///
|
||||
/// Photos live on the photo volume, and that volume is not part of the FUSE
|
||||
/// tree: a photo has no `nodes` row, so every authority comes back
|
||||
/// [`WriteAuthority::Unknown`] and the plain check refuses the whole
|
||||
/// library. Membership of our own timeline is the authority that does exist
|
||||
/// here — the timeline is enumerated from our photo share, so a uid in it is
|
||||
/// ours to trash. Anything else (an album photo on someone else's volume)
|
||||
/// still fails closed.
|
||||
fn require_photo_writable(&self, uid: &NodeUid) -> Result<(), fuser::Errno> {
|
||||
match self.uid_write_authority(uid) {
|
||||
WriteAuthority::Writable => Ok(()),
|
||||
WriteAuthority::Denied => Err(fuser::Errno::EACCES),
|
||||
WriteAuthority::Unknown => {
|
||||
let key = uid.to_string();
|
||||
let known = self
|
||||
.db
|
||||
.photos_by_uid(std::slice::from_ref(&key))
|
||||
.map(|rows| !rows.is_empty())
|
||||
.unwrap_or(false);
|
||||
if known {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(fuser::Errno::EACCES)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trash photos by uid, and forget them locally.
|
||||
///
|
||||
/// The gallery is not part of the FUSE mount, so this is the photos-side
|
||||
/// counterpart of [`Core::delete`]: same destination (Proton trash, so the
|
||||
/// Trash page can restore them), addressed by uid because that is all a tile
|
||||
/// has.
|
||||
///
|
||||
/// The server answers per node, and so does this: a batch where one photo
|
||||
/// fails still removes the ones that went, and the caller is told which is
|
||||
/// which. Only what actually left the server is forgotten locally.
|
||||
///
|
||||
/// A photo that is one file of a group takes its whole group with it. The
|
||||
/// gallery shows a RAW+JPEG shot as one tile, so deleting that tile has to
|
||||
/// delete the shot — leaving the RAW behind would put the photo back on the
|
||||
/// Raw tab and nowhere else.
|
||||
pub(crate) fn trash_photos(&self, uids: &[NodeUid]) -> CoreResult<TrashOutcome> {
|
||||
if uids.is_empty() {
|
||||
return Ok((Vec::new(), Vec::new()));
|
||||
}
|
||||
let uids = &self.expand_photo_groups(uids);
|
||||
for uid in uids {
|
||||
self.require_photo_writable(uid)
|
||||
.map_err(|errno| self.errno_error(errno, "trash access"))?;
|
||||
}
|
||||
let outcomes = self
|
||||
.rt
|
||||
.block_on(self.client.trash_nodes(uids))
|
||||
.map_err(|e| CoreError::from_api(&e, "trash"))?;
|
||||
let (trashed, failed) = pdfs_core::batch::split(outcomes);
|
||||
|
||||
let trashed: Vec<String> = trashed.iter().map(|uid| uid.to_string()).collect();
|
||||
for uid in &trashed {
|
||||
if let Some(parsed) = parse_uid(uid) {
|
||||
self.cache.evict(&parsed);
|
||||
self.evict_reader(&parsed);
|
||||
}
|
||||
}
|
||||
// The timeline is served from the database, so removing the rows is what
|
||||
// makes the photos disappear now rather than at the next refresh.
|
||||
if let Err(error) = self.db.photos_delete(&trashed) {
|
||||
warn!(%error, "trashed photos could not be removed from the timeline");
|
||||
}
|
||||
self.invalidate_photos();
|
||||
self.invalidate_albums();
|
||||
self.invalidate_trash();
|
||||
|
||||
let failed = failed
|
||||
.into_iter()
|
||||
.map(|(uid, error)| (uid.to_string(), error.to_string()))
|
||||
.collect();
|
||||
Ok((trashed, failed))
|
||||
}
|
||||
|
||||
/// Thumbnails for `uids`, served from the cache, fetched from the server for
|
||||
/// whatever is missing, and — for the photos the server has no thumbnail for
|
||||
/// at all — *generated locally* from the full file (see
|
||||
@@ -498,8 +637,36 @@ impl Core {
|
||||
///
|
||||
/// A photo absent from the persisted timeline is skipped: its capture time is
|
||||
/// the cache's validity tag, and guessing that would poison the cache.
|
||||
pub(crate) fn photo_thumbs(&self, uids: &[NodeUid]) -> Vec<PhotoThumb> {
|
||||
///
|
||||
/// A camera raw is served by the JPEG of the same shot when there is one:
|
||||
/// the two files show one picture, and the JPEG is already decodable in
|
||||
/// milliseconds where the raw costs a full download and a raw decode. The
|
||||
/// tile still answers under the uid that was asked for.
|
||||
pub(crate) fn photo_thumbs(&self, requested: &[NodeUid]) -> Vec<PhotoThumb> {
|
||||
let ttype = ThumbnailType::Thumbnail.as_i32();
|
||||
// Only a raw can be stood in for, and that is one batched read; the
|
||||
// per-shot group lookup is then paid for raws alone, not for a page of
|
||||
// ordinary photos.
|
||||
let asked: Vec<String> = requested.iter().map(|u| u.to_string()).collect();
|
||||
let raws: HashSet<String> = self
|
||||
.db
|
||||
.photos_by_uid(&asked)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|photo| photo.kind == PhotoKind::Raw)
|
||||
.map(|photo| photo.uid)
|
||||
.collect();
|
||||
let sources: Vec<NodeUid> = requested
|
||||
.iter()
|
||||
.map(|uid| {
|
||||
if raws.contains(&uid.to_string()) {
|
||||
self.thumb_source(uid)
|
||||
} else {
|
||||
uid.clone()
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let uids = &sources;
|
||||
let keys: Vec<String> = uids.iter().map(|u| u.to_string()).collect();
|
||||
let mut stored = self.db.photos_by_uid(&keys).unwrap_or_default();
|
||||
// Album covers and the contents of an album shared with us are not in our
|
||||
@@ -582,19 +749,47 @@ impl Core {
|
||||
}
|
||||
|
||||
let pending = self.thumb_gen.lock();
|
||||
uids.iter()
|
||||
.map(|uid| PhotoThumb {
|
||||
uid: uid.to_string(),
|
||||
path: tags.get(&uid.to_string()).and_then(|&tag| {
|
||||
requested
|
||||
.iter()
|
||||
.zip(uids)
|
||||
.map(|(asked, source)| PhotoThumb {
|
||||
uid: asked.to_string(),
|
||||
path: tags.get(&source.to_string()).and_then(|&tag| {
|
||||
self.cache
|
||||
.cached_thumbnail_path(uid, ttype, tag)
|
||||
.cached_thumbnail_path(source, ttype, tag)
|
||||
.map(|p| p.display().to_string())
|
||||
}),
|
||||
pending: pending.contains(uid),
|
||||
pending: pending.contains(source),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Which file's thumbnail stands for `uid` on a tile.
|
||||
///
|
||||
/// Itself, for all but a camera raw that was written next to a JPEG of the
|
||||
/// same shot. For that one, the JPEG: same picture, already compressed,
|
||||
/// often carrying a server thumbnail the raw does not have — so the Raw tab
|
||||
/// costs no raw download and no raw decode. A raw that stands alone keeps
|
||||
/// its own local generation.
|
||||
fn thumb_source(&self, uid: &NodeUid) -> NodeUid {
|
||||
let members = match self.db.photos_group(&uid.to_string()) {
|
||||
Ok(members) if members.len() > 1 => members,
|
||||
_ => return uid.clone(),
|
||||
};
|
||||
let key = uid.to_string();
|
||||
if !members
|
||||
.iter()
|
||||
.any(|photo| photo.uid == key && photo.kind == PhotoKind::Raw)
|
||||
{
|
||||
return uid.clone();
|
||||
}
|
||||
members
|
||||
.iter()
|
||||
.find(|photo| photo.kind != PhotoKind::Raw)
|
||||
.and_then(|photo| parse_uid(&photo.uid))
|
||||
.unwrap_or_else(|| uid.clone())
|
||||
}
|
||||
|
||||
/// Thumbnails for image files shown by the regular file browser, Shared and
|
||||
/// Trash pages. Unlike [`Core::photo_thumbs`], these nodes need not be in the
|
||||
/// Photos timeline: the request supplies each file's modification time as
|
||||
@@ -1209,6 +1404,7 @@ impl Core {
|
||||
/// Whether the listing stamped under `key` is older than `ttl` (or was never
|
||||
/// fetched).
|
||||
pub(crate) async fn refresh_timeline(&self) -> CoreResult<bool> {
|
||||
let started = Instant::now();
|
||||
let photos = self.photos();
|
||||
if photos
|
||||
.get_photos_root()
|
||||
@@ -1225,15 +1421,47 @@ impl Core {
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "timeline"))?;
|
||||
|
||||
// The timeline DTO carries only a uid and capture time, but the Photos
|
||||
// page has to split into Photos / Videos / Raw — which needs each photo's
|
||||
// name and media type. Resolve those in batches off the request path.
|
||||
// Best-effort: a photo whose node we fail to resolve keeps whatever was
|
||||
// learned before (or classifies from nothing, i.e. a still photo), so a
|
||||
// partial resolve never blanks the timeline.
|
||||
let uids: Vec<NodeUid> = items.iter().map(|it| it.uid.clone()).collect();
|
||||
let mut meta: HashMap<String, (Option<String>, Option<String>, bool)> = HashMap::new();
|
||||
// The timeline DTO carries only a uid and capture time, but the gallery
|
||||
// has to split into its tabs — which needs each photo's name and media
|
||||
// type. Resolve those in batches off the request path. Best-effort: a
|
||||
// photo whose node we fail to resolve keeps whatever was learned before
|
||||
// (or classifies from nothing, i.e. a still photo), so a partial resolve
|
||||
// never blanks the timeline.
|
||||
//
|
||||
// Only photos the daemon has never read, and ones a remote event marked
|
||||
// stale, are read. Resolving all of them every time is what made a
|
||||
// 12k-photo refresh take minutes (`docs/BUGS.md` B92): this metadata does
|
||||
// not change on its own, and when it does change the event feed names the
|
||||
// uid that changed (`Db::photos_mark_unresolved`).
|
||||
let known: HashSet<String> = self
|
||||
.db
|
||||
.photos_resolved()
|
||||
.map_err(CoreError::from)?
|
||||
.into_iter()
|
||||
.collect();
|
||||
let uids: Vec<NodeUid> = items
|
||||
.iter()
|
||||
.filter(|it| !known.contains(&it.uid.to_string()))
|
||||
.map(|it| it.uid.clone())
|
||||
.collect();
|
||||
let resolved_at = now_ms();
|
||||
let mut meta: HashMap<String, PhotoMeta> = HashMap::new();
|
||||
// A main photo names its related photos rather than the other way round,
|
||||
// so the relation is recorded from whichever end resolves — the two ends
|
||||
// can land in different chunks, and one of them may not resolve at all.
|
||||
let mut main_of: HashMap<String, String> = HashMap::new();
|
||||
let mut stopped = false;
|
||||
for chunk in uids.chunks(TIMELINE_ENRICH_CHUNK) {
|
||||
// Teardown cancels the runtime under this loop, and every chunk
|
||||
// still in flight then fails its decrypt task — a burst of
|
||||
// "skipping undecryptable photo ... task was cancelled" warnings
|
||||
// that read like corruption and are only a shutdown. Stop asking
|
||||
// for chunks instead. What resolved so far is kept below; the rest
|
||||
// stay unresolved and are read by the next refresh.
|
||||
if self.shutdown.is_stopping() {
|
||||
stopped = true;
|
||||
break;
|
||||
}
|
||||
match photos.enumerate_nodes(chunk).await {
|
||||
Ok(nodes) => {
|
||||
for node in nodes {
|
||||
@@ -1248,9 +1476,24 @@ impl Core {
|
||||
.photo
|
||||
.as_ref()
|
||||
.is_some_and(|p| p.tags.contains(&PhotoTag::Favorite));
|
||||
let key = node.uid.to_string();
|
||||
let content_hash = node.photo.as_ref().and_then(|p| p.content_hash.clone());
|
||||
if let Some(photo) = node.photo.as_ref() {
|
||||
if let Some(main) = photo.main_photo_uid.as_ref() {
|
||||
main_of.insert(key.clone(), main.to_string());
|
||||
}
|
||||
for related in &photo.related_photo_uids {
|
||||
main_of.insert(related.to_string(), key.clone());
|
||||
}
|
||||
}
|
||||
meta.insert(
|
||||
node.uid.to_string(),
|
||||
(Some(node.name), media_type, favorite),
|
||||
key,
|
||||
PhotoMeta {
|
||||
name: Some(node.name),
|
||||
media_type,
|
||||
favorite,
|
||||
content_hash,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1262,19 +1505,43 @@ impl Core {
|
||||
.iter()
|
||||
.map(|it| {
|
||||
let key = it.uid.to_string();
|
||||
let main_uid = main_of.get(&key).cloned();
|
||||
// An unresolved photo keeps whatever was learned before, for the
|
||||
// favourite flag as much as for the name and media type.
|
||||
match meta.get(&key).cloned() {
|
||||
Some((name, media_type, favorite)) => {
|
||||
(key, it.capture_time, name, media_type, Some(favorite))
|
||||
}
|
||||
None => (key, it.capture_time, None, None, None),
|
||||
Some(m) => db::TimelineRow {
|
||||
name: m.name,
|
||||
media_type: m.media_type,
|
||||
favorite: Some(m.favorite),
|
||||
content_hash: m.content_hash,
|
||||
main_uid,
|
||||
resolved_at: Some(resolved_at),
|
||||
..db::TimelineRow::new(key, it.capture_time)
|
||||
},
|
||||
None => db::TimelineRow {
|
||||
main_uid,
|
||||
..db::TimelineRow::new(key, it.capture_time)
|
||||
},
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
self.db.photos_replace(&rows).map_err(CoreError::from)?;
|
||||
let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 1);
|
||||
let _ = self.db.set_state_i64(PHOTOS_SYNCED_MS, now_ms());
|
||||
// How long this took is the first thing to know when a refresh looks
|
||||
// like it never happened, and the chunk count is what it scales with.
|
||||
info!(
|
||||
photos = rows.len(),
|
||||
// `resolved` is what came back, which is short of `wanted` when
|
||||
// shutdown cut the pass off or a chunk failed.
|
||||
resolved = meta.len(),
|
||||
wanted = uids.len(),
|
||||
skipped = rows.len().saturating_sub(uids.len()),
|
||||
chunks = uids.len().div_ceil(TIMELINE_ENRICH_CHUNK),
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
stopped,
|
||||
"photos timeline refreshed"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -1293,6 +1560,12 @@ impl Core {
|
||||
});
|
||||
}
|
||||
|
||||
/// Whether a timeline refresh is running. A front-end that asked for one
|
||||
/// watches this to know when the new timeline is worth re-reading.
|
||||
pub(crate) fn timeline_refreshing(&self) -> bool {
|
||||
self.timeline_refreshing.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Download a photo's full content into the content cache, returning its
|
||||
/// on-disk path (served from cache when a fresh blob already exists).
|
||||
pub(crate) fn open_photo(&self, uid: &NodeUid) -> CoreResult<PathBuf> {
|
||||
|
||||
@@ -222,8 +222,13 @@ impl Core {
|
||||
tags_to_add: vec![PhotoTag::Favorite],
|
||||
tags_to_remove: Vec::new(),
|
||||
};
|
||||
if let Err(e) = self.rt.block_on(photos.update_photos(&[update])) {
|
||||
warn!(name = candidate.name, error = %e, "cannot re-favourite re-dated photo");
|
||||
match self.rt.block_on(photos.update_photos(&[update])) {
|
||||
// As in `set_photo_favorite`: claim our own echo, so the tag we
|
||||
// just set does not come back as a remote change.
|
||||
Ok(_) => self.note_self_change(&new_uid),
|
||||
Err(e) => {
|
||||
warn!(name = candidate.name, error = %e, "cannot re-favourite re-dated photo")
|
||||
}
|
||||
}
|
||||
}
|
||||
for album in &candidate.albums {
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
//! The thread that watches the daemon for the thing the daemon cannot report
|
||||
//! about itself: that it has stopped making progress.
|
||||
//!
|
||||
//! Three jobs, all cheap, all on one timer:
|
||||
//!
|
||||
//! 1. **Say so in the journal.** A worker that has held one job for minutes, a
|
||||
//! queue that is not draining, a control request that never returned — each
|
||||
//! gets a WARN naming it. The 2026-09-18 hang left nothing in the journal at
|
||||
//! all; this is what turns the next one into a one-line diagnosis.
|
||||
//! 2. **Ping the systemd watchdog**, but only after a real round trip over the
|
||||
//! control socket. Pinging unconditionally from a thread that does nothing
|
||||
//! else would keep the unit "healthy" through exactly the hang it is meant to
|
||||
//! catch: the probe is what makes the ping mean "a client can still talk to
|
||||
//! this daemon".
|
||||
//! 3. **Sample memory.** That hang peaked at 5.2 GiB resident, and nobody knew
|
||||
//! until it was over.
|
||||
|
||||
use std::io::{BufRead, BufReader, Write as _};
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use pdfs_core::control::Request as CtlRequest;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::Core;
|
||||
use super::systemd;
|
||||
|
||||
/// How often the supervisor looks. Fast enough that a stall is named while the
|
||||
/// user is still watching it, slow enough to cost nothing.
|
||||
const TICK: Duration = Duration::from_secs(15);
|
||||
|
||||
/// How long one job may hold a worker, or one control request may run, before it
|
||||
/// is called out. Generous: a large upload chunk or a cold folder listing on a
|
||||
/// slow link is minutes of legitimate work, and a false WARN every few minutes
|
||||
/// would train everyone to ignore the real one.
|
||||
const STALL_AFTER: Duration = Duration::from_secs(120);
|
||||
|
||||
/// How often resident size goes into the log.
|
||||
const RSS_EVERY: Duration = Duration::from_secs(300);
|
||||
|
||||
/// How long the liveness probe waits for the daemon to answer its own control
|
||||
/// socket. Well under the watchdog period, so a probe that times out still
|
||||
/// leaves time for the manager to hear nothing and act.
|
||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Run until shutdown. Spawned once per mount, from [`super::mount::mount`].
|
||||
pub(crate) fn run(core: Core, control_socket: PathBuf) {
|
||||
let watchdog = systemd::watchdog_period();
|
||||
if let Some(period) = watchdog {
|
||||
info!(period_secs = period.as_secs(), "systemd watchdog enabled");
|
||||
}
|
||||
|
||||
let mut last_rss = Instant::now();
|
||||
let mut last_ping = Instant::now();
|
||||
// Per-lane completion counts from the previous tick, to tell a queue that is
|
||||
// being worked through from one that is not moving at all.
|
||||
let mut last_completed = (0_u64, 0_u64);
|
||||
|
||||
while core.shutdown.sleep(TICK) {
|
||||
let report = core.diagnostics();
|
||||
let stalled = report_stalls(&report, last_completed);
|
||||
last_completed = (report.meta_completed, report.transfer_completed);
|
||||
|
||||
if last_rss.elapsed() >= RSS_EVERY {
|
||||
info!(
|
||||
rss_bytes = report.rss_bytes,
|
||||
pending_ops = report.pending_ops,
|
||||
meta_queued = report.meta_queued,
|
||||
transfer_queued = report.transfer_queued,
|
||||
stalled,
|
||||
"daemon resource sample"
|
||||
);
|
||||
last_rss = Instant::now();
|
||||
}
|
||||
|
||||
// A stalled lane is deliberately *not* a reason to withhold the ping:
|
||||
// the mount may still be answering everything else, and a restart would
|
||||
// throw away in-flight uploads. The probe decides that; a stall only
|
||||
// gets said out loud.
|
||||
if let Some(period) = watchdog
|
||||
&& last_ping.elapsed() >= period
|
||||
{
|
||||
if probe(&control_socket) {
|
||||
systemd::watchdog_ping();
|
||||
} else {
|
||||
warn!(
|
||||
"daemon did not answer its own control socket; withholding the watchdog ping"
|
||||
);
|
||||
}
|
||||
last_ping = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// WARN about anything that has not moved for [`STALL_AFTER`]. Returns whether
|
||||
/// anything was reported, so the caller can weigh it.
|
||||
fn report_stalls(report: &pdfs_core::control::Diagnostics, last_completed: (u64, u64)) -> bool {
|
||||
let threshold = STALL_AFTER.as_secs();
|
||||
let mut stalled = false;
|
||||
|
||||
for worker in &report.workers {
|
||||
if worker.busy && worker.age_secs >= threshold {
|
||||
stalled = true;
|
||||
warn!(
|
||||
worker = %worker.name,
|
||||
job = %worker.label,
|
||||
age_secs = worker.age_secs,
|
||||
"a fuse worker has held the same job for a long time"
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(oldest) = report.inflight.first()
|
||||
&& oldest.age_secs >= threshold
|
||||
{
|
||||
stalled = true;
|
||||
warn!(
|
||||
request = %oldest.kind,
|
||||
age_secs = oldest.age_secs,
|
||||
in_flight = report.inflight.len(),
|
||||
"a control request has been running for a long time"
|
||||
);
|
||||
}
|
||||
// A backed-up lane is normal; a backed-up lane that finished nothing since
|
||||
// the last tick is not.
|
||||
if report.meta_queued > 0 && report.meta_completed == last_completed.0 {
|
||||
stalled = true;
|
||||
warn!(
|
||||
queued = report.meta_queued,
|
||||
"the metadata lane has not completed a job since the last check"
|
||||
);
|
||||
}
|
||||
if report.transfer_queued > 0 && report.transfer_completed == last_completed.1 {
|
||||
stalled = true;
|
||||
warn!(
|
||||
queued = report.transfer_queued,
|
||||
"the transfer lane has not completed a job since the last check"
|
||||
);
|
||||
}
|
||||
stalled
|
||||
}
|
||||
|
||||
/// One round trip over the control socket: connect, ask for the cheapest
|
||||
/// request there is, read the answer.
|
||||
///
|
||||
/// This is the liveness the watchdog ping stands for. It covers the accept loop,
|
||||
/// the handler pool and the reply path — everything a `pdfs` command or the GUI
|
||||
/// depends on — without touching the network or the database.
|
||||
fn probe(control_socket: &Path) -> bool {
|
||||
let Ok(stream) = UnixStream::connect(control_socket) else {
|
||||
return false;
|
||||
};
|
||||
if stream.set_read_timeout(Some(PROBE_TIMEOUT)).is_err()
|
||||
|| stream.set_write_timeout(Some(PROBE_TIMEOUT)).is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Ok(request) = serde_json::to_string(&CtlRequest::Diagnostics) else {
|
||||
return false;
|
||||
};
|
||||
let mut stream = stream;
|
||||
if stream.write_all(format!("{request}\n").as_bytes()).is_err() {
|
||||
return false;
|
||||
}
|
||||
let mut line = String::new();
|
||||
BufReader::new(stream).read_line(&mut line).is_ok() && !line.trim().is_empty()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pdfs_core::control::{Diagnostics, InflightRequest, WorkerState};
|
||||
|
||||
fn busy_worker(age_secs: u64) -> WorkerState {
|
||||
WorkerState {
|
||||
name: "pdfs-fuse-3".into(),
|
||||
busy: true,
|
||||
label: "read".into(),
|
||||
age_secs,
|
||||
}
|
||||
}
|
||||
|
||||
/// A worker doing normal work must not be reported: a WARN every tick is a
|
||||
/// WARN nobody reads.
|
||||
#[test]
|
||||
fn ordinary_work_is_not_a_stall() {
|
||||
let report = Diagnostics {
|
||||
workers: vec![busy_worker(5)],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!report_stalls(&report, (0, 0)));
|
||||
}
|
||||
|
||||
/// A worker on the same job past the threshold is the shape of the hang.
|
||||
#[test]
|
||||
fn a_long_held_job_is_a_stall() {
|
||||
let report = Diagnostics {
|
||||
workers: vec![busy_worker(STALL_AFTER.as_secs() + 1)],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(report_stalls(&report, (0, 0)));
|
||||
}
|
||||
|
||||
/// So is a control request that never returned.
|
||||
#[test]
|
||||
fn a_long_running_request_is_a_stall() {
|
||||
let report = Diagnostics {
|
||||
inflight: vec![InflightRequest {
|
||||
kind: "OpenFile".into(),
|
||||
age_secs: STALL_AFTER.as_secs() + 1,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(report_stalls(&report, (0, 0)));
|
||||
}
|
||||
|
||||
/// A queue that is draining is busy, not stalled — that is what comparing
|
||||
/// the completion counts is for.
|
||||
#[test]
|
||||
fn a_draining_queue_is_not_a_stall() {
|
||||
let report = Diagnostics {
|
||||
transfer_queued: 12,
|
||||
transfer_completed: 40,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!report_stalls(&report, (0, 30)));
|
||||
assert!(report_stalls(&report, (0, 40)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
//! The `sd_notify` half of the service contract, without linking libsystemd.
|
||||
//!
|
||||
//! The protocol is one datagram of `KEY=VALUE` lines to the socket in
|
||||
//! `$NOTIFY_SOCKET`. That is little enough to write directly, and writing it
|
||||
//! directly keeps the daemon buildable on a machine without systemd headers —
|
||||
//! everything here degrades to "no socket, do nothing" off systemd.
|
||||
|
||||
use std::os::unix::net::UnixDatagram;
|
||||
use std::time::Duration;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
/// Send one notification. `false` when there is no manager to send it to, or the
|
||||
/// send failed — a caller has nothing useful to do about either.
|
||||
pub(crate) fn notify(message: &str) -> bool {
|
||||
let Ok(socket_path) = std::env::var("NOTIFY_SOCKET") else {
|
||||
return false;
|
||||
};
|
||||
if socket_path.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let Ok(socket) = UnixDatagram::unbound() else {
|
||||
return false;
|
||||
};
|
||||
if let Some(name) = socket_path.strip_prefix('@') {
|
||||
// Abstract namespace: the address is a NUL byte followed by the name,
|
||||
// which `std`'s path-based API cannot express.
|
||||
send_abstract(&socket, name, message)
|
||||
} else {
|
||||
socket.send_to(message.as_bytes(), &socket_path).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// `sendto` on an abstract-namespace address, which needs a hand-built
|
||||
/// `sockaddr_un` because the first byte of the name is NUL.
|
||||
fn send_abstract(socket: &UnixDatagram, name: &str, message: &str) -> bool {
|
||||
use std::os::fd::AsRawFd;
|
||||
|
||||
let mut addr: libc::sockaddr_un = unsafe { std::mem::zeroed() };
|
||||
addr.sun_family = libc::AF_UNIX as libc::sa_family_t;
|
||||
let bytes = name.as_bytes();
|
||||
// One byte for the leading NUL, and the path must still be addressable.
|
||||
if bytes.len() + 1 > addr.sun_path.len() {
|
||||
return false;
|
||||
}
|
||||
for (slot, byte) in addr.sun_path.iter_mut().skip(1).zip(bytes) {
|
||||
*slot = *byte as libc::c_char;
|
||||
}
|
||||
let len = std::mem::size_of::<libc::sa_family_t>() + 1 + bytes.len();
|
||||
let sent = unsafe {
|
||||
libc::sendto(
|
||||
socket.as_raw_fd(),
|
||||
message.as_ptr().cast(),
|
||||
message.len(),
|
||||
0,
|
||||
std::ptr::from_ref(&addr).cast(),
|
||||
len as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
sent >= 0
|
||||
}
|
||||
|
||||
/// Tell the manager the daemon is up and serving. Called once the mount is live,
|
||||
/// so `systemctl --user start` returns when the mount actually works rather than
|
||||
/// when the process exists.
|
||||
pub(crate) fn ready() {
|
||||
if notify("READY=1\n") {
|
||||
debug!("notified systemd: ready");
|
||||
}
|
||||
}
|
||||
|
||||
/// Tell the manager the daemon is still healthy.
|
||||
pub(crate) fn watchdog_ping() {
|
||||
let _ = notify("WATCHDOG=1\n");
|
||||
}
|
||||
|
||||
/// Tell the manager the daemon is stopping, so a slow teardown is not mistaken
|
||||
/// for a hang by the watchdog it is racing.
|
||||
pub(crate) fn stopping() {
|
||||
let _ = notify("STOPPING=1\n");
|
||||
}
|
||||
|
||||
/// How often the manager expects a ping, from `$WATCHDOG_USEC`, or `None` when
|
||||
/// the watchdog is off or belongs to another process.
|
||||
///
|
||||
/// Reports *half* the configured interval, the interval systemd documents for
|
||||
/// pinging: a ping period equal to the deadline would restart the daemon on the
|
||||
/// first late scheduling.
|
||||
pub(crate) fn watchdog_period() -> Option<Duration> {
|
||||
// `WATCHDOG_PID` is set when the manager wants a specific process to answer;
|
||||
// in that case a ping from anyone else is not the one it is waiting for.
|
||||
if let Ok(pid) = std::env::var("WATCHDOG_PID")
|
||||
&& pid.parse::<u32>().ok() != Some(std::process::id())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let usec: u64 = std::env::var("WATCHDOG_USEC").ok()?.parse().ok()?;
|
||||
(usec > 0).then(|| Duration::from_micros(usec / 2))
|
||||
}
|
||||
+333
-33
@@ -12,6 +12,8 @@
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use parking_lot::{Condvar, Mutex};
|
||||
use tracing::warn;
|
||||
@@ -51,8 +53,25 @@ pub(crate) const FUSE_WORKERS: usize = 11;
|
||||
/// exactly the blocking this split exists to prevent.
|
||||
const META_WORKERS: usize = 3;
|
||||
|
||||
/// Queue depth that gets a line in the log, and the step at which it repeats.
|
||||
/// Every job holds a `Reply` and a few fields, so depth is cheap — but a lane
|
||||
/// this far behind is worth knowing about before the user reports a slow mount.
|
||||
const QUEUE_WARN_STEP: usize = 256;
|
||||
|
||||
/// Whether a queue reaching `depth` is one of the depths worth logging.
|
||||
fn crosses_warn_step(depth: usize) -> bool {
|
||||
depth >= QUEUE_WARN_STEP && depth.is_multiple_of(QUEUE_WARN_STEP)
|
||||
}
|
||||
|
||||
type Job = Box<dyn FnOnce() + Send + 'static>;
|
||||
|
||||
/// A queued job and the name its caller gave it, so a job that never finishes
|
||||
/// can be named in the diagnostics instead of showing up as "a worker is busy".
|
||||
struct Queued {
|
||||
label: &'static str,
|
||||
job: Job,
|
||||
}
|
||||
|
||||
/// Which lane a job belongs in.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum Lane {
|
||||
@@ -65,12 +84,63 @@ pub(crate) enum Lane {
|
||||
|
||||
#[derive(Default)]
|
||||
struct Queues {
|
||||
meta: VecDeque<Job>,
|
||||
transfer: VecDeque<Job>,
|
||||
meta: VecDeque<Queued>,
|
||||
transfer: VecDeque<Queued>,
|
||||
/// Set when the pool is dropped, to wake and retire every worker.
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// What one worker thread is doing, published for [`Workers::snapshot`].
|
||||
///
|
||||
/// Every field is an atomic (or a lock the worker only ever holds for the length
|
||||
/// of a pointer write) because the whole point of the snapshot is to answer
|
||||
/// *while the daemon is wedged*: a diagnostic that waits on the same lock the
|
||||
/// stuck job holds reports nothing at exactly the moment it is needed.
|
||||
struct Slot {
|
||||
name: String,
|
||||
/// True between picking a job up and finishing it.
|
||||
busy: AtomicBool,
|
||||
/// When the current state (busy or idle) began, as milliseconds since the
|
||||
/// pool started. Monotonic, so an age is always a real duration.
|
||||
since_ms: AtomicU64,
|
||||
/// The label of the job in hand. Behind a mutex because a `&'static str` is
|
||||
/// two words and there is no atomic for that; the worker holds it only to
|
||||
/// swap the pointer, and the snapshot never blocks on it (see
|
||||
/// [`Workers::snapshot`]).
|
||||
label: Mutex<&'static str>,
|
||||
}
|
||||
|
||||
impl Slot {
|
||||
fn new(name: String, now_ms: u64) -> Self {
|
||||
Self {
|
||||
name,
|
||||
busy: AtomicBool::new(false),
|
||||
since_ms: AtomicU64::new(now_ms),
|
||||
label: Mutex::new(""),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One worker's state at the moment [`Workers::snapshot`] read it.
|
||||
pub(crate) struct WorkerSnapshot {
|
||||
pub(crate) name: String,
|
||||
pub(crate) busy: bool,
|
||||
/// How long the worker has been in that state.
|
||||
pub(crate) age_ms: u64,
|
||||
/// The job in hand, or `""` when idle. `"?"` when the label could not be
|
||||
/// read without blocking.
|
||||
pub(crate) label: String,
|
||||
}
|
||||
|
||||
/// The pool's state at the moment [`Workers::snapshot`] read it.
|
||||
pub(crate) struct PoolSnapshot {
|
||||
pub(crate) workers: Vec<WorkerSnapshot>,
|
||||
pub(crate) meta_queued: usize,
|
||||
pub(crate) transfer_queued: usize,
|
||||
pub(crate) meta_completed: u64,
|
||||
pub(crate) transfer_completed: u64,
|
||||
}
|
||||
|
||||
/// The queues and the two wait sets over them.
|
||||
///
|
||||
/// Two condvars, not one, because the workers are not interchangeable: a
|
||||
@@ -86,6 +156,22 @@ struct Pool {
|
||||
meta_cv: Condvar,
|
||||
/// Waited on by the general workers.
|
||||
general_cv: Condvar,
|
||||
/// Queue depths, mirrored outside the lock so a snapshot can read them
|
||||
/// without waiting on whatever is holding it.
|
||||
meta_queued: AtomicUsize,
|
||||
transfer_queued: AtomicUsize,
|
||||
/// Jobs finished per lane. A depth that stays high while this stays still is
|
||||
/// a stalled lane rather than a busy one.
|
||||
meta_completed: AtomicU64,
|
||||
transfer_completed: AtomicU64,
|
||||
slots: Vec<Slot>,
|
||||
started: Instant,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
fn now_ms(&self) -> u64 {
|
||||
self.started.elapsed().as_millis() as u64
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded thread pool behind the network-touching FUSE handlers.
|
||||
@@ -94,25 +180,46 @@ struct Pool {
|
||||
/// on-demand sync folder), so the bound is per daemon rather than per mount.
|
||||
pub(crate) struct Workers {
|
||||
inner: Arc<Pool>,
|
||||
/// Kept so teardown can *join* the workers, not merely flag them. A worker
|
||||
/// still inside a job when the process drops its tokio runtime panics with
|
||||
/// "A Tokio 1.x context was found, but it is being shutdown" (seen
|
||||
/// 2026-09-18 09:58:35), which is a teardown ordering bug rather than a real
|
||||
/// failure.
|
||||
handles: Mutex<Vec<std::thread::JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl Workers {
|
||||
pub(crate) fn new(n: usize) -> std::io::Result<Self> {
|
||||
// Never leave the pool without a general worker, however small `n` is.
|
||||
let meta_workers = META_WORKERS.min(n.saturating_sub(1));
|
||||
let names: Vec<String> = (0..n)
|
||||
.map(|i| {
|
||||
format!(
|
||||
"pdfs-fuse-{}{i}",
|
||||
if i < meta_workers { "meta-" } else { "" }
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let inner = Arc::new(Pool {
|
||||
queues: Mutex::new(Queues::default()),
|
||||
meta_cv: Condvar::new(),
|
||||
general_cv: Condvar::new(),
|
||||
meta_queued: AtomicUsize::new(0),
|
||||
transfer_queued: AtomicUsize::new(0),
|
||||
meta_completed: AtomicU64::new(0),
|
||||
transfer_completed: AtomicU64::new(0),
|
||||
slots: names
|
||||
.iter()
|
||||
.map(|name| Slot::new(name.clone(), 0))
|
||||
.collect(),
|
||||
started: Instant::now(),
|
||||
});
|
||||
// Never leave the pool without a general worker, however small `n` is.
|
||||
let meta_workers = META_WORKERS.min(n.saturating_sub(1));
|
||||
for i in 0..n {
|
||||
let mut handles = Vec::with_capacity(n);
|
||||
for (i, name) in names.iter().enumerate() {
|
||||
let inner = inner.clone();
|
||||
let meta_only = i < meta_workers;
|
||||
std::thread::Builder::new()
|
||||
.name(format!(
|
||||
"pdfs-fuse-{}{i}",
|
||||
if meta_only { "meta-" } else { "" }
|
||||
))
|
||||
let handle = std::thread::Builder::new()
|
||||
.name(name.clone())
|
||||
.spawn(move || {
|
||||
let cv = if meta_only {
|
||||
&inner.meta_cv
|
||||
@@ -121,17 +228,20 @@ impl Workers {
|
||||
};
|
||||
loop {
|
||||
let mut q = inner.queues.lock();
|
||||
let job = loop {
|
||||
let picked = loop {
|
||||
// A general worker takes transfers first: metadata
|
||||
// already has threads of its own, so draining the
|
||||
// bulk queue is the useful thing for it to do.
|
||||
let picked = if meta_only {
|
||||
q.meta.pop_front()
|
||||
let taken = if meta_only {
|
||||
q.meta.pop_front().map(|job| (Lane::Meta, job))
|
||||
} else {
|
||||
q.transfer.pop_front().or_else(|| q.meta.pop_front())
|
||||
q.transfer
|
||||
.pop_front()
|
||||
.map(|job| (Lane::Transfer, job))
|
||||
.or_else(|| q.meta.pop_front().map(|job| (Lane::Meta, job)))
|
||||
};
|
||||
if let Some(job) = picked {
|
||||
break Some(job);
|
||||
if let Some(taken) = taken {
|
||||
break Some(taken);
|
||||
}
|
||||
if q.closed {
|
||||
break None;
|
||||
@@ -139,7 +249,15 @@ impl Workers {
|
||||
cv.wait(&mut q);
|
||||
};
|
||||
drop(q);
|
||||
let Some(job) = job else { break };
|
||||
let Some((lane, queued)) = picked else { break };
|
||||
match lane {
|
||||
Lane::Meta => inner.meta_queued.fetch_sub(1, Ordering::Relaxed),
|
||||
Lane::Transfer => inner.transfer_queued.fetch_sub(1, Ordering::Relaxed),
|
||||
};
|
||||
let slot = &inner.slots[i];
|
||||
*slot.label.lock() = queued.label;
|
||||
slot.since_ms.store(inner.now_ms(), Ordering::Relaxed);
|
||||
slot.busy.store(true, Ordering::Relaxed);
|
||||
// A panicking handler must not cost the pool a worker
|
||||
// for the rest of the run. The dropped `Reply` answers
|
||||
// EIO on its own, so the caller of the failed op is
|
||||
@@ -164,11 +282,70 @@ impl Workers {
|
||||
// poisoning at all. Anything new reached from a job owes
|
||||
// the same check — an SDK-side `.lock().unwrap()`
|
||||
// silently invalidates this comment.
|
||||
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job));
|
||||
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(queued.job));
|
||||
slot.busy.store(false, Ordering::Relaxed);
|
||||
slot.since_ms.store(inner.now_ms(), Ordering::Relaxed);
|
||||
*slot.label.lock() = "";
|
||||
match lane {
|
||||
Lane::Meta => inner.meta_completed.fetch_add(1, Ordering::Relaxed),
|
||||
Lane::Transfer => {
|
||||
inner.transfer_completed.fetch_add(1, Ordering::Relaxed)
|
||||
}
|
||||
};
|
||||
}
|
||||
})?;
|
||||
handles.push(handle);
|
||||
}
|
||||
Ok(Self { inner })
|
||||
Ok(Self {
|
||||
inner,
|
||||
handles: Mutex::new(handles),
|
||||
})
|
||||
}
|
||||
|
||||
/// Stop accepting work, wake every worker, and wait up to `deadline` for the
|
||||
/// jobs in flight to finish.
|
||||
///
|
||||
/// Bounded because a join is only worth what it costs: a worker wedged on a
|
||||
/// network call would otherwise hold the whole teardown, and the user asked
|
||||
/// for the daemon to stop. What is stuck gets named in the log, and the
|
||||
/// process continues shutting down around it.
|
||||
pub(crate) fn stop_and_join(&self, deadline: std::time::Duration) {
|
||||
self.close();
|
||||
let handles: Vec<_> = std::mem::take(&mut *self.handles.lock());
|
||||
if handles.is_empty() {
|
||||
return;
|
||||
}
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let joiner = std::thread::Builder::new()
|
||||
.name("pdfs-fuse-join".into())
|
||||
.spawn(move || {
|
||||
for handle in handles {
|
||||
let _ = handle.join();
|
||||
}
|
||||
let _ = tx.send(());
|
||||
});
|
||||
if joiner.is_err() {
|
||||
return;
|
||||
}
|
||||
if rx.recv_timeout(deadline).is_err() {
|
||||
let busy: Vec<String> = self
|
||||
.snapshot()
|
||||
.workers
|
||||
.into_iter()
|
||||
.filter(|worker| worker.busy)
|
||||
.map(|worker| format!("{}={}", worker.name, worker.label))
|
||||
.collect();
|
||||
warn!(
|
||||
stuck = busy.join(","),
|
||||
"fuse workers did not finish in time; shutting down around them"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.inner.queues.lock().closed = true;
|
||||
self.inner.meta_cv.notify_all();
|
||||
self.inner.general_cv.notify_all();
|
||||
}
|
||||
|
||||
/// Queue `job` in `lane`.
|
||||
@@ -179,7 +356,15 @@ impl Workers {
|
||||
/// a pending job holds a `Reply` and a few fields, while the 4 MiB block
|
||||
/// buffer is allocated inside the job once it runs, and the SDK's in-flight
|
||||
/// semaphore is what bounds how many of those exist at once.
|
||||
pub(crate) fn run(&self, lane: Lane, job: impl FnOnce() + Send + 'static) {
|
||||
/// The queue is *not* bounded, and the depth warning above is deliberately
|
||||
/// all this does about it: the only caller able to feel backpressure here is
|
||||
/// fuser's dispatch loop, and blocking that is the stall the whole module
|
||||
/// exists to avoid. A depth that keeps climbing is a symptom to read in the
|
||||
/// log, not something to fix by stopping the mount.
|
||||
///
|
||||
/// `label` names the work for the diagnostics (`pdfs diagnostics`) and for
|
||||
/// the stall warnings: it is what a hung daemon reports instead of "busy".
|
||||
pub(crate) fn run(&self, lane: Lane, label: &'static str, job: impl FnOnce() + Send + 'static) {
|
||||
let mut q = self.inner.queues.lock();
|
||||
if q.closed {
|
||||
// Pre-pool behaviour: a shut-down pool degrades to a slow mount
|
||||
@@ -189,11 +374,31 @@ impl Workers {
|
||||
job();
|
||||
return;
|
||||
}
|
||||
match lane {
|
||||
Lane::Meta => q.meta.push_back(Box::new(job)),
|
||||
Lane::Transfer => q.transfer.push_back(Box::new(job)),
|
||||
}
|
||||
let queued = Queued {
|
||||
label,
|
||||
job: Box::new(job),
|
||||
};
|
||||
let depth = match lane {
|
||||
Lane::Meta => {
|
||||
q.meta.push_back(queued);
|
||||
self.inner.meta_queued.fetch_add(1, Ordering::Relaxed) + 1
|
||||
}
|
||||
Lane::Transfer => {
|
||||
q.transfer.push_back(queued);
|
||||
self.inner.transfer_queued.fetch_add(1, Ordering::Relaxed) + 1
|
||||
}
|
||||
};
|
||||
drop(q);
|
||||
if crosses_warn_step(depth) {
|
||||
warn!(
|
||||
lane = if matches!(lane, Lane::Meta) {
|
||||
"meta"
|
||||
} else {
|
||||
"transfer"
|
||||
},
|
||||
depth, label, "fuse worker queue is deep"
|
||||
);
|
||||
}
|
||||
match lane {
|
||||
// Either class can serve metadata, and only one of them needs to:
|
||||
// whichever wakes first takes it, and the other finds the queue
|
||||
@@ -209,13 +414,50 @@ impl Workers {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What every worker is doing right now, without taking the queue lock.
|
||||
///
|
||||
/// A diagnostic is worth having exactly when the daemon is stuck, so this
|
||||
/// reads atomics only, and `try_lock`s each label rather than waiting for
|
||||
/// it: a label is held for one pointer write, so failing to get it means a
|
||||
/// racing worker, not a wedged one, and `"?"` is a better answer than a
|
||||
/// diagnostics request that hangs too.
|
||||
pub(crate) fn snapshot(&self) -> PoolSnapshot {
|
||||
let now = self.inner.now_ms();
|
||||
let workers = self
|
||||
.inner
|
||||
.slots
|
||||
.iter()
|
||||
.map(|slot| {
|
||||
let label = slot
|
||||
.label
|
||||
.try_lock()
|
||||
.map_or_else(|| "?".to_string(), |label| (*label).to_string());
|
||||
WorkerSnapshot {
|
||||
name: slot.name.clone(),
|
||||
busy: slot.busy.load(Ordering::Relaxed),
|
||||
age_ms: now.saturating_sub(slot.since_ms.load(Ordering::Relaxed)),
|
||||
label,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
PoolSnapshot {
|
||||
workers,
|
||||
meta_queued: self.inner.meta_queued.load(Ordering::Relaxed),
|
||||
transfer_queued: self.inner.transfer_queued.load(Ordering::Relaxed),
|
||||
meta_completed: self.inner.meta_completed.load(Ordering::Relaxed),
|
||||
transfer_completed: self.inner.transfer_completed.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Workers {
|
||||
/// A backstop for the paths that never call
|
||||
/// [`stop_and_join`](Workers::stop_and_join) — tests, and a mount that fails
|
||||
/// before teardown is wired up. Flag and wake only: a `Drop` that blocks on
|
||||
/// a network call is worse than a worker outliving its pool by a moment.
|
||||
fn drop(&mut self) {
|
||||
self.inner.queues.lock().closed = true;
|
||||
self.inner.meta_cv.notify_all();
|
||||
self.inner.general_cv.notify_all();
|
||||
self.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,6 +467,64 @@ mod tests {
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// The depth warning fires on the step, and only on the step: a lane that
|
||||
/// oscillates around the threshold must not log on every single job.
|
||||
#[test]
|
||||
fn queue_depth_warns_on_each_step() {
|
||||
assert!(!crosses_warn_step(1));
|
||||
assert!(!crosses_warn_step(QUEUE_WARN_STEP - 1));
|
||||
assert!(crosses_warn_step(QUEUE_WARN_STEP));
|
||||
assert!(!crosses_warn_step(QUEUE_WARN_STEP + 1));
|
||||
assert!(crosses_warn_step(QUEUE_WARN_STEP * 3));
|
||||
}
|
||||
|
||||
/// Teardown must return even while a job is running, and must not leave the
|
||||
/// pool accepting work.
|
||||
#[test]
|
||||
fn stopping_is_bounded_and_closes_the_pool() {
|
||||
let pool = Workers::new(2).unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
pool.run(Lane::Meta, "test", move || tx.send(()).unwrap());
|
||||
rx.recv_timeout(Duration::from_secs(5)).expect("it runs");
|
||||
|
||||
pool.stop_and_join(Duration::from_secs(5));
|
||||
|
||||
// A job queued after the stop runs inline rather than disappearing.
|
||||
let (tx, rx) = mpsc::channel();
|
||||
pool.run(Lane::Meta, "test", move || tx.send(()).unwrap());
|
||||
rx.recv_timeout(Duration::from_secs(5))
|
||||
.expect("a closed pool serves inline");
|
||||
}
|
||||
|
||||
/// A diagnostic is only useful if it names the work. A running job must show
|
||||
/// up under its label, and the queue behind it must show up as depth.
|
||||
#[test]
|
||||
fn a_snapshot_names_the_running_job_and_the_queue_behind_it() {
|
||||
let pool = Workers::new(2).unwrap();
|
||||
let (release_tx, release_rx) = mpsc::channel::<()>();
|
||||
let release_rx = Arc::new(Mutex::new(release_rx));
|
||||
let (started_tx, started_rx) = mpsc::channel();
|
||||
{
|
||||
let release_rx = release_rx.clone();
|
||||
pool.run(Lane::Transfer, "read", move || {
|
||||
started_tx.send(()).unwrap();
|
||||
let _ = release_rx.lock().recv();
|
||||
});
|
||||
}
|
||||
started_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.expect("the job starts");
|
||||
|
||||
let snapshot = pool.snapshot();
|
||||
assert!(
|
||||
snapshot.workers.iter().any(|w| w.busy && w.label == "read"),
|
||||
"the running job is named"
|
||||
);
|
||||
assert_eq!(snapshot.transfer_completed, 0);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
}
|
||||
|
||||
/// Audit A6. Transfers filling every general worker must not delay a
|
||||
/// metadata job: that is what the reserved threads are for.
|
||||
#[test]
|
||||
@@ -240,7 +540,7 @@ mod tests {
|
||||
for _ in 0..general {
|
||||
let release_rx = release_rx.clone();
|
||||
let started_tx = started_tx.clone();
|
||||
pool.run(Lane::Transfer, move || {
|
||||
pool.run(Lane::Transfer, "test", move || {
|
||||
started_tx.send(()).unwrap();
|
||||
let _ = release_rx.lock().recv();
|
||||
});
|
||||
@@ -255,13 +555,13 @@ mod tests {
|
||||
// in the way that used to starve metadata.
|
||||
for _ in 0..16 {
|
||||
let release_rx = release_rx.clone();
|
||||
pool.run(Lane::Transfer, move || {
|
||||
pool.run(Lane::Transfer, "test", move || {
|
||||
let _ = release_rx.lock().recv();
|
||||
});
|
||||
}
|
||||
|
||||
let (meta_tx, meta_rx) = mpsc::channel();
|
||||
pool.run(Lane::Meta, move || meta_tx.send(()).unwrap());
|
||||
pool.run(Lane::Meta, "test", move || meta_tx.send(()).unwrap());
|
||||
meta_rx
|
||||
.recv_timeout(Duration::from_secs(5))
|
||||
.expect("a metadata job runs while every transfer thread is stuck");
|
||||
@@ -279,7 +579,7 @@ mod tests {
|
||||
// must have been served by the general worker.
|
||||
let pool = Workers::new(1).unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
pool.run(Lane::Meta, move || tx.send(()).unwrap());
|
||||
pool.run(Lane::Meta, "test", move || tx.send(()).unwrap());
|
||||
rx.recv_timeout(Duration::from_secs(5))
|
||||
.expect("a general worker takes metadata when no transfer is waiting");
|
||||
}
|
||||
@@ -288,9 +588,9 @@ mod tests {
|
||||
#[test]
|
||||
fn a_panicking_job_does_not_retire_its_worker() {
|
||||
let pool = Workers::new(1).unwrap();
|
||||
pool.run(Lane::Meta, || panic!("boom"));
|
||||
pool.run(Lane::Meta, "test", || panic!("boom"));
|
||||
let (tx, rx) = mpsc::channel();
|
||||
pool.run(Lane::Meta, move || tx.send(()).unwrap());
|
||||
pool.run(Lane::Meta, "test", move || tx.send(()).unwrap());
|
||||
rx.recv_timeout(Duration::from_secs(5))
|
||||
.expect("the pool still serves work after a panic");
|
||||
}
|
||||
|
||||
+223
-41
@@ -78,6 +78,15 @@ const CONNECT_RETRY_INTERVAL: Duration = Duration::from_secs(2);
|
||||
/// reload by clearing the page's timestamp.
|
||||
const PAGE_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// How often the gallery asks whether the timeline refresh it started has
|
||||
/// finished.
|
||||
const PHOTOS_REFRESH_POLL: Duration = Duration::from_secs(2);
|
||||
|
||||
/// How long the gallery keeps following one timeline refresh. A library of tens
|
||||
/// of thousands of photos is re-read in batches over several minutes; past this
|
||||
/// the daemon carries on and the page catches up on its next load.
|
||||
const PHOTOS_REFRESH_FOLLOW_LIMIT: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
/// All widgets the periodic refresh and the action handlers mutate, plus the
|
||||
/// resolved paths they act on. Wrapped in an [`Rc`] so handlers and the timeout
|
||||
/// closure share one instance.
|
||||
@@ -162,6 +171,22 @@ impl Ui {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark a cached thumbnail as freshly used, so eviction takes the tiles
|
||||
/// nobody has looked at rather than the oldest ones.
|
||||
///
|
||||
/// Insertion order alone evicts the top of the timeline first — exactly what
|
||||
/// a scroll back up then asks for again.
|
||||
fn touch_texture(&self, uid: &str) {
|
||||
let mut order = self.gallery.photo_tex_order.borrow_mut();
|
||||
if order.back().is_some_and(|back| back == uid) {
|
||||
return;
|
||||
}
|
||||
if let Some(at) = order.iter().position(|held| held == uid) {
|
||||
order.remove(at);
|
||||
order.push_back(uid.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> glib::ExitCode {
|
||||
@@ -237,15 +262,19 @@ fn load_proton_theme() {
|
||||
.viewer-status {{ color: white; font-size: 1rem; background-color: rgba(0, 0, 0, 0.75); padding: 12px 24px; border-radius: 12px; }}\n\
|
||||
.viewer-info-panel {{ background-color: @window_bg_color; border-left: 1px solid alpha(currentColor, 0.12); }}\n\
|
||||
.gallery-day {{ font-size: 0.82rem; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: alpha(currentColor, 0.65); padding: 4px 2px 8px 2px; }}\n\
|
||||
.photo-tile {{ padding: 0; margin: 0; min-height: 0; min-width: 0; border-radius: 14px; background: alpha(currentColor, 0.06); box-shadow: none; transition: box-shadow 180ms ease, background 180ms ease; }}\n\
|
||||
.photo-tile {{ padding: 0; margin: 0; min-height: 0; min-width: 0; border-radius: 6px; background: none; box-shadow: none; transition: box-shadow 180ms ease; }}\n\
|
||||
.photo-tile:hover {{ box-shadow: 0 8px 22px rgba(0, 0, 0, 0.45); }}\n\
|
||||
.photo-tile:focus {{ outline: 2px solid {PROTON_PURPLE}; outline-offset: -2px; }}\n\
|
||||
.photo-thumb {{ transition: transform 220ms ease; }}\n\
|
||||
.photo-tile:hover .photo-thumb {{ transform: scale(1.06); }}\n\
|
||||
.photo-check {{ color: white; background: rgba(0, 0, 0, 0.5); border-radius: 999px; padding: 3px; }}\n\
|
||||
.photo-check-on {{ background: {PROTON_PURPLE}; }}\n\
|
||||
.photo-tile-selected {{ outline: 3px solid {PROTON_PURPLE}; outline-offset: -3px; }}\n\
|
||||
.photo-placeholder {{ color: alpha(currentColor, 0.35); background: alpha(currentColor, 0.07); }}\n\
|
||||
.photo-caption {{ font-size: 0.78rem; color: white; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); padding: 22px 10px 6px 10px; opacity: 0; transition: opacity 180ms ease; }}\n\
|
||||
.photo-video-badge {{ color: white; background: rgba(0, 0, 0, 0.45); border-radius: 999px; padding: 8px; min-width: 20px; min-height: 20px; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.5); transition: background 160ms ease; }}\n\
|
||||
.photo-tile:hover .photo-video-badge {{ background: alpha({PROTON_PURPLE}, 0.85); }}\n\
|
||||
.photo-group-badge {{ font-size: 0.68rem; font-weight: 700; letter-spacing: 0.04em; color: white; background: rgba(0, 0, 0, 0.55); border-radius: 6px; padding: 1px 5px; margin: 6px; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.9); }}\n\
|
||||
.photo-tile:hover .photo-caption {{ opacity: 1; background: linear-gradient(to top, rgba(0, 0, 0, 0.55), rgba(0, 0, 0, 0)); }}\n\
|
||||
.album-card {{ padding: 0; border-radius: 16px; transition: background 180ms ease; }}\n\
|
||||
.album-card:hover {{ background: alpha(currentColor, 0.07); }}\n\
|
||||
@@ -443,7 +472,11 @@ fn build_window(app: &adw::Application) {
|
||||
gallery: GalleryState {
|
||||
model: gallery_widgets.model.clone(),
|
||||
groups: gallery_widgets.groups.clone(),
|
||||
tile: Cell::new(TILE_DEFAULT),
|
||||
row_height: Cell::new(ROW_DEFAULT),
|
||||
learned_ratios: RefCell::new(HashMap::new()),
|
||||
scrolling_down: Cell::new(true),
|
||||
scroll_offset: Cell::new(0.0),
|
||||
assumed_ratios: RefCell::new(HashSet::new()),
|
||||
content: gallery_widgets.content.clone(),
|
||||
status: gallery_widgets.status.clone(),
|
||||
retry: gallery_widgets.retry.clone(),
|
||||
@@ -486,6 +519,12 @@ fn build_window(app: &adw::Application) {
|
||||
relayout_source: RefCell::new(None),
|
||||
bound: RefCell::new(BTreeMap::new()),
|
||||
list: gallery_widgets.list.clone(),
|
||||
selecting: Cell::new(false),
|
||||
selected: RefCell::new(HashSet::new()),
|
||||
select_btn: gallery_widgets.select_btn.clone(),
|
||||
select_bar: gallery_widgets.select_bar.clone(),
|
||||
select_label: gallery_widgets.select_label.clone(),
|
||||
select_trash: gallery_widgets.select_trash.clone(),
|
||||
},
|
||||
shared: SharedState {
|
||||
content: shared_widgets.content.clone(),
|
||||
@@ -583,7 +622,12 @@ fn build_window(app: &adw::Application) {
|
||||
);
|
||||
wire_details(&ui);
|
||||
wire_search(&ui);
|
||||
wire_gallery(&ui, &gallery_widgets.list, &gallery_widgets.scroll);
|
||||
wire_gallery(
|
||||
&ui,
|
||||
&gallery_widgets.list,
|
||||
&gallery_widgets.scroll,
|
||||
&gallery_widgets.select_done,
|
||||
);
|
||||
wire_gallery_empty(
|
||||
&ui,
|
||||
&gallery_widgets.empty_upload,
|
||||
@@ -804,7 +848,7 @@ fn reload_current_page(ui: &Rc<Ui>) {
|
||||
}
|
||||
// One scope covers the whole photos view, so Refresh reloads whichever of
|
||||
// the two — album grid or timeline/album — is actually on screen.
|
||||
Some("gallery") => refresh_then(ui, RefreshScope::Photos, |ui| {
|
||||
Some("gallery") => refresh_then(ui, RefreshScope::Photos { full: false }, |ui| {
|
||||
if ui.gallery.content.visible_child_name().as_deref() == Some("albums") {
|
||||
load_albums(ui);
|
||||
} else {
|
||||
@@ -839,11 +883,54 @@ fn reload_current_page(ui: &Rc<Ui>) {
|
||||
/// how to report an unreachable daemon on its own page, and a refresh that fails
|
||||
/// silently would read as a dead button.
|
||||
fn refresh_then(ui: &Rc<Ui>, scope: RefreshScope, load: fn(&Rc<Ui>)) {
|
||||
// A photos refresh answers as soon as it has started, because re-reading a
|
||||
// large library takes minutes. The page is therefore loaded twice: once from
|
||||
// what the daemon already holds, and again when the refresh has landed.
|
||||
let follow = matches!(scope, RefreshScope::Photos { .. });
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::Refresh { scope });
|
||||
let ui = ui.clone();
|
||||
glib::spawn_future_local(async move {
|
||||
let _ = rx.recv().await;
|
||||
load(&ui);
|
||||
if follow {
|
||||
follow_photos_refresh(&ui, load);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Watch the timeline refresh the daemon just started, and load the page again
|
||||
/// when it finishes.
|
||||
///
|
||||
/// Giving up after [`PHOTOS_REFRESH_FOLLOW_LIMIT`] is not a failure: the refresh
|
||||
/// keeps running in the daemon and the next visit to the page shows its result.
|
||||
/// It only stops this front-end from polling a daemon forever.
|
||||
fn follow_photos_refresh(ui: &Rc<Ui>, load: fn(&Rc<Ui>)) {
|
||||
let ui = ui.clone();
|
||||
let give_up_at = Instant::now() + PHOTOS_REFRESH_FOLLOW_LIMIT;
|
||||
glib::spawn_future_local(async move {
|
||||
loop {
|
||||
glib::timeout_future(PHOTOS_REFRESH_POLL).await;
|
||||
// The person may have left the gallery in the meantime; reloading a
|
||||
// page nobody is looking at would only cost the daemon work.
|
||||
if ui.stack.visible_child_name().as_deref() != Some("gallery") {
|
||||
return;
|
||||
}
|
||||
let rx = spawn_request(ui.dirs.control_socket(), Request::PhotosRefreshStatus);
|
||||
match rx.recv().await {
|
||||
Ok(Ok(Response::PhotosRefresh { running: true })) => {
|
||||
if Instant::now() >= give_up_at {
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(Ok(Response::PhotosRefresh { running: false })) => {
|
||||
load(&ui);
|
||||
return;
|
||||
}
|
||||
// An older daemon does not know the request, and anything else
|
||||
// means we cannot tell — either way, stop asking.
|
||||
_ => return,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1170,54 +1257,149 @@ fn human_bytes(bytes: u64) -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The full width one row of `columns` tiles occupies, gaps included.
|
||||
fn row_width(columns: usize, edge: i32) -> i32 {
|
||||
edge * columns as i32 + TILE_GAP * (columns as i32 - 1)
|
||||
/// The full width a row occupies, gaps included.
|
||||
fn row_width(widths: &[i32]) -> i32 {
|
||||
widths.iter().sum::<i32>() + TILE_GAP * (widths.len() as i32 - 1)
|
||||
}
|
||||
|
||||
/// Ratios of a typical mixed day: landscape phone shots, a portrait, a
|
||||
/// square crop.
|
||||
fn mixed() -> Vec<f64> {
|
||||
vec![
|
||||
1.5, 1.5, 0.75, 1.0, 1.33, 1.5, 0.75, 1.78, 1.0, 1.5, 1.5, 1.33,
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_grid_spans_the_content_width() {
|
||||
// Whatever the width, the columns plus their gaps land on it (bar the
|
||||
// few px integer division cannot divide), so no ragged right margin.
|
||||
fn every_full_row_spans_the_content_width() {
|
||||
// The point of justifying: no ragged right margin at any width, at any
|
||||
// zoom, for any mix of shapes.
|
||||
for width in [640, 900, 1000, 1440, 1920, 2560] {
|
||||
let (columns, edge) = plan_grid(TILE_DEFAULT, width);
|
||||
let used = row_width(columns, edge);
|
||||
assert!(
|
||||
used <= width && width - used < columns as i32,
|
||||
"{columns} x {edge}px = {used}px does not span {width}px"
|
||||
);
|
||||
let rows = plan_rows(&mixed(), width, ROW_DEFAULT);
|
||||
// The last row is deliberately short, so it is not part of this.
|
||||
for (_, widths) in rows.iter().take(rows.len() - 1) {
|
||||
assert_eq!(row_width(widths), width, "row does not span {width}px");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiles_stay_near_the_target_size() {
|
||||
// The column count rounds and the edge absorbs the remainder, so a tile
|
||||
// is never more than one gap-and-a-bit away from what was asked for.
|
||||
let (columns, edge) = plan_grid(TILE_DEFAULT, 1920);
|
||||
assert!(columns >= 7, "1920px should hold several 220px tiles");
|
||||
assert!(
|
||||
(edge - TILE_DEFAULT).abs() < TILE_DEFAULT / 3,
|
||||
"{edge}px is not near the {TILE_DEFAULT}px target"
|
||||
fn a_tile_keeps_its_photos_shape() {
|
||||
// Each tile is its photo's ratio at the row's height, which is what
|
||||
// makes cropping unnecessary.
|
||||
let ratios = mixed();
|
||||
let rows = plan_rows(&ratios, 1440, ROW_DEFAULT);
|
||||
let (height, widths) = &rows[0];
|
||||
for (width, ratio) in widths.iter().zip(&ratios).take(widths.len() - 1) {
|
||||
let laid_out = *width as f64 / *height as f64;
|
||||
assert!((laid_out - ratio).abs() < 0.02, "{laid_out} is not {ratio}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rows_land_near_the_target_height() {
|
||||
// A row is closed as soon as it no longer fits at the target, so it is
|
||||
// never taller than the target and never far below it.
|
||||
for target in [ROW_MIN, ROW_DEFAULT, ROW_MAX] {
|
||||
let rows = plan_rows(&mixed(), 1600, target);
|
||||
for (height, _) in rows.iter().take(rows.len() - 1) {
|
||||
assert!(
|
||||
*height <= target,
|
||||
"{height}px exceeds the {target}px target"
|
||||
);
|
||||
assert!(*height > target / 2, "{height}px is far under {target}px");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_short_day_is_not_stretched_across_the_window() {
|
||||
// Two photos are two photos, not two half-window tiles.
|
||||
let rows = plan_rows(&[1.5, 1.5], 1920, ROW_DEFAULT);
|
||||
assert_eq!(rows.len(), 1);
|
||||
let (height, widths) = &rows[0];
|
||||
assert_eq!(*height, ROW_DEFAULT);
|
||||
assert!(row_width(widths) < 1920 / 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zooming_in_puts_fewer_photos_in_a_row() {
|
||||
let small = plan_rows(&mixed(), 1200, ROW_MIN);
|
||||
let big = plan_rows(&mixed(), 1200, ROW_MAX);
|
||||
assert!(small[0].1.len() > big[0].1.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_photo_is_laid_out_exactly_once() {
|
||||
let ratios = mixed();
|
||||
let placed: usize = plan_rows(&ratios, 1000, ROW_DEFAULT)
|
||||
.iter()
|
||||
.map(|(_, widths)| widths.len())
|
||||
.sum();
|
||||
assert_eq!(placed, ratios.len());
|
||||
assert!(plan_rows(&[], 1000, ROW_DEFAULT).is_empty());
|
||||
}
|
||||
|
||||
/// One file of a shot, for the group-switch tests.
|
||||
fn member(uid: &str, name: Option<&str>, kind: PhotoKind) -> PhotoItem {
|
||||
PhotoItem {
|
||||
uid: uid.into(),
|
||||
capture_time: 0,
|
||||
thumb_path: None,
|
||||
name: name.map(str::to_string),
|
||||
ratio: None,
|
||||
no_thumb: false,
|
||||
kind,
|
||||
favorite: false,
|
||||
group_size: 2,
|
||||
has_raw: kind == PhotoKind::Raw,
|
||||
}
|
||||
}
|
||||
|
||||
/// The lightbox steps through a shot's files and comes back round, so a
|
||||
/// burst of three is as reachable as a RAW+JPEG pair.
|
||||
#[test]
|
||||
fn the_group_switch_cycles_through_every_file() {
|
||||
let members = [
|
||||
member("jpeg", Some("IMG_1.JPG"), PhotoKind::Photo),
|
||||
member("raw", Some("IMG_1.CR2"), PhotoKind::Raw),
|
||||
member("clip", None, PhotoKind::Video),
|
||||
];
|
||||
let next = |uid| next_group_member(&members, uid).map(|item| item.uid.as_str());
|
||||
assert_eq!(next("jpeg"), Some("raw"));
|
||||
assert_eq!(next("raw"), Some("clip"));
|
||||
assert_eq!(next("clip"), Some("jpeg"));
|
||||
// A photo that is not in the group leaves the button alone rather than
|
||||
// jumping somewhere arbitrary.
|
||||
assert_eq!(next("elsewhere"), None);
|
||||
assert!(next_group_member(&[], "jpeg").is_none());
|
||||
}
|
||||
|
||||
/// The switch names what it will show; a file whose name the daemon has not
|
||||
/// resolved still gets a sentence rather than a blank.
|
||||
#[test]
|
||||
fn a_group_member_without_a_name_is_still_described() {
|
||||
assert_eq!(
|
||||
member_label(&member("raw", Some("IMG_1.CR2"), PhotoKind::Raw)),
|
||||
"IMG_1.CR2"
|
||||
);
|
||||
assert_eq!(
|
||||
member_label(&member("raw", None, PhotoKind::Raw)),
|
||||
"the raw file"
|
||||
);
|
||||
assert_eq!(
|
||||
member_label(&member("clip", Some(""), PhotoKind::Video)),
|
||||
"the video"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zooming_in_widens_the_tiles_and_drops_columns() {
|
||||
let (wide_columns, small_edge) = plan_grid(TILE_MIN, 1200);
|
||||
let (few_columns, big_edge) = plan_grid(TILE_MAX, 1200);
|
||||
assert!(wide_columns > few_columns);
|
||||
assert!(big_edge > small_edge);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_narrower_than_one_tile_still_gets_a_column() {
|
||||
// A single column at whatever fits, rather than zero columns (which would
|
||||
// divide by zero) or a zero-px tile.
|
||||
let (columns, edge) = plan_grid(TILE_DEFAULT, 40);
|
||||
assert_eq!(columns, 1);
|
||||
assert!(edge > 0);
|
||||
let (columns, edge) = plan_grid(TILE_DEFAULT, 0);
|
||||
assert_eq!(columns, 1);
|
||||
assert!(edge > 0);
|
||||
fn a_window_narrower_than_one_photo_still_lays_it_out() {
|
||||
// A tile of at least one px, rather than a zero-width widget or a
|
||||
// division by zero.
|
||||
for width in [0, 40] {
|
||||
let rows = plan_rows(&[1.5, 1.5, 1.5], width, ROW_DEFAULT);
|
||||
assert!(rows.iter().all(|(h, w)| *h > 0 && w.iter().all(|w| *w > 0)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,11 @@ pub(crate) struct Viewer {
|
||||
/// uses for its pin switch, so painting a photo doesn't fire a round-trip.
|
||||
pub(crate) favorite: gtk4::ToggleButton,
|
||||
pub(crate) favorite_suppress: Cell<bool>,
|
||||
/// The button that switches between the files of one shot, and the files it
|
||||
/// switches between (the JPEG and the RAW of a photo, a live photo and its
|
||||
/// clip). Empty, and the button hidden, for a photo stored as one file.
|
||||
pub(crate) group_btn: gtk4::Button,
|
||||
pub(crate) group: RefCell<Vec<PhotoItem>>,
|
||||
/// uid of the photo currently on screen.
|
||||
pub(crate) uid: RefCell<String>,
|
||||
/// On-disk path of the full-size photo, once it has been downloaded.
|
||||
@@ -81,6 +86,59 @@ fn set_gallery_favorite(ui: &Rc<Ui>, uid: &str, favorite: bool) {
|
||||
/// Show the photo behind `uid`: paint its (already cached) thumbnail immediately
|
||||
/// so the lightbox never opens on a blank screen, ask the daemon for the
|
||||
/// full-size file, and swap it in — plus its EXIF — when it lands.
|
||||
/// Ask the daemon which files this shot was stored as, and offer a switch when
|
||||
/// there is more than one. The grid shows such a shot as a single tile, so the
|
||||
/// lightbox is the only place the RAW is reachable.
|
||||
fn load_photo_group(ui: &Rc<Ui>, viewer: &Rc<Viewer>, uid: &str) {
|
||||
viewer.group.borrow_mut().clear();
|
||||
viewer.group_btn.set_visible(false);
|
||||
let rx = spawn_request(
|
||||
ui.dirs.control_socket(),
|
||||
Request::PhotoGroup {
|
||||
uid: uid.to_string(),
|
||||
},
|
||||
);
|
||||
let viewer = viewer.clone();
|
||||
let uid = uid.to_string();
|
||||
glib::spawn_future_local(async move {
|
||||
let Ok(Ok(Response::Photos { items, .. })) = rx.recv().await else {
|
||||
return;
|
||||
};
|
||||
// A late reply for a photo the user has already left must not relabel
|
||||
// the button under the one now on screen.
|
||||
if *viewer.uid.borrow() != uid || items.len() < 2 {
|
||||
return;
|
||||
}
|
||||
if let Some(next) = next_group_member(&items, &uid) {
|
||||
viewer
|
||||
.group_btn
|
||||
.set_tooltip_text(Some(&format!("Show {}", member_label(next))));
|
||||
}
|
||||
*viewer.group.borrow_mut() = items;
|
||||
viewer.group_btn.set_visible(true);
|
||||
});
|
||||
}
|
||||
|
||||
/// The member after `uid`, wrapping round — the button steps through a shot's
|
||||
/// files rather than toggling two of them, so a burst of three works too.
|
||||
pub(crate) fn next_group_member<'a>(members: &'a [PhotoItem], uid: &str) -> Option<&'a PhotoItem> {
|
||||
let at = members.iter().position(|item| item.uid == uid)?;
|
||||
members.get((at + 1) % members.len())
|
||||
}
|
||||
|
||||
/// What to call one file of a shot: its name, or its kind when the daemon has
|
||||
/// not resolved a name for it.
|
||||
pub(crate) fn member_label(item: &PhotoItem) -> String {
|
||||
match item.name.as_deref() {
|
||||
Some(name) if !name.is_empty() => name.to_string(),
|
||||
_ => match item.kind {
|
||||
PhotoKind::Raw => "the raw file".to_string(),
|
||||
PhotoKind::Video => "the video".to_string(),
|
||||
PhotoKind::Photo => "the photo".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn load_photo(ui: &Rc<Ui>, viewer: &Rc<Viewer>, uid: String) {
|
||||
viewer.spinner.set_visible(true);
|
||||
viewer.spinner.start();
|
||||
@@ -96,6 +154,7 @@ pub(crate) fn load_photo(ui: &Rc<Ui>, viewer: &Rc<Viewer>, uid: String) {
|
||||
.and_downcast::<BoxedAnyObject>()
|
||||
.is_some_and(|boxed| boxed.borrow::<PhotoItem>().favorite);
|
||||
show_favorite(viewer, favorite);
|
||||
load_photo_group(ui, viewer, &uid);
|
||||
|
||||
// The thumbnail the gallery already decoded stands in for the full photo
|
||||
// while it downloads: blurry for a moment beats black for a second.
|
||||
@@ -503,7 +562,13 @@ pub(crate) fn open_photo_viewer(ui: &Rc<Ui>, initial_uid: String) {
|
||||
favorite_btn.add_css_class("flat");
|
||||
favorite_btn.add_css_class("viewer-action-btn");
|
||||
|
||||
// Only shown for a shot stored as more than one file; `load_photo_group`
|
||||
// decides that per photo.
|
||||
let group_btn = action("view-paged-symbolic", "Other files of this photo");
|
||||
group_btn.set_visible(false);
|
||||
|
||||
let download_btn = action("document-save-symbolic", "Save a copy…");
|
||||
let delete_btn = action("user-trash-symbolic", "Move to Trash (Delete)");
|
||||
let open_ext_btn = action("document-open-symbolic", "Open with another app");
|
||||
let close_btn = action("window-close-symbolic", "Close (Esc)");
|
||||
close_btn.add_css_class("viewer-close-btn");
|
||||
@@ -513,9 +578,11 @@ pub(crate) fn open_photo_viewer(ui: &Rc<Ui>, initial_uid: String) {
|
||||
top_bar.set_valign(gtk4::Align::Start);
|
||||
top_bar.set_hexpand(true);
|
||||
top_bar.append(&titles);
|
||||
top_bar.append(&group_btn);
|
||||
top_bar.append(&favorite_btn);
|
||||
top_bar.append(&info_toggle);
|
||||
top_bar.append(&download_btn);
|
||||
top_bar.append(&delete_btn);
|
||||
top_bar.append(&open_ext_btn);
|
||||
top_bar.append(&close_btn);
|
||||
overlay.add_overlay(&top_bar);
|
||||
@@ -614,6 +681,8 @@ pub(crate) fn open_photo_viewer(ui: &Rc<Ui>, initial_uid: String) {
|
||||
info_map: info_map.clone(),
|
||||
favorite: favorite_btn.clone(),
|
||||
favorite_suppress: Cell::new(false),
|
||||
group_btn: group_btn.clone(),
|
||||
group: RefCell::new(Vec::new()),
|
||||
coords: RefCell::new(None),
|
||||
uid: RefCell::new(initial_uid.clone()),
|
||||
path: RefCell::new(None),
|
||||
@@ -695,6 +764,30 @@ pub(crate) fn open_photo_viewer(ui: &Rc<Ui>, initial_uid: String) {
|
||||
});
|
||||
});
|
||||
|
||||
// Deleting the photo on screen closes the lightbox: what it was showing is
|
||||
// gone, and leaving it open on a trashed photo invites a second Delete on
|
||||
// something the user cannot see.
|
||||
let ui_delete = ui.clone();
|
||||
let viewer_delete = viewer.clone();
|
||||
let w_delete = window.clone();
|
||||
delete_btn.connect_clicked(move |_| {
|
||||
let uid = viewer_delete.uid.borrow().clone();
|
||||
w_delete.close();
|
||||
trash_photos(&ui_delete, vec![uid]);
|
||||
});
|
||||
|
||||
let ui_group = ui.clone();
|
||||
let viewer_group = viewer.clone();
|
||||
group_btn.connect_clicked(move |_| {
|
||||
let uid = viewer_group.uid.borrow().clone();
|
||||
let next =
|
||||
next_group_member(&viewer_group.group.borrow(), &uid).map(|item| item.uid.clone());
|
||||
if let Some(next) = next {
|
||||
*viewer_group.uid.borrow_mut() = next.clone();
|
||||
load_photo(&ui_group, &viewer_group, next);
|
||||
}
|
||||
});
|
||||
|
||||
let w_download = window.clone();
|
||||
let viewer_download = viewer.clone();
|
||||
download_btn.connect_clicked(move |_| {
|
||||
@@ -757,6 +850,11 @@ pub(crate) fn open_photo_viewer(ui: &Rc<Ui>, initial_uid: String) {
|
||||
w_key.fullscreen();
|
||||
}
|
||||
}
|
||||
Some("Delete" | "KP_Delete") => {
|
||||
let uid = viewer_key.uid.borrow().clone();
|
||||
w_key.close();
|
||||
trash_photos(&ui_key, vec![uid]);
|
||||
}
|
||||
Some("Escape" | "q") => w_key.close(),
|
||||
Some("w") if ctrl => w_key.close(),
|
||||
_ => return glib::Propagation::Proceed,
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+330
@@ -12,6 +12,336 @@ Conventions:
|
||||
|
||||
---
|
||||
|
||||
## B97 — Photo refreshes stack up on the control socket, and shutdown turns the resolve pass into a burst of "undecryptable photo" warnings
|
||||
|
||||
**Status:** Fixed (unverified) — the daemon carrying the change has not been driven against a real
|
||||
library yet.
|
||||
**Found:** 2026-09-19, while reading the journal for issues that were still open. `pdfs diagnostics`
|
||||
had reported a `Refresh` running for 23 minutes with `in_flight` at 5, and every mount that ended
|
||||
in a `systemctl --user restart` was followed by ~95 `skipping undecryptable photo` warnings.
|
||||
|
||||
**Where:** `crates/pdfs-fuse/src/control.rs` (`Refresh` handler), `crates/pdfs-fuse/src/photos.rs`
|
||||
(`refresh_timeline`), `crates/pdfs-fuse/src/diagnostics.rs` (`request_kind`).
|
||||
|
||||
**Cause.** Two unrelated faults with one symptom each.
|
||||
|
||||
The stall: every `Refresh` arm is individually cheap — a directory refresh resolves and
|
||||
invalidates one listing, a photos refresh clears one state key and spawns the real work — but
|
||||
nothing stopped several of them running at once. A front end that asks again while the first
|
||||
refresh is still going gets a second handler that does the whole thing again: `invalidate_photos`,
|
||||
optionally `photos_mark_all_unresolved` over the entire timeline, `invalidate_albums`. Each of
|
||||
those takes the database lock the already-running refresh holds for minutes at a time, so the
|
||||
duplicates queue behind it, hold a control handler slot, and appear as a wedged `Refresh`. The
|
||||
report could not even say which view was wedged: `request_kind` reads only serde's external tag,
|
||||
so all three scopes logged as a bare `Refresh`.
|
||||
|
||||
The warning burst: `refresh_timeline` resolves photo metadata in chunks and keeps going through
|
||||
teardown. The runtime is cancelled under it, every decrypt task in flight fails with "task was
|
||||
cancelled", and the SDK logs one `skipping undecryptable photo` per node. Nothing was corrupt —
|
||||
the library was fine on the next mount, because `photos_replace` preserves prior metadata and
|
||||
leaves `resolved_at` null so the next refresh re-reads exactly those photos.
|
||||
|
||||
**What changed.** A refresh scope is now single-flight: `RefreshGuard` holds the scope key
|
||||
(`dir:<path>`, `trash`, `photos`, `photos:full`) for as long as one is being served, and a second
|
||||
request for that same key is answered `refreshing` immediately instead of repeating the work. A
|
||||
non-full photos refresh is additionally skipped while `timeline_refreshing()` is true, since that
|
||||
work outlives the request that started it. `request_kind` now names the scope, so the diagnostics
|
||||
report says `Refresh(Photos)` rather than `Refresh`. And the resolve loop stops asking for chunks
|
||||
once `Shutdown::is_stopping()`, logging `stopped` and the resolved-versus-wanted counts on the way
|
||||
out.
|
||||
|
||||
---
|
||||
|
||||
## B96 — Staged bytes that name no node are never retired, and are re-announced on every boot
|
||||
|
||||
**Status:** Fixed (unverified).
|
||||
**Found:** 2026-09-19, in the journal: the same "staged bytes belong to no queued upload" warning
|
||||
appeared once per blob per mount, forever. The staging directory held 138 blobs / 575 MiB, of
|
||||
which 95 (~445 MiB) referenced no node at all; the oldest were two months old.
|
||||
|
||||
**Where:** `crates/pdfs-fuse/src/lib.rs` (`reconcile_staging`).
|
||||
|
||||
**Cause.** Startup reconciliation is deliberately conservative: staged bytes are the only copy of
|
||||
an un-uploaded write, so a blob it cannot match to a queued op is kept rather than deleted. But
|
||||
"kept" had no end. A blob whose node is gone from the database can never be matched by any future
|
||||
boot, so the warning was both permanent and per-blob, and the bytes accumulated with no ceiling.
|
||||
|
||||
**What changed.** A blob that names no node and whose mtime is older than `STAGING_ORPHAN_RETAIN`
|
||||
(30 days) is retired, with its size logged. A month is long enough that a write stranded by a bug
|
||||
can still be recovered by hand. Younger orphans are still kept, but they are now counted and
|
||||
reported as one aggregated warning with one example path, not one line each.
|
||||
|
||||
---
|
||||
|
||||
## B95 — A real `index.tmp` is treated as a browser tempfile and parked forever
|
||||
|
||||
**Status:** Fixed (unverified).
|
||||
**Found:** 2026-09-19, while working out why 43 uploads were parked. Most of them were
|
||||
`index.tmp` and `preview.tmp` files from a Google Photos Takeout import — ordinary files whose
|
||||
final name simply ends in `.tmp`.
|
||||
|
||||
**Where:** `crates/pdfs-core/src/syncignore.rs` (`TRANSIENT_SUFFIXES`).
|
||||
|
||||
**Cause.** One suffix list served two mechanisms with very different consequences. On a mirror
|
||||
mount, calling a name transient means "do not reconcile it yet", which is reversible and costs
|
||||
nothing if wrong. On an on-demand mount it means "park the create until the writer renames it"
|
||||
(B70) — and a file whose real, final name ends in `.tmp` is never renamed, so the park stands
|
||||
forever and the bytes never go up.
|
||||
|
||||
**What changed.** `.tmp` and `.temp` are out of `TRANSIENT_SUFFIXES`. The remaining suffixes
|
||||
(`.crdownload`, `.part`, `.partial`, `.download`, `.swp`, `.swx`) belong to writers that always
|
||||
rename. A `.tmp` write from a program that does rename is still handled — by the rename itself,
|
||||
which supersedes the create.
|
||||
|
||||
---
|
||||
|
||||
## B94 — A parked create has no exit but the rename, so an abandoned one is stuck forever
|
||||
|
||||
**Status:** Fixed (unverified).
|
||||
**Found:** 2026-09-19, while reading the journal: 43 rows in `pending_op` carried the
|
||||
`PARK_UNTIL` sentinel, some for weeks. `pdfs status` reported them as parked uploads that never
|
||||
moved.
|
||||
|
||||
**Where:** `crates/pdfs-core/src/db/ops.rs` (`parked_create_ops`, `PARK_EXPIRY_MS`),
|
||||
`crates/pdfs-fuse/src/drain.rs` (`sweep_parked_creates`).
|
||||
|
||||
**Cause.** B70 parks a transient create by writing `PARK_UNTIL` into `next_attempt_at`, and the
|
||||
only thing that clears it is the transient-to-final rename (`Db::set_create_hold`). If the rename
|
||||
never happens — the writer crashed, the user deleted the temp file, or the name was never
|
||||
transient in the first place (B95) — the row is invisible to `claim_next_due_op` and stays in the
|
||||
queue permanently, holding its staged blob.
|
||||
|
||||
**What changed.** The primary drain worker sweeps parked creates when the queue is idle, which is
|
||||
exactly when a park that has outlived its writer matters. A parked create whose node is gone from
|
||||
the database has its queued ops discarded. One whose node still exists, that is older than
|
||||
`PARK_EXPIRY_MS` (1 hour) and that nobody has open, is un-parked rather than dropped — drain
|
||||
invariant 1 says staged bytes are only dropped once their op has landed, so the sweep lets the
|
||||
bytes through under the transient name instead of destroying them. An open node is left parked
|
||||
whatever its age: it is still being written.
|
||||
|
||||
---
|
||||
|
||||
## B93 — A photo whose node would not resolve lost its name, and with it its tab and its group
|
||||
|
||||
**Status:** Fixed (unverified) — found and fixed while making the timeline refresh incremental
|
||||
(B92); no library with an undecryptable photo in it has been driven against the fix.
|
||||
**Found:** 2026-09-19, by reading `photos_replace` before making a refresh skip photos it has
|
||||
already resolved. Skipping one means writing a `TimelineRow` that carries only the uid and the
|
||||
capture time, which is exactly the row a failed resolve already produced.
|
||||
|
||||
**Where:** `crates/pdfs-core/src/db/photos.rs` (`Learned`, `photos_replace`).
|
||||
|
||||
**Cause.** `photos_replace` keeps what a refresh could not learn — ratio, thumbnail verdict, media
|
||||
type, favourite flag, content hash, photo relation all survive a `None`. The `name` did not:
|
||||
`Learned` never held it, the SELECT never read it, and the INSERT bound `row.name` straight
|
||||
through. So a photo the SDK could not decrypt (it logs and skips the node) had its stored name
|
||||
overwritten with `NULL` on every refresh. That is not a cosmetic loss — both the tab split and the
|
||||
grouping are derived from the name. `PhotoKind::classify` is extension-first, so a nameless RAW
|
||||
file classifies as a still photo and leaves the Raw tab; `name_stem` returns `None`, so the RAW and
|
||||
the JPEG of one shot stop being one shot; and `redate` skips nameless photos outright.
|
||||
|
||||
**What changed.** `Learned` carries the name like everything else, and the kind and the group stem
|
||||
are computed from the carried name rather than the incoming row. A row that *was* resolved is now
|
||||
authoritative instead: its fields are taken as they are, so a photo unlinked from its main on
|
||||
another device finally loses its `main_uid` — a carry could never express "the server says none".
|
||||
|
||||
---
|
||||
|
||||
## B92 — Refreshing the gallery reported "Resource temporarily unavailable"
|
||||
|
||||
**Status:** Fixed (unverified) — the daemon carrying the change has not been driven against a real
|
||||
library yet.
|
||||
**Found:** 2026-09-19, while checking why a 12,445-photo timeline still showed the pre-grouping
|
||||
layout after pressing Refresh. `pdfs refresh photos` failed after about two minutes with:
|
||||
|
||||
```
|
||||
Caused by:
|
||||
0: io: Resource temporarily unavailable (os error 11)
|
||||
1: Resource temporarily unavailable (os error 11)
|
||||
```
|
||||
|
||||
**Where:** `crates/pdfs-fuse/src/control.rs` (`RefreshScope::Photos`),
|
||||
`crates/pdfs-core/src/control.rs` (`READ_TIMEOUT`, the new `PhotosRefreshStatus`),
|
||||
`crates/pdfs-gui/src/app/main.rs` (`refresh_then`), `crates/pdfs-cli/src/main.rs` (`cmd_refresh`).
|
||||
|
||||
**Cause.** `Refresh { Photos }` awaited `refresh_timeline()` before replying, on the reasoning that
|
||||
somebody who presses Refresh wants the truth rather than a stale page. But the refresh resolves
|
||||
every photo's node in batches of `TIMELINE_ENRICH_CHUNK` (200), so a 12,445-photo library is 63
|
||||
server round-trips — minutes of work. The client's read bound is `READ_TIMEOUT` = 120 s, so the
|
||||
socket timed out with `EAGAIN` while the refresh was running perfectly. In the GUI the timeout was
|
||||
invisible: `refresh_then` ignores the reply and loads the page anyway, which re-read the *old*
|
||||
timeline and made Refresh look like a dead button.
|
||||
|
||||
**What changed.** The daemon starts the refresh (`spawn_timeline_refresh`, which already dedupes)
|
||||
and replies `refreshing` at once. `Request::PhotosRefreshStatus` reports whether one is running;
|
||||
the gallery polls it every 2 s and reloads the page when it clears, and `pdfs refresh photos`
|
||||
polls in the same way so the command still means "the timeline is up to date when I return".
|
||||
`refresh_timeline` now logs `photos`, `chunks` and `elapsed_ms`.
|
||||
|
||||
**What changed, second pass (2026-09-19).** The refresh is incremental. `photos.resolved_at`
|
||||
(schema v31) records that a photo's node was read, and `refresh_timeline` resolves only the photos
|
||||
that have no such mark — the ones it has never read, and the ones a remote event marked stale.
|
||||
A library that has already been read costs one `enumerate_timeline` and no `enumerate_nodes` call
|
||||
at all.
|
||||
|
||||
The favourite question resolved by moving it off the refresh and onto the event feed:
|
||||
`classify_photo_events` turns a `NodeUpdated` into `photos_mark_unresolved(uid)` instead of a
|
||||
blanket "re-read everything", so a favourite toggled on a phone costs one node read rather than
|
||||
63. A local toggle is claimed as our own echo (`note_self_change`), because the feed replays it and
|
||||
a `NodeUpdated` carries nothing that says what changed. `ContinuityLost` / `ScopeAccessLost` and
|
||||
`pdfs refresh photos --full` drop every mark, which is the one way back to a whole-library read.
|
||||
`TIMELINE_ENRICH_CHUNK` is 150, matching the SDK's own `MAX_BATCH_COUNT` — a chunk of 200 was split
|
||||
there into a full request and a 50-uid tail.
|
||||
|
||||
**Still open.** `invalidate_albums()` drives the same pattern per album
|
||||
(`crates/pdfs-fuse/src/albums.rs:96` and `:191`). It is bounded by album size rather than library
|
||||
size, so it is nowhere near as expensive, but the technique that fixed this would apply there too.
|
||||
|
||||
---
|
||||
|
||||
## B91 — The daemon hung overnight, and nothing could say what it was waiting on
|
||||
|
||||
**Status:** Open — the hang itself is undiagnosed. What is fixed (unverified) is the
|
||||
*undiagnosability*: the daemon now reports what it is doing, warns when it stops making progress,
|
||||
and is restarted by systemd when it stops answering.
|
||||
**Found:** 2026-09-18, from a user report — the mount stopped responding and only came back after
|
||||
`systemctl --user restart`.
|
||||
**Where:** `crates/pdfs-fuse/src/{workers.rs,diagnostics.rs,supervisor.rs,systemd.rs,mount.rs}`,
|
||||
`packaging/proton-drive.service`.
|
||||
|
||||
**Symptom.** The mount stops answering. The process is alive and is not spinning; there is no
|
||||
panic, no error, and no log line after the moment it stopped. The accounting for the hung run:
|
||||
|
||||
```
|
||||
Consumed 43min 39.045s CPU time over 20h 8min 15.733s wall clock time,
|
||||
5.2G memory peak, 65.7M memory swap peak
|
||||
```
|
||||
|
||||
It went silent at 17:10 and stayed that way until the manual stop at 13:18:54 the next day. A
|
||||
restart picked everything back up, which rules out corrupt local state.
|
||||
|
||||
**What made it undiagnosable.** Nothing in the daemon could be asked what it was doing:
|
||||
|
||||
- the worker pool published no state at all — not which threads were busy, not with what, not for
|
||||
how long, not how deep the queues were;
|
||||
- control requests were not tracked, so a handler that never returned looked exactly like a
|
||||
handler that was never asked for;
|
||||
- there was no watchdog, so a daemon that stopped answering stayed "active (running)" for twenty
|
||||
hours;
|
||||
- resident size was never sampled, so the 5.2 G peak was only visible in systemd's post-mortem.
|
||||
|
||||
**Related, and a real bug on its own.** At 2026-09-18 09:58:35 a meta worker panicked during
|
||||
shutdown:
|
||||
|
||||
```
|
||||
thread 'pdfs-fuse-meta-0' (3044) panicked at ...:
|
||||
A Tokio 1.x context was found, but it is being shutdown.
|
||||
```
|
||||
|
||||
The pool was flagged closed at teardown but never joined, so the process could drop the tokio
|
||||
runtime while a worker was still inside a job. `catch_unwind` caught it, which is why it was only
|
||||
ever a log line — but it is teardown ordering, not an error.
|
||||
|
||||
**What changed.**
|
||||
|
||||
- `Workers` publishes per-thread state (busy, job label, age) plus queue depths and per-lane
|
||||
completion counts, all outside the queue lock, and every FUSE handler labels the job it hands
|
||||
over.
|
||||
- `Request::Diagnostics` / `pdfs diagnostics` reports that, the in-flight control requests with
|
||||
their ages, the handler count against its limit, resident size and the pending-op count. It is
|
||||
built to answer while the daemon is wedged: atomics and `try_lock` only, with the one database
|
||||
read on a throwaway thread behind a 500 ms deadline.
|
||||
- A supervisor thread WARNs when a worker has held one job for 120 s, when a control request has
|
||||
run that long, or when a lane has a queue but completed nothing since the last tick; it samples
|
||||
RSS every 5 minutes.
|
||||
- The supervisor answers the systemd watchdog, but only after a real round trip over the control
|
||||
socket — so the ping means "a client can still talk to this daemon" rather than "a thread is
|
||||
still scheduled". `WatchdogSec=120` and `Restart=always` turn the next hang into a restart
|
||||
instead of a morning of downtime, and `MemoryHigh=2G`/`MemoryMax=6G` bound the growth.
|
||||
- Teardown joins the worker pool, with a 10 s deadline, before the caller drops the runtime.
|
||||
|
||||
**Still open.** None of this says *why* it hung. The candidates the evidence does not yet separate:
|
||||
the 5.2 G working set (a leak, or a legitimately huge listing retained), a lock held across an
|
||||
await somewhere in the control plane, and the ~150 `rt.block_on` sites on the control path that
|
||||
have no op-level deadline of their own — each individual HTTP call is bounded by the SDK's 30 s
|
||||
API / 300 s storage timeouts, but a sequence of them is not. The next occurrence should be a
|
||||
one-line answer from `pdfs diagnostics` or from the stall WARN.
|
||||
|
||||
---
|
||||
|
||||
## B90 — The local-file indexer walks into foreign FUSE mounts, and a dead one makes the daemon unkillable
|
||||
|
||||
**Status:** Fixed (unverified) — the walker no longer stats a nested mountpoint. The stuck-daemon
|
||||
half cannot be fixed in userspace at all; see "Why there is no second line of defence".
|
||||
**Found:** 2026-09-16, from a user report of `systemctl --user restart` failing in a loop.
|
||||
**Where:** `crates/pdfs-core/src/localindex.rs` (`default_excludes`, `scan`),
|
||||
`crates/pdfs-fuse/src/background.rs` (`scan_local_once`).
|
||||
|
||||
**Symptom.** Stopping the service times out, SIGKILL does not help, and every subsequent start
|
||||
fails with `another Proton Drive daemon is already using .../cache.db`:
|
||||
|
||||
```
|
||||
systemd[2196]: proton-drive.service: State 'final-sigterm' timed out. Killing.
|
||||
systemd[2196]: proton-drive.service: Killing process 2221 (pdfs) with signal SIGKILL.
|
||||
systemd[2196]: proton-drive.service: Processes still around after final SIGKILL. Entering failed mode.
|
||||
systemd[2196]: proton-drive.service: Unit process 2221 (pdfs) remains running after unit stopped.
|
||||
pdfs[73921]: ERROR mount failed; retrying in 5s error="open cache db: another Proton Drive daemon
|
||||
is already using /home/narl/.local/state/proton-drive-linux/cache.db"
|
||||
```
|
||||
|
||||
**Diagnosis.** The surviving process is a zombie that still has a live thread:
|
||||
|
||||
```
|
||||
$ cat /proc/2221/status | grep -E 'State|Threads'
|
||||
State: Z (zombie)
|
||||
Threads: 2
|
||||
$ cat /proc/2221/task/73231/comm; cat /proc/2221/task/73231/stack
|
||||
pdfs-localindex
|
||||
request_wait_answer
|
||||
```
|
||||
|
||||
`pdfs-localindex` is parked in a FUSE request nobody will answer, in uninterruptible `D` state, so
|
||||
it ignores SIGKILL. The thread group leader exits, the thread does not, the process cannot be
|
||||
reaped — and it keeps the `cache.db` lock, which is what turns one stuck scan into a permanent
|
||||
restart loop. The same dead connection also blocked an unrelated `duf` in `fuse_statfs`
|
||||
(`/sys/fs/fuse/connections/69/waiting` was 2).
|
||||
|
||||
**Cause.** `scan_local_once` walks `$HOME`, and `default_excludes` only excluded the Drive
|
||||
mountpoint plus our own state and cache dirs. The reporter's home also held
|
||||
`~/remote/narl.io` (sshfs) and `~/remote/gdrive` (rclone) — third-party FUSE mounts the walk
|
||||
descends into.
|
||||
|
||||
`scan` does set `same_file_system(true)`, which looks like it should already cover this. It does
|
||||
not: in `ignore` 0.4.33 the check is `is_same_file_system` → `device_num(path)` →
|
||||
`path.metadata()` (`walk.rs:2164`), i.e. **a stat of the mountpoint**, performed in `run_one`
|
||||
*after* the directory has been queued. To discover that a mount is foreign, the walker first has
|
||||
to touch it. Against a network FUSE mount whose server is gone, that stat never returns.
|
||||
|
||||
**Fix.** `localindex::nested_mount_points` reads `/proc/self/mounts` and returns every mountpoint
|
||||
strictly below the scan root; `scan_local_once` appends those to the excludes. `filter_entry` runs
|
||||
before the device stat (`walk.rs:1926`, before `send`), so a path-only rejection keeps the walker
|
||||
from ever touching the mount. Mountpoint fields are octal-unescaped, because a `\040` left in
|
||||
place would silently fail to match and put the mount back in the walk. Re-read per scan, not
|
||||
cached at startup: a mount that appeared since the daemon started is the likeliest to be a
|
||||
half-alive network filesystem.
|
||||
|
||||
This changes nothing about what gets indexed. A nested mount is a different device, so
|
||||
`same_file_system(true)` was always going to skip it — the exclude only removes the stat that had
|
||||
to happen first.
|
||||
|
||||
**Why there is no second line of defence.** Once a thread is in uninterruptible FUSE wait, no
|
||||
userspace change rescues it: not a join timeout, not `std::process::exit`, not SIGKILL. The kernel
|
||||
will not reap the task until the syscall returns, which needs someone to answer the request or
|
||||
abort the connection (`echo 1 > /sys/fs/fuse/connections/<id>/abort`, root-only). Avoiding entry is
|
||||
the entire fix. That makes any *new* code path that stats a user-supplied or mount-crossing path
|
||||
from a daemon thread a repeat of this bug — mirror-folder scanning is the obvious next candidate.
|
||||
|
||||
**Recovery for an already-wedged daemon.** As root, abort the connection with waiters
|
||||
(`grep . /sys/fs/fuse/connections/*/waiting` finds it), which releases the thread, lets the zombie
|
||||
be reaped and frees the `cache.db` lock. A reboot also clears it.
|
||||
|
||||
---
|
||||
|
||||
## B89 — The recorded SDK dependency is three minor versions out of date in the working memory
|
||||
|
||||
**Status:** Open — documentation only, but it misdirects every reader of the SDK code.
|
||||
|
||||
+129
-2
@@ -9,9 +9,136 @@ release ships. Migrations are forward-only — a database written by a newer cli
|
||||
refuse-to-open, not a downgrade, so rolling back a release means restoring the cache from
|
||||
scratch (user data in `staging/` and `recovery/` is never touched by this).
|
||||
|
||||
## [Unreleased]
|
||||
## [1.11.1] — 2026-09-19
|
||||
|
||||
Nothing yet.
|
||||
Schema: **30** (unchanged).
|
||||
|
||||
### Fixed
|
||||
- **An upload parked behind a temp-file rename no longer stays parked forever.** A create for a
|
||||
name that looks like a writer's temporary file waits for the rename to the final name. If that
|
||||
rename never came — the writer crashed, the file was deleted, or the name was never temporary —
|
||||
the upload sat in the queue permanently, holding its staged bytes. The queue now sweeps parked
|
||||
creates while it is idle: one whose file is gone is dropped, and one that has been parked for
|
||||
over an hour with nobody holding it open is uploaded under the name it has. An open file is
|
||||
left alone however old it is (B94).
|
||||
- **A file whose real name ends in `.tmp` uploads normally.** `.tmp` and `.temp` were treated as
|
||||
writer temp-file suffixes, so ordinary files — a Google Photos Takeout `index.tmp`, for
|
||||
instance — were parked waiting for a rename that was never going to happen (B95).
|
||||
- **Staged bytes that belong to nothing are cleaned up.** A staged blob that matches no queued
|
||||
upload and names no node used to be kept forever and warned about once per blob on every mount.
|
||||
Blobs older than 30 days are now retired with their size logged; younger ones are kept, but
|
||||
reported as a single line instead of one per blob (B96).
|
||||
- **Refreshing the gallery repeatedly no longer wedges the control socket.** Each refresh scope
|
||||
now runs one at a time; asking again while one is running is answered immediately rather than
|
||||
repeating work that contends with it for the database. `pdfs diagnostics` also names the scope
|
||||
of a running refresh (B97).
|
||||
- **Shutting down during a photo refresh no longer logs a wall of "undecryptable photo"
|
||||
warnings.** The metadata pass stops when teardown begins instead of running into the cancelled
|
||||
runtime. Nothing was wrong with those photos; the next refresh reads them again (B97).
|
||||
|
||||
## [1.11.0] — 2026-09-19
|
||||
|
||||
Schema: **30** (`photos.content_hash` / `main_uid` / `group_key`); SDK bumped to `proton-sdk` /
|
||||
`proton-drive-rs` **0.6.5**.
|
||||
|
||||
### Changed
|
||||
- **The gallery lays photos out in justified rows.** Every tile is now its own photo's shape:
|
||||
photos are taken in capture order until they no longer fit the target height, and the row is
|
||||
then scaled to land exactly on the right margin. Nothing is cropped and nothing is
|
||||
letterboxed, which is what the square centre-cropped tiles used to do to portraits and
|
||||
panoramas. The last row of a day stays at the target height rather than being stretched, and
|
||||
Ctrl+scroll / Ctrl+± now changes that target height. Gaps and tile corners are smaller, so the
|
||||
page reads as photos rather than as cards.
|
||||
- **Thumbnails stay on screen while you scroll.** A decoded thumbnail now lands in the texture
|
||||
cache even when the tile that asked for it has already been recycled, the cache is a real LRU
|
||||
holding 1500 textures instead of 600, and the rows just past the edge of the window are
|
||||
fetched ahead of the scroll direction. Scrolling fast through a large timeline no longer
|
||||
leaves a screen of blank tiles behind.
|
||||
- **Refresh on the Photos page really refreshes.** Instead of only clearing the freshness stamp
|
||||
and letting the next read serve the stale page, it starts the timeline re-read and follows it:
|
||||
the page paints from what the daemon holds, then paints again when the new timeline lands. The
|
||||
timeline is also considered stale after 60 s rather than 5 minutes. `pdfs refresh photos`
|
||||
blocks until the re-read is done, as it reads.
|
||||
|
||||
- **Restore puts back the whole tree.** The trash is a flat list, but you delete a shape: a
|
||||
folder, and often things you had already deleted inside it. Restoring a folder now also
|
||||
restores the items that were trashed inside it, and restoring one of those items first
|
||||
restores the trashed folders above it — otherwise the file came back into a folder that is
|
||||
still in the trash, which is to say nowhere you can see it. Parents are restored before their
|
||||
children, and each node is still restored individually, so one node the server refuses does
|
||||
not cancel the rest.
|
||||
|
||||
### Added
|
||||
- **A RAW and its JPEG are one photo.** A camera that writes both put two tiles of the same shot
|
||||
in the timeline; now it puts one, badged `RAW`, and the lightbox has a control that switches
|
||||
between the files. The Raw tab still lists every raw file individually, because that is what
|
||||
the tab is for. Grouping follows the server's own photo relation where it has one — which
|
||||
also collapses live photos and bursts — and otherwise pairs files with the same name taken on
|
||||
the same day where one is raw and the other is not. Deleting the tile deletes the whole shot,
|
||||
and the confirmation says how many files that is.
|
||||
- **Delete from the gallery.** Select photos with the Select button, with Ctrl+click or with
|
||||
Shift+click, then Move to Trash — or press Delete in the lightbox. The tiles leave the grid
|
||||
immediately with an Undo toast, and the photos land in Proton's trash, where they stay
|
||||
recoverable. A photo the server refuses to trash comes back into the grid.
|
||||
- **A photo deleted on your phone disappears here.** The daemon now follows the photos volume's
|
||||
own event stream, so a photo trashed on another device leaves the gallery within about ten
|
||||
seconds instead of waiting for the timeline to go stale.
|
||||
|
||||
- **`pdfs diagnostics`.** Reports what the daemon is doing right now: every worker thread with the job
|
||||
it holds and for how long, queue depths per lane, the control requests still in flight, resident
|
||||
size and the pending-op count. Answers even while the daemon is wedged, which is the only time
|
||||
anyone asks (B91).
|
||||
- **A stall watchdog.** The daemon warns in the journal when a worker has held one job for two
|
||||
minutes, when a control request has run that long, or when a queue is not draining — and it now
|
||||
answers the systemd watchdog, but only after a real round trip over its own control socket. A
|
||||
daemon that stops answering is restarted instead of sitting there looking healthy (B91).
|
||||
|
||||
### Fixed
|
||||
- **A Pixel raw capture groups with its JPEG.** Pixel names the two files of one shot
|
||||
`PXL_….RAW-01.COVER.jpg` and `PXL_….RAW-02.ORIGINAL.dng`, so their stems differ and the pairing
|
||||
rule never saw them as one shot. The part-of-a-shot marker is now cut off before the names are
|
||||
compared, and the grid shows the cover JPEG once. The grouping is computed by a timeline
|
||||
refresh, so an existing library pairs up after the next one.
|
||||
- **The Raw tab no longer decodes a raw it does not have to.** A raw file that was written next
|
||||
to a JPEG of the same shot is now given that JPEG's thumbnail, which the server usually already
|
||||
has. No raw download and no raw decode for those tiles.
|
||||
- **Photos can be moved to Trash from the gallery again.** Photos live on the photo volume, which
|
||||
the mount never holds, so the write check found no node to consult and refused every photo with
|
||||
"trash access: not allowed". Membership of our own timeline now answers for them; an album photo
|
||||
on someone else's volume is still refused.
|
||||
- **A photos refresh no longer fails with "Resource temporarily unavailable" (B92).** Re-reading a
|
||||
twelve-thousand-photo library resolves the nodes in batches of 200 and takes minutes, but the
|
||||
daemon held the control reply until it had finished — well past the client's 120 s read bound.
|
||||
The caller therefore saw `io: Resource temporarily unavailable (os error 11)` for a refresh that
|
||||
was running perfectly, and the gallery reloaded from the old timeline. The daemon now answers as
|
||||
soon as the refresh is running, and the new `PhotosRefreshStatus` request reports when it is
|
||||
done. Each refresh also logs its photo count, batch count and duration.
|
||||
- **A worker no longer panics during shutdown (B91).** The FUSE worker pool is joined before the
|
||||
tokio runtime is dropped, so a worker still inside a job does not find a runtime that is going
|
||||
away underneath it.
|
||||
- **A permanently rejected block upload is no longer retried (SDK 0.6.5).** The SDK now knows
|
||||
which API errors are worth replaying: 4xx is permanent, except 404, 408 and 429.
|
||||
- **Long downloads no longer fail on expired block URLs (SDK 0.6.5).** A download whose block
|
||||
URLs lapse mid-transfer re-mints them once instead of failing, on the whole-file path and on
|
||||
public links, as reads already did.
|
||||
- **A long rate-limit answer can no longer look like a hang (SDK 0.6.5).** A server-supplied
|
||||
`Retry-After` is now capped at 120 s.
|
||||
|
||||
## [1.10.2] — 2026-09-16
|
||||
|
||||
Fixes a daemon that could not be stopped or restarted. No schema change.
|
||||
Schema: **28**; no SDK change.
|
||||
|
||||
### Fixed
|
||||
- **A dead network mount in your home directory no longer wedges the daemon (B90).** The
|
||||
"This computer" file indexer walked `$HOME` and descended into other FUSE mounts — an sshfs
|
||||
whose server had gone away, an unresponsive rclone. Stat'ing one of those blocks
|
||||
uninterruptibly, so the indexer thread stopped responding to SIGKILL, the daemon could never
|
||||
exit, `systemctl --user stop` timed out, and the half-dead process kept the cache database lock
|
||||
that the next start needed: every restart then failed with "another Proton Drive daemon is
|
||||
already using cache.db". The indexer now skips mountpoints below your home directory by path,
|
||||
without touching them. Nothing changes about which files are indexed — those mounts were always
|
||||
excluded from the results, just not before they had been stat'ed.
|
||||
|
||||
## [1.10.1] — 2026-09-15
|
||||
|
||||
|
||||
@@ -142,6 +142,27 @@ The gallery can mark and filter favourites, through the SDK's photo-tag API (`up
|
||||
|
||||
Favouriting a photo that is not on this account's own photos volume (shared with us, or album-only) needs it re-encrypted for our timeline root, which the SDK does not implement; the daemon surfaces that as an error rather than silently doing nothing.
|
||||
|
||||
### Justified Gallery, Gallery Delete (Implemented)
|
||||
**Files**: [`photos.rs`](../crates/pdfs-gui/src/app/pages/photos.rs), [`photo_viewer.rs`](../crates/pdfs-gui/src/app/pages/photo_viewer.rs), [`photos.rs`](../crates/pdfs-fuse/src/photos.rs), [`background.rs`](../crates/pdfs-fuse/src/background.rs)
|
||||
|
||||
The gallery lays each day out in justified rows (`justify_rows` / `plan_rows` / `fit_row`): photos are taken in capture order until their summed aspect ratio no longer fits the target row height, and the row is then scaled so it ends exactly on the content width. A tile is therefore its own photo's shape, and `ContentFit::Contain` has nothing left to crop. The last row of a day is left at the target height rather than stretched, because a day holding two photos is a short day, not a layout fault.
|
||||
|
||||
- **Ratios that are not known yet.** `PhotoItem::ratio` is persisted, but a photo that has never been decoded has none. Those are laid out square and remembered in `assumed_ratios`; when a decode proves the real shape the day re-flows on the existing debounce timer, so a screenful of decodes costs one re-flow rather than one per photo. Ratios are clamped to 0.4–3.0 so a single panorama cannot decide what a row looks like.
|
||||
- **Thumbnails survive recycling.** A reply for a tile that has already been re-bound still lands in the texture cache, which is an LRU of 1500 textures keyed by uid, and the rows just outside the realised range are prefetched in the scroll direction.
|
||||
- **Delete is uid-addressed.** The photos volume is not in the FUSE mount, so the path-based `Request::Delete` cannot reach it. `Request::TrashNodes { uids }` trashes through the SDK and answers `Response::Trashed { trashed, failed }`; the daemon then drops those rows with `Db::photos_delete` (album membership included) so the gallery does not wait for a timeline refresh. The GUI removes the tiles optimistically with an Undo toast and puts back anything the server refused.
|
||||
- **Remote deletions arrive on their own.** The Drive volume and the photos volume have separate event streams, so `run_photos_event_sync` polls the photos volume with its own cursor (`photos_event_cursor` in the state table). Trash and delete events remove rows; anything else only invalidates freshness, because there is no inode space to converge on this volume. Together with a 60 s `TIMELINE_TTL` and a `RefreshScope::Photos` that now awaits the refresh, a photo deleted on a phone leaves the grid in about ten seconds.
|
||||
|
||||
### RAW + JPEG Grouping (Implemented)
|
||||
**Files**: [`photos.rs`](../crates/pdfs-core/src/db/photos.rs), [`migrations.rs`](../crates/pdfs-core/src/db/migrations.rs), [`photos.rs`](../crates/pdfs-fuse/src/photos.rs), [`photo_viewer.rs`](../crates/pdfs-gui/src/app/pages/photo_viewer.rs)
|
||||
|
||||
Schema **v30** adds `content_hash`, `main_uid` and an indexed `group_key` to `photos`. The first two come from the server's `PhotoProperties` and are filled by the enrichment pass in `refresh_timeline` that already resolves each photo's name, media type and favourite tag; `group_key` is computed in `photos_replace`, where the whole timeline is in hand.
|
||||
|
||||
- **Precedence.** The server relation first (`main_photo_uid` and `related_photo_uids`, read from whichever end resolves — the two can land in different enrichment chunks); then same capture day, same case-insensitive name stem, and one member raw while the other is not; otherwise the photo is its own group. The union-find pass is over the timeline in memory, so it costs one pass per refresh rather than a query per photo.
|
||||
- **Representative.** The non-raw member when there is one — a JPEG decodes in milliseconds and is what the person expects to see. `group_key` holds that member's uid, so "is this the tile" is `uid = group_key`, an index scan.
|
||||
- **What is grouped and what is not.** `photos_page`, `photos_months` and `photos_counts` count groups, except on the Raw tab, which lists files. Album pages are never grouped: an album is a list someone made. The relation is learned-and-kept like `media_type`, so a refresh that could not resolve a node does not break a group up.
|
||||
- **Deleting.** `Core::trash_photos` expands each uid to its group before trashing, because the grid shows the shot as one tile — leaving the RAW behind would put the photo back on the Raw tab and nowhere else. The GUI's confirmation names the file count.
|
||||
- **The other files.** `Request::PhotoGroup { uid }` answers with the group's members as ordinary `PhotoItem`s, which is what the lightbox's switch steps through.
|
||||
|
||||
### Persistent SDK Entity Cache (Implemented)
|
||||
**Files**: [`sdkcache.rs`](../crates/pdfs-core/src/sdkcache.rs), [`auth.rs`](../crates/pdfs-core/src/auth.rs), [`background.rs`](../crates/pdfs-fuse/src/background.rs)
|
||||
|
||||
@@ -158,6 +179,8 @@ Needed a small additive SDK change (`ProtonDriveClient::with_entity_repository`,
|
||||
|
||||
SDK 0.5.0 made `trash_nodes` / `restore_nodes` / `delete_nodes` report one outcome per node instead of failing the whole call (mirroring upstream's streamed `NodeActionResult`). `pdfs_core::batch::into_unit` collapses the single-node calls back to a `Result`; `batch::split` handles a collected batch.
|
||||
|
||||
A restore is expanded before it is sent (`expand_restore`, `pdfs-fuse/src/lib.rs`): the persisted trash listing carries each row's `parent_uid` (schema 29), so a restore takes the connected piece of the trashed tree — every trashed descendant of what was asked for, and every trashed ancestor above it — and sends it shallowest wave first, because the server cannot put a node back under a parent that is still trashed. A uid the listing does not know about (it is materialised in chunks, and can be stale) is restored on its own rather than dropped.
|
||||
|
||||
The trash view's restore and permanent-delete use the SDK's **streaming** variants (`restore_nodes_streaming` / `delete_nodes_streaming`) and apply local state per node as each batch lands rather than after the last one. For a permanent delete — which is irreversible — that means a daemon interrupted mid-batch has forgotten exactly the nodes the server destroyed, no more and no fewer. Both report per-node failures and count only what actually succeeded; only a batch where nothing succeeded is an error.
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
# Build + install: cd packaging && makepkg -fi
|
||||
|
||||
pkgname=proton-drive-linux
|
||||
pkgver=1.10.1
|
||||
pkgver=1.11.1
|
||||
pkgrel=1
|
||||
pkgdesc="Unofficial Proton Drive client for Linux: files-on-demand FUSE mount, CLI, GTK4 app + tray"
|
||||
arch=('x86_64')
|
||||
|
||||
@@ -6,12 +6,28 @@ After=graphical-session.target
|
||||
PartOf=graphical-session.target
|
||||
|
||||
[Service]
|
||||
# Deliberately not Type=notify: the daemon waits for a stored login before it
|
||||
# mounts anything, and a machine that boots before the user logs in would then
|
||||
# fail the unit on the start timeout. NotifyAccess=all gets the daemon a
|
||||
# NOTIFY_SOCKET all the same, which is all the watchdog below needs.
|
||||
Type=simple
|
||||
NotifyAccess=all
|
||||
# The daemon waits for a stored login, mounts ~/ProtonDrive, and keeps it
|
||||
# mounted. SIGTERM (from `systemctl --user stop`) triggers a clean lazy unmount.
|
||||
ExecStart=/usr/bin/pdfs daemon
|
||||
Restart=on-failure
|
||||
# Always, not on-failure: a daemon that hangs is not a failure by exit code, and
|
||||
# on 2026-09-18 one had to be restarted by hand after wedging overnight.
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
# The supervisor thread pings this, but only after a round trip over the control
|
||||
# socket — so a daemon that stops answering its own clients is restarted instead
|
||||
# of sitting there looking alive. It pings at half this interval.
|
||||
WatchdogSec=120
|
||||
# Backstop against unbounded growth: the hung run peaked at 5.2G resident.
|
||||
# MemoryHigh throttles and reclaims first; MemoryMax is the hard stop, above any
|
||||
# legitimate working set, so it only fires on a leak.
|
||||
MemoryHigh=2G
|
||||
MemoryMax=6G
|
||||
# Give the lazy unmount time to finish before SIGKILL.
|
||||
TimeoutStopSec=15
|
||||
|
||||
|
||||
Reference in new issue
Block a user