From f0a43fb9234acd744e1c55d6bd668133f8748210 Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Mon, 5 Oct 2026 03:48:13 +0200 Subject: [PATCH] fix(fuse): changes that failed on a short outage go out once the link is back (B189) --- crates/pdfs-fuse/src/drain.rs | 15 +++++++++++-- crates/pdfs-fuse/src/sim/daemon.rs | 36 ++++++++++++++++++++++++++++++ docs/BUGS.md | 19 ++++++++++++++++ docs/CHANGELOG.md | 2 ++ docs/MILESTONE-3.0.0-STATUS.md | 5 +++-- 5 files changed, 73 insertions(+), 4 deletions(-) diff --git a/crates/pdfs-fuse/src/drain.rs b/crates/pdfs-fuse/src/drain.rs index d1433cf..a2e912b 100644 --- a/crates/pdfs-fuse/src/drain.rs +++ b/crates/pdfs-fuse/src/drain.rs @@ -432,6 +432,13 @@ impl Core { { debug!(uid = %op.uid, error = %e, "waking the ops waiting for a name failed"); } + // A drain that cannot reach Drive has lost the link as surely as + // a syscall that cannot. Unmarked, a short outage only the drain + // met had no probe to see the link return, and the op sat out + // its backoff long after (`docs/BUGS.md` B189). + if let Some(proton) = proton_error(e.as_ref()) { + self.lost_link(proton, "drain"); + } // A failure to reach Drive says nothing about an earlier refusal, // so it keeps the issue that refusal recorded. if let Some(issue) = refusal(e.as_ref()) { @@ -2654,12 +2661,16 @@ fn moved_elsewhere(meta: &RenameMeta, node: &Node, parent: &NodeUid, name: &str) !at(&meta.original_parent_uid, original_name) && !at(&parent.to_string(), name) } +/// The answer from Drive somewhere in `e`'s chain, if it got that far. +fn proton_error<'e>(e: &'e (dyn std::error::Error + 'static)) -> Option<&'e ProtonError> { + std::iter::successors(Some(e), |e| e.source()).find_map(|e| e.downcast_ref::()) +} + /// What a failed attempt says about its op's sync issue: `None` when Drive /// could not be reached, which says nothing; otherwise the issue Drive's answer /// is, or `Some(None)` for a failure that is not a refusal. fn refusal(e: &(dyn std::error::Error + 'static)) -> Option> { - let proton = std::iter::successors(Some(e), |e| e.source()) - .find_map(|e| e.downcast_ref::())?; + let proton = proton_error(e)?; if is_network_error(proton) { return None; } diff --git a/crates/pdfs-fuse/src/sim/daemon.rs b/crates/pdfs-fuse/src/sim/daemon.rs index 4dcce2d..b290303 100644 --- a/crates/pdfs-fuse/src/sim/daemon.rs +++ b/crates/pdfs-fuse/src/sim/daemon.rs @@ -476,6 +476,42 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[test] + #[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"] + fn a_write_that_failed_on_a_short_outage_goes_out_soon_after_the_link_returns() { + // Only the drain met the outage, and it never took the mount offline: + // no probe saw the link come back, and the write sat out its backoff + // (B189). + let drive = FakeDrive::new(); + let dir = scratch("short-outage"); + let daemon = Daemon::start(&dir, drive.client(1, Faults::lan())).unwrap(); + std::fs::write(daemon.mountpoint.join("g.txt"), b"g").unwrap(); + assert!(wait_until(Duration::from_secs(30), || { + daemon.pending().is_ok_and(|items| items.is_empty()) && drive.tree().len() == 1 + })); + + daemon.client.set_online(false); + std::fs::write(daemon.mountpoint.join("f.txt"), b"0123456789").unwrap(); + assert!(wait_until(Duration::from_secs(30), || { + daemon + .pending() + .is_ok_and(|items| items.iter().any(|item| item.attempts > 0)) + })); + daemon.client.set_online(true); + assert!( + wait_until(Duration::from_secs(3), || { + daemon.pending().is_ok_and(|items| items.is_empty()) + && drive.tree().get("f.txt") + == Some(&Entry::File(Arc::new(b"0123456789".to_vec()))) + }), + "{:?}", + daemon.pending() + ); + + assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown))); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] #[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"] fn a_restart_does_not_bring_back_the_write_of_a_file_deleted_while_open() { diff --git a/docs/BUGS.md b/docs/BUGS.md index 99baeca..a043a9d 100644 --- a/docs/BUGS.md +++ b/docs/BUGS.md @@ -12,6 +12,25 @@ Conventions: --- +## B189 — A change that failed on a short outage waits out its retry long after the link is back + +**Status:** Fixed (unverified). +**Found:** 2026-10-05, in the confirmation run on 2baad67 (`three_clients_one_writer_each` +seed 6, every replay). A client's link was down for 0.3 s, during which it made a folder and a +file. Their uploads failed once, and 4 s after the link came back they were still queued. + +**Where:** the failure path of `Core::run_pending_drain` in `crates/pdfs-fuse/src/drain.rs`. + +**Cause.** Every syscall and the event poll take the mount offline when Drive does not answer, +and the probe that brings it back makes every backed-off op due at once. The drain was the one +caller that did not. An outage that only the drain met left the mount online, so nothing probed +for the link's return, and the ops sat out a backoff that doubles with each failure, up to five +minutes. + +**Fix.** A drain attempt that cannot reach Drive takes the mount offline like any other call. +Test `a_write_that_failed_on_a_short_outage_goes_out_soon_after_the_link_returns` in +`crates/pdfs-fuse/src/sim/daemon.rs`. + ## B188 — A restart can leave a queued write retrying forever on a file deleted while open **Status:** Fixed (unverified). diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0df8617..30260c4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -111,6 +111,8 @@ by its local id, in the queue, the stored tree and pins alike. A write into it no longer failed with "Input/output error" (B186). - **An uploaded file no longer leaves a copy of itself in the cache** until the next start (B187). +- **Changes that failed during a short outage go out soon after the connection is back**, not + after a retry delay of up to five minutes (B189). - **A file renamed just as its upload finished keeps its new name.** The mount no longer showed the old name, and a second rename no longer failed with "No such file or directory" (B152, B172). diff --git a/docs/MILESTONE-3.0.0-STATUS.md b/docs/MILESTONE-3.0.0-STATUS.md index ba892b8..c9446c2 100644 --- a/docs/MILESTONE-3.0.0-STATUS.md +++ b/docs/MILESTONE-3.0.0-STATUS.md @@ -9,7 +9,7 @@ Tracks `docs/MILESTONE-3.0.0.md`. Update it with every commit that moves a phase - **Phase 2**: local ids, inode = local id, one tree for every mount (2.10.0). - **Phase 3, first part**: mkdir, create, rename, unlink and rmdir on the mount are recorded locally and sent from the queue; `"local_first": false` brings back the old path (5e0f30c). -- Fixes from the simulation and account runs since then: B125, B127, B130 and B141 to B188. +- Fixes from the simulation and account runs since then: B125, B127, B130 and B141 to B189. - **Sync issues, first cut** (§6.2): refusals sorted into issues, shown at once in the app, tray and CLI; Export button and `pdfs sync export`; `pdfs sync issues`; Discard button and `pdfs sync discard`; a warning badge on the node in the file browser. @@ -114,7 +114,8 @@ Tracks `docs/MILESTONE-3.0.0.md`. Update it with every commit that moves a phase a retry after a write replaced its blob (B185, fixed), and a file deleted offline while open that lost the bytes its handle read (B186, fixed; older than 3.0.0). The run on 2baad67 found that a restart before such a file's trash went out queued - its kept bytes again, for good (B188, fixed). + its kept bytes again, for good (B188, fixed), and changes that sat out their retry + after an outage only the drain had met (B189, fixed; older than 3.0.0). - [ ] A clean account run. The one on a6f36d5 (2026-10-04, LAN: 427 passed, 12 failed) failed B113 and B121 on every on-demand mount, two mirror-to-mirror move cases, and the journal check. B121 was a bug (B149, fixed); B113's case still expected creates