added fuse compliance testing

This commit is contained in:
nvrl committed 2026-07-22 03:04:40 +02:00
1 parent bef39f75cf
commit 544d0913e8
7 files changed
+570 -37

No files matched your search

+5
View File
@@ -94,6 +94,11 @@ For the local metadata database and content cache:
the database while it runs, so it is a deliberate operation rather than
something the daemon does on a timer.
Before a release, run the real-kernel acceptance suite against a dedicated test
account and mount. See [`docs/TESTING.md`](docs/TESTING.md); it includes optional
two-mount convergence checks and confines destructive operations to a fresh
test directory.
## Scripting (`--json`)
Query commands accept a global `--json` flag and emit machine-readable output:
+251 -32
View File
@@ -1,7 +1,9 @@
//! `pdfs` — command-line front-end for the Proton Drive Linux client.
use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::Write;
use std::os::unix::fs::FileTypeExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -1740,16 +1742,95 @@ fn cmd_cache_inspect(deep: bool) -> Result<()> {
}
}
/// One line of the `diagnose` report.
fn check(label: &str, ok: bool, detail: impl AsRef<str>) -> bool {
let mark = if ok { "ok " } else { "FAIL" };
#[derive(Default)]
struct DiagnoseReport {
warnings: usize,
failures: usize,
}
impl DiagnoseReport {
fn finding(&mut self, level: DiagnoseLevel, label: &str, detail: impl AsRef<str>) {
let mark = match level {
DiagnoseLevel::Ok => "ok ",
DiagnoseLevel::Warn => {
self.warnings += 1;
"WARN"
}
DiagnoseLevel::Fail => {
self.failures += 1;
"FAIL"
}
};
let detail = detail.as_ref();
if detail.is_empty() {
println!("[{mark}] {label}");
} else {
println!("[{mark}] {label}: {detail}");
}
ok
}
fn check(&mut self, label: &str, ok: bool, detail: impl AsRef<str>) {
self.finding(
if ok {
DiagnoseLevel::Ok
} else {
DiagnoseLevel::Fail
},
label,
detail,
);
}
}
#[derive(Clone, Copy)]
enum DiagnoseLevel {
Ok,
Warn,
Fail,
}
fn executable_on_path(name: &str) -> bool {
std::env::var_os("PATH").is_some_and(|paths| {
std::env::split_paths(&paths)
.any(|dir| std::fs::metadata(dir.join(name)).is_ok_and(|m| m.is_file()))
})
}
fn mounted_fs_type(path: &Path) -> Option<String> {
let wanted = path.canonicalize().ok()?;
let mounts = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
mounts
.lines()
.filter_map(|line| {
let fields: Vec<_> = line.split_whitespace().collect();
let separator = fields.iter().position(|field| *field == "-")?;
let encoded = *fields.get(4)?;
let decoded = encoded.replace("\\040", " ").replace("\\011", "\t");
let mountpoint = PathBuf::from(decoded);
if mountpoint.canonicalize().ok()? == wanted {
Some(fields.get(separator + 1)?.to_string())
} else {
None
}
})
.next()
}
/// Create, sync, and remove a new file without ever replacing an existing one.
fn writable_probe(dir: &Path) -> std::io::Result<()> {
let name = format!(".pdfs-diagnose-probe-{}", std::process::id());
let path = dir.join(name);
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)?;
let result = (|| {
file.write_all(b"probe")?;
file.sync_all()
})();
drop(file);
let remove = std::fs::remove_file(path);
result.and(remove)
}
/// Self-check the installation, reporting each finding and exiting non-zero if
@@ -1759,58 +1840,80 @@ fn check(label: &str, ok: bool, detail: impl AsRef<str>) -> bool {
/// situation a user runs this in. A missing daemon is reported as a finding,
/// not an error that aborts the rest of the report.
fn cmd_diagnose() -> Result<()> {
let mut all_ok = true;
let mut report = DiagnoseReport::default();
let dirs = AppDirs::new().context("resolve application directories")?;
let config = dirs.load_config();
println!("Paths");
let state = dirs.state_dir();
all_ok &= check(
report.check(
" state dir",
state.is_dir(),
format!("{}", state.display()),
);
let cache_dir = dirs.cache_dir();
all_ok &= check(
report.check(
" cache dir",
cache_dir.is_dir(),
format!("{}", cache_dir.display()),
);
// Writability is checked by actually writing: permission bits do not
// account for a full disk, a read-only remount, or an immutable flag.
let probe = state.join(".pdfs-diagnose-probe");
let writable = std::fs::write(&probe, b"probe").is_ok();
let _ = std::fs::remove_file(&probe);
all_ok &= check(" state dir writable", writable, "");
match writable_probe(&state) {
Ok(()) => report.finding(
DiagnoseLevel::Ok,
" state dir writable",
"write and fsync succeeded",
),
Err(e) => report.finding(DiagnoseLevel::Fail, " state dir writable", e.to_string()),
}
let db_path = dirs.db_path();
let db_size = std::fs::metadata(&db_path).map(|m| m.len()).unwrap_or(0);
all_ok &= check(
report.check(
" database",
db_path.is_file(),
format!("{} ({})", db_path.display(), human_bytes(db_size)),
);
println!("\nFUSE");
let fuse = Path::new("/dev/fuse");
report.check(" /dev/fuse", fuse.exists(), fuse.display().to_string());
let has_fusermount = executable_on_path("fusermount3") || executable_on_path("fusermount");
report.check(
" unmount helper",
has_fusermount,
"fusermount3 or fusermount on PATH",
);
println!("\nAccount");
match pdfs_core::auth::load() {
Ok(session) => {
all_ok &= check(" keyring session", true, session.username);
report.finding(DiagnoseLevel::Ok, " keyring session", session.username);
}
Err(e) => {
// A locked keyring and an absent login look the same from here, so
// the detail carries the distinction rather than the verdict.
all_ok &= check(" keyring session", false, format!("{e}"));
report.finding(DiagnoseLevel::Fail, " keyring session", format!("{e}"));
}
}
println!("\nDaemon");
let socket = dirs.control_socket();
let socket_exists = socket.exists();
check(
let socket_kind_ok =
std::fs::symlink_metadata(&socket).is_ok_and(|m| m.file_type().is_socket());
report.finding(
if socket_exists && !socket_kind_ok {
DiagnoseLevel::Fail
} else if socket_exists {
DiagnoseLevel::Ok
} else {
DiagnoseLevel::Warn
},
" control socket",
socket_exists,
format!("{}", socket.display()),
socket.display().to_string(),
);
// A stale socket file with no listener is the common failure, so the reply
// is what decides this rather than the file's existence.
@@ -1822,47 +1925,127 @@ fn cmd_diagnose() -> Result<()> {
pending_changes,
..
}) => {
check(" daemon responding", true, "");
check(" mounted at", true, mountpoint);
check(" network", online, if online { "" } else { "offline" });
report.finding(DiagnoseLevel::Ok, " daemon responding", "");
let live_type = mounted_fs_type(Path::new(&mountpoint));
let is_fuse = live_type
.as_deref()
.is_some_and(|kind| kind.starts_with("fuse"));
report.finding(
if is_fuse {
DiagnoseLevel::Ok
} else {
DiagnoseLevel::Fail
},
" active mount",
live_type.map_or_else(
|| format!("{mountpoint} is not mounted"),
|kind| format!("{mountpoint} ({kind})"),
),
);
report.finding(
if online {
DiagnoseLevel::Ok
} else {
DiagnoseLevel::Warn
},
" network",
if online {
"online"
} else {
"offline; cached data remains available"
},
);
let pending = pending_uploads + pending_changes;
check(
report.finding(
if pending == 0 {
DiagnoseLevel::Ok
} else {
DiagnoseLevel::Warn
},
" queued writes",
true,
if pending == 0 {
"none".to_string()
} else {
format!("{pending} waiting to upload")
},
);
match control_request(CtlRequest::CacheInspect { deep: true }) {
Ok(CtlResponse::CacheReport {
integrity_problems, ..
}) if integrity_problems.is_empty() => report.finding(
DiagnoseLevel::Ok,
" database integrity",
"SQLite integrity_check passed",
),
Ok(CtlResponse::CacheReport {
integrity_problems, ..
}) => report.finding(
DiagnoseLevel::Fail,
" database integrity",
format!(
"{} problem(s): {}",
integrity_problems.len(),
integrity_problems.join("; ")
),
),
Ok(other) => report.finding(
DiagnoseLevel::Fail,
" database integrity",
format!("unexpected: {other:?}"),
),
Err(e) => report.finding(
DiagnoseLevel::Fail,
" database integrity",
format!("check failed: {e:#}"),
),
}
}
Ok(other) => {
all_ok &= check(
report.finding(
DiagnoseLevel::Fail,
" daemon responding",
false,
format!("unexpected: {other:?}"),
);
}
Err(e) => {
// Not a failure of the installation: a stopped daemon is a normal
// state, and the rest of the report is still worth printing.
check(" daemon responding", false, format!("{e:#}"));
println!(" (start it with `systemctl --user start pdfs.service`)");
report.finding(
DiagnoseLevel::Warn,
" daemon responding",
format!("stopped or unreachable: {e:#}"),
);
println!(" (start it with `systemctl --user start proton-drive.service`)");
}
}
let mountpoint = dirs.resolved_mountpoint(&config);
check(
" mountpoint exists",
mountpoint.is_dir(),
format!("{}", mountpoint.display()),
report.finding(
if mountpoint.is_dir() {
DiagnoseLevel::Ok
} else {
DiagnoseLevel::Warn
},
" configured mountpoint",
mountpoint.display().to_string(),
);
if all_ok {
if report.failures == 0 {
if report.warnings == 0 {
println!("\nNo problems found.");
} else {
println!(
"\nNo critical problems found; {} warning(s) need attention.",
report.warnings
);
}
Ok(())
} else {
bail!("one or more checks failed; see the report above")
bail!(
"{} check(s) failed and {} warning(s) were reported; see above",
report.failures,
report.warnings
)
}
}
@@ -1873,3 +2056,39 @@ fn control_request(req: CtlRequest) -> Result<CtlResponse> {
pdfs_core::control::send(&socket, &req)
.with_context(|| format!("talk to mount daemon at {}", socket.display()))
}
#[cfg(test)]
mod diagnose_tests {
use super::*;
#[test]
fn finding_severity_is_counted_centrally() {
let mut report = DiagnoseReport::default();
report.finding(DiagnoseLevel::Ok, "ok", "");
report.finding(DiagnoseLevel::Warn, "warn", "");
report.finding(DiagnoseLevel::Fail, "fail", "");
assert_eq!(report.warnings, 1);
assert_eq!(report.failures, 1);
}
#[test]
fn writable_probe_never_replaces_an_existing_file() {
let dir = std::env::temp_dir().join(format!(
"pdfs-diagnose-test-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
std::fs::create_dir_all(&dir).unwrap();
let collision = dir.join(format!(".pdfs-diagnose-probe-{}", std::process::id()));
std::fs::write(&collision, b"keep me").unwrap();
assert_eq!(
writable_probe(&dir).unwrap_err().kind(),
std::io::ErrorKind::AlreadyExists
);
assert_eq!(std::fs::read(&collision).unwrap(), b"keep me");
std::fs::remove_file(collision).unwrap();
std::fs::remove_dir(dir).unwrap();
}
}
+64 -4
View File
@@ -129,6 +129,25 @@ impl Filesystem for ProtonFs {
reply.opened(FileHandle(0), FopenFlags::empty());
return;
}
// A previous close may already have newer bytes queued locally while
// the server still holds the older revision. That staged blob, not the
// remote, is the base for this handle. In particular, `truncate file`
// opens for writing and then shrinks via setattr; starting its scratch
// as a sparse zero file would turn the preserved prefix into zeros.
//
// Complete pending blobs are the common case and can be copied without
// any network access. An incomplete blob still has gaps referring to
// the remote base; stacking another write on it cannot be represented
// safely by WriteHandle, so fail the open rather than risk corruption.
let pending_base = self.core.pending.lock().get(&uid).cloned();
if pending_base.as_ref().is_some_and(|p| !p.meta.complete) {
if let Some(entry) = self.core.state.lock().entries.get_mut(&ino.0) {
entry.open_count = entry.open_count.saturating_sub(1);
}
error!(%uid, "refusing write over incomplete queued revision");
reply.error(Errno::EIO);
return;
}
let (file, path) = match self.core.cache.create_scratch() {
Ok(x) => x,
Err(e) => {
@@ -140,6 +159,20 @@ impl Filesystem for ProtonFs {
return;
}
};
let mut initial_written = Intervals::default();
if let Some(pending) = &pending_base {
if let Err(e) = std::fs::copy(&pending.path, &path) {
if let Some(entry) = self.core.state.lock().entries.get_mut(&ino.0) {
entry.open_count = entry.open_count.saturating_sub(1);
}
let _ = std::fs::remove_file(&path);
error!(%uid, source = %pending.path.display(), error = %e,
"copy queued revision into write scratch failed");
reply.error(Errno::EIO);
return;
}
initial_written.add(0, pending.meta.len);
}
let mut st = self.core.state.lock();
let fh = st.next_fh;
st.next_fh += 1;
@@ -149,7 +182,7 @@ impl Filesystem for ProtonFs {
uid,
file: Arc::new(file),
path,
written: Intervals::default(),
written: initial_written,
// Starts at the current size; reads in [0, base_size) come from
// the base until overwritten.
len: base_size,
@@ -1045,17 +1078,44 @@ impl Filesystem for ProtonFs {
reply.error(Errno::EIO);
return;
}
if newname != name
&& let Err(e) = self
if newname != name {
// A move changes the node's encrypted name requirements. Proton's
// link-details read can remain briefly stale immediately after the
// move, making the following rename fail with InvalidRequirements
// ("out of date"). Re-read/retry that narrow eventual-consistency
// error; other refusals are stable and must surface immediately.
let mut attempts = 0u32;
let renamed = loop {
match self
.core
.rt
.block_on(self.core.client.rename_node(&uid, &newname, None))
{
error!(%uid, error = %e, "rename failed");
Ok(()) => break Ok(()),
Err(e)
if newparent != parent
&& api_code(&e) == Some(ResponseCode::InvalidRequirements)
&& attempts < 4 =>
{
attempts += 1;
std::thread::sleep(Duration::from_millis(100 * (1 << attempts)));
}
Err(e) => break Err(e),
}
};
if let Err(e) = renamed {
error!(%uid, attempts, error = %e, "rename after move failed");
// The move half already landed. Keep our inode model honest
// even though the combined POSIX operation has to return EIO.
self.core
.state
.lock()
.relocate(ino, parent, newparent, &new_parent_uid, &name);
self.core.restore_replaced(victim.as_ref(), &newname);
reply.error(Errno::EIO);
return;
}
}
self.core
.state
.lock()
+44
View File
@@ -1585,3 +1585,47 @@ continue DB/client activity after teardown.
handles, and ordered shutdown. Repeated mount/unmount tests should return thread
counts to baseline and observe no post-unmount DB work.
---
## B45 — Truncate After a Queued Rewrite Replaces the Prefix With Zeros
**Status:** Fixed and live-verified (2026-07-22)
**Found by:** `scripts/fuse-acceptance.sh`
**Where:** `crates/pdfs-fuse/src/filesystem.rs`, write-open scratch setup
**Repro:** Write a file, immediately replace its contents, then run
`truncate -s 4 file` before the queued replacement drains. Reading it returned
four NUL bytes instead of the first four replacement bytes.
**Cause:** A new write handle always treated the server revision as its base.
When a newer complete revision was still queued locally, the handle's scratch
file began sparse and zero-filled. Shrinking it preserved those scratch zeros,
not the prefix of the locally authoritative queued revision.
**Fix:** A write opened over a complete queued revision seeds its scratch file
from that staged blob and marks the copied range authored. Stacking a write over
an incomplete queued revision is refused rather than guessed. Live acceptance
verified the exact rewrite-then-truncate sequence returns `b"a mu"`.
---
## B46 — Combined Cross-Directory Move and Rename Can Fail Out of Date
**Status:** Fixed and live-verified (2026-07-22)
**Found by:** `scripts/fuse-acceptance.sh`
**Where:** `crates/pdfs-fuse/src/filesystem.rs`, `ProtonFs::rename`
**Repro:** `mv source victim`, replacing an existing file, followed immediately
by `mv victim dir/moved`. Proton accepted the move half of the second operation,
then rejected its rename half with `InvalidRequirements` (HTTP 422, "out of
date"), surfaced as `EIO`.
**Cause:** Moving changes the node's encrypted-name requirements. The following
link-details read can briefly observe the pre-move state, so the new name is
signed against stale requirements.
**Fix:** Retry only `InvalidRequirements` from the rename-after-move step with a
short bounded backoff. Stable API failures are not retried. If every retry
fails, adopt the already-completed move in inode state before returning `EIO`,
so the local model never claims the node remains in its old parent. The full
live suite subsequently passed the replacement and combined move/rename cases.
+55
View File
@@ -0,0 +1,55 @@
# Testing
The normal Rust suite exercises the metadata database, sync planner, offline
queue, write staging, cache, and FUSE handler state without requiring a Proton
account:
```console
cargo test --workspace
cargo clippy --workspace --all-targets -- -D warnings
```
## Live FUSE acceptance suite
Kernel behavior and remote convergence need a real mount. The live suite runs
destructive POSIX operations only below a fresh `pdfs-acceptance-*` directory.
It refuses paths that are not FUSE mounts and removes its directory on success,
failure, or interruption.
Use a dedicated test account without irreplaceable data:
```console
scripts/fuse-acceptance.sh /mnt/on-demand-testmount
```
Passing more paths runs the same filesystem suite independently against each.
The first path must be the FUSE mount under test; later paths may be another
FUSE mount or a normal local mirror filesystem:
```console
PDFS_ACCEPTANCE_SYNC_TIMEOUT=180 \
scripts/fuse-acceptance.sh /mnt/on-demand-testmount /mnt/testmount
```
If two paths really are views of the **same remote folder**, enable an
additional byte-for-byte convergence check explicitly:
```console
PDFS_ACCEPTANCE_CONVERGENCE=1 \
scripts/fuse-acceptance.sh /mnt/first-view /mnt/second-view
```
Do not enable that flag merely because two sync folders belong to the same
account. Their configured remote UIDs must be identical.
The suite covers create/read/rewrite/truncate, multi-megabyte binary content,
unaligned random writes, rename replacement, cross-directory moves, important
POSIX error cases, open-then-unlink behavior, concurrent writers, directory
enumeration, and optional cross-mount byte-for-byte convergence. A pass is
strong evidence for those paths, not a blanket data-loss guarantee. Before a
release, also run the recovery drills in `docs/RECOVERY.md` and inspect
`docs/BUGS.md` for live-verification items.
The runner accepts already-mounted paths instead of starting a daemon. This
keeps authentication and keyring setup explicit and prevents it from silently
reusing or replacing the normal application state directory.
+1 -1
View File
@@ -9,7 +9,7 @@
# Build + install: cd packaging && makepkg -fi
pkgname=proton-drive-linux
pkgver=0.7.0
pkgver=0.7.1
pkgrel=1
pkgdesc="Proton Drive client for Linux: FUSE files-on-demand mount, CLI, GTK4 app + tray"
arch=('x86_64')
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env bash
# Destructive only inside a uniquely named directory created by this script.
set -Eeuo pipefail
usage() {
echo "usage: $0 MOUNTPOINT [MOUNTPOINT ...]" >&2
echo "example: $0 /mnt/on-demand-testmount /mnt/testmount" >&2
exit 2
}
(( $# > 0 )) || usage
command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 2; }
command -v findmnt >/dev/null || { echo "findmnt is required" >&2; exit 2; }
mounts=("$@")
run_id="pdfs-acceptance-$(date +%Y%m%dT%H%M%S)-$$"
roots=()
cleanup() {
local root
for root in "${roots[@]}"; do
[[ -n "$root" && "$root" == */pdfs-acceptance-* ]] || continue
rm -rf -- "$root" 2>/dev/null || true
done
}
trap cleanup EXIT INT TERM
for index in "${!mounts[@]}"; do
mountpoint="${mounts[$index]}"
[[ -d "$mountpoint" ]] || { echo "not a directory: $mountpoint" >&2; exit 2; }
fs_type="$(findmnt -T "$mountpoint" -n -o FSTYPE | head -n 1)"
if (( index == 0 )) && [[ "$fs_type" != fuse* ]]; then
echo "refusing primary non-FUSE path $mountpoint (type: ${fs_type:-unknown})" >&2
exit 2
fi
roots+=("${mountpoint%/}/$run_id")
done
step() { printf '[test] %s\n' "$1"; }
assert_text() {
python3 - "$1" "$2" <<'PY'
import sys
p, expected = sys.argv[1], sys.argv[2].encode()
with open(p, "rb") as f: got = f.read()
if got != expected:
raise AssertionError(
f"{p}: expected {expected!r} ({len(expected)} bytes), "
f"got {got!r} ({len(got)} bytes)"
)
PY
}
run_suite() {
local primary="$1"
printf '[mount] testing %s\n' "${primary%/*}"
mkdir -- "$primary"
step "create, read, overwrite, truncate"
printf 'alpha\n' > "$primary/plain.txt"
assert_text "$primary/plain.txt" $'alpha\n'
printf 'a much longer replacement\n' > "$primary/plain.txt"
truncate -s 4 "$primary/plain.txt"
assert_text "$primary/plain.txt" "a mu"
step "binary round-trip and random-access writes"
python3 - "$primary/binary.dat" <<'PY'
import hashlib, sys
p = sys.argv[1]
data = bytes(range(256)) * 16384
with open(p, "wb") as f: f.write(data)
with open(p, "r+b", buffering=0) as f:
f.seek(4093); f.write(b"boundary-write")
with open(p, "rb") as f: got = f.read()
expected = data[:4093] + b"boundary-write" + data[4107:]
assert got == expected
print(hashlib.sha256(got).hexdigest())
PY
step "rename, move, and replacement semantics"
mkdir -- "$primary/dir-a" "$primary/dir-b"
printf 'source' > "$primary/source"
printf 'victim' > "$primary/victim"
mv -- "$primary/source" "$primary/victim"
assert_text "$primary/victim" "source"
[[ ! -e "$primary/source" ]]
mv -- "$primary/victim" "$primary/dir-a/moved"
assert_text "$primary/dir-a/moved" "source"
step "POSIX error behavior"
if rmdir -- "$primary/dir-a/moved" 2>/dev/null; then echo "rmdir removed a file" >&2; exit 1; fi
if rm -- "$primary/dir-a" 2>/dev/null; then echo "rm removed a directory" >&2; exit 1; fi
if mv -- "$primary/dir-a" "$primary/dir-a/child" 2>/dev/null; then
echo "moving a directory below itself succeeded" >&2; exit 1
fi
step "open file remains readable after unlink"
python3 - "$primary/open-unlink" <<'PY'
import os, sys
p = sys.argv[1]; data = b"held-open\0" * 4096
with open(p, "wb") as f: f.write(data)
f = open(p, "rb", buffering=0); os.unlink(p)
assert not os.path.exists(p) and f.read() == data
f.close()
PY
step "concurrent independent writers"
for i in $(seq 1 12); do
(python3 - "$primary/concurrent-$i" "$i" <<'PY'
import sys
p, seed = sys.argv[1], int(sys.argv[2]); data = bytes([seed]) * (131072 + seed)
with open(p, "wb") as f: f.write(data)
with open(p, "rb") as f: assert f.read() == data
PY
) &
done
wait
step "directory enumeration agrees with stat"
python3 - "$primary" <<'PY'
import os, sys
for name in os.listdir(sys.argv[1]): os.lstat(os.path.join(sys.argv[1], name))
PY
}
for root in "${roots[@]}"; do
run_suite "$root"
done
if [[ "${PDFS_ACCEPTANCE_CONVERGENCE:-0}" == 1 ]] && (( ${#roots[@]} > 1 )); then
step "cross-mount convergence"
primary="${roots[0]}"
expected="$(sha256sum "$primary/binary.dat" | cut -d' ' -f1)"
for secondary in "${roots[@]:1}"; do
deadline=$((SECONDS + ${PDFS_ACCEPTANCE_SYNC_TIMEOUT:-120}))
while (( SECONDS < deadline )); do
if [[ -f "$secondary/binary.dat" ]] &&
[[ "$(sha256sum "$secondary/binary.dat" 2>/dev/null | cut -d' ' -f1)" == "$expected" ]]; then break; fi
sleep 2
done
[[ -f "$secondary/binary.dat" ]] || { echo "did not appear on $secondary" >&2; exit 1; }
[[ "$(sha256sum "$secondary/binary.dat" | cut -d' ' -f1)" == "$expected" ]] || {
echo "content mismatch on $secondary" >&2; exit 1
}
done
fi
step "cleanup"
cleanup
roots=()
echo "PASS: FUSE acceptance suite completed ($run_id)"