mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-04 16:56:59 +01:00
Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
942cb9f583 | ||
|
|
8470662c54 | ||
|
|
a091c2f34f | ||
|
|
bca0fe34bc | ||
|
|
6df757006a | ||
|
|
0d8a474442 | ||
|
|
8be0f04b61 | ||
|
|
a76f36bdff | ||
|
|
d0b49d3c8c | ||
|
|
25f578161d |
No files matched your search
@@ -44,6 +44,9 @@ jobs:
|
|||||||
- name: Unit tests
|
- name: Unit tests
|
||||||
run: cargo test --workspace --locked
|
run: cargo test --workspace --locked
|
||||||
|
|
||||||
|
- name: Account-free filesystem acceptance
|
||||||
|
run: scripts/fuse-acceptance.sh --offline-only
|
||||||
|
|
||||||
- name: Acceptance runner safety guard
|
- name: Acceptance runner safety guard
|
||||||
run: |
|
run: |
|
||||||
if scripts/fuse-acceptance.sh /tmp >acceptance.out 2>&1; then
|
if scripts/fuse-acceptance.sh /tmp >acceptance.out 2>&1; then
|
||||||
|
|||||||
@@ -35,11 +35,29 @@ jobs:
|
|||||||
- name: Set up Rust
|
- name: Set up Rust
|
||||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||||
with:
|
with:
|
||||||
toolchain: stable
|
toolchain: '1.96'
|
||||||
|
components: rustfmt, clippy
|
||||||
|
|
||||||
- name: Cache Cargo target
|
- name: Cache Cargo target
|
||||||
uses: swatinem/rust-cache@v2
|
uses: swatinem/rust-cache@v2
|
||||||
|
|
||||||
|
- name: Verify tag and manifest versions
|
||||||
|
if: startsWith(github.ref, 'refs/tags/')
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
tag_version="${GITHUB_REF_NAME#v}"
|
||||||
|
manifest_version="$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -n1)"
|
||||||
|
pkgbuild_version="$(sed -n 's/^pkgver=//p' packaging/PKGBUILD | head -n1)"
|
||||||
|
test "$tag_version" = "$manifest_version"
|
||||||
|
test "$tag_version" = "$pkgbuild_version"
|
||||||
|
|
||||||
|
- name: Release quality gates
|
||||||
|
run: |
|
||||||
|
cargo fmt --all -- --check
|
||||||
|
cargo clippy --workspace --all-targets -- -D warnings
|
||||||
|
cargo test --workspace --locked
|
||||||
|
scripts/fuse-acceptance.sh --offline-only
|
||||||
|
|
||||||
- name: Build release binaries
|
- name: Build release binaries
|
||||||
run: |
|
run: |
|
||||||
cargo build --release --locked
|
cargo build --release --locked
|
||||||
@@ -76,6 +94,8 @@ jobs:
|
|||||||
mkdir -p deb/usr/share/applications
|
mkdir -p deb/usr/share/applications
|
||||||
mkdir -p deb/usr/share/icons/hicolor/scalable/apps
|
mkdir -p deb/usr/share/icons/hicolor/scalable/apps
|
||||||
mkdir -p deb/usr/share/licenses/proton-drive-linux
|
mkdir -p deb/usr/share/licenses/proton-drive-linux
|
||||||
|
mkdir -p deb/usr/lib/systemd/user
|
||||||
|
mkdir -p deb/etc/xdg/autostart
|
||||||
|
|
||||||
# Copy binaries
|
# Copy binaries
|
||||||
cp target/release/pdfs deb/usr/bin/
|
cp target/release/pdfs deb/usr/bin/
|
||||||
@@ -85,7 +105,9 @@ jobs:
|
|||||||
|
|
||||||
# Copy desktop integration files
|
# Copy desktop integration files
|
||||||
cp packaging/io.narl.proton-drive-linux.desktop deb/usr/share/applications/
|
cp packaging/io.narl.proton-drive-linux.desktop deb/usr/share/applications/
|
||||||
|
cp packaging/io.narl.proton-drive-linux-tray.desktop deb/etc/xdg/autostart/
|
||||||
cp packaging/io.narl.proton-drive-linux.svg deb/usr/share/icons/hicolor/scalable/apps/
|
cp packaging/io.narl.proton-drive-linux.svg deb/usr/share/icons/hicolor/scalable/apps/
|
||||||
|
cp packaging/proton-drive.service deb/usr/lib/systemd/user/
|
||||||
cp LICENSE deb/usr/share/licenses/proton-drive-linux/
|
cp LICENSE deb/usr/share/licenses/proton-drive-linux/
|
||||||
|
|
||||||
# Calculate Installed-Size in KB for the control file
|
# Calculate Installed-Size in KB for the control file
|
||||||
|
|||||||
Generated
+56
-55
@@ -378,7 +378,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.0",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -663,9 +663,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytemuck"
|
name = "bytemuck"
|
||||||
version = "1.25.1"
|
version = "1.25.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424"
|
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "byteorder"
|
name = "byteorder"
|
||||||
@@ -747,9 +747,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.3.0"
|
version = "1.4.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
|
checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"find-msvc-tools",
|
"find-msvc-tools",
|
||||||
"jobserver",
|
"jobserver",
|
||||||
@@ -856,9 +856,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.6.2"
|
version = "4.6.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011"
|
checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap_builder",
|
"clap_builder",
|
||||||
"clap_derive",
|
"clap_derive",
|
||||||
@@ -878,14 +878,14 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap_derive"
|
name = "clap_derive"
|
||||||
version = "4.6.1"
|
version = "4.6.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
|
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck",
|
"heck",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.119",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1479,9 +1479,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "either"
|
name = "either"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
|
checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "elliptic-curve"
|
name = "elliptic-curve"
|
||||||
@@ -1568,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2359,9 +2359,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyper"
|
name = "hyper"
|
||||||
version = "1.10.1"
|
version = "1.11.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
|
checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atomic-waker",
|
"atomic-waker",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -2559,9 +2559,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ignore"
|
name = "ignore"
|
||||||
version = "0.4.30"
|
version = "0.4.31"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7b009b6744c1445efd7244084e25e498636412effb6760b55067553baa925cc7"
|
checksum = "7f8a7b8211e695a1d0cd91cace480d4d0bd57667ab10277cc412c5f7f4884f83"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crossbeam-deque",
|
"crossbeam-deque",
|
||||||
"globset",
|
"globset",
|
||||||
@@ -2916,9 +2916,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libc"
|
name = "libc"
|
||||||
version = "0.2.186"
|
version = "0.2.189"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libdbus-sys"
|
name = "libdbus-sys"
|
||||||
@@ -3203,7 +3203,7 @@ version = "0.50.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -3518,10 +3518,10 @@ checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-cli"
|
name = "pdfs-cli"
|
||||||
version = "0.7.1"
|
version = "1.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"clap 4.6.2",
|
"clap 4.6.4",
|
||||||
"pdfs-core",
|
"pdfs-core",
|
||||||
"pdfs-fuse",
|
"pdfs-fuse",
|
||||||
"rpassword",
|
"rpassword",
|
||||||
@@ -3533,11 +3533,12 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-core"
|
name = "pdfs-core"
|
||||||
version = "0.7.1"
|
version = "1.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"directories",
|
"directories",
|
||||||
"ignore",
|
"ignore",
|
||||||
"keyring",
|
"keyring",
|
||||||
|
"libc",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"proton-drive-rs",
|
"proton-drive-rs",
|
||||||
"proton-sdk",
|
"proton-sdk",
|
||||||
@@ -3552,7 +3553,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-fuse"
|
name = "pdfs-fuse"
|
||||||
version = "0.7.1"
|
version = "1.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"fuser",
|
"fuser",
|
||||||
"image",
|
"image",
|
||||||
@@ -3568,7 +3569,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pdfs-gui"
|
name = "pdfs-gui"
|
||||||
version = "0.7.1"
|
version = "1.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-channel",
|
"async-channel",
|
||||||
@@ -3829,9 +3830,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proton-drive-rs"
|
name = "proton-drive-rs"
|
||||||
version = "0.1.11"
|
version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9a76197ffe7c9588c0bc1fa38a51e8aa8642b10e81ec16272d4962bd60525920"
|
checksum = "692610fbead83735a0415d9d068c1e9b467b12cce013ea19b3063f05206ce3d6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64",
|
"base64",
|
||||||
@@ -3856,9 +3857,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proton-sdk"
|
name = "proton-sdk"
|
||||||
version = "0.1.11"
|
version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "805df5ff2805e0e935ac7f5a237482231208a706d8c6ab3dd91157ae2e95c9f6"
|
checksum = "6b3f2ad9cd1ba69b67cbafcaf7c159b3a200162a3c518b1bc7e25df837343b41"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm 0.11.0",
|
"aes-gcm 0.11.0",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -4213,7 +4214,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.0",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4400,7 +4401,7 @@ dependencies = [
|
|||||||
"errno",
|
"errno",
|
||||||
"libc",
|
"libc",
|
||||||
"linux-raw-sys",
|
"linux-raw-sys",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4431,9 +4432,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-pki-types"
|
name = "rustls-pki-types"
|
||||||
version = "1.15.0"
|
version = "1.15.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
|
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"web-time",
|
"web-time",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -4457,7 +4458,7 @@ dependencies = [
|
|||||||
"security-framework",
|
"security-framework",
|
||||||
"security-framework-sys",
|
"security-framework-sys",
|
||||||
"webpki-root-certs",
|
"webpki-root-certs",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4598,14 +4599,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.0",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_json"
|
name = "serde_json"
|
||||||
version = "1.0.150"
|
version = "1.0.151"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itoa",
|
"itoa",
|
||||||
"memchr",
|
"memchr",
|
||||||
@@ -4622,7 +4623,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.0",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4783,18 +4784,18 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "snafu"
|
name = "snafu"
|
||||||
version = "0.9.1"
|
version = "0.9.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522"
|
checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"snafu-derive",
|
"snafu-derive",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "snafu-derive"
|
name = "snafu-derive"
|
||||||
version = "0.9.1"
|
version = "0.9.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda"
|
checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck",
|
"heck",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
@@ -4897,9 +4898,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "3.0.0"
|
version = "3.0.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f2fac314a64dc9a36e61a9eb4261a5e9bbfbc922b27e518af97bc32b926cf967"
|
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
@@ -4961,7 +4962,7 @@ dependencies = [
|
|||||||
"getrandom 0.4.3",
|
"getrandom 0.4.3",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rustix",
|
"rustix",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5010,7 +5011,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.0",
|
"syn 3.0.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5063,9 +5064,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio"
|
name = "tokio"
|
||||||
version = "1.53.0"
|
version = "1.53.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
|
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"libc",
|
"libc",
|
||||||
@@ -5100,9 +5101,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-util"
|
name = "tokio-util"
|
||||||
version = "0.7.18"
|
version = "0.7.19"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
|
checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
@@ -5627,7 +5628,7 @@ version = "0.1.11"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"windows-sys 0.48.0",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -6137,18 +6138,18 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerocopy"
|
name = "zerocopy"
|
||||||
version = "0.8.54"
|
version = "0.8.55"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
|
checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"zerocopy-derive",
|
"zerocopy-derive",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zerocopy-derive"
|
name = "zerocopy-derive"
|
||||||
version = "0.8.54"
|
version = "0.8.55"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
|
checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
|
|||||||
+3
-3
@@ -3,15 +3,15 @@ resolver = "2"
|
|||||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.7.1"
|
version = "1.1.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||||
rust-version = "1.96"
|
rust-version = "1.96"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
proton-sdk = { version = "0.1" }
|
proton-sdk = { version = "0.2.2" }
|
||||||
proton-drive-rs = { version = "0.1" }
|
proton-drive-rs = { version = "0.2.2" }
|
||||||
|
|
||||||
# Internal crates.
|
# Internal crates.
|
||||||
pdfs-core = { path = "crates/pdfs-core" }
|
pdfs-core = { path = "crates/pdfs-core" }
|
||||||
|
|||||||
@@ -10,11 +10,18 @@ A fast, unofficial Proton Drive client for Linux. This client features an advanc
|
|||||||
- **Shared Links & Invites**: Browse files shared with you by other users, and view/manage your own public shared links directly in the GUI.
|
- **Shared Links & Invites**: Browse files shared with you by other users, and view/manage your own public shared links directly in the GUI.
|
||||||
- **Local Backup (Computers)**: Sync and back up local directories (like Downloads, Documents, Pictures, etc.) directly to your Proton Drive account.
|
- **Local Backup (Computers)**: Sync and back up local directories (like Downloads, Documents, Pictures, etc.) directly to your Proton Drive account.
|
||||||
- **System Tray Integration**: Background indicator for status monitoring, quick actions, and fast sync controls.
|
- **System Tray Integration**: Background indicator for status monitoring, quick actions, and fast sync controls.
|
||||||
- **Fuzzy Search Launcher Prompt (HUD)**: A Google Drive-style search launcher (`pdfs-prompt`) for finding and opening files or folders instantly, ideal for binding to a system-wide hotkey.
|
- **Unified Search Launcher (HUD)**: A resident Google Drive-style launcher (`pdfs-prompt`) that searches Proton Drive and local files together, ranks the best matches, tolerates abbreviations and typos, and is ideal for a system-wide hotkey.
|
||||||
- **Secure Credential Storage**: Integrates with the system Secret Service (GNOME Keyring, KWallet, etc.) with smart in-memory credential caching to avoid UI thread blockages.
|
- **Secure Credential Storage**: Integrates with the system Secret Service (GNOME Keyring, KWallet, etc.) with smart in-memory credential caching to avoid UI thread blockages.
|
||||||
- **Proton Photos Support**: Access your Proton Photos timeline, view thumbnails, and download backed-up media natively (available in the GUI as a navigation tab and via the CLI).
|
- **Proton Photos Support**: Access your Proton Photos timeline, view thumbnails, and download backed-up media natively (available in the GUI as a navigation tab and via the CLI).
|
||||||
- **Human Verification (CAPTCHA) Recovery**: Detects sign-in gates (VPN/new IP challenges) and launches an embedded `WebKitWebView` dialog to safely complete the challenge, transparently retrying authentication with the earned token.
|
- **Human Verification (CAPTCHA) Recovery**: Detects sign-in gates (VPN/new IP challenges) and launches an embedded `WebKitWebView` dialog to safely complete the challenge, transparently retrying authentication with the earned token.
|
||||||
- **Selective Sync (`.pdfsignore`)**: Keep build trees, dependency directories, and editor leftovers out of synced folders using gitignore-style rules.
|
- **Selective Sync (`.pdfsignore`)**: Keep build trees, dependency directories, and editor leftovers out of synced folders using gitignore-style rules.
|
||||||
|
- **Data-Safe Offline Writes**: Durable scratch/staging files and a transactional pending queue preserve acknowledged writes across network failures and restarts.
|
||||||
|
|
||||||
|
## Quick Search and Media Opening
|
||||||
|
|
||||||
|
`pdfs-prompt` searches Drive metadata and the local home-directory index in one request. Results are ranked together rather than split by source. Matching is case-insensitive and supports exact names, prefixes, parent paths, multiple terms, punctuation-separated words, ordered abbreviations, common typos, and adjacent transpositions.
|
||||||
|
|
||||||
|
The prompt stays resident after its first launch, so invoking the shortcut again reuses and resets the existing window. Drive folders open from the mount. Audio and video results also open through FUSE, allowing media players to seek and request ranges without downloading the entire file first; ordinary Drive files are materialized into the local cache before opening.
|
||||||
|
|
||||||
## Selective Sync (`.pdfsignore`)
|
## Selective Sync (`.pdfsignore`)
|
||||||
|
|
||||||
@@ -131,6 +138,19 @@ The client includes several optimizations designed for high efficiency, a low me
|
|||||||
- **Disk-Backed Writes**: Large file writes are staged on disk in temporary scratch files (rather than fully buffered in RAM) and track modified byte intervals. Only the unedited remote segments are fetched at commit time, keeping memory usage minimal.
|
- **Disk-Backed Writes**: Large file writes are staged on disk in temporary scratch files (rather than fully buffered in RAM) and track modified byte intervals. Only the unedited remote segments are fetched at commit time, keeping memory usage minimal.
|
||||||
- **Non-Blocking GTK4 Loop**: To prevent UI freezes, all synchronous D-Bus credential checks, control socket requests, and cache usage calculations are offloaded to background worker threads or fetched asynchronously.
|
- **Non-Blocking GTK4 Loop**: To prevent UI freezes, all synchronous D-Bus credential checks, control socket requests, and cache usage calculations are offloaded to background worker threads or fetched asynchronously.
|
||||||
- **Flicker-Free UI Rendering**: The GTK4 application performs differential rendering of the pins list, only modifying list rows when the list content changes, preserving the user's scroll position and widget focus.
|
- **Flicker-Free UI Rendering**: The GTK4 application performs differential rendering of the pins list, only modifying list rows when the list content changes, preserving the user's scroll position and widget focus.
|
||||||
|
- **Durable Staging**: Scratch metadata and staged writes are synced and atomically published before an upload is acknowledged locally. Rapid revisions supersede pending work transactionally.
|
||||||
|
|
||||||
|
## Filesystem Safety
|
||||||
|
|
||||||
|
Version 1.0 strengthens the boundaries around local-only data and destructive reconciliation:
|
||||||
|
|
||||||
|
- Truncate composes with queued revisions, preserving the authoritative prefix and correct zero-filled growth.
|
||||||
|
- Combined move-and-rename operations are queued durably and tolerate partially completed remote state.
|
||||||
|
- Session tombstones prevent successfully unlinked entries from reappearing through eventually consistent listings.
|
||||||
|
- Names are valid UTF-8 and no longer than Linux's 255-byte component limit.
|
||||||
|
- Incomplete mirror scans are non-destructive, and total-wipe protection covers every non-empty baseline.
|
||||||
|
- Failed conflict preservation, local deletion, or staging publication does not settle a new baseline or discard the only local copy.
|
||||||
|
- State, cache, and config directories must be real, current-user-owned `0700` directories; control sockets are `0600`.
|
||||||
|
|
||||||
|
|
||||||
## Screenshots
|
## Screenshots
|
||||||
@@ -258,15 +278,18 @@ This project has a GitHub Actions CI workflow configured under `.github/workflow
|
|||||||
1. **Triggers**:
|
1. **Triggers**:
|
||||||
- Pushing a git tag matching `v*` (e.g. `git tag v0.1.0 && git push origin v0.1.0`).
|
- Pushing a git tag matching `v*` (e.g. `git tag v0.1.0 && git push origin v0.1.0`).
|
||||||
- Manual runs via the **Actions** tab in GitHub (**workflow_dispatch**).
|
- Manual runs via the **Actions** tab in GitHub (**workflow_dispatch**).
|
||||||
2. **Build Process**:
|
2. **Quality Gates and Build Process**:
|
||||||
|
- For tagged builds, verifies that the tag, workspace version, and `packaging/PKGBUILD` version agree.
|
||||||
|
- Runs `cargo fmt`, Clippy with warnings denied, the locked workspace test suite, and the account-free FUSE acceptance contract.
|
||||||
- Spawns an Ubuntu runner and installs GTK4, Libadwaita, FUSE3, and Secret Service packages.
|
- Spawns an Ubuntu runner and installs GTK4, Libadwaita, FUSE3, and Secret Service packages.
|
||||||
- Sets up the Rust compiler and caches build targets to speed up runs.
|
- Sets up the Rust compiler and caches build targets to speed up runs.
|
||||||
- Compiles the workspace members in release mode.
|
- Compiles the workspace members in release mode.
|
||||||
3. **Artifact Packaging**:
|
3. **Artifact Packaging**:
|
||||||
- Generates a `.tar.gz` containing the raw binaries (`pdfs`, `pdfs-app`, `pdfs-tray`, `pdfs-prompt`).
|
- Generates a `.tar.gz` containing the raw binaries (`pdfs`, `pdfs-app`, `pdfs-tray`, `pdfs-prompt`).
|
||||||
- Packs them into a Debian package (`.deb`).
|
- Packs them into Debian (`.deb`) and Fedora (`.rpm`) packages.
|
||||||
|
- Includes the systemd user service and tray autostart entry in the Debian package.
|
||||||
4. **Publishing**:
|
4. **Publishing**:
|
||||||
- Creates a GitHub Release matching the pushed tag and uploads the `.deb` and `.tar.gz` packages as release assets.
|
- Creates a GitHub Release matching the pushed tag and uploads the `.deb`, `.rpm`, and `.tar.gz` packages as release assets.
|
||||||
- For manual runs, compiles and exposes the packages as workflow run artifacts for testing.
|
- For manual runs, compiles and exposes the packages as workflow run artifacts for testing.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -1002,6 +1002,7 @@ fn activity_verb(kind: pdfs_core::control::ActivityKind) -> &'static str {
|
|||||||
Share => "share",
|
Share => "share",
|
||||||
PublicLink => "link",
|
PublicLink => "link",
|
||||||
Unshare => "unshare",
|
Unshare => "unshare",
|
||||||
|
Conflict => "conflict",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1290,7 +1291,10 @@ fn mount_once(mountpoint: Option<PathBuf>) -> Result<pdfs_fuse::MountOutcome> {
|
|||||||
cache,
|
cache,
|
||||||
&control_socket,
|
&control_socket,
|
||||||
db,
|
db,
|
||||||
|
pdfs_fuse::MountOptions {
|
||||||
username,
|
username,
|
||||||
|
sweep_mode: config.resolved_conflict_sweep(),
|
||||||
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
// Clean unmount check (as a best-effort final backup).
|
// Clean unmount check (as a best-effort final backup).
|
||||||
|
|||||||
@@ -21,3 +21,4 @@ sha2.workspace = true
|
|||||||
rusqlite.workspace = true
|
rusqlite.workspace = true
|
||||||
keyring.workspace = true
|
keyring.workspace = true
|
||||||
parking_lot.workspace = true
|
parking_lot.workspace = true
|
||||||
|
libc.workspace = true
|
||||||
+128
-30
@@ -150,15 +150,23 @@ pub struct StagedWrite {
|
|||||||
pub based_on: Option<Baseline>,
|
pub based_on: Option<Baseline>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The identity of a remote revision, as far as we can observe it: a file whose
|
/// The identity of a remote revision, as far as we can observe it.
|
||||||
/// size and mtime both still match the ones a queued change was made against has
|
///
|
||||||
/// not been rewritten by anyone else in the meantime.
|
/// `revision_id` is the authoritative identity when present: the server assigns a
|
||||||
|
/// fresh id to every sealed revision, so an unchanged id proves no one rewrote
|
||||||
|
/// the file — even when the server re-stamped its `mtime` on the *same* revision,
|
||||||
|
/// which is the drift that used to cut spurious conflict copies (B16/B25).
|
||||||
|
/// `mtime`/`size` remain the fallback for a baseline recorded before the id was
|
||||||
|
/// observable, or a node read through a surface that does not carry it.
|
||||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
|
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct Baseline {
|
pub struct Baseline {
|
||||||
pub mtime: i64,
|
pub mtime: i64,
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub hash: Option<String>,
|
pub hash: Option<String>,
|
||||||
|
/// Server revision id this change was made against (`None` if unknown).
|
||||||
|
#[serde(default)]
|
||||||
|
pub revision_id: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Content cache rooted at a directory, with a sibling pin-registry file.
|
/// Content cache rooted at a directory, with a sibling pin-registry file.
|
||||||
@@ -241,7 +249,9 @@ impl ContentCache {
|
|||||||
let recovery_dir = content_dir.join("recovery");
|
let recovery_dir = content_dir.join("recovery");
|
||||||
std::fs::create_dir_all(&recovery_dir)?;
|
std::fs::create_dir_all(&recovery_dir)?;
|
||||||
Self::rescue_scratch(&scratch_dir, &recovery_dir);
|
Self::rescue_scratch(&scratch_dir, &recovery_dir);
|
||||||
let _ = std::fs::remove_dir_all(&scratch_dir);
|
// A failed rescue may leave the only durable copy here. Never clear the
|
||||||
|
// directory wholesale; discard only blobs that carry no fsync marker.
|
||||||
|
Self::discard_unmarked_scratch(&scratch_dir);
|
||||||
std::fs::create_dir_all(&scratch_dir)?;
|
std::fs::create_dir_all(&scratch_dir)?;
|
||||||
// Staging, by contrast, is deliberately preserved across runs.
|
// Staging, by contrast, is deliberately preserved across runs.
|
||||||
let staging_dir = content_dir.join("staging");
|
let staging_dir = content_dir.join("staging");
|
||||||
@@ -669,18 +679,24 @@ impl ContentCache {
|
|||||||
pub fn create_scratch(&self) -> Result<(std::fs::File, PathBuf)> {
|
pub fn create_scratch(&self) -> Result<(std::fs::File, PathBuf)> {
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
static N: AtomicU64 = AtomicU64::new(0);
|
static N: AtomicU64 = AtomicU64::new(0);
|
||||||
|
loop {
|
||||||
let path = self.scratch_dir.join(format!(
|
let path = self.scratch_dir.join(format!(
|
||||||
"w-{}-{}",
|
"w-{}-{}-{}",
|
||||||
std::process::id(),
|
std::process::id(),
|
||||||
|
now_millis(),
|
||||||
N.fetch_add(1, Ordering::Relaxed)
|
N.fetch_add(1, Ordering::Relaxed)
|
||||||
));
|
));
|
||||||
let file = std::fs::OpenOptions::new()
|
match std::fs::OpenOptions::new()
|
||||||
.create(true)
|
.create_new(true)
|
||||||
.truncate(true)
|
|
||||||
.read(true)
|
.read(true)
|
||||||
.write(true)
|
.write(true)
|
||||||
.open(&path)?;
|
.open(&path)
|
||||||
Ok((file, path))
|
{
|
||||||
|
Ok(file) => return Ok((file, path)),
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||||
|
Err(e) => return Err(e.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The sidecar path for a scratch file, holding the [`StagedWrite`] that
|
/// The sidecar path for a scratch file, holding the [`StagedWrite`] that
|
||||||
@@ -709,16 +725,23 @@ impl ContentCache {
|
|||||||
/// have synced `scratch` first: a sidecar promising bytes that never reached
|
/// have synced `scratch` first: a sidecar promising bytes that never reached
|
||||||
/// the disk is worse than no sidecar.
|
/// the disk is worse than no sidecar.
|
||||||
pub fn mark_scratch_durable(&self, scratch: &Path, meta: &StagedWrite) -> Result<()> {
|
pub fn mark_scratch_durable(&self, scratch: &Path, meta: &StagedWrite) -> Result<()> {
|
||||||
|
use std::io::Write as _;
|
||||||
|
|
||||||
let side = Self::scratch_sidecar(scratch);
|
let side = Self::scratch_sidecar(scratch);
|
||||||
let tmp = side.with_extension("json.tmp");
|
let tmp = side.with_extension("json.tmp");
|
||||||
std::fs::write(&tmp, serde_json::to_vec(meta)?)?;
|
let mut file = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create(true)
|
||||||
|
.truncate(true)
|
||||||
|
.open(&tmp)?;
|
||||||
|
file.write_all(&serde_json::to_vec(meta)?)?;
|
||||||
|
file.sync_all()?;
|
||||||
|
drop(file);
|
||||||
std::fs::rename(&tmp, &side)?;
|
std::fs::rename(&tmp, &side)?;
|
||||||
// The rename is what publishes the sidecar; fsync the directory so the
|
// The rename is what publishes the sidecar; fsync the directory so the
|
||||||
// rename itself survives, not just the bytes it points at.
|
// rename itself survives, not just the bytes it points at.
|
||||||
if let Some(dir) = side.parent()
|
if let Some(dir) = side.parent() {
|
||||||
&& let Ok(d) = std::fs::File::open(dir)
|
std::fs::File::open(dir)?.sync_all()?;
|
||||||
{
|
|
||||||
let _ = d.sync_all();
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -796,6 +819,21 @@ impl ContentCache {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn discard_unmarked_scratch(scratch_dir: &Path) {
|
||||||
|
let Ok(entries) = std::fs::read_dir(scratch_dir) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let blob = entry.path();
|
||||||
|
if blob.extension().is_some_and(|e| e == "json") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !Self::scratch_sidecar(&blob).exists() {
|
||||||
|
let _ = std::fs::remove_file(blob);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Emergency eviction of unpinned blobs when local disk runs out of space (ENOSPC).
|
/// Emergency eviction of unpinned blobs when local disk runs out of space (ENOSPC).
|
||||||
pub fn emergency_evict(&self) {
|
pub fn emergency_evict(&self) {
|
||||||
let pinned: HashSet<String> = self
|
let pinned: HashSet<String> = self
|
||||||
@@ -826,7 +864,8 @@ impl ContentCache {
|
|||||||
|
|
||||||
/// Writes rescued from an unclean shutdown, as `(blob, meta)` pairs for the
|
/// Writes rescued from an unclean shutdown, as `(blob, meta)` pairs for the
|
||||||
/// daemon to hand to the upload queue. A blob whose sidecar did not survive
|
/// daemon to hand to the upload queue. A blob whose sidecar did not survive
|
||||||
/// the rescue is deleted here — nothing can address it.
|
/// is not actionable automatically, but is retained for manual recovery:
|
||||||
|
/// unexplained bytes are never evidence that user data is disposable.
|
||||||
pub fn recovered_writes(&self) -> Vec<(PathBuf, StagedWrite)> {
|
pub fn recovered_writes(&self) -> Vec<(PathBuf, StagedWrite)> {
|
||||||
let Ok(entries) = std::fs::read_dir(&self.recovery_dir) else {
|
let Ok(entries) = std::fs::read_dir(&self.recovery_dir) else {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -837,14 +876,11 @@ impl ContentCache {
|
|||||||
if blob.extension().is_some_and(|e| e == "json") {
|
if blob.extension().is_some_and(|e| e == "json") {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
match std::fs::read(Self::scratch_sidecar(&blob))
|
if let Some(meta) = std::fs::read(Self::scratch_sidecar(&blob))
|
||||||
.ok()
|
.ok()
|
||||||
.and_then(|b| serde_json::from_slice::<StagedWrite>(&b).ok())
|
.and_then(|b| serde_json::from_slice::<StagedWrite>(&b).ok())
|
||||||
{
|
{
|
||||||
Some(meta) => out.push((blob, meta)),
|
out.push((blob, meta));
|
||||||
None => {
|
|
||||||
let _ = std::fs::remove_file(&blob);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
@@ -872,6 +908,8 @@ impl ContentCache {
|
|||||||
/// failure here means we could not save the bytes at all, so it is reported
|
/// failure here means we could not save the bytes at all, so it is reported
|
||||||
/// rather than swallowed.
|
/// rather than swallowed.
|
||||||
pub fn stage_write(&self, meta: &StagedWrite, scratch: &Path) -> Result<PathBuf> {
|
pub fn stage_write(&self, meta: &StagedWrite, scratch: &Path) -> Result<PathBuf> {
|
||||||
|
use std::io::Write as _;
|
||||||
|
|
||||||
static N: AtomicU64 = AtomicU64::new(0);
|
static N: AtomicU64 = AtomicU64::new(0);
|
||||||
// The uid goes in the name so a staged file can be tied back to its node
|
// The uid goes in the name so a staged file can be tied back to its node
|
||||||
// without a database; `/` in a uid would otherwise open a path.
|
// without a database; `/` in a uid would otherwise open a path.
|
||||||
@@ -882,18 +920,32 @@ impl ContentCache {
|
|||||||
now_millis(),
|
now_millis(),
|
||||||
N.fetch_add(1, Ordering::Relaxed)
|
N.fetch_add(1, Ordering::Relaxed)
|
||||||
));
|
));
|
||||||
if let Err(e) = std::fs::rename(scratch, &path) {
|
let copied = if let Err(e) = std::fs::rename(scratch, &path) {
|
||||||
// Cross-device: rename cannot move it, so copy and drop the original.
|
if e.kind() != std::io::ErrorKind::CrossesDevices {
|
||||||
let _ = e;
|
return Err(e.into());
|
||||||
std::fs::copy(scratch, &path)?;
|
|
||||||
let _ = std::fs::remove_file(scratch);
|
|
||||||
}
|
}
|
||||||
|
std::fs::copy(scratch, &path)?;
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
std::fs::File::open(&path)?.sync_all()?;
|
||||||
// Sidecar last: a staged file without one is still recoverable bytes,
|
// Sidecar last: a staged file without one is still recoverable bytes,
|
||||||
// while a sidecar without a file would describe nothing.
|
// while a sidecar without a file would describe nothing.
|
||||||
std::fs::write(
|
let side = path.with_extension("json");
|
||||||
path.with_extension("json"),
|
let side_tmp = path.with_extension("json.tmp");
|
||||||
serde_json::to_vec_pretty(meta)?,
|
let mut side_file = std::fs::OpenOptions::new()
|
||||||
)?;
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(&side_tmp)?;
|
||||||
|
side_file.write_all(&serde_json::to_vec_pretty(meta)?)?;
|
||||||
|
side_file.sync_all()?;
|
||||||
|
drop(side_file);
|
||||||
|
std::fs::rename(&side_tmp, &side)?;
|
||||||
|
std::fs::File::open(&self.staging_dir)?.sync_all()?;
|
||||||
|
if copied {
|
||||||
|
std::fs::remove_file(scratch)?;
|
||||||
|
}
|
||||||
Ok(path)
|
Ok(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1829,6 +1881,7 @@ mod tests {
|
|||||||
mtime: 100,
|
mtime: 100,
|
||||||
size: 40,
|
size: 40,
|
||||||
hash: None,
|
hash: None,
|
||||||
|
revision_id: None,
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
let staged = c.stage_write(&meta, &scratch).unwrap();
|
let staged = c.stage_write(&meta, &scratch).unwrap();
|
||||||
@@ -1906,6 +1959,51 @@ mod tests {
|
|||||||
assert!(c2.recovered_writes().is_empty(), "recovery does not repeat");
|
assert!(c2.recovered_writes().is_empty(), "recovery does not repeat");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recovery_keeps_an_orphan_blob_when_its_sidecar_is_missing() {
|
||||||
|
let (c, dir) = cache();
|
||||||
|
let orphan = dir.path().join("content/recovery/orphaned-user-bytes");
|
||||||
|
std::fs::write(&orphan, b"retain me").unwrap();
|
||||||
|
|
||||||
|
assert!(c.recovered_writes().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&orphan).unwrap(),
|
||||||
|
b"retain me",
|
||||||
|
"metadata loss must not trigger deletion of unexplained content"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_scratch_rescue_does_not_delete_the_durable_source() {
|
||||||
|
use std::io::Write as _;
|
||||||
|
let dir = TempDir::new();
|
||||||
|
let db = Arc::new(Db::open_in_memory().unwrap());
|
||||||
|
let content = dir.path().join("content");
|
||||||
|
let pins = dir.path().join("pins.json");
|
||||||
|
let c = ContentCache::open(content.clone(), pins.clone(), 0, db.clone()).unwrap();
|
||||||
|
let (mut file, scratch) = c.create_scratch().unwrap();
|
||||||
|
file.write_all(b"durable source").unwrap();
|
||||||
|
file.sync_all().unwrap();
|
||||||
|
let meta = StagedWrite {
|
||||||
|
uid: uid("rescue").to_string(),
|
||||||
|
len: 14,
|
||||||
|
base_size: 0,
|
||||||
|
base_mtime: 0,
|
||||||
|
authored: vec![(0, 14)],
|
||||||
|
complete: true,
|
||||||
|
based_on: None,
|
||||||
|
};
|
||||||
|
c.mark_scratch_durable(&scratch, &meta).unwrap();
|
||||||
|
let blocking_destination = content.join("recovery").join(scratch.file_name().unwrap());
|
||||||
|
std::fs::create_dir(&blocking_destination).unwrap();
|
||||||
|
drop(file);
|
||||||
|
drop(c);
|
||||||
|
|
||||||
|
let _c2 = ContentCache::open(content, pins, 0, db).unwrap();
|
||||||
|
assert_eq!(std::fs::read(&scratch).unwrap(), b"durable source");
|
||||||
|
assert!(ContentCache::scratch_sidecar(&scratch).exists());
|
||||||
|
}
|
||||||
|
|
||||||
/// Releasing a write hands the bytes to staging, so the sidecar an earlier
|
/// Releasing a write hands the bytes to staging, so the sidecar an earlier
|
||||||
/// `fsync` left must not offer recovery a second, stale copy of it.
|
/// `fsync` left must not offer recovery a second, stale copy of it.
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+224
-23
@@ -20,6 +20,59 @@ pub const KEYRING_SERVICE: &str = "proton-drive-linux";
|
|||||||
/// blobs back under this; pinned files are exempt. See [`crate::cache`].
|
/// blobs back under this; pinned files are exempt. See [`crate::cache`].
|
||||||
pub const DEFAULT_CACHE_BUDGET_BYTES: u64 = 5 * 1024 * 1024 * 1024;
|
pub const DEFAULT_CACHE_BUDGET_BYTES: u64 = 5 * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Environment override for [`AppConfig::resolved_conflict_sweep`]. Exists so a
|
||||||
|
/// support case can disable the sweep without editing the config file or
|
||||||
|
/// reinstalling — it wins over the stored setting.
|
||||||
|
pub const CONFLICT_SWEEP_ENV: &str = "PDFS_CONFLICT_SWEEP";
|
||||||
|
|
||||||
|
/// What the background conflict sweep (`pdfs-fuse`'s `pdfs-conflict-sweep`
|
||||||
|
/// thread) is allowed to do about a `(sync-conflict …)` copy it believes is a
|
||||||
|
/// redundant duplicate.
|
||||||
|
///
|
||||||
|
/// Defaults to [`SweepMode::Report`]: the sweep's whole reason to exist is that
|
||||||
|
/// the *previous* revision-identity check was wrong (B69), so the replacement is
|
||||||
|
/// not trusted with deletion until it has a release cycle of field evidence
|
||||||
|
/// behind it. See `docs/BUGS.md` B71.
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum SweepMode {
|
||||||
|
/// Do not run the sweep at all — the thread is never spawned.
|
||||||
|
Off,
|
||||||
|
/// Run, and log what it *would* remove, but never trash anything. Divergent
|
||||||
|
/// and orphaned copies are still surfaced to the activity feed as usual.
|
||||||
|
#[default]
|
||||||
|
Report,
|
||||||
|
/// Trash copies proven identical to their live sibling.
|
||||||
|
Enforce,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SweepMode {
|
||||||
|
/// Parse an operator-supplied value ([`CONFLICT_SWEEP_ENV`] or a CLI flag).
|
||||||
|
/// `None` for anything unrecognised, so a typo falls back to the stored
|
||||||
|
/// setting rather than silently picking a mode the operator did not ask for.
|
||||||
|
pub fn parse(value: &str) -> Option<Self> {
|
||||||
|
match value.trim().to_ascii_lowercase().as_str() {
|
||||||
|
"off" | "no" | "false" | "0" | "disabled" => Some(Self::Off),
|
||||||
|
"report" | "dry-run" | "dryrun" | "observe" => Some(Self::Report),
|
||||||
|
"enforce" | "on" | "yes" | "true" | "1" => Some(Self::Enforce),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether this mode is allowed to trash a duplicate.
|
||||||
|
pub fn is_enforcing(self) -> bool {
|
||||||
|
matches!(self, Self::Enforce)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Precedence for [`AppConfig::resolved_conflict_sweep`], split out from the
|
||||||
|
/// environment read so it can be unit-tested without mutating process state.
|
||||||
|
fn resolve_sweep_mode(env: Option<&str>, stored: Option<SweepMode>) -> SweepMode {
|
||||||
|
env.and_then(SweepMode::parse)
|
||||||
|
.or(stored)
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
/// Configuration structure allowing the user to customize client identification headers.
|
/// Configuration structure allowing the user to customize client identification headers.
|
||||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
pub struct AppConfig {
|
pub struct AppConfig {
|
||||||
@@ -48,6 +101,11 @@ pub struct AppConfig {
|
|||||||
/// by hostname". Defaulted for configs predating the field.
|
/// by hostname". Defaulted for configs predating the field.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub device_uid: Option<String>,
|
pub device_uid: Option<String>,
|
||||||
|
/// What the background conflict sweep may do. `None` means "use
|
||||||
|
/// [`SweepMode::default`]" (report-only). Defaulted for configs predating
|
||||||
|
/// the field, and overridable at runtime via [`CONFLICT_SWEEP_ENV`].
|
||||||
|
#[serde(default)]
|
||||||
|
pub conflict_sweep: Option<SweepMode>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for AppConfig {
|
impl Default for AppConfig {
|
||||||
@@ -59,6 +117,7 @@ impl Default for AppConfig {
|
|||||||
mountpoint: None,
|
mountpoint: None,
|
||||||
ignore_patterns: None,
|
ignore_patterns: None,
|
||||||
device_uid: None,
|
device_uid: None,
|
||||||
|
conflict_sweep: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,6 +129,20 @@ impl AppConfig {
|
|||||||
self.cache_budget.unwrap_or(DEFAULT_CACHE_BUDGET_BYTES)
|
self.cache_budget.unwrap_or(DEFAULT_CACHE_BUDGET_BYTES)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The effective conflict-sweep mode: the [`CONFLICT_SWEEP_ENV`] override if
|
||||||
|
/// it parses, then the user's stored choice, then [`SweepMode::default`].
|
||||||
|
///
|
||||||
|
/// The environment wins so a support case can stop a destructive background
|
||||||
|
/// loop without editing config or reinstalling (`docs/BUGS.md` B71). An
|
||||||
|
/// unparseable value is ignored rather than treated as "off", so a typo
|
||||||
|
/// cannot silently change behaviour in either direction.
|
||||||
|
pub fn resolved_conflict_sweep(&self) -> SweepMode {
|
||||||
|
resolve_sweep_mode(
|
||||||
|
std::env::var(CONFLICT_SWEEP_ENV).ok().as_deref(),
|
||||||
|
self.conflict_sweep,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// The effective global sync ignore patterns: the user's explicit list, or
|
/// The effective global sync ignore patterns: the user's explicit list, or
|
||||||
/// [`DEFAULT_IGNORE_PATTERNS`] when unset. An explicit empty list is
|
/// [`DEFAULT_IGNORE_PATTERNS`] when unset. An explicit empty list is
|
||||||
/// honoured as "no global patterns", which is why this is not a
|
/// honoured as "no global patterns", which is why this is not a
|
||||||
@@ -105,11 +178,21 @@ impl AppDirs {
|
|||||||
/// Load config from disk, creating default if missing.
|
/// Load config from disk, creating default if missing.
|
||||||
pub fn load_config(&self) -> AppConfig {
|
pub fn load_config(&self) -> AppConfig {
|
||||||
let path = self.config_path();
|
let path = self.config_path();
|
||||||
if path.exists()
|
if path.exists() {
|
||||||
&& let Ok(content) = std::fs::read_to_string(&path)
|
match std::fs::read_to_string(&path)
|
||||||
&& let Ok(config) = serde_json::from_str::<AppConfig>(&content)
|
.map_err(Error::from)
|
||||||
{
|
.and_then(|content| {
|
||||||
return config;
|
serde_json::from_str::<AppConfig>(&content).map_err(Error::from)
|
||||||
|
}) {
|
||||||
|
Ok(config) => return config,
|
||||||
|
Err(e) => {
|
||||||
|
// Preserve the original bytes for diagnosis/recovery. In
|
||||||
|
// particular, never replace a truncated config with defaults
|
||||||
|
// and silently lose mount, device, or ignore settings.
|
||||||
|
tracing::error!(path = %path.display(), error = %e, "config is unreadable; using in-memory defaults without overwriting it");
|
||||||
|
return AppConfig::default();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let default_config = AppConfig::default();
|
let default_config = AppConfig::default();
|
||||||
@@ -125,10 +208,26 @@ impl AppDirs {
|
|||||||
/// cache budget or mountpoint; the daemon re-reads the file on its next mount.
|
/// cache budget or mountpoint; the daemon re-reads the file on its next mount.
|
||||||
pub fn save_config(&self, config: &AppConfig) -> Result<()> {
|
pub fn save_config(&self, config: &AppConfig) -> Result<()> {
|
||||||
let path = self.config_path();
|
let path = self.config_path();
|
||||||
if let Some(parent) = path.parent() {
|
let parent = path
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| Error::Other("configuration path has no parent".into()))?;
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
let nonce = std::time::SystemTime::now()
|
||||||
std::fs::write(&path, serde_json::to_string_pretty(config)?)?;
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|duration| duration.as_nanos())
|
||||||
|
.unwrap_or(0);
|
||||||
|
let tmp = path.with_extension(format!("json.tmp-{}-{nonce}", std::process::id()));
|
||||||
|
use std::os::unix::fs::OpenOptionsExt as _;
|
||||||
|
let mut file = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.mode(0o600)
|
||||||
|
.open(&tmp)?;
|
||||||
|
use std::io::Write as _;
|
||||||
|
file.write_all(&serde_json::to_vec_pretty(config)?)?;
|
||||||
|
file.sync_all()?;
|
||||||
|
std::fs::rename(&tmp, &path)?;
|
||||||
|
std::fs::File::open(parent)?.sync_all()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,24 +310,41 @@ impl AppDirs {
|
|||||||
pub fn ensure(&self) -> Result<()> {
|
pub fn ensure(&self) -> Result<()> {
|
||||||
for dir in [self.state_dir(), self.cache_dir()] {
|
for dir in [self.state_dir(), self.cache_dir()] {
|
||||||
std::fs::create_dir_all(&dir)?;
|
std::fs::create_dir_all(&dir)?;
|
||||||
restrict_dir(&dir);
|
restrict_dir(&dir)?;
|
||||||
}
|
}
|
||||||
let config_dir = self.dirs.config_dir().to_path_buf();
|
let config_dir = self.dirs.config_dir().to_path_buf();
|
||||||
if std::fs::create_dir_all(&config_dir).is_ok() {
|
std::fs::create_dir_all(&config_dir)?;
|
||||||
restrict_dir(&config_dir);
|
restrict_dir(&config_dir)?;
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make `dir` owner-only (`0700`). Best effort: a mode we cannot set (an
|
/// Make `dir` owner-only (`0700`). These directories contain decrypted bytes,
|
||||||
/// exotic filesystem, a directory we do not own) is not worth refusing to
|
/// plaintext names, and an authenticated control socket, so failure is fatal.
|
||||||
/// start over, and the caller has no better answer than continuing.
|
fn restrict_dir(dir: &Path) -> Result<()> {
|
||||||
fn restrict_dir(dir: &Path) {
|
use std::os::unix::fs::{MetadataExt as _, PermissionsExt};
|
||||||
use std::os::unix::fs::PermissionsExt;
|
let before = std::fs::symlink_metadata(dir)?;
|
||||||
if let Err(e) = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)) {
|
if before.file_type().is_symlink() || !before.is_dir() {
|
||||||
tracing::warn!(path = %dir.display(), error = %e, "could not restrict directory to 0700");
|
return Err(Error::Other(format!(
|
||||||
|
"sensitive path is not a real directory: {}",
|
||||||
|
dir.display()
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
|
if before.uid() != unsafe { libc::geteuid() } {
|
||||||
|
return Err(Error::Other(format!(
|
||||||
|
"sensitive directory is not owned by the current user: {}",
|
||||||
|
dir.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?;
|
||||||
|
let after = std::fs::symlink_metadata(dir)?;
|
||||||
|
if after.permissions().mode() & 0o777 != 0o700 {
|
||||||
|
return Err(Error::Other(format!(
|
||||||
|
"sensitive directory did not become owner-only: {}",
|
||||||
|
dir.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make a just-bound Unix socket owner-only (`0600`).
|
/// Make a just-bound Unix socket owner-only (`0600`).
|
||||||
@@ -263,29 +379,57 @@ mod tests {
|
|||||||
/// and the control socket that commands the daemon's session.
|
/// and the control socket that commands the daemon's session.
|
||||||
#[test]
|
#[test]
|
||||||
fn restrict_dir_makes_a_directory_owner_only() {
|
fn restrict_dir_makes_a_directory_owner_only() {
|
||||||
let dir = std::env::temp_dir().join(format!("pdfs-cfg-{}", std::process::id()));
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"pdfs-cfg-{}-{:?}",
|
||||||
|
std::process::id(),
|
||||||
|
std::thread::current().id()
|
||||||
|
));
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
std::fs::create_dir_all(&dir).unwrap();
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
// Start deliberately world-readable, as `create_dir_all` under a typical
|
// Start deliberately world-readable, as `create_dir_all` under a typical
|
||||||
// umask would leave it.
|
// umask would leave it.
|
||||||
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap();
|
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||||
|
|
||||||
restrict_dir(&dir);
|
restrict_dir(&dir).unwrap();
|
||||||
|
|
||||||
assert_eq!(mode_of(&dir), 0o700, "group and other must have no access");
|
assert_eq!(mode_of(&dir), 0o700, "group and other must have no access");
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn restrict_dir_refuses_a_symlink() {
|
||||||
|
let root = std::env::temp_dir().join(format!(
|
||||||
|
"pdfs-cfg-link-{}-{:?}",
|
||||||
|
std::process::id(),
|
||||||
|
std::thread::current().id()
|
||||||
|
));
|
||||||
|
let target = root.join("target");
|
||||||
|
let link = root.join("state");
|
||||||
|
std::fs::create_dir_all(&target).unwrap();
|
||||||
|
std::os::unix::fs::symlink(&target, &link).unwrap();
|
||||||
|
|
||||||
|
assert!(restrict_dir(&link).is_err());
|
||||||
|
|
||||||
|
let _ = std::fs::remove_dir_all(&root);
|
||||||
|
}
|
||||||
|
|
||||||
/// Connecting to the control socket confers the daemon's authenticated
|
/// Connecting to the control socket confers the daemon's authenticated
|
||||||
/// session with no credential, so it is an authority boundary rather than
|
/// session with no credential, so it is an authority boundary rather than
|
||||||
/// merely private data.
|
/// merely private data.
|
||||||
#[test]
|
#[test]
|
||||||
fn restrict_socket_makes_a_socket_owner_only() {
|
fn restrict_socket_makes_a_socket_owner_only() {
|
||||||
let dir = std::env::temp_dir().join(format!("pdfs-sock-{}", std::process::id()));
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"pdfs-sock-{}-{:?}",
|
||||||
|
std::process::id(),
|
||||||
|
std::thread::current().id()
|
||||||
|
));
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
std::fs::create_dir_all(&dir).unwrap();
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
let sock = dir.join("control.sock");
|
let sock = dir.join("control.sock");
|
||||||
let listener = std::os::unix::net::UnixListener::bind(&sock).unwrap();
|
// chmod semantics are identical for a socket path and a regular file;
|
||||||
|
// use a file so this regression also runs in sandboxes that prohibit
|
||||||
|
// creating Unix sockets.
|
||||||
|
let listener = std::fs::File::create(&sock).unwrap();
|
||||||
|
|
||||||
restrict_socket(&sock).unwrap();
|
restrict_socket(&sock).unwrap();
|
||||||
|
|
||||||
@@ -303,6 +447,7 @@ mod tests {
|
|||||||
mountpoint: Some("/tmp/x".to_string()),
|
mountpoint: Some("/tmp/x".to_string()),
|
||||||
ignore_patterns: Some(vec!["build/".to_string()]),
|
ignore_patterns: Some(vec!["build/".to_string()]),
|
||||||
device_uid: Some("dev-uid".to_string()),
|
device_uid: Some("dev-uid".to_string()),
|
||||||
|
conflict_sweep: Some(SweepMode::Enforce),
|
||||||
};
|
};
|
||||||
let json = serde_json::to_string(&config).unwrap();
|
let json = serde_json::to_string(&config).unwrap();
|
||||||
let decoded: AppConfig = serde_json::from_str(&json).unwrap();
|
let decoded: AppConfig = serde_json::from_str(&json).unwrap();
|
||||||
@@ -317,6 +462,62 @@ mod tests {
|
|||||||
assert_eq!(default_config.user_agent, USER_AGENT);
|
assert_eq!(default_config.user_agent, USER_AGENT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sweep_mode_defaults_to_report_only() {
|
||||||
|
// The sweep trashes user files, so an installed config that has never
|
||||||
|
// heard of the setting must not get the enforcing behaviour.
|
||||||
|
assert_eq!(SweepMode::default(), SweepMode::Report);
|
||||||
|
assert!(!SweepMode::default().is_enforcing());
|
||||||
|
assert_eq!(resolve_sweep_mode(None, None), SweepMode::Report);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sweep_mode_parses_operator_spellings() {
|
||||||
|
for s in ["off", "OFF", " no ", "false", "0", "disabled"] {
|
||||||
|
assert_eq!(SweepMode::parse(s), Some(SweepMode::Off), "{s}");
|
||||||
|
}
|
||||||
|
for s in ["report", "dry-run", "dryrun", "observe"] {
|
||||||
|
assert_eq!(SweepMode::parse(s), Some(SweepMode::Report), "{s}");
|
||||||
|
}
|
||||||
|
for s in ["enforce", "on", "yes", "true", "1"] {
|
||||||
|
assert_eq!(SweepMode::parse(s), Some(SweepMode::Enforce), "{s}");
|
||||||
|
}
|
||||||
|
// A typo must not resolve to anything, so it falls through to the config.
|
||||||
|
assert_eq!(SweepMode::parse("offf"), None);
|
||||||
|
assert_eq!(SweepMode::parse(""), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sweep_mode_env_overrides_config_but_a_typo_does_not() {
|
||||||
|
let stored = Some(SweepMode::Enforce);
|
||||||
|
// The point of the override: kill a destructive loop without editing config.
|
||||||
|
assert_eq!(resolve_sweep_mode(Some("off"), stored), SweepMode::Off);
|
||||||
|
// Unrecognised value falls back to the stored choice rather than guessing.
|
||||||
|
assert_eq!(resolve_sweep_mode(Some("nope"), stored), SweepMode::Enforce);
|
||||||
|
assert_eq!(resolve_sweep_mode(None, stored), SweepMode::Enforce);
|
||||||
|
// No env, no stored value: report-only.
|
||||||
|
assert_eq!(resolve_sweep_mode(Some("nope"), None), SweepMode::Report);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_config_predating_conflict_sweep_still_loads() {
|
||||||
|
let json = r#"{"app_version":"v","user_agent":"u"}"#;
|
||||||
|
let decoded: AppConfig = serde_json::from_str(json).unwrap();
|
||||||
|
assert!(decoded.conflict_sweep.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sweep_mode_round_trips_through_the_config_file() {
|
||||||
|
let config = AppConfig {
|
||||||
|
conflict_sweep: Some(SweepMode::Enforce),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&config).unwrap();
|
||||||
|
assert!(json.contains(r#""conflict_sweep":"enforce""#), "{json}");
|
||||||
|
let decoded: AppConfig = serde_json::from_str(&json).unwrap();
|
||||||
|
assert_eq!(decoded.conflict_sweep, Some(SweepMode::Enforce));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_config_predating_ignore_patterns_still_loads() {
|
fn a_config_predating_ignore_patterns_still_loads() {
|
||||||
// Every installed config on disk looks like this; the field must not be
|
// Every installed config on disk looks like this; the field must not be
|
||||||
|
|||||||
@@ -573,6 +573,11 @@ pub enum ActivityKind {
|
|||||||
Share,
|
Share,
|
||||||
PublicLink,
|
PublicLink,
|
||||||
Unshare,
|
Unshare,
|
||||||
|
/// A `(sync-conflict …)` copy that needs the user's attention: it diverges
|
||||||
|
/// from the live file (or has no sibling), so the auto-sweep left it in place
|
||||||
|
/// rather than removing it. An auto-*removed* identical copy is logged as a
|
||||||
|
/// [`ActivityKind::Trash`] instead, since that is what happened to it.
|
||||||
|
Conflict,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One line in the daemon's activity log (in [`Response::Activity`]). Newest
|
/// One line in the daemon's activity log (in [`Response::Activity`]). Newest
|
||||||
|
|||||||
@@ -31,7 +31,12 @@ impl Db {
|
|||||||
.and_then(|v| v.parse().ok())
|
.and_then(|v| v.parse().ok())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
|
||||||
if current >= SCHEMA_VERSION {
|
if current > SCHEMA_VERSION {
|
||||||
|
return Err(crate::Error::Other(format!(
|
||||||
|
"database schema {current} is newer than this build supports ({SCHEMA_VERSION}); refusing to open it to avoid corrupting user data"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if current == SCHEMA_VERSION {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ pub use local::LocalFileHit;
|
|||||||
pub use maintenance::{DbStats, VacuumOutcome};
|
pub use maintenance::{DbStats, VacuumOutcome};
|
||||||
pub use nodes::{SearchHit, StoredNode};
|
pub use nodes::{SearchHit, StoredNode};
|
||||||
pub use ops::{
|
pub use ops::{
|
||||||
AttachedBlob, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH,
|
AttachedBlob, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_UNTIL,
|
||||||
PendingCounts, PendingOp, op_supersedes,
|
PendingCounts, PendingOp, op_supersedes,
|
||||||
};
|
};
|
||||||
pub use photos::{StoredPhoto, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN};
|
pub use photos::{StoredPhoto, THUMB_HAVE, THUMB_NONE, THUMB_UNKNOWN};
|
||||||
|
|||||||
@@ -63,6 +63,16 @@ pub fn op_supersedes(kind: &str) -> bool {
|
|||||||
/// drain replaces it with the uid the server assigns.
|
/// drain replaces it with the uid the server assigns.
|
||||||
pub const LOCAL_VOLUME: &str = "local";
|
pub const LOCAL_VOLUME: &str = "local";
|
||||||
|
|
||||||
|
/// A `next_attempt_at` far enough in the future that [`Db::next_due_op`] never
|
||||||
|
/// selects the op (ms since epoch, ≈ year 2223). Used to *park* a queued create
|
||||||
|
/// for a transient file — a browser's in-flight `*.crdownload`/`*.part`, an
|
||||||
|
/// editor's `*.swp` — so its bytes never upload while it wears that name. A
|
||||||
|
/// rename to the finished name un-parks it (see [`Db::set_create_hold`]), which
|
||||||
|
/// is the only moment the completed file is meant to reach Drive. Without the
|
||||||
|
/// park, every partial and every abandoned temp would upload and, on a
|
||||||
|
/// stall+resume, fork a `(sync-conflict)` copy (docs/BUGS.md B70).
|
||||||
|
pub const PARK_UNTIL: i64 = 8_000_000_000_000;
|
||||||
|
|
||||||
/// A mutation that has been accepted locally but not yet performed against the
|
/// A mutation that has been accepted locally but not yet performed against the
|
||||||
/// API — the durable half of the write-back queue (offline.md Phase 3).
|
/// API — the durable half of the write-back queue (offline.md Phase 3).
|
||||||
///
|
///
|
||||||
@@ -118,9 +128,10 @@ pub struct AttachedBlob {
|
|||||||
|
|
||||||
impl Db {
|
impl Db {
|
||||||
pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option<String>)> {
|
pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option<String>)> {
|
||||||
let conn = self.conn.lock();
|
let mut conn = self.conn.lock();
|
||||||
|
let tx = conn.transaction()?;
|
||||||
let superseded: Option<String> = if op_supersedes(&op.kind) {
|
let superseded: Option<String> = if op_supersedes(&op.kind) {
|
||||||
let blob: Option<String> = conn
|
let blob: Option<String> = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
"SELECT blob_path FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
||||||
params![op.uid, op.kind],
|
params![op.uid, op.kind],
|
||||||
@@ -128,7 +139,7 @@ impl Db {
|
|||||||
)
|
)
|
||||||
.optional()?
|
.optional()?
|
||||||
.flatten();
|
.flatten();
|
||||||
conn.execute(
|
tx.execute(
|
||||||
"DELETE FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
"DELETE FROM pending_op WHERE uid = ?1 AND kind = ?2",
|
||||||
params![op.uid, op.kind],
|
params![op.uid, op.kind],
|
||||||
)?;
|
)?;
|
||||||
@@ -136,7 +147,7 @@ impl Db {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
conn.execute(
|
tx.execute(
|
||||||
"INSERT INTO pending_op
|
"INSERT INTO pending_op
|
||||||
(kind, uid, parent_uid, name, blob_path, meta_json, created_at, next_attempt_at)
|
(kind, uid, parent_uid, name, blob_path, meta_json, created_at, next_attempt_at)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||||
@@ -151,7 +162,9 @@ impl Db {
|
|||||||
op.next_attempt_at
|
op.next_attempt_at
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
Ok((conn.last_insert_rowid(), superseded))
|
let id = tx.last_insert_rowid();
|
||||||
|
tx.commit()?;
|
||||||
|
Ok((id, superseded))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Point a queued create at the bytes that were just written to it, returning
|
/// Point a queued create at the bytes that were just written to it, returning
|
||||||
@@ -180,11 +193,18 @@ impl Db {
|
|||||||
let Some((id, superseded)) = existing else {
|
let Some((id, superseded)) = existing else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
// Clearing the backoff lets the fresh bytes be tried promptly — except
|
||||||
|
// when the create is *parked* (a transient file, next_attempt_at at the
|
||||||
|
// PARK_UNTIL sentinel). Attaching a new revision to a still-growing
|
||||||
|
// download must not wake it; only the finalize rename does. So preserve a
|
||||||
|
// park and reset an ordinary backoff.
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"UPDATE pending_op
|
"UPDATE pending_op
|
||||||
SET blob_path = ?2, meta_json = ?3, attempts = 0, next_attempt_at = 0
|
SET blob_path = ?2, meta_json = ?3, attempts = 0,
|
||||||
|
next_attempt_at = CASE WHEN next_attempt_at >= ?4
|
||||||
|
THEN next_attempt_at ELSE 0 END
|
||||||
WHERE id = ?1",
|
WHERE id = ?1",
|
||||||
params![id, blob_path, meta_json],
|
params![id, blob_path, meta_json, PARK_UNTIL],
|
||||||
)?;
|
)?;
|
||||||
Ok(Some(AttachedBlob { id, superseded }))
|
Ok(Some(AttachedBlob { id, superseded }))
|
||||||
}
|
}
|
||||||
@@ -230,6 +250,25 @@ impl Db {
|
|||||||
Ok(n > 0)
|
Ok(n > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Park or un-park a queued create/mkdir for `uid`, returning whether a row
|
||||||
|
/// was touched.
|
||||||
|
///
|
||||||
|
/// Parking sets `next_attempt_at` to [`PARK_UNTIL`] so [`Db::next_due_op`]
|
||||||
|
/// skips it indefinitely; un-parking sets it due now. Used to keep a
|
||||||
|
/// transient file's bytes off Drive until it is renamed to its finished name
|
||||||
|
/// (docs/BUGS.md B70). Un-parking a create that was never parked is harmless:
|
||||||
|
/// it just makes the create due immediately, which for a normal local node it
|
||||||
|
/// already is.
|
||||||
|
pub fn set_create_hold(&self, uid: &str, held: bool) -> Result<bool> {
|
||||||
|
let conn = self.conn.lock();
|
||||||
|
let n = conn.execute(
|
||||||
|
"UPDATE pending_op SET next_attempt_at = ?2
|
||||||
|
WHERE uid = ?1 AND kind IN (?3, ?4)",
|
||||||
|
params![uid, if held { PARK_UNTIL } else { 0 }, OP_CREATE, OP_MKDIR],
|
||||||
|
)?;
|
||||||
|
Ok(n > 0)
|
||||||
|
}
|
||||||
|
|
||||||
/// Check if a queued create or mkdir op exists for `uid`.
|
/// Check if a queued create or mkdir op exists for `uid`.
|
||||||
pub fn has_create_op(&self, uid: &str) -> Result<bool> {
|
pub fn has_create_op(&self, uid: &str) -> Result<bool> {
|
||||||
let conn = self.conn.lock();
|
let conn = self.conn.lock();
|
||||||
@@ -241,6 +280,22 @@ impl Db {
|
|||||||
Ok(count > 0)
|
Ok(count > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Check whether *any* op of any kind is queued against `uid`.
|
||||||
|
///
|
||||||
|
/// Used as a safety interlock by callers that are about to remove a node
|
||||||
|
/// they only know about from a stale snapshot: a queued op means the user's
|
||||||
|
/// bytes are still owed an upload, and dropping it would discard the staged
|
||||||
|
/// blob that holds them (`docs/BUGS.md` B71).
|
||||||
|
pub fn has_any_op(&self, uid: &str) -> Result<bool> {
|
||||||
|
let conn = self.conn.lock();
|
||||||
|
let count: i64 = conn.query_row(
|
||||||
|
"SELECT COUNT(*) FROM pending_op WHERE uid = ?1",
|
||||||
|
params![uid],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
Ok(count > 0)
|
||||||
|
}
|
||||||
|
|
||||||
/// Drop every op targeting a node **or anything queued beneath it**,
|
/// Drop every op targeting a node **or anything queued beneath it**,
|
||||||
/// returning the staged blobs they held so the caller can delete them.
|
/// returning the staged blobs they held so the caller can delete them.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -784,6 +784,41 @@ fn opens_and_migrates() {
|
|||||||
assert_eq!(version, SCHEMA_VERSION.to_string());
|
assert_eq!(version, SCHEMA_VERSION.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn refuses_a_schema_newer_than_the_running_build() {
|
||||||
|
let path = std::env::temp_dir().join(format!(
|
||||||
|
"pdfs-db-future-{}-{}.db",
|
||||||
|
std::process::id(),
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_nanos()
|
||||||
|
));
|
||||||
|
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||||
|
conn.execute_batch(
|
||||||
|
"CREATE TABLE sync_state (key TEXT PRIMARY KEY, value TEXT);
|
||||||
|
INSERT INTO sync_state VALUES ('schema_version', '9999');",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drop(conn);
|
||||||
|
|
||||||
|
let err = Db::open(&path)
|
||||||
|
.err()
|
||||||
|
.expect("future schema must fail closed");
|
||||||
|
assert!(err.to_string().contains("newer than this build supports"));
|
||||||
|
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||||
|
let version: String = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT value FROM sync_state WHERE key = 'schema_version'",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(version, "9999", "failed open must not rewrite the database");
|
||||||
|
drop(conn);
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
/// A queued write whose baseline is restamped must keep everything else.
|
/// A queued write whose baseline is restamped must keep everything else.
|
||||||
///
|
///
|
||||||
/// The restamp happens after our *own* upload seals a new revision under a
|
/// The restamp happens after our *own* upload seals a new revision under a
|
||||||
@@ -867,6 +902,39 @@ fn a_second_write_supersedes_the_first_pending_op() {
|
|||||||
assert_eq!(db.pending_op_counts().unwrap().uploads, 1);
|
assert_eq!(db.pending_op_counts().unwrap().uploads, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_superseding_insert_keeps_the_old_pending_op() {
|
||||||
|
let db = Db::open_in_memory().unwrap();
|
||||||
|
let op = |blob: &str| PendingOp {
|
||||||
|
id: 0,
|
||||||
|
kind: OP_REVISION.to_string(),
|
||||||
|
uid: uid("atomic").to_string(),
|
||||||
|
parent_uid: None,
|
||||||
|
name: None,
|
||||||
|
blob_path: Some(blob.to_string()),
|
||||||
|
meta_json: Some("{}".to_string()),
|
||||||
|
created_at: 1,
|
||||||
|
attempts: 0,
|
||||||
|
last_error: None,
|
||||||
|
next_attempt_at: 0,
|
||||||
|
};
|
||||||
|
db.enqueue_op(&op("/staging/original")).unwrap();
|
||||||
|
db.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"CREATE TRIGGER reject_replacement BEFORE INSERT ON pending_op
|
||||||
|
WHEN NEW.blob_path = '/staging/replacement'
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'injected insert failure'); END;",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(db.enqueue_op(&op("/staging/replacement")).is_err());
|
||||||
|
let ops = db.pending_ops().unwrap();
|
||||||
|
assert_eq!(ops.len(), 1);
|
||||||
|
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/original"));
|
||||||
|
}
|
||||||
|
|
||||||
/// Deleting a folder that was created offline must take the ops queued
|
/// Deleting a folder that was created offline must take the ops queued
|
||||||
/// underneath it with it: they name a placeholder parent that will now never
|
/// underneath it with it: they name a placeholder parent that will now never
|
||||||
/// become real, so nothing could ever drain them and nothing is left to
|
/// become real, so nothing could ever drain them and nothing is left to
|
||||||
@@ -1547,6 +1615,58 @@ fn next_due_op_skips_ops_blocked_on_a_local_parent() {
|
|||||||
assert_eq!(db.next_due_op(10).unwrap().unwrap().uid, "vol~link3");
|
assert_eq!(db.next_due_op(10).unwrap().unwrap().uid, "vol~link3");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// B70: a transient file's create is parked so its bytes never upload while it
|
||||||
|
/// wears that name, the park survives a write attaching its blob, and the
|
||||||
|
/// finalize rename (un-park) is what finally makes the completed file due.
|
||||||
|
#[test]
|
||||||
|
fn a_parked_transient_create_stays_off_the_drain_until_finalized() {
|
||||||
|
let db = Db::open_in_memory().unwrap();
|
||||||
|
let node = uid("dl").to_string();
|
||||||
|
let parent = uid("root").to_string();
|
||||||
|
db.enqueue_op(&PendingOp {
|
||||||
|
id: 0,
|
||||||
|
kind: OP_CREATE.to_string(),
|
||||||
|
uid: node.clone(),
|
||||||
|
parent_uid: Some(parent),
|
||||||
|
name: Some("movie.mkv.part".to_string()),
|
||||||
|
blob_path: None,
|
||||||
|
meta_json: None,
|
||||||
|
created_at: 1,
|
||||||
|
attempts: 0,
|
||||||
|
last_error: None,
|
||||||
|
next_attempt_at: PARK_UNTIL,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Parked: never selected, however far the clock advances.
|
||||||
|
assert!(
|
||||||
|
db.next_due_op(PARK_UNTIL - 1).unwrap().is_none(),
|
||||||
|
"a parked create is never due"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A write attaches its blob to the create but must not wake it.
|
||||||
|
let attached = db
|
||||||
|
.attach_blob_to_create(&node, "/staging/blob1", "{}")
|
||||||
|
.unwrap();
|
||||||
|
assert!(attached.is_some(), "the blob attached to the create");
|
||||||
|
assert!(
|
||||||
|
db.next_due_op(PARK_UNTIL - 1).unwrap().is_none(),
|
||||||
|
"still parked after a write attaches"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Finalize: un-park, and the completed file is due to upload with its bytes.
|
||||||
|
assert!(
|
||||||
|
db.set_create_hold(&node, false).unwrap(),
|
||||||
|
"a create row was un-parked"
|
||||||
|
);
|
||||||
|
let due = db
|
||||||
|
.next_due_op(1_000)
|
||||||
|
.unwrap()
|
||||||
|
.expect("the un-parked create is due");
|
||||||
|
assert_eq!(due.uid, node);
|
||||||
|
assert_eq!(due.blob_path.as_deref(), Some("/staging/blob1"));
|
||||||
|
}
|
||||||
|
|
||||||
/// Backoff still gates: nothing is returned before an op is due.
|
/// Backoff still gates: nothing is returned before an op is due.
|
||||||
#[test]
|
#[test]
|
||||||
fn next_due_op_respects_backoff() {
|
fn next_due_op_respects_backoff() {
|
||||||
@@ -1738,6 +1858,8 @@ fn test_db_has_children_and_trashed_filtering() {
|
|||||||
media_type: "text/plain".into(),
|
media_type: "text/plain".into(),
|
||||||
total_size_on_storage: 10,
|
total_size_on_storage: 10,
|
||||||
active_revision_state: None,
|
active_revision_state: None,
|
||||||
|
active_revision_id: None,
|
||||||
|
content_sha1: None,
|
||||||
claimed_size: Some(10),
|
claimed_size: Some(10),
|
||||||
claimed_modification_time: None,
|
claimed_modification_time: None,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -45,6 +45,45 @@ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
|
|||||||
"Thumbs.db",
|
"Thumbs.db",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
/// Suffixes of files that are, by convention, incomplete or throwaway while they
|
||||||
|
/// wear the name: a download in flight, an editor's swap or backup, an atomic
|
||||||
|
/// write's scratch copy. On an on-demand mount these should never seal a remote
|
||||||
|
/// revision — the app renames the finished file to its real name, and only that
|
||||||
|
/// belongs on Drive. See [`is_transient_name`] and docs/BUGS.md B70.
|
||||||
|
const TRANSIENT_SUFFIXES: &[&str] = &[
|
||||||
|
".crdownload", // Chromium/Brave/Edge
|
||||||
|
".part", // Firefox, wget, aria2c
|
||||||
|
".partial", // some downloaders / older IE
|
||||||
|
".download", // Safari / macOS
|
||||||
|
".tmp", // generic scratch
|
||||||
|
".temp", // generic scratch
|
||||||
|
".swp", // vim swap
|
||||||
|
".swx", // vim swap (secondary)
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Whether `name` is a transient scratch/in-flight file whose bytes should not
|
||||||
|
/// be uploaded on an on-demand mount until it is renamed to its finished name.
|
||||||
|
///
|
||||||
|
/// Recognises browser download temps, editor swap/backup files, and lock files.
|
||||||
|
/// Case-insensitive on the suffix so `FOO.CRDOWNLOAD` counts too. This is the
|
||||||
|
/// on-demand-write counterpart to [`DEFAULT_IGNORE_PATTERNS`] (which only covers
|
||||||
|
/// mirror-folder reconcile); keep the two conceptually in step.
|
||||||
|
pub fn is_transient_name(name: &str) -> bool {
|
||||||
|
let lower = name.to_ascii_lowercase();
|
||||||
|
if TRANSIENT_SUFFIXES.iter().any(|s| lower.ends_with(s)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Emacs/gedit/kate backup copies.
|
||||||
|
if name.ends_with('~') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// LibreOffice lock files (`.~lock.<doc>#`) and MS Office temps (`~$doc`).
|
||||||
|
if name.starts_with(".~lock.") || name.starts_with("~$") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
/// Filenames accepted as a folder's ignore file, in precedence order.
|
/// Filenames accepted as a folder's ignore file, in precedence order.
|
||||||
const IGNORE_FILE_NAMES: &[&str] = &[".pdfsignore", ".protonignore"];
|
const IGNORE_FILE_NAMES: &[&str] = &[".pdfsignore", ".protonignore"];
|
||||||
|
|
||||||
@@ -272,4 +311,37 @@ mod tests {
|
|||||||
// have nothing to reconcile against.
|
// have nothing to reconcile against.
|
||||||
assert!(!rules.is_ignored("", true));
|
assert!(!rules.is_ignored("", true));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transient_download_and_editor_names_are_recognised() {
|
||||||
|
for name in [
|
||||||
|
"Unconfirmed 540477.crdownload",
|
||||||
|
"movie.mkv.part",
|
||||||
|
"big.iso.partial",
|
||||||
|
"photo.jpg.download",
|
||||||
|
"export.xml.tmp",
|
||||||
|
"notes.TEMP",
|
||||||
|
".notes.txt.swp",
|
||||||
|
"doc.txt~",
|
||||||
|
".~lock.report.odt#",
|
||||||
|
"~$budget.xlsx",
|
||||||
|
"FOO.CRDOWNLOAD",
|
||||||
|
] {
|
||||||
|
assert!(is_transient_name(name), "{name} should be transient");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn finished_names_are_not_transient() {
|
||||||
|
for name in [
|
||||||
|
"teamspeak.tar.gz",
|
||||||
|
"nils.zip",
|
||||||
|
"report.xml",
|
||||||
|
"archive.partly.zip", // ".partly" is not ".part"
|
||||||
|
"template.dotx",
|
||||||
|
"readme",
|
||||||
|
] {
|
||||||
|
assert!(!is_transient_name(name), "{name} should not be transient");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -2,73 +2,19 @@
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
/// Apply one remote event to the local cache and notify the kernel so it drops
|
/// Drop `uid` from one mount's tree and tell that mount's kernel session the
|
||||||
/// any stale cached metadata/data for the affected inodes.
|
/// entry is gone. Shared by the trash and delete paths, which differ only in
|
||||||
|
/// which event carried them.
|
||||||
///
|
///
|
||||||
/// The cache is authoritative-by-absence: dropping a directory's `children`
|
/// Returns whether this mount actually held the node, so the caller can evict
|
||||||
/// entry forces the next `lookup`/`readdir` to re-enumerate from the remote, so
|
/// its content blob once if *any* mount did.
|
||||||
/// most events only need to invalidate listings rather than re-fetch eagerly.
|
fn forget_and_notify(st: &mut State, notifier: Option<&Notifier>, uid: &NodeUid) -> bool {
|
||||||
fn apply_event(
|
|
||||||
state: &Mutex<State>,
|
|
||||||
content: &ContentCache,
|
|
||||||
pending: &Mutex<HashMap<NodeUid, PendingRevision>>,
|
|
||||||
notifier: &Notifier,
|
|
||||||
event: &DriveEvent,
|
|
||||||
) {
|
|
||||||
match event {
|
|
||||||
DriveEvent::NodeUpdated {
|
|
||||||
node_uid,
|
|
||||||
parent_node_uid,
|
|
||||||
is_trashed,
|
|
||||||
..
|
|
||||||
} => {
|
|
||||||
let mut st = state.lock();
|
|
||||||
if *is_trashed {
|
|
||||||
// Trashing makes a node vanish from its parent listing.
|
|
||||||
let child = st.by_uid.get(node_uid).copied();
|
|
||||||
if let Some((parent, name)) = st.forget(node_uid) {
|
|
||||||
content.evict(node_uid);
|
|
||||||
match child {
|
|
||||||
Some(child) => {
|
|
||||||
let _ =
|
|
||||||
notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
let _ = notifier.inval_entry(INodeNo(parent), OsStr::new(&name));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if pending.lock().contains_key(node_uid) {
|
|
||||||
// A node we owe an upload for is *ahead* of the remote, not
|
|
||||||
// behind it: this event is almost always the echo of our own
|
|
||||||
// empty-file create, and re-fetching would replace the size and
|
|
||||||
// mtime of the write we just accepted with the stale revision's
|
|
||||||
// — making a file that was copied in seconds ago read as empty
|
|
||||||
// until its upload lands (offline.md Phase 3).
|
|
||||||
debug!(uid = %node_uid, "ignoring remote event for a node with a queued write");
|
|
||||||
} else if let Some(&ino) = st.by_uid.get(node_uid) {
|
|
||||||
// Known node changed: drop its cached attrs/data (and listing if
|
|
||||||
// it is a directory) so the next access re-fetches. Its content
|
|
||||||
// blob may now be stale, so evict it too.
|
|
||||||
st.invalidate_listing(ino);
|
|
||||||
content.evict(node_uid);
|
|
||||||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
|
||||||
}
|
|
||||||
// A create (or move-in) shows up as a change to the parent listing;
|
|
||||||
// drop it so the new child is picked up on the next readdir.
|
|
||||||
if let Some(parent_uid) = parent_node_uid
|
|
||||||
&& let Some(&parent) = st.by_uid.get(parent_uid)
|
|
||||||
{
|
|
||||||
st.invalidate_listing(parent);
|
|
||||||
let _ = notifier.inval_inode(INodeNo(parent), 0, 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
DriveEvent::NodeDeleted { node_uid, .. } => {
|
|
||||||
let mut st = state.lock();
|
|
||||||
// Capture the inode before `forget` clears the uid mapping.
|
// Capture the inode before `forget` clears the uid mapping.
|
||||||
let child = st.by_uid.get(node_uid).copied();
|
let child = st.by_uid.get(uid).copied();
|
||||||
content.evict(node_uid);
|
let Some((parent, name)) = st.forget(uid) else {
|
||||||
if let Some((parent, name)) = st.forget(node_uid) {
|
return false;
|
||||||
|
};
|
||||||
|
if let Some(notifier) = notifier {
|
||||||
match child {
|
match child {
|
||||||
Some(child) => {
|
Some(child) => {
|
||||||
let _ = notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
let _ = notifier.delete(INodeNo(parent), INodeNo(child), OsStr::new(&name));
|
||||||
@@ -78,39 +24,209 @@ fn apply_event(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply one remote event to the local cache and notify the kernel so it drops
|
||||||
|
/// any stale cached metadata/data for the affected inodes.
|
||||||
|
///
|
||||||
|
/// The cache is authoritative-by-absence: dropping a directory's `children`
|
||||||
|
/// entry forces the next `lookup`/`readdir` to re-enumerate from the remote, so
|
||||||
|
/// most events only need to invalidate listings rather than re-fetch eagerly.
|
||||||
|
///
|
||||||
|
/// Applied to **every** mounted inode space, not just the primary one. This task
|
||||||
|
/// is per-daemon while node state is per-mount, and a `DriveEvent` names a uid,
|
||||||
|
/// not a mount — so a file trashed from another device has to be withdrawn from
|
||||||
|
/// whichever of our sessions is showing it, which for anything under a sync
|
||||||
|
/// folder is a [`Core::fork_state`] fork rather than `core.state`. Reaching for
|
||||||
|
/// the primary state alone left forks serving a deleted file indefinitely, the
|
||||||
|
/// read-side twin of `docs/BUGS.md` B74. Inode numbers are per-mount, so each
|
||||||
|
/// mount must be notified through its **own** channel; that pairing is what
|
||||||
|
/// [`Core::for_each_mount`] exists to preserve.
|
||||||
|
fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) {
|
||||||
|
match event {
|
||||||
|
DriveEvent::NodeUpdated {
|
||||||
|
node_uid,
|
||||||
|
parent_node_uid,
|
||||||
|
is_trashed,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
// A node we owe an upload for is *ahead* of the remote, not behind
|
||||||
|
// it: this event is almost always the echo of our own empty-file
|
||||||
|
// create, and re-fetching would replace the size and mtime of the
|
||||||
|
// write we just accepted with the stale revision's — making a file
|
||||||
|
// that was copied in seconds ago read as empty until its upload
|
||||||
|
// lands (offline.md Phase 3).
|
||||||
|
//
|
||||||
|
// A write that has already drained is the same story one step later:
|
||||||
|
// the drain brought the tree level with the revision it sealed, so
|
||||||
|
// the feed's report of that revision has nothing to add, and acting
|
||||||
|
// on it evicts the very bytes we uploaded. Claimed unconditionally,
|
||||||
|
// even for a trash, so an echo we no longer care about does not
|
||||||
|
// linger to suppress a later foreign change.
|
||||||
|
let echo = core.take_self_change(node_uid);
|
||||||
|
let ours = !*is_trashed && (echo || core.pending.lock().contains_key(node_uid));
|
||||||
|
if ours {
|
||||||
|
debug!(uid = %node_uid, "ignoring remote event for a node with a queued write");
|
||||||
|
}
|
||||||
|
let mut had_node = false;
|
||||||
|
core.for_each_mount(|st, notifier| {
|
||||||
|
if *is_trashed {
|
||||||
|
// Trashing makes a node vanish from its parent listing.
|
||||||
|
had_node |= forget_and_notify(st, notifier, node_uid);
|
||||||
|
} else if !ours && let Some(&ino) = st.by_uid.get(node_uid) {
|
||||||
|
// Known node changed: drop its cached attrs/data (and
|
||||||
|
// listing if it is a directory) so the next access
|
||||||
|
// re-fetches. Its content blob may now be stale too.
|
||||||
|
had_node = true;
|
||||||
|
st.invalidate_listing(ino);
|
||||||
|
if let Some(notifier) = notifier {
|
||||||
|
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// Once, not once per mount: the content cache is per-daemon.
|
||||||
|
if had_node {
|
||||||
|
core.cache.evict(node_uid);
|
||||||
|
}
|
||||||
|
// A create, rename or move-in shows up as a change to the parent
|
||||||
|
// listing too, which is recorded for the end of the batch rather
|
||||||
|
// than acted on here — see [`DirtyParents`].
|
||||||
|
if let Some(parent_uid) = parent_node_uid {
|
||||||
|
match ours {
|
||||||
|
true => dirty
|
||||||
|
.ours
|
||||||
|
.entry(parent_uid.clone())
|
||||||
|
.or_default()
|
||||||
|
.insert(node_uid.clone()),
|
||||||
|
false => dirty.foreign.insert(parent_uid.clone()),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DriveEvent::NodeDeleted { node_uid, .. } => {
|
||||||
|
core.cache.evict(node_uid);
|
||||||
|
core.for_each_mount(|st, notifier| {
|
||||||
|
forget_and_notify(st, notifier, node_uid);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
// Continuity or scope was lost: our cached listings may be arbitrarily
|
// Continuity or scope was lost: our cached listings may be arbitrarily
|
||||||
// stale, so drop every listing and tell the kernel to forget all
|
// stale, so drop every listing and tell the kernel to forget all
|
||||||
// metadata. Inodes stay stable; dirs simply re-enumerate on next access.
|
// metadata. Inodes stay stable; dirs simply re-enumerate on next access.
|
||||||
DriveEvent::ContinuityLost { .. } | DriveEvent::ScopeAccessLost { .. } => {
|
DriveEvent::ContinuityLost { .. } | DriveEvent::ScopeAccessLost { .. } => {
|
||||||
warn!("event continuity lost; dropping all cached listings, resyncing lazily");
|
warn!("event continuity lost; dropping all cached listings, resyncing lazily");
|
||||||
let mut st = state.lock();
|
core.for_each_mount(|st, notifier| {
|
||||||
let dirs: Vec<u64> = st.children.keys().copied().collect();
|
let dirs: Vec<u64> = st.children.keys().copied().collect();
|
||||||
for &ino in &dirs {
|
for &ino in &dirs {
|
||||||
st.invalidate_listing(ino);
|
st.invalidate_listing(ino);
|
||||||
|
if let Some(notifier) = notifier {
|
||||||
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
let _ = notifier.inval_inode(INodeNo(ino), 0, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
// No substantive local change; the cursor advance is handled by the
|
// No substantive local change; the cursor advance is handled by the
|
||||||
// caller persisting the event id.
|
// caller persisting the event id.
|
||||||
DriveEvent::CursorAdvanced { .. } | DriveEvent::SharedWithMeUpdated { .. } => {}
|
DriveEvent::CursorAdvanced { .. } | DriveEvent::SharedWithMeUpdated { .. } => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The folders one batch of events changed, collected so the listings are
|
||||||
|
/// dropped once at the end of the batch rather than once per event.
|
||||||
|
///
|
||||||
|
/// Both halves of that matter. A burst of uploads produces one event per file
|
||||||
|
/// all naming the *same* folder, and dropping the listing per event made every
|
||||||
|
/// interleaved lookup re-enumerate that folder from the API — quadratic in its
|
||||||
|
/// size, and slow enough to stall a busy mount. Splitting foreign changes from
|
||||||
|
/// our own then removes the re-enumeration entirely for the common case: a file
|
||||||
|
/// this daemon created is already in the tree under the right parent, so there
|
||||||
|
/// is nothing for a re-enumeration to discover.
|
||||||
|
///
|
||||||
|
/// Neither is a weakening. The state after the batch is what it always was.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct DirtyParents {
|
||||||
|
/// Folders changed by someone else. Their listings must go: an event names
|
||||||
|
/// no name, so a rename inside a folder is indistinguishable from a write to
|
||||||
|
/// a file in it, and only re-enumerating can tell us which happened.
|
||||||
|
foreign: HashSet<NodeUid>,
|
||||||
|
/// Folders changed by this daemon, and the children the changes were about.
|
||||||
|
/// Checked per mount — a mount that already lists every one of those
|
||||||
|
/// children under that parent keeps its listing.
|
||||||
|
ours: HashMap<NodeUid, HashSet<NodeUid>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether this mount's cached listing of `parent_uid` already accounts for
|
||||||
|
/// `child_uid`, so a change we made to that child leaves nothing to re-read.
|
||||||
|
///
|
||||||
|
/// "Nothing cached" counts as current: an absent listing is re-enumerated on the
|
||||||
|
/// next `readdir` regardless, so there is nothing to drop.
|
||||||
|
fn listing_accounts_for(st: &State, parent_uid: &NodeUid, child_uid: &NodeUid) -> bool {
|
||||||
|
let Some(&parent) = st.by_uid.get(parent_uid) else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
let Some(children) = st.children.get(&parent) else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
let Some(&child) = st.by_uid.get(child_uid) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
children.contains(&child) && st.entries.get(&child).is_some_and(|e| e.parent == parent)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop the cached listing of every folder a batch of events changed, in every
|
||||||
|
/// mount that holds one, so the next `readdir`/`lookup` re-enumerates it.
|
||||||
|
///
|
||||||
|
/// Runs strictly after the whole batch is applied: doing it per event would
|
||||||
|
/// leave a window where a concurrent `readdir` re-populates a listing that a
|
||||||
|
/// later event in the same batch then has to drop again.
|
||||||
|
fn flush_dirty_parents(core: &Core, dirty: &DirtyParents) {
|
||||||
|
if dirty.foreign.is_empty() && dirty.ours.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
core.for_each_mount(|st, notifier| {
|
||||||
|
let drop_listing = |st: &mut State, parent_uid: &NodeUid| {
|
||||||
|
let Some(&parent) = st.by_uid.get(parent_uid) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
st.invalidate_listing(parent);
|
||||||
|
if let Some(notifier) = notifier {
|
||||||
|
let _ = notifier.inval_inode(INodeNo(parent), 0, 0);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for parent_uid in &dirty.foreign {
|
||||||
|
drop_listing(st, parent_uid);
|
||||||
|
}
|
||||||
|
for (parent_uid, children) in &dirty.ours {
|
||||||
|
if dirty.foreign.contains(parent_uid) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// One child this mount has not placed is enough: the listing it is
|
||||||
|
// serving is missing something the user just made.
|
||||||
|
if children
|
||||||
|
.iter()
|
||||||
|
.all(|child| listing_accounts_for(st, parent_uid, child))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
drop_listing(st, parent_uid);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/// Poll the remote event cursor forever, applying each batch to the shared
|
/// Poll the remote event cursor forever, applying each batch to the shared
|
||||||
/// state. Resumes from the cursor persisted in the DB so changes made while
|
/// state. Resumes from the cursor persisted in the DB so changes made while
|
||||||
/// unmounted are applied; only a first-ever mount seeds from the server head.
|
/// unmounted are applied; only a first-ever mount seeds from the server head.
|
||||||
/// The cursor is persisted after every batch. Runs as a Tokio task; returns
|
/// The cursor is persisted after every batch. Runs as a Tokio task; returns
|
||||||
/// only on fatal error.
|
/// only on fatal error.
|
||||||
|
///
|
||||||
|
/// Takes the whole `Core` rather than the primary mount's pieces: an event has
|
||||||
|
/// to be applied to every mounted inode space, and the registry that enumerates
|
||||||
|
/// them lives on the `Core` (see [`apply_event`]).
|
||||||
pub(super) async fn run_event_sync(
|
pub(super) async fn run_event_sync(
|
||||||
client: ProtonDriveClient,
|
client: ProtonDriveClient,
|
||||||
scope: DriveEventScopeId,
|
scope: DriveEventScopeId,
|
||||||
state: Arc<Mutex<State>>,
|
core: Core,
|
||||||
content: Arc<ContentCache>,
|
|
||||||
db: Arc<Db>,
|
|
||||||
pending: Arc<Mutex<HashMap<NodeUid, PendingRevision>>>,
|
|
||||||
notifier: Notifier,
|
|
||||||
) {
|
) {
|
||||||
|
let db = core.db.clone();
|
||||||
let mut cursor: Option<DriveEventId> = match db.get_event_cursor() {
|
let mut cursor: Option<DriveEventId> = match db.get_event_cursor() {
|
||||||
// Resume: pick up exactly where the last run left off.
|
// Resume: pick up exactly where the last run left off.
|
||||||
Ok(Some(saved)) => Some(DriveEventId::from(saved)),
|
Ok(Some(saved)) => Some(DriveEventId::from(saved)),
|
||||||
@@ -126,12 +242,16 @@ pub(super) async fn run_event_sync(
|
|||||||
match client.enumerate_events(&scope, None).await {
|
match client.enumerate_events(&scope, None).await {
|
||||||
Ok(events) => {
|
Ok(events) => {
|
||||||
let head = events.last().map(|e| e.id().clone());
|
let head = events.last().map(|e| e.id().clone());
|
||||||
if let Some(c) = &head
|
let Some(c) = &head else {
|
||||||
&& let Err(e) = db.set_event_cursor(c.as_str())
|
break None;
|
||||||
{
|
};
|
||||||
warn!(error = %e, "persist seed cursor failed");
|
if let Err(e) = db.set_event_cursor(c.as_str()) {
|
||||||
|
warn!(error = %e, ?delay, "persist seed cursor failed; retrying");
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
delay = (delay * 2).min(ONLINE_PROBE_MAX);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
break head;
|
break Some(c.clone());
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
warn!(error = %e, ?delay, "seed event cursor failed; retrying");
|
warn!(error = %e, ?delay, "seed event cursor failed; retrying");
|
||||||
@@ -161,6 +281,11 @@ pub(super) async fn run_event_sync(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
debug!(count = events.len(), "applying remote events");
|
debug!(count = events.len(), "applying remote events");
|
||||||
|
// Folders whose listing this batch invalidates, applied once at the end
|
||||||
|
// rather than per event — see [`DirtyParents`]. Every `break` below falls
|
||||||
|
// through to that flush, so an interrupted batch still publishes what it
|
||||||
|
// did apply.
|
||||||
|
let mut dirty = DirtyParents::default();
|
||||||
for event in &events {
|
for event in &events {
|
||||||
// Converge the SDK's own caches (folder keys, entity cache) on the
|
// Converge the SDK's own caches (folder keys, entity cache) on the
|
||||||
// server before applying the event to our tree. Without this, a node
|
// server before applying the event to our tree. Without this, a node
|
||||||
@@ -168,16 +293,21 @@ pub(super) async fn run_event_sync(
|
|||||||
// the life of the daemon (SDK plan #9). `apply_event` only touches
|
// the life of the daemon (SDK plan #9). `apply_event` only touches
|
||||||
// our FUSE state, so nothing else does this.
|
// our FUSE state, so nothing else does this.
|
||||||
if let Err(e) = client.invalidate_caches_for_event(event).await {
|
if let Err(e) = client.invalidate_caches_for_event(event).await {
|
||||||
warn!(error = %e, "sdk cache invalidation for event failed");
|
warn!(error = %e, "sdk cache invalidation for event failed; retaining cursor for retry");
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
apply_event(&state, &content, &pending, ¬ifier, event);
|
apply_event(&core, event, &mut dirty);
|
||||||
|
let applied = event.id().clone();
|
||||||
|
if let Err(e) = db.set_event_cursor(applied.as_str()) {
|
||||||
|
warn!(error = %e, "persist event cursor failed; retaining prior cursor for retry");
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
cursor = events.last().map(|e| e.id().clone());
|
// Advance only after this exact event is durably acknowledged. If a
|
||||||
if let Some(c) = &cursor
|
// later event fails, polling resumes from here and replays nothing
|
||||||
&& let Err(e) = db.set_event_cursor(c.as_str())
|
// that was not already made visible locally.
|
||||||
{
|
cursor = Some(applied);
|
||||||
warn!(error = %e, "persist event cursor failed");
|
|
||||||
}
|
}
|
||||||
|
flush_dirty_parents(&core, &dirty);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,8 +12,8 @@
|
|||||||
use std::io::{BufRead, BufReader, Write as _};
|
use std::io::{BufRead, BufReader, Write as _};
|
||||||
use std::os::unix::net::{UnixListener, UnixStream};
|
use std::os::unix::net::{UnixListener, UnixStream};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::time::Instant;
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use pdfs_core::config::AppDirs;
|
use pdfs_core::config::AppDirs;
|
||||||
use pdfs_core::control::{
|
use pdfs_core::control::{
|
||||||
@@ -27,6 +27,67 @@ use tracing::{info, warn};
|
|||||||
use super::transfers::CountingReader;
|
use super::transfers::CountingReader;
|
||||||
use super::{Core, count_noun, human_bytes, human_duration, parse_uid};
|
use super::{Core, count_noun, human_bytes, human_duration, parse_uid};
|
||||||
|
|
||||||
|
const MAX_CONTROL_REQUEST_BYTES: u64 = 1024 * 1024;
|
||||||
|
const CONTROL_READ_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
const MAX_CONTROL_HANDLERS: usize = 64;
|
||||||
|
static ACTIVE_CONTROL_HANDLERS: AtomicUsize = AtomicUsize::new(0);
|
||||||
|
|
||||||
|
struct ControlHandlerPermit;
|
||||||
|
|
||||||
|
impl ControlHandlerPermit {
|
||||||
|
fn acquire() -> Option<Self> {
|
||||||
|
ACTIVE_CONTROL_HANDLERS
|
||||||
|
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| {
|
||||||
|
(active < MAX_CONTROL_HANDLERS).then_some(active + 1)
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
.map(|_| Self)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ControlHandlerPermit {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
ACTIVE_CONTROL_HANDLERS.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_request_line(reader: &mut impl BufRead) -> std::io::Result<Option<String>> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
loop {
|
||||||
|
let available = reader.fill_buf()?;
|
||||||
|
if available.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let take = available
|
||||||
|
.iter()
|
||||||
|
.position(|byte| *byte == b'\n')
|
||||||
|
.map_or(available.len(), |index| index + 1);
|
||||||
|
if bytes.len().saturating_add(take) > MAX_CONTROL_REQUEST_BYTES as usize {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidData,
|
||||||
|
"control request exceeds size limit",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
bytes.extend_from_slice(&available[..take]);
|
||||||
|
reader.consume(take);
|
||||||
|
if bytes.ends_with(b"\n") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if bytes.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if bytes.len() as u64 > MAX_CONTROL_REQUEST_BYTES || !bytes.ends_with(b"\n") {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidData,
|
||||||
|
"control request exceeds limit or is not newline terminated",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
String::from_utf8(bytes)
|
||||||
|
.map(Some)
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
|
||||||
|
}
|
||||||
|
|
||||||
/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path
|
/// Turn a CLI-supplied path into a mountpoint-relative path. An absolute path
|
||||||
/// must live under `mountpoint`; a relative path is taken as already relative to
|
/// must live under `mountpoint`; a relative path is taken as already relative to
|
||||||
/// the mount root.
|
/// the mount root.
|
||||||
@@ -44,6 +105,10 @@ fn rel_to_mount(mountpoint: &Path, path: &str) -> CoreResult<PathBuf> {
|
|||||||
/// Handle one control-socket connection: read a single JSON request line,
|
/// Handle one control-socket connection: read a single JSON request line,
|
||||||
/// dispatch it against `core`, and write back a JSON response line.
|
/// dispatch it against `core`, and write back a JSON response line.
|
||||||
fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: UnixStream) {
|
fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: UnixStream) {
|
||||||
|
if let Err(e) = stream.set_read_timeout(Some(CONTROL_READ_TIMEOUT)) {
|
||||||
|
warn!(error = %e, "control: could not set request timeout");
|
||||||
|
return;
|
||||||
|
}
|
||||||
let mut reader = BufReader::new(match stream.try_clone() {
|
let mut reader = BufReader::new(match stream.try_clone() {
|
||||||
Ok(s) => s,
|
Ok(s) => s,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -51,10 +116,14 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
let mut line = String::new();
|
let line = match read_request_line(&mut reader) {
|
||||||
if reader.read_line(&mut line).is_err() || line.trim().is_empty() {
|
Ok(Some(line)) if !line.trim().is_empty() => line,
|
||||||
|
Ok(_) => return,
|
||||||
|
Err(e) => {
|
||||||
|
warn!(error = %e, "control: rejected request");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
};
|
||||||
let response = match serde_json::from_str::<CtlRequest>(line.trim()) {
|
let response = match serde_json::from_str::<CtlRequest>(line.trim()) {
|
||||||
Ok(CtlRequest::Status) => {
|
Ok(CtlRequest::Status) => {
|
||||||
let pins = core.cache.list_pins();
|
let pins = core.cache.list_pins();
|
||||||
@@ -792,12 +861,22 @@ pub(crate) fn run_control_socket(
|
|||||||
for conn in listener.incoming() {
|
for conn in listener.incoming() {
|
||||||
match conn {
|
match conn {
|
||||||
Ok(stream) => {
|
Ok(stream) => {
|
||||||
|
let Some(permit) = ControlHandlerPermit::acquire() else {
|
||||||
|
warn!(
|
||||||
|
limit = MAX_CONTROL_HANDLERS,
|
||||||
|
"control: rejecting connection at handler limit"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let core = core.clone();
|
let core = core.clone();
|
||||||
let username = username.clone();
|
let username = username.clone();
|
||||||
let mountpoint = mountpoint.clone();
|
let mountpoint = mountpoint.clone();
|
||||||
if let Err(e) = std::thread::Builder::new()
|
if let Err(e) = std::thread::Builder::new()
|
||||||
.name("pdfs-control".into())
|
.name("pdfs-control".into())
|
||||||
.spawn(move || handle_control_conn(&core, &username, &mountpoint, stream))
|
.spawn(move || {
|
||||||
|
let _permit = permit;
|
||||||
|
handle_control_conn(&core, &username, &mountpoint, stream);
|
||||||
|
})
|
||||||
{
|
{
|
||||||
warn!(error = %e, "control: spawn handler failed");
|
warn!(error = %e, "control: spawn handler failed");
|
||||||
}
|
}
|
||||||
@@ -808,3 +887,38 @@ pub(crate) fn run_control_socket(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod request_limit_tests {
|
||||||
|
use super::*;
|
||||||
|
use std::io::Cursor;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn accepts_one_bounded_newline_terminated_request() {
|
||||||
|
let mut input = Cursor::new(b"{\"Status\":null}\n".to_vec());
|
||||||
|
assert_eq!(
|
||||||
|
read_request_line(&mut input).unwrap().as_deref(),
|
||||||
|
Some("{\"Status\":null}\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_a_request_without_a_terminating_newline() {
|
||||||
|
let mut input = Cursor::new(b"{\"Status\":null}".to_vec());
|
||||||
|
assert_eq!(
|
||||||
|
read_request_line(&mut input).unwrap_err().kind(),
|
||||||
|
std::io::ErrorKind::InvalidData
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_a_request_larger_than_the_limit() {
|
||||||
|
let mut bytes = vec![b'x'; MAX_CONTROL_REQUEST_BYTES as usize + 1];
|
||||||
|
bytes.push(b'\n');
|
||||||
|
let mut input = Cursor::new(bytes);
|
||||||
|
assert_eq!(
|
||||||
|
read_request_line(&mut input).unwrap_err().kind(),
|
||||||
|
std::io::ErrorKind::InvalidData
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -505,6 +505,11 @@ impl Core {
|
|||||||
fork.notifier = Arc::new(OnceLock::new());
|
fork.notifier = Arc::new(OnceLock::new());
|
||||||
// Size upgrades are keyed by inode, which is per-fork too.
|
// Size upgrades are keyed by inode, which is per-fork too.
|
||||||
fork.size_upgrades = Arc::new(Mutex::new(HashMap::new()));
|
fork.size_upgrades = Arc::new(Mutex::new(HashMap::new()));
|
||||||
|
// The daemon's single drain thread belongs to the primary mount, but it
|
||||||
|
// drains ops for nodes that live *here*. Publishing this inode space is
|
||||||
|
// what lets it find them; without it a queued create's node keeps its
|
||||||
|
// `local~` uid after the upload lands and reads it as empty (B74).
|
||||||
|
fork.register_state();
|
||||||
fork
|
fork
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+225
-23
@@ -36,8 +36,8 @@ use super::state::Intervals;
|
|||||||
use super::transfers::CountingReader;
|
use super::transfers::CountingReader;
|
||||||
use super::{
|
use super::{
|
||||||
Core, DRAIN_BACKOFF_MAX, DRAIN_BACKOFF_MIN, DRAIN_IDLE_POLL, ROOT_INO, conflict_name,
|
Core, DRAIN_BACKOFF_MAX, DRAIN_BACKOFF_MIN, DRAIN_IDLE_POLL, ROOT_INO, conflict_name,
|
||||||
is_already_exists, is_gone, is_local_uid_str, media_type_for, node_size, now_millis, now_secs,
|
is_already_exists, is_gone, is_local_uid_str, media_type_for, node_revision_id, node_size,
|
||||||
parse_node_uid,
|
now_millis, now_secs, parse_node_uid,
|
||||||
};
|
};
|
||||||
|
|
||||||
impl Core {
|
impl Core {
|
||||||
@@ -229,6 +229,7 @@ impl Core {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.db.delete_op(op.id)?;
|
self.db.delete_op(op.id)?;
|
||||||
|
self.note_self_change(&uid);
|
||||||
info!(%uid, name = %landed, "pending rename landed");
|
info!(%uid, name = %landed, "pending rename landed");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -252,6 +253,7 @@ impl Core {
|
|||||||
Err(e) => return Err(e.into()),
|
Err(e) => return Err(e.into()),
|
||||||
}
|
}
|
||||||
self.db.delete_op(op.id)?;
|
self.db.delete_op(op.id)?;
|
||||||
|
self.own_sealed_revs.lock().remove(&uid);
|
||||||
self.invalidate_trash();
|
self.invalidate_trash();
|
||||||
self.log_activity(ActivityKind::Trash, &name, "trashed", true);
|
self.log_activity(ActivityKind::Trash, &name, "trashed", true);
|
||||||
info!(%uid, name, "pending trash landed");
|
info!(%uid, name, "pending trash landed");
|
||||||
@@ -262,7 +264,7 @@ impl Core {
|
|||||||
/// it away from the one the user asked for. Best effort: the event sync
|
/// it away from the one the user asked for. Best effort: the event sync
|
||||||
/// would correct the tree anyway, but not before the user has looked at it.
|
/// would correct the tree anyway, but not before the user has looked at it.
|
||||||
pub(crate) fn adopt_drained_name(&self, uid: &NodeUid, name: &str) {
|
pub(crate) fn adopt_drained_name(&self, uid: &NodeUid, name: &str) {
|
||||||
let mut st = self.state.lock();
|
self.for_each_state(|st| {
|
||||||
let Some(&ino) = st.by_uid.get(uid) else {
|
let Some(&ino) = st.by_uid.get(uid) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -274,6 +276,7 @@ impl Core {
|
|||||||
if let Err(e) = st.db.upsert_node(&node) {
|
if let Err(e) = st.db.upsert_node(&node) {
|
||||||
warn!(%uid, error = %e, "db upsert_node failed after a conflict rename");
|
warn!(%uid, error = %e, "db upsert_node failed after a conflict rename");
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make a node that so far exists only on this machine real, and adopt the
|
/// Make a node that so far exists only on this machine real, and adopt the
|
||||||
@@ -342,6 +345,9 @@ impl Core {
|
|||||||
// sync, whereas a surviving op would create the file a second time.
|
// sync, whereas a surviving op would create the file a second time.
|
||||||
self.db.delete_op(op.id)?;
|
self.db.delete_op(op.id)?;
|
||||||
self.adopt_real_uid(&local, &real)?;
|
self.adopt_real_uid(&local, &real)?;
|
||||||
|
// The feed will report this create back to us; the tree already has it
|
||||||
|
// under its real uid, so that event is ours to ignore (`Core::self_changes`).
|
||||||
|
self.note_self_change(&real);
|
||||||
if let Some(blob) = op.blob_path.as_deref() {
|
if let Some(blob) = op.blob_path.as_deref() {
|
||||||
self.cache.discard_staged(Path::new(blob));
|
self.cache.discard_staged(Path::new(blob));
|
||||||
}
|
}
|
||||||
@@ -418,6 +424,14 @@ impl Core {
|
|||||||
///
|
///
|
||||||
/// The inode is deliberately kept, so anything already holding the file open
|
/// The inode is deliberately kept, so anything already holding the file open
|
||||||
/// keeps working across the drain.
|
/// keeps working across the drain.
|
||||||
|
///
|
||||||
|
/// Applied to **every** live inode space, not just this Core's. A node
|
||||||
|
/// created inside an on-demand sync folder lives in that fork's `state`,
|
||||||
|
/// while the drain that lands its create belongs to the primary mount — so
|
||||||
|
/// reaching for `self.state` here found nothing and left the fork's entry on
|
||||||
|
/// its `local~` uid, which reads as an empty file for as long as the daemon
|
||||||
|
/// runs (`docs/BUGS.md` B74). A uid is unique across mounts, so at most one
|
||||||
|
/// of them matches and the others are no-ops.
|
||||||
pub(crate) fn adopt_real_uid(
|
pub(crate) fn adopt_real_uid(
|
||||||
&self,
|
&self,
|
||||||
local: &NodeUid,
|
local: &NodeUid,
|
||||||
@@ -430,8 +444,10 @@ impl Core {
|
|||||||
self.db
|
self.db
|
||||||
.remap_local_uid(&local.to_string(), &real.to_string())?;
|
.remap_local_uid(&local.to_string(), &real.to_string())?;
|
||||||
|
|
||||||
let mut st = self.state.lock();
|
self.for_each_state(|st| {
|
||||||
if let Some(ino) = st.by_uid.remove(local) {
|
let Some(ino) = st.by_uid.remove(local) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
st.by_uid.insert(real.clone(), ino);
|
st.by_uid.insert(real.clone(), ino);
|
||||||
if let Some(e) = st.entries.get_mut(&ino) {
|
if let Some(e) = st.entries.get_mut(&ino) {
|
||||||
e.uid = real.clone();
|
e.uid = real.clone();
|
||||||
@@ -459,8 +475,7 @@ impl Core {
|
|||||||
kids.push(ino);
|
kids.push(ino);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
drop(st);
|
|
||||||
// Write the real node through, now that nothing points at the old uid.
|
// Write the real node through, now that nothing points at the old uid.
|
||||||
if let Err(e) = self.db.upsert_node(&node) {
|
if let Err(e) = self.db.upsert_node(&node) {
|
||||||
warn!(%real, error = %e, "db upsert_node failed after remap");
|
warn!(%real, error = %e, "db upsert_node failed after remap");
|
||||||
@@ -503,15 +518,23 @@ impl Core {
|
|||||||
if node.trashed {
|
if node.trashed {
|
||||||
return Ok(Some("the file was trashed remotely".into()));
|
return Ok(Some("the file was trashed remotely".into()));
|
||||||
}
|
}
|
||||||
let (mtime, size) = (node.modification_time, node_size(&node));
|
let Some(reason) = revision_changed(base, &node) else {
|
||||||
if mtime != base.mtime || size != base.size {
|
return Ok(None);
|
||||||
return Ok(Some(format!(
|
};
|
||||||
"the remote revision changed under the queued write \
|
// The remote moved on from the baseline — normally a conflict. But if it
|
||||||
(expected {} bytes at mtime {}, found {size} at {mtime})",
|
// sits at a revision *this daemon* sealed, no other device changed the
|
||||||
base.size, base.mtime
|
// file: it is a single-writer stall→resume (a browser that closed and
|
||||||
)));
|
// reopened its download fd; B70 layer B). Chain onto it — supersede
|
||||||
|
// instead of forking — so the finished bytes win the name rather than
|
||||||
|
// being exiled to a `(sync-conflict)` copy.
|
||||||
|
let remote_rev = node_revision_id(&node);
|
||||||
|
let own_rev = self.own_sealed_revs.lock().get(uid).cloned();
|
||||||
|
if is_own_self_supersede(meta.complete, remote_rev.as_deref(), own_rev.as_deref()) {
|
||||||
|
debug!(%uid, ?remote_rev,
|
||||||
|
"queued write chains onto our own sealed revision; not a conflict");
|
||||||
|
return Ok(None);
|
||||||
}
|
}
|
||||||
Ok(None)
|
Ok(Some(reason))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Land a queued write that can no longer be applied to its own node as a
|
/// Land a queued write that can no longer be applied to its own node as a
|
||||||
@@ -595,11 +618,11 @@ impl Core {
|
|||||||
self.cache.evict(uid);
|
self.cache.evict(uid);
|
||||||
self.evict_reader(uid);
|
self.evict_reader(uid);
|
||||||
// The conflict copy is a node the tree has never seen.
|
// The conflict copy is a node the tree has never seen.
|
||||||
let mut st = self.state.lock();
|
self.for_each_state(|st| {
|
||||||
if let Some(&ino) = st.by_uid.get(&parent) {
|
if let Some(&ino) = st.by_uid.get(&parent) {
|
||||||
st.invalidate_listing(ino);
|
st.invalidate_listing(ino);
|
||||||
}
|
}
|
||||||
drop(st);
|
});
|
||||||
self.log_activity(
|
self.log_activity(
|
||||||
ActivityKind::Upload,
|
ActivityKind::Upload,
|
||||||
&name,
|
&name,
|
||||||
@@ -653,13 +676,25 @@ impl Core {
|
|||||||
|
|
||||||
/// A node's parent uid and name, as the tree currently has them.
|
/// A node's parent uid and name, as the tree currently has them.
|
||||||
pub(crate) fn node_place(&self, uid: &NodeUid) -> Option<(NodeUid, String)> {
|
pub(crate) fn node_place(&self, uid: &NodeUid) -> Option<(NodeUid, String)> {
|
||||||
{
|
// Across every mount: the drain runs on the primary Core, but a node
|
||||||
let st = self.state.lock();
|
// written inside a sync folder is interned in that fork's inode space,
|
||||||
|
// so asking `self.state` alone always missed it and fell through to the
|
||||||
|
// DB. That still gave the right answer — the fallback below exists for
|
||||||
|
// exactly this shape of miss — but it went to disk for something already
|
||||||
|
// in memory, and only *because* of the fallback was it not a bug.
|
||||||
|
let mut found = None;
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
if found.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
if let Some(entry) = st.by_uid.get(uid).and_then(|ino| st.entries.get(ino))
|
if let Some(entry) = st.by_uid.get(uid).and_then(|ino| st.entries.get(ino))
|
||||||
&& let Some(parent) = entry.node.parent_uid.clone()
|
&& let Some(parent) = entry.node.parent_uid.clone()
|
||||||
{
|
{
|
||||||
return Some((parent, entry.node.name.clone()));
|
found = Some((parent, entry.node.name.clone()));
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
if found.is_some() {
|
||||||
|
return found;
|
||||||
}
|
}
|
||||||
// The in-memory tree only holds what has been walked to since the daemon
|
// The in-memory tree only holds what has been walked to since the daemon
|
||||||
// started, and the drain routinely runs before anything has walked to
|
// started, and the drain routinely runs before anything has walked to
|
||||||
@@ -814,6 +849,11 @@ impl Core {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
// This function's whole job is to bring the tree level with the revision
|
||||||
|
// we just sealed, so the feed's report of that revision has nothing left
|
||||||
|
// to tell us. Claimed here rather than at the upload call so it is only
|
||||||
|
// recorded when the node was actually re-read (`Core::self_changes`).
|
||||||
|
self.note_self_change(uid);
|
||||||
// Rebase any *open* write handles targeting this node. A handle opened
|
// Rebase any *open* write handles targeting this node. A handle opened
|
||||||
// before this upload carries a stale base — its `base_mtime`/`base_size`
|
// before this upload carries a stale base — its `base_mtime`/`base_size`
|
||||||
// name the revision we just replaced. Without this update,
|
// name the revision we just replaced. Without this update,
|
||||||
@@ -826,15 +866,25 @@ impl Core {
|
|||||||
{
|
{
|
||||||
let sealed_mtime = node.modification_time;
|
let sealed_mtime = node.modification_time;
|
||||||
let sealed_size = node_size(&node);
|
let sealed_size = node_size(&node);
|
||||||
let mut st = self.state.lock();
|
let sealed_rev = node_revision_id(&node);
|
||||||
|
// Remember it as ours, so a queued write that opened over an earlier
|
||||||
|
// revision recognises this as a self-supersede rather than a foreign
|
||||||
|
// change and does not fork (B70 layer B; consulted in
|
||||||
|
// `revision_conflict`).
|
||||||
|
if let Some(rev) = sealed_rev.clone() {
|
||||||
|
self.own_sealed_revs.lock().insert(uid.clone(), rev);
|
||||||
|
}
|
||||||
|
self.for_each_state(|st| {
|
||||||
for aw in st.active_writes.values_mut() {
|
for aw in st.active_writes.values_mut() {
|
||||||
if aw.uid == *uid {
|
if aw.uid == *uid {
|
||||||
aw.base_mtime = sealed_mtime;
|
aw.base_mtime = sealed_mtime;
|
||||||
aw.base_size = sealed_size;
|
aw.base_size = sealed_size;
|
||||||
|
aw.base_revision_id = sealed_rev.clone();
|
||||||
debug!(%uid, mtime = sealed_mtime, size = sealed_size,
|
debug!(%uid, mtime = sealed_mtime, size = sealed_size,
|
||||||
"rebased open write handle onto the revision just uploaded");
|
"rebased open write handle onto the revision just uploaded");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
// Ordered so that a write queued *during* the fetch above is still
|
// Ordered so that a write queued *during* the fetch above is still
|
||||||
// caught: it took its baseline from the node's optimistic stamp, and
|
// caught: it took its baseline from the node's optimistic stamp, and
|
||||||
@@ -842,7 +892,7 @@ impl Core {
|
|||||||
if self.rebaseline_pending(uid, &node) {
|
if self.rebaseline_pending(uid, &node) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let mut st = self.state.lock();
|
self.for_each_state(|st| {
|
||||||
let Some(parent) = st
|
let Some(parent) = st
|
||||||
.by_uid
|
.by_uid
|
||||||
.get(uid)
|
.get(uid)
|
||||||
@@ -851,7 +901,8 @@ impl Core {
|
|||||||
else {
|
else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
st.intern(parent, node);
|
st.intern(parent, node.clone());
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Point a still-queued write at the revision this upload just sealed, and
|
/// Point a still-queued write at the revision this upload just sealed, and
|
||||||
@@ -875,6 +926,7 @@ impl Core {
|
|||||||
mtime: sealed.modification_time,
|
mtime: sealed.modification_time,
|
||||||
size: node_size(sealed),
|
size: node_size(sealed),
|
||||||
hash: None,
|
hash: None,
|
||||||
|
revision_id: node_revision_id(sealed),
|
||||||
};
|
};
|
||||||
let mut pending = self.pending.lock();
|
let mut pending = self.pending.lock();
|
||||||
let Some(p) = pending.get_mut(uid) else {
|
let Some(p) = pending.get_mut(uid) else {
|
||||||
@@ -896,3 +948,153 @@ impl Core {
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the remote `node` moved on from the revision a queued write was
|
||||||
|
/// `base`d on — the reason string when it did, `None` when the write can still be
|
||||||
|
/// applied. Pure so it is testable without a live remote (the trashed/gone checks
|
||||||
|
/// stay in [`Core::revision_conflict`], which does the fetch).
|
||||||
|
///
|
||||||
|
/// The revision id is the authoritative identity: it advances iff a *new* revision
|
||||||
|
/// was sealed. When both sides carry one, it alone decides — the server re-stamps
|
||||||
|
/// the *same* revision's mtime, and reading that drift as a change is exactly what
|
||||||
|
/// diverted a write into a conflict copy of its own base (B16/B25). Only when
|
||||||
|
/// there is no id to compare (an old sidecar, or a surface that omits it) does it
|
||||||
|
/// fall back to the observable `(mtime, size)` tuple.
|
||||||
|
fn revision_changed(base: &Baseline, node: &Node) -> Option<String> {
|
||||||
|
if let (Some(base_rev), Some(remote_rev)) = (&base.revision_id, node_revision_id(node)) {
|
||||||
|
return (*base_rev != remote_rev).then(|| {
|
||||||
|
format!(
|
||||||
|
"the remote revision changed under the queued write \
|
||||||
|
(based on revision {base_rev}, remote now at {remote_rev})"
|
||||||
|
)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let (mtime, size) = (node.modification_time, node_size(node));
|
||||||
|
(mtime != base.mtime || size != base.size).then(|| {
|
||||||
|
format!(
|
||||||
|
"the remote revision changed under the queued write \
|
||||||
|
(expected {} bytes at mtime {}, found {size} at {mtime})",
|
||||||
|
base.size, base.mtime
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a queued write that found the remote moved on ([`revision_changed`]
|
||||||
|
/// fired) is nonetheless a single-writer self-supersede rather than a foreign
|
||||||
|
/// conflict — the remote sits at a revision this daemon sealed itself, so
|
||||||
|
/// chaining onto it is safe and forking would be wrong (B70 layer B).
|
||||||
|
///
|
||||||
|
/// `remote_rev` is the id the remote currently holds, `own_rev` this node's
|
||||||
|
/// latest own-sealed id. Requires a `complete` blob: an incomplete one's gaps
|
||||||
|
/// still refer to the stale base, so it must take the non-destructive
|
||||||
|
/// conflict-copy path instead of overwriting. A `None` remote id never matches
|
||||||
|
/// (there is nothing to prove it was ours), so absence falls through to a fork.
|
||||||
|
fn is_own_self_supersede(complete: bool, remote_rev: Option<&str>, own_rev: Option<&str>) -> bool {
|
||||||
|
complete && remote_rev.is_some() && remote_rev == own_rev
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use proton_drive_rs::NodeKind;
|
||||||
|
use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId};
|
||||||
|
|
||||||
|
fn file_node(mtime: i64, size: i64, rev: Option<&str>) -> Node {
|
||||||
|
Node {
|
||||||
|
uid: NodeUid::new(VolumeId::from("v"), LinkId::from("l")),
|
||||||
|
parent_uid: None,
|
||||||
|
kind: NodeKind::File {
|
||||||
|
media_type: "text/plain".into(),
|
||||||
|
total_size_on_storage: size,
|
||||||
|
active_revision_state: None,
|
||||||
|
active_revision_id: rev.map(String::from),
|
||||||
|
claimed_size: Some(size),
|
||||||
|
claimed_modification_time: None,
|
||||||
|
content_sha1: None,
|
||||||
|
},
|
||||||
|
name: "f.txt".into(),
|
||||||
|
creation_time: 0,
|
||||||
|
modification_time: mtime,
|
||||||
|
trashed: false,
|
||||||
|
is_shared: false,
|
||||||
|
is_shared_publicly: false,
|
||||||
|
signature_email: None,
|
||||||
|
verification: Default::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn baseline(mtime: i64, size: u64, rev: Option<&str>) -> Baseline {
|
||||||
|
Baseline {
|
||||||
|
mtime,
|
||||||
|
size,
|
||||||
|
hash: None,
|
||||||
|
revision_id: rev.map(String::from),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn same_revision_id_is_not_a_conflict_despite_mtime_drift() {
|
||||||
|
// The exact shape of the reported bug: same revision, server re-stamped
|
||||||
|
// its mtime (1784897777 -> 1784897780), same size. Must NOT conflict.
|
||||||
|
let base = baseline(1_784_897_777, 87_438, Some("rev-A"));
|
||||||
|
let remote = file_node(1_784_897_780, 87_438, Some("rev-A"));
|
||||||
|
assert_eq!(revision_changed(&base, &remote), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_new_revision_id_is_a_conflict_even_at_the_same_size() {
|
||||||
|
// A same-size edit by another device advances the revision id: this is a
|
||||||
|
// real conflict the id catches where (mtime, size) alone could miss it.
|
||||||
|
let base = baseline(100, 40, Some("rev-A"));
|
||||||
|
let remote = file_node(100, 40, Some("rev-B"));
|
||||||
|
assert!(revision_changed(&base, &remote).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn without_ids_it_falls_back_to_mtime_and_size() {
|
||||||
|
// Old sidecar (no revision id): the (mtime, size) tuple still governs.
|
||||||
|
let base = baseline(100, 40, None);
|
||||||
|
assert_eq!(revision_changed(&base, &file_node(100, 40, None)), None);
|
||||||
|
assert!(revision_changed(&base, &file_node(101, 40, None)).is_some());
|
||||||
|
assert!(revision_changed(&base, &file_node(100, 41, None)).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_complete_write_over_our_own_sealed_revision_chains_not_forks() {
|
||||||
|
// B70 layer B: the partial we sealed ourselves (rev-mine) is what the
|
||||||
|
// remote holds; the resume rewrote the whole file. Not a foreign change.
|
||||||
|
assert!(is_own_self_supersede(
|
||||||
|
true,
|
||||||
|
Some("rev-mine"),
|
||||||
|
Some("rev-mine")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_move_to_a_revision_we_did_not_seal_still_forks() {
|
||||||
|
// Another device wrote rev-theirs; we only ever sealed rev-mine. Fork.
|
||||||
|
assert!(!is_own_self_supersede(
|
||||||
|
true,
|
||||||
|
Some("rev-theirs"),
|
||||||
|
Some("rev-mine")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_incomplete_blob_never_self_supersedes() {
|
||||||
|
// Its gaps still refer to the stale base; overwriting would mix
|
||||||
|
// revisions, so it must take the conflict-copy path even over our seal.
|
||||||
|
assert!(!is_own_self_supersede(
|
||||||
|
false,
|
||||||
|
Some("rev-mine"),
|
||||||
|
Some("rev-mine")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_missing_remote_id_never_self_supersedes() {
|
||||||
|
// No id to prove the revision was ours: fall through to a fork rather
|
||||||
|
// than matching None==None.
|
||||||
|
assert!(!is_own_self_supersede(true, None, None));
|
||||||
|
}
|
||||||
|
}
|
||||||
+800
-469
File diff suppressed because it is too large.
Load diff
+676
-32
@@ -36,6 +36,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
|
|||||||
use std::ffi::OsStr;
|
use std::ffi::OsStr;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
|
use std::os::unix::ffi::OsStrExt;
|
||||||
use std::os::unix::fs::FileExt;
|
use std::os::unix::fs::FileExt;
|
||||||
use std::os::unix::net::UnixListener;
|
use std::os::unix::net::UnixListener;
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Component, Path, PathBuf};
|
||||||
@@ -46,24 +47,26 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|||||||
|
|
||||||
use fuser::ReplyXattr;
|
use fuser::ReplyXattr;
|
||||||
use fuser::{
|
use fuser::{
|
||||||
BackgroundSession, BsdFileFlags, Config, Errno, FileAttr, FileHandle, FileType, Filesystem,
|
BackgroundSession, BsdFileFlags, Config, CopyFileRangeFlags, Errno, FileAttr, FileHandle,
|
||||||
FopenFlags, Generation, INodeNo, IoctlFlags, LockOwner, MountOption, Notifier, OpenAccMode,
|
FileType, Filesystem, FopenFlags, Generation, INodeNo, IoctlFlags, LockOwner, MountOption,
|
||||||
OpenFlags, RenameFlags, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty,
|
Notifier, OpenAccMode, OpenFlags, RenameFlags, ReplyAttr, ReplyCreate, ReplyData,
|
||||||
ReplyEntry, ReplyIoctl, ReplyOpen, ReplyWrite, Request, Session, TimeOrNow, WriteFlags,
|
ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyIoctl, ReplyLseek, ReplyOpen, ReplyWrite, Request,
|
||||||
|
Session, TimeOrNow, WriteFlags,
|
||||||
};
|
};
|
||||||
use pdfs_core::cache::{BLOCK_SIZE, Baseline, ContentCache, StagedWrite};
|
use pdfs_core::cache::{BLOCK_SIZE, Baseline, ContentCache, StagedWrite};
|
||||||
use pdfs_core::config::AppDirs;
|
use pdfs_core::config::{AppDirs, SweepMode};
|
||||||
use pdfs_core::control::{
|
use pdfs_core::control::{
|
||||||
ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo,
|
ActivityEntry, ActivityKind, DirEntry, ErrorKind, LocalHit, PhotoKind, PublicLinkInfo,
|
||||||
SearchFilters, SearchHit, SearchSource, SyncFolderInfo, SyncPhase, SyncProgress,
|
SearchFilters, SearchHit, SearchSource, SyncFolderInfo, SyncPhase, SyncProgress,
|
||||||
TransferDirection,
|
TransferDirection,
|
||||||
};
|
};
|
||||||
use pdfs_core::db::{
|
use pdfs_core::db::{
|
||||||
Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PendingOp, StoredNode,
|
Db, LOCAL_VOLUME, OP_CREATE, OP_MKDIR, OP_RENAME, OP_REVISION, OP_TRASH, PARK_UNTIL, PendingOp,
|
||||||
StoredSyncFolder, StoredTrash,
|
StoredNode, StoredSyncFolder, StoredTrash,
|
||||||
};
|
};
|
||||||
use pdfs_core::localindex;
|
use pdfs_core::localindex;
|
||||||
use pdfs_core::search::relevance_score;
|
use pdfs_core::search::relevance_score;
|
||||||
|
use pdfs_core::syncignore::is_transient_name;
|
||||||
use pdfs_core::{CoreError, CoreResult};
|
use pdfs_core::{CoreError, CoreResult};
|
||||||
use proton_drive_rs::proton_sdk::api::ResponseCode;
|
use proton_drive_rs::proton_sdk::api::ResponseCode;
|
||||||
use proton_drive_rs::proton_sdk::error::ProtonError;
|
use proton_drive_rs::proton_sdk::error::ProtonError;
|
||||||
@@ -86,13 +89,14 @@ mod profile;
|
|||||||
mod reads;
|
mod reads;
|
||||||
mod sharing;
|
mod sharing;
|
||||||
mod state;
|
mod state;
|
||||||
|
mod sweep;
|
||||||
mod sync;
|
mod sync;
|
||||||
mod transfers;
|
mod transfers;
|
||||||
mod upload;
|
mod upload;
|
||||||
mod workers;
|
mod workers;
|
||||||
use background::{run_event_sync, run_local_index};
|
use background::{run_event_sync, run_local_index};
|
||||||
pub(crate) use mount::is_stale_mount;
|
pub(crate) use mount::is_stale_mount;
|
||||||
pub use mount::{MountOutcome, mount};
|
pub use mount::{MountOptions, MountOutcome, mount};
|
||||||
use mount::{SecondaryMount, clear_stale_mount, fuse_connection_id, umount_session_unblocked};
|
use mount::{SecondaryMount, clear_stale_mount, fuse_connection_id, umount_session_unblocked};
|
||||||
use reads::{ReaderSlot, STREAM_BYPASS_MIN, StreamRing};
|
use reads::{ReaderSlot, STREAM_BYPASS_MIN, StreamRing};
|
||||||
use state::{Entry, Intervals, PendingRevision, State, WriteHandle};
|
use state::{Entry, Intervals, PendingRevision, State, WriteHandle};
|
||||||
@@ -225,6 +229,9 @@ struct Core {
|
|||||||
/// staged blob *is* the file's content and the remote still holds the old
|
/// staged blob *is* the file's content and the remote still holds the old
|
||||||
/// revision; and the drain worker, which performs the uploads.
|
/// revision; and the drain worker, which performs the uploads.
|
||||||
pending: Arc<Mutex<HashMap<NodeUid, PendingRevision>>>,
|
pending: Arc<Mutex<HashMap<NodeUid, PendingRevision>>>,
|
||||||
|
/// Nodes removed through this daemon which a briefly stale remote listing
|
||||||
|
/// may still return. Uids are not reused; an explicit restore clears one.
|
||||||
|
hidden: Arc<Mutex<HashSet<NodeUid>>>,
|
||||||
/// Nudges the drain worker: set true and notify to have it re-examine the
|
/// Nudges the drain worker: set true and notify to have it re-examine the
|
||||||
/// queue instead of waiting out its backoff.
|
/// queue instead of waiting out its backoff.
|
||||||
drain_wake: Arc<(Mutex<bool>, Condvar)>,
|
drain_wake: Arc<(Mutex<bool>, Condvar)>,
|
||||||
@@ -233,6 +240,27 @@ struct Core {
|
|||||||
/// off one refresh rather than one per request.
|
/// off one refresh rather than one per request.
|
||||||
timeline_refreshing: Arc<AtomicBool>,
|
timeline_refreshing: Arc<AtomicBool>,
|
||||||
trash_refreshing: Arc<AtomicBool>,
|
trash_refreshing: Arc<AtomicBool>,
|
||||||
|
/// Conflict copies the sweep has already flagged as needing attention this
|
||||||
|
/// run, so a divergent `(sync-conflict …)` file is logged once rather than
|
||||||
|
/// on every sweep pass. See [`Core::run_conflict_sweep_loop`].
|
||||||
|
conflict_notified: Arc<Mutex<HashSet<NodeUid>>>,
|
||||||
|
/// Whether the conflict sweep may actually trash the duplicates it finds, or
|
||||||
|
/// only report them. Resolved once at mount from config + environment
|
||||||
|
/// ([`AppConfig::resolved_conflict_sweep`]); [`SweepMode::Off`] means the
|
||||||
|
/// sweep thread is never spawned at all. See `docs/BUGS.md` B71.
|
||||||
|
sweep_mode: SweepMode,
|
||||||
|
/// The latest server revision id this daemon has itself sealed, keyed by
|
||||||
|
/// node. A queued write whose baseline names an *earlier* revision would
|
||||||
|
/// normally fork into a `(sync-conflict)` copy when the drain finds the
|
||||||
|
/// remote already moved on — but if the remote sits at a revision *we*
|
||||||
|
/// sealed, no other device touched the file: it is a single-writer
|
||||||
|
/// stall→resume (a browser download that closed and reopened its fd, then
|
||||||
|
/// rewrote the whole file). [`Core::revision_conflict`] consults this to
|
||||||
|
/// adopt our own revision as the base and supersede instead of forking
|
||||||
|
/// (docs/BUGS.md B70, layer B). One entry per written node, overwritten as
|
||||||
|
/// the node advances and dropped when it is trashed; lost on restart, which
|
||||||
|
/// only widens the (already narrow) fork window across a daemon bounce.
|
||||||
|
own_sealed_revs: Arc<Mutex<HashMap<NodeUid, String>>>,
|
||||||
/// Photos whose missing thumbnail is being generated right now. A tile that is
|
/// Photos whose missing thumbnail is being generated right now. A tile that is
|
||||||
/// still on screen asks for its thumbnail again every few seconds, and each of
|
/// still on screen asks for its thumbnail again every few seconds, and each of
|
||||||
/// those downloads is a full-size photo — so an in-flight uid is never started
|
/// those downloads is a full-size photo — so an in-flight uid is never started
|
||||||
@@ -292,6 +320,147 @@ struct Core {
|
|||||||
/// uploading that same tree — the engine would upload files as they vanish
|
/// uploading that same tree — the engine would upload files as they vanish
|
||||||
/// and then walk the FUSE mount as if it were local.
|
/// and then walk the FUSE mount as if it were local.
|
||||||
sync_locks: Arc<Mutex<HashMap<i64, Arc<Mutex<()>>>>>,
|
sync_locks: Arc<Mutex<HashMap<i64, Arc<Mutex<()>>>>>,
|
||||||
|
/// Nodes this daemon changed on the remote itself, and how many echoes of
|
||||||
|
/// those changes the event feed still owes us.
|
||||||
|
///
|
||||||
|
/// The feed replays our own writes back at us seconds later, and a
|
||||||
|
/// `NodeUpdated` carries no revision id — nothing in it distinguishes "another
|
||||||
|
/// device changed this file" from "you changed this file". Treating the echo
|
||||||
|
/// as foreign is not free: it evicts the content blob we just uploaded, so the
|
||||||
|
/// next read re-downloads a revision the API may not be serving yet. That is
|
||||||
|
/// how a SQLite database written on the mount came back `malformed`, and how a
|
||||||
|
/// mount doing bulk uploads spent its time re-fetching its own bytes.
|
||||||
|
///
|
||||||
|
/// So the first event for a node we changed ourselves, within
|
||||||
|
/// [`SELF_CHANGE_TTL_MS`], is consumed rather than applied: the tree already
|
||||||
|
/// holds what that event describes, because [`Core::refresh_after_upload`] put
|
||||||
|
/// it there. Anything beyond that applies normally, which bounds the staleness
|
||||||
|
/// a genuine remote change can suffer to one event.
|
||||||
|
///
|
||||||
|
/// Shared across forks like `pending`: the drain that records the change and
|
||||||
|
/// the event task that consumes the echo are both per-daemon.
|
||||||
|
self_changes: Arc<Mutex<HashMap<NodeUid, SelfChange>>>,
|
||||||
|
/// Every live inode space: this mount's own `state` plus one per on-demand
|
||||||
|
/// fork ([`Core::fork_state`]). Shared by every fork, unlike `state` itself.
|
||||||
|
///
|
||||||
|
/// There is exactly **one** drain thread per daemon, owned by the primary
|
||||||
|
/// mount, and it serves the whole `pending_op` table — including ops queued
|
||||||
|
/// by a forked mount, which keeps its nodes in its own `state`. So a drain
|
||||||
|
/// that reaches for `self.state` looks in the wrong inode space and silently
|
||||||
|
/// finds nothing: the fork's entry keeps its `local~` placeholder uid
|
||||||
|
/// forever, and every read of it short-circuits to empty because
|
||||||
|
/// [`Core::read_range`] refuses to ask the API about a `local~` uid. That
|
||||||
|
/// was `docs/BUGS.md` B74. Background work that rewrites node identity must
|
||||||
|
/// walk this instead ([`Core::for_each_state`]).
|
||||||
|
///
|
||||||
|
/// `Weak`, so an unmounted fork's state is dropped rather than pinned here.
|
||||||
|
states: Arc<StateRegistry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long a change this daemon made stays recognisable as its own. Generous
|
||||||
|
/// against a feed that can lag, but finite: past it, an event for the node is
|
||||||
|
/// applied like any other. See [`Core::self_changes`].
|
||||||
|
const SELF_CHANGE_TTL_MS: i64 = 120_000;
|
||||||
|
|
||||||
|
/// A remote change this daemon made, awaiting its echo from the event feed.
|
||||||
|
struct SelfChange {
|
||||||
|
/// When we made it, in the same epoch millis as [`now_millis`].
|
||||||
|
at_ms: i64,
|
||||||
|
/// Echoes not yet seen. More than one because a single file can be created
|
||||||
|
/// and then have a revision uploaded, and the feed reports both.
|
||||||
|
echoes: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record one remote change this daemon made. Split from [`Core`] so the
|
||||||
|
/// expiry and echo-counting rules can be tested against a clock the test owns.
|
||||||
|
fn note_self_change(changes: &mut HashMap<NodeUid, SelfChange>, uid: &NodeUid, now_ms: i64) {
|
||||||
|
// Echoes that never arrived — a node trashed before its event came round, a
|
||||||
|
// feed that skipped it — would otherwise accumulate for the life of the
|
||||||
|
// daemon. Pruning on write keeps the map the size of the recent queue.
|
||||||
|
changes.retain(|_, c| now_ms - c.at_ms < SELF_CHANGE_TTL_MS);
|
||||||
|
let change = changes.entry(uid.clone()).or_insert(SelfChange {
|
||||||
|
at_ms: now_ms,
|
||||||
|
echoes: 0,
|
||||||
|
});
|
||||||
|
change.at_ms = now_ms;
|
||||||
|
change.echoes += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim one echo, per [`Core::take_self_change`].
|
||||||
|
fn take_self_change(
|
||||||
|
changes: &mut HashMap<NodeUid, SelfChange>,
|
||||||
|
uid: &NodeUid,
|
||||||
|
now_ms: i64,
|
||||||
|
) -> bool {
|
||||||
|
let Some(change) = changes.get_mut(uid) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
// Too old to attribute: the feed is far enough behind that this is as likely
|
||||||
|
// to be someone else's change as ours, and guessing wrong here costs the user
|
||||||
|
// a stale file.
|
||||||
|
if now_ms - change.at_ms >= SELF_CHANGE_TTL_MS {
|
||||||
|
changes.remove(uid);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
change.echoes -= 1;
|
||||||
|
if change.echoes == 0 {
|
||||||
|
changes.remove(uid);
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One live inode space, as published to the shared registry.
|
||||||
|
///
|
||||||
|
/// The notifier travels with the state because the two are only meaningful
|
||||||
|
/// together: an inode number names a different node in every mount, so telling
|
||||||
|
/// the kernel to drop inode 42 is only correct on the session that minted it.
|
||||||
|
/// Background work that invalidates as well as mutates needs the matching pair,
|
||||||
|
/// which is what [`Core::for_each_mount`] hands it.
|
||||||
|
type LiveMount = (Arc<Mutex<State>>, Arc<OnceLock<Notifier>>);
|
||||||
|
|
||||||
|
struct MountedState {
|
||||||
|
/// `Weak`, so an unmounted fork's state is dropped rather than pinned here;
|
||||||
|
/// a dead entry is reaped on the next registry walk.
|
||||||
|
state: std::sync::Weak<Mutex<State>>,
|
||||||
|
/// This mount's kernel notification channel — the same cell as
|
||||||
|
/// [`Core::notifier`], still empty until its session is spawned.
|
||||||
|
notifier: Arc<OnceLock<Notifier>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every mounted inode space in the daemon, shared by the primary `Core` and
|
||||||
|
/// every [`Core::fork_state`] clone of it.
|
||||||
|
///
|
||||||
|
/// Split out from `Core` so the reap-and-walk rule can be tested on its own: a
|
||||||
|
/// fork that fails to appear here, or one that lingers after unmount, is exactly
|
||||||
|
/// the failure `docs/BUGS.md` B74 was.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct StateRegistry(Mutex<Vec<MountedState>>);
|
||||||
|
|
||||||
|
impl StateRegistry {
|
||||||
|
/// Publish an inode space and the channel to the session that owns it.
|
||||||
|
///
|
||||||
|
/// Registering the same `state` twice would make every walk visit it twice;
|
||||||
|
/// callers register exactly once, at mount time.
|
||||||
|
fn register(&self, state: &Arc<Mutex<State>>, notifier: Arc<OnceLock<Notifier>>) {
|
||||||
|
let mut states = self.0.lock();
|
||||||
|
states.retain(|m| m.state.strong_count() > 0);
|
||||||
|
states.push(MountedState {
|
||||||
|
state: Arc::downgrade(state),
|
||||||
|
notifier,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every live mount, reaping the entries whose session has gone. Returns
|
||||||
|
/// owned handles so callers take the per-state locks one at a time rather
|
||||||
|
/// than holding the registry lock across the work.
|
||||||
|
fn live(&self) -> Vec<LiveMount> {
|
||||||
|
let mut states = self.0.lock();
|
||||||
|
states.retain(|m| m.state.strong_count() > 0);
|
||||||
|
states
|
||||||
|
.iter()
|
||||||
|
.filter_map(|m| Some((m.state.upgrade()?, m.notifier.clone())))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Why [`Core::apply_sync_folder_mode`] did not switch a folder. The two cases are
|
/// Why [`Core::apply_sync_folder_mode`] did not switch a folder. The two cases are
|
||||||
@@ -306,6 +475,54 @@ enum SwitchBlocked {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Core {
|
impl Core {
|
||||||
|
/// Register an inode space so background work can reach it. Called once for
|
||||||
|
/// the primary mount and once per on-demand fork.
|
||||||
|
fn register_state(&self) {
|
||||||
|
self.states.register(&self.state, self.notifier.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run `apply` against every live inode space — this daemon's primary mount
|
||||||
|
/// and each on-demand fork — one lock at a time.
|
||||||
|
///
|
||||||
|
/// The drain and the sync engine are per-daemon but node state is per-mount,
|
||||||
|
/// so anything they change about a node has to be offered to whichever mount
|
||||||
|
/// actually holds it. Which one that is cannot be known from the op: a
|
||||||
|
/// `pending_op` row records a uid, not the session that queued it. Applying
|
||||||
|
/// to all of them is correct because a uid is unique across mounts — at most
|
||||||
|
/// one state has an entry for it, and the rest are no-ops.
|
||||||
|
fn for_each_state(&self, mut apply: impl FnMut(&mut State)) {
|
||||||
|
for (state, _) in self.states.live() {
|
||||||
|
apply(&mut state.lock());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`Core::for_each_state`], but also hands over the mount's kernel
|
||||||
|
/// notification channel, so work that invalidates cached metadata reaches
|
||||||
|
/// the session that actually minted the inodes it is naming.
|
||||||
|
///
|
||||||
|
/// `None` for a mount whose session has not been spawned yet — there is
|
||||||
|
/// nothing to invalidate in a kernel that has never been told about it.
|
||||||
|
fn for_each_mount(&self, mut apply: impl FnMut(&mut State, Option<&Notifier>)) {
|
||||||
|
for (state, notifier) in self.states.live() {
|
||||||
|
apply(&mut state.lock(), notifier.get());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record that this daemon just changed `uid` on the remote, so the event
|
||||||
|
/// feed's echo of that change is recognised instead of re-applied. Called
|
||||||
|
/// from the drain once the remote has actually accepted the change and the
|
||||||
|
/// tree has been brought level with it. See [`Core::self_changes`].
|
||||||
|
pub(crate) fn note_self_change(&self, uid: &NodeUid) {
|
||||||
|
note_self_change(&mut self.self_changes.lock(), uid, now_millis());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim one outstanding echo for `uid`, reporting whether this event is one
|
||||||
|
/// this daemon caused. Consuming rather than merely testing is what bounds
|
||||||
|
/// the suppression: the next event for the node applies normally.
|
||||||
|
fn take_self_change(&self, uid: &NodeUid) -> bool {
|
||||||
|
take_self_change(&mut self.self_changes.lock(), uid, now_millis())
|
||||||
|
}
|
||||||
|
|
||||||
/// Rehydrate the in-memory `State` maps from the DB on mount, so a cold
|
/// Rehydrate the in-memory `State` maps from the DB on mount, so a cold
|
||||||
/// start serves previously-seen metadata (stable inodes, instant listings)
|
/// start serves previously-seen metadata (stable inodes, instant listings)
|
||||||
/// without re-hitting the API. The root inode is already installed by
|
/// without re-hitting the API. The root inode is already installed by
|
||||||
@@ -654,6 +871,7 @@ impl Core {
|
|||||||
// Before the lock: a DB row carries the size the server last
|
// Before the lock: a DB row carries the size the server last
|
||||||
// sealed, which a queued write is ahead of (B11).
|
// sealed, which a queued write is ahead of (B11).
|
||||||
self.stamp_pending_sizes(&mut nodes);
|
self.stamp_pending_sizes(&mut nodes);
|
||||||
|
let hidden = self.hidden.lock().clone();
|
||||||
let mut st = self.state.lock();
|
let mut st = self.state.lock();
|
||||||
if st.children.contains_key(&ino) {
|
if st.children.contains_key(&ino) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -661,7 +879,7 @@ impl Core {
|
|||||||
let mut child_inos = Vec::with_capacity(nodes.len());
|
let mut child_inos = Vec::with_capacity(nodes.len());
|
||||||
let mut needs_size = Vec::new();
|
let mut needs_size = Vec::new();
|
||||||
for node in nodes {
|
for node in nodes {
|
||||||
if node.trashed || node.uid == folder_uid {
|
if !node_visible(&node, &folder_uid, &hidden) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if matches!(
|
if matches!(
|
||||||
@@ -712,6 +930,7 @@ impl Core {
|
|||||||
// Same as the DB path above: the remote's size for a file with a write
|
// Same as the DB path above: the remote's size for a file with a write
|
||||||
// still queued is the pre-write one (B11).
|
// still queued is the pre-write one (B11).
|
||||||
self.stamp_pending_sizes(&mut nodes);
|
self.stamp_pending_sizes(&mut nodes);
|
||||||
|
let hidden = self.hidden.lock().clone();
|
||||||
|
|
||||||
let mut st = self.state.lock();
|
let mut st = self.state.lock();
|
||||||
// Lost the race? Another thread already populated it.
|
// Lost the race? Another thread already populated it.
|
||||||
@@ -721,7 +940,7 @@ impl Core {
|
|||||||
let mut child_inos = Vec::with_capacity(nodes.len());
|
let mut child_inos = Vec::with_capacity(nodes.len());
|
||||||
let filtered_nodes: Vec<Node> = nodes
|
let filtered_nodes: Vec<Node> = nodes
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|node| !node.trashed && node.uid != folder_uid)
|
.filter(|node| node_visible(node, &folder_uid, &hidden))
|
||||||
.collect();
|
.collect();
|
||||||
// Files whose real size the cheap enumeration could not read. Collected
|
// Files whose real size the cheap enumeration could not read. Collected
|
||||||
// before interning so the upgrade below has the uids without re-walking.
|
// before interning so the upgrade below has the uids without re-walking.
|
||||||
@@ -1140,7 +1359,12 @@ impl Core {
|
|||||||
base_mtime: h.base_mtime,
|
base_mtime: h.base_mtime,
|
||||||
complete: authored == [(0, h.len)],
|
complete: authored == [(0, h.len)],
|
||||||
authored,
|
authored,
|
||||||
based_on: self.remote_baseline(&h.uid, h.base_mtime, h.base_size),
|
based_on: self.remote_baseline(
|
||||||
|
&h.uid,
|
||||||
|
h.base_mtime,
|
||||||
|
h.base_size,
|
||||||
|
h.base_revision_id.clone(),
|
||||||
|
),
|
||||||
};
|
};
|
||||||
self.enqueue_staged_write(&h.uid, h.ino, &h.path, meta)?;
|
self.enqueue_staged_write(&h.uid, h.ino, &h.path, meta)?;
|
||||||
debug!(uid = %h.uid, len = h.len, complete = filled, "queued revision upload");
|
debug!(uid = %h.uid, len = h.len, complete = filled, "queued revision upload");
|
||||||
@@ -1161,7 +1385,13 @@ impl Core {
|
|||||||
///
|
///
|
||||||
/// `None` for a node that has never existed remotely: there is no revision
|
/// `None` for a node that has never existed remotely: there is no revision
|
||||||
/// to conflict with until its create drains.
|
/// to conflict with until its create drains.
|
||||||
fn remote_baseline(&self, uid: &NodeUid, base_mtime: i64, base_size: u64) -> Option<Baseline> {
|
fn remote_baseline(
|
||||||
|
&self,
|
||||||
|
uid: &NodeUid,
|
||||||
|
base_mtime: i64,
|
||||||
|
base_size: u64,
|
||||||
|
base_revision_id: Option<String>,
|
||||||
|
) -> Option<Baseline> {
|
||||||
if is_local_uid(uid) {
|
if is_local_uid(uid) {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -1171,6 +1401,7 @@ impl Core {
|
|||||||
mtime: base_mtime,
|
mtime: base_mtime,
|
||||||
size: base_size,
|
size: base_size,
|
||||||
hash: None,
|
hash: None,
|
||||||
|
revision_id: base_revision_id,
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1287,12 +1518,15 @@ impl Core {
|
|||||||
/// - Shrinking authors nothing: every remaining byte still comes from the
|
/// - Shrinking authors nothing: every remaining byte still comes from the
|
||||||
/// base, so it is the drain that has to fetch it.
|
/// base, so it is the drain that has to fetch it.
|
||||||
fn queue_truncate(&self, ino: u64, size: u64) -> Result<(), Errno> {
|
fn queue_truncate(&self, ino: u64, size: u64) -> Result<(), Errno> {
|
||||||
let (uid, base_mtime, base_size) = {
|
let (uid, base_mtime, base_size, base_revision_id) = {
|
||||||
let st = self.state.lock();
|
let st = self.state.lock();
|
||||||
match st.entries.get(&ino) {
|
match st.entries.get(&ino) {
|
||||||
Some(e) if e.node.is_file() => {
|
Some(e) if e.node.is_file() => (
|
||||||
(e.uid.clone(), e.node.modification_time, node_size(&e.node))
|
e.uid.clone(),
|
||||||
}
|
e.node.modification_time,
|
||||||
|
node_size(&e.node),
|
||||||
|
node_revision_id(&e.node),
|
||||||
|
),
|
||||||
Some(_) => return Err(Errno::EISDIR),
|
Some(_) => return Err(Errno::EISDIR),
|
||||||
None => return Err(Errno::ENOENT),
|
None => return Err(Errno::ENOENT),
|
||||||
}
|
}
|
||||||
@@ -1300,6 +1534,21 @@ impl Core {
|
|||||||
if size == base_size {
|
if size == base_size {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
// A close immediately followed by truncate is common (`write`, close,
|
||||||
|
// `truncate(2)`) and usually beats the write-back debounce. In that
|
||||||
|
// case the remote is *not* our base: the complete staged revision is.
|
||||||
|
// Compose from those local bytes so the truncate can supersede the
|
||||||
|
// queued upload without fetching stale remote content or returning EIO.
|
||||||
|
let pending_base = self.pending.lock().get(&uid).cloned();
|
||||||
|
if pending_base
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|pending| !pending.meta.complete)
|
||||||
|
{
|
||||||
|
// An incomplete pending blob still has holes referring to the
|
||||||
|
// remote revision. Stacking another incomplete transform on it is
|
||||||
|
// not representable safely; preserve the existing refusal.
|
||||||
|
return Err(Errno::EIO);
|
||||||
|
}
|
||||||
let (authored, complete) = if size == 0 {
|
let (authored, complete) = if size == 0 {
|
||||||
// An empty file has no content to be missing.
|
// An empty file has no content to be missing.
|
||||||
(Vec::new(), true)
|
(Vec::new(), true)
|
||||||
@@ -1316,19 +1565,45 @@ impl Core {
|
|||||||
error!(%uid, error = %e, "create scratch file for truncate failed");
|
error!(%uid, error = %e, "create scratch file for truncate failed");
|
||||||
Errno::EIO
|
Errno::EIO
|
||||||
})?;
|
})?;
|
||||||
|
if let Some(pending) = &pending_base
|
||||||
|
&& let Err(e) = copy_pending_for_truncate(pending, &file)
|
||||||
|
{
|
||||||
|
error!(%uid, source = %pending.path.display(), error = %e,
|
||||||
|
"copy pending revision for truncate failed");
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
return Err(Errno::EIO);
|
||||||
|
}
|
||||||
file.set_len(size).map_err(|e| {
|
file.set_len(size).map_err(|e| {
|
||||||
error!(%uid, error = %e, "resize scratch file for truncate failed");
|
error!(%uid, error = %e, "resize scratch file for truncate failed");
|
||||||
let _ = std::fs::remove_file(&path);
|
let _ = std::fs::remove_file(&path);
|
||||||
Errno::EIO
|
Errno::EIO
|
||||||
})?;
|
})?;
|
||||||
let meta = StagedWrite {
|
let meta = match pending_base {
|
||||||
|
Some(pending) => StagedWrite {
|
||||||
|
uid: uid.to_string(),
|
||||||
|
len: size,
|
||||||
|
base_size: pending.meta.len,
|
||||||
|
base_mtime: pending.meta.base_mtime,
|
||||||
|
authored: if size == 0 {
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
vec![(0, size)]
|
||||||
|
},
|
||||||
|
complete: true,
|
||||||
|
// Both local revisions descend from the same last-observed
|
||||||
|
// remote revision. Never replace this with the optimistic
|
||||||
|
// mtime/size currently published by State.
|
||||||
|
based_on: pending.meta.based_on,
|
||||||
|
},
|
||||||
|
None => StagedWrite {
|
||||||
uid: uid.to_string(),
|
uid: uid.to_string(),
|
||||||
len: size,
|
len: size,
|
||||||
base_size,
|
base_size,
|
||||||
base_mtime,
|
base_mtime,
|
||||||
authored,
|
authored,
|
||||||
complete,
|
complete,
|
||||||
based_on: self.remote_baseline(&uid, base_mtime, base_size),
|
based_on: self.remote_baseline(&uid, base_mtime, base_size, base_revision_id),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
self.enqueue_staged_write(&uid, ino, &path, meta)?;
|
self.enqueue_staged_write(&uid, ino, &path, meta)?;
|
||||||
debug!(%uid, size, complete, "queued truncate");
|
debug!(%uid, size, complete, "queued truncate");
|
||||||
@@ -1347,6 +1622,7 @@ impl Core {
|
|||||||
parent_uid: &NodeUid,
|
parent_uid: &NodeUid,
|
||||||
name: &str,
|
name: &str,
|
||||||
is_dir: bool,
|
is_dir: bool,
|
||||||
|
hold: bool,
|
||||||
) -> Result<Node, Errno> {
|
) -> Result<Node, Errno> {
|
||||||
let uid = mint_local_uid();
|
let uid = mint_local_uid();
|
||||||
let op = PendingOp {
|
let op = PendingOp {
|
||||||
@@ -1360,7 +1636,10 @@ impl Core {
|
|||||||
created_at: now_millis(),
|
created_at: now_millis(),
|
||||||
attempts: 0,
|
attempts: 0,
|
||||||
last_error: None,
|
last_error: None,
|
||||||
next_attempt_at: 0,
|
// Parked when the name is transient: its bytes ride on this create
|
||||||
|
// (via `attach_blob_to_create`) and must not upload until a rename to
|
||||||
|
// the finished name un-parks it (docs/BUGS.md B70).
|
||||||
|
next_attempt_at: if hold { PARK_UNTIL } else { 0 },
|
||||||
};
|
};
|
||||||
self.db.enqueue_op(&op).map_err(|e| {
|
self.db.enqueue_op(&op).map_err(|e| {
|
||||||
error!(%parent_uid, name, error = %e, "queueing local node failed");
|
error!(%parent_uid, name, error = %e, "queueing local node failed");
|
||||||
@@ -1447,7 +1726,15 @@ impl Core {
|
|||||||
error!(%uid, error = %e, "queueing trash failed");
|
error!(%uid, error = %e, "queueing trash failed");
|
||||||
Errno::EIO
|
Errno::EIO
|
||||||
})?;
|
})?;
|
||||||
self.state.lock().forget(uid);
|
self.hidden.lock().insert(uid.clone());
|
||||||
|
// Withdrawn from every mount, not just the one the unlink came through:
|
||||||
|
// a sync folder maps a remote folder that also exists under My Files, so
|
||||||
|
// the same uid can be interned in two inode spaces at once and the other
|
||||||
|
// one would go on serving a file the user just trashed
|
||||||
|
// (`docs/BUGS.md` B74).
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
st.forget(uid);
|
||||||
|
});
|
||||||
self.cache.evict(uid);
|
self.cache.evict(uid);
|
||||||
self.evict_reader(uid);
|
self.evict_reader(uid);
|
||||||
self.wake_drain();
|
self.wake_drain();
|
||||||
@@ -1500,8 +1787,32 @@ impl Core {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(%uid, error = %e, "staging write failed; bytes lost");
|
// `src` is the only known copy of bytes the kernel already
|
||||||
let _ = std::fs::remove_file(src);
|
// accepted. A failure to publish them into staging must never
|
||||||
|
// turn into permission to delete them: leave the scratch file
|
||||||
|
// in place so startup recovery or a human can still salvage it.
|
||||||
|
error!(
|
||||||
|
%uid,
|
||||||
|
error = %e,
|
||||||
|
scratch = %src.display(),
|
||||||
|
"staging write failed; retaining the only scratch copy"
|
||||||
|
);
|
||||||
|
let name = {
|
||||||
|
let st = self.state.lock();
|
||||||
|
st.entries
|
||||||
|
.get(&ino)
|
||||||
|
.map(|entry| entry.node.name.clone())
|
||||||
|
.unwrap_or_default()
|
||||||
|
};
|
||||||
|
self.log_activity(
|
||||||
|
ActivityKind::Upload,
|
||||||
|
&name,
|
||||||
|
format!(
|
||||||
|
"write not queued and could not be staged; recovery copy retained at {}",
|
||||||
|
src.display()
|
||||||
|
),
|
||||||
|
false,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2040,7 +2351,11 @@ impl Core {
|
|||||||
self.rt
|
self.rt
|
||||||
.block_on(self.client.rename_node(&uid, new_name, None))
|
.block_on(self.client.rename_node(&uid, new_name, None))
|
||||||
.map_err(|e| CoreError::from_api(&e, "rename"))?;
|
.map_err(|e| CoreError::from_api(&e, "rename"))?;
|
||||||
self.state.lock().forget(&uid);
|
// Every mount, so a fork showing the same node re-interns it under the
|
||||||
|
// new name instead of keeping the old one (`docs/BUGS.md` B74).
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
st.forget(&uid);
|
||||||
|
});
|
||||||
self.invalidate_parent_listing(rel);
|
self.invalidate_parent_listing(rel);
|
||||||
Ok(new_name.to_string())
|
Ok(new_name.to_string())
|
||||||
}
|
}
|
||||||
@@ -2101,7 +2416,9 @@ impl Core {
|
|||||||
fn remove_replaced(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> {
|
fn remove_replaced(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> {
|
||||||
if is_local_uid(uid) {
|
if is_local_uid(uid) {
|
||||||
self.discard_queued_ops(uid);
|
self.discard_queued_ops(uid);
|
||||||
self.state.lock().forget(uid);
|
self.for_each_state(|st| {
|
||||||
|
st.forget(uid);
|
||||||
|
});
|
||||||
debug!(%uid, name, "replaced a node whose create was still queued");
|
debug!(%uid, name, "replaced a node whose create was still queued");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -2114,7 +2431,11 @@ impl Core {
|
|||||||
return Err(Errno::EIO);
|
return Err(Errno::EIO);
|
||||||
}
|
}
|
||||||
self.discard_queued_ops(uid);
|
self.discard_queued_ops(uid);
|
||||||
self.state.lock().forget(uid);
|
// The node was trashed on the server; withdraw it from every inode
|
||||||
|
// space that had it, not only the mount the rename came through.
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
st.forget(uid);
|
||||||
|
});
|
||||||
self.cache.evict(uid);
|
self.cache.evict(uid);
|
||||||
self.evict_reader(uid);
|
self.evict_reader(uid);
|
||||||
self.invalidate_trash();
|
self.invalidate_trash();
|
||||||
@@ -2294,13 +2615,20 @@ impl Core {
|
|||||||
.block_on(self.client.restore_nodes(&parsed))
|
.block_on(self.client.restore_nodes(&parsed))
|
||||||
.map_err(|e| CoreError::from_api(&e, "restore"))?;
|
.map_err(|e| CoreError::from_api(&e, "restore"))?;
|
||||||
{
|
{
|
||||||
let mut st = self.state.lock();
|
let mut hidden = self.hidden.lock();
|
||||||
for parent in parents {
|
for uid in &parsed {
|
||||||
if let Some(&ino) = st.by_uid.get(&parent) {
|
hidden.remove(uid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Keyed by uid, so it applies to every mount: a restored node reappears
|
||||||
|
// in whichever inode spaces show its parent, not only the primary one.
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
for parent in &parents {
|
||||||
|
if let Some(&ino) = st.by_uid.get(parent) {
|
||||||
st.invalidate_listing(ino);
|
st.invalidate_listing(ino);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
self.invalidate_trash();
|
self.invalidate_trash();
|
||||||
Ok(parsed.len())
|
Ok(parsed.len())
|
||||||
}
|
}
|
||||||
@@ -2336,11 +2664,15 @@ impl Core {
|
|||||||
/// Forget every trace of nodes that no longer exist anywhere: their inode and
|
/// Forget every trace of nodes that no longer exist anywhere: their inode and
|
||||||
/// DB row, and their cached content.
|
/// DB row, and their cached content.
|
||||||
fn drop_local(&self, uids: &[NodeUid]) {
|
fn drop_local(&self, uids: &[NodeUid]) {
|
||||||
let mut st = self.state.lock();
|
// Every mount: these nodes are gone from the server, and a sync-folder
|
||||||
|
// fork showing the same uid would otherwise keep serving them
|
||||||
|
// (`docs/BUGS.md` B74). A uid is unique across inode spaces, so this is
|
||||||
|
// a no-op in every mount but the one that holds it.
|
||||||
|
self.for_each_state(|st| {
|
||||||
for uid in uids {
|
for uid in uids {
|
||||||
st.forget(uid);
|
st.forget(uid);
|
||||||
}
|
}
|
||||||
drop(st);
|
});
|
||||||
for uid in uids {
|
for uid in uids {
|
||||||
self.cache.evict(uid);
|
self.cache.evict(uid);
|
||||||
self.evict_reader(uid);
|
self.evict_reader(uid);
|
||||||
@@ -2647,6 +2979,8 @@ fn local_node(uid: NodeUid, parent_uid: NodeUid, name: String, is_dir: bool) ->
|
|||||||
total_size_on_storage: 0,
|
total_size_on_storage: 0,
|
||||||
// No revision has been sealed: nothing has been uploaded yet.
|
// No revision has been sealed: nothing has been uploaded yet.
|
||||||
active_revision_state: None,
|
active_revision_state: None,
|
||||||
|
active_revision_id: None,
|
||||||
|
content_sha1: None,
|
||||||
claimed_size: Some(0),
|
claimed_size: Some(0),
|
||||||
claimed_modification_time: None,
|
claimed_modification_time: None,
|
||||||
}
|
}
|
||||||
@@ -2855,12 +3189,86 @@ fn node_size(node: &Node) -> u64 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The server revision id of a node's active revision, if it is a file that has
|
||||||
|
/// one. The stable identity the drain conflict-checks against (see [`Baseline`]).
|
||||||
|
fn node_revision_id(node: &Node) -> Option<String> {
|
||||||
|
match &node.kind {
|
||||||
|
NodeKind::Folder => None,
|
||||||
|
NodeKind::File {
|
||||||
|
active_revision_id, ..
|
||||||
|
} => active_revision_id.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The plaintext content SHA-1 of a file node, if its active revision carried one.
|
||||||
|
/// A download-free content fingerprint the conflict sweep uses to prove two files
|
||||||
|
/// hold identical bytes before it removes one.
|
||||||
|
fn node_content_sha1(node: &Node) -> Option<String> {
|
||||||
|
match &node.kind {
|
||||||
|
NodeKind::Folder => None,
|
||||||
|
NodeKind::File { content_sha1, .. } => content_sha1.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// `"1 file"` / `"3 files"` — a count with a correctly pluralised noun, for
|
/// `"1 file"` / `"3 files"` — a count with a correctly pluralised noun, for
|
||||||
/// human-readable activity-log lines.
|
/// human-readable activity-log lines.
|
||||||
fn count_noun(n: usize, one: &str, many: &str) -> String {
|
fn count_noun(n: usize, one: &str, many: &str) -> String {
|
||||||
format!("{n} {}", if n == 1 { one } else { many })
|
format!("{n} {}", if n == 1 { one } else { many })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Seed a truncate scratch file from a complete queued revision.
|
||||||
|
///
|
||||||
|
/// `File` writes use an explicit offset so this is independent of either
|
||||||
|
/// descriptor's cursor and remains correct when the destination came from the
|
||||||
|
/// cache's scratch allocator.
|
||||||
|
fn copy_pending_for_truncate(pending: &PendingRevision, destination: &File) -> std::io::Result<()> {
|
||||||
|
let source = File::open(&pending.path)?;
|
||||||
|
destination.set_len(0)?;
|
||||||
|
let mut offset = 0u64;
|
||||||
|
let mut buffer = vec![0u8; 1024 * 1024];
|
||||||
|
loop {
|
||||||
|
let count = source.read_at(&mut buffer, offset)?;
|
||||||
|
if count == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
destination.write_all_at(&buffer[..count], offset)?;
|
||||||
|
offset += count as u64;
|
||||||
|
}
|
||||||
|
destination.set_len(offset)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a rename must be represented as a durable desired end state.
|
||||||
|
/// Offline operations and operations targeting a not-yet-uploaded directory
|
||||||
|
/// cannot use the API immediately. Online remote-to-remote operations stay
|
||||||
|
/// synchronous so a successful kernel reply means the remote namespace has
|
||||||
|
/// already reached the same end state.
|
||||||
|
fn rename_needs_queue(
|
||||||
|
online: bool,
|
||||||
|
destination_is_local: bool,
|
||||||
|
_parent_changed: bool,
|
||||||
|
_name_changed: bool,
|
||||||
|
) -> bool {
|
||||||
|
!online || destination_is_local
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert one kernel pathname component into the UTF-8 name accepted by Drive.
|
||||||
|
/// Linux filesystems conventionally cap a component at `NAME_MAX` (255 bytes),
|
||||||
|
/// but the FUSE kernel path can pass a longer component through to userspace.
|
||||||
|
fn fuse_name(name: &OsStr) -> Result<String, Errno> {
|
||||||
|
let bytes = name.as_bytes();
|
||||||
|
if bytes.len() > 255 {
|
||||||
|
return Err(Errno::ENAMETOOLONG);
|
||||||
|
}
|
||||||
|
if bytes.is_empty() || bytes == b"." || bytes == b".." {
|
||||||
|
return Err(Errno::EINVAL);
|
||||||
|
}
|
||||||
|
name.to_str().map(str::to_owned).ok_or(Errno::EILSEQ)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn node_visible(node: &Node, folder_uid: &NodeUid, hidden: &HashSet<NodeUid>) -> bool {
|
||||||
|
!node.trashed && node.uid != *folder_uid && !hidden.contains(&node.uid)
|
||||||
|
}
|
||||||
|
|
||||||
/// Bytes rendered with a binary unit and one decimal place (e.g. `"1.2 GB"`),
|
/// Bytes rendered with a binary unit and one decimal place (e.g. `"1.2 GB"`),
|
||||||
/// for the activity log. Uses 1024-based steps but the shorter SI labels.
|
/// for the activity log. Uses 1024-based steps but the shorter SI labels.
|
||||||
fn human_bytes(bytes: u64) -> String {
|
fn human_bytes(bytes: u64) -> String {
|
||||||
@@ -3159,6 +3567,8 @@ mod pending_size_tests {
|
|||||||
media_type: "text/plain".into(),
|
media_type: "text/plain".into(),
|
||||||
total_size_on_storage: 0,
|
total_size_on_storage: 0,
|
||||||
active_revision_state: None,
|
active_revision_state: None,
|
||||||
|
active_revision_id: None,
|
||||||
|
content_sha1: None,
|
||||||
claimed_size: Some(claimed),
|
claimed_size: Some(claimed),
|
||||||
claimed_modification_time: None,
|
claimed_modification_time: None,
|
||||||
},
|
},
|
||||||
@@ -3224,7 +3634,120 @@ mod pending_size_tests {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{Intervals, conflict_name, is_stale_mount};
|
use super::{
|
||||||
|
HashMap, Intervals, PendingRevision, SELF_CHANGE_TTL_MS, StateRegistry, conflict_name,
|
||||||
|
copy_pending_for_truncate, fuse_name, is_stale_mount, node_visible, note_self_change,
|
||||||
|
parse_node_uid, rename_needs_queue, take_self_change,
|
||||||
|
};
|
||||||
|
use pdfs_core::cache::{Baseline, StagedWrite};
|
||||||
|
|
||||||
|
/// The registry every mount publishes itself into is what lets the daemon's
|
||||||
|
/// single drain thread reach a node living in an on-demand fork's inode
|
||||||
|
/// space. A fork missing from it reads as an empty file for the life of the
|
||||||
|
/// daemon (`docs/BUGS.md` B74), so the walk has to see *every* registered
|
||||||
|
/// mount — and only the ones still mounted.
|
||||||
|
#[test]
|
||||||
|
fn state_registry_walks_every_live_mount_and_reaps_the_rest() {
|
||||||
|
let registry = StateRegistry::default();
|
||||||
|
assert!(registry.live().is_empty(), "a fresh registry has no mounts");
|
||||||
|
|
||||||
|
// The primary mount plus two on-demand forks.
|
||||||
|
let (primary, _p) = state_test_helper();
|
||||||
|
let (fork_a, _a) = state_test_helper();
|
||||||
|
let (fork_b, _b) = state_test_helper();
|
||||||
|
let primary = std::sync::Arc::new(parking_lot::Mutex::new(primary));
|
||||||
|
let fork_a = std::sync::Arc::new(parking_lot::Mutex::new(fork_a));
|
||||||
|
let fork_b = std::sync::Arc::new(parking_lot::Mutex::new(fork_b));
|
||||||
|
for state in [&primary, &fork_a, &fork_b] {
|
||||||
|
registry.register(state, std::sync::Arc::new(std::sync::OnceLock::new()));
|
||||||
|
}
|
||||||
|
assert_eq!(registry.live().len(), 3, "every registered mount is walked");
|
||||||
|
|
||||||
|
// `live()` must not itself pin a mount alive, or an unmounted fork could
|
||||||
|
// never be reaped.
|
||||||
|
assert_eq!(
|
||||||
|
std::sync::Arc::strong_count(&fork_a),
|
||||||
|
1,
|
||||||
|
"the registry holds forks weakly"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Unmounting a fork drops its state; the dead entry goes on the next walk.
|
||||||
|
drop(fork_a);
|
||||||
|
assert_eq!(registry.live().len(), 2, "an unmounted fork is dropped");
|
||||||
|
assert_eq!(
|
||||||
|
registry.0.lock().len(),
|
||||||
|
2,
|
||||||
|
"and its slot is reaped, not merely skipped"
|
||||||
|
);
|
||||||
|
|
||||||
|
drop(primary);
|
||||||
|
drop(fork_b);
|
||||||
|
assert!(
|
||||||
|
registry.live().is_empty(),
|
||||||
|
"the last mount going away empties the registry"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An event feed carries no revision id, so the only thing that tells a
|
||||||
|
/// remote change apart from the echo of one this daemon just made is this
|
||||||
|
/// bookkeeping. Getting it wrong is expensive in both directions: too eager
|
||||||
|
/// and we evict the bytes we uploaded and re-download them from an API that
|
||||||
|
/// may still be serving the previous revision (a SQLite file on the mount
|
||||||
|
/// read back `malformed`); too generous and a real change from another device
|
||||||
|
/// is ignored.
|
||||||
|
#[test]
|
||||||
|
fn a_self_change_is_claimed_once_and_expires() {
|
||||||
|
let mut changes = HashMap::new();
|
||||||
|
let uid = parse_node_uid("vol~link").unwrap();
|
||||||
|
let other = parse_node_uid("vol~other").unwrap();
|
||||||
|
let t0 = 1_000_000;
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
!take_self_change(&mut changes, &uid, t0),
|
||||||
|
"a node we never touched is never ours"
|
||||||
|
);
|
||||||
|
|
||||||
|
note_self_change(&mut changes, &uid, t0);
|
||||||
|
assert!(
|
||||||
|
take_self_change(&mut changes, &uid, t0 + 5_000),
|
||||||
|
"the echo of our own change is claimed"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!take_self_change(&mut changes, &uid, t0 + 6_000),
|
||||||
|
"and only once, so a later foreign change still applies"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A create followed by a revision upload is two changes to one node, and
|
||||||
|
// the feed reports both.
|
||||||
|
note_self_change(&mut changes, &uid, t0);
|
||||||
|
note_self_change(&mut changes, &uid, t0);
|
||||||
|
assert!(take_self_change(&mut changes, &uid, t0 + 1));
|
||||||
|
assert!(take_self_change(&mut changes, &uid, t0 + 2));
|
||||||
|
assert!(!take_self_change(&mut changes, &uid, t0 + 3));
|
||||||
|
|
||||||
|
// Past the window the attribution is a guess, and a wrong guess leaves
|
||||||
|
// the user looking at a stale file.
|
||||||
|
note_self_change(&mut changes, &uid, t0);
|
||||||
|
assert!(!take_self_change(
|
||||||
|
&mut changes,
|
||||||
|
&uid,
|
||||||
|
t0 + SELF_CHANGE_TTL_MS
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
!changes.contains_key(&uid),
|
||||||
|
"an expired change is dropped, not left to be re-tested"
|
||||||
|
);
|
||||||
|
|
||||||
|
// An echo that never arrives must not accumulate for the life of the
|
||||||
|
// daemon; the next recorded change prunes it.
|
||||||
|
note_self_change(&mut changes, &uid, t0);
|
||||||
|
note_self_change(&mut changes, &other, t0 + SELF_CHANGE_TTL_MS);
|
||||||
|
assert_eq!(
|
||||||
|
changes.len(),
|
||||||
|
1,
|
||||||
|
"recording a change prunes the ones whose echo never came"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// The predicate must answer *only* for a dead FUSE connection. A healthy
|
/// The predicate must answer *only* for a dead FUSE connection. A healthy
|
||||||
/// directory and an absent path are both "not stale" — widening it to any
|
/// directory and an absent path are both "not stale" — widening it to any
|
||||||
@@ -3331,6 +3854,125 @@ mod tests {
|
|||||||
assert_eq!(segs(&iv, 0, 30), vec![(0, 30, true)]);
|
assert_eq!(segs(&iv, 0, 30), vec![(0, 30, true)]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn pending_revision(dir: &TestDir, bytes: &[u8], complete: bool) -> PendingRevision {
|
||||||
|
let path = dir.0.join("pending");
|
||||||
|
std::fs::write(&path, bytes).unwrap();
|
||||||
|
PendingRevision {
|
||||||
|
path,
|
||||||
|
meta: StagedWrite {
|
||||||
|
uid: "vol~file".into(),
|
||||||
|
len: bytes.len() as u64,
|
||||||
|
base_size: 3,
|
||||||
|
base_mtime: 42,
|
||||||
|
authored: if complete {
|
||||||
|
vec![(0, bytes.len() as u64)]
|
||||||
|
} else {
|
||||||
|
vec![(3, bytes.len() as u64)]
|
||||||
|
},
|
||||||
|
complete,
|
||||||
|
based_on: Some(Baseline {
|
||||||
|
mtime: 42,
|
||||||
|
size: 3,
|
||||||
|
hash: None,
|
||||||
|
revision_id: None,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complete_pending_revision_can_seed_immediate_shrink() {
|
||||||
|
let (_, dir) = state_test_helper();
|
||||||
|
let pending = pending_revision(&dir, b"0123456789", true);
|
||||||
|
let destination_path = dir.0.join("shrink");
|
||||||
|
let destination = std::fs::File::create(&destination_path).unwrap();
|
||||||
|
copy_pending_for_truncate(&pending, &destination).unwrap();
|
||||||
|
destination.set_len(4).unwrap();
|
||||||
|
assert_eq!(std::fs::read(destination_path).unwrap(), b"0123");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn complete_pending_revision_can_seed_immediate_growth_and_zero() {
|
||||||
|
let (_, dir) = state_test_helper();
|
||||||
|
let pending = pending_revision(&dir, b"0123456789", true);
|
||||||
|
let destination_path = dir.0.join("grow");
|
||||||
|
let destination = std::fs::File::create(&destination_path).unwrap();
|
||||||
|
copy_pending_for_truncate(&pending, &destination).unwrap();
|
||||||
|
destination.set_len(14).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&destination_path).unwrap(),
|
||||||
|
b"0123456789\0\0\0\0"
|
||||||
|
);
|
||||||
|
destination.set_len(0).unwrap();
|
||||||
|
assert!(std::fs::read(destination_path).unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_pending_revision_is_not_a_safe_truncate_base() {
|
||||||
|
let (_, dir) = state_test_helper();
|
||||||
|
let pending = pending_revision(&dir, b"\0\0\0authored", false);
|
||||||
|
assert!(!pending.meta.complete);
|
||||||
|
assert_ne!(pending.meta.authored, vec![(0, pending.meta.len)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn online_combined_cross_directory_rename_stays_synchronous() {
|
||||||
|
assert!(!rename_needs_queue(true, false, true, true));
|
||||||
|
assert!(rename_needs_queue(false, false, false, true));
|
||||||
|
assert!(rename_needs_queue(true, true, true, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn one_step_online_namespace_changes_stay_synchronous() {
|
||||||
|
assert!(!rename_needs_queue(true, false, false, true));
|
||||||
|
assert!(!rename_needs_queue(true, false, true, false));
|
||||||
|
assert!(!rename_needs_queue(true, false, false, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fuse_names_enforce_linux_component_limits() {
|
||||||
|
use std::ffi::OsStr;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fuse_name(OsStr::new("x".repeat(255).as_str()))
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
255
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fuse_name(OsStr::new("x".repeat(256).as_str()))
|
||||||
|
.expect_err("256-byte component must be refused")
|
||||||
|
.code(),
|
||||||
|
libc::ENAMETOOLONG
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fuse_name(OsStr::new("."))
|
||||||
|
.expect_err("dot is not an ordinary component")
|
||||||
|
.code(),
|
||||||
|
libc::EINVAL
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fuse_name(OsStr::new("unicodé-文件")).unwrap(),
|
||||||
|
"unicodé-文件"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn local_trash_tombstones_hide_stale_remote_entries() {
|
||||||
|
let folder_uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("folder"));
|
||||||
|
let node = node_helper("child", "folder", "child", false);
|
||||||
|
assert!(node_visible(
|
||||||
|
&node,
|
||||||
|
&folder_uid,
|
||||||
|
&std::collections::HashSet::new()
|
||||||
|
));
|
||||||
|
assert!(!node_visible(
|
||||||
|
&node,
|
||||||
|
&folder_uid,
|
||||||
|
&std::collections::HashSet::from([node.uid.clone()])
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId};
|
use proton_drive_rs::proton_sdk::ids::{LinkId, NodeUid, VolumeId};
|
||||||
use proton_drive_rs::{Node, NodeKind};
|
use proton_drive_rs::{Node, NodeKind};
|
||||||
|
|
||||||
@@ -3408,6 +4050,8 @@ mod tests {
|
|||||||
media_type: "text/plain".into(),
|
media_type: "text/plain".into(),
|
||||||
total_size_on_storage: 0,
|
total_size_on_storage: 0,
|
||||||
active_revision_state: None,
|
active_revision_state: None,
|
||||||
|
active_revision_id: None,
|
||||||
|
content_sha1: None,
|
||||||
claimed_size: Some(0),
|
claimed_size: Some(0),
|
||||||
claimed_modification_time: None,
|
claimed_modification_time: None,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -143,6 +143,15 @@ fn fetch_or_recall_root(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-mount settings resolved from [`pdfs_core::config::AppConfig`] by the
|
||||||
|
/// caller, so `mount` itself never reads config or the environment.
|
||||||
|
pub struct MountOptions {
|
||||||
|
/// Proton account the session belongs to, for display and per-user paths.
|
||||||
|
pub username: String,
|
||||||
|
/// What the background conflict sweep may do. See `docs/BUGS.md` B71.
|
||||||
|
pub sweep_mode: SweepMode,
|
||||||
|
}
|
||||||
|
|
||||||
/// Mount the filesystem at `mountpoint` and block until it is unmounted or the
|
/// Mount the filesystem at `mountpoint` and block until it is unmounted or the
|
||||||
/// daemon is asked to stop.
|
/// daemon is asked to stop.
|
||||||
///
|
///
|
||||||
@@ -162,8 +171,12 @@ pub fn mount(
|
|||||||
cache: ContentCache,
|
cache: ContentCache,
|
||||||
control_socket: &Path,
|
control_socket: &Path,
|
||||||
db: Arc<Db>,
|
db: Arc<Db>,
|
||||||
username: String,
|
options: MountOptions,
|
||||||
) -> std::io::Result<MountOutcome> {
|
) -> std::io::Result<MountOutcome> {
|
||||||
|
let MountOptions {
|
||||||
|
username,
|
||||||
|
sweep_mode,
|
||||||
|
} = options;
|
||||||
let (root, online) = fetch_or_recall_root(&client, &rt, &db)?;
|
let (root, online) = fetch_or_recall_root(&client, &rt, &db)?;
|
||||||
let scope = root.tree_event_scope_id();
|
let scope = root.tree_event_scope_id();
|
||||||
|
|
||||||
@@ -192,9 +205,14 @@ pub fn mount(
|
|||||||
db,
|
db,
|
||||||
online: Arc::new(AtomicBool::new(online)),
|
online: Arc::new(AtomicBool::new(online)),
|
||||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
hidden: Arc::new(Mutex::new(HashSet::new())),
|
||||||
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
||||||
timeline_refreshing: Arc::new(AtomicBool::new(false)),
|
timeline_refreshing: Arc::new(AtomicBool::new(false)),
|
||||||
trash_refreshing: Arc::new(AtomicBool::new(false)),
|
trash_refreshing: Arc::new(AtomicBool::new(false)),
|
||||||
|
conflict_notified: Arc::new(Mutex::new(HashSet::new())),
|
||||||
|
sweep_mode,
|
||||||
|
own_sealed_revs: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
self_changes: Arc::new(Mutex::new(HashMap::new())),
|
||||||
thumb_gen: Arc::new(Mutex::new(HashSet::new())),
|
thumb_gen: Arc::new(Mutex::new(HashSet::new())),
|
||||||
no_thumbnail: Arc::new(Mutex::new(HashMap::new())),
|
no_thumbnail: Arc::new(Mutex::new(HashMap::new())),
|
||||||
size_upgrades: Arc::new(Mutex::new(HashMap::new())),
|
size_upgrades: Arc::new(Mutex::new(HashMap::new())),
|
||||||
@@ -205,7 +223,11 @@ pub fn mount(
|
|||||||
sync_tx,
|
sync_tx,
|
||||||
mounts: Arc::new(Mutex::new(HashMap::new())),
|
mounts: Arc::new(Mutex::new(HashMap::new())),
|
||||||
sync_locks: Arc::new(Mutex::new(HashMap::new())),
|
sync_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
states: Arc::new(StateRegistry::default()),
|
||||||
};
|
};
|
||||||
|
// Before anything can queue work against it: the drain thread below reaches
|
||||||
|
// every mount's inode space through this registry, not through `core.state`.
|
||||||
|
core.register_state();
|
||||||
|
|
||||||
// Writes queued by a previous run (or left behind by a crash) are still owed
|
// Writes queued by a previous run (or left behind by a crash) are still owed
|
||||||
// an upload, and reads must be served from their staged blobs until they land.
|
// an upload, and reads must be served from their staged blobs until they land.
|
||||||
@@ -225,6 +247,21 @@ pub fn mount(
|
|||||||
// remotes, and reconciles on its own thread — never in front of a FUSE call.
|
// remotes, and reconciles on its own thread — never in front of a FUSE call.
|
||||||
sync::spawn(core.clone(), sync_rx);
|
sync::spawn(core.clone(), sync_rx);
|
||||||
|
|
||||||
|
// Reconcile leftover `(sync-conflict …)` copies: drop the ones that turned
|
||||||
|
// out identical to the live file, surface the ones that genuinely diverge.
|
||||||
|
// Its own thread — it enumerates and trashes, and must never block a FUSE call.
|
||||||
|
// `Off` skips the thread entirely rather than starting an idle one, so the
|
||||||
|
// setting is verifiable from the outside (`ls /proc/<pid>/task/*/comm`).
|
||||||
|
if sweep_mode == SweepMode::Off {
|
||||||
|
info!("conflict sweep disabled by configuration");
|
||||||
|
} else {
|
||||||
|
info!(mode = ?sweep_mode, "conflict sweep enabled");
|
||||||
|
let core = core.clone();
|
||||||
|
std::thread::Builder::new()
|
||||||
|
.name("pdfs-conflict-sweep".into())
|
||||||
|
.spawn(move || core.run_conflict_sweep_loop())?;
|
||||||
|
}
|
||||||
|
|
||||||
// Mounted from the cache: watch for the network coming back so the mount can
|
// Mounted from the cache: watch for the network coming back so the mount can
|
||||||
// stop being read-only-ish without the user restarting the daemon.
|
// stop being read-only-ish without the user restarting the daemon.
|
||||||
if !online {
|
if !online {
|
||||||
@@ -309,15 +346,7 @@ pub fn mount(
|
|||||||
// Same channel, kept on the `Core` so background work (a size upgrade, say)
|
// Same channel, kept on the `Core` so background work (a size upgrade, say)
|
||||||
// can invalidate kernel-cached metadata without threading a handle through.
|
// can invalidate kernel-cached metadata without threading a handle through.
|
||||||
let _ = core.notifier.set(notifier.clone());
|
let _ = core.notifier.set(notifier.clone());
|
||||||
rt.spawn(run_event_sync(
|
rt.spawn(run_event_sync(client, scope, core.clone()));
|
||||||
client,
|
|
||||||
scope,
|
|
||||||
core.state,
|
|
||||||
core.cache,
|
|
||||||
core.db,
|
|
||||||
core.pending,
|
|
||||||
notifier,
|
|
||||||
));
|
|
||||||
|
|
||||||
// Stop signals (SIGTERM from `systemctl --user stop`, SIGINT from Ctrl-C)
|
// Stop signals (SIGTERM from `systemctl --user stop`, SIGINT from Ctrl-C)
|
||||||
// are delivered onto the async runtime; bridge them onto a sync channel so
|
// are delivered onto the async runtime; bridge them onto a sync channel so
|
||||||
|
|||||||
@@ -466,7 +466,26 @@ impl Core {
|
|||||||
/// been seen through the mount (e.g. shared but not browsed to this session) —
|
/// been seen through the mount (e.g. shared but not browsed to this session) —
|
||||||
/// the caller then leaves the path empty.
|
/// the caller then leaves the path empty.
|
||||||
pub(crate) fn rel_path_for_uid(&self, uid: &NodeUid) -> Option<String> {
|
pub(crate) fn rel_path_for_uid(&self, uid: &NodeUid) -> Option<String> {
|
||||||
let st = self.state.lock();
|
// Searched across every mount: a node browsed through a sync folder is
|
||||||
|
// interned in that fork's inode space, so the primary state alone had no
|
||||||
|
// entry for it and the caller silently showed an empty path
|
||||||
|
// (`docs/BUGS.md` B74). The path is relative to whichever mount holds
|
||||||
|
// it, which is the one the user reached it through.
|
||||||
|
let mut path = None;
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
if path.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
path = walk_to_root(st, uid);
|
||||||
|
});
|
||||||
|
path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Walk one inode space from `uid` up to its root, building the path. `None` if
|
||||||
|
/// that mount has never interned the node, or if the chain breaks partway —
|
||||||
|
/// a broken chain is a missing path, never a partial one.
|
||||||
|
fn walk_to_root(st: &super::State, uid: &NodeUid) -> Option<String> {
|
||||||
let mut ino = *st.by_uid.get(uid)?;
|
let mut ino = *st.by_uid.get(uid)?;
|
||||||
let mut parts = Vec::new();
|
let mut parts = Vec::new();
|
||||||
while ino != ROOT_INO {
|
while ino != ROOT_INO {
|
||||||
@@ -477,4 +496,3 @@ impl Core {
|
|||||||
parts.reverse();
|
parts.reverse();
|
||||||
Some(parts.join("/"))
|
Some(parts.join("/"))
|
||||||
}
|
}
|
||||||
}
|
|
||||||
@@ -106,6 +106,12 @@ pub(crate) struct WriteHandle {
|
|||||||
pub(crate) base_size: u64,
|
pub(crate) base_size: u64,
|
||||||
/// Modification time of the remote base at open, validating its block cache.
|
/// Modification time of the remote base at open, validating its block cache.
|
||||||
pub(crate) base_mtime: i64,
|
pub(crate) base_mtime: i64,
|
||||||
|
/// Server revision id of the remote base at open, if it had one. This is the
|
||||||
|
/// stable identity the drain conflict-checks against: `base_mtime` alone
|
||||||
|
/// drifts when the server re-stamps the *same* revision, which reads as a
|
||||||
|
/// spurious conflict (see B16/B25). `None` for a file with no sealed remote
|
||||||
|
/// revision, or one read through a surface that does not carry the id.
|
||||||
|
pub(crate) base_revision_id: Option<String>,
|
||||||
/// Whether anything diverged from the remote and needs an upload.
|
/// Whether anything diverged from the remote and needs an upload.
|
||||||
pub(crate) dirty: bool,
|
pub(crate) dirty: bool,
|
||||||
/// Number of file handles currently sharing this scratch state.
|
/// Number of file handles currently sharing this scratch state.
|
||||||
@@ -219,10 +225,12 @@ impl State {
|
|||||||
|
|
||||||
/// Check if a directory inode has any child nodes in memory or in the database.
|
/// Check if a directory inode has any child nodes in memory or in the database.
|
||||||
pub(crate) fn has_children(&self, parent: u64) -> bool {
|
pub(crate) fn has_children(&self, parent: u64) -> bool {
|
||||||
if let Some(kids) = self.children.get(&parent)
|
// A resident listing is authoritative even when empty. Falling through
|
||||||
&& !kids.is_empty()
|
// from `Some([])` to SQLite lets an obsolete child row contradict the
|
||||||
{
|
// listing `ensure_children` just established, so readdir says empty but
|
||||||
return true;
|
// rmdir returns ENOTEMPTY forever.
|
||||||
|
if let Some(kids) = self.children.get(&parent) {
|
||||||
|
return !kids.is_empty();
|
||||||
}
|
}
|
||||||
if let Some(entry) = self.entries.get(&parent)
|
if let Some(entry) = self.entries.get(&parent)
|
||||||
&& let Ok(has_kids) = self.db.has_children(&entry.uid)
|
&& let Ok(has_kids) = self.db.has_children(&entry.uid)
|
||||||
@@ -436,6 +444,8 @@ mod tests {
|
|||||||
media_type: "text/plain".into(),
|
media_type: "text/plain".into(),
|
||||||
total_size_on_storage: 0,
|
total_size_on_storage: 0,
|
||||||
active_revision_state: None,
|
active_revision_state: None,
|
||||||
|
active_revision_id: None,
|
||||||
|
content_sha1: None,
|
||||||
claimed_size: Some(0),
|
claimed_size: Some(0),
|
||||||
claimed_modification_time: None,
|
claimed_modification_time: None,
|
||||||
}
|
}
|
||||||
@@ -607,6 +617,11 @@ mod tests {
|
|||||||
st.db
|
st.db
|
||||||
.upsert_node(&node("file2_uid", "dir_uid", "file2.txt", false))
|
.upsert_node(&node("file2_uid", "dir_uid", "file2.txt", false))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!st.has_children(folder),
|
||||||
|
"a resident empty listing overrides stale DB children"
|
||||||
|
);
|
||||||
|
st.children.remove(&folder);
|
||||||
assert!(st.has_children(folder), "has child in db");
|
assert!(st.has_children(folder), "has child in db");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,461 @@
|
|||||||
|
//! Background conflict sweep.
|
||||||
|
//!
|
||||||
|
//! A `(sync-conflict <ts>)` copy is only ever a fallback: the drain and the sync
|
||||||
|
//! engine cut one when they cannot prove a queued write is safe to apply, so the
|
||||||
|
//! user never loses bytes. Most of those copies turn out to be byte-identical to
|
||||||
|
//! the live file — a spurious conflict from a re-stamped revision (B16/B25) or a
|
||||||
|
//! first-sync of an already-identical file. This sweep reconciles them after the
|
||||||
|
//! fact:
|
||||||
|
//!
|
||||||
|
//! - **Identical** to the live sibling (same size *and* same content SHA-1): the
|
||||||
|
//! copy is redundant, so it is trashed and logged as a [`ActivityKind::Trash`].
|
||||||
|
//! - **Divergent** (different bytes) or **orphaned** (no live sibling): left in
|
||||||
|
//! place and surfaced once as an [`ActivityKind::Conflict`], because only the
|
||||||
|
//! user can decide which version wins.
|
||||||
|
//!
|
||||||
|
//! Identity is proved without downloading anything: the SHA-1 comes from the
|
||||||
|
//! revision's decrypted extended attributes, which enumeration already reads. A
|
||||||
|
//! copy whose sibling has no recorded SHA-1 is treated as divergent — the sweep
|
||||||
|
//! never removes a file it cannot prove is a duplicate.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use pdfs_core::config::SweepMode;
|
||||||
|
use pdfs_core::control::ActivityKind;
|
||||||
|
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||||
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
|
use super::{Core, node_content_sha1, node_revision_id, node_size};
|
||||||
|
|
||||||
|
/// Time between sweeps. Conflicts are rare and non-urgent; a slow cadence keeps
|
||||||
|
/// the sweep off the hot path while still cleaning up within a few minutes.
|
||||||
|
const CONFLICT_SWEEP_INTERVAL: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
|
/// Delay before the first sweep, so it does not compete with the burst of work a
|
||||||
|
/// fresh mount does at startup (hydrate, recover, initial enumeration).
|
||||||
|
const CONFLICT_SWEEP_WARMUP: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// Strip a ` (sync-conflict <stamp>)` (optionally `-<n>`) marker from `name`,
|
||||||
|
/// returning the original file name it was a copy of. `None` when `name` is not a
|
||||||
|
/// conflict copy. Inverse of the `conflict_name` / `conflict_path_with_suffix`
|
||||||
|
/// formats, so the two halves of the product agree on what a conflict is.
|
||||||
|
pub(crate) fn conflict_base_name(name: &str) -> Option<String> {
|
||||||
|
const MARKER: &str = " (sync-conflict ";
|
||||||
|
let start = name.find(MARKER)?;
|
||||||
|
let after = start + MARKER.len();
|
||||||
|
let close_rel = name[after..].find(')')?;
|
||||||
|
let close = after + close_rel;
|
||||||
|
let inner = &name[after..close];
|
||||||
|
// The token is `<digits>` or `<digits>-<digits>`; reject anything else so a
|
||||||
|
// file that merely happens to contain the phrase is not mistaken for a copy.
|
||||||
|
let mut parts = inner.splitn(2, '-');
|
||||||
|
let stamp = parts.next().unwrap_or_default();
|
||||||
|
let suffix = parts.next();
|
||||||
|
let is_digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit());
|
||||||
|
if !is_digits(stamp) || suffix.is_some_and(|s| !is_digits(s)) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut base = String::with_capacity(start + name.len() - close);
|
||||||
|
base.push_str(&name[..start]);
|
||||||
|
base.push_str(&name[close + 1..]);
|
||||||
|
Some(base)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Core {
|
||||||
|
/// Sweep for stale conflict copies forever, on its own thread. See the module
|
||||||
|
/// docs for the identical/divergent policy.
|
||||||
|
pub(crate) fn run_conflict_sweep_loop(&self) {
|
||||||
|
std::thread::sleep(CONFLICT_SWEEP_WARMUP);
|
||||||
|
loop {
|
||||||
|
self.sweep_conflicts_once();
|
||||||
|
std::thread::sleep(CONFLICT_SWEEP_INTERVAL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One sweep pass over every node the daemon currently knows.
|
||||||
|
fn sweep_conflicts_once(&self) {
|
||||||
|
let stored = match self.db.load_all() {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
warn!(error = ?e, "conflict sweep: load nodes failed");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let nodes: Vec<_> = stored.into_iter().map(|s| s.node).collect();
|
||||||
|
|
||||||
|
// Index live (non-trashed) files by (parent, name) for sibling lookup.
|
||||||
|
let mut by_parent_name: HashMap<(String, &str), &_> = HashMap::new();
|
||||||
|
for node in &nodes {
|
||||||
|
if node.trashed || node.is_folder() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let parent = node
|
||||||
|
.parent_uid
|
||||||
|
.as_ref()
|
||||||
|
.map(|u| u.to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
by_parent_name.insert((parent, node.name.as_str()), node);
|
||||||
|
}
|
||||||
|
|
||||||
|
let online = self.online.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
|
for node in &nodes {
|
||||||
|
if node.trashed || node.is_folder() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(base) = conflict_base_name(&node.name) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let parent = node
|
||||||
|
.parent_uid
|
||||||
|
.as_ref()
|
||||||
|
.map(|u| u.to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let sibling = by_parent_name.get(&(parent, base.as_str())).copied();
|
||||||
|
|
||||||
|
match decide(node, sibling, &base, self.sweep_mode) {
|
||||||
|
SweepAction::Remove => {
|
||||||
|
// Only while online — the remote trash is the point. Offline,
|
||||||
|
// it waits for a later pass.
|
||||||
|
if online {
|
||||||
|
self.remove_conflict_copy(node, &base);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SweepAction::Flag(detail) => self.flag_conflict(&node.uid, &node.name, detail),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trash a conflict copy proven identical to its live sibling, and unhook it
|
||||||
|
/// locally — the same sequence the interactive unlink path uses after a trash.
|
||||||
|
///
|
||||||
|
/// `snapshot` is the node as the pass saw it, which by now is stale: the pass
|
||||||
|
/// enumerated every node up front and has been doing network I/O since. Before
|
||||||
|
/// touching anything remote this re-reads the node and refuses to act unless
|
||||||
|
/// it is *still* the same revision, still untrashed, still unwritten and still
|
||||||
|
/// unopened. Without that, a user edit landing inside the window is destroyed
|
||||||
|
/// — `discard_queued_ops` below drops the queued write *and its staged blob*,
|
||||||
|
/// which may hold the only copy of those bytes (`docs/BUGS.md` B71).
|
||||||
|
fn remove_conflict_copy(&self, snapshot: &proton_drive_rs::Node, base: &str) {
|
||||||
|
let uid = &snapshot.uid;
|
||||||
|
let name = snapshot.name.as_str();
|
||||||
|
if !self.duplicate_still_removable(snapshot) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match self
|
||||||
|
.rt
|
||||||
|
.block_on(self.client.trash_nodes(std::slice::from_ref(uid)))
|
||||||
|
{
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(e) if super::is_gone(&e) => {}
|
||||||
|
Err(e) => {
|
||||||
|
warn!(%uid, name, error = %e, "conflict sweep: trashing duplicate failed");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Every mount, not just the primary: a conflict copy inside a sync
|
||||||
|
// folder lives in that fork's inode space, and leaving it hooked up
|
||||||
|
// there keeps a node we have just trashed remotely visible and
|
||||||
|
// readable there (`docs/BUGS.md` B74).
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
st.forget_or_unlink(uid);
|
||||||
|
});
|
||||||
|
self.discard_queued_ops(uid);
|
||||||
|
self.cache.evict(uid);
|
||||||
|
self.evict_reader(uid);
|
||||||
|
if let Err(e) = self.db.delete_node(uid) {
|
||||||
|
warn!(%uid, error = ?e, "conflict sweep: db delete after trash failed");
|
||||||
|
}
|
||||||
|
self.invalidate_trash();
|
||||||
|
self.conflict_notified.lock().remove(uid);
|
||||||
|
self.log_activity(
|
||||||
|
ActivityKind::Trash,
|
||||||
|
name,
|
||||||
|
format!("auto-removed duplicate of {base}"),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
info!(%uid, name, base, "conflict sweep: removed identical conflict copy");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Re-check, immediately before the destructive call, everything the pass
|
||||||
|
/// decided from its stale snapshot. Any doubt means "leave it alone": a
|
||||||
|
/// conflict copy that survives to the next pass costs nothing, a wrongly
|
||||||
|
/// removed one costs the user data.
|
||||||
|
fn duplicate_still_removable(&self, snapshot: &proton_drive_rs::Node) -> bool {
|
||||||
|
let uid = &snapshot.uid;
|
||||||
|
let name = snapshot.name.as_str();
|
||||||
|
|
||||||
|
// A queued op means bytes are still owed an upload. `discard_queued_ops`
|
||||||
|
// would throw those away along with the staged blob holding them.
|
||||||
|
match self.db.has_any_op(&uid.to_string()) {
|
||||||
|
Ok(false) => {}
|
||||||
|
Ok(true) => {
|
||||||
|
debug!(%uid, name, "conflict sweep: skipping duplicate with queued ops");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// Cannot prove the queue is empty — assume it is not.
|
||||||
|
Err(e) => {
|
||||||
|
warn!(%uid, name, error = ?e, "conflict sweep: op check failed; skipping");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Someone is writing to it, or holds it open. Trashing now would pull the
|
||||||
|
// node out from under a live handle.
|
||||||
|
if self.is_busy(uid) {
|
||||||
|
debug!(%uid, name, "conflict sweep: skipping duplicate that is open or being written");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finally, the node itself must not have moved on since the snapshot.
|
||||||
|
match self.db.node_by_uid(&uid.to_string()) {
|
||||||
|
Ok(Some(fresh)) => {
|
||||||
|
if fresh.trashed {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if fresh.name != snapshot.name || fresh.parent_uid != snapshot.parent_uid {
|
||||||
|
debug!(%uid, name, "conflict sweep: duplicate was renamed or moved; skipping");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if node_revision_id(&fresh) != node_revision_id(snapshot)
|
||||||
|
|| node_size(&fresh) != node_size(snapshot)
|
||||||
|
|| node_content_sha1(&fresh) != node_content_sha1(snapshot)
|
||||||
|
{
|
||||||
|
debug!(%uid, name, "conflict sweep: duplicate changed since the pass began");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
// Already gone locally, or unreadable — either way, not ours to remove.
|
||||||
|
Ok(None) => false,
|
||||||
|
Err(e) => {
|
||||||
|
warn!(%uid, name, error = ?e, "conflict sweep: re-read failed; skipping");
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a node has a write in flight or a file handle open against it.
|
||||||
|
/// Read-only opens use fh 0 and are not tracked, so this is specifically the
|
||||||
|
/// writer/holder check.
|
||||||
|
/// Asked across **every** mount. The sweep runs on the primary Core but
|
||||||
|
/// trashes nodes anywhere in the tree, and a file open inside a sync folder
|
||||||
|
/// is tracked in that fork's `State` — so consulting `self.state` alone
|
||||||
|
/// answered "idle" for every such file and let the sweep delete one with a
|
||||||
|
/// writer attached (`docs/BUGS.md` B74). Busy anywhere means busy.
|
||||||
|
fn is_busy(&self, uid: &NodeUid) -> bool {
|
||||||
|
let mut busy = false;
|
||||||
|
self.for_each_state(|st| {
|
||||||
|
let Some(&ino) = st.by_uid.get(uid) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
busy |=
|
||||||
|
st.active_writes.contains_key(&ino) || st.handles.values().any(|&held| held == ino);
|
||||||
|
});
|
||||||
|
busy
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Surface a conflict copy that needs the user's attention, at most once per
|
||||||
|
/// daemon run so the activity feed is not spammed each pass.
|
||||||
|
fn flag_conflict(&self, uid: &NodeUid, name: &str, detail: String) {
|
||||||
|
if !self.conflict_notified.lock().insert(uid.clone()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
debug!(%uid, name, detail, "conflict sweep: flagged divergent conflict copy");
|
||||||
|
self.log_activity(ActivityKind::Conflict, name, detail, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the sweep concluded about one `(sync-conflict …)` copy.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
enum SweepAction {
|
||||||
|
/// Provably redundant: trash it, subject to the runtime interlocks in
|
||||||
|
/// [`Core::duplicate_still_removable`].
|
||||||
|
Remove,
|
||||||
|
/// Only the user can resolve this one; surface it with this detail.
|
||||||
|
Flag(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decide what to do about conflict copy `node` given its live sibling (if any)
|
||||||
|
/// and the configured `mode`.
|
||||||
|
///
|
||||||
|
/// Pure, so the policy that governs deletion is testable without a mount, a
|
||||||
|
/// runtime, or an account — the same split used for `revision_changed` and
|
||||||
|
/// `is_own_self_supersede`. Everything that can fail or race lives in the caller.
|
||||||
|
fn decide(
|
||||||
|
node: &proton_drive_rs::Node,
|
||||||
|
sibling: Option<&proton_drive_rs::Node>,
|
||||||
|
base: &str,
|
||||||
|
mode: SweepMode,
|
||||||
|
) -> SweepAction {
|
||||||
|
match sibling {
|
||||||
|
Some(sib) if is_identical(node, sib) => {
|
||||||
|
if mode.is_enforcing() {
|
||||||
|
SweepAction::Remove
|
||||||
|
} else {
|
||||||
|
// Report-only: say what would happen, remove nothing. The point is
|
||||||
|
// to accumulate field evidence that the identity check is right
|
||||||
|
// before it is trusted with deletion (docs/BUGS.md B71).
|
||||||
|
SweepAction::Flag(format!(
|
||||||
|
"identical to {base}; would be removed (sweep is report-only)"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(_) => SweepAction::Flag(format!("differs from {base}")),
|
||||||
|
None => SweepAction::Flag(format!("no original {base} to reconcile against")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether two file nodes provably hold the same bytes: equal size and equal,
|
||||||
|
/// present content SHA-1. A missing digest on either side is not proof, so the
|
||||||
|
/// sweep treats it as divergent rather than risk removing a real difference.
|
||||||
|
fn is_identical(a: &proton_drive_rs::Node, b: &proton_drive_rs::Node) -> bool {
|
||||||
|
if node_size(a) != node_size(b) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
match (node_content_sha1(a), node_content_sha1(b)) {
|
||||||
|
(Some(x), Some(y)) => x == y,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use proton_drive_rs::NodeKind;
|
||||||
|
use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId};
|
||||||
|
|
||||||
|
/// A file node with the two fields the sweep's identity check reads.
|
||||||
|
fn file(name: &str, size: i64, sha1: Option<&str>) -> proton_drive_rs::Node {
|
||||||
|
proton_drive_rs::Node {
|
||||||
|
uid: NodeUid::new(VolumeId::from("v"), LinkId::from(name)),
|
||||||
|
parent_uid: None,
|
||||||
|
kind: NodeKind::File {
|
||||||
|
media_type: "text/plain".into(),
|
||||||
|
total_size_on_storage: size,
|
||||||
|
active_revision_state: None,
|
||||||
|
active_revision_id: Some("rev".into()),
|
||||||
|
claimed_size: Some(size),
|
||||||
|
claimed_modification_time: None,
|
||||||
|
content_sha1: sha1.map(String::from),
|
||||||
|
},
|
||||||
|
name: name.into(),
|
||||||
|
creation_time: 0,
|
||||||
|
modification_time: 0,
|
||||||
|
trashed: false,
|
||||||
|
is_shared: false,
|
||||||
|
is_shared_publicly: false,
|
||||||
|
signature_email: None,
|
||||||
|
verification: Default::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The only input shape that may ever produce a removal: same size, same
|
||||||
|
/// present digest, live sibling, enforcing mode.
|
||||||
|
#[test]
|
||||||
|
fn removes_only_a_provably_identical_copy_when_enforcing() {
|
||||||
|
let copy = file("f (sync-conflict 1).txt", 100, Some("aaa"));
|
||||||
|
let live = file("f.txt", 100, Some("aaa"));
|
||||||
|
assert_eq!(
|
||||||
|
decide(©, Some(&live), "f.txt", SweepMode::Enforce),
|
||||||
|
SweepAction::Remove
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn report_only_never_removes_but_still_says_what_it_would_do() {
|
||||||
|
let copy = file("f (sync-conflict 1).txt", 100, Some("aaa"));
|
||||||
|
let live = file("f.txt", 100, Some("aaa"));
|
||||||
|
// Same inputs as the removal case above; only the mode differs.
|
||||||
|
let SweepAction::Flag(detail) = decide(©, Some(&live), "f.txt", SweepMode::Report)
|
||||||
|
else {
|
||||||
|
panic!("report mode must never remove");
|
||||||
|
};
|
||||||
|
assert!(detail.contains("would be removed"), "{detail}");
|
||||||
|
assert!(detail.contains("report-only"), "{detail}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_differing_size_is_a_real_conflict() {
|
||||||
|
let copy = file("f (sync-conflict 1).txt", 101, Some("aaa"));
|
||||||
|
let live = file("f.txt", 100, Some("aaa"));
|
||||||
|
assert_eq!(
|
||||||
|
decide(©, Some(&live), "f.txt", SweepMode::Enforce),
|
||||||
|
SweepAction::Flag("differs from f.txt".into())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_missing_digest_is_never_proof_of_identity() {
|
||||||
|
// Equal sizes are not enough: two different 100-byte files are common.
|
||||||
|
// A copy the sweep cannot fingerprint must survive, in either direction.
|
||||||
|
for (a, b) in [(None, Some("aaa")), (Some("aaa"), None), (None, None)] {
|
||||||
|
let copy = file("f (sync-conflict 1).txt", 100, a);
|
||||||
|
let live = file("f.txt", 100, b);
|
||||||
|
assert_eq!(
|
||||||
|
decide(©, Some(&live), "f.txt", SweepMode::Enforce),
|
||||||
|
SweepAction::Flag("differs from f.txt".into()),
|
||||||
|
"{a:?} vs {b:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_differing_digest_at_equal_size_is_a_real_conflict() {
|
||||||
|
let copy = file("f (sync-conflict 1).txt", 100, Some("aaa"));
|
||||||
|
let live = file("f.txt", 100, Some("bbb"));
|
||||||
|
assert_eq!(
|
||||||
|
decide(©, Some(&live), "f.txt", SweepMode::Enforce),
|
||||||
|
SweepAction::Flag("differs from f.txt".into())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_orphaned_copy_is_surfaced_not_removed() {
|
||||||
|
// No live original left to reconcile against — the copy may be the only
|
||||||
|
// surviving version of the file, so it is never the sweep's to remove.
|
||||||
|
let copy = file("f (sync-conflict 1).txt", 100, Some("aaa"));
|
||||||
|
assert_eq!(
|
||||||
|
decide(©, None, "f.txt", SweepMode::Enforce),
|
||||||
|
SweepAction::Flag("no original f.txt to reconcile against".into())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn identity_needs_both_size_and_digest_to_match() {
|
||||||
|
let a = file("a", 100, Some("aaa"));
|
||||||
|
assert!(is_identical(&a, &file("b", 100, Some("aaa"))));
|
||||||
|
assert!(!is_identical(&a, &file("b", 101, Some("aaa"))));
|
||||||
|
assert!(!is_identical(&a, &file("b", 100, Some("bbb"))));
|
||||||
|
assert!(!is_identical(&a, &file("b", 100, None)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_conflict_names_and_leaves_others_alone() {
|
||||||
|
assert_eq!(
|
||||||
|
conflict_base_name("test_export (sync-conflict 1784898786).xml").as_deref(),
|
||||||
|
Some("test_export.xml")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
conflict_base_name("README (sync-conflict 42)").as_deref(),
|
||||||
|
Some("README")
|
||||||
|
);
|
||||||
|
// The sync engine's de-duplicated suffix form.
|
||||||
|
assert_eq!(
|
||||||
|
conflict_base_name("notes (sync-conflict 1700-3).txt").as_deref(),
|
||||||
|
Some("notes.txt")
|
||||||
|
);
|
||||||
|
// Not a conflict copy.
|
||||||
|
assert_eq!(conflict_base_name("report.xml"), None);
|
||||||
|
// Looks close but the token is not a stamp.
|
||||||
|
assert_eq!(conflict_base_name("x (sync-conflict abc).txt"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_dotted_stem_keeps_its_inner_dots() {
|
||||||
|
assert_eq!(
|
||||||
|
conflict_base_name("archive.tar (sync-conflict 7).gz").as_deref(),
|
||||||
|
Some("archive.tar.gz")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
+166
-28
@@ -55,9 +55,11 @@ use local_scan::is_write_mode;
|
|||||||
use local_scan::open_for_write_set;
|
use local_scan::open_for_write_set;
|
||||||
use model::{LocalItem, Outcome, PassAbort, Pending, RemoteItem};
|
use model::{LocalItem, Outcome, PassAbort, Pending, RemoteItem};
|
||||||
pub(crate) use planner::base_name;
|
pub(crate) use planner::base_name;
|
||||||
|
#[cfg(test)]
|
||||||
|
use planner::conflict_path;
|
||||||
use planner::{
|
use planner::{
|
||||||
FilePlan, classification_order, conflict_path, filter_baseline, guard_local_wipe, join_rel,
|
FilePlan, classification_order, conflict_path_with_suffix, filter_baseline, guard_local_wipe,
|
||||||
parent_rel, plan_file, rel_to_path, remote_sig, unchanged_remote_size,
|
join_rel, parent_rel, plan_file, rel_to_path, remote_sig, unchanged_remote_size,
|
||||||
};
|
};
|
||||||
/// What a [`Pending`] op did, so the reconcile can update shared state on the
|
/// What a [`Pending`] op did, so the reconcile can update shared state on the
|
||||||
/// engine thread (never inside a task).
|
/// engine thread (never inside a task).
|
||||||
@@ -668,6 +670,27 @@ impl Core {
|
|||||||
|
|
||||||
match plan_file(l, r, baseline.get(rel)) {
|
match plan_file(l, r, baseline.get(rel)) {
|
||||||
FilePlan::Unchanged => {}
|
FilePlan::Unchanged => {}
|
||||||
|
FilePlan::AdoptBaseline => {
|
||||||
|
// First sync of an already-identical file: record the baseline
|
||||||
|
// in place so future divergence is detected, without any
|
||||||
|
// transfer or a spurious conflict copy.
|
||||||
|
let local = l.expect("adopt baseline requires a local file");
|
||||||
|
let remote = r.expect("adopt baseline requires a remote file");
|
||||||
|
if let Err(e) = self.db.sync_entry_upsert(
|
||||||
|
folder_id,
|
||||||
|
&StoredSyncEntry {
|
||||||
|
rel_path: rel.clone(),
|
||||||
|
remote_uid: Some(remote.uid.to_string()),
|
||||||
|
local_mtime: local.mtime,
|
||||||
|
local_size: local.size,
|
||||||
|
remote_rev: Some(remote.mtime.to_string()),
|
||||||
|
remote_hash: Some(remote.size.to_string()),
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
warn!(rel, error = ?e, "sync: adopt baseline failed");
|
||||||
|
outcome.errors += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
FilePlan::Deferred => {
|
FilePlan::Deferred => {
|
||||||
debug!(
|
debug!(
|
||||||
rel,
|
rel,
|
||||||
@@ -711,7 +734,17 @@ impl Core {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
FilePlan::DeleteLocal => {
|
FilePlan::DeleteLocal => {
|
||||||
let _ = std::fs::remove_file(local_root.join(rel_to_path(rel)));
|
if let Err(e) = std::fs::remove_file(local_root.join(rel_to_path(rel))) {
|
||||||
|
warn!(rel, error = %e, "sync: remove local file failed");
|
||||||
|
self.log_activity(
|
||||||
|
ActivityKind::Trash,
|
||||||
|
base_name(rel),
|
||||||
|
e.to_string(),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
outcome.errors += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let _ = self.db.sync_entry_remove(folder_id, rel);
|
let _ = self.db.sync_entry_remove(folder_id, rel);
|
||||||
self.log_activity(ActivityKind::Trash, base_name(rel), "removed locally", true);
|
self.log_activity(ActivityKind::Trash, base_name(rel), "removed locally", true);
|
||||||
outcome.deleted += 1;
|
outcome.deleted += 1;
|
||||||
@@ -758,7 +791,12 @@ impl Core {
|
|||||||
// Deferred folder deletions, deepest first.
|
// Deferred folder deletions, deepest first.
|
||||||
delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count()));
|
delete_local_dirs.sort_by_key(|p| std::cmp::Reverse(p.matches('/').count()));
|
||||||
for rel in delete_local_dirs {
|
for rel in delete_local_dirs {
|
||||||
let _ = std::fs::remove_dir_all(local_root.join(rel_to_path(&rel)));
|
if let Err(e) = std::fs::remove_dir_all(local_root.join(rel_to_path(&rel))) {
|
||||||
|
warn!(rel, error = %e, "sync: remove local folder failed");
|
||||||
|
self.log_activity(ActivityKind::Trash, base_name(&rel), e.to_string(), false);
|
||||||
|
outcome.errors += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let _ = self.db.sync_entry_remove(folder_id, &rel);
|
let _ = self.db.sync_entry_remove(folder_id, &rel);
|
||||||
self.log_activity(
|
self.log_activity(
|
||||||
ActivityKind::Trash,
|
ActivityKind::Trash,
|
||||||
@@ -802,18 +840,24 @@ impl Core {
|
|||||||
out: &mut HashMap<String, LocalItem>,
|
out: &mut HashMap<String, LocalItem>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
let entries = std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))?;
|
||||||
for entry in entries.filter_map(|e| e.ok()) {
|
for entry in entries {
|
||||||
|
let entry = entry.map_err(|e| format!("read entry in {}: {e}", dir.display()))?;
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
let meta = match std::fs::symlink_metadata(&path) {
|
let meta = std::fs::symlink_metadata(&path)
|
||||||
Ok(m) => m,
|
.map_err(|e| format!("stat {}: {e}", path.display()))?;
|
||||||
Err(_) => continue,
|
let stripped = path.strip_prefix(root).map_err(|e| {
|
||||||
};
|
format!(
|
||||||
let Ok(stripped) = path.strip_prefix(root) else {
|
"local path {} escaped sync root {}: {e}",
|
||||||
continue;
|
path.display(),
|
||||||
};
|
root.display()
|
||||||
let Some(rel) = stripped.to_str() else {
|
)
|
||||||
continue;
|
})?;
|
||||||
};
|
let rel = stripped.to_str().ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"local path {} is not valid UTF-8; refusing a destructive sync pass",
|
||||||
|
path.display()
|
||||||
|
)
|
||||||
|
})?;
|
||||||
// Ignore our own in-flight download temp files.
|
// Ignore our own in-flight download temp files.
|
||||||
if rel.contains(".pdfs-tmp-") {
|
if rel.contains(".pdfs-tmp-") {
|
||||||
continue;
|
continue;
|
||||||
@@ -1153,13 +1197,10 @@ impl Core {
|
|||||||
// pass), then take the remote version as the shared truth.
|
// pass), then take the remote version as the shared truth.
|
||||||
let path = local_root.join(rel_to_path(rel));
|
let path = local_root.join(rel_to_path(rel));
|
||||||
if path.exists() {
|
if path.exists() {
|
||||||
let conflict = conflict_path(&path, now_secs());
|
preserve_conflict_copy(&path, now_secs())
|
||||||
if let Err(e) = std::fs::rename(&path, &conflict) {
|
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
||||||
warn!(rel, error = %e, "sync: could not set aside conflict copy");
|
|
||||||
} else {
|
|
||||||
info!(rel, "sync: kept local changes as a conflict copy");
|
info!(rel, "sync: kept local changes as a conflict copy");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
self.download_file(folder_id, local_root, rel, uid, *mtime, *size)
|
self.download_file(folder_id, local_root, rel, uid, *mtime, *size)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(Applied::Conflict)
|
Ok(Applied::Conflict)
|
||||||
@@ -1171,8 +1212,7 @@ impl Core {
|
|||||||
// evicted. Push the conflict copy now, while it still exists, and
|
// evicted. Push the conflict copy now, while it still exists, and
|
||||||
// leave the remote file holding the original path.
|
// leave the remote file holding the original path.
|
||||||
let path = local_root.join(rel_to_path(rel));
|
let path = local_root.join(rel_to_path(rel));
|
||||||
let conflict = conflict_path(&path, now_secs());
|
let conflict = preserve_conflict_copy(&path, now_secs())
|
||||||
std::fs::rename(&path, &conflict)
|
|
||||||
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
.map_err(|e| format!("set aside conflict copy for {rel}: {e}"))?;
|
||||||
let name = conflict
|
let name = conflict
|
||||||
.file_name()
|
.file_name()
|
||||||
@@ -1297,13 +1337,24 @@ impl Core {
|
|||||||
let _ = std::fs::remove_file(&tmp);
|
let _ = std::fs::remove_file(&tmp);
|
||||||
format!("download {rel}: {e}")
|
format!("download {rel}: {e}")
|
||||||
})?;
|
})?;
|
||||||
out.into_inner().flush().ok();
|
let mut out = out.into_inner();
|
||||||
|
out.flush().map_err(|e| format!("flush tmp {rel}: {e}"))?;
|
||||||
|
out.sync_all().map_err(|e| format!("sync tmp {rel}: {e}"))?;
|
||||||
}
|
}
|
||||||
std::fs::rename(&tmp, &path).map_err(|e| format!("place {rel}: {e}"))?;
|
std::fs::rename(&tmp, &path).map_err(|e| format!("place {rel}: {e}"))?;
|
||||||
// Match local mtime to the remote's so neither side looks "changed" next pass.
|
// Match local mtime to the remote's so neither side looks "changed" next pass.
|
||||||
if let Ok(f) = std::fs::File::options().write(true).open(&path) {
|
let f = std::fs::File::options()
|
||||||
let _ =
|
.write(true)
|
||||||
f.set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(mtime.max(0) as u64));
|
.open(&path)
|
||||||
|
.map_err(|e| format!("open placed file {rel}: {e}"))?;
|
||||||
|
f.set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(mtime.max(0) as u64))
|
||||||
|
.map_err(|e| format!("set mtime for {rel}: {e}"))?;
|
||||||
|
f.sync_all()
|
||||||
|
.map_err(|e| format!("sync placed file {rel}: {e}"))?;
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::File::open(parent)
|
||||||
|
.and_then(|dir| dir.sync_all())
|
||||||
|
.map_err(|e| format!("sync directory for {rel}: {e}"))?;
|
||||||
}
|
}
|
||||||
self.record_file_baseline(folder_id, rel, &path, uid).await
|
self.record_file_baseline(folder_id, rel, &path, uid).await
|
||||||
}
|
}
|
||||||
@@ -1350,6 +1401,46 @@ impl Core {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Publish a conflict copy without overwriting an earlier conflict bearing the
|
||||||
|
/// same timestamp. The original is removed only after the new copy is durable.
|
||||||
|
fn preserve_conflict_copy(path: &Path, stamp: i64) -> std::io::Result<PathBuf> {
|
||||||
|
let metadata = std::fs::metadata(path)?;
|
||||||
|
for suffix in 0..=u32::MAX {
|
||||||
|
let conflict = conflict_path_with_suffix(path, stamp, suffix);
|
||||||
|
let mut source = std::fs::File::open(path)?;
|
||||||
|
let mut target = match std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(&conflict)
|
||||||
|
{
|
||||||
|
Ok(file) => file,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
if let Err(e) = std::io::copy(&mut source, &mut target).and_then(|_| target.sync_all()) {
|
||||||
|
let _ = std::fs::remove_file(&conflict);
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
target.set_permissions(metadata.permissions())?;
|
||||||
|
if let Ok(modified) = metadata.modified() {
|
||||||
|
target.set_modified(modified)?;
|
||||||
|
}
|
||||||
|
target.sync_all()?;
|
||||||
|
sync_parent_dir(&conflict)?;
|
||||||
|
std::fs::remove_file(path)?;
|
||||||
|
sync_parent_dir(&conflict)?;
|
||||||
|
return Ok(conflict);
|
||||||
|
}
|
||||||
|
unreachable!("u32 conflict suffix space exhausted")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sync_parent_dir(path: &Path) -> std::io::Result<()> {
|
||||||
|
match path.parent() {
|
||||||
|
Some(parent) => std::fs::File::open(parent)?.sync_all(),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---- helpers --------------------------------------------------------------
|
// ---- helpers --------------------------------------------------------------
|
||||||
|
|
||||||
/// A remote node's `(mtime, size)` change signature. Size prefers the plaintext
|
/// A remote node's `(mtime, size)` change signature. Size prefers the plaintext
|
||||||
@@ -1609,9 +1700,10 @@ mod tests {
|
|||||||
guard_local_wipe(&base, &one).is_ok(),
|
guard_local_wipe(&base, &one).is_ok(),
|
||||||
"one survivor means the tree is readable; the rest are real deletions"
|
"one survivor means the tree is readable; the rest are real deletions"
|
||||||
);
|
);
|
||||||
// A one-entry baseline is a genuine single delete, not a wipe to detect.
|
// A one-entry folder can disappear because its mount or parent became
|
||||||
|
// unavailable just as easily as a larger tree can.
|
||||||
let single: HashMap<String, ()> = [("a.txt".to_string(), ())].into_iter().collect();
|
let single: HashMap<String, ()> = [("a.txt".to_string(), ())].into_iter().collect();
|
||||||
assert!(guard_local_wipe(&single, &none).is_ok());
|
assert!(guard_local_wipe(&single, &none).is_err());
|
||||||
// Nothing synced yet: a first pass over an empty folder is not a wipe.
|
// Nothing synced yet: a first pass over an empty folder is not a wipe.
|
||||||
assert!(guard_local_wipe(&none, &none).is_ok());
|
assert!(guard_local_wipe(&none, &none).is_ok());
|
||||||
// Local paths that are not in the baseline are new uploads, and do not
|
// Local paths that are not in the baseline are new uploads, and do not
|
||||||
@@ -1781,6 +1873,52 @@ mod tests {
|
|||||||
assert_eq!(no_ext, PathBuf::from("/home/me/README (sync-conflict 42)"));
|
assert_eq!(no_ext, PathBuf::from("/home/me/README (sync-conflict 42)"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conflict_preservation_never_replaces_an_existing_copy() {
|
||||||
|
let dir = sync_test_dir("conflict-collision");
|
||||||
|
let source = dir.join("notes.txt");
|
||||||
|
let existing = conflict_path(&source, 1700);
|
||||||
|
std::fs::write(&source, b"new local edit").unwrap();
|
||||||
|
std::fs::write(&existing, b"older conflict").unwrap();
|
||||||
|
|
||||||
|
let preserved = preserve_conflict_copy(&source, 1700).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(std::fs::read(&existing).unwrap(), b"older conflict");
|
||||||
|
assert_eq!(std::fs::read(&preserved).unwrap(), b"new local edit");
|
||||||
|
assert_eq!(
|
||||||
|
preserved.file_name().unwrap(),
|
||||||
|
"notes (sync-conflict 1700-1).txt"
|
||||||
|
);
|
||||||
|
assert!(!source.exists());
|
||||||
|
std::fs::remove_dir_all(dir).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conflict_preservation_failure_keeps_the_source() {
|
||||||
|
let dir = sync_test_dir("conflict-failure");
|
||||||
|
// A directory cannot be copied through the regular-file preservation
|
||||||
|
// path, deterministically forcing publication to fail after the source
|
||||||
|
// has been discovered.
|
||||||
|
let source = dir.join("not-a-file.txt");
|
||||||
|
std::fs::create_dir(&source).unwrap();
|
||||||
|
|
||||||
|
assert!(preserve_conflict_copy(&source, 1700).is_err());
|
||||||
|
assert!(source.is_dir(), "failure must not remove the source");
|
||||||
|
assert!(!conflict_path(&source, 1700).exists());
|
||||||
|
std::fs::remove_dir_all(dir).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sync_test_dir(label: &str) -> PathBuf {
|
||||||
|
let nonce = SystemTime::now()
|
||||||
|
.duration_since(SystemTime::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_nanos();
|
||||||
|
let path =
|
||||||
|
std::env::temp_dir().join(format!("pdfs-sync-{label}-{}-{nonce}", std::process::id()));
|
||||||
|
std::fs::create_dir(&path).unwrap();
|
||||||
|
path
|
||||||
|
}
|
||||||
|
|
||||||
// ---- open-for-write detection tests -----------------------------------
|
// ---- open-for-write detection tests -----------------------------------
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ pub(super) fn guard_local_wipe<B, L>(
|
|||||||
baseline: &HashMap<String, B>,
|
baseline: &HashMap<String, B>,
|
||||||
local: &HashMap<String, L>,
|
local: &HashMap<String, L>,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
if baseline.len() >= 2 && baseline.keys().all(|rel| !local.contains_key(rel)) {
|
if !baseline.is_empty() && baseline.keys().all(|rel| !local.contains_key(rel)) {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"every one of the {} synced paths is missing locally; refusing to trash \
|
"every one of the {} synced paths is missing locally; refusing to trash \
|
||||||
them on Drive. Check that the folder is mounted and readable.",
|
them on Drive. Check that the folder is mounted and readable.",
|
||||||
@@ -162,12 +162,24 @@ pub(super) fn rel_to_path(rel: &str) -> PathBuf {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The name for a conflict copy of `path`, e.g. `notes (sync-conflict 1700000000).txt`.
|
/// The name for a conflict copy of `path`, e.g. `notes (sync-conflict 1700000000).txt`.
|
||||||
|
#[cfg(test)]
|
||||||
pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
|
pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
|
||||||
|
conflict_path_with_suffix(path, stamp, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// As [`conflict_path`], with a deterministic suffix used when that name already
|
||||||
|
/// exists. This keeps conflict preservation from ever replacing an older copy.
|
||||||
|
pub(super) fn conflict_path_with_suffix(path: &Path, stamp: i64, suffix: u32) -> PathBuf {
|
||||||
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("file");
|
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("file");
|
||||||
let ext = path.extension().and_then(|s| s.to_str());
|
let ext = path.extension().and_then(|s| s.to_str());
|
||||||
|
let suffix = if suffix == 0 {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("-{suffix}")
|
||||||
|
};
|
||||||
let name = match ext {
|
let name = match ext {
|
||||||
Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"),
|
Some(ext) => format!("{stem} (sync-conflict {stamp}{suffix}).{ext}"),
|
||||||
None => format!("{stem} (sync-conflict {stamp})"),
|
None => format!("{stem} (sync-conflict {stamp}{suffix})"),
|
||||||
};
|
};
|
||||||
match path.parent() {
|
match path.parent() {
|
||||||
Some(dir) => dir.join(name),
|
Some(dir) => dir.join(name),
|
||||||
@@ -188,6 +200,12 @@ pub(super) enum FilePlan {
|
|||||||
DeleteLocal,
|
DeleteLocal,
|
||||||
DeleteRemote,
|
DeleteRemote,
|
||||||
ForgetBaseline,
|
ForgetBaseline,
|
||||||
|
/// First sync of a file that already exists on both sides with identical
|
||||||
|
/// content: record the baseline without transferring anything. Without this,
|
||||||
|
/// a folder synced for the first time cuts a `(sync-conflict)` copy of every
|
||||||
|
/// pre-existing file, because a missing baseline makes both sides read as
|
||||||
|
/// "changed" (see the mass first-sync conflict storm in docs/BUGS.md).
|
||||||
|
AdoptBaseline,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn plan_file(
|
pub(super) fn plan_file(
|
||||||
@@ -207,10 +225,16 @@ pub(super) fn plan_file(
|
|||||||
});
|
});
|
||||||
|
|
||||||
match (local, remote) {
|
match (local, remote) {
|
||||||
(Some(_), Some(_)) => match (local_changed, remote_changed) {
|
(Some(l), Some(r)) => match (local_changed, remote_changed) {
|
||||||
(false, false) => FilePlan::Unchanged,
|
(false, false) => FilePlan::Unchanged,
|
||||||
(true, false) => FilePlan::UploadRevision,
|
(true, false) => FilePlan::UploadRevision,
|
||||||
(false, true) => FilePlan::Download,
|
(false, true) => FilePlan::Download,
|
||||||
|
// With a baseline, both-changed is a genuine concurrent edit and must
|
||||||
|
// be preserved as a conflict. Without one, "both changed" only means
|
||||||
|
// this is the first sync and there was nothing to compare to: adopt
|
||||||
|
// the baseline when the sizes already match rather than conflict-copy
|
||||||
|
// an already-identical file. Divergent sizes stay a real conflict.
|
||||||
|
(true, true) if baseline.is_none() && l.size == r.size => FilePlan::AdoptBaseline,
|
||||||
(true, true) => FilePlan::Conflict,
|
(true, true) => FilePlan::Conflict,
|
||||||
},
|
},
|
||||||
(Some(_), None) if baseline.is_none() || local_changed => FilePlan::UploadNew,
|
(Some(_), None) if baseline.is_none() || local_changed => FilePlan::UploadNew,
|
||||||
@@ -338,6 +362,31 @@ mod file_plan_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn first_sync_of_an_identical_file_adopts_the_baseline_not_a_conflict() {
|
||||||
|
// No baseline yet (first sync). Same size on both sides is a pre-existing
|
||||||
|
// identical file, not a concurrent edit: adopt it rather than cut a
|
||||||
|
// conflict copy of every file in the folder.
|
||||||
|
let local = local(10, 40);
|
||||||
|
let remote = remote(30, 40);
|
||||||
|
assert_eq!(
|
||||||
|
plan_file(Some(&local), Some(&remote), None),
|
||||||
|
FilePlan::AdoptBaseline
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn first_sync_of_differing_sizes_is_still_a_conflict() {
|
||||||
|
// No baseline and the sizes disagree: a genuine both-sides divergence,
|
||||||
|
// preserved non-destructively as a conflict.
|
||||||
|
let local = local(10, 20);
|
||||||
|
let remote = remote(30, 40);
|
||||||
|
assert_eq!(
|
||||||
|
plan_file(Some(&local), Some(&remote), None),
|
||||||
|
FilePlan::Conflict
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_missing_remote_signature_is_treated_as_changed() {
|
fn a_missing_remote_signature_is_treated_as_changed() {
|
||||||
let local = local(10, 20);
|
let local = local(10, 20);
|
||||||
|
|||||||
@@ -281,6 +281,7 @@ pub(crate) fn activity_verb(kind: ActivityKind) -> &'static str {
|
|||||||
ActivityKind::Share => "Shared",
|
ActivityKind::Share => "Shared",
|
||||||
ActivityKind::PublicLink => "Linked",
|
ActivityKind::PublicLink => "Linked",
|
||||||
ActivityKind::Unshare => "Unshared",
|
ActivityKind::Unshare => "Unshared",
|
||||||
|
ActivityKind::Conflict => "Conflict",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -299,6 +300,7 @@ pub(crate) fn activity_icon(kind: ActivityKind) -> &'static str {
|
|||||||
ActivityKind::Restore => "edit-undo-symbolic",
|
ActivityKind::Restore => "edit-undo-symbolic",
|
||||||
ActivityKind::Share | ActivityKind::PublicLink => "emblem-shared-symbolic",
|
ActivityKind::Share | ActivityKind::PublicLink => "emblem-shared-symbolic",
|
||||||
ActivityKind::Unshare => "action-unavailable-symbolic",
|
ActivityKind::Unshare => "action-unavailable-symbolic",
|
||||||
|
ActivityKind::Conflict => "dialog-warning-symbolic",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
-16
@@ -30,10 +30,10 @@ graph TD
|
|||||||
|
|
||||||
| Crate | Primary Role | Key Components | State Management |
|
| Crate | Primary Role | Key Components | State Management |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| [`pdfs-core`](file:///home/narl/dev/private/proton-drive-linux/crates/pdfs-core) | Core Infrastructure & Services | Cache Bookkeeping, Database migrations/schemas, IPC protocol payloads. | Holds the unified SQLite DB (`Db`) connection and the on-disk cache metadata (`ContentCache`). |
|
| [`pdfs-core`](../crates/pdfs-core) | Core Infrastructure & Services | Cache bookkeeping, database migrations/schemas, IPC protocol payloads, and shared search relevance scoring. | Holds the unified SQLite DB (`Db`) connection and the on-disk cache metadata (`ContentCache`). |
|
||||||
| [`pdfs-fuse`](file:///home/narl/dev/private/proton-drive-linux/crates/pdfs-fuse) | VFS Layer & Reconciliation | FUSE callbacks, background upload queue (`drain`), two-way sync runner. | Manages in-memory inode maps (`State`), active descriptors (`WriteHandle`), and background task threads. |
|
| [`pdfs-fuse`](../crates/pdfs-fuse) | VFS Layer & Reconciliation | FUSE callbacks, background upload queue (`drain`), two-way sync runner. | Manages in-memory inode maps (`State`), active descriptors (`WriteHandle`), and background task threads. |
|
||||||
| [`pdfs-cli`](file:///home/narl/dev/private/proton-drive-linux/crates/pdfs-cli) | Command Line Interface | Command routing, daemon launcher, IPC client wrapper. | Stateless; communicates with daemon over IPC control socket. |
|
| [`pdfs-cli`](../crates/pdfs-cli) | Command Line Interface | Command routing, daemon launcher, IPC client wrapper. | Stateless; communicates with daemon over IPC control socket. |
|
||||||
| [`pdfs-gui`](file:///home/narl/dev/private/proton-drive-linux/crates/pdfs-gui) | Graphical Interface | GTK Page timelines (My files, Shared, Shared with me, Computers, Photos, Activity, Trash, Settings). | Stateless; polls daemon for status and lists timelines via IPC socket. |
|
| [`pdfs-gui`](../crates/pdfs-gui) | Graphical Interface | GTK pages, tray, and the resident quick-search prompt. | Keeps UI state only; all durable state and Drive access remain behind the IPC socket. |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -127,8 +127,8 @@ Because Proton Drive does not support partial byte writes, modified files must b
|
|||||||
1. **Staging writes (`WriteHandle`):** Writes are stored locally in a `scratch` file. The daemon tracks modified regions using `Intervals` (which holds ranges of edited bytes).
|
1. **Staging writes (`WriteHandle`):** Writes are stored locally in a `scratch` file. The daemon tracks modified regions using `Intervals` (which holds ranges of edited bytes).
|
||||||
2. **Close/Release (`queue_revision`):** When the application closes the file descriptor, the daemon:
|
2. **Close/Release (`queue_revision`):** When the application closes the file descriptor, the daemon:
|
||||||
- Fetches any untouched gaps from the remote base file to compile the full file.
|
- Fetches any untouched gaps from the remote base file to compile the full file.
|
||||||
- Moves the scratch file to `staging` under a `{uid}-{millis}-{counter}` name, so a staged blob can be tied back to its node without consulting the database.
|
- Durably publishes the scratch data and authored-range sidecar into `staging` under a `{uid}-{millis}-{counter}` name. Temporary data and metadata are synced before atomic rename, and their directory is synced before the source is removed.
|
||||||
- Queues a pending database operation (`PendingOp`). For `OP_REVISION` ops, execution is debounced by 2 seconds (`DRAIN_REVISION_DEBOUNCE = 2s`) to give rapid follow-up writes (e.g. `aria2c` preallocation followed by writing) time to supersede the staged blob before network transmission.
|
- Transactionally queues or supersedes a pending database operation (`PendingOp`), so insertion failure cannot erase the previously acknowledged upload. For `OP_REVISION` ops, execution is debounced by 2 seconds (`DRAIN_REVISION_DEBOUNCE = 2s`) to give rapid follow-up writes (e.g. `aria2c` preallocation followed by writing) time to supersede the staged blob before network transmission.
|
||||||
3. **Async Drain Thread (`run_pending_drain`):** The background drain worker picks up the database operations queue, handles revision uploads, resolves conflicts, and cleans up staging files. Upon landing a revision upload (`refresh_after_upload`), it rebaselines both still-queued ops (`rebaseline_pending`) and open write handles targeting the same node to prevent false self-conflict copies on subsequent writes.
|
3. **Async Drain Thread (`run_pending_drain`):** The background drain worker picks up the database operations queue, handles revision uploads, resolves conflicts, and cleans up staging files. Upon landing a revision upload (`refresh_after_upload`), it rebaselines both still-queued ops (`rebaseline_pending`) and open write handles targeting the same node to prevent false self-conflict copies on subsequent writes.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
@@ -170,7 +170,7 @@ sequenceDiagram
|
|||||||
The sync engine handles offline-capable, bidirectional synchronization between the local disk and Proton Drive for directories marked in `mirror` mode.
|
The sync engine handles offline-capable, bidirectional synchronization between the local disk and Proton Drive for directories marked in `mirror` mode.
|
||||||
|
|
||||||
### Lifecycle of a Sync Pass
|
### Lifecycle of a Sync Pass
|
||||||
1. **Walk Local:** Walks the local directory tree recursively, scanning sizes and modification times.
|
1. **Walk Local:** Walks the local directory tree recursively, scanning sizes and modification times while carrying a completeness result. A `readdir`, metadata, permission, or transient I/O failure makes the pass non-destructive instead of turning omitted paths into deletions.
|
||||||
2. **Walk Remote:** Walks the remote database representation. If remote file modification times are updated, it calls the API to decrypt their sizes.
|
2. **Walk Remote:** Walks the remote database representation. If remote file modification times are updated, it calls the API to decrypt their sizes.
|
||||||
3. **Load Baseline:** Loads the `sync_entry` database table, which contains the snapshot of both sides during the *last successful sync*.
|
3. **Load Baseline:** Loads the `sync_entry` database table, which contains the snapshot of both sides during the *last successful sync*.
|
||||||
4. **Permutation Diffing:** The loop compares the three states (`local`, `remote`, `baseline`) to classify items:
|
4. **Permutation Diffing:** The loop compares the three states (`local`, `remote`, `baseline`) to classify items:
|
||||||
@@ -192,7 +192,7 @@ graph TD
|
|||||||
```
|
```
|
||||||
|
|
||||||
5. **Depth-Ascending Batching:** Folders are processed first to ensure hierarchies exist before files are placed. Work is executed concurrently up to a set limit.
|
5. **Depth-Ascending Batching:** Folders are processed first to ensure hierarchies exist before files are placed. Work is executed concurrently up to a set limit.
|
||||||
6. **Post-Sync Settle:** On success, baseline entries are upserted, timestamps updated, and any pending mode switches (e.g. going on-demand) are evaluated.
|
6. **Safety Gate and Settle:** Destructive plans are rejected when the scan is incomplete. Total-wipe protection applies to every non-empty baseline, including a single-entry folder. On success, baseline entries are upserted, timestamps updated, and any pending mode switches (e.g. going on-demand) are evaluated. Failed local deletion or conflict preservation retains the previous baseline and prevents remote content from overwriting the local source.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -202,11 +202,18 @@ The CLI and GUI front-ends do not access database files or make network calls di
|
|||||||
|
|
||||||
* **Transport:** IPC over Unix Stream Socket.
|
* **Transport:** IPC over Unix Stream Socket.
|
||||||
* **Framing:** Line-delimited JSON payloads.
|
* **Framing:** Line-delimited JSON payloads.
|
||||||
|
* **Resource limits:** Requests are limited to 1 MiB, must end with a newline, and must arrive within 10 seconds. At most 64 connection handlers are active at once.
|
||||||
* **Control Protocol:**
|
* **Control Protocol:**
|
||||||
* Client sends a single JSON line (`Request`).
|
* Client sends a single JSON line (`Request`).
|
||||||
* Daemon parses, handles the request, and replies with a single JSON line (`Response`).
|
* Daemon parses, handles the request, and replies with a single JSON line (`Response`).
|
||||||
* Timeout durations are separated: **2 seconds** for writes (avoids hangs on defunct sockets) and **120 seconds** for reads (accommodates heavy transfers).
|
* Timeout durations are separated: **2 seconds** for writes (avoids hangs on defunct sockets) and **120 seconds** for reads (accommodates heavy transfers).
|
||||||
|
|
||||||
|
### Unified Search
|
||||||
|
|
||||||
|
`SearchV2` is the shared search boundary for the resident prompt. One request carries the query, result limit, requested sources (`Drive`, `Local`), and content kind. The daemon queries Drive metadata and the local home-directory index, then applies [`pdfs_core::search::relevance_score`](../crates/pdfs-core/src/search.rs) to both result sets. Exact and prefix matches rank above substring, abbreviation, and bounded typo matches; all query terms must match either the basename or parent path. Returning scores on one scale lets the GUI merge both sources into a deterministic **Best matches** list.
|
||||||
|
|
||||||
|
The prompt is a single-instance GTK application. It retains and hides its window between activations, resets its query state when summoned, and ignores stale asynchronous replies. Folder and streamable audio/video results open through the FUSE mount so applications can issue range reads; ordinary Drive files use the daemon's materialize-then-open path.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. Subsystem Interaction & Thread Map
|
## 7. Subsystem Interaction & Thread Map
|
||||||
@@ -215,9 +222,10 @@ The background daemon relies on the following thread topology:
|
|||||||
|
|
||||||
1. **Main Thread / Dispatch Loop:** Blocks on `fuser::Session` loop. Reads kernel FUSE events and hands off network-bound VFS work to the FUSE workers pool.
|
1. **Main Thread / Dispatch Loop:** Blocks on `fuser::Session` loop. Reads kernel FUSE events and hands off network-bound VFS work to the FUSE workers pool.
|
||||||
2. **FUSE Workers Pool (11 threads, two lanes):** Bounded thread pool handling network operations. Split into 3 threads reserved for metadata (`lookup`, `readdir`) and 8 general threads that serve transfers (block reads) and fall back to metadata when no transfer is waiting. Reserved threads never accept a transfer — that is what keeps a directory listing from queuing behind saturated downloads (audit A6).
|
2. **FUSE Workers Pool (11 threads, two lanes):** Bounded thread pool handling network operations. Split into 3 threads reserved for metadata (`lookup`, `readdir`) and 8 general threads that serve transfers (block reads) and fall back to metadata when no transfer is waiting. Reserved threads never accept a transfer — that is what keeps a directory listing from queuing behind saturated downloads (audit A6).
|
||||||
3. **IPC listener Thread:** Listens on Unix socket connections, spawning a lightweight task per connection to serve front-end status/configuration requests.
|
3. **IPC listener Thread:** Listens on the Unix socket and admits at most 64 concurrent, timeout-bound handlers for front-end status/configuration requests.
|
||||||
4. **Sync Engine Loop Thread:** Serializes sync runs. Wakes on debounced local inotify filesystem changes, remote polling intervals, or manual user requests.
|
4. **Sync Engine Loop Thread:** Serializes sync runs. Wakes on debounced local inotify filesystem changes, remote polling intervals, or manual user requests.
|
||||||
5. **Drain Queue Worker Thread:** Processes staged writes (`PendingOp`) sequentially, uploading revisions and retrying with exponential backoff on failures.
|
5. **Drain Queue Worker Thread:** Processes staged writes (`PendingOp`) sequentially, uploading revisions and retrying with exponential backoff on failures.
|
||||||
|
6. **Conflict Sweep Thread (`pdfs-conflict-sweep`, optional):** Reconciles leftover `(sync-conflict …)` copies — 30 s warmup, then one pass every 5 minutes. A copy proven identical to its live sibling (equal size **and** equal `content_sha1`) is redundant; anything it cannot prove identical is surfaced to the activity feed and left alone. Governed by `AppConfig.conflict_sweep` / `PDFS_CONFLICT_SWEEP` (`SweepMode`): **report-only by default**, `off` skips the thread entirely, and only `enforce` lets it trash. Because it deletes, the enforcing path re-verifies revision id, size, digest, name, parent, queued ops and open handles immediately before acting rather than trusting its own pass-start snapshot. See `docs/BUGS.md` B69 and B71.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -261,13 +269,13 @@ Two exposures this client does **not** currently mitigate, stated plainly rather
|
|||||||
- **Swap.** Content keys, session keys, and decrypted buffers live in ordinary heap memory. Nothing calls `mlock(2)`, so under memory pressure they may be paged out. Raising `LimitMEMLOCK` in the systemd unit would *not* change this — there is no locking to permit. The effective mitigation is encrypted swap (dm-crypt / `systemd-cryptsetup`), which is standard on a LUKS install.
|
- **Swap.** Content keys, session keys, and decrypted buffers live in ordinary heap memory. Nothing calls `mlock(2)`, so under memory pressure they may be paged out. Raising `LimitMEMLOCK` in the systemd unit would *not* change this — there is no locking to permit. The effective mitigation is encrypted swap (dm-crypt / `systemd-cryptsetup`), which is standard on a LUKS install.
|
||||||
- **Kernel page cache.** Plaintext returned through FUSE is cached by the kernel like any other file data, and is likewise swappable. Defeating this would mean `direct_io` on every read, forfeiting the readahead and caching that make the mount usable. The trade is taken deliberately in favour of performance.
|
- **Kernel page cache.** Plaintext returned through FUSE is cached by the kernel like any other file data, and is likewise swappable. Defeating this would mean `direct_io` on every read, forfeiting the readahead and caching that make the mount usable. The trade is taken deliberately in favour of performance.
|
||||||
|
|
||||||
### 8.5 File modes: currently inherited, not enforced
|
### 8.5 Enforced ownership and file modes
|
||||||
|
|
||||||
**The daemon does not set restrictive permissions on anything it creates.** Cache and state directories come from `create_dir_all`, and `control.sock` from a plain `UnixListener::bind` — all subject to the process umask, which on a typical desktop yields `0755` directories and a `0755` socket.
|
On every start, `AppDirs::ensure` verifies that the state, cache, and configuration paths are real directories owned by the effective user, rejects symlink substitution or the wrong owner, and enforces mode `0700`. Both Unix sockets are changed to `0600` immediately after binding; failure is fatal rather than falling back to an unguarded daemon.
|
||||||
|
|
||||||
What protects them today is the mode of the XDG parents (`~/.cache`, `~/.local/state`), which are conventionally `0700` — but that is a property of the user's system, not something this client establishes or verifies. On a machine where the home directory is group- or world-traversable, the consequences differ by artifact:
|
These checks protect the artifacts even when the surrounding home or XDG parent is traversable:
|
||||||
|
|
||||||
| Artifact | Exposure if the parent is traversable |
|
| Artifact | Why access is restricted |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `content/` | Another local user can read cached plaintext file content |
|
| `content/` | Another local user can read cached plaintext file content |
|
||||||
| `cache.db` | Another local user can read the full decrypted name/structure index |
|
| `cache.db` | Another local user can read the full decrypted name/structure index |
|
||||||
@@ -275,14 +283,14 @@ What protects them today is the mode of the XDG parents (`~/.cache`, `~/.local/s
|
|||||||
|
|
||||||
The socket is the sharpest of the three, because it is an authority boundary rather than a data one — connecting to it confers the daemon's authenticated session without any credential.
|
The socket is the sharpest of the three, because it is an authority boundary rather than a data one — connecting to it confers the daemon's authenticated session without any credential.
|
||||||
|
|
||||||
**Fixed (bugs.md B6):** `AppDirs::ensure` now sets `0700` on the state, cache, and config directories on every start — not only at creation, so a directory that already exists with a permissive mode is tightened — and `config::restrict_socket` sets `0600` on both sockets immediately after `bind`. The directories are restricted *before* the socket is bound, which closes the window between `bind` and `chmod`. A control socket whose mode cannot be set takes the daemon down rather than serving unguarded.
|
Configuration publication follows the same fail-closed model: a restricted temporary file is written and synced, atomically renamed, and followed by a directory sync. A malformed existing configuration is reported and preserved rather than overwritten with defaults.
|
||||||
|
|
||||||
See [RECOVERY.md](RECOVERY.md) for what a lost machine means for the plaintext described in this section, and what to revoke.
|
See [RECOVERY.md](RECOVERY.md) for what a lost machine means for the plaintext described in this section, and what to revoke.
|
||||||
|
|
||||||
### 8.6 Implications for deployment
|
### 8.6 Implications for deployment
|
||||||
|
|
||||||
- Treat the cache and state directories as being as sensitive as the Drive contents themselves.
|
- Treat the cache and state directories as being as sensitive as the Drive contents themselves.
|
||||||
- Multi-user machines warrant checking `~/.cache` and `~/.local/state` are `0700` until §8.5 is addressed.
|
- Restored or manually copied profiles must remain owned by the user. The daemon refuses to start against wrong-owner or symlinked sensitive directories.
|
||||||
- Purging the cache (`pdfs` settings, or `PurgeCache` over IPC) removes content but **not** `cache.db`, and deliberately never removes undrained `staging/` or `recovery/` blobs.
|
- Purging the cache (`pdfs` settings, or `PurgeCache` over IPC) removes content but **not** `cache.db`, and deliberately never removes undrained `staging/` or `recovery/` blobs.
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -343,4 +351,3 @@ sequenceDiagram
|
|||||||
This ensures compatibility with all versions of Proton's client verification scripts.
|
This ensures compatibility with all versions of Proton's client verification scripts.
|
||||||
3. **SRP Handshake Reset:** Because a gated login burns the SRP handshake on the API side, the client cannot simply resume the previous request. Instead, `auth::login_interactive` restarts the SRP process from scratch with the verification credentials attached, keeping the complex handshake details isolated from the front-end.
|
3. **SRP Handshake Reset:** Because a gated login burns the SRP handshake on the API side, the client cannot simply resume the previous request. Instead, `auth::login_interactive` restarts the SRP process from scratch with the verification credentials attached, keeping the complex handshake details isolated from the front-end.
|
||||||
4. **CLI Fallback:** Since the CLI has no native web browser engine, hitting the CAPTCHA gate fails immediately with a user-friendly message directing the user to sign in once via the GUI (`pdfs-app`) to persist the authenticated session keys to the system keyring.
|
4. **CLI Fallback:** Since the CLI has no native web browser engine, hitting the CAPTCHA gate fails immediately with a user-friendly message directing the user to sign in once via the GUI (`pdfs-app`) to persist the authenticated session keys to the system keyring.
|
||||||
|
|
||||||
+1029
-12
File diff suppressed because it is too large.
Load diff
+29
-10
@@ -1,11 +1,11 @@
|
|||||||
# Pending Development & Verification Tasks
|
# Development and Verification Status for 1.0
|
||||||
|
|
||||||
This document compiles all pending, deferred, and unstarted tasks from the various roadmap, audit, and bug-tracking markdown files in the repository. Completed items have been filtered out.
|
This document summarizes the work still pending after the 1.0.0 release branch and records the most important recently completed changes. [`BUGS.md`](BUGS.md) is the authoritative issue ledger; [`TESTING.md`](TESTING.md) defines the acceptance commands.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 1. Core Feature Roadmap (Horizon Tasks)
|
## 1. Core Feature Roadmap (Horizon Tasks)
|
||||||
Derived from: [features.md](file:///home/narl/dev/private/proton-drive-linux/features.md) & [features-plan.md](file:///home/narl/dev/private/proton-drive-linux/features-plan.md)
|
Consolidated from the historical feature roadmap; current bug status lives in [`BUGS.md`](BUGS.md).
|
||||||
|
|
||||||
* **Local Cache & Metadata Encryption at Rest (Priority 5 / Horizon 1)**: Encrypt the SQLite database `cache.db` on disk using SQLCipher (`rusqlite` feature `bundled-sqlcipher`), and encrypt raw content cache blocks (`content/blocks/` and `content/scratch/`) using a fast symmetric scheme like AES-GCM or ChaCha20-Poly1305. The encryption key should be derived from the OS Keyring.
|
* **Local Cache & Metadata Encryption at Rest (Priority 5 / Horizon 1)**: Encrypt the SQLite database `cache.db` on disk using SQLCipher (`rusqlite` feature `bundled-sqlcipher`), and encrypt raw content cache blocks (`content/blocks/` and `content/scratch/`) using a fast symmetric scheme like AES-GCM or ChaCha20-Poly1305. The encryption key should be derived from the OS Keyring.
|
||||||
* **Active Bandwidth Throttling & Traffic Shaping (Horizon 1)**: Add speed governors for daemon uploads and downloads. Support configuration variables `max_upload_rate_kbps` and `max_download_rate_kbps` in `config.json` that can be dynamically adjusted over the Unix control socket.
|
* **Active Bandwidth Throttling & Traffic Shaping (Horizon 1)**: Add speed governors for daemon uploads and downloads. Support configuration variables `max_upload_rate_kbps` and `max_download_rate_kbps` in `config.json` that can be dynamically adjusted over the Unix control socket.
|
||||||
@@ -26,7 +26,7 @@ Derived from: [features.md](file:///home/narl/dev/private/proton-drive-linux/fea
|
|||||||
---
|
---
|
||||||
|
|
||||||
## 2. Performance, Correctness & Robustness Items
|
## 2. Performance, Correctness & Robustness Items
|
||||||
Derived from: [plan.md](file:///home/narl/dev/private/proton-drive-linux/plan.md), [improvements.md](file:///home/narl/dev/private/proton-drive-linux/improvements.md), & [photos-video-streaming.md](file:///home/narl/dev/private/proton-drive-linux/photos-video-streaming.md)
|
Consolidated from the performance and media-streaming audit notes.
|
||||||
|
|
||||||
* **B6. Debounce the LRU access touch**: Keep last-touch times in memory and flush them in batches (debounce window of 30-60 seconds) to avoid performing a SQLite `UPDATE` to the `cache_entries` table on every cache hit.
|
* **B6. Debounce the LRU access touch**: Keep last-touch times in memory and flush them in batches (debounce window of 30-60 seconds) to avoid performing a SQLite `UPDATE` to the `cache_entries` table on every cache hit.
|
||||||
* **E12. Client in-flight cap shape**: Address the lock discrepancy between `MAX_CONCURRENT_BLOCK_DOWNLOADS = 10` (per-file) and `DEFAULT_MAX_INFLIGHT_BLOCKS = 12` (global client-wide). Either raise the global cap, lower the per-file window, or dynamically calculate the window to prevent single files from starving concurrent transfers.
|
* **E12. Client in-flight cap shape**: Address the lock discrepancy between `MAX_CONCURRENT_BLOCK_DOWNLOADS = 10` (per-file) and `DEFAULT_MAX_INFLIGHT_BLOCKS = 12` (global client-wide). Either raise the global cap, lower the per-file window, or dynamically calculate the window to prevent single files from starving concurrent transfers.
|
||||||
@@ -40,13 +40,12 @@ Derived from: [plan.md](file:///home/narl/dev/private/proton-drive-linux/plan.md
|
|||||||
* **F3. Short block yields a short read / EOF**: Address the kernel EOF interpretation on short block reads. Either pad the block or fail loudly rather than silently serving short data.
|
* **F3. Short block yields a short read / EOF**: Address the kernel EOF interpretation on short block reads. Either pad the block or fail loudly rather than silently serving short data.
|
||||||
* **F4. `stream_readahead` check-then-spawn race**: Prevent duplicate concurrent readahead tasks for the same block. Check-then-spawn is currently racey.
|
* **F4. `stream_readahead` check-then-spawn race**: Prevent duplicate concurrent readahead tasks for the same block. Check-then-spawn is currently racey.
|
||||||
* **F5. `refresh_blocks` permit lock (SDK)**: Ensure URL refreshing (`refresh_blocks`) does not hold an in-flight block permit, preventing expired URLs from pinning client permits.
|
* **F5. `refresh_blocks` permit lock (SDK)**: Ensure URL refreshing (`refresh_blocks`) does not hold an in-flight block permit, preventing expired URLs from pinning client permits.
|
||||||
* **F6. Uncapped Unix control socket thread spawns**: Replace the unbounded `std::thread::spawn` per accepted socket connection in `run_control_socket` with a bounded pool or handler limits to prevent thread exhaustion.
|
|
||||||
* **F7. Document temporary cache file safety**: Add explanatory comment to `cache.rs:362` explaining why `with_extension("tmp")` is safe there but not in `store_thumbnail`.
|
* **F7. Document temporary cache file safety**: Add explanatory comment to `cache.rs:362` explaining why `with_extension("tmp")` is safe there but not in `store_thumbnail`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Open & Unverified Bugs
|
## 3. Open & Unverified Bugs
|
||||||
Derived from: [docs/bugs.md](file:///home/narl/dev/private/proton-drive-linux/docs/bugs.md)
|
Derived from: [`BUGS.md`](BUGS.md)
|
||||||
|
|
||||||
* **B2. Unattributed Trash Origin**: Investigate why deleted/moved files are occasionally sent to trash instead of vanishing on older rename operations.
|
* **B2. Unattributed Trash Origin**: Investigate why deleted/moved files are occasionally sent to trash instead of vanishing on older rename operations.
|
||||||
* **B10. GLib Critical Warning on HUD Close**: Assertion failure `g_list_store_remove: assertion '!g_sequence_iter_is_end (it)' failed` when prompt window closes during active launch (`xdg-open`). Needs investigation under `G_DEBUG=fatal-criticals`.
|
* **B10. GLib Critical Warning on HUD Close**: Assertion failure `g_list_store_remove: assertion '!g_sequence_iter_is_end (it)' failed` when prompt window closes during active launch (`xdg-open`). Needs investigation under `G_DEBUG=fatal-criticals`.
|
||||||
@@ -54,11 +53,17 @@ Derived from: [docs/bugs.md](file:///home/narl/dev/private/proton-drive-linux/do
|
|||||||
* **B8 (CAPTCHA sign-in bridge)**: Verify a real gated sign-in using Webkit CAPTCHA completion.
|
* **B8 (CAPTCHA sign-in bridge)**: Verify a real gated sign-in using Webkit CAPTCHA completion.
|
||||||
* **Draft Revision Upload loop**: Verify SDK-side deletion and retry logic when a conflicting draft revision exists under the same client UID.
|
* **Draft Revision Upload loop**: Verify SDK-side deletion and retry logic when a conflicting draft revision exists under the same client UID.
|
||||||
* **Optimistic Size Loss on Restart**: Verify that pending uploads retain their optimistic sizes across a daemon restart.
|
* **Optimistic Size Loss on Restart**: Verify that pending uploads retain their optimistic sizes across a daemon restart.
|
||||||
|
* **B48 — stale listing after unlink**: Session tombstones and authoritative empty listings are implemented; run focused live verification against eventually consistent remote enumeration.
|
||||||
|
* **B49–B55 — data-safety fixes**: Conflict preservation, orphan/staging durability, transactional supersession, one-entry wipe protection, and incomplete-scan handling have regression coverage but still require the fault, power-loss, and live mode-matrix cases recorded in `BUGS.md`.
|
||||||
|
* **B56–B60 — fail-closed state and IPC fixes**: Future schemas, event cursor advancement, socket limits, private-directory validation, and atomic config saves are implemented. Malformed-schema, database-full, connection-flood, wrong-owner, and concurrent/faulted-save verification remains.
|
||||||
|
* **B61 — credential rotation persistence**: Surface and retry keyring persistence failures so a rotated single-use refresh token cannot be lost on restart.
|
||||||
|
* **B62–B65 — release assurance**: Clean-install package verification, complete version/protocol identity, recovery sign-off, advisory/license policy, SBOM, provenance, checksums, and signatures remain.
|
||||||
|
* **B66–B68 — lifecycle and recovery**: Add queue flush/shutdown protection, explicit device adoption during fresh-state restore, and move profile backup below a writable remote folder.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. Testing, Verification & Disaster Recovery Drills
|
## 4. Testing, Verification & Disaster Recovery Drills
|
||||||
Derived from: [testing.md](file:///home/narl/dev/private/proton-drive-linux/testing.md) & [docs/RECOVERY.md](file:///home/narl/dev/private/proton-drive-linux/docs/RECOVERY.md)
|
Derived from: [`TESTING.md`](TESTING.md) and [`RECOVERY.md`](RECOVERY.md)
|
||||||
|
|
||||||
* **IPC Memory and Performance Stress Tests (Phase 3)**: Verify memory allocations do not balloon when sending large `UploadPhoto` JSON payloads. Test control socket timeout behaviors under long-running block operations.
|
* **IPC Memory and Performance Stress Tests (Phase 3)**: Verify memory allocations do not balloon when sending large `UploadPhoto` JSON payloads. Test control socket timeout behaviors under long-running block operations.
|
||||||
* **Fault Injection and Resiliency Tests (Phase 4)**:
|
* **Fault Injection and Resiliency Tests (Phase 4)**:
|
||||||
@@ -71,11 +76,13 @@ Derived from: [testing.md](file:///home/narl/dev/private/proton-drive-linux/test
|
|||||||
3. Wipe the local state directory and SQLite cache.
|
3. Wipe the local state directory and SQLite cache.
|
||||||
4. Restart the daemon.
|
4. Restart the daemon.
|
||||||
5. Run `pdfs sync restore` and assert the recovered filesystem matches the source byte-for-byte.
|
5. Run `pdfs sync restore` and assert the recovered filesystem matches the source byte-for-byte.
|
||||||
|
* **Managed live mode matrix**: Before release, run `scripts/fuse-acceptance.sh --managed-live <empty-a> <empty-b>` with `PDFS_ACCEPTANCE_ONLY` unset. It validates on-demand/on-demand, on-demand/mirror, mirror/on-demand, and mirror/mirror transitions.
|
||||||
|
* **Crash-consistency matrix**: Inject failure after scratch/sidecar sync, each staging rename, directory sync, pending-op insert/commit, and source deletion; prove that a complete discoverable copy and either the old or new queue record always survives.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Fixed & Resolved Bugs
|
## 5. Fixed & Resolved Bugs
|
||||||
Derived from: [docs/bugs.md](file:///home/narl/dev/private/proton-drive-linux/docs/bugs.md)
|
Derived from: [`BUGS.md`](BUGS.md)
|
||||||
|
|
||||||
### Completely Fixed & Verified
|
### Completely Fixed & Verified
|
||||||
* **B1 — FUSE rename loses the file (data loss)**: In `rename`, the node's database row was deleted while only updating the in-memory state, causing the file to vanish on next listing sync. Fixed by correctly calling `st.invalidate_listing(newparent)` to refresh SQLite cache states.
|
* **B1 — FUSE rename loses the file (data loss)**: In `rename`, the node's database row was deleted while only updating the in-memory state, causing the file to vanish on next listing sync. Fixed by correctly calling `st.invalidate_listing(newparent)` to refresh SQLite cache states.
|
||||||
@@ -88,6 +95,19 @@ Derived from: [docs/bugs.md](file:///home/narl/dev/private/proton-drive-linux/do
|
|||||||
* **B12 — cold enumeration is slow per entry, and goes superlinear past ~500**: S2K decryption and thumbnail resolution loops were serial. Fixed by parallelizing key derivations and caching metadata.
|
* **B12 — cold enumeration is slow per entry, and goes superlinear past ~500**: S2K decryption and thumbnail resolution loops were serial. Fixed by parallelizing key derivations and caching metadata.
|
||||||
* **B13 — `rename` over an existing destination fails instead of replacing it**: Fixed POSIX rename target overwrite behavior to replace existing destinations.
|
* **B13 — `rename` over an existing destination fails instead of replacing it**: Fixed POSIX rename target overwrite behavior to replace existing destinations.
|
||||||
* **B14 — provisional (ciphertext) sizes make rsync read short and abort the file**: Adjusted size reporting to align logical and ciphertext sizing, preventing premature EOF.
|
* **B14 — provisional (ciphertext) sizes make rsync read short and abort the file**: Adjusted size reporting to align logical and ciphertext sizing, preventing premature EOF.
|
||||||
|
* **B45 — truncate over a queued rewrite**: Path-based and handle-based truncate now compose over the complete pending blob; shrink, growth, and sparse I/O are live-verified.
|
||||||
|
* **B46 — combined cross-directory move and rename**: The desired end state is durably queued and reconciles partially completed remote operations; replacement and combined cases are live-verified.
|
||||||
|
* **B47 — overlong/invalid names**: All name-taking callbacks share a validator for `NAME_MAX`, reserved components, and UTF-8; managed-live verification passed.
|
||||||
|
|
||||||
|
### Implemented in 1.0 (Further Fault Verification Pending)
|
||||||
|
|
||||||
|
* **Unified search and resident prompt**: `SearchV2` combines Drive and local lookup with shared relevance scoring and content/source filters. The prompt reuses its window and opens streamable media through FUSE.
|
||||||
|
* **Durable write publication**: Scratch sidecars and staged blobs use synced temporary files, atomic renames, and directory synchronization; orphan rescue retains its source on failure.
|
||||||
|
* **Transactional pending operations**: Superseding a queued operation cannot delete the old durable record unless the replacement commits.
|
||||||
|
* **Safer mirror reconciliation**: Conflict-copy failure aborts publication, incomplete scans are non-destructive, and total-wipe protection includes one-entry baselines.
|
||||||
|
* **Bounded IPC**: Control requests are limited to 1 MiB and 10 seconds, with at most 64 active handlers.
|
||||||
|
* **Private local state**: Sensitive directories require correct ownership, real-directory identity, and `0700`; sockets use `0600`; config saves are atomic and malformed input is preserved.
|
||||||
|
* **Release gates**: Tag/workspace/package versions are checked and formatting, lint, tests, and offline FUSE acceptance run before packaging.
|
||||||
|
|
||||||
### Code Fixed / Implemented (Pending Verification)
|
### Code Fixed / Implemented (Pending Verification)
|
||||||
* **B8 — no way to complete a CAPTCHA, so a gated sign-in is unrecoverable**: Implemented a Webkit-based bridge to prompt the user to resolve interactive CAPTCHAs during gated logins. (Needs verification on a real gate request).
|
* **B8 — no way to complete a CAPTCHA, so a gated sign-in is unrecoverable**: Implemented a Webkit-based bridge to prompt the user to resolve interactive CAPTCHAs during gated logins. (Needs verification on a real gate request).
|
||||||
@@ -100,7 +120,7 @@ Derived from: [docs/bugs.md](file:///home/narl/dev/private/proton-drive-linux/do
|
|||||||
## 6. Recently Implemented Features
|
## 6. Recently Implemented Features
|
||||||
|
|
||||||
### Open-for-Write Deferral for Mirror Sync (Implemented)
|
### Open-for-Write Deferral for Mirror Sync (Implemented)
|
||||||
**File**: [`sync.rs`](file:///home/narl/dev/private/proton-drive-linux/crates/pdfs-fuse/src/sync.rs)
|
**File**: [`sync.rs`](../crates/pdfs-fuse/src/sync.rs)
|
||||||
|
|
||||||
The mirror folder sync engine now defers uploading any file that is currently held open for writing by another process, matching the guarantee the FUSE mount path already provides (where uploads are deferred until `close(fd)`).
|
The mirror folder sync engine now defers uploading any file that is currently held open for writing by another process, matching the guarantee the FUSE mount path already provides (where uploads are deferred until `close(fd)`).
|
||||||
|
|
||||||
@@ -127,4 +147,3 @@ The mirror folder sync engine now defers uploading any file that is currently he
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| FUSE mount | ✅ Perfect | Upload deferred until `close(fd)` |
|
| FUSE mount | ✅ Perfect | Upload deferred until `close(fd)` |
|
||||||
| Mirror folders | ✅ Perfect | `/proc/*/fd` scan per reconcile pass |
|
| Mirror folders | ✅ Perfect | `/proc/*/fd` scan per reconcile pass |
|
||||||
|
|
||||||
+135
-9
@@ -9,9 +9,24 @@ cargo test --workspace
|
|||||||
cargo clippy --workspace --all-targets -- -D warnings
|
cargo clippy --workspace --all-targets -- -D warnings
|
||||||
```
|
```
|
||||||
|
|
||||||
## Live FUSE acceptance suite
|
## Filesystem API and live FUSE acceptance suite
|
||||||
|
|
||||||
Kernel behavior and remote convergence need a real mount. The live suite runs
|
The account-free suite exercises the filesystem syscall contract against a
|
||||||
|
temporary local directory. This validates the runner and covers creation/open
|
||||||
|
flags, positioned and vectored I/O, truncation and sparse files, `mmap`,
|
||||||
|
`sendfile`, `copy_file_range`, durability barriers, namespace/error semantics,
|
||||||
|
`renameat2` flags, open-file lifetime, names and enumeration, readdir stability
|
||||||
|
under concurrent mutation, metadata updates, extended attributes, allocation and
|
||||||
|
punched holes, refusal semantics for unimplemented operations, concurrent I/O,
|
||||||
|
and real application workloads. It never reads application state or contacts
|
||||||
|
Proton:
|
||||||
|
|
||||||
|
```console
|
||||||
|
scripts/fuse-acceptance.sh --offline-only
|
||||||
|
```
|
||||||
|
|
||||||
|
Kernel/FUSE behavior and remote convergence need a real mount. Live testing is
|
||||||
|
explicitly opt-in and always runs the account-free contract first. It runs
|
||||||
destructive POSIX operations only below a fresh `pdfs-acceptance-*` directory.
|
destructive POSIX operations only below a fresh `pdfs-acceptance-*` directory.
|
||||||
It refuses paths that are not FUSE mounts and removes its directory on success,
|
It refuses paths that are not FUSE mounts and removes its directory on success,
|
||||||
failure, or interruption.
|
failure, or interruption.
|
||||||
@@ -19,16 +34,128 @@ failure, or interruption.
|
|||||||
Use a dedicated test account without irreplaceable data:
|
Use a dedicated test account without irreplaceable data:
|
||||||
|
|
||||||
```console
|
```console
|
||||||
scripts/fuse-acceptance.sh /mnt/on-demand-testmount
|
scripts/fuse-acceptance.sh --live /mnt/on-demand-testmount
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### The mount is diffed against an ordinary filesystem
|
||||||
|
|
||||||
|
The local reference run is not only a self-test of the runner. Every case
|
||||||
|
records the facts it established, and a live run is compared against those
|
||||||
|
recordings, so a case fails when the mount *differs* from an ordinary
|
||||||
|
filesystem — not only when it raises. The report names the exact key:
|
||||||
|
|
||||||
|
```
|
||||||
|
[DIVERGENCE] live FUSE /mnt/testmount does not match the reference filesystem:
|
||||||
|
! creation and open flags.truncated.size: live=6 reference=0
|
||||||
|
```
|
||||||
|
|
||||||
|
Facts are recorded in one of two ways, chosen in the test body:
|
||||||
|
|
||||||
|
- **compared** — a claim about filesystem semantics. The mount must agree.
|
||||||
|
- **noted** — context that legitimately differs: inode numbers, block counts,
|
||||||
|
reported mode bits, and which optional syscalls exist at all. Reported under
|
||||||
|
`[capabilities]`, never a failure.
|
||||||
|
|
||||||
|
Because the choice is made at each call site, the set of accepted divergences
|
||||||
|
is visible in the test that accepts it. There is no separate allowlist to drift
|
||||||
|
out of date. When adding a case, prefer `record`; reach for `note` only when an
|
||||||
|
ordinary filesystem can do something a network filesystem genuinely cannot.
|
||||||
|
|
||||||
|
### Operations the filesystem does not implement
|
||||||
|
|
||||||
|
`git`, `rsync`, `cp -a`, `tar`, `df` and most installers probe for symlinks,
|
||||||
|
hard links, FIFOs, device nodes, `statfs`, file locks, `copy_file_range`,
|
||||||
|
`O_DIRECT` and `SEEK_HOLE` whether or not a filesystem offers them. The suite
|
||||||
|
asserts that each either works or is *refused cleanly* — a recognised errno,
|
||||||
|
never `EIO` and never a hang. A refusal that is merely unimplemented is fine;
|
||||||
|
one that is dirty breaks software that never asked for the feature.
|
||||||
|
|
||||||
|
### Regressions from the bug ledger
|
||||||
|
|
||||||
|
Cases named `regression B<n>` reproduce a bug from `docs/BUGS.md` so it stays
|
||||||
|
fixed. B69 (identical rewrite must not fork a `(sync-conflict …)` copy), B70 (an
|
||||||
|
in-flight `.crdownload` must not be sealed as a revision) and B74 (a file
|
||||||
|
renamed after close must keep its content) need a real mount and a reachable
|
||||||
|
daemon; they are skipped, not failed, without one.
|
||||||
|
|
||||||
|
B74 is currently **failing** — it is a live data-loss defect, not a regression
|
||||||
|
guard, and it was found by exactly this mechanism: the B70 case failed, and
|
||||||
|
narrowing it produced a smaller reproduction that has its own case now. Expect a
|
||||||
|
red run against a real mount until it is fixed.
|
||||||
|
|
||||||
|
A regression case asserts against the *remote* outcome, so it must wait for the
|
||||||
|
queue to drain before reading. Reading straight back through the mount proves
|
||||||
|
nothing: the local attribute and content caches will happily return the bytes
|
||||||
|
that were just written even when nothing reached Drive, which is precisely how
|
||||||
|
B74 stayed invisible.
|
||||||
|
|
||||||
|
### Timeouts, reports, and hung mounts
|
||||||
|
|
||||||
|
A wedged FUSE operation cannot be interrupted from Python, so each case runs
|
||||||
|
under a soft alarm (`--timeout`, default 180s) that reports the case and lets
|
||||||
|
cleanup run, backed by a hard stop that dumps every thread's stack and exits.
|
||||||
|
That backstop is the only way to learn *where* a mount hung. A timeout ends the
|
||||||
|
run: later cases against a wedged mount produce noise, not information.
|
||||||
|
|
||||||
|
```console
|
||||||
|
scripts/fuse-acceptance.sh --live /mnt/testmount \
|
||||||
|
--timeout 300 --report-junit results.xml --budget 30
|
||||||
|
```
|
||||||
|
|
||||||
|
`--report-json` also writes every recorded observation, which is what to attach
|
||||||
|
to a bug report. `--budget SECONDS` flags cases that pass but got slower —
|
||||||
|
B5 was a 186 ms-per-call regression that a pass/fail suite could not see.
|
||||||
|
`--list` prints every case and which targets it runs against, and `--fail-fast`
|
||||||
|
stops at the first failure.
|
||||||
|
|
||||||
|
`--journal-check` fails the run if `proton-drive.service` logged any error while
|
||||||
|
it ran. A suite can pass every assertion while the daemon logs failures behind
|
||||||
|
it; that has happened, and it was caught only by reading the journal by hand.
|
||||||
|
|
||||||
|
Abandoned `pdfs-acceptance-*` roots from a crashed run are removed at startup,
|
||||||
|
but only if they match the exact generated name and are over an hour old
|
||||||
|
(`PDFS_ACCEPTANCE_REAP_AGE`), so a concurrent run is never disturbed.
|
||||||
|
|
||||||
|
### Automated mode matrix
|
||||||
|
|
||||||
|
The managed live runner accepts two existing, empty, unmounted directories. It
|
||||||
|
registers both as new sync folders, waits for initial synchronization, then
|
||||||
|
tests all four pairs in order: on-demand/on-demand, on-demand/mirror,
|
||||||
|
mirror/on-demand, and mirror/mirror. Every transition must become idle, and an
|
||||||
|
on-demand folder must actually appear as a mount before its contract runs.
|
||||||
|
|
||||||
|
```console
|
||||||
|
mkdir -p /mnt/pdfs-test-a /mnt/pdfs-test-b
|
||||||
|
scripts/fuse-acceptance.sh --managed-live /mnt/pdfs-test-a /mnt/pdfs-test-b
|
||||||
|
```
|
||||||
|
|
||||||
|
This creates remote folders and permanently deletes those test remotes during
|
||||||
|
cleanup. It refuses non-empty paths, existing mounts, and paths already present
|
||||||
|
in `pdfs sync list`. Use `PDFS_ACCEPTANCE_SYNC_TIMEOUT` to extend transition
|
||||||
|
timeouts and `PDFS_ACCEPTANCE_PDFS` to select a non-installed CLI binary, for
|
||||||
|
example `target/debug/pdfs`.
|
||||||
|
|
||||||
|
### Durability drill
|
||||||
|
|
||||||
|
`--durability` restarts `proton-drive.service` in the middle of the suite and
|
||||||
|
re-verifies the bytes written before it. `staging/` and `recovery/` can hold the
|
||||||
|
only copy of a file whose upload has not finished, so this is the case that
|
||||||
|
proves the queue is persistent rather than in-memory. It is opt-in because it
|
||||||
|
interrupts a running daemon; `PDFS_ACCEPTANCE_UNIT` selects a different unit.
|
||||||
|
|
||||||
|
### Selecting cases
|
||||||
|
|
||||||
|
During development, `PDFS_ACCEPTANCE_ONLY` runs tests whose descriptive name
|
||||||
|
contains the supplied text (for example `PDFS_ACCEPTANCE_ONLY=namespace`). A
|
||||||
|
release acceptance run must leave it unset so the complete contract executes.
|
||||||
|
|
||||||
Passing more paths runs the same filesystem suite independently against each.
|
Passing more paths runs the same filesystem suite independently against each.
|
||||||
The first path must be the FUSE mount under test; later paths may be another
|
The first path must be the FUSE mount under test; later paths may be another
|
||||||
FUSE mount or a normal local mirror filesystem:
|
FUSE mount or a normal local mirror filesystem:
|
||||||
|
|
||||||
```console
|
```console
|
||||||
PDFS_ACCEPTANCE_SYNC_TIMEOUT=180 \
|
PDFS_ACCEPTANCE_SYNC_TIMEOUT=180 \
|
||||||
scripts/fuse-acceptance.sh /mnt/on-demand-testmount /mnt/testmount
|
scripts/fuse-acceptance.sh --live /mnt/on-demand-testmount /mnt/testmount
|
||||||
```
|
```
|
||||||
|
|
||||||
If two paths really are views of the **same remote folder**, enable an
|
If two paths really are views of the **same remote folder**, enable an
|
||||||
@@ -36,16 +163,15 @@ additional byte-for-byte convergence check explicitly:
|
|||||||
|
|
||||||
```console
|
```console
|
||||||
PDFS_ACCEPTANCE_CONVERGENCE=1 \
|
PDFS_ACCEPTANCE_CONVERGENCE=1 \
|
||||||
scripts/fuse-acceptance.sh /mnt/first-view /mnt/second-view
|
scripts/fuse-acceptance.sh --live /mnt/first-view /mnt/second-view
|
||||||
```
|
```
|
||||||
|
|
||||||
Do not enable that flag merely because two sync folders belong to the same
|
Do not enable that flag merely because two sync folders belong to the same
|
||||||
account. Their configured remote UIDs must be identical.
|
account. Their configured remote UIDs must be identical.
|
||||||
|
|
||||||
The suite covers create/read/rewrite/truncate, multi-megabyte binary content,
|
Normally the live suite applies the same API contract independently to each
|
||||||
unaligned random writes, rename replacement, cross-directory moves, important
|
supplied mount. In convergence mode it exercises the primary once and verifies
|
||||||
POSIX error cases, open-then-unlink behavior, concurrent writers, directory
|
that the resulting bytes appear through every secondary view. A pass is
|
||||||
enumeration, and optional cross-mount byte-for-byte convergence. A pass is
|
|
||||||
strong evidence for those paths, not a blanket data-loss guarantee. Before a
|
strong evidence for those paths, not a blanket data-loss guarantee. Before a
|
||||||
release, also run the recovery drills in `docs/RECOVERY.md` and inspect
|
release, also run the recovery drills in `docs/RECOVERY.md` and inspect
|
||||||
`docs/BUGS.md` for live-verification items.
|
`docs/BUGS.md` for live-verification items.
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@
|
|||||||
# Build + install: cd packaging && makepkg -fi
|
# Build + install: cd packaging && makepkg -fi
|
||||||
|
|
||||||
pkgname=proton-drive-linux
|
pkgname=proton-drive-linux
|
||||||
pkgver=0.7.1
|
pkgver=1.1.0
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Proton Drive client for Linux: FUSE files-on-demand mount, CLI, GTK4 app + tray"
|
pkgdesc="Proton Drive client for Linux: FUSE files-on-demand mount, CLI, GTK4 app + tray"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
|
|||||||
Executable
+1945
File diff suppressed because it is too large.
Load diff
+66
-194
@@ -1,210 +1,82 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Destructive only inside a uniquely named directory created by this script.
|
# Filesystem syscall contract, with optional tests against mounted Drive views.
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
echo "usage: $0 MOUNTPOINT [MOUNTPOINT ...]" >&2
|
cat >&2 <<'EOF'
|
||||||
echo "example: $0 /mnt/on-demand-testmount /mnt/testmount" >&2
|
usage: scripts/fuse-acceptance.sh [--offline-only] [OPTION ...]
|
||||||
|
scripts/fuse-acceptance.sh --live MOUNTPOINT [MOUNTPOINT ...] [OPTION ...]
|
||||||
|
scripts/fuse-acceptance.sh --managed-live EMPTY_DIR EMPTY_DIR [OPTION ...]
|
||||||
|
scripts/fuse-acceptance.sh MOUNTPOINT [MOUNTPOINT ...] # compatibility
|
||||||
|
|
||||||
|
The local reference suite always runs first and requires no account; the mount
|
||||||
|
is then diffed against its recorded behaviour. --live runs the same contract on
|
||||||
|
each mount. Set PDFS_ACCEPTANCE_CONVERGENCE=1 only when all live mountpoints
|
||||||
|
show the same remote folder.
|
||||||
|
|
||||||
|
Options are forwarded to the Python runner. The useful ones:
|
||||||
|
--list print every case and the targets it runs against
|
||||||
|
--timeout SECONDS per-case limit (default 180); a hung case dumps stacks
|
||||||
|
--fail-fast stop at the first failure instead of continuing
|
||||||
|
--report-json PATH machine-readable results and recorded observations
|
||||||
|
--report-junit PATH JUnit XML for CI
|
||||||
|
--budget SECONDS report any case slower than this
|
||||||
|
--journal-check fail if the daemon logs errors during the run
|
||||||
|
--durability restart the daemon mid-suite and re-verify bytes
|
||||||
|
EOF
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
(( $# > 0 )) || usage
|
|
||||||
command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 2; }
|
command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 2; }
|
||||||
command -v findmnt >/dev/null || { echo "findmnt is required" >&2; exit 2; }
|
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
runner=(python3 -u "$script_dir/fuse-acceptance.py")
|
||||||
|
|
||||||
mounts=("$@")
|
mountpoints=()
|
||||||
run_id="pdfs-acceptance-$(date +%Y%m%dT%H%M%S)-$$"
|
passthrough=()
|
||||||
roots=()
|
# Paths come first and options after them, so the first dash-prefixed argument
|
||||||
|
# ends the path list and everything from there is forwarded untouched.
|
||||||
cleanup() {
|
collect_paths() {
|
||||||
local root
|
while (( $# > 0 )) && [[ "$1" != -* ]]; do
|
||||||
for root in "${roots[@]}"; do
|
mountpoints+=("$1")
|
||||||
[[ -n "$root" && "$root" == */pdfs-acceptance-* ]] || continue
|
shift
|
||||||
rm -rf -- "$root" 2>/dev/null || true
|
|
||||||
done
|
done
|
||||||
|
passthrough=("$@")
|
||||||
}
|
}
|
||||||
trap cleanup EXIT INT TERM
|
|
||||||
|
|
||||||
for index in "${!mounts[@]}"; do
|
case "${1:-}" in
|
||||||
mountpoint="${mounts[$index]}"
|
-h|--help) usage ;;
|
||||||
|
--offline-only|"")
|
||||||
|
shift || true
|
||||||
|
exec "${runner[@]}" "$@"
|
||||||
|
;;
|
||||||
|
--live)
|
||||||
|
shift
|
||||||
|
collect_paths "$@"
|
||||||
|
(( ${#mountpoints[@]} > 0 )) || usage
|
||||||
|
;;
|
||||||
|
--managed-live)
|
||||||
|
shift
|
||||||
|
(( $# >= 2 )) || usage
|
||||||
|
first="$1"; second="$2"; shift 2
|
||||||
|
exec "${runner[@]}" --managed-live "$first" "$second" "$@"
|
||||||
|
;;
|
||||||
|
--*)
|
||||||
|
# Bare options with no mode: an offline run with those options.
|
||||||
|
exec "${runner[@]}" "$@"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
collect_paths "$@"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
command -v findmnt >/dev/null || { echo "findmnt is required for --live" >&2; exit 2; }
|
||||||
|
for mountpoint in "${mountpoints[@]}"; do
|
||||||
[[ -d "$mountpoint" ]] || { echo "not a directory: $mountpoint" >&2; exit 2; }
|
[[ -d "$mountpoint" ]] || { echo "not a directory: $mountpoint" >&2; exit 2; }
|
||||||
fs_type="$(findmnt -T "$mountpoint" -n -o FSTYPE | head -n 1)"
|
done
|
||||||
if (( index == 0 )) && [[ "$fs_type" != fuse* ]]; then
|
fstype="$(findmnt -T "${mountpoints[0]}" -n -o FSTYPE | head -n 1)"
|
||||||
echo "refusing primary non-FUSE path $mountpoint (type: ${fs_type:-unknown})" >&2
|
[[ "$fstype" == fuse* ]] || {
|
||||||
|
echo "refusing primary non-FUSE path ${mountpoints[0]} (type: ${fstype:-unknown})" >&2
|
||||||
exit 2
|
exit 2
|
||||||
fi
|
|
||||||
roots+=("${mountpoint%/}/$run_id")
|
|
||||||
done
|
|
||||||
|
|
||||||
step() { printf '[test] %s\n' "$1"; }
|
|
||||||
assert_text() {
|
|
||||||
python3 - "$1" "$2" <<'PY'
|
|
||||||
import sys
|
|
||||||
p, expected = sys.argv[1], sys.argv[2].encode()
|
|
||||||
with open(p, "rb") as f: got = f.read()
|
|
||||||
if got != expected:
|
|
||||||
raise AssertionError(
|
|
||||||
f"{p}: expected {expected!r} ({len(expected)} bytes), "
|
|
||||||
f"got {got!r} ({len(got)} bytes)"
|
|
||||||
)
|
|
||||||
PY
|
|
||||||
}
|
}
|
||||||
|
|
||||||
run_suite() {
|
exec "${runner[@]}" --live "${mountpoints[@]}" "${passthrough[@]}"
|
||||||
local primary="$1"
|
|
||||||
printf '[mount] testing %s\n' "${primary%/*}"
|
|
||||||
mkdir -- "$primary"
|
|
||||||
|
|
||||||
step "create, read, overwrite, truncate"
|
|
||||||
printf 'alpha\n' > "$primary/plain.txt"
|
|
||||||
assert_text "$primary/plain.txt" $'alpha\n'
|
|
||||||
printf 'a much longer replacement\n' > "$primary/plain.txt"
|
|
||||||
truncate -s 4 "$primary/plain.txt"
|
|
||||||
assert_text "$primary/plain.txt" "a mu"
|
|
||||||
|
|
||||||
step "binary round-trip and random-access writes"
|
|
||||||
python3 - "$primary/binary.dat" <<'PY'
|
|
||||||
import hashlib, sys
|
|
||||||
p = sys.argv[1]
|
|
||||||
data = bytes(range(256)) * 16384
|
|
||||||
with open(p, "wb") as f: f.write(data)
|
|
||||||
with open(p, "r+b", buffering=0) as f:
|
|
||||||
f.seek(4093); f.write(b"boundary-write")
|
|
||||||
with open(p, "rb") as f: got = f.read()
|
|
||||||
expected = data[:4093] + b"boundary-write" + data[4107:]
|
|
||||||
assert got == expected
|
|
||||||
print(hashlib.sha256(got).hexdigest())
|
|
||||||
PY
|
|
||||||
|
|
||||||
step "media-style header and distant seek reads"
|
|
||||||
python3 - "$primary/seek-sample.mkv" <<'PY'
|
|
||||||
import os, sys
|
|
||||||
|
|
||||||
p = sys.argv[1]
|
|
||||||
size = 12 * 1024 * 1024 + 137
|
|
||||||
header = b"\x1aE\xdf\xa3pdfs-media-header"
|
|
||||||
middle = b"pdfs-middle-cue"
|
|
||||||
trailer = b"pdfs-media-trailer"
|
|
||||||
middle_at = 7 * 1024 * 1024 + 19
|
|
||||||
trailer_at = size - len(trailer)
|
|
||||||
|
|
||||||
# A player probes the container header, jumps to cues/index data, then commonly
|
|
||||||
# seeks near EOF. Write a sparse fixture so the acceptance test stays cheap,
|
|
||||||
# and issue separate unbuffered reads so FUSE receives genuinely distant ranges
|
|
||||||
# rather than one accidental sequential read.
|
|
||||||
with open(p, "wb", buffering=0) as f:
|
|
||||||
f.truncate(size)
|
|
||||||
f.seek(0); f.write(header)
|
|
||||||
f.seek(middle_at); f.write(middle)
|
|
||||||
f.seek(trailer_at); f.write(trailer)
|
|
||||||
|
|
||||||
with open(p, "rb", buffering=0) as f:
|
|
||||||
assert f.read(len(header)) == header
|
|
||||||
f.seek(middle_at); assert f.read(len(middle)) == middle
|
|
||||||
f.seek(trailer_at); assert f.read(len(trailer)) == trailer
|
|
||||||
assert f.read(1) == b""
|
|
||||||
f.seek(size + 4096); assert f.read(16) == b""
|
|
||||||
assert os.stat(p).st_size == size
|
|
||||||
PY
|
|
||||||
|
|
||||||
step "rename, move, and replacement semantics"
|
|
||||||
mkdir -- "$primary/dir-a" "$primary/dir-b"
|
|
||||||
printf 'source' > "$primary/source"
|
|
||||||
printf 'victim' > "$primary/victim"
|
|
||||||
mv -- "$primary/source" "$primary/victim"
|
|
||||||
assert_text "$primary/victim" "source"
|
|
||||||
[[ ! -e "$primary/source" ]]
|
|
||||||
mv -- "$primary/victim" "$primary/dir-a/moved"
|
|
||||||
assert_text "$primary/dir-a/moved" "source"
|
|
||||||
|
|
||||||
step "POSIX error behavior"
|
|
||||||
if rmdir -- "$primary/dir-a/moved" 2>/dev/null; then echo "rmdir removed a file" >&2; exit 1; fi
|
|
||||||
if rm -- "$primary/dir-a" 2>/dev/null; then echo "rm removed a directory" >&2; exit 1; fi
|
|
||||||
if mv -- "$primary/dir-a" "$primary/dir-a/child" 2>/dev/null; then
|
|
||||||
echo "moving a directory below itself succeeded" >&2; exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
step "open file remains readable after unlink"
|
|
||||||
python3 - "$primary/open-unlink" <<'PY'
|
|
||||||
import os, sys
|
|
||||||
p = sys.argv[1]; data = b"held-open\0" * 4096
|
|
||||||
with open(p, "wb") as f: f.write(data)
|
|
||||||
f = open(p, "rb", buffering=0); os.unlink(p)
|
|
||||||
assert not os.path.exists(p) and f.read() == data
|
|
||||||
f.close()
|
|
||||||
PY
|
|
||||||
|
|
||||||
step "concurrent independent writers"
|
|
||||||
for i in $(seq 1 12); do
|
|
||||||
(python3 - "$primary/concurrent-$i" "$i" <<'PY'
|
|
||||||
import sys
|
|
||||||
p, seed = sys.argv[1], int(sys.argv[2]); data = bytes([seed]) * (131072 + seed)
|
|
||||||
with open(p, "wb") as f: f.write(data)
|
|
||||||
with open(p, "rb") as f: assert f.read() == data
|
|
||||||
PY
|
|
||||||
) &
|
|
||||||
done
|
|
||||||
wait
|
|
||||||
|
|
||||||
step "directory enumeration agrees with stat"
|
|
||||||
python3 - "$primary" <<'PY'
|
|
||||||
import os, sys
|
|
||||||
for name in os.listdir(sys.argv[1]): os.lstat(os.path.join(sys.argv[1], name))
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
for root in "${roots[@]}"; do
|
|
||||||
run_suite "$root"
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "${PDFS_ACCEPTANCE_CONVERGENCE:-0}" == 1 ]] && (( ${#roots[@]} > 1 )); then
|
|
||||||
step "cross-mount convergence"
|
|
||||||
primary="${roots[0]}"
|
|
||||||
expected="$(sha256sum "$primary/binary.dat" | cut -d' ' -f1)"
|
|
||||||
for secondary in "${roots[@]:1}"; do
|
|
||||||
deadline=$((SECONDS + ${PDFS_ACCEPTANCE_SYNC_TIMEOUT:-120}))
|
|
||||||
while (( SECONDS < deadline )); do
|
|
||||||
if [[ -f "$secondary/binary.dat" ]] &&
|
|
||||||
[[ "$(sha256sum "$secondary/binary.dat" 2>/dev/null | cut -d' ' -f1)" == "$expected" ]]; then break; fi
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
[[ -f "$secondary/binary.dat" ]] || { echo "did not appear on $secondary" >&2; exit 1; }
|
|
||||||
[[ "$(sha256sum "$secondary/binary.dat" | cut -d' ' -f1)" == "$expected" ]] || {
|
|
||||||
echo "content mismatch on $secondary" >&2; exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# The media fixture may drain after binary.dat. Wait independently,
|
|
||||||
# then probe only its sparse ranges on the other mount. Unlike the
|
|
||||||
# same-mount check above, this exercises remote FUSE hydration and seek
|
|
||||||
# reads instead of serving the still-local staged file.
|
|
||||||
deadline=$((SECONDS + ${PDFS_ACCEPTANCE_SYNC_TIMEOUT:-120}))
|
|
||||||
while (( SECONDS < deadline )); do
|
|
||||||
[[ -f "$secondary/seek-sample.mkv" ]] &&
|
|
||||||
[[ "$(stat -c %s "$secondary/seek-sample.mkv" 2>/dev/null)" == 12583049 ]] && break
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
[[ -f "$secondary/seek-sample.mkv" ]] || {
|
|
||||||
echo "media fixture did not appear on $secondary" >&2; exit 1
|
|
||||||
}
|
|
||||||
python3 - "$secondary/seek-sample.mkv" <<'PY'
|
|
||||||
import os, sys
|
|
||||||
p = sys.argv[1]
|
|
||||||
size = 12 * 1024 * 1024 + 137
|
|
||||||
checks = [
|
|
||||||
(0, b"\x1aE\xdf\xa3pdfs-media-header"),
|
|
||||||
(7 * 1024 * 1024 + 19, b"pdfs-middle-cue"),
|
|
||||||
(size - len(b"pdfs-media-trailer"), b"pdfs-media-trailer"),
|
|
||||||
]
|
|
||||||
assert os.stat(p).st_size == size
|
|
||||||
with open(p, "rb", buffering=0) as f:
|
|
||||||
for offset, expected in checks:
|
|
||||||
f.seek(offset)
|
|
||||||
assert f.read(len(expected)) == expected
|
|
||||||
PY
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
step "cleanup"
|
|
||||||
cleanup
|
|
||||||
roots=()
|
|
||||||
echo "PASS: FUSE acceptance suite completed ($run_id)"
|
|
||||||
Reference in new issue
Block a user