mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-20 18:13:49 +01:00
Compare commits
270
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
233d9e9e51 | ||
|
|
26272cda6c | ||
|
|
370abd58dc | ||
|
|
24594b24ae | ||
|
|
04f0713012 | ||
|
|
cc020bfb54 | ||
|
|
1e6043d12a | ||
|
|
834684f6d7 | ||
|
|
1c2a66169f | ||
|
|
cb5ff8b8da | ||
|
|
c57d98799a | ||
|
|
7d9d5850a4 | ||
|
|
a69ebbdd10 | ||
|
|
ddf7335894 | ||
|
|
140bd2eee3 | ||
|
|
2462cb24bd | ||
|
|
a1dd2b5a9b | ||
|
|
e9d6d4a907 | ||
|
|
6fa91e83f1 | ||
|
|
f239dffbcc | ||
|
|
e011a0d89f | ||
|
|
f2c4c05f27 | ||
|
|
8598aff7dd | ||
|
|
38a8db0e3e | ||
|
|
db8c0cb382 | ||
|
|
a7baf2313c | ||
|
|
7e0d7d020e | ||
|
|
0df4b3c5f9 | ||
|
|
14b9e0fe64 | ||
|
|
64864562ef | ||
|
|
ce3bfa217d | ||
|
|
593df5c401 | ||
|
|
fd6b2d3e85 | ||
|
|
34902390f3 | ||
|
|
1dcf7c84ca | ||
|
|
a05772aa13 | ||
|
|
2f2fe96cdf | ||
|
|
2ec3b7ff62 | ||
|
|
38bd282d3a | ||
|
|
d773ba15d3 | ||
|
|
d93982baa9 | ||
|
|
3f5001b2d2 | ||
|
|
2308021aa2 | ||
|
|
602f1c9577 | ||
|
|
9132c7ebd5 | ||
|
|
94304ccc1a | ||
|
|
74cd6fd2a9 | ||
|
|
4b669168ee | ||
|
|
ba5b6a8bf2 | ||
|
|
3295eda1f0 | ||
|
|
41212efdf3 | ||
|
|
5f0b15fe70 | ||
|
|
033c8d3d71 | ||
|
|
5c184013f1 | ||
|
|
6be8a8f9bc | ||
|
|
7d445bb02c | ||
|
|
57acb0b849 | ||
|
|
68ff8c9e80 | ||
|
|
4552093c53 | ||
|
|
f2a9a0f64b | ||
|
|
e571f22d5d | ||
|
|
1c50797772 | ||
|
|
0183f17435 | ||
|
|
14a663cee9 | ||
|
|
3f45e89391 | ||
|
|
b49631e18f | ||
|
|
ad648ab5cc | ||
|
|
f25c2459a0 | ||
|
|
ce35a83a04 | ||
|
|
762d06765e | ||
|
|
2b95708260 | ||
|
|
c78bb529bc | ||
|
|
8d824af656 | ||
|
|
a0ab243be6 | ||
|
|
c77c548d7e | ||
|
|
2930b457d4 | ||
|
|
d9da0e2263 | ||
|
|
d4a14a17a8 | ||
|
|
5ca0272f15 | ||
|
|
800705ccb0 | ||
|
|
7ab14bb8f7 | ||
|
|
9c8b6680a7 | ||
|
|
bbd913b87d | ||
|
|
04f4610e4d | ||
|
|
0a5afd0b6a | ||
|
|
163c54a1e9 | ||
|
|
0d7c6e7bae | ||
|
|
10252fa84b | ||
|
|
bf9ec0862a | ||
|
|
d0d0c73a51 | ||
|
|
129ad1a50d | ||
|
|
506a610273 | ||
|
|
5048492a83 | ||
|
|
321547fb77 | ||
|
|
a06a8150fb | ||
|
|
c4bce50d35 | ||
|
|
d2b5916b99 | ||
|
|
4626ce17a7 | ||
|
|
630f316fe6 | ||
|
|
801681f747 | ||
|
|
e7382f2c7e | ||
|
|
7046f0f3ef | ||
|
|
37e8dae61b | ||
|
|
c52f2098f2 | ||
|
|
f4a2674b9c | ||
|
|
347861bbc1 | ||
|
|
c57763b07f | ||
|
|
846dfd8d14 | ||
|
|
2ae99de844 | ||
|
|
69ff216e7d | ||
|
|
b3ea564c6b | ||
|
|
463c97a14d | ||
|
|
40aa8016ea | ||
|
|
d1b7535696 | ||
|
|
ba414c4446 | ||
|
|
5b7f17bdaf | ||
|
|
73068f5868 | ||
|
|
b093010b64 | ||
|
|
ab8fe0e61d | ||
|
|
b8295373f7 | ||
|
|
5bc0b5c58a | ||
|
|
73af39ee65 | ||
|
|
48aaad23ce | ||
|
|
176112a9cb | ||
|
|
7957e1ed59 | ||
|
|
53543c91f9 | ||
|
|
fccb3d9561 | ||
|
|
6f2b4bb80a | ||
|
|
a617b2b410 | ||
|
|
bdbfc0bfd4 | ||
|
|
e9577f953f | ||
|
|
437d14b0f5 | ||
|
|
b40ca797d9 | ||
|
|
c64bb23e3c | ||
|
|
8d4a005aa2 | ||
|
|
9460c93b50 | ||
|
|
639ca359fa | ||
|
|
bd50ba5959 | ||
|
|
27201914ee | ||
|
|
2775f0681d | ||
|
|
97277d41dd | ||
|
|
d49738bf0b | ||
|
|
48945b9c97 | ||
|
|
b860ac3b49 | ||
|
|
42f3a18177 | ||
|
|
6f504d2e83 | ||
|
|
8c1c37a928 | ||
|
|
e4d2340bbf | ||
|
|
c1b72e455e | ||
|
|
2e3b7748e6 | ||
|
|
7f4f11ef18 | ||
|
|
cc9f7c6d8a | ||
|
|
9add185c1f | ||
|
|
0e7ffe1261 | ||
|
|
a5acae41b3 | ||
|
|
ea10a43216 | ||
|
|
917e7268f9 | ||
|
|
9089c8d13c | ||
|
|
071a11ef65 | ||
|
|
4edcff3f87 | ||
|
|
ee59e338b6 | ||
|
|
c1d9538669 | ||
|
|
a73ed34421 | ||
|
|
9c30ff8276 | ||
|
|
d1d4d23dd5 | ||
|
|
f717a692ff | ||
|
|
e065586d7e | ||
|
|
fd80c34152 | ||
|
|
e66308498b | ||
|
|
29b87ece72 | ||
|
|
fac8a144e5 | ||
|
|
eb25c16fe6 | ||
|
|
8f771fdcc4 | ||
|
|
c1911ea1de | ||
|
|
179b6555c4 | ||
|
|
e6971dd702 | ||
|
|
770d4fda38 | ||
|
|
11d03091a7 | ||
|
|
7a0f1ebd6f | ||
|
|
950056b48d | ||
|
|
05b84bf796 | ||
|
|
7cd516a164 | ||
|
|
c3c536892e | ||
|
|
cc9db51b78 | ||
|
|
7384d8e97d | ||
|
|
4bf176a631 | ||
|
|
aa0c60f5a2 | ||
|
|
47934b7ddb | ||
|
|
48c8910e7b | ||
|
|
6ec8043934 | ||
|
|
99707c48fa | ||
|
|
5557aa66d5 | ||
|
|
6456f503f1 | ||
|
|
4836c5a84a | ||
|
|
937a0a6824 | ||
|
|
fcbb15cb4b | ||
|
|
8aca7836f9 | ||
|
|
60a5a98174 | ||
|
|
51800c77d7 | ||
|
|
518763c5c9 | ||
|
|
ef0f046939 | ||
|
|
519ba0591a | ||
|
|
42b2f69fa0 | ||
|
|
0204f69e9e | ||
|
|
d4ef9116c8 | ||
|
|
f8c60c9753 | ||
|
|
607972cf1e | ||
|
|
96b70600f0 | ||
|
|
b0ff6ee441 | ||
|
|
bca22056b0 | ||
|
|
830cfd9faa | ||
|
|
77dac97afe | ||
|
|
2bea1f7672 | ||
|
|
0f279c7536 | ||
|
|
d6bc921a4b | ||
|
|
3221e4aa97 | ||
|
|
3d96cdbab1 | ||
|
|
18cc2ea498 | ||
|
|
a9cb389f04 | ||
|
|
923c47acb8 | ||
|
|
478a0e6cdd | ||
|
|
2633b4ed72 | ||
|
|
b46abfc07c | ||
|
|
fd78f088ec | ||
|
|
005f7ded1f | ||
|
|
3e4886f965 | ||
|
|
aac53e3139 | ||
|
|
c2bf87596f | ||
|
|
69d4feaa07 | ||
|
|
d481dcc15c | ||
|
|
e0442c56b9 | ||
|
|
ee76f4ac60 | ||
|
|
f6c651b06b | ||
|
|
8b7d7ae87f | ||
|
|
160626b29d | ||
|
|
f1f1f8ad4d | ||
|
|
bb28a18f49 | ||
|
|
a3c0c9ee9b | ||
|
|
91fa1614ed | ||
|
|
87361a4573 | ||
|
|
6d0b6ad553 | ||
|
|
170c3f01ff | ||
|
|
976ad2aa04 | ||
|
|
b73cd087aa | ||
|
|
e402981708 | ||
|
|
453c439b74 | ||
|
|
ceba0f5849 | ||
|
|
3e0db8cf46 | ||
|
|
2a86830e72 | ||
|
|
43031ecf99 | ||
|
|
2b6c37f543 | ||
|
|
3458e2e72d | ||
|
|
0ac5928138 | ||
|
|
beb3b2e4e4 | ||
|
|
bf43476e88 | ||
|
|
6363125fec | ||
|
|
51d114a8da | ||
|
|
31be27f30e | ||
|
|
4c498d559c | ||
|
|
7fd3439b60 | ||
|
|
c4847c4d52 | ||
|
|
ad49e32de9 | ||
|
|
19854b2d76 | ||
|
|
085335ca14 | ||
|
|
9fb7e84d41 | ||
|
|
7ebe8e5210 | ||
|
|
922399b9a2 | ||
|
|
3b80e9bc8f | ||
|
|
dc8eb27c99 | ||
|
|
d13e74fc12 |
No files matched your search
@@ -3,6 +3,9 @@ contact_links:
|
||||
- name: Security vulnerability
|
||||
url: https://github.com/narrrl/proton-drive-linux/security/advisories/new
|
||||
about: Report privately — never in a public issue.
|
||||
- name: Troubleshooting and FAQ
|
||||
url: https://github.com/narrrl/proton-drive-linux/wiki/Troubleshooting
|
||||
about: Fixes for common problems, and what to include in a bug report.
|
||||
- name: Known issues
|
||||
url: https://github.com/narrrl/proton-drive-linux/blob/main/docs/BUGS.md
|
||||
about: The authoritative ledger of tracked bugs and their status.
|
||||
|
||||
@@ -21,6 +21,7 @@ cargo test --workspace --locked
|
||||
- [ ] Touches recovery, staging, or anything that can be the only copy of user data
|
||||
- [ ] Changes packaging, the systemd unit, or desktop entries
|
||||
- [ ] Changes the control protocol (daemon and front ends updated together)
|
||||
- [ ] Adds or renames a page in `docs/` (listed in `SECTIONS` in `scripts/publish-wiki.sh`, which publishes it to the wiki)
|
||||
- [ ] None of the above
|
||||
|
||||
<!-- Screenshots for GTK changes. -->
|
||||
@@ -44,8 +44,11 @@ jobs:
|
||||
- name: Formatting
|
||||
run: cargo fmt --all -- --check
|
||||
|
||||
- name: Shell scripts of the release tarball
|
||||
run: shellcheck packaging/tarball/*.sh
|
||||
|
||||
- name: Lints
|
||||
run: cargo clippy --workspace --all-targets -- -D warnings
|
||||
run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
|
||||
- name: Unit tests
|
||||
run: cargo test --workspace --locked
|
||||
@@ -66,8 +69,9 @@ jobs:
|
||||
|
||||
# The seeded simulation runs (docs/TESTING.md): real FUSE
|
||||
# daemons against an in-memory Drive, so they need /dev/fuse but no account.
|
||||
# They run on pull requests too: most data-loss fixes are only exercised
|
||||
# here, and a change that breaks one should not reach main first.
|
||||
simulation:
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
|
||||
@@ -101,14 +105,17 @@ jobs:
|
||||
- name: Simulation runs
|
||||
run: 'cargo test -p pdfs-fuse --lib --locked sim:: -- --ignored --test-threads=1'
|
||||
|
||||
# A hung step writes every daemon thread's stack next to the run's
|
||||
# state; without them a hang that does not replay cannot be told apart.
|
||||
- name: Keep the stacks of a hung run
|
||||
# Retain the operation trace and daemon logs for the failing seed, plus
|
||||
# every daemon thread's stack if the watchdog caught a hang.
|
||||
- name: Keep simulation failure diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: simulation-stacks
|
||||
path: /tmp/pdfs-sim-*/stacks.txt
|
||||
name: simulation-diagnostics
|
||||
path: |
|
||||
/tmp/pdfs-sim-*/steps.log
|
||||
/tmp/pdfs-sim-*/daemon.log
|
||||
/tmp/pdfs-sim-*/stacks.txt
|
||||
if-no-files-found: ignore
|
||||
|
||||
# Keeps the release-profile dependencies cached on main, where the release
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
name: COPR
|
||||
|
||||
on:
|
||||
# Called by release.yml once the GitHub release and its assets exist: COPR
|
||||
# downloads the tag archive and the vendored crates from the release.
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
COPR_CONFIG:
|
||||
required: true
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Release to build, such as 3.5.0. It must already have its vendor tarball.'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: copr
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build in COPR
|
||||
runs-on: ubuntu-24.04
|
||||
container: fedora:latest
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
COPR_PROJECT: narl-rs/proton-drive-linux
|
||||
|
||||
steps:
|
||||
- name: Install packaging tools
|
||||
run: dnf -y install copr-cli rpm-build rpmdevtools
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Configure COPR access
|
||||
env:
|
||||
COPR_CONFIG: ${{ secrets.COPR_CONFIG }}
|
||||
run: |
|
||||
install -d -m700 ~/.config
|
||||
printf '%s\n' "$COPR_CONFIG" > ~/.config/copr
|
||||
chmod 600 ~/.config/copr
|
||||
|
||||
# COPR rebuilds the source package without our --define, so the version
|
||||
# has to be written into the spec itself.
|
||||
- name: Build the source package
|
||||
run: |
|
||||
version="${VERSION#v}"
|
||||
spec=packaging/proton-drive-linux.spec
|
||||
sed -i -E "s/^(%\{!\?version: %global version )[^}]*\}/\1${version}}/" "$spec"
|
||||
grep -qF "%global version ${version}}" "$spec"
|
||||
mkdir -p rpmbuild/SOURCES
|
||||
spectool -g -C rpmbuild/SOURCES "$spec"
|
||||
rpmbuild -bs --define "_topdir $PWD/rpmbuild" "$spec"
|
||||
|
||||
# Waits for the build, so a failed COPR build fails this job.
|
||||
- name: Submit to COPR
|
||||
run: copr-cli build "$COPR_PROJECT" rpmbuild/SRPMS/*.src.rpm
|
||||
@@ -0,0 +1,59 @@
|
||||
name: Nix
|
||||
|
||||
# Builds the flake (docs/INSTALL.md#nixos-flake) whenever its recipe, the lock
|
||||
# files or the code change, so a NixOS user's `nix build` keeps working. Run it
|
||||
# from the Actions tab for any branch to get a build to try.
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'flake.nix'
|
||||
- 'flake.lock'
|
||||
- 'packaging/nix/**'
|
||||
- 'Cargo.lock'
|
||||
- 'Cargo.toml'
|
||||
- 'crates/**'
|
||||
- 'po/**'
|
||||
- '.github/workflows/nix.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'flake.nix'
|
||||
- 'flake.lock'
|
||||
- 'packaging/nix/**'
|
||||
- 'Cargo.lock'
|
||||
- '.github/workflows/nix.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: nix-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: cachix/install-nix-action@v31
|
||||
|
||||
# Pulls from the project's binary cache, and pushes what this run built
|
||||
# when the token is there; a fork's pull request has none and only pulls.
|
||||
- uses: cachix/cachix-action@v17
|
||||
with:
|
||||
name: proton-drive-linux
|
||||
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
|
||||
|
||||
- name: Check the flake
|
||||
run: nix flake check --print-build-logs --no-build
|
||||
|
||||
- name: Build the package
|
||||
run: nix build --print-build-logs .#proton-drive-linux
|
||||
|
||||
- name: Smoke test
|
||||
run: |
|
||||
./result/bin/pdfs --version
|
||||
for bin in pdfs-app pdfs-tray pdfs-prompt; do test -x "result/bin/$bin"; done
|
||||
grep -q "ExecStart=$(readlink -f result)/bin/pdfs" result/lib/systemd/user/proton-drive.service
|
||||
ls result/share/locale/*/LC_MESSAGES/pdfs.mo
|
||||
@@ -52,8 +52,10 @@ jobs:
|
||||
tag_version="${GITHUB_REF_NAME#v}"
|
||||
manifest_version="$(sed -n 's/^version = "\([^"]*\)"/\1/p' Cargo.toml | head -n1)"
|
||||
pkgbuild_version="$(sed -n 's/^pkgver=//p' packaging/PKGBUILD | head -n1)"
|
||||
spec_version="$(sed -n 's/^%{!?version: %global version \([^}]*\)}/\1/p' packaging/proton-drive-linux.spec)"
|
||||
test "$tag_version" = "$manifest_version"
|
||||
test "$tag_version" = "$pkgbuild_version"
|
||||
test "$tag_version" = "$spec_version"
|
||||
|
||||
- name: Release quality gates
|
||||
run: |
|
||||
@@ -117,17 +119,22 @@ jobs:
|
||||
echo "VERSION=$VERSION" >> $GITHUB_ENV
|
||||
echo "Building version: $VERSION"
|
||||
|
||||
- name: Package raw binaries (.tar.gz)
|
||||
# Binaries plus everything around them, with install.sh for a home
|
||||
# directory install (immutable distributions, anything without a package).
|
||||
- name: Package the installable tarball (.tar.gz)
|
||||
run: |
|
||||
mkdir -p dist/binaries
|
||||
cp target/release/pdfs dist/binaries/
|
||||
cp target/release/pdfs-tray dist/binaries/
|
||||
cp target/release/pdfs-app dist/binaries/
|
||||
cp target/release/pdfs-prompt dist/binaries/
|
||||
po/build.sh dist/binaries/locale
|
||||
|
||||
# Create tarball
|
||||
tar -czf proton-drive-linux-${{ env.VERSION }}-x86_64.tar.gz -C dist/binaries .
|
||||
tarball=$(packaging/tarball/build.sh target/release "${{ env.VERSION }}" .)
|
||||
packaging/tarball/test.sh "$tarball"
|
||||
|
||||
# Every crate the build needs, so COPR can build the spec without network.
|
||||
- name: Package vendored crates (.tar.xz)
|
||||
run: |
|
||||
mkdir -p dist/.cargo
|
||||
(
|
||||
cd dist
|
||||
cargo vendor --locked --manifest-path ../Cargo.toml vendor > .cargo/config.toml
|
||||
)
|
||||
tar -cJf proton-drive-linux-vendor-${{ env.VERSION }}.tar.xz -C dist vendor .cargo/config.toml
|
||||
|
||||
- name: Package Debian package (.deb)
|
||||
run: |
|
||||
@@ -149,9 +156,14 @@ jobs:
|
||||
cp packaging/io.narl.proton-drive-linux.desktop deb/usr/share/applications/
|
||||
cp packaging/io.narl.proton-drive-linux-tray.desktop deb/etc/xdg/autostart/
|
||||
cp packaging/io.narl.proton-drive-linux.svg deb/usr/share/icons/hicolor/scalable/apps/
|
||||
install -Dm644 packaging/io.narl.proton-drive-linux.policy deb/usr/share/polkit-1/actions/io.narl.proton-drive-linux.policy
|
||||
cp packaging/proton-drive.service deb/usr/lib/systemd/user/
|
||||
cp LICENSE deb/usr/share/licenses/proton-drive-linux/
|
||||
po/build.sh deb/usr/share/locale
|
||||
packaging/completions.sh target/release/pdfs deb/usr/share
|
||||
|
||||
# Stops the running daemon before removal, so the mount stops mirroring deletes
|
||||
install -m755 packaging/deb/prerm deb/DEBIAN/prerm
|
||||
|
||||
# Calculate Installed-Size in KB for the control file
|
||||
INSTALLED_SIZE=$(du -sk deb | cut -f1)
|
||||
@@ -164,7 +176,7 @@ jobs:
|
||||
Priority: optional
|
||||
Architecture: amd64
|
||||
Maintainer: Nils Pukropp <contact@narl.io>
|
||||
Depends: fuse3, libgtk-4-1 (>= 4.12.0), libadwaita-1-0 (>= 1.5.0), libsecret-1-0, libwebkitgtk-6.0-4, libimage-exiftool-perl
|
||||
Depends: polkitd, fuse3, libgtk-4-1 (>= 4.12.0), libadwaita-1-0 (>= 1.5.0), libsecret-1-0, libwebkitgtk-6.0-4, libimage-exiftool-perl
|
||||
Installed-Size: ${INSTALLED_SIZE}
|
||||
Homepage: https://proton-drive.narl.io
|
||||
Description: Unofficial Proton Drive client for Linux
|
||||
@@ -227,27 +239,75 @@ jobs:
|
||||
path: |
|
||||
*.deb
|
||||
*.tar.gz
|
||||
*.tar.xz
|
||||
*.rpm
|
||||
if-no-files-found: error
|
||||
|
||||
# The flake is the Nix release: a tag is what `github:narrrl/proton-drive-linux/vX.Y.Z`
|
||||
# pins, so it has to build before the release is announced.
|
||||
nix:
|
||||
name: Build the Nix package
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: cachix/install-nix-action@v31
|
||||
|
||||
# Pulls from the project's binary cache, and pushes what this run built
|
||||
# when the token is there; a fork's pull request has none and only pulls.
|
||||
- uses: cachix/cachix-action@v17
|
||||
with:
|
||||
name: proton-drive-linux
|
||||
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
|
||||
|
||||
- name: Build the flake
|
||||
run: nix build --print-build-logs .#proton-drive-linux
|
||||
|
||||
- name: Check the version
|
||||
run: |
|
||||
built="$(./result/bin/pdfs --version)"
|
||||
echo "$built"
|
||||
case "$GITHUB_REF" in
|
||||
refs/tags/v*) grep -qF "${GITHUB_REF_NAME#v}" <<< "$built" ;;
|
||||
esac
|
||||
|
||||
publish:
|
||||
name: Create GitHub Release
|
||||
needs: [checks, build-release]
|
||||
needs: [checks, build-release, nix]
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Download packages
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: proton-drive-linux-builds
|
||||
|
||||
# The release notes are this version's section of the changelog, from its
|
||||
# heading up to the next one.
|
||||
- name: Extract release notes
|
||||
run: |
|
||||
version="${GITHUB_REF_NAME#v}"
|
||||
awk -v heading="## [$version]" \
|
||||
'index($0, heading) == 1 { found = 1; print; next }
|
||||
found && /^## \[/ { exit }
|
||||
found' docs/CHANGELOG.md > release-notes.md
|
||||
if [ ! -s release-notes.md ]; then
|
||||
echo "::error::docs/CHANGELOG.md has no section for $version"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
body_path: release-notes.md
|
||||
files: |
|
||||
*.deb
|
||||
*.tar.gz
|
||||
*.tar.xz
|
||||
*.rpm
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -263,3 +323,21 @@ jobs:
|
||||
version: ${{ github.ref_name }}
|
||||
secrets:
|
||||
AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
|
||||
|
||||
copr:
|
||||
name: Build COPR packages
|
||||
needs: publish
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: ./.github/workflows/copr.yml
|
||||
with:
|
||||
version: ${{ github.ref_name }}
|
||||
secrets:
|
||||
COPR_CONFIG: ${{ secrets.COPR_CONFIG }}
|
||||
|
||||
wiki:
|
||||
name: Publish the wiki
|
||||
needs: publish
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: ./.github/workflows/wiki.yml
|
||||
with:
|
||||
ref: ${{ github.ref_name }}
|
||||
@@ -0,0 +1,77 @@
|
||||
name: Wiki
|
||||
|
||||
# docs/ is the source of the GitHub wiki (docs/DEVELOPMENT.md#common-changes).
|
||||
# Every change to the documentation checks its links; release.yml calls this
|
||||
# once the release is published, so the wiki documents the latest release. Run
|
||||
# it from the Actions tab to publish main in between.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- 'images/**'
|
||||
- 'README.md'
|
||||
- 'CONTRIBUTING.md'
|
||||
- 'SECURITY.md'
|
||||
- 'scripts/publish-wiki.sh'
|
||||
- '.github/workflows/wiki.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- 'images/**'
|
||||
- 'README.md'
|
||||
- 'CONTRIBUTING.md'
|
||||
- 'SECURITY.md'
|
||||
- 'scripts/publish-wiki.sh'
|
||||
- '.github/workflows/wiki.yml'
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Tag or branch to publish'
|
||||
required: true
|
||||
type: string
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Tag or branch to publish'
|
||||
required: false
|
||||
default: 'main'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check documentation links
|
||||
if: inputs.ref == ''
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Check links
|
||||
run: scripts/publish-wiki.sh --check "$RUNNER_TEMP"
|
||||
|
||||
publish:
|
||||
name: Publish to the wiki
|
||||
# Set only by release.yml and the Actions tab, so pushes and pull requests
|
||||
# never publish.
|
||||
if: inputs.ref != ''
|
||||
runs-on: ubuntu-24.04
|
||||
concurrency:
|
||||
group: wiki-publish
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Publish
|
||||
env:
|
||||
WIKI_REMOTE: https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.wiki.git
|
||||
run: |
|
||||
git config --global user.name 'github-actions[bot]'
|
||||
git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
scripts/publish-wiki.sh --push "$RUNNER_TEMP"
|
||||
@@ -3,3 +3,5 @@
|
||||
*.pdb
|
||||
.env
|
||||
/rpmbuild
|
||||
/result
|
||||
/result-*
|
||||
+8
-6
@@ -6,10 +6,10 @@ deliberately high — everything else is ordinary Rust review.
|
||||
|
||||
## Before you start
|
||||
|
||||
- Read [`AGENTS.md`](AGENTS.md) for repository conventions,
|
||||
[`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md) for building, running and debugging from source,
|
||||
and [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for how the daemon, front ends, and SDK fit
|
||||
together. [`docs/README.md`](docs/README.md) indexes the rest.
|
||||
- Read [`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md) for building, running and debugging from
|
||||
source, and [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for how the daemon, front ends, and
|
||||
SDK fit together. [`docs/README.md`](docs/README.md) indexes the rest. The same pages are on
|
||||
the [wiki](https://github.com/narrrl/proton-drive-linux/wiki).
|
||||
- Check [`docs/BUGS.md`](docs/BUGS.md) — it is the authoritative issue ledger, and your problem may
|
||||
already be tracked there with context.
|
||||
- For anything larger than a fix, open an issue first so the design can be discussed before you
|
||||
@@ -58,8 +58,10 @@ runs are destructive: use a dedicated test account, never one holding data you c
|
||||
catalog in `po/LINGUAS` in the same change. See [`docs/TRANSLATING.md`](docs/TRANSLATING.md).
|
||||
The CLI and daemon messages stay in English.
|
||||
- **Document what users see.** A new command, setting or behaviour belongs in the matching page
|
||||
under `docs/` (user guide, CLI reference or configuration) and in the `[Unreleased]` section of
|
||||
[`docs/CHANGELOG.md`](docs/CHANGELOG.md).
|
||||
under `docs/` (a user guide topic, the CLI reference or configuration) and in the `[Unreleased]`
|
||||
section of [`docs/CHANGELOG.md`](docs/CHANGELOG.md). Each release publishes `docs/` to the
|
||||
wiki. A new page goes into `SECTIONS` in `scripts/publish-wiki.sh`, and
|
||||
`scripts/publish-wiki.sh --check` must pass. CI runs it on every change to `docs/`.
|
||||
|
||||
## Commits and pull requests
|
||||
|
||||
|
||||
Generated
+628
-425
File diff suppressed because it is too large.
Load diff
+37
-6
@@ -3,7 +3,7 @@ resolver = "2"
|
||||
members = ["crates/pdfs-core", "crates/pdfs-fuse", "crates/pdfs-cli", "crates/pdfs-gui"]
|
||||
|
||||
[workspace.package]
|
||||
version = "3.1.2"
|
||||
version = "3.9.0"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
authors = ["Nils Pukropp <contact@narl.io>"]
|
||||
@@ -15,8 +15,11 @@ keywords = ["proton", "drive", "fuse", "sync", "linux"]
|
||||
categories = ["filesystem", "command-line-utilities"]
|
||||
|
||||
[workspace.dependencies]
|
||||
proton-sdk = { version = "0.8.1" }
|
||||
proton-drive-rs = { version = "0.8.1" }
|
||||
proton-sdk = { version = "0.12.1" }
|
||||
proton-drive-rs = { version = "0.12.1" }
|
||||
proton-gallery-app-rs = "0.1.0"
|
||||
proton-gallery-rs = "0.1.1"
|
||||
chrono = { version = "0.4", default-features = false, features = ["std"] }
|
||||
|
||||
# Internal crates.
|
||||
pdfs-core = { path = "crates/pdfs-core" }
|
||||
@@ -33,15 +36,17 @@ anyhow = "1"
|
||||
# wedge every later acquisition of it: `Workers` catches the unwind to keep the
|
||||
# pool alive, and `std::sync::Mutex` would defeat that by poisoning the state
|
||||
# the recovered worker needs.
|
||||
parking_lot = "0.12"
|
||||
parking_lot = "0.13"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
directories = "6"
|
||||
ignore = "0.4"
|
||||
# Command lines typed into Preferences, split and shown the way a shell would.
|
||||
shell-words = "1"
|
||||
sha2 = "0.11"
|
||||
# Google Takeout exports are zip sets; the importer reads their central
|
||||
# directories and streams entries out without extracting to disk first.
|
||||
zip = { version = "8", default-features = false, features = ["deflate"] }
|
||||
zip = { version = "9", default-features = false, features = ["deflate"] }
|
||||
# Proton's photo duplicate detection is keyed on the plaintext SHA-1, so an
|
||||
# import has to speak SHA-1 whatever its age.
|
||||
sha1 = "0.11"
|
||||
@@ -49,15 +54,41 @@ sha1 = "0.11"
|
||||
# formats a camera or phone actually produces — the rest of the crate's codecs
|
||||
# would be dead weight in the daemon.
|
||||
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "tiff", "bmp", "rayon"] }
|
||||
rusqlite = { version = "0.40", features = ["bundled"] }
|
||||
# Cover pictures of songs, read from their tags (ID3, FLAC, Ogg, MP4, APE) for
|
||||
# the song's thumbnail. Its default feature is required: 0.25 does not build
|
||||
# without it.
|
||||
lofty = "0.25"
|
||||
rusqlite = { version = "0.41", features = ["bundled"] }
|
||||
# keyring 4 split into `keyring-core` plus per-platform stores. The libdbus
|
||||
# Secret Service store is the one keyring 3's `sync-secret-service` wrapped, and
|
||||
# with no `target` it matches items v3 wrote, so existing logins survive.
|
||||
keyring-core = "1"
|
||||
# The keyring store all three apps share: reconnects after a Secret Service
|
||||
# restart (B285) and replaces items instead of editing them (B291).
|
||||
proton-auth-rs = "0.1"
|
||||
proton-base-rs = "0.1"
|
||||
dbus-secret-service-keyring-store = { version = "1", features = ["crypto-rust"] }
|
||||
# The same Secret Service client the store wraps, used directly to replace an
|
||||
# item instead of editing it, which oo7 before 0.7.0 kept only in memory (B291).
|
||||
dbus-secret-service = { version = "4.1", features = ["crypto-rust"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
# `unstable-dynamic`: Tab calls `pdfs` back, so arguments complete from the
|
||||
# running service. Only `pdfs-cli/src/complete.rs` uses it.
|
||||
clap_complete = { version = "4.6", features = ["unstable-dynamic"] }
|
||||
rpassword = "7"
|
||||
async-trait = "0.1"
|
||||
futures = "0.3"
|
||||
fuser = "0.18"
|
||||
libc = "0.2"
|
||||
|
||||
[profile.release]
|
||||
# No package ships a separate debug package (the PKGBUILD sets `!debug`, the
|
||||
# RPM spec sets `debug_package` to nil), and the unstripped binaries weighed
|
||||
# 43 MB (`pdfs`), 24 MB (`pdfs-app`) and 12 MB (`pdfs-tray`).
|
||||
strip = "symbols"
|
||||
# Off on purpose: LTO has broken GTK links for this workspace, which is why
|
||||
# the packages turn it off as well.
|
||||
lto = false
|
||||
# `Workers` and the thumbnail build catch panics to keep the daemon alive, so
|
||||
# unwinding must stay on.
|
||||
panic = "unwind"
|
||||
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
|
||||
<img src="images/logo.svg" alt="" width="112" height="112">
|
||||
<a href="images/logo.svg"><img src="images/logo.svg" alt="Proton Drive for Linux icon" width="128" height="128"></a>
|
||||
|
||||
# Proton Drive for Linux
|
||||
|
||||
@@ -73,15 +73,31 @@ scriptable CLI.
|
||||
|
||||
| Distribution | Package |
|
||||
|---|---|
|
||||
| Debian, Ubuntu 24.04 or newer | `.deb` from the [latest release](https://github.com/narrrl/proton-drive-linux/releases/latest) |
|
||||
| Fedora 44 or newer | `.rpm` from the [latest release](https://github.com/narrrl/proton-drive-linux/releases/latest) |
|
||||
| Fedora 44 or newer | [`narl-rs/proton-drive-linux`](https://copr.fedorainfracloud.org/coprs/narl-rs/proton-drive-linux/) in COPR, updated with `dnf upgrade` |
|
||||
| Arch Linux | [`proton-drive-for-linux`](https://aur.archlinux.org/packages/proton-drive-for-linux) (also `-bin` and `-git`) in the AUR |
|
||||
| Anything else | [Build from source](docs/INSTALL.md#build-from-source) |
|
||||
| Debian, Ubuntu 24.04 or newer | `.deb` from the [latest release](https://github.com/narrrl/proton-drive-linux/releases/latest) |
|
||||
| NixOS | The flake in this repository: see [Installation → NixOS](docs/INSTALL.md#nixos-flake) |
|
||||
| Fedora Atomic (Silverblue, Kinoite, Bazzite, Bluefin) | The COPR repository, layered with `rpm-ostree`: see [Installation → Fedora Atomic](docs/INSTALL.md#fedora-atomic-silverblue-kinoite-bazzite-bluefin) |
|
||||
| Anything else, immutable ones included | `.tar.gz` from the [latest release](https://github.com/narrrl/proton-drive-linux/releases/latest), installed into your home folder by its `install.sh`: see [Installation → Other distributions](docs/INSTALL.md#other-distributions), or [build from source](docs/INSTALL.md#build-from-source) |
|
||||
|
||||
**Fedora**
|
||||
|
||||
```bash
|
||||
sudo apt install ./proton-drive-linux_*.deb # Debian, Ubuntu
|
||||
sudo dnf install ./proton-drive-linux-*.rpm # Fedora
|
||||
paru -S proton-drive-for-linux # Arch
|
||||
sudo dnf copr enable narl-rs/proton-drive-linux
|
||||
sudo dnf install proton-drive-linux
|
||||
```
|
||||
|
||||
**Arch Linux**
|
||||
|
||||
```bash
|
||||
paru -S proton-drive-for-linux
|
||||
```
|
||||
|
||||
**Debian and Ubuntu:** download the `.deb` from the
|
||||
[latest release](https://github.com/narrrl/proton-drive-linux/releases/latest), then:
|
||||
|
||||
```bash
|
||||
sudo apt install ./proton-drive-linux_*_amd64.deb
|
||||
```
|
||||
|
||||
Then open **Proton Drive** from your applications menu and sign in. The app starts the background
|
||||
@@ -96,18 +112,25 @@ systemctl --user enable --now proton-drive.service
|
||||
pdfs status
|
||||
```
|
||||
|
||||
[docs/INSTALL.md](docs/INSTALL.md) has the requirements, optional extras (tray on GNOME, RAW and
|
||||
video thumbnails), headless use and uninstalling.
|
||||
New to the client? [Getting started](docs/GETTING_STARTED.md) walks through the first steps.
|
||||
[Installation](docs/INSTALL.md) has the optional extras (tray on GNOME, RAW and video
|
||||
thumbnails), updating, headless use and uninstalling.
|
||||
|
||||
## Documentation
|
||||
|
||||
The documentation is in [`docs/`](docs/README.md) and on the
|
||||
[wiki](https://github.com/narrrl/proton-drive-linux/wiki).
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| [Getting started](docs/GETTING_STARTED.md) | From installing to your first synced folder |
|
||||
| [User guide](docs/USER_GUIDE.md) | Everything the app, tray and launcher do |
|
||||
| [Command-line reference](docs/CLI.md) | Every `pdfs` command and scripting with `--json` |
|
||||
| [FAQ](docs/FAQ.md) | Short answers to common questions |
|
||||
| [Command-line reference](docs/CLI.md) | Every `pdfs` command, Tab completion and scripting with `--json` |
|
||||
| [Configuration](docs/CONFIGURATION.md) | `config.json`, file locations, environment variables, the service |
|
||||
| [Troubleshooting](docs/TROUBLESHOOTING.md) | Diagnosing problems and fixing common ones |
|
||||
| [Recovering on a new computer](docs/RECOVERY.md) | Getting your synced folders back after a reinstall or a lost machine |
|
||||
| [Security model](docs/SECURITY_MODEL.md) | What protects your files, what stays on this computer, what the client connects to |
|
||||
| [Changelog](docs/CHANGELOG.md) | What changed in each release |
|
||||
|
||||
For contributors: [development guide](docs/DEVELOPMENT.md), [architecture](docs/ARCHITECTURE.md),
|
||||
@@ -149,7 +172,8 @@ keys before it leaves the machine, through a Rust implementation of Proton's Dri
|
||||
|
||||
Files you open are decrypted into a local cache so programs can read them. What that means for
|
||||
the security of the machine is spelled out in the
|
||||
[threat model](docs/ARCHITECTURE.md#10-threat-model-what-this-client-writes-to-disk-in-plaintext).
|
||||
[security model](docs/SECURITY_MODEL.md#what-is-kept-on-this-computer), and in full detail in
|
||||
the [threat model](docs/ARCHITECTURE.md#10-threat-model-what-this-client-writes-to-disk-in-plaintext).
|
||||
|
||||
## Contributing
|
||||
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
|
||||
Fixes land on the latest release. Please reproduce on the most recent version before reporting.
|
||||
|
||||
For what the client protects and what it does not, written for users, see the
|
||||
[security model](docs/SECURITY_MODEL.md).
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
**Do not open a public issue.** Report privately through GitHub's
|
||||
|
||||
@@ -21,6 +21,7 @@ anyhow.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
clap.workspace = true
|
||||
clap_complete.workspace = true
|
||||
rpassword.workspace = true
|
||||
serde_json.workspace = true
|
||||
zeroize.workspace = true
|
||||
@@ -0,0 +1,731 @@
|
||||
//! Shell completion for `pdfs`.
|
||||
//!
|
||||
//! Built on clap_complete's dynamic engine: the script a shell loads is a thin
|
||||
//! stub that calls `pdfs` back on every Tab with the words typed so far, and
|
||||
//! `pdfs` answers from its own clap definition. That lets an argument complete
|
||||
//! from the running service as well as from the command tree — `pdfs sync rm
|
||||
//! <Tab>` offers the synced-folder ids with their paths, `pdfs pin Doc<Tab>`
|
||||
//! walks My files.
|
||||
//!
|
||||
//! Every value that comes from the service is best effort. With no service, a
|
||||
//! slow one or an error, a completer offers nothing: a Tab that hangs or prints
|
||||
//! an error into the prompt is worse than one that does nothing.
|
||||
//!
|
||||
//! The engine sits behind clap_complete's `unstable-dynamic` feature, so all of
|
||||
//! this crate's use of it is kept to this module.
|
||||
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use clap::CommandFactory;
|
||||
use clap_complete::engine::{CompletionCandidate, PathCompleter, ValueCompleter};
|
||||
use clap_complete::env::{Bash, CompleteEnv, Elvish, EnvCompleter, Fish, Powershell, Zsh};
|
||||
use pdfs_core::control::{
|
||||
AlbumInfo, BookmarkInfo, ConflictInfo, DeviceInfo, DirEntry, InvitationInfo, PendingOpInfo,
|
||||
PublicLinkInfo, Request, Response, RevisionInfo, ShareEntry, ShareEntryKind, SyncFolderInfo,
|
||||
};
|
||||
|
||||
/// The environment variable that turns a run of `pdfs` into a completion
|
||||
/// request. Not clap's generic `COMPLETE`: the same binary is the systemd
|
||||
/// daemon, and an unrelated `COMPLETE` in its environment must not stop it.
|
||||
pub const VAR: &str = "PDFS_COMPLETE";
|
||||
|
||||
/// How long a completer waits for the service. A Tab is interactive, so a
|
||||
/// listing the service cannot give soon is better skipped. Long enough for the
|
||||
/// listings that ask Proton (computers take over a second).
|
||||
const SERVICE_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// How long a listing that asks Proton is reused. Every Tab is a new `pdfs`, so
|
||||
/// without it `pdfs devices rm <Tab>`, a letter and another Tab would wait for
|
||||
/// Proton twice. A uid that went away in the meantime only fails the command.
|
||||
const CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Answer a completion request and exit, if this run is one. Must run before
|
||||
/// anything writes to stdout.
|
||||
pub fn handle_request() {
|
||||
CompleteEnv::with_factory(crate::Cli::command)
|
||||
.var(VAR)
|
||||
.complete();
|
||||
}
|
||||
|
||||
/// The shells `pdfs completions` writes a script for.
|
||||
#[derive(Clone, Copy, clap::ValueEnum)]
|
||||
pub enum ShellArg {
|
||||
Bash,
|
||||
Zsh,
|
||||
Fish,
|
||||
Elvish,
|
||||
Powershell,
|
||||
}
|
||||
|
||||
impl ShellArg {
|
||||
fn completer(self) -> &'static dyn EnvCompleter {
|
||||
match self {
|
||||
ShellArg::Bash => &Bash,
|
||||
ShellArg::Zsh => &Zsh,
|
||||
ShellArg::Fish => &Fish,
|
||||
ShellArg::Elvish => &Elvish,
|
||||
ShellArg::Powershell => &Powershell,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the script that registers `pdfs` completion with `shell`.
|
||||
///
|
||||
/// The script calls `pdfs` from `PATH`, not this binary's path, so one written
|
||||
/// at packaging time keeps working once installed.
|
||||
pub fn write_script(shell: ShellArg, out: &mut dyn std::io::Write) -> std::io::Result<()> {
|
||||
shell
|
||||
.completer()
|
||||
.write_registration(VAR, "pdfs", "pdfs", "pdfs", out)?;
|
||||
if let ShellArg::Zsh = shell {
|
||||
// Installed as `_pdfs` in site-functions, the script is autoloaded as
|
||||
// that function's body on the first Tab. The body above only defines
|
||||
// and registers the completer, so this line also runs it, or the first
|
||||
// Tab would complete nothing. Sourced, `funcstack` names no `_pdfs`.
|
||||
writeln!(
|
||||
out,
|
||||
"if [ \"$funcstack[1]\" = \"_pdfs\" ]; then\n _clap_dynamic_completer_pdfs \"$@\"\nfi"
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Talking to the service
|
||||
|
||||
/// One request to the service, or `None` if there is no answer worth showing.
|
||||
fn ask(request: Request) -> Option<Response> {
|
||||
let socket = pdfs_core::config::AppDirs::new().ok()?.control_socket();
|
||||
ask_at(&socket, &request)
|
||||
}
|
||||
|
||||
fn ask_at(socket: &Path, request: &Request) -> Option<Response> {
|
||||
match pdfs_core::control::send_with_timeout(socket, request, SERVICE_TIMEOUT) {
|
||||
Ok(Response::Error { .. }) | Err(_) => None,
|
||||
Ok(response) => Some(response),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`ask`] for a listing that comes from Proton rather than the local
|
||||
/// database, reusing an answer younger than [`CACHE_TTL`]. `key` names the
|
||||
/// listing's cache file.
|
||||
fn ask_cached(key: &str, request: Request) -> Option<Response> {
|
||||
let file = cache_file(key);
|
||||
if let Some(hit) = file.as_deref().and_then(read_fresh) {
|
||||
return Some(hit);
|
||||
}
|
||||
let response = ask(request)?;
|
||||
if let Some(file) = &file {
|
||||
write_cache(file, &response);
|
||||
}
|
||||
Some(response)
|
||||
}
|
||||
|
||||
/// In `$XDG_RUNTIME_DIR`: private to the user, in memory and gone at logout,
|
||||
/// which suits names from the account. With no runtime dir, nothing is cached.
|
||||
fn cache_file(key: &str) -> Option<PathBuf> {
|
||||
let runtime = PathBuf::from(std::env::var_os("XDG_RUNTIME_DIR")?);
|
||||
Some(
|
||||
runtime
|
||||
.join("proton-drive-linux")
|
||||
.join(format!("completion-{key}.json")),
|
||||
)
|
||||
}
|
||||
|
||||
fn read_fresh(file: &Path) -> Option<Response> {
|
||||
let age = std::fs::metadata(file)
|
||||
.ok()?
|
||||
.modified()
|
||||
.ok()?
|
||||
.elapsed()
|
||||
.ok()?;
|
||||
if age > CACHE_TTL {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_slice(&std::fs::read(file).ok()?).ok()
|
||||
}
|
||||
|
||||
/// Best effort: a cache that cannot be written only costs the next Tab a wait.
|
||||
fn write_cache(file: &Path, response: &Response) {
|
||||
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
|
||||
|
||||
let (Some(dir), Ok(json)) = (file.parent(), serde_json::to_vec(response)) else {
|
||||
return;
|
||||
};
|
||||
let _ = std::fs::DirBuilder::new()
|
||||
.recursive(true)
|
||||
.mode(0o700)
|
||||
.create(dir);
|
||||
let staging = file.with_extension(format!("{}.tmp", std::process::id()));
|
||||
let written = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(&staging)
|
||||
.and_then(|mut f| std::io::Write::write_all(&mut f, &json));
|
||||
if written.is_err() || std::fs::rename(&staging, file).is_err() {
|
||||
let _ = std::fs::remove_file(&staging);
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate(value: impl Into<OsString>, help: impl Into<String>) -> CompletionCandidate {
|
||||
let help = help.into();
|
||||
let candidate = CompletionCandidate::new(value);
|
||||
if help.is_empty() {
|
||||
candidate
|
||||
} else {
|
||||
candidate.help(Some(help.into()))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Paths in Proton Drive
|
||||
|
||||
/// A path in the mount: an absolute path completes from the local filesystem
|
||||
/// (through the mount), anything else from the service's listing of My files,
|
||||
/// which is what the commands resolve a relative path against.
|
||||
pub fn drive_path(current: &OsStr) -> Vec<CompletionCandidate> {
|
||||
complete_drive(current, false)
|
||||
}
|
||||
|
||||
/// [`drive_path`] for an argument that must be a folder.
|
||||
pub fn drive_dir(current: &OsStr) -> Vec<CompletionCandidate> {
|
||||
complete_drive(current, true)
|
||||
}
|
||||
|
||||
/// `pdfs refresh`: a folder, or one of the two named caches.
|
||||
pub fn refresh_target(current: &OsStr) -> Vec<CompletionCandidate> {
|
||||
let typed = current.to_string_lossy();
|
||||
let mut candidates: Vec<_> = [
|
||||
("trash", "the trash listing"),
|
||||
("photos", "the photos timeline"),
|
||||
]
|
||||
.into_iter()
|
||||
.filter(|(name, _)| name.starts_with(typed.as_ref()))
|
||||
.map(|(name, help)| candidate(name, help))
|
||||
.collect();
|
||||
candidates.extend(drive_dir(current));
|
||||
candidates
|
||||
}
|
||||
|
||||
fn complete_drive(current: &OsStr, dirs_only: bool) -> Vec<CompletionCandidate> {
|
||||
let Some(word) = current.to_str() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if word.starts_with('/') || word.starts_with('~') {
|
||||
let local = if dirs_only {
|
||||
PathCompleter::dir()
|
||||
} else {
|
||||
PathCompleter::any()
|
||||
};
|
||||
return local.complete(current);
|
||||
}
|
||||
let (parent, prefix) = split_drive_word(word);
|
||||
let listing = Request::ListDir {
|
||||
path: parent.trim_end_matches('/').to_owned(),
|
||||
};
|
||||
match ask(listing) {
|
||||
Some(Response::Entries { entries }) => {
|
||||
drive_candidates(parent, prefix, &entries, dirs_only)
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a typed drive path into the folder to list (with its trailing `/`)
|
||||
/// and the start of the name being completed in it.
|
||||
fn split_drive_word(word: &str) -> (&str, &str) {
|
||||
match word.rfind('/') {
|
||||
Some(at) => word.split_at(at + 1),
|
||||
None => ("", word),
|
||||
}
|
||||
}
|
||||
|
||||
/// The entries of `parent` whose name starts with `prefix`. A folder ends in
|
||||
/// `/`, which tells every shell not to add a space after it, so the next Tab
|
||||
/// goes on into it.
|
||||
fn drive_candidates(
|
||||
parent: &str,
|
||||
prefix: &str,
|
||||
entries: &[DirEntry],
|
||||
dirs_only: bool,
|
||||
) -> Vec<CompletionCandidate> {
|
||||
entries
|
||||
.iter()
|
||||
.filter(|e| e.name.starts_with(prefix) && (e.is_dir || !dirs_only))
|
||||
.map(|e| {
|
||||
let slash = if e.is_dir { "/" } else { "" };
|
||||
CompletionCandidate::new(format!("{parent}{}{slash}", e.name))
|
||||
.hide(e.name.starts_with('.') && !prefix.starts_with('.'))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ids and uids from the service's listings
|
||||
|
||||
pub fn sync_folder_id() -> Vec<CompletionCandidate> {
|
||||
match ask(Request::ListSyncFolders) {
|
||||
Some(Response::SyncFolders { items }) => sync_folder_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sync_folder_candidates(items: &[SyncFolderInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|f| candidate(f.id.to_string(), format!("{} ({})", f.local_path, f.mode)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn queue_id() -> Vec<CompletionCandidate> {
|
||||
match ask(Request::ListPendingOps) {
|
||||
Some(Response::PendingOps { items }) => queue_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn queue_candidates(items: &[PendingOpInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|op| candidate(op.id.to_string(), format!("{} {}", op.kind, op.path)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn device_uid() -> Vec<CompletionCandidate> {
|
||||
match ask_cached("devices", Request::ListDevices) {
|
||||
Some(Response::Devices { items }) => device_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn device_candidates(items: &[DeviceInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|d| {
|
||||
let this = if d.this_device {
|
||||
" (this computer)"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
candidate(d.uid.as_str(), format!("{}{this}", d.name))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn trash_uid() -> Vec<CompletionCandidate> {
|
||||
match ask_cached("trash", Request::ListTrash) {
|
||||
Some(Response::Entries { entries }) => trash_candidates(&entries),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn trash_candidates(entries: &[DirEntry]) -> Vec<CompletionCandidate> {
|
||||
entries
|
||||
.iter()
|
||||
.map(|e| {
|
||||
let help = match e.trashed_from.as_deref() {
|
||||
Some(from) if !from.is_empty() => format!("{} (from {from})", e.name),
|
||||
_ => e.name.clone(),
|
||||
};
|
||||
candidate(e.uid.as_str(), help)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn album_uid() -> Vec<CompletionCandidate> {
|
||||
match ask_cached("albums", Request::PhotoAlbums) {
|
||||
Some(Response::Albums { items, .. }) => album_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn album_candidates(items: &[AlbumInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|a| candidate(a.uid.as_str(), a.name.as_str()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn shared_uid() -> Vec<CompletionCandidate> {
|
||||
match ask_cached("shared-with-me", Request::ListSharedWithMe) {
|
||||
Some(Response::Entries { entries }) => shared_candidates(&entries),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn shared_candidates(entries: &[DirEntry]) -> Vec<CompletionCandidate> {
|
||||
entries
|
||||
.iter()
|
||||
.map(|e| {
|
||||
let help = if e.shared_by.is_empty() {
|
||||
e.name.clone()
|
||||
} else {
|
||||
format!("{} (from {})", e.name, e.shared_by)
|
||||
};
|
||||
candidate(e.uid.as_str(), help)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn invitation_id() -> Vec<CompletionCandidate> {
|
||||
match ask_cached("invitations", Request::ListInvitations) {
|
||||
Some(Response::Invitations { items }) => invitation_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn invitation_candidates(items: &[InvitationInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|i| {
|
||||
let help = match i.name.as_deref() {
|
||||
Some(name) => format!("{name} from {}", i.inviter_email),
|
||||
None => format!("from {}", i.inviter_email),
|
||||
};
|
||||
candidate(i.id.as_str(), help)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn bookmark_token() -> Vec<CompletionCandidate> {
|
||||
match ask_cached("bookmarks", Request::ListBookmarks) {
|
||||
Some(Response::Bookmarks { items }) => bookmark_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn bookmark_candidates(items: &[BookmarkInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|b| candidate(b.token.as_str(), b.name.as_deref().unwrap_or(&b.url)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn conflict_path() -> Vec<CompletionCandidate> {
|
||||
match ask(Request::ListConflicts) {
|
||||
Some(Response::Conflicts { items }) => conflict_candidates(&items),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn conflict_candidates(items: &[ConflictInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|c| candidate(c.path.as_str(), format!("copy of {}", c.original_path)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ids that belong to the path typed before them
|
||||
|
||||
/// A version of the file named earlier on the line (`pdfs versions restore
|
||||
/// <path> <Tab>`).
|
||||
pub fn version_id(current: &OsStr) -> Vec<CompletionCandidate> {
|
||||
let Some(path) = typed_path() else {
|
||||
return Vec::new();
|
||||
};
|
||||
match ask(Request::ListRevisions { path }) {
|
||||
Some(Response::Revisions { items }) => starting_with(current, revision_candidates(&items)),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn revision_candidates(items: &[RevisionInfo]) -> Vec<CompletionCandidate> {
|
||||
items
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let current = if r.is_active { ", current" } else { "" };
|
||||
let size = r.claimed_size.unwrap_or(r.size_on_storage).max(0);
|
||||
candidate(
|
||||
r.id.as_str(),
|
||||
format!(
|
||||
"{}, {}{current}",
|
||||
crate::format_epoch_utc(r.created),
|
||||
crate::human_bytes(size as u64)
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A member or invitation on the share of the path typed earlier on the line.
|
||||
pub fn share_entry_id(current: &OsStr) -> Vec<CompletionCandidate> {
|
||||
match share_of_typed_path() {
|
||||
Some((entries, _)) => starting_with(current, share_entry_candidates(&entries)),
|
||||
None => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn share_entry_candidates(entries: &[ShareEntry]) -> Vec<CompletionCandidate> {
|
||||
entries
|
||||
.iter()
|
||||
.map(|e| {
|
||||
// The kind is the next argument, so the help names it in the words
|
||||
// that argument takes.
|
||||
let kind = match e.kind {
|
||||
ShareEntryKind::Member => "member",
|
||||
ShareEntryKind::ProtonInvite => "proton",
|
||||
ShareEntryKind::ExternalInvite => "external",
|
||||
};
|
||||
candidate(e.id.as_str(), format!("{} ({}, {kind})", e.email, e.role))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The public link of the path typed earlier on the line.
|
||||
pub fn public_link_id(current: &OsStr) -> Vec<CompletionCandidate> {
|
||||
match share_of_typed_path() {
|
||||
Some((_, Some(link))) => starting_with(current, vec![public_link_candidate(&link)]),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Described without its URL: the fragment is the link's password, and a Tab
|
||||
/// menu is on screen for anyone watching.
|
||||
fn public_link_candidate(link: &PublicLinkInfo) -> CompletionCandidate {
|
||||
let help = match link.expires {
|
||||
Some(expires) => format!(
|
||||
"{} link, expires {}",
|
||||
link.role,
|
||||
crate::format_epoch_utc(expires)
|
||||
),
|
||||
None => format!("{} link", link.role),
|
||||
};
|
||||
candidate(link.id.as_str(), help)
|
||||
}
|
||||
|
||||
fn share_of_typed_path() -> Option<(Vec<ShareEntry>, Option<PublicLinkInfo>)> {
|
||||
match ask(Request::ListShare {
|
||||
path: typed_path()?,
|
||||
})? {
|
||||
Response::Share { entries, link } => Some((entries, link)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// [`clap_complete::engine::ArgValueCompleter`] candidates are not filtered by
|
||||
/// the engine, unlike [`clap_complete::engine::ArgValueCandidates`].
|
||||
fn starting_with(
|
||||
current: &OsStr,
|
||||
candidates: Vec<CompletionCandidate>,
|
||||
) -> Vec<CompletionCandidate> {
|
||||
let typed = current.to_string_lossy();
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|c| c.get_value().to_string_lossy().starts_with(typed.as_ref()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The `path` argument of the command being completed, as the service takes
|
||||
/// it. A completer only receives the word under the cursor, so this reads the
|
||||
/// line from the process arguments, where the shell's stub passes it.
|
||||
fn typed_path() -> Option<String> {
|
||||
let args: Vec<OsString> = std::env::args_os().collect();
|
||||
let index = std::env::var("_CLAP_COMPLETE_INDEX")
|
||||
.ok()
|
||||
.and_then(|i| i.parse().ok());
|
||||
path_in(words_before_cursor(&args, index))
|
||||
}
|
||||
|
||||
/// The words of the line before the one being completed. The stub runs
|
||||
/// `pdfs -- <words>`; bash, zsh and elvish say which word the cursor is on,
|
||||
/// fish and PowerShell pass the line cut at the cursor, so it is the last.
|
||||
fn words_before_cursor(args: &[OsString], index: Option<usize>) -> Vec<OsString> {
|
||||
let start = args
|
||||
.iter()
|
||||
.position(|a| a == "--")
|
||||
.map_or(args.len(), |at| at + 1);
|
||||
let mut words = args[start..].to_vec();
|
||||
let index = index.unwrap_or(words.len().saturating_sub(1));
|
||||
words.truncate(index);
|
||||
words
|
||||
}
|
||||
|
||||
/// Parse `words` as far as they go and take the innermost command's `path`.
|
||||
fn path_in(words: Vec<OsString>) -> Option<String> {
|
||||
let matches = crate::Cli::command()
|
||||
.ignore_errors(true)
|
||||
.try_get_matches_from(words)
|
||||
.ok()?;
|
||||
let mut leaf = &matches;
|
||||
while let Some((_, sub)) = leaf.subcommand() {
|
||||
leaf = sub;
|
||||
}
|
||||
let path = leaf.try_get_one::<PathBuf>("path").ok().flatten()?;
|
||||
crate::path_arg(path).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn values(candidates: &[CompletionCandidate]) -> Vec<String> {
|
||||
candidates
|
||||
.iter()
|
||||
.map(|c| c.get_value().to_string_lossy().into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn help(candidate: &CompletionCandidate) -> String {
|
||||
candidate
|
||||
.get_help()
|
||||
.map(ToString::to_string)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn entry(name: &str, is_dir: bool) -> DirEntry {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0,
|
||||
"modified": 0,
|
||||
"pinned": false,
|
||||
"uid": format!("vol~{name}"),
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn words(line: &[&str]) -> Vec<OsString> {
|
||||
line.iter().map(OsString::from).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_command_definition_is_valid() {
|
||||
crate::Cli::command().debug_assert();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completions_script_is_written_for_every_shell() {
|
||||
for shell in [
|
||||
ShellArg::Bash,
|
||||
ShellArg::Zsh,
|
||||
ShellArg::Fish,
|
||||
ShellArg::Elvish,
|
||||
ShellArg::Powershell,
|
||||
] {
|
||||
let mut script = Vec::new();
|
||||
write_script(shell, &mut script).unwrap();
|
||||
let script = String::from_utf8(script).unwrap();
|
||||
assert!(script.contains(VAR), "{script}");
|
||||
assert!(script.contains("pdfs"), "{script}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_path_splits_parent_and_prefix() {
|
||||
assert_eq!(split_drive_word(""), ("", ""));
|
||||
assert_eq!(split_drive_word("Doc"), ("", "Doc"));
|
||||
assert_eq!(split_drive_word("Documents/"), ("Documents/", ""));
|
||||
assert_eq!(split_drive_word("Documents/Rep"), ("Documents/", "Rep"));
|
||||
assert_eq!(split_drive_word("a/b/c"), ("a/b/", "c"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_dirs_end_in_a_slash() {
|
||||
let entries = [
|
||||
entry("Reports", true),
|
||||
entry("Receipt.pdf", false),
|
||||
entry("Other", true),
|
||||
];
|
||||
let all = drive_candidates("Documents/", "Re", &entries, false);
|
||||
assert_eq!(
|
||||
values(&all),
|
||||
["Documents/Reports/", "Documents/Receipt.pdf"]
|
||||
);
|
||||
let dirs = drive_candidates("Documents/", "", &entries, true);
|
||||
assert_eq!(values(&dirs), ["Documents/Reports/", "Documents/Other/"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hidden_drive_entries_need_a_typed_dot() {
|
||||
let entries = [entry(".git", true)];
|
||||
assert!(drive_candidates("", "", &entries, false)[0].is_hide_set());
|
||||
assert!(!drive_candidates("", ".", &entries, false)[0].is_hide_set());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_folder_ids_complete_with_their_local_path() {
|
||||
let folders: Vec<SyncFolderInfo> = serde_json::from_value(serde_json::json!([{
|
||||
"id": 3,
|
||||
"local_path": "/home/me/Documents",
|
||||
"remote_uid": "vol~docs",
|
||||
"mode": "mirror",
|
||||
"state": "idle",
|
||||
"last_sync": 0,
|
||||
}]))
|
||||
.unwrap();
|
||||
let candidates = sync_folder_candidates(&folders);
|
||||
assert_eq!(values(&candidates), ["3"]);
|
||||
assert_eq!(help(&candidates[0]), "/home/me/Documents (mirror)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trash_uids_complete_with_their_name() {
|
||||
let mut from_docs = entry("a.pdf", false);
|
||||
from_docs.trashed_from = Some("Documents".into());
|
||||
let candidates = trash_candidates(&[from_docs, entry("b", true)]);
|
||||
assert_eq!(values(&candidates), ["vol~a.pdf", "vol~b"]);
|
||||
assert_eq!(help(&candidates[0]), "a.pdf (from Documents)");
|
||||
assert_eq!(help(&candidates[1]), "b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completer_without_a_service_offers_nothing() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-complete-{}", std::process::id()));
|
||||
let socket = dir.join("control.sock");
|
||||
assert!(ask_at(&socket, &Request::ListSyncFolders).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_listing_is_reused_until_it_is_stale() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-complete-cache-{}", std::process::id()));
|
||||
let file = dir.join("completion-devices.json");
|
||||
write_cache(&file, &Response::Devices { items: Vec::new() });
|
||||
assert!(matches!(read_fresh(&file), Some(Response::Devices { .. })));
|
||||
let mode = std::os::unix::fs::PermissionsExt::mode(
|
||||
&std::fs::metadata(&file).unwrap().permissions(),
|
||||
);
|
||||
assert_eq!(mode & 0o777, 0o600);
|
||||
|
||||
let stale = std::time::SystemTime::now() - CACHE_TTL - Duration::from_secs(1);
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(&file)
|
||||
.unwrap()
|
||||
.set_modified(stale)
|
||||
.unwrap();
|
||||
assert!(read_fresh(&file).is_none());
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn words_before_cursor_stop_at_the_cursor() {
|
||||
let args = words(&[
|
||||
"pdfs", "--", "pdfs", "versions", "restore", "a.txt", "", "x",
|
||||
]);
|
||||
assert_eq!(
|
||||
words_before_cursor(&args, Some(4)),
|
||||
words(&["pdfs", "versions", "restore", "a.txt"])
|
||||
);
|
||||
// fish: the line arrives cut at the cursor, the current word last.
|
||||
let fish = words(&["pdfs", "--", "pdfs", "versions", "restore", "a.txt", "ab"]);
|
||||
assert_eq!(
|
||||
words_before_cursor(&fish, None),
|
||||
words(&["pdfs", "versions", "restore", "a.txt"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completing_words_finds_the_path_before_the_id() {
|
||||
let line = words(&["pdfs", "versions", "restore", "Documents/a.txt"]);
|
||||
assert_eq!(path_in(line).as_deref(), Some("Documents/a.txt"));
|
||||
let line = words(&["pdfs", "--json", "share-role", "Shared/x"]);
|
||||
assert_eq!(path_in(line).as_deref(), Some("Shared/x"));
|
||||
let line = words(&["pdfs", "public-link", "update", "--role", "editor", "a"]);
|
||||
assert_eq!(path_in(line).as_deref(), Some("a"));
|
||||
assert_eq!(path_in(words(&["pdfs", "versions", "restore"])), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
//! Log priorities for journald.
|
||||
//!
|
||||
//! journald files each line a service writes at the unit's default priority,
|
||||
//! info, unless the line opens with a syslog `<N>` prefix. Without one every
|
||||
//! WARN and ERROR the daemon logs is stored as info, and the
|
||||
//! `journalctl -p warning` that the troubleshooting guide asks for shows
|
||||
//! nothing (`docs/BUGS.md` B258).
|
||||
|
||||
use std::fmt;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
use tracing::{Event, Level, Subscriber};
|
||||
use tracing_subscriber::fmt::format::Writer;
|
||||
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
|
||||
/// Whether standard output is the journal stream systemd connected this
|
||||
/// process to.
|
||||
///
|
||||
/// `JOURNAL_STREAM` alone does not say so: a terminal started from a user unit
|
||||
/// inherits it, and every `pdfs` run in that terminal would print the
|
||||
/// prefixes. systemd sets it to the stream's `device:inode`, so it is compared
|
||||
/// against what fd 1 actually is.
|
||||
pub(crate) fn stdout_is_journal() -> bool {
|
||||
let Some(stream) = std::env::var_os("JOURNAL_STREAM") else {
|
||||
return false;
|
||||
};
|
||||
let Ok(stdout) = std::fs::metadata("/proc/self/fd/1") else {
|
||||
return false;
|
||||
};
|
||||
stream
|
||||
.to_str()
|
||||
.is_some_and(|value| names_stream(value, stdout.dev(), stdout.ino()))
|
||||
}
|
||||
|
||||
/// Whether a `JOURNAL_STREAM` value names the file `dev`:`ino`.
|
||||
fn names_stream(value: &str, dev: u64, ino: u64) -> bool {
|
||||
value
|
||||
.split_once(':')
|
||||
.is_some_and(|(d, i)| d.parse() == Ok(dev) && i.parse() == Ok(ino))
|
||||
}
|
||||
|
||||
/// The syslog priority journald should file a `level` line at.
|
||||
fn priority(level: &Level) -> u8 {
|
||||
match *level {
|
||||
Level::ERROR => 3,
|
||||
Level::WARN => 4,
|
||||
Level::INFO => 6,
|
||||
Level::DEBUG | Level::TRACE => 7,
|
||||
}
|
||||
}
|
||||
|
||||
/// An event format that opens each line with its `<N>` priority, then writes
|
||||
/// the line as `F` would.
|
||||
pub(crate) struct Prioritized<F>(pub(crate) F);
|
||||
|
||||
impl<S, N, F> FormatEvent<S, N> for Prioritized<F>
|
||||
where
|
||||
S: Subscriber + for<'a> LookupSpan<'a>,
|
||||
N: for<'a> FormatFields<'a> + 'static,
|
||||
F: FormatEvent<S, N>,
|
||||
{
|
||||
fn format_event(
|
||||
&self,
|
||||
ctx: &FmtContext<'_, S, N>,
|
||||
mut writer: Writer<'_>,
|
||||
event: &Event<'_>,
|
||||
) -> fmt::Result {
|
||||
write!(writer, "<{}>", priority(event.metadata().level()))?;
|
||||
self.0.format_event(ctx, writer, event)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Buf(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl io::Write for Buf {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_line_opens_with_the_priority_of_its_level() {
|
||||
let buf = Buf(Arc::new(Mutex::new(Vec::new())));
|
||||
let writer = buf.clone();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_max_level(Level::DEBUG)
|
||||
.with_ansi(false)
|
||||
.with_writer(move || writer.clone())
|
||||
.event_format(Prioritized(tracing_subscriber::fmt::format()))
|
||||
.finish();
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::error!("e");
|
||||
tracing::warn!("w");
|
||||
tracing::info!("i");
|
||||
tracing::debug!("d");
|
||||
});
|
||||
let out = String::from_utf8(buf.0.lock().unwrap().clone()).unwrap();
|
||||
let prefixes: Vec<&str> = out.lines().map(|line| &line[..3]).collect();
|
||||
assert_eq!(prefixes, ["<3>", "<4>", "<6>", "<7>"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn journal_stream_must_name_the_same_file() {
|
||||
assert!(names_stream("8:12345", 8, 12345));
|
||||
assert!(!names_stream("8:12345", 8, 54321));
|
||||
assert!(!names_stream("9:12345", 8, 12345));
|
||||
assert!(!names_stream("8", 8, 12345));
|
||||
assert!(!names_stream("", 8, 12345));
|
||||
}
|
||||
}
|
||||
+1422
-158
File diff suppressed because it is too large.
Load diff
@@ -12,6 +12,8 @@ description = "Core auth/session/config for the Proton Drive Linux client"
|
||||
[dependencies]
|
||||
proton-sdk.workspace = true
|
||||
proton-drive-rs.workspace = true
|
||||
proton-gallery-rs.workspace = true
|
||||
chrono.workspace = true
|
||||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -19,12 +21,14 @@ thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
directories.workspace = true
|
||||
ignore.workspace = true
|
||||
shell-words.workspace = true
|
||||
sha2.workspace = true
|
||||
sha1.workspace = true
|
||||
zip.workspace = true
|
||||
rusqlite.workspace = true
|
||||
keyring-core.workspace = true
|
||||
proton-auth-rs = { workspace = true, features = ["keyring"] }
|
||||
proton-base-rs.workspace = true
|
||||
dbus-secret-service-keyring-store.workspace = true
|
||||
dbus-secret-service.workspace = true
|
||||
parking_lot.workspace = true
|
||||
async-trait.workspace = true
|
||||
libc.workspace = true
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,250 +0,0 @@
|
||||
AD Andorra
|
||||
AE United Arab Emirates
|
||||
AF Afghanistan
|
||||
AG Antigua and Barbuda
|
||||
AI Anguilla
|
||||
AL Albania
|
||||
AM Armenia
|
||||
AO Angola
|
||||
AQ Antarctica
|
||||
AR Argentina
|
||||
AS American Samoa
|
||||
AT Austria
|
||||
AU Australia
|
||||
AW Aruba
|
||||
AX Åland Islands
|
||||
AZ Azerbaijan
|
||||
BA Bosnia and Herzegovina
|
||||
BB Barbados
|
||||
BD Bangladesh
|
||||
BE Belgium
|
||||
BF Burkina Faso
|
||||
BG Bulgaria
|
||||
BH Bahrain
|
||||
BI Burundi
|
||||
BJ Benin
|
||||
BL Saint Barthélemy
|
||||
BM Bermuda
|
||||
BN Brunei Darussalam
|
||||
BO Bolivia, Plurinational State of
|
||||
BQ Bonaire, Sint Eustatius and Saba
|
||||
BR Brazil
|
||||
BS Bahamas
|
||||
BT Bhutan
|
||||
BV Bouvet Island
|
||||
BW Botswana
|
||||
BY Belarus
|
||||
BZ Belize
|
||||
CA Canada
|
||||
CC Cocos (Keeling) Islands
|
||||
CD Congo, The Democratic Republic of the
|
||||
CF Central African Republic
|
||||
CG Congo
|
||||
CH Switzerland
|
||||
CI Côte d'Ivoire
|
||||
CK Cook Islands
|
||||
CL Chile
|
||||
CM Cameroon
|
||||
CN China
|
||||
CO Colombia
|
||||
CR Costa Rica
|
||||
CU Cuba
|
||||
CV Cabo Verde
|
||||
CW Curaçao
|
||||
CX Christmas Island
|
||||
CY Cyprus
|
||||
CZ Czechia
|
||||
DE Germany
|
||||
DJ Djibouti
|
||||
DK Denmark
|
||||
DM Dominica
|
||||
DO Dominican Republic
|
||||
DZ Algeria
|
||||
EC Ecuador
|
||||
EE Estonia
|
||||
EG Egypt
|
||||
EH Western Sahara
|
||||
ER Eritrea
|
||||
ES Spain
|
||||
ET Ethiopia
|
||||
FI Finland
|
||||
FJ Fiji
|
||||
FK Falkland Islands (Malvinas)
|
||||
FM Micronesia, Federated States of
|
||||
FO Faroe Islands
|
||||
FR France
|
||||
GA Gabon
|
||||
GB United Kingdom
|
||||
GD Grenada
|
||||
GE Georgia
|
||||
GF French Guiana
|
||||
GG Guernsey
|
||||
GH Ghana
|
||||
GI Gibraltar
|
||||
GL Greenland
|
||||
GM Gambia
|
||||
GN Guinea
|
||||
GP Guadeloupe
|
||||
GQ Equatorial Guinea
|
||||
GR Greece
|
||||
GS South Georgia and the South Sandwich Islands
|
||||
GT Guatemala
|
||||
GU Guam
|
||||
GW Guinea-Bissau
|
||||
GY Guyana
|
||||
HK Hong Kong
|
||||
HM Heard Island and McDonald Islands
|
||||
HN Honduras
|
||||
HR Croatia
|
||||
HT Haiti
|
||||
HU Hungary
|
||||
ID Indonesia
|
||||
IE Ireland
|
||||
IL Israel
|
||||
IM Isle of Man
|
||||
IN India
|
||||
IO British Indian Ocean Territory
|
||||
IQ Iraq
|
||||
IR Iran, Islamic Republic of
|
||||
IS Iceland
|
||||
IT Italy
|
||||
JE Jersey
|
||||
JM Jamaica
|
||||
JO Jordan
|
||||
JP Japan
|
||||
KE Kenya
|
||||
KG Kyrgyzstan
|
||||
KH Cambodia
|
||||
KI Kiribati
|
||||
KM Comoros
|
||||
KN Saint Kitts and Nevis
|
||||
KP Korea, Democratic People's Republic of
|
||||
KR Korea, Republic of
|
||||
KW Kuwait
|
||||
KY Cayman Islands
|
||||
KZ Kazakhstan
|
||||
LA Lao People's Democratic Republic
|
||||
LB Lebanon
|
||||
LC Saint Lucia
|
||||
LI Liechtenstein
|
||||
LK Sri Lanka
|
||||
LR Liberia
|
||||
LS Lesotho
|
||||
LT Lithuania
|
||||
LU Luxembourg
|
||||
LV Latvia
|
||||
LY Libya
|
||||
MA Morocco
|
||||
MC Monaco
|
||||
MD Moldova, Republic of
|
||||
ME Montenegro
|
||||
MF Saint Martin (French part)
|
||||
MG Madagascar
|
||||
MH Marshall Islands
|
||||
MK North Macedonia
|
||||
ML Mali
|
||||
MM Myanmar
|
||||
MN Mongolia
|
||||
MO Macao
|
||||
MP Northern Mariana Islands
|
||||
MQ Martinique
|
||||
MR Mauritania
|
||||
MS Montserrat
|
||||
MT Malta
|
||||
MU Mauritius
|
||||
MV Maldives
|
||||
MW Malawi
|
||||
MX Mexico
|
||||
MY Malaysia
|
||||
MZ Mozambique
|
||||
NA Namibia
|
||||
NC New Caledonia
|
||||
NE Niger
|
||||
NF Norfolk Island
|
||||
NG Nigeria
|
||||
NI Nicaragua
|
||||
NL Netherlands
|
||||
NO Norway
|
||||
NP Nepal
|
||||
NR Nauru
|
||||
NU Niue
|
||||
NZ New Zealand
|
||||
OM Oman
|
||||
PA Panama
|
||||
PE Peru
|
||||
PF French Polynesia
|
||||
PG Papua New Guinea
|
||||
PH Philippines
|
||||
PK Pakistan
|
||||
PL Poland
|
||||
PM Saint Pierre and Miquelon
|
||||
PN Pitcairn
|
||||
PR Puerto Rico
|
||||
PS Palestine, State of
|
||||
PT Portugal
|
||||
PW Palau
|
||||
PY Paraguay
|
||||
QA Qatar
|
||||
RE Réunion
|
||||
RO Romania
|
||||
RS Serbia
|
||||
RU Russian Federation
|
||||
RW Rwanda
|
||||
SA Saudi Arabia
|
||||
SB Solomon Islands
|
||||
SC Seychelles
|
||||
SD Sudan
|
||||
SE Sweden
|
||||
SG Singapore
|
||||
SH Saint Helena, Ascension and Tristan da Cunha
|
||||
SI Slovenia
|
||||
SJ Svalbard and Jan Mayen
|
||||
SK Slovakia
|
||||
SL Sierra Leone
|
||||
SM San Marino
|
||||
SN Senegal
|
||||
SO Somalia
|
||||
SR Suriname
|
||||
SS South Sudan
|
||||
ST Sao Tome and Principe
|
||||
SV El Salvador
|
||||
SX Sint Maarten (Dutch part)
|
||||
SY Syrian Arab Republic
|
||||
SZ Eswatini
|
||||
TC Turks and Caicos Islands
|
||||
TD Chad
|
||||
TF French Southern Territories
|
||||
TG Togo
|
||||
TH Thailand
|
||||
TJ Tajikistan
|
||||
TK Tokelau
|
||||
TL Timor-Leste
|
||||
TM Turkmenistan
|
||||
TN Tunisia
|
||||
TO Tonga
|
||||
TR Türkiye
|
||||
TT Trinidad and Tobago
|
||||
TV Tuvalu
|
||||
TW Taiwan, Province of China
|
||||
TZ Tanzania, United Republic of
|
||||
UA Ukraine
|
||||
UG Uganda
|
||||
UM United States Minor Outlying Islands
|
||||
US United States
|
||||
UY Uruguay
|
||||
UZ Uzbekistan
|
||||
VA Holy See (Vatican City State)
|
||||
VC Saint Vincent and the Grenadines
|
||||
VE Venezuela, Bolivarian Republic of
|
||||
VG Virgin Islands, British
|
||||
VI Virgin Islands, U.S.
|
||||
VN Viet Nam
|
||||
VU Vanuatu
|
||||
WF Wallis and Futuna
|
||||
WS Samoa
|
||||
XK Kosovo
|
||||
YE Yemen
|
||||
YT Mayotte
|
||||
ZA South Africa
|
||||
ZM Zambia
|
||||
ZW Zimbabwe
|
||||
|
@@ -0,0 +1,925 @@
|
||||
//! A stand-in daemon that serves made-up data, for the README screenshots.
|
||||
//!
|
||||
//! `scripts/screenshots.sh` runs it; see `docs/DEVELOPMENT.md`. It answers the
|
||||
//! control socket the way the real daemon does, for the requests the captured
|
||||
//! pages send, with an invented account: no Proton login, no FUSE mount and
|
||||
//! nothing private. Every name, place and number below is made up.
|
||||
//!
|
||||
//! ```text
|
||||
//! demo_daemon serve <photos-dir> answer the control socket until killed
|
||||
//! demo_daemon seed-keyring store a fake session, so the app signs in
|
||||
//! ```
|
||||
//!
|
||||
//! Both refuse to run outside a throwaway environment: `PDFS_DEMO_ROOT` must be
|
||||
//! set and every XDG directory must lie inside it, so the real daemon's socket
|
||||
//! is never touched (`HOME` may instead name a folder that does not exist, the
|
||||
//! demo user's, so paths read as theirs), and `seed-keyring` refuses the session bus the script was
|
||||
//! started on (`PDFS_DEMO_OUTER_BUS`), so the fake session only ever lands in a
|
||||
//! private keyring.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::{Datelike, Local, TimeZone};
|
||||
use pdfs_core::cache::Pin;
|
||||
use pdfs_core::config::{AppDirs, KEYRING_SERVICE};
|
||||
use pdfs_core::control::{
|
||||
ActivityEntry, ActivityKind, AlbumInfo, DeviceInfo, DirEntry, Event, LocalHit, PhotoItem,
|
||||
PhotoKind, PhotoLocation, PhotoMonth, PhotoThumb, PlaceInfo, Request, Response, SearchHit,
|
||||
SyncFolderInfo, Topic,
|
||||
};
|
||||
use pdfs_core::mounts::{MountAccess, MountKind, MountMode, MountSpec};
|
||||
|
||||
const USERNAME: &str = "alex.example";
|
||||
const MOUNTPOINT: &str = "/home/alex/ProtonDrive";
|
||||
const GIB: u64 = 1 << 30;
|
||||
const DAY: i64 = 86_400;
|
||||
|
||||
/// How many photos the timeline holds. The demo pictures repeat, but not
|
||||
/// within a screen.
|
||||
const PHOTOS: usize = 1_840;
|
||||
|
||||
fn main() {
|
||||
if let Err(e) = run() {
|
||||
eprintln!("demo_daemon: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<(), String> {
|
||||
check_sandbox()?;
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
match args.iter().map(String::as_str).collect::<Vec<_>>()[..] {
|
||||
["serve", photos] => serve(Path::new(photos)),
|
||||
["seed-keyring"] => seed_keyring(),
|
||||
_ => Err("usage: demo_daemon serve <photos-dir> | seed-keyring".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuse to run anywhere but inside the throwaway root the script made.
|
||||
fn check_sandbox() -> Result<(), String> {
|
||||
let root = std::env::var_os("PDFS_DEMO_ROOT")
|
||||
.map(PathBuf::from)
|
||||
.ok_or("PDFS_DEMO_ROOT is not set; run scripts/screenshots.sh")?;
|
||||
let home = std::env::var_os("HOME").map(PathBuf::from);
|
||||
if home.is_none_or(|home| home.exists() && !home.starts_with(&root)) {
|
||||
return Err("HOME must lie inside PDFS_DEMO_ROOT or not exist".into());
|
||||
}
|
||||
for var in [
|
||||
"XDG_CONFIG_HOME",
|
||||
"XDG_STATE_HOME",
|
||||
"XDG_CACHE_HOME",
|
||||
"XDG_DATA_HOME",
|
||||
] {
|
||||
let inside = std::env::var_os(var)
|
||||
.map(PathBuf::from)
|
||||
.is_some_and(|dir| dir.starts_with(&root));
|
||||
if !inside {
|
||||
return Err(format!("{var} must lie inside PDFS_DEMO_ROOT"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Store a made-up session where the app looks for one. Its keys "unlock"
|
||||
/// without a prompt; nothing ever uses them against Proton.
|
||||
fn seed_keyring() -> Result<(), String> {
|
||||
let bus = std::env::var("DBUS_SESSION_BUS_ADDRESS").unwrap_or_default();
|
||||
let outer =
|
||||
std::env::var("PDFS_DEMO_OUTER_BUS").map_err(|_| "PDFS_DEMO_OUTER_BUS is not set")?;
|
||||
if bus.is_empty() || bus == outer {
|
||||
return Err("refusing to write a fake session to your own keyring".into());
|
||||
}
|
||||
let session = serde_json::json!({
|
||||
"session_id": "demo-session",
|
||||
"username": USERNAME,
|
||||
"user_id": "demo-user",
|
||||
"access_token": "demo",
|
||||
"refresh_token": "demo",
|
||||
"scopes": ["full", "self", "drive"],
|
||||
"password_mode": 1,
|
||||
"key_passphrases": { "demo-key": "ZGVtbw==" },
|
||||
});
|
||||
let store: Arc<keyring_core::CredentialStore> =
|
||||
dbus_secret_service_keyring_store::Store::new().map_err(|e| e.to_string())?;
|
||||
keyring_core::set_default_store(store);
|
||||
keyring_core::Entry::new(KEYRING_SERVICE, "session")
|
||||
.and_then(|entry| entry.set_password(&session.to_string()))
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn serve(photos: &Path) -> Result<(), String> {
|
||||
let demo = Arc::new(Demo::new(photos)?);
|
||||
let dirs = AppDirs::new().map_err(|e| e.to_string())?;
|
||||
dirs.ensure().map_err(|e| e.to_string())?;
|
||||
let socket = dirs.control_socket();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let listener = UnixListener::bind(&socket).map_err(|e| format!("{}: {e}", socket.display()))?;
|
||||
std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o600))
|
||||
.map_err(|e| e.to_string())?;
|
||||
eprintln!("demo_daemon: serving {}", socket.display());
|
||||
for stream in listener.incoming().flatten() {
|
||||
let demo = Arc::clone(&demo);
|
||||
std::thread::spawn(move || {
|
||||
if let Err(e) = connection(&demo, stream) {
|
||||
eprintln!("demo_daemon: {e}");
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn connection(demo: &Demo, stream: UnixStream) -> std::io::Result<()> {
|
||||
let mut reader = BufReader::new(stream.try_clone()?);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line)?;
|
||||
let request: Request = match serde_json::from_str(line.trim()) {
|
||||
Ok(request) => request,
|
||||
Err(e) => {
|
||||
eprintln!("demo_daemon: unreadable request: {e}");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let mut out = stream;
|
||||
if matches!(request, Request::Subscribe) {
|
||||
return subscription(demo, &mut out);
|
||||
}
|
||||
send(&mut out, &demo.answer(&request))
|
||||
}
|
||||
|
||||
/// What the real daemon sends a new subscriber, then a heartbeat now and then
|
||||
/// so the window never counts the daemon as gone.
|
||||
fn subscription(demo: &Demo, out: &mut UnixStream) -> std::io::Result<()> {
|
||||
send(out, &Event::Status(Box::new(demo.status())))?;
|
||||
send(
|
||||
out,
|
||||
&Event::Transfers {
|
||||
items: Vec::new(),
|
||||
jobs: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
send(
|
||||
out,
|
||||
&Event::Changed {
|
||||
topics: Topic::ALL.to_vec(),
|
||||
},
|
||||
)?;
|
||||
loop {
|
||||
std::thread::sleep(Duration::from_secs(10));
|
||||
send(out, &Event::Heartbeat)?;
|
||||
}
|
||||
}
|
||||
|
||||
fn send(out: &mut UnixStream, value: &impl serde::Serialize) -> std::io::Result<()> {
|
||||
let mut line = serde_json::to_vec(value)?;
|
||||
line.push(b'\n');
|
||||
out.write_all(&line)?;
|
||||
out.flush()
|
||||
}
|
||||
|
||||
/// One invented account.
|
||||
struct Demo {
|
||||
now: i64,
|
||||
pictures: Vec<PathBuf>,
|
||||
photos: Vec<PhotoItem>,
|
||||
/// Each photo's place, by index into [`PLACES`].
|
||||
place_of: Vec<Option<usize>>,
|
||||
}
|
||||
|
||||
/// Towns the photos were "taken" in: name, ISO country, latitude, longitude.
|
||||
const PLACES: &[(&str, &str, f64, f64)] = &[
|
||||
("Lisbon", "PT", 38.7223, -9.1393),
|
||||
("Porto", "PT", 41.1579, -8.6291),
|
||||
("Barcelona", "ES", 41.3874, 2.1686),
|
||||
("Paris", "FR", 48.8566, 2.3522),
|
||||
("Amsterdam", "NL", 52.3676, 4.9041),
|
||||
("Copenhagen", "DK", 55.6761, 12.5683),
|
||||
("Berlin", "DE", 52.5200, 13.4050),
|
||||
("Prague", "CZ", 50.0755, 14.4378),
|
||||
("Vienna", "AT", 48.2082, 16.3738),
|
||||
("Florence", "IT", 43.7696, 11.2558),
|
||||
("Rome", "IT", 41.9028, 12.4964),
|
||||
("Thira", "GR", 36.4166, 25.4318),
|
||||
];
|
||||
|
||||
const ALBUMS: &[&str] = &[
|
||||
"Summer in Portugal",
|
||||
"Family",
|
||||
"Hiking",
|
||||
"Food",
|
||||
"City trips",
|
||||
"Best of 2025",
|
||||
];
|
||||
|
||||
impl Demo {
|
||||
fn new(dir: &Path) -> Result<Self, String> {
|
||||
let mut pictures: Vec<PathBuf> = std::fs::read_dir(dir)
|
||||
.map_err(|e| format!("{}: {e}", dir.display()))?
|
||||
.flatten()
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.extension().is_some_and(|ext| ext == "jpg"))
|
||||
.collect();
|
||||
pictures.sort();
|
||||
if pictures.is_empty() {
|
||||
return Err(format!("no .jpg pictures in {}", dir.display()));
|
||||
}
|
||||
let now = Local::now().timestamp();
|
||||
let mut demo = Demo {
|
||||
now,
|
||||
pictures,
|
||||
photos: Vec::new(),
|
||||
place_of: Vec::new(),
|
||||
};
|
||||
demo.make_photos();
|
||||
Ok(demo)
|
||||
}
|
||||
|
||||
/// The timeline: newest first, a few a week back three years, plus a
|
||||
/// photo on this day in each of the last three years.
|
||||
fn make_photos(&mut self) {
|
||||
let mut seed = 0x5eed_u64;
|
||||
let mut next = move || {
|
||||
seed = seed
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
(seed >> 33) as i64
|
||||
};
|
||||
let today = Local::now().date_naive();
|
||||
let noon = |years_back: i32| {
|
||||
let day = today
|
||||
.with_year(today.year() - years_back)
|
||||
.unwrap_or(today - chrono::Days::new(365 * years_back as u64));
|
||||
Local
|
||||
.from_local_datetime(&day.and_hms_opt(11, 30, 0).unwrap_or_default())
|
||||
.single()
|
||||
.map_or(0, |t| t.timestamp())
|
||||
};
|
||||
// Three years, so a few a day.
|
||||
let step = 3 * 365 * DAY / PHOTOS as i64;
|
||||
let mut times: Vec<i64> = (0..PHOTOS as i64)
|
||||
.map(|i| self.now - 3 * 3600 - i * step - next() % step)
|
||||
.collect();
|
||||
times.extend([noon(1), noon(2), noon(3)]);
|
||||
times.sort_unstable_by(|a, b| b.cmp(a));
|
||||
for (i, time) in times.into_iter().enumerate() {
|
||||
let picture = &self.pictures[i % self.pictures.len()];
|
||||
let kind = match i % 23 {
|
||||
7 | 15 => PhotoKind::Video,
|
||||
19 => PhotoKind::Raw,
|
||||
_ => PhotoKind::Photo,
|
||||
};
|
||||
let place = (i % 5 != 4).then_some((i / 7) % PLACES.len());
|
||||
self.photos.push(PhotoItem {
|
||||
uid: format!("demo-photos~{i}"),
|
||||
capture_time: time,
|
||||
thumb_path: Some(picture.display().to_string()),
|
||||
name: Some(format!(
|
||||
"IMG_{}.{}",
|
||||
20_000 + i,
|
||||
if kind == PhotoKind::Video {
|
||||
"MP4"
|
||||
} else {
|
||||
"JPG"
|
||||
}
|
||||
)),
|
||||
ratio: Some(if picture.to_string_lossy().ends_with("-p.jpg") {
|
||||
2.0 / 3.0
|
||||
} else {
|
||||
1.5
|
||||
}),
|
||||
no_thumb: false,
|
||||
kind,
|
||||
favorite: i % 11 == 3,
|
||||
group_size: 0,
|
||||
has_raw: kind == PhotoKind::Raw,
|
||||
place_id: place.map(|p| p as u32 + 1),
|
||||
});
|
||||
self.place_of.push(place);
|
||||
}
|
||||
}
|
||||
|
||||
fn status(&self) -> Response {
|
||||
Response::Status {
|
||||
username: USERNAME.into(),
|
||||
mountpoint: MOUNTPOINT.into(),
|
||||
pinned: 3,
|
||||
used: 6 * GIB + 300 * (1 << 20),
|
||||
budget: 20 * GIB,
|
||||
pins: self.pins(),
|
||||
online: true,
|
||||
pending_uploads: 0,
|
||||
pending_changes: 0,
|
||||
parked_uploads: 0,
|
||||
failing_ops: 0,
|
||||
failing_error: None,
|
||||
staged_bytes: 0,
|
||||
staged_oldest_secs: 0,
|
||||
paused: false,
|
||||
paused_until: None,
|
||||
session_expired: false,
|
||||
tokens_unsaved: false,
|
||||
version: env!("CARGO_PKG_VERSION").into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn answer(&self, request: &Request) -> Response {
|
||||
match request {
|
||||
Request::Status => self.status(),
|
||||
Request::AccountQuota => Response::AccountQuota {
|
||||
max_space: 500 * GIB as i64,
|
||||
used_space: 137 * GIB as i64,
|
||||
},
|
||||
Request::ListDir { path } => Response::Entries {
|
||||
entries: self.dir(path),
|
||||
},
|
||||
Request::ReserveFileThumbGeneration => Response::FileThumbGeneration { generation: 1 },
|
||||
Request::FileThumbs { items, .. } => Response::Thumbs {
|
||||
items: items
|
||||
.iter()
|
||||
.map(|item| PhotoThumb {
|
||||
uid: item.uid.clone(),
|
||||
path: self.file_picture(&item.uid),
|
||||
pending: false,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
Request::PhotosTimeline {
|
||||
offset,
|
||||
limit,
|
||||
kind,
|
||||
favorites,
|
||||
..
|
||||
} => {
|
||||
let items: Vec<PhotoItem> = self
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| kind.is_none_or(|k| p.kind == k))
|
||||
.filter(|p| !favorites || p.favorite)
|
||||
.skip(*offset)
|
||||
.take(*limit)
|
||||
.cloned()
|
||||
.collect();
|
||||
Response::Photos {
|
||||
available: true,
|
||||
items,
|
||||
counts: Some(self.counts()),
|
||||
}
|
||||
}
|
||||
Request::PhotoMonths { kind } => Response::PhotoMonths {
|
||||
months: self.months(*kind),
|
||||
},
|
||||
Request::PhotosOnThisDay { .. } => {
|
||||
let today = Local::now();
|
||||
let items = self
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| {
|
||||
let at = Local.timestamp_opt(p.capture_time, 0).single();
|
||||
at.is_some_and(|at| {
|
||||
at.year() < today.year()
|
||||
&& at.month() == today.month()
|
||||
&& at.day() == today.day()
|
||||
})
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
Response::Photos {
|
||||
available: true,
|
||||
items,
|
||||
counts: None,
|
||||
}
|
||||
}
|
||||
Request::PhotoThumbs { uids } => Response::Thumbs {
|
||||
items: uids
|
||||
.iter()
|
||||
.map(|uid| PhotoThumb {
|
||||
uid: uid.clone(),
|
||||
path: self
|
||||
.photos
|
||||
.iter()
|
||||
.find(|p| &p.uid == uid)
|
||||
.and_then(|p| p.thumb_path.clone()),
|
||||
pending: false,
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
Request::PhotoAlbums => Response::Albums {
|
||||
available: true,
|
||||
items: self.albums(),
|
||||
},
|
||||
Request::AlbumPhotos { uid, offset, limit } => {
|
||||
let index: usize = uid
|
||||
.rsplit('~')
|
||||
.next()
|
||||
.and_then(|n| n.parse().ok())
|
||||
.unwrap_or(0);
|
||||
Response::Photos {
|
||||
available: true,
|
||||
items: self
|
||||
.album_photos(index)
|
||||
.skip(*offset)
|
||||
.take(*limit)
|
||||
.cloned()
|
||||
.collect(),
|
||||
counts: None,
|
||||
}
|
||||
}
|
||||
Request::PhotoPlaces => Response::Places {
|
||||
items: self.places(),
|
||||
mapping: None,
|
||||
},
|
||||
Request::PlaceLocations { id } => Response::PhotoLocations {
|
||||
items: self.place_locations(*id),
|
||||
},
|
||||
Request::PlacePhotos { id, offset, limit } => Response::Photos {
|
||||
available: true,
|
||||
items: self
|
||||
.in_place(*id)
|
||||
.map(|(_, p)| p)
|
||||
.skip(*offset)
|
||||
.take(*limit)
|
||||
.cloned()
|
||||
.collect(),
|
||||
counts: None,
|
||||
},
|
||||
Request::ListDevices => Response::Devices {
|
||||
items: self.devices(),
|
||||
},
|
||||
Request::ListSyncFolders => Response::SyncFolders {
|
||||
items: self.sync_folders(),
|
||||
},
|
||||
Request::ListLocations => Response::Locations {
|
||||
items: self.locations(),
|
||||
},
|
||||
// Every conflict in the history is resolved: an open one would
|
||||
// badge every page.
|
||||
Request::ListConflicts => Response::Conflicts { items: Vec::new() },
|
||||
Request::ListPendingOps => Response::PendingOps { items: Vec::new() },
|
||||
Request::ListActivity { limit } => Response::Activity {
|
||||
items: self.activity().into_iter().take(*limit).collect(),
|
||||
},
|
||||
Request::ListInvitations => Response::Invitations { items: Vec::new() },
|
||||
Request::ListBookmarks => Response::Bookmarks { items: Vec::new() },
|
||||
Request::GetQueueStatus => Response::Transfers {
|
||||
items: Vec::new(),
|
||||
jobs: Vec::new(),
|
||||
},
|
||||
Request::ListPins => Response::Pins { pins: self.pins() },
|
||||
Request::SearchV2 { query, limit, .. } => Response::SearchResultsV2 {
|
||||
drive_hits: self.search(query, *limit),
|
||||
local_hits: self.local_search(query),
|
||||
local_indexing: false,
|
||||
},
|
||||
Request::Search { query, limit, .. } => Response::SearchResults {
|
||||
hits: self.search(query, *limit),
|
||||
},
|
||||
Request::ImportStatus => Response::ImportStatus {
|
||||
running: false,
|
||||
summary: None,
|
||||
progress: None,
|
||||
resumable: Vec::new(),
|
||||
},
|
||||
Request::PhotosRefreshStatus => Response::PhotosRefresh { running: false },
|
||||
other => {
|
||||
eprintln!("demo_daemon: no demo answer for {}", request_name(other));
|
||||
Response::Ok {
|
||||
message: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn counts(&self) -> (usize, usize, usize) {
|
||||
let of = |kind| self.photos.iter().filter(|p| p.kind == kind).count();
|
||||
(
|
||||
of(PhotoKind::Photo),
|
||||
of(PhotoKind::Video),
|
||||
of(PhotoKind::Raw),
|
||||
)
|
||||
}
|
||||
|
||||
fn months(&self, kind: Option<PhotoKind>) -> Vec<PhotoMonth> {
|
||||
let mut months: BTreeMap<(i32, i32), usize> = BTreeMap::new();
|
||||
for photo in self
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| kind.is_none_or(|k| p.kind == k))
|
||||
{
|
||||
if let Some(at) = Local.timestamp_opt(photo.capture_time, 0).single() {
|
||||
*months.entry((at.year(), at.month() as i32)).or_default() += 1;
|
||||
}
|
||||
}
|
||||
months
|
||||
.into_iter()
|
||||
.rev()
|
||||
.map(|((year, month), count)| PhotoMonth { year, month, count })
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn albums(&self) -> Vec<AlbumInfo> {
|
||||
ALBUMS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, name)| AlbumInfo {
|
||||
uid: format!("demo-album~{i}"),
|
||||
name: (*name).into(),
|
||||
photo_count: self.album_photos(i).count(),
|
||||
cover_uid: self.album_photos(i).next().map(|p| p.uid.clone()),
|
||||
last_activity: Some(self.now - (i as i64 * 9 + 2) * DAY),
|
||||
shared: i == 1,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn album_photos(&self, album: usize) -> impl Iterator<Item = &PhotoItem> {
|
||||
let stride = 3 + album;
|
||||
self.photos
|
||||
.iter()
|
||||
.skip(album * 2)
|
||||
.step_by(stride)
|
||||
.take(40 + album * 17)
|
||||
}
|
||||
|
||||
fn in_place(&self, id: u32) -> impl Iterator<Item = (usize, &PhotoItem)> {
|
||||
self.photos
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(move |(i, _)| self.place_of[*i].is_some_and(|p| p as u32 + 1 == id))
|
||||
}
|
||||
|
||||
fn places(&self) -> Vec<PlaceInfo> {
|
||||
PLACES
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, (name, country, latitude, longitude))| {
|
||||
let id = i as u32 + 1;
|
||||
let count = self.in_place(id).count();
|
||||
let cover = self.in_place(id).next()?.1.clone();
|
||||
Some(PlaceInfo {
|
||||
id,
|
||||
name: (*name).into(),
|
||||
country: (*country).into(),
|
||||
latitude: *latitude,
|
||||
longitude: *longitude,
|
||||
photo_count: count,
|
||||
cover,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn place_locations(&self, id: u32) -> Vec<PhotoLocation> {
|
||||
let Some((_, _, latitude, longitude)) = PLACES.get(id as usize - 1) else {
|
||||
return Vec::new();
|
||||
};
|
||||
self.in_place(id)
|
||||
.map(|(i, photo)| {
|
||||
// Spread around the centre, the same way on every run.
|
||||
let angle = i as f64 * 2.399;
|
||||
let reach = 0.004 + (i % 9) as f64 * 0.003;
|
||||
PhotoLocation {
|
||||
photo: photo.clone(),
|
||||
latitude: latitude + reach * angle.sin(),
|
||||
longitude: longitude + reach * angle.cos(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// My files: folders first, then a few files, some of them pictures with
|
||||
/// thumbnails. Any folder below the root holds the same few documents.
|
||||
fn dir(&self, path: &str) -> Vec<DirEntry> {
|
||||
let folder = |name: &str, days: i64| DirEntry {
|
||||
name: name.into(),
|
||||
is_dir: true,
|
||||
modified: self.now - days * DAY,
|
||||
uid: format!("demo-files~{path}/{name}"),
|
||||
path: join(path, name),
|
||||
..Default::default()
|
||||
};
|
||||
let file = |name: &str, size: u64, days: i64| DirEntry {
|
||||
name: name.into(),
|
||||
is_dir: false,
|
||||
size,
|
||||
modified: self.now - days * DAY - 4000,
|
||||
uid: format!("demo-files~{path}/{name}"),
|
||||
path: join(path, name),
|
||||
..Default::default()
|
||||
};
|
||||
if !path.trim_matches('/').is_empty() {
|
||||
return vec![
|
||||
file("Notes.md", 4_812, 1),
|
||||
file("Overview.pdf", 912_384, 6),
|
||||
file("Budget.ods", 38_112, 12),
|
||||
];
|
||||
}
|
||||
let mut entries = vec![
|
||||
folder("Documents", 1),
|
||||
folder("Music", 30),
|
||||
folder("Pictures", 2),
|
||||
folder("Projects", 4),
|
||||
folder("Recipes", 60),
|
||||
folder("Taxes", 120),
|
||||
folder("Videos", 14),
|
||||
folder("Work", 3),
|
||||
];
|
||||
let mut pinned = file("Lisbon sunset.jpg", 4_218_112, 20);
|
||||
pinned.pinned = true;
|
||||
pinned.cached = true;
|
||||
entries.extend([
|
||||
file("Apartment lease.pdf", 1_482_011, 40),
|
||||
file("Budget 2026.ods", 52_340, 2),
|
||||
file("Cliffs.jpg", 3_844_201, 33),
|
||||
file("Garden.jpg", 2_904_877, 9),
|
||||
pinned,
|
||||
file("Meeting notes.md", 6_120, 0),
|
||||
file("Mountain lake.jpg", 5_120_330, 61),
|
||||
file("Packing list.txt", 1_204, 5),
|
||||
file("Road trip.jpg", 3_211_008, 75),
|
||||
file("Thesis draft.odt", 812_448, 18),
|
||||
]);
|
||||
entries
|
||||
}
|
||||
|
||||
/// The picture behind a demo file, for its thumbnail.
|
||||
fn file_picture(&self, uid: &str) -> Option<String> {
|
||||
let name = uid.rsplit('/').next()?;
|
||||
if !name.ends_with(".jpg") {
|
||||
return None;
|
||||
}
|
||||
let index = name.bytes().map(usize::from).sum::<usize>() % self.pictures.len();
|
||||
Some(self.pictures[index].display().to_string())
|
||||
}
|
||||
|
||||
fn pins(&self) -> Vec<Pin> {
|
||||
[
|
||||
("Documents", true),
|
||||
("Work/Quarterly report.odt", false),
|
||||
("Lisbon sunset.jpg", false),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(path, dir)| Pin {
|
||||
uid: format!("demo-files~/{path}"),
|
||||
path: (*path).into(),
|
||||
recursive: dir,
|
||||
is_dir: Some(dir),
|
||||
cached: true,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn devices(&self) -> Vec<DeviceInfo> {
|
||||
vec![
|
||||
DeviceInfo {
|
||||
uid: "demo-devices~1".into(),
|
||||
name: "alex-laptop".into(),
|
||||
device_type: "Linux".into(),
|
||||
last_sync: Some(self.now - 240),
|
||||
this_device: true,
|
||||
adopted: true,
|
||||
},
|
||||
DeviceInfo {
|
||||
uid: "demo-devices~2".into(),
|
||||
name: "studio-desktop".into(),
|
||||
device_type: "Windows".into(),
|
||||
last_sync: Some(self.now - 2 * DAY - 3 * 3600),
|
||||
this_device: false,
|
||||
adopted: false,
|
||||
},
|
||||
DeviceInfo {
|
||||
uid: "demo-devices~3".into(),
|
||||
name: "old-macbook".into(),
|
||||
device_type: "MacOs".into(),
|
||||
last_sync: Some(self.now - 210 * DAY),
|
||||
this_device: false,
|
||||
adopted: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// This computer's synced folders: mirrors, and one kept online only.
|
||||
const FOLDERS: &[(&str, &str)] = &[
|
||||
("/home/alex/Documents", "mirror"),
|
||||
("/home/alex/Pictures/Camera", "mirror"),
|
||||
("/home/alex/Music", "ondemand"),
|
||||
("/home/alex/Projects", "mirror"),
|
||||
];
|
||||
|
||||
fn sync_folders(&self) -> Vec<SyncFolderInfo> {
|
||||
Self::FOLDERS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (path, mode))| SyncFolderInfo {
|
||||
id: i as i64 + 1,
|
||||
kind: "device".into(),
|
||||
local_path: (*path).into(),
|
||||
remote_uid: format!("demo-sync~{i}"),
|
||||
mode: (*mode).into(),
|
||||
pending_mode: None,
|
||||
state: "idle".into(),
|
||||
last_sync: self.now - 240 - i as i64 * 60,
|
||||
progress: None,
|
||||
paused: false,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn locations(&self) -> Vec<MountSpec> {
|
||||
let mut items = vec![MountSpec {
|
||||
id: 1,
|
||||
kind: MountKind::MyFiles,
|
||||
local_path: MOUNTPOINT.into(),
|
||||
root_uid: "demo-files~".into(),
|
||||
root_share_id: String::new(),
|
||||
mode: MountMode::OnDemand,
|
||||
access: MountAccess::Rw,
|
||||
state: "idle".into(),
|
||||
last_sync: self.now - 60,
|
||||
pending_mode: None,
|
||||
mounted: true,
|
||||
progress: None,
|
||||
paused: false,
|
||||
}];
|
||||
items.extend(
|
||||
Self::FOLDERS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (path, mode))| MountSpec {
|
||||
id: i as i64 + 2,
|
||||
kind: MountKind::Device {
|
||||
sync_folder_id: i as i64 + 1,
|
||||
},
|
||||
local_path: (*path).into(),
|
||||
root_uid: format!("demo-sync~{i}"),
|
||||
root_share_id: String::new(),
|
||||
mode: MountMode::from(*mode),
|
||||
access: MountAccess::Rw,
|
||||
state: "idle".into(),
|
||||
last_sync: self.now - 240 - i as i64 * 60,
|
||||
pending_mode: None,
|
||||
mounted: *mode == "ondemand",
|
||||
progress: None,
|
||||
paused: false,
|
||||
}),
|
||||
);
|
||||
items
|
||||
}
|
||||
|
||||
fn activity(&self) -> Vec<ActivityEntry> {
|
||||
let at = |minutes: i64| self.now - minutes * 60;
|
||||
let entry = |time, kind, target: &str, detail: &str, ok| ActivityEntry {
|
||||
time,
|
||||
kind,
|
||||
target: target.into(),
|
||||
detail: detail.into(),
|
||||
ok,
|
||||
};
|
||||
let conflict = |time, name: &str, ext: &str| {
|
||||
entry(
|
||||
time,
|
||||
ActivityKind::Conflict,
|
||||
&format!("{name} (sync-conflict {time}).{ext}"),
|
||||
&format!("differs from {name}.{ext}"),
|
||||
false,
|
||||
)
|
||||
};
|
||||
vec![
|
||||
entry(at(2), ActivityKind::Upload, "Meeting notes.md", "", true),
|
||||
entry(
|
||||
at(14),
|
||||
ActivityKind::Share,
|
||||
"Budget 2026.ods",
|
||||
"1 recipient(s) as editor",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(38),
|
||||
ActivityKind::Upload,
|
||||
"Pictures/Camera/IMG_20481.JPG",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(39),
|
||||
ActivityKind::Upload,
|
||||
"Pictures/Camera/IMG_20480.JPG",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(52),
|
||||
ActivityKind::Rename,
|
||||
"Thesis draft.odt",
|
||||
"was thesis-v3.odt",
|
||||
true,
|
||||
),
|
||||
conflict(at(180), "Quarterly report", "odt"),
|
||||
entry(
|
||||
at(240),
|
||||
ActivityKind::Move,
|
||||
"Taxes/2025",
|
||||
"from Documents",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(26 * 60),
|
||||
ActivityKind::PublicLink,
|
||||
"Road trip.jpg",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
conflict(at(26 * 60 + 5), "Packing list", "txt"),
|
||||
entry(at(27 * 60), ActivityKind::CreateFolder, "Recipes", "", true),
|
||||
conflict(at(28 * 60), "Garden plan", "svg"),
|
||||
entry(at(29 * 60), ActivityKind::Trash, "Old invoices", "", true),
|
||||
entry(
|
||||
at(30 * 60),
|
||||
ActivityKind::Restore,
|
||||
"Apartment lease.pdf",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
entry(
|
||||
at(50 * 60),
|
||||
ActivityKind::Upload,
|
||||
"Videos/Harbour at dusk.mp4",
|
||||
"",
|
||||
true,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
/// Hits for the search launcher: every demo file whose name holds the
|
||||
/// query, or a few to show when nothing does.
|
||||
fn search(&self, query: &str, limit: usize) -> Vec<SearchHit> {
|
||||
let names = [
|
||||
("Work/Quarterly report.odt", false, 180_912),
|
||||
("Work/Quarterly report 2025.pdf", false, 1_204_334),
|
||||
("Work/Reports", true, 0),
|
||||
("Documents/Expense report March.ods", false, 44_201),
|
||||
("Documents/Lab report.pdf", false, 2_301_442),
|
||||
("Projects/Annual report draft.md", false, 18_400),
|
||||
("Taxes/2025/Tax report.pdf", false, 640_008),
|
||||
("Pictures/Report cover.jpg", false, 2_801_220),
|
||||
];
|
||||
let needle = query.to_lowercase();
|
||||
names
|
||||
.iter()
|
||||
.filter(|(path, _, _)| needle.is_empty() || path.to_lowercase().contains(&needle))
|
||||
.take(limit)
|
||||
.enumerate()
|
||||
.map(|(i, (path, is_dir, size))| SearchHit {
|
||||
name: path.rsplit('/').next().unwrap_or(path).into(),
|
||||
path: (*path).into(),
|
||||
is_dir: *is_dir,
|
||||
size: *size,
|
||||
modified: self.now - (i as i64 * 5 + 1) * DAY,
|
||||
pinned: i == 0,
|
||||
cached: i == 0,
|
||||
uid: format!("demo-files~/{path}"),
|
||||
mounted_path: Some(format!("{MOUNTPOINT}/{path}")),
|
||||
score: 100 - i as i64,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn local_search(&self, query: &str) -> Vec<LocalHit> {
|
||||
let needle = query.to_lowercase();
|
||||
[("/home/alex/Downloads/report-template.ott", 22_018)]
|
||||
.iter()
|
||||
.filter(|(path, _)| path.to_lowercase().contains(&needle))
|
||||
.map(|(path, size)| LocalHit {
|
||||
name: path.rsplit('/').next().unwrap_or(path).into(),
|
||||
path: (*path).into(),
|
||||
is_dir: false,
|
||||
size: *size,
|
||||
modified: self.now - 4 * DAY,
|
||||
score: 10,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn join(dir: &str, name: &str) -> String {
|
||||
let dir = dir.trim_matches('/');
|
||||
if dir.is_empty() {
|
||||
name.into()
|
||||
} else {
|
||||
format!("{dir}/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
/// The variant name of `request`, for the log of requests the demo ignores.
|
||||
fn request_name(request: &Request) -> String {
|
||||
let json = serde_json::to_string(request).unwrap_or_default();
|
||||
json.trim_start_matches(['{', '"'])
|
||||
.split('"')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
+908
-56
File diff suppressed because it is too large.
Load diff
+116
-63
@@ -579,6 +579,7 @@ impl ContentCache {
|
||||
// the disk.
|
||||
cache.reseed_totals();
|
||||
cache.import_legacy_pins()?;
|
||||
cache.set_budget(max_bytes);
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
@@ -621,10 +622,20 @@ impl ContentCache {
|
||||
for entry in rd.flatten() {
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else { continue };
|
||||
// A `.meta` tags the blob named like it without the suffix. One
|
||||
// whose blob is gone is what an eviction leaves between its two
|
||||
// removals, by a crash or by a store of the same uid landing in
|
||||
// between, and nothing else would ever remove it.
|
||||
if let Some(blob) = name.strip_suffix(".meta") {
|
||||
if !dir.join(blob).exists() {
|
||||
let _ = std::fs::remove_file(entry.path());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// `.geom` is a block-geometry sidecar (see `GeometryMeta`), tens
|
||||
// of bytes and not a cached block; like `.meta` it is neither
|
||||
// budgeted nor evictable on its own.
|
||||
if name.ends_with(".meta") || name.ends_with(".geom") {
|
||||
if name.ends_with(".geom") {
|
||||
continue;
|
||||
}
|
||||
// A `.tmp` is a staging file from a store that never completed
|
||||
@@ -1143,16 +1154,18 @@ impl ContentCache {
|
||||
};
|
||||
let mut skipped = 0usize;
|
||||
while running > cap {
|
||||
// Fetch a batch past whatever we have already skipped as exempt.
|
||||
let want = skipped + EVICT_BATCH;
|
||||
let Ok(batch) = self.db.cache_eviction_candidates(kind, want) else {
|
||||
// A whole batch can be pinned. Keep looking until the index has
|
||||
// no entries past the ones we have skipped.
|
||||
let Ok(batch) = self
|
||||
.db
|
||||
.cache_eviction_candidates(kind, EVICT_BATCH, skipped)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if batch.len() <= skipped {
|
||||
break; // nothing new to consider
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
let mut evicted_any = false;
|
||||
for (key, size) in batch.into_iter().skip(skipped) {
|
||||
for (key, size) in batch {
|
||||
if running <= cap {
|
||||
break;
|
||||
}
|
||||
@@ -1160,14 +1173,18 @@ impl ContentCache {
|
||||
skipped += 1; // counts toward the total, never a victim
|
||||
continue;
|
||||
}
|
||||
let _ = std::fs::remove_file(dir.join(&key));
|
||||
if let Err(error) = std::fs::remove_file(dir.join(&key))
|
||||
&& error.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
let _ = std::fs::remove_file(dir.join(format!("{key}.meta")));
|
||||
let _ = self.db.cache_remove(&key);
|
||||
if self.db.cache_remove(&key).is_err() {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
running = running.saturating_sub(size);
|
||||
evicted_any = true;
|
||||
}
|
||||
if !evicted_any {
|
||||
break; // every remaining candidate is exempt
|
||||
}
|
||||
}
|
||||
// The index is the truth; realign the fast path with it.
|
||||
@@ -1395,7 +1412,7 @@ impl ContentCache {
|
||||
.collect();
|
||||
let content_dir = self.content_dir.clone();
|
||||
// Force pool enforcement ignoring current cap check to free disk space immediately
|
||||
let Ok(list) = self.db.cache_eviction_candidates(KIND_BLOB, 10) else {
|
||||
let Ok(list) = self.db.cache_eviction_candidates(KIND_BLOB, 10, 0) else {
|
||||
return;
|
||||
};
|
||||
for (key, bytes) in list {
|
||||
@@ -1873,13 +1890,13 @@ impl ContentCache {
|
||||
// Pinned blobs (by cache key) are exempt from eviction. Resolves direct
|
||||
// pins plus every descendant of a recursively-pinned folder, hashed into
|
||||
// cache keys to match on-disk filenames.
|
||||
let pinned: HashSet<String> = self
|
||||
.db
|
||||
.pinned_uids()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|uid| Self::key_str(uid))
|
||||
.collect();
|
||||
let pinned = match self.db.pinned_uids() {
|
||||
Ok(uids) => uids.iter().map(|uid| Self::key_str(uid)).collect(),
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "reading pins for cache eviction failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let content_dir = self.content_dir.clone();
|
||||
self.enforce_pool(KIND_BLOB, &content_dir, Some(&pinned));
|
||||
}
|
||||
@@ -2000,8 +2017,7 @@ impl ContentCache {
|
||||
}
|
||||
|
||||
/// Retune the soft byte cap (`0` = unlimited) at runtime and immediately
|
||||
/// enforce it: blobs and blocks are LRU-evicted back under the new cap (a
|
||||
/// lower cap frees space now; a higher one is a no-op until the next store).
|
||||
/// enforce it across all three pools.
|
||||
/// Called by the daemon when the Settings page changes the cache budget.
|
||||
pub fn set_budget(&self, bytes: u64) {
|
||||
self.max_bytes.store(bytes, Ordering::Relaxed);
|
||||
@@ -2235,19 +2251,72 @@ mod tests {
|
||||
// crash) but the same on-disk cache: reconcile repopulates from disk.
|
||||
let db2 = Arc::new(Db::open_in_memory().unwrap());
|
||||
assert!(db2.cache_entries_by_kind("blob").unwrap().is_empty());
|
||||
let c = ContentCache::open(content, pins, 8, db2.clone()).unwrap();
|
||||
let c = ContentCache::open(content, pins, 0, db2.clone()).unwrap();
|
||||
assert_eq!(db2.cache_entries_by_kind("blob").unwrap().len(), 3);
|
||||
|
||||
// With the index rebuilt, a further store enforces the budget against the
|
||||
// reconciled entries instead of leaking them.
|
||||
c.store(&uid("d"), 1, 4, b"dddd").unwrap();
|
||||
let total: u64 = db2
|
||||
.cache_entries_by_kind("blob")
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|(_, s)| s)
|
||||
.sum();
|
||||
assert!(total <= 8, "budget enforced after reconcile, total={total}");
|
||||
assert_eq!(c.usage(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reopening_trims_all_cache_pools_even_when_the_oldest_batch_is_pinned() {
|
||||
let (c, _dir) = cache();
|
||||
for i in 0..EVICT_BATCH {
|
||||
let u = uid(&format!("pinned-{i}"));
|
||||
c.store(&u, 1, 1, b"p").unwrap();
|
||||
c.add_pin(&u, Path::new("pinned"), false).unwrap();
|
||||
std::fs::File::open(c.blob_path(&u))
|
||||
.unwrap()
|
||||
.set_modified(UNIX_EPOCH)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let unpinned = uid("unpinned");
|
||||
let payload = [0u8; 64];
|
||||
let size = payload.len() as u64;
|
||||
c.store(&unpinned, 1, size, &payload).unwrap();
|
||||
c.store_block(&unpinned, 1, size, blk(size, 0), &payload)
|
||||
.unwrap();
|
||||
c.store_thumbnail(&unpinned, 1, 1, &payload).unwrap();
|
||||
let recovery = c.recovery_dir.join("recovered.bin");
|
||||
std::fs::write(&recovery, b"recovered write").unwrap();
|
||||
|
||||
let budget = (EVICT_BATCH as u64 * 100).div_ceil(SPLIT_BLOB);
|
||||
let reopened = ContentCache::open(
|
||||
c.content_dir.clone(),
|
||||
c.pins_path.clone(),
|
||||
budget,
|
||||
c.db.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(reopened.usage(), EVICT_BATCH as u64);
|
||||
assert!(!reopened.blob_path(&unpinned).exists());
|
||||
for i in 0..EVICT_BATCH {
|
||||
assert!(reopened.is_cached(&uid(&format!("pinned-{i}")), 1, 1));
|
||||
}
|
||||
assert_eq!(std::fs::read(recovery).unwrap(), b"recovered write");
|
||||
}
|
||||
|
||||
/// An eviction removes the blob, then its tag. A crash in between, or a
|
||||
/// store of the same uid landing in between, left the tag on its own, and
|
||||
/// nothing removed it.
|
||||
#[test]
|
||||
fn a_tag_without_its_blob_is_swept_at_open() {
|
||||
let dir = TempDir::new();
|
||||
let content = dir.path().join("content");
|
||||
let pins = dir.path().join("pins.json");
|
||||
let db = Arc::new(Db::open_in_memory().unwrap());
|
||||
{
|
||||
let c = ContentCache::open(content.clone(), pins.clone(), 0, db.clone()).unwrap();
|
||||
c.store(&uid("kept"), 1, 4, b"kept").unwrap();
|
||||
c.store(&uid("gone"), 1, 4, b"gone").unwrap();
|
||||
std::fs::remove_file(c.blob_path(&uid("gone"))).unwrap();
|
||||
}
|
||||
|
||||
let c = ContentCache::open(content, pins, 0, db).unwrap();
|
||||
assert!(!c.meta_path(&uid("gone")).exists());
|
||||
assert!(c.meta_path(&uid("kept")).exists());
|
||||
assert_eq!(c.read_range(&uid("kept"), 1, 4, 0, 4).unwrap(), b"kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2309,42 +2378,26 @@ mod tests {
|
||||
/// call also needs.
|
||||
#[test]
|
||||
fn budget_check_is_free_when_under_budget() {
|
||||
use std::time::Instant;
|
||||
|
||||
// Cap far above what we store, so every store stays under budget.
|
||||
let (c, _d) = cache_capped(1 << 30);
|
||||
let payload = vec![0u8; 4096];
|
||||
|
||||
// Cost of a store against a near-empty index...
|
||||
let t = Instant::now();
|
||||
for i in 0..500u64 {
|
||||
for i in 0..8u64 {
|
||||
c.store_block(&uid("a"), 1, 1 << 30, blk(1 << 30, i), &payload)
|
||||
.unwrap();
|
||||
}
|
||||
let empty = t.elapsed() / 500;
|
||||
|
||||
// ...and against one with a few thousand entries in it.
|
||||
for i in 500..2500u64 {
|
||||
c.store_block(&uid("a"), 1, 1 << 30, blk(1 << 30, i), &payload)
|
||||
.unwrap();
|
||||
}
|
||||
let t = Instant::now();
|
||||
for i in 2500..3000u64 {
|
||||
c.store_block(&uid("a"), 1, 1 << 30, blk(1 << 30, i), &payload)
|
||||
.unwrap();
|
||||
}
|
||||
let full = t.elapsed() / 500;
|
||||
// Plant an index row far over the cap behind the running total's back,
|
||||
// and older than every real block. Any budget check that consults the
|
||||
// index would see the pool over budget, evict, and reseed the total.
|
||||
c.db.cache_touch("planted", KIND_BLOCK, 1 << 40, 0).unwrap();
|
||||
|
||||
println!("B4: store_block under budget — {empty:?} at ~0 entries, {full:?} at 2500");
|
||||
// A ratio, not a wall-clock bound: the claim is that the under-budget
|
||||
// path does not grow with the index, and only a ratio tests that. An
|
||||
// absolute threshold measures the machine's load instead, and this test
|
||||
// runs alongside the rest of the suite.
|
||||
assert!(
|
||||
full < empty * 3,
|
||||
"under-budget store should not scale with the index: \
|
||||
{empty:?} at ~0 entries vs {full:?} at 2500"
|
||||
);
|
||||
c.store_block(&uid("a"), 1, 1 << 30, blk(1 << 30, 8), &payload)
|
||||
.unwrap();
|
||||
|
||||
// Not a timing ratio: that measured the CI runner's load and flaked.
|
||||
assert_eq!(c.block_bytes.load(Ordering::Relaxed), 9 * 4096);
|
||||
let entries = c.db.cache_entries_by_kind(KIND_BLOCK).unwrap();
|
||||
assert_eq!(entries.len(), 10, "under-budget store evicted: {entries:?}");
|
||||
}
|
||||
|
||||
/// The running totals are a fast path, not a second source of truth: after
|
||||
|
||||
+459
-48
@@ -163,6 +163,14 @@ pub struct AppConfig {
|
||||
/// Defaulted for configs predating the field.
|
||||
#[serde(default)]
|
||||
pub files_view: FilesView,
|
||||
/// How the desktop app's Photos page lays out its grid. Defaulted for
|
||||
/// configs predating the field.
|
||||
#[serde(default)]
|
||||
pub photos_view: PhotosView,
|
||||
/// The desktop app's main window size, remembered between runs. Defaulted
|
||||
/// for configs predating the field.
|
||||
#[serde(default)]
|
||||
pub window: WindowState,
|
||||
/// The user hid the tray icon from its own menu. The tray exits at start
|
||||
/// while this is set, and the app does not spawn it; Preferences turns it
|
||||
/// back on. Defaulted for configs predating the field.
|
||||
@@ -186,6 +194,24 @@ pub struct AppConfig {
|
||||
/// the field.
|
||||
#[serde(default)]
|
||||
pub local_first: Option<bool>,
|
||||
/// Ask GitHub once a day whether a newer release is out, and say so in the
|
||||
/// desktop app. Off by default: GitHub then sees the address and version
|
||||
/// asking. Never used where a package manager installs updates (see
|
||||
/// [`crate::update::updates_managed`]). Defaulted for configs predating the
|
||||
/// field.
|
||||
#[serde(default)]
|
||||
pub update_check: bool,
|
||||
/// When the update check last asked GitHub, in Unix seconds, so a restart
|
||||
/// does not ask again within [`crate::update::CHECK_INTERVAL`]. Defaulted
|
||||
/// for configs predating the field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub update_last_checked: Option<i64>,
|
||||
/// The latest release that check found, such as `3.6.0`, so the app can
|
||||
/// say so again after a restart without asking. Older than or equal to the
|
||||
/// running version once the update is installed. Defaulted for configs
|
||||
/// predating the field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub update_latest: Option<String>,
|
||||
}
|
||||
|
||||
/// The My Files page's view choices, remembered between runs.
|
||||
@@ -198,6 +224,10 @@ pub struct FilesView {
|
||||
pub descending: bool,
|
||||
/// Folders ahead of files, whatever the sort.
|
||||
pub folders_first: bool,
|
||||
/// The icon grid's thumbnail edge in logical pixels. `None` means the
|
||||
/// app's default.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub icon_size: Option<i32>,
|
||||
}
|
||||
|
||||
impl Default for FilesView {
|
||||
@@ -207,6 +237,57 @@ impl Default for FilesView {
|
||||
sort: FileSort::Name,
|
||||
descending: false,
|
||||
folders_first: true,
|
||||
icon_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The Photos page's grid choices, remembered between runs.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
#[serde(default)]
|
||||
pub struct PhotosView {
|
||||
/// The zoom level: the row height the grid aims for, in logical pixels.
|
||||
/// `None` means the app's default.
|
||||
pub row_height: Option<i32>,
|
||||
/// Crop every photo to a square tile instead of showing it at its own
|
||||
/// shape.
|
||||
pub square: bool,
|
||||
}
|
||||
|
||||
/// The desktop app's main window, as it was when it last closed.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[serde(default)]
|
||||
pub struct WindowState {
|
||||
/// The unmaximised size, in logical pixels.
|
||||
pub width: i32,
|
||||
pub height: i32,
|
||||
pub maximized: bool,
|
||||
}
|
||||
|
||||
impl WindowState {
|
||||
/// Below this the saved size is taken for damage and the default is used.
|
||||
pub const MIN_WIDTH: i32 = 480;
|
||||
pub const MIN_HEIGHT: i32 = 400;
|
||||
|
||||
/// This state, with a size too small to use replaced by the default.
|
||||
pub fn usable(self) -> Self {
|
||||
if self.width < Self::MIN_WIDTH || self.height < Self::MIN_HEIGHT {
|
||||
Self {
|
||||
maximized: self.maximized,
|
||||
..Self::default()
|
||||
}
|
||||
} else {
|
||||
self
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WindowState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
width: 980,
|
||||
height: 680,
|
||||
maximized: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,6 +391,134 @@ impl AppConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply `edit` to the ignore patterns in effect in the config at `path`,
|
||||
/// under the config lock, and return the saved config. A list equal to the
|
||||
/// built-in one is stored as unset, so it follows the built-in list in later
|
||||
/// releases.
|
||||
pub fn change_ignore_patterns_at(
|
||||
path: &Path,
|
||||
edit: impl FnOnce(&mut Vec<String>),
|
||||
) -> Result<AppConfig> {
|
||||
update_config_at(path, |config| {
|
||||
let mut list = config.resolved_ignore_patterns();
|
||||
edit(&mut list);
|
||||
let builtin = list.len() == DEFAULT_IGNORE_PATTERNS.len()
|
||||
&& list
|
||||
.iter()
|
||||
.zip(DEFAULT_IGNORE_PATTERNS)
|
||||
.all(|(a, b)| a == b);
|
||||
config.ignore_patterns = (!builtin).then_some(list);
|
||||
config.clone()
|
||||
})
|
||||
}
|
||||
|
||||
/// Fail when the config file at `path` exists but does not parse, so that
|
||||
/// [`AppDirs::save_config`] never writes over settings it could not read.
|
||||
fn refuse_to_replace_unreadable(path: &Path) -> Result<()> {
|
||||
match std::fs::read_to_string(path) {
|
||||
Ok(content) => match serde_json::from_str::<AppConfig>(&content) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(Error::Other(format!(
|
||||
"{} cannot be read ({e}); fix it or move it aside before changing settings",
|
||||
path.display()
|
||||
))),
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`AppDirs::load_config`] for the file at `path`.
|
||||
fn load_config_at(path: &Path) -> AppConfig {
|
||||
if path.exists() {
|
||||
match std::fs::read_to_string(path)
|
||||
.map_err(Error::from)
|
||||
.and_then(|content| serde_json::from_str::<AppConfig>(&content).map_err(Error::from))
|
||||
{
|
||||
Ok(config) => return config,
|
||||
Err(e) => {
|
||||
// Preserve the original bytes for diagnosis/recovery. In
|
||||
// particular, never replace a truncated config with defaults
|
||||
// and silently lose mount, device, or ignore settings.
|
||||
// `save_config` refuses to write over it for the same reason.
|
||||
tracing::error!(path = %path.display(), error = %e, "config is unreadable; using in-memory defaults without overwriting it");
|
||||
return AppConfig::default();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let default_config = AppConfig::default();
|
||||
if let Err(e) = save_config_at(path, &default_config) {
|
||||
tracing::warn!(path = %path.display(), error = %e, "could not write the default config");
|
||||
}
|
||||
default_config
|
||||
}
|
||||
|
||||
/// [`AppDirs::save_config`] for the file at `path`.
|
||||
fn save_config_at(path: &Path, config: &AppConfig) -> Result<()> {
|
||||
refuse_to_replace_unreadable(path)?;
|
||||
proton_base_rs::atomic::write(path, &serde_json::to_vec_pretty(config)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// [`AppDirs::update_config`] for the file at `path`.
|
||||
fn update_config_at<R>(path: &Path, change: impl FnOnce(&mut AppConfig) -> R) -> Result<R> {
|
||||
let _lock = lock_config(path)?;
|
||||
// Refuse before reading: an unreadable file loads as defaults, and the
|
||||
// change must not be applied to those.
|
||||
refuse_to_replace_unreadable(path)?;
|
||||
let mut config = load_config_at(path);
|
||||
let out = change(&mut config);
|
||||
save_config_at(path, &config)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// How long [`lock_config`] waits for another writer. A change holds the lock
|
||||
/// for one small read and write; anything longer is a stuck process, and the
|
||||
/// setting is better reported as not saved than waited on forever.
|
||||
const CONFIG_LOCK_WAIT: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
/// Take the lock that orders changes to the config file at `path`, held until
|
||||
/// the returned file is dropped.
|
||||
///
|
||||
/// A separate `config.json.lock`, never the config file itself: that file is
|
||||
/// replaced by a rename on every save, so a lock on it would guard a file
|
||||
/// nobody reads any more.
|
||||
fn lock_config(path: &Path) -> Result<std::fs::File> {
|
||||
use std::os::fd::AsRawFd as _;
|
||||
use std::os::unix::fs::OpenOptionsExt as _;
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut name = path.as_os_str().to_owned();
|
||||
name.push(".lock");
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.truncate(false)
|
||||
.mode(0o600)
|
||||
.open(PathBuf::from(name))?;
|
||||
let deadline = std::time::Instant::now() + CONFIG_LOCK_WAIT;
|
||||
loop {
|
||||
// SAFETY: `file` owns the descriptor and outlives the call.
|
||||
if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 {
|
||||
return Ok(file);
|
||||
}
|
||||
let e = std::io::Error::last_os_error();
|
||||
if e.raw_os_error() != Some(libc::EWOULDBLOCK) {
|
||||
return Err(Error::Io(e));
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
return Err(Error::Other(format!(
|
||||
"{} is being changed by another process; try again",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolved XDG locations for state, cache, and the default mountpoint.
|
||||
pub struct AppDirs {
|
||||
dirs: ProjectDirs,
|
||||
@@ -329,58 +538,73 @@ impl AppDirs {
|
||||
|
||||
/// Load config from disk, creating default if missing.
|
||||
pub fn load_config(&self) -> AppConfig {
|
||||
let path = self.config_path();
|
||||
if path.exists() {
|
||||
match std::fs::read_to_string(&path)
|
||||
.map_err(Error::from)
|
||||
.and_then(|content| {
|
||||
serde_json::from_str::<AppConfig>(&content).map_err(Error::from)
|
||||
}) {
|
||||
Ok(config) => return config,
|
||||
Err(e) => {
|
||||
// Preserve the original bytes for diagnosis/recovery. In
|
||||
// particular, never replace a truncated config with defaults
|
||||
// and silently lose mount, device, or ignore settings.
|
||||
tracing::error!(path = %path.display(), error = %e, "config is unreadable; using in-memory defaults without overwriting it");
|
||||
return AppConfig::default();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let default_config = AppConfig::default();
|
||||
if let Ok(content) = serde_json::to_string_pretty(&default_config) {
|
||||
let _ = std::fs::create_dir_all(self.dirs.config_dir());
|
||||
let _ = std::fs::write(&path, content);
|
||||
}
|
||||
default_config
|
||||
load_config_at(&self.config_path())
|
||||
}
|
||||
|
||||
/// Persist `config` to [`config_path`](Self::config_path), creating the
|
||||
/// config dir if missing. The Settings page calls this after editing the
|
||||
/// cache budget or mountpoint; the daemon re-reads the file on its next mount.
|
||||
/// config dir if missing.
|
||||
///
|
||||
/// Refuses while the file on disk does not parse: [`load_config`](Self::load_config)
|
||||
/// gives defaults for a file it cannot read, so the first change after the
|
||||
/// file was damaged would write defaults over the mountpoint, the device
|
||||
/// and the ignore rules the user had (`docs/BUGS.md` B223).
|
||||
///
|
||||
/// Writes the whole file. To change a setting, use
|
||||
/// [`update_config`](Self::update_config), which does not lose another
|
||||
/// process's change.
|
||||
pub fn save_config(&self, config: &AppConfig) -> Result<()> {
|
||||
let path = self.config_path();
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| Error::Other("configuration path has no parent".into()))?;
|
||||
std::fs::create_dir_all(parent)?;
|
||||
let nonce = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|duration| duration.as_nanos())
|
||||
.unwrap_or(0);
|
||||
let tmp = path.with_extension(format!("json.tmp-{}-{nonce}", std::process::id()));
|
||||
use std::os::unix::fs::OpenOptionsExt as _;
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(&tmp)?;
|
||||
use std::io::Write as _;
|
||||
file.write_all(&serde_json::to_vec_pretty(config)?)?;
|
||||
file.sync_all()?;
|
||||
std::fs::rename(&tmp, &path)?;
|
||||
std::fs::File::open(parent)?.sync_all()?;
|
||||
Ok(())
|
||||
save_config_at(&self.config_path(), config)
|
||||
}
|
||||
|
||||
/// Change settings in place: read the file, apply `change`, write it back,
|
||||
/// all under a lock, and return what `change` returned.
|
||||
///
|
||||
/// The app, the tray and the daemon each write the file. Without the lock,
|
||||
/// two of them changing different settings at once both read the old file,
|
||||
/// and the one that wrote last undid the other's change (`docs/BUGS.md`
|
||||
/// B292).
|
||||
pub fn update_config<R>(&self, change: impl FnOnce(&mut AppConfig) -> R) -> Result<R> {
|
||||
update_config_at(&self.config_path(), change)
|
||||
}
|
||||
|
||||
/// Apply `edit` to the ignore patterns in effect for every synced folder
|
||||
/// and save them, returning the saved config. See
|
||||
/// [`change_ignore_patterns_at`].
|
||||
pub fn change_ignore_patterns(&self, edit: impl FnOnce(&mut Vec<String>)) -> Result<AppConfig> {
|
||||
change_ignore_patterns_at(&self.config_path(), edit)
|
||||
}
|
||||
|
||||
/// This machine's id (e.g. ~/.config/proton-drive-linux/machine-id). See
|
||||
/// [`machine_id`](Self::machine_id).
|
||||
pub fn machine_id_path(&self) -> PathBuf {
|
||||
self.dirs.config_dir().join("machine-id")
|
||||
}
|
||||
|
||||
/// A random id for this machine, made on first use and kept in
|
||||
/// [`machine_id_path`](Self::machine_id_path).
|
||||
///
|
||||
/// It names this machine's profile in a device it may share with other
|
||||
/// computers. Random rather than `/etc/machine-id`, so nothing that
|
||||
/// identifies the host goes to Drive. In a file of its own rather than
|
||||
/// `config.json`, because the app saves the config from a copy it loaded
|
||||
/// earlier, and that would drop an id the daemon had just added.
|
||||
pub fn machine_id(&self) -> Result<String> {
|
||||
let path = self.machine_id_path();
|
||||
match std::fs::read_to_string(&path) {
|
||||
Ok(id) if is_machine_id(id.trim()) => return Ok(id.trim().to_string()),
|
||||
Ok(_) => {
|
||||
tracing::warn!(path = %path.display(), "machine id is unusable; making a new one");
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
let mut bytes = [0u8; 16];
|
||||
use std::io::Read as _;
|
||||
std::fs::File::open("/dev/urandom")?.read_exact(&mut bytes)?;
|
||||
let id: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
|
||||
// Not secret, and other tools may read it; keep it world-readable.
|
||||
proton_base_rs::atomic::write_with_mode(&path, format!("{id}\n").as_bytes(), 0o644)?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Persistent state: inode map / cache index DB lives here.
|
||||
@@ -418,6 +642,12 @@ impl AppDirs {
|
||||
self.state_dir().join("control.sock")
|
||||
}
|
||||
|
||||
/// The Google Photos import's last run: its archives and how it ended, so
|
||||
/// an import cut short by a restart can be resumed.
|
||||
pub fn takeout_import_path(&self) -> PathBuf {
|
||||
self.state_dir().join("takeout-import.json")
|
||||
}
|
||||
|
||||
/// Unix domain socket the tray uses to ensure single instance.
|
||||
pub fn tray_socket(&self) -> PathBuf {
|
||||
self.state_dir().join("tray.sock")
|
||||
@@ -471,6 +701,12 @@ impl AppDirs {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `id` is what [`AppDirs::machine_id`] writes: 32 lowercase hex
|
||||
/// digits.
|
||||
fn is_machine_id(id: &str) -> bool {
|
||||
id.len() == 32 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
|
||||
}
|
||||
|
||||
/// Make `dir` owner-only (`0700`). These directories contain decrypted bytes,
|
||||
/// plaintext names, and an authenticated control socket, so failure is fatal.
|
||||
fn restrict_dir(dir: &Path) -> Result<()> {
|
||||
@@ -526,6 +762,111 @@ mod tests {
|
||||
std::fs::metadata(p).unwrap().permissions().mode() & 0o777
|
||||
}
|
||||
|
||||
/// Two writers changing different settings at once both keep their change
|
||||
/// (`docs/BUGS.md` B292).
|
||||
#[test]
|
||||
fn concurrent_changes_to_different_settings_are_all_kept() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"pdfs-cfg-race-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("config.json");
|
||||
|
||||
let a = {
|
||||
let path = path.clone();
|
||||
std::thread::spawn(move || {
|
||||
for i in 1..=100u64 {
|
||||
update_config_at(&path, |c| c.upload_limit = Some(i)).unwrap();
|
||||
}
|
||||
})
|
||||
};
|
||||
let b = {
|
||||
let path = path.clone();
|
||||
std::thread::spawn(move || {
|
||||
for i in 1..=100u64 {
|
||||
update_config_at(&path, |c| c.download_limit = Some(i)).unwrap();
|
||||
}
|
||||
})
|
||||
};
|
||||
a.join().unwrap();
|
||||
b.join().unwrap();
|
||||
|
||||
let config = load_config_at(&path);
|
||||
assert_eq!(config.upload_limit, Some(100));
|
||||
assert_eq!(config.download_limit, Some(100));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignore_patterns_back_at_the_built_in_list_are_stored_as_unset() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"pdfs-cfg-ignore-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("config.json");
|
||||
|
||||
let config = change_ignore_patterns_at(&path, |list| list.push("*.db".into())).unwrap();
|
||||
assert_eq!(
|
||||
config.ignore_patterns.as_ref().unwrap().last().unwrap(),
|
||||
"*.db"
|
||||
);
|
||||
let config = change_ignore_patterns_at(&path, |list| list.retain(|p| p != "*.db")).unwrap();
|
||||
assert_eq!(config.ignore_patterns, None);
|
||||
assert_eq!(load_config_at(&path).ignore_patterns, None);
|
||||
// An empty list is a choice, not the built-in one.
|
||||
let config = change_ignore_patterns_at(&path, Vec::clear).unwrap();
|
||||
assert_eq!(config.ignore_patterns, Some(Vec::new()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_change_is_never_applied_over_an_unreadable_config() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"pdfs-cfg-update-guard-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("config.json");
|
||||
std::fs::write(&path, br#"{"mountpoint": "/home/me/Pro"#).unwrap();
|
||||
|
||||
assert!(update_config_at(&path, |c| c.tray_hidden = true).is_err());
|
||||
assert_eq!(
|
||||
std::fs::read(&path).unwrap(),
|
||||
br#"{"mountpoint": "/home/me/Pro"#
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A damaged config must not be replaced by defaults the next time a
|
||||
/// setting changes (`docs/BUGS.md` B223).
|
||||
#[test]
|
||||
fn an_unreadable_config_is_never_written_over() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"pdfs-cfg-guard-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("config.json");
|
||||
|
||||
assert!(refuse_to_replace_unreadable(&path).is_ok(), "no file yet");
|
||||
std::fs::write(&path, serde_json::to_vec(&AppConfig::default()).unwrap()).unwrap();
|
||||
assert!(refuse_to_replace_unreadable(&path).is_ok());
|
||||
std::fs::write(&path, br#"{"mountpoint": "/home/me/Pro"#).unwrap();
|
||||
assert!(refuse_to_replace_unreadable(&path).is_err(), "truncated");
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// `docs/BUGS.md` B6. A permissive mode here is not a cosmetic problem: these
|
||||
/// directories hold decrypted content, a plaintext index of every node name,
|
||||
/// and the control socket that commands the daemon's session.
|
||||
@@ -610,11 +951,24 @@ mod tests {
|
||||
sort: FileSort::Modified,
|
||||
descending: true,
|
||||
folders_first: false,
|
||||
icon_size: Some(192),
|
||||
},
|
||||
photos_view: PhotosView {
|
||||
row_height: Some(240),
|
||||
square: true,
|
||||
},
|
||||
window: WindowState {
|
||||
width: 1200,
|
||||
height: 800,
|
||||
maximized: true,
|
||||
},
|
||||
tray_hidden: true,
|
||||
language: Some("de".to_string()),
|
||||
online_map: true,
|
||||
local_first: Some(false),
|
||||
update_check: true,
|
||||
update_last_checked: Some(1_760_000_000),
|
||||
update_latest: Some("3.6.0".to_string()),
|
||||
};
|
||||
let json = serde_json::to_string(&config).unwrap();
|
||||
let decoded: AppConfig = serde_json::from_str(&json).unwrap();
|
||||
@@ -626,10 +980,47 @@ mod tests {
|
||||
assert_eq!(decoded.proton_theme, Some(true));
|
||||
assert!(decoded.tray_hidden);
|
||||
assert!(decoded.online_map);
|
||||
assert!(decoded.update_check);
|
||||
assert_eq!(decoded.update_last_checked, Some(1_760_000_000));
|
||||
assert_eq!(decoded.update_latest.as_deref(), Some("3.6.0"));
|
||||
assert_eq!(decoded.language.as_deref(), Some("de"));
|
||||
assert_eq!(decoded.upload_limit, Some(500_000));
|
||||
assert_eq!(decoded.download_limit, None);
|
||||
assert_eq!(decoded.files_view, config.files_view);
|
||||
assert_eq!(decoded.photos_view, config.photos_view);
|
||||
assert_eq!(decoded.window, config.window);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_config_without_a_window_opens_at_the_default_size() {
|
||||
let decoded: AppConfig =
|
||||
serde_json::from_str(r#"{"app_version":"a","user_agent":"b"}"#).unwrap();
|
||||
assert_eq!(decoded.window, WindowState::default());
|
||||
let partial: WindowState = serde_json::from_str(r#"{"maximized":true}"#).unwrap();
|
||||
assert_eq!((partial.width, partial.height), (980, 680));
|
||||
assert!(partial.maximized);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_window_saved_too_small_opens_at_the_default_size() {
|
||||
let tiny = WindowState {
|
||||
width: 0,
|
||||
height: 500,
|
||||
maximized: true,
|
||||
};
|
||||
assert_eq!(
|
||||
tiny.usable(),
|
||||
WindowState {
|
||||
maximized: true,
|
||||
..WindowState::default()
|
||||
}
|
||||
);
|
||||
let fine = WindowState {
|
||||
width: 640,
|
||||
height: 480,
|
||||
maximized: false,
|
||||
};
|
||||
assert_eq!(fine.usable(), fine);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -640,6 +1031,17 @@ mod tests {
|
||||
let partial: FilesView = serde_json::from_str(r#"{"sort":"size"}"#).unwrap();
|
||||
assert_eq!(partial.sort, FileSort::Size);
|
||||
assert!(partial.folders_first);
|
||||
assert_eq!(partial.icon_size, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_config_without_a_photos_view_gets_the_default() {
|
||||
let decoded: AppConfig =
|
||||
serde_json::from_str(r#"{"app_version":"a","user_agent":"b"}"#).unwrap();
|
||||
assert_eq!(decoded.photos_view, PhotosView::default());
|
||||
let partial: PhotosView = serde_json::from_str(r#"{"square":true}"#).unwrap();
|
||||
assert!(partial.square);
|
||||
assert_eq!(partial.row_height, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -760,6 +1162,15 @@ mod tests {
|
||||
assert_eq!(decoded.conflict_sweep, Some(SweepMode::Enforce));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_machine_id_is_32_lowercase_hex_digits() {
|
||||
assert!(is_machine_id("0123456789abcdef0123456789abcdef"));
|
||||
assert!(!is_machine_id("0123456789ABCDEF0123456789ABCDEF"));
|
||||
assert!(!is_machine_id("0123456789abcdef"));
|
||||
assert!(!is_machine_id("../../../../etc/passwd/0123456789"));
|
||||
assert!(!is_machine_id(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_config_predating_ignore_patterns_still_loads() {
|
||||
// Every installed config on disk looks like this; the field must not be
|
||||
|
||||
+843
-137
File diff suppressed because it is too large.
Load diff
@@ -31,8 +31,30 @@ pub struct StoredAlbum {
|
||||
impl Db {
|
||||
/// Replace the album listing wholesale, in the given order.
|
||||
pub fn albums_replace(&self, albums: &[StoredAlbum]) -> Result<()> {
|
||||
self.albums_replace_impl(albums, None).map(|_| ())
|
||||
}
|
||||
|
||||
pub fn albums_replace_at(
|
||||
&self,
|
||||
albums: &[StoredAlbum],
|
||||
generation: i64,
|
||||
now: i64,
|
||||
) -> Result<bool> {
|
||||
self.albums_replace_impl(albums, Some((generation, now)))
|
||||
}
|
||||
|
||||
fn albums_replace_impl(
|
||||
&self,
|
||||
albums: &[StoredAlbum],
|
||||
stamp: Option<(i64, i64)>,
|
||||
) -> Result<bool> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
if let Some((generation, _)) = stamp
|
||||
&& super::photos::photos_generation(&tx)? != generation
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
// Contents of an album that no longer exists (unshared, deleted) go with
|
||||
// it — otherwise they would linger forever, keyed by an album uid no
|
||||
// listing mentions again.
|
||||
@@ -56,7 +78,11 @@ impl Db {
|
||||
let mut stmt = tx.prepare(
|
||||
"INSERT INTO albums
|
||||
(uid, name, photo_count, cover_uid, last_activity, shared, seq)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||
SELECT ?1,?2,?3,
|
||||
CASE WHEN EXISTS(SELECT 1 FROM photo_tombstones WHERE uid=?4)
|
||||
THEN (SELECT uid FROM album_photos WHERE album_uid=?1 ORDER BY seq LIMIT 1) ELSE ?4 END,
|
||||
?5,?6,?7
|
||||
WHERE NOT EXISTS(SELECT 1 FROM photo_tombstones WHERE uid=?1)",
|
||||
)?;
|
||||
for (seq, album) in albums.iter().enumerate() {
|
||||
stmt.execute(params![
|
||||
@@ -70,8 +96,11 @@ impl Db {
|
||||
])?;
|
||||
}
|
||||
}
|
||||
if let Some((_, now)) = stamp {
|
||||
super::photos::gallery_state(&tx, "albums_synced_ms", now)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// The persisted album listing, in stored order.
|
||||
@@ -111,15 +140,44 @@ impl Db {
|
||||
album_uid: &str,
|
||||
items: &[(String, i64, Option<String>, Option<String>)],
|
||||
) -> Result<()> {
|
||||
self.album_photos_replace_impl(album_uid, items, None)
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
pub fn album_photos_replace_at(
|
||||
&self,
|
||||
album_uid: &str,
|
||||
items: &[(String, i64, Option<String>, Option<String>)],
|
||||
generation: i64,
|
||||
now: i64,
|
||||
) -> Result<bool> {
|
||||
self.album_photos_replace_impl(album_uid, items, Some((generation, now)))
|
||||
}
|
||||
|
||||
fn album_photos_replace_impl(
|
||||
&self,
|
||||
album_uid: &str,
|
||||
items: &[(String, i64, Option<String>, Option<String>)],
|
||||
stamp: Option<(i64, i64)>,
|
||||
) -> Result<bool> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let learned: HashMap<String, (Option<f64>, i64, Option<String>)> = {
|
||||
if let Some((generation, _)) = stamp
|
||||
&& super::photos::photos_generation(&tx)? != generation
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
type StoredPhotoMetadata = (Option<f64>, i64, Option<String>, Option<String>);
|
||||
let learned: HashMap<String, StoredPhotoMetadata> = {
|
||||
let mut stmt = tx.prepare(
|
||||
"SELECT uid, ratio, thumb_state, media_type FROM album_photos
|
||||
"SELECT uid, ratio, thumb_state, media_type, name FROM album_photos
|
||||
WHERE album_uid = ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map([album_uid], |r| {
|
||||
Ok((r.get::<_, String>(0)?, (r.get(1)?, r.get(2)?, r.get(3)?)))
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
(r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?),
|
||||
))
|
||||
})?;
|
||||
rows.collect::<rusqlite::Result<_>>()?
|
||||
};
|
||||
@@ -128,15 +186,17 @@ impl Db {
|
||||
{
|
||||
let mut stmt = tx.prepare(
|
||||
"INSERT INTO album_photos
|
||||
(album_uid, uid, capture_time, name, media_type, kind, ratio, thumb_state, seq)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
(album_uid, uid, capture_time, name, media_type, kind, ratio, thumb_state, seq, resolved)
|
||||
SELECT ?1,?2,?3,?4,?5,?6,?7,?8,?9,?10
|
||||
WHERE NOT EXISTS(SELECT 1 FROM photo_tombstones WHERE uid IN (?1,?2))",
|
||||
)?;
|
||||
for (seq, (uid, capture_time, name, media_type)) in items.iter().enumerate() {
|
||||
let (ratio, thumb_state, learned_media) =
|
||||
learned
|
||||
.get(uid)
|
||||
.cloned()
|
||||
.unwrap_or((None, THUMB_UNKNOWN, None));
|
||||
let (ratio, thumb_state, learned_media, learned_name) = learned
|
||||
.get(uid)
|
||||
.cloned()
|
||||
.unwrap_or((None, THUMB_UNKNOWN, None, None));
|
||||
let resolved = name.is_some();
|
||||
let name = name.clone().or(learned_name);
|
||||
let media_type = media_type.clone().or(learned_media);
|
||||
let kind =
|
||||
crate::control::PhotoKind::classify(name.as_deref(), media_type.as_deref());
|
||||
@@ -150,11 +210,32 @@ impl Db {
|
||||
ratio,
|
||||
thumb_state,
|
||||
seq as i64,
|
||||
resolved,
|
||||
])?;
|
||||
}
|
||||
}
|
||||
if let Some((_, now)) = stamp {
|
||||
super::photos::gallery_state(&tx, &format!("album_synced_ms:{album_uid}"), now)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Reuse current timeline metadata and resolved album-only metadata.
|
||||
pub fn album_photo_metadata(
|
||||
&self,
|
||||
album: &str,
|
||||
) -> Result<HashMap<String, (String, Option<String>)>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid,name,media_type FROM album_photos
|
||||
WHERE album_uid=?1 AND resolved=1 AND name IS NOT NULL
|
||||
UNION ALL SELECT uid,name,media_type FROM photos
|
||||
WHERE resolved_at IS NOT NULL AND name IS NOT NULL",
|
||||
)?;
|
||||
Ok(stmt
|
||||
.query_map([album], |r| Ok((r.get(0)?, (r.get(1)?, r.get(2)?))))?
|
||||
.collect::<rusqlite::Result<_>>()?)
|
||||
}
|
||||
|
||||
/// One page of an album's photos, in stored order.
|
||||
@@ -163,31 +244,47 @@ impl Db {
|
||||
album_uid: &str,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
self.album_photos_page_visible(album_uid, offset, limit, "[]")
|
||||
}
|
||||
|
||||
pub fn album_photos_page_visible(
|
||||
&self,
|
||||
album_uid: &str,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
excluded: &str,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT uid, capture_time, name, ratio, thumb_state, kind FROM album_photos
|
||||
WHERE album_uid = ?1 ORDER BY seq LIMIT ?2 OFFSET ?3",
|
||||
WHERE album_uid = ?1 AND uid NOT IN (SELECT value FROM json_each(?4))
|
||||
ORDER BY seq LIMIT ?2 OFFSET ?3",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map(params![album_uid, limit as i64, offset as i64], |r| {
|
||||
Ok(StoredPhoto {
|
||||
uid: r.get(0)?,
|
||||
capture_time: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
ratio: r.get(3)?,
|
||||
thumb_state: r.get(4)?,
|
||||
kind: crate::control::PhotoKind::from_i64(r.get(5)?),
|
||||
// Album rows have no favorite state of their own: an album
|
||||
// photo on someone else's volume is not in our timeline, and
|
||||
// favouriting one is not supported yet anyway.
|
||||
favorite: false,
|
||||
// Nor are album rows grouped: an album is a list the user
|
||||
// made, and it shows exactly what they put in it.
|
||||
group_size: 1,
|
||||
has_raw: false,
|
||||
})
|
||||
})?
|
||||
.query_map(
|
||||
params![album_uid, limit as i64, offset as i64, excluded],
|
||||
|r| {
|
||||
Ok(StoredPhoto {
|
||||
uid: r.get(0)?,
|
||||
capture_time: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
ratio: r.get(3)?,
|
||||
thumb_state: r.get(4)?,
|
||||
kind: crate::control::PhotoKind::from_i64(r.get(5)?),
|
||||
// Album rows have no favorite state of their own: an album
|
||||
// photo on someone else's volume is not in our timeline, and
|
||||
// favouriting one is not supported yet anyway.
|
||||
favorite: false,
|
||||
// Nor are album rows grouped: an album is a list the user
|
||||
// made, and it shows exactly what they put in it.
|
||||
group_size: 1,
|
||||
has_raw: false,
|
||||
// Album rows carry no location.
|
||||
place_id: None,
|
||||
})
|
||||
},
|
||||
)?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
@@ -247,6 +344,8 @@ impl Db {
|
||||
// made, and it shows exactly what they put in it.
|
||||
group_size: 1,
|
||||
has_raw: false,
|
||||
// Album rows carry no location.
|
||||
place_id: None,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
|
||||
@@ -108,7 +108,8 @@ impl Db {
|
||||
Ok(total.max(0) as u64)
|
||||
}
|
||||
|
||||
/// The `limit` least-recently-accessed entries of `kind`, oldest first.
|
||||
/// The `limit` least-recently-accessed entries of `kind` after `offset`,
|
||||
/// oldest first.
|
||||
///
|
||||
/// Eviction wants a handful of victims, not the whole table. Bounded so a
|
||||
/// pass over a large cache does not read every row to drop a few, and so the
|
||||
@@ -118,13 +119,14 @@ impl Db {
|
||||
&self,
|
||||
kind: &str,
|
||||
limit: usize,
|
||||
offset: usize,
|
||||
) -> Result<Vec<(String, u64)>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare_cached(
|
||||
"SELECT cache_key, size_bytes FROM cache_entries
|
||||
WHERE kind = ?1 ORDER BY last_accessed ASC LIMIT ?2",
|
||||
WHERE kind = ?1 ORDER BY last_accessed ASC LIMIT ?2 OFFSET ?3",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![kind, limit as i64], |r| {
|
||||
let rows = stmt.query_map(params![kind, limit as i64, offset as i64], |r| {
|
||||
Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?.max(0) as u64))
|
||||
})?;
|
||||
let mut out = Vec::new();
|
||||
|
||||
@@ -111,9 +111,80 @@ impl Db {
|
||||
)?;
|
||||
let count: i64 = tx.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?;
|
||||
tx.commit()?;
|
||||
drop(conn);
|
||||
self.compact_search_indexes_if_due(finished_at)?;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Close a scan that did not see everything: it stopped at the entry cap,
|
||||
/// or a batch failed to write. Nothing is pruned, since a path it missed is
|
||||
/// not a path that is gone; the next complete scan prunes. Returns the
|
||||
/// number of indexed entries.
|
||||
///
|
||||
/// Pruning after such a scan dropped every entry it had not reached, and
|
||||
/// the next scan put them back. That churn is most of what grew
|
||||
/// `local_fts` to 16 times its size (`docs/BUGS.md` B217).
|
||||
pub fn local_finish_partial_scan(&self, finished_at: i64) -> Result<i64> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('local_indexed_at', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
[finished_at.to_string()],
|
||||
)?;
|
||||
let count: i64 = conn.query_row("SELECT COUNT(*) FROM local_files", [], |r| r.get(0))?;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Compact both full-text indexes, at most once a day.
|
||||
///
|
||||
/// FTS5 keeps a deleted entry's data until the segments holding it are
|
||||
/// merged, and with the few large segments these indexes settle into that
|
||||
/// is close to never. Every file the local scan sees disappear or appear
|
||||
/// leaves some behind, and every node row the tree replaces does too: on
|
||||
/// one machine `local_fts` held 317 MiB for 40,000 paths, against 20 MiB
|
||||
/// rebuilt, and a search took four times as long (`docs/BUGS.md` B217).
|
||||
///
|
||||
/// `local_fts` is rebuilt from `local_files`, which also drops anything
|
||||
/// left over, and `nodes_fts`, which has no content table to rebuild from,
|
||||
/// is merged into one segment. Each runs in its own transaction on the
|
||||
/// write connection, about half a second on an index that size; once a
|
||||
/// day keeps the leftovers small without paying that at every scan.
|
||||
pub fn compact_search_indexes_if_due(&self, now: i64) -> Result<()> {
|
||||
const EVERY_SECS: i64 = 24 * 60 * 60;
|
||||
let last: i64 = self
|
||||
.read()
|
||||
.query_row(
|
||||
"SELECT value FROM sync_state WHERE key = 'fts_compacted_at'",
|
||||
[],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.optional()?
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(0);
|
||||
if now - last < EVERY_SECS {
|
||||
return Ok(());
|
||||
}
|
||||
let started = std::time::Instant::now();
|
||||
{
|
||||
let conn = self.conn.lock();
|
||||
conn.execute("INSERT INTO local_fts(local_fts) VALUES('rebuild')", [])?;
|
||||
}
|
||||
{
|
||||
let conn = self.conn.lock();
|
||||
conn.execute("INSERT INTO nodes_fts(nodes_fts) VALUES('optimize')", [])?;
|
||||
conn.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('fts_compacted_at', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
[now.to_string()],
|
||||
)?;
|
||||
}
|
||||
tracing::info!(
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
"compacted the search indexes"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// When the last local scan completed (epoch seconds), or `None` if the index
|
||||
/// has never been built. The daemon uses this to decide whether a fresh mount
|
||||
/// needs an immediate rescan or can serve the existing index.
|
||||
|
||||
@@ -91,8 +91,13 @@ impl Db {
|
||||
/// Returns an empty vector when the database is sound; otherwise each entry
|
||||
/// is one problem SQLite found. Read-only and safe against a live daemon,
|
||||
/// though it reads every page, so it is not free on a large file.
|
||||
///
|
||||
/// Runs on a read connection: it reads every page, which took 34 s on a
|
||||
/// 567 MiB file, and on the write connection every write in the daemon
|
||||
/// waited that long behind it (`docs/BUGS.md` B218). WAL lets it read a
|
||||
/// consistent snapshot while the writer goes on.
|
||||
pub fn integrity_check(&self) -> Result<Vec<String>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare("PRAGMA integrity_check")?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
|
||||
let mut problems = Vec::new();
|
||||
@@ -107,9 +112,11 @@ impl Db {
|
||||
Ok(problems)
|
||||
}
|
||||
|
||||
/// Size and row-count snapshot for `pdfs cache inspect`.
|
||||
/// Size and row-count snapshot for `pdfs cache inspect`. On a read
|
||||
/// connection, for the reason [`integrity_check`](Self::integrity_check)
|
||||
/// gives: counting the rows of every table reads most of the file.
|
||||
pub fn stats(&self) -> Result<DbStats> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.read();
|
||||
let page_count: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?;
|
||||
let free_pages: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?;
|
||||
let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
|
||||
@@ -218,6 +225,40 @@ mod tests {
|
||||
assert!(db.integrity_check().unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// `pdfs diagnose` and `pdfs cache inspect` read the whole file. Behind the
|
||||
/// write lock they froze every write in the daemon for as long as that
|
||||
/// took (`docs/BUGS.md` B218).
|
||||
#[test]
|
||||
fn the_integrity_check_and_stats_do_not_wait_for_the_writer() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-check-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let db = std::sync::Arc::new(Db::open(&path).unwrap());
|
||||
let writer = db.conn.lock();
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let reader = std::sync::Arc::clone(&db);
|
||||
std::thread::spawn(move || {
|
||||
let checked = reader.integrity_check().map(|p| p.is_empty());
|
||||
let counted = reader.stats().map(|s| s.page_count > 0);
|
||||
let _ = tx.send((checked.ok(), counted.ok()));
|
||||
});
|
||||
let answer = rx.recv_timeout(std::time::Duration::from_secs(10));
|
||||
drop(writer);
|
||||
assert_eq!(answer.ok(), Some((Some(true), Some(true))));
|
||||
drop(db);
|
||||
for suffix in ["", "-wal", "-shm"] {
|
||||
let mut file = path.as_os_str().to_owned();
|
||||
file.push(suffix);
|
||||
let _ = std::fs::remove_file(std::path::PathBuf::from(file));
|
||||
}
|
||||
let _ = std::fs::remove_file(path.with_extension("db.lock"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_report_the_schema_and_the_known_tables() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
|
||||
@@ -11,7 +11,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
/// Current schema version. Bump on every forward migration added below.
|
||||
pub(super) const SCHEMA_VERSION: i64 = 42;
|
||||
pub(super) const SCHEMA_VERSION: i64 = 46;
|
||||
|
||||
impl Db {
|
||||
pub(super) fn migrate(&self) -> Result<()> {
|
||||
@@ -440,6 +440,67 @@ impl Db {
|
||||
tx.execute_batch(MIGRATION_V42)?;
|
||||
}
|
||||
}
|
||||
if current < 43 {
|
||||
let has_ops: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_ops {
|
||||
tx.execute_batch(MIGRATION_V43)?;
|
||||
}
|
||||
}
|
||||
if current < 44 {
|
||||
// Same guards as V26-V42.
|
||||
let has_column: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('sync_entry') \
|
||||
WHERE name = 'remote_revision_id'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_entries: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_entry'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_entries && !has_column {
|
||||
tx.execute_batch(MIGRATION_V44)?;
|
||||
}
|
||||
}
|
||||
if current < 45 {
|
||||
let has_folders: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_folder'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
let has_kind: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM pragma_table_info('sync_folder') WHERE name = 'kind'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_folders && !has_kind {
|
||||
tx.execute_batch(MIGRATION_V45_FOLDERS)?;
|
||||
let has_mounts: bool = tx.query_row(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'mount'",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)? > 0;
|
||||
if has_mounts {
|
||||
tx.execute_batch(MIGRATION_V45_MOUNTS)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
if current < 46 {
|
||||
tx.execute_batch(MIGRATION_V46)?;
|
||||
let has_resolved: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('album_photos') WHERE name='resolved')",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if !has_resolved {
|
||||
tx.execute_batch(MIGRATION_V46_ALBUM_PHOTOS)?;
|
||||
}
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -1330,3 +1391,92 @@ const MIGRATION_V42: &str = "
|
||||
ALTER TABLE pending_op ADD COLUMN sent_sha1 TEXT;
|
||||
ALTER TABLE pending_op ADD COLUMN sent_to TEXT;
|
||||
";
|
||||
|
||||
/// Schema v43: queued ops by kind and name.
|
||||
///
|
||||
/// Before each create, rename and trash the drain asks whether a queued trash
|
||||
/// or rename holds the name it wants, and a released name wakes the ops
|
||||
/// waiting for it. Each of those scanned the whole queue, so a large offline
|
||||
/// batch made draining it quadratic. The kind leads, so a lookup of every
|
||||
/// queued rename uses the index too.
|
||||
const MIGRATION_V43: &str = "
|
||||
CREATE INDEX IF NOT EXISTS pending_op_kind_name ON pending_op(kind, name);
|
||||
";
|
||||
|
||||
/// Schema v44: a synced file's baseline remembers the revision it was synced
|
||||
/// at.
|
||||
///
|
||||
/// A mirrored file edited here is uploaded as a new revision of whatever is
|
||||
/// current on Drive, and another computer syncing the same folder may have
|
||||
/// committed one since this pass looked. The revision id is what the upload
|
||||
/// checks first, so that edit becomes a conflict copy instead of being
|
||||
/// replaced (B249). `NULL` means "this row predates the column", and the check
|
||||
/// falls back to the modification time and size; each row gains a real value
|
||||
/// the next time its path is synced.
|
||||
const MIGRATION_V44: &str = "
|
||||
ALTER TABLE sync_entry ADD COLUMN remote_revision_id TEXT;
|
||||
";
|
||||
|
||||
/// Schema v45: distinguish device backups from My files attachments. Existing
|
||||
/// released rows are device backups. Attachments cannot evict their local tree.
|
||||
const MIGRATION_V45_FOLDERS: &str = "
|
||||
ALTER TABLE sync_folder ADD COLUMN kind TEXT NOT NULL DEFAULT 'device'
|
||||
CHECK (kind IN ('device', 'attached'));
|
||||
CREATE TRIGGER sync_folder_attached_insert
|
||||
BEFORE INSERT ON sync_folder
|
||||
WHEN NEW.kind = 'attached' AND (NEW.mode != 'mirror' OR NEW.pending_mode IS NOT NULL)
|
||||
BEGIN SELECT RAISE(ABORT, 'attached folders only support mirror mode'); END;
|
||||
CREATE TRIGGER sync_folder_attached_update
|
||||
BEFORE UPDATE ON sync_folder
|
||||
WHEN NEW.kind = 'attached' AND (NEW.mode != 'mirror' OR NEW.pending_mode IS NOT NULL)
|
||||
BEGIN SELECT RAISE(ABORT, 'attached folders only support mirror mode'); END;
|
||||
";
|
||||
|
||||
const MIGRATION_V45_MOUNTS: &str = "
|
||||
DROP TRIGGER mount_sync_folder_insert;
|
||||
DROP INDEX mount_myfiles;
|
||||
DROP INDEX mount_device;
|
||||
DROP INDEX mount_shared;
|
||||
ALTER TABLE mount RENAME TO mount_v44;
|
||||
CREATE TABLE mount (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('myfiles', 'device', 'attached', 'shared')),
|
||||
sync_folder_id INTEGER REFERENCES sync_folder(id) ON DELETE CASCADE,
|
||||
share_root_uid TEXT,
|
||||
local_path TEXT NOT NULL DEFAULT '',
|
||||
root_uid TEXT NOT NULL DEFAULT '',
|
||||
root_share_id TEXT NOT NULL DEFAULT '',
|
||||
mode TEXT NOT NULL DEFAULT 'unknown',
|
||||
access TEXT NOT NULL DEFAULT 'rw' CHECK (access IN ('rw', 'ro')),
|
||||
CHECK (
|
||||
(kind = 'myfiles' AND sync_folder_id IS NULL AND share_root_uid IS NULL)
|
||||
OR (kind IN ('device', 'attached') AND sync_folder_id IS NOT NULL AND share_root_uid IS NULL)
|
||||
OR (kind = 'shared' AND sync_folder_id IS NULL AND share_root_uid IS NOT NULL)
|
||||
)
|
||||
);
|
||||
INSERT INTO mount SELECT * FROM mount_v44;
|
||||
DROP TABLE mount_v44;
|
||||
CREATE UNIQUE INDEX mount_myfiles ON mount(kind) WHERE kind = 'myfiles';
|
||||
CREATE UNIQUE INDEX mount_device ON mount(sync_folder_id) WHERE kind IN ('device', 'attached');
|
||||
CREATE UNIQUE INDEX mount_shared ON mount(share_root_uid) WHERE kind = 'shared';
|
||||
CREATE TRIGGER mount_sync_folder_insert
|
||||
AFTER INSERT ON sync_folder
|
||||
BEGIN
|
||||
INSERT INTO mount (kind, sync_folder_id) VALUES (NEW.kind, NEW.id);
|
||||
END;
|
||||
CREATE TRIGGER mount_sync_folder_kind_update
|
||||
AFTER UPDATE OF kind ON sync_folder
|
||||
BEGIN
|
||||
UPDATE mount SET kind = NEW.kind WHERE sync_folder_id = NEW.id;
|
||||
END;
|
||||
";
|
||||
|
||||
/// Keep deletion evidence across listing races and remember unresolved album metadata.
|
||||
const MIGRATION_V46: &str = "
|
||||
CREATE TABLE IF NOT EXISTS photo_tombstones(uid TEXT PRIMARY KEY);
|
||||
";
|
||||
|
||||
const MIGRATION_V46_ALBUM_PHOTOS: &str = "
|
||||
ALTER TABLE album_photos ADD COLUMN resolved INTEGER NOT NULL DEFAULT 0;
|
||||
UPDATE album_photos SET resolved=1 WHERE name IS NOT NULL;
|
||||
";
|
||||
+186
-21
@@ -179,6 +179,45 @@ impl Drop for Reader<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The write connection with `synchronous=FULL`, back to NORMAL on drop.
|
||||
///
|
||||
/// NORMAL in WAL mode keeps a commit across a crash of the process but not
|
||||
/// across a power loss: the WAL is synced only at checkpoints. That is fine for
|
||||
/// metadata the next listing re-fetches, and not for the queue. A staged blob
|
||||
/// is synced before its op is queued, and the op row is what makes it
|
||||
/// reachable; losing that row loses a write `close()` already reported as
|
||||
/// done. A FULL commit syncs the WAL, which covers every NORMAL commit before
|
||||
/// it as well.
|
||||
pub(super) struct Durable<'a>(parking_lot::MutexGuard<'a, Connection>);
|
||||
|
||||
impl<'a> Durable<'a> {
|
||||
pub(super) fn new(conn: parking_lot::MutexGuard<'a, Connection>) -> Result<Self> {
|
||||
conn.pragma_update(None, "synchronous", "FULL")?;
|
||||
Ok(Self(conn))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for Durable<'_> {
|
||||
type Target = Connection;
|
||||
|
||||
fn deref(&self) -> &Connection {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::DerefMut for Durable<'_> {
|
||||
fn deref_mut(&mut self) -> &mut Connection {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Durable<'_> {
|
||||
fn drop(&mut self) {
|
||||
// A failure leaves the writer at FULL: slower, never less safe.
|
||||
let _ = self.0.pragma_update(None, "synchronous", "NORMAL");
|
||||
}
|
||||
}
|
||||
|
||||
/// Open a second connection to the same file for reads only.
|
||||
///
|
||||
/// `SQLITE_OPEN_READ_ONLY` plus `query_only` is belt and braces: the flag stops
|
||||
@@ -216,18 +255,34 @@ impl Db {
|
||||
let conn = match Self::open_configured(path) {
|
||||
Ok(conn) => conn,
|
||||
Err(e) if is_corrupt(&e) => {
|
||||
// Every byte in here is derived from the account or from the
|
||||
// content cache directory, so the recovery is to start over —
|
||||
// loudly, and keeping the damaged file for a post-mortem rather
|
||||
// than deleting evidence.
|
||||
// Most of what is in here is derived from the account and is
|
||||
// fetched again. The queue is not: it holds writes accepted
|
||||
// here that Drive does not have yet, and the mirror baselines
|
||||
// and pins are this machine's alone. So the damaged file is
|
||||
// moved aside, kept as evidence, and those tables are copied
|
||||
// out of it into the new one as far as SQLite can still read
|
||||
// them (`docs/BUGS.md` B216).
|
||||
tracing::error!(
|
||||
db = %path.display(),
|
||||
error = %e,
|
||||
"cache database is corrupt — moving it aside and rebuilding \
|
||||
(metadata will be re-fetched; no user data is stored here)"
|
||||
"cache database is corrupt; moving it aside, rebuilding, and \
|
||||
copying the upload queue, sync baselines and pins out of it"
|
||||
);
|
||||
quarantine_corrupt_db(path)?;
|
||||
Self::open_configured(path)?
|
||||
let damaged = quarantine_corrupt_db(path)?;
|
||||
let conn = Self::open_configured(path)?;
|
||||
for (table, result) in salvage(&conn, &damaged) {
|
||||
match result {
|
||||
Ok(rows) => {
|
||||
tracing::warn!(table, rows, "salvaged from the damaged database")
|
||||
}
|
||||
Err(error) => tracing::error!(
|
||||
table,
|
||||
%error,
|
||||
"could not salvage from the damaged database"
|
||||
),
|
||||
}
|
||||
}
|
||||
conn
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
@@ -254,6 +309,12 @@ impl Db {
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
/// The write connection for a commit that accepts a user's change into
|
||||
/// the queue. See [`Durable`].
|
||||
pub(super) fn durable(&self) -> Result<Durable<'_>> {
|
||||
Durable::new(self.conn.lock())
|
||||
}
|
||||
|
||||
/// A connection for a `SELECT`-only method.
|
||||
///
|
||||
/// Reads see the last committed state, which is the same thing they saw
|
||||
@@ -281,7 +342,8 @@ impl Db {
|
||||
fn open_configured(path: &Path) -> Result<Connection> {
|
||||
let conn = Connection::open(path)?;
|
||||
// WAL: readers never block the single writer. NORMAL sync is the
|
||||
// standard durability/throughput tradeoff for WAL.
|
||||
// standard durability/throughput tradeoff for WAL; the queue commits
|
||||
// that need more raise it to FULL for themselves (`Durable`).
|
||||
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||
conn.pragma_update(None, "synchronous", "NORMAL")?;
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
@@ -383,6 +445,13 @@ fn is_in_memory(path: &Path) -> bool {
|
||||
/// SQLite uses POSIX record locks, so the two never interact, and a lock file
|
||||
/// left behind carries no state — the lock lives in the kernel and dies with
|
||||
/// the process, however it exits.
|
||||
///
|
||||
/// A refusal is retried for [`LOCK_RETRY`] before it counts as another
|
||||
/// daemon. A child this process spawns holds a copy of every descriptor from
|
||||
/// its fork until its exec closes them, and `flock` belongs to the open file,
|
||||
/// not the descriptor. So a lock the process has just let go of can stay held
|
||||
/// for that moment, and a reopen right after a drop is refused
|
||||
/// (`docs/BUGS.md` B290).
|
||||
fn acquire_single_writer_lock(path: &Path) -> Result<Option<File>> {
|
||||
if is_in_memory(path) {
|
||||
return Ok(None);
|
||||
@@ -394,22 +463,32 @@ fn acquire_single_writer_lock(path: &Path) -> Result<Option<File>> {
|
||||
.truncate(false)
|
||||
.write(true)
|
||||
.open(&lock_path)?;
|
||||
// SAFETY: `file` owns the descriptor and outlives the call.
|
||||
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if rc != 0 {
|
||||
let deadline = std::time::Instant::now() + LOCK_RETRY;
|
||||
loop {
|
||||
// SAFETY: `file` owns the descriptor and outlives the call.
|
||||
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
|
||||
if rc == 0 {
|
||||
return Ok(Some(file));
|
||||
}
|
||||
let e = std::io::Error::last_os_error();
|
||||
if e.raw_os_error() == Some(libc::EWOULDBLOCK) {
|
||||
if e.raw_os_error() != Some(libc::EWOULDBLOCK) {
|
||||
return Err(Error::Io(e));
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
return Err(Error::Other(format!(
|
||||
"another Proton Drive daemon is already using {} — \
|
||||
stop it first (`systemctl --user stop proton-drive.service`)",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
return Err(Error::Io(e));
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
}
|
||||
Ok(Some(file))
|
||||
}
|
||||
|
||||
/// How long a refused single-writer lock is retried; far longer than a fork
|
||||
/// takes to reach its exec, far shorter than anyone notices.
|
||||
const LOCK_RETRY: std::time::Duration = std::time::Duration::from_millis(500);
|
||||
|
||||
fn lock_path_for(path: &Path) -> PathBuf {
|
||||
let mut name = path.as_os_str().to_owned();
|
||||
name.push(".lock");
|
||||
@@ -428,12 +507,96 @@ fn is_corrupt(e: &Error) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// Move a damaged database (and its WAL sidecars) out of the way so the next
|
||||
/// open starts from empty.
|
||||
/// The tables [`salvage`] copies out of a damaged database, in order, each
|
||||
/// with the rows it keeps.
|
||||
///
|
||||
/// Renamed rather than deleted: this is a cache, but it is also the only
|
||||
/// evidence of what went wrong, and it costs one file the user can remove.
|
||||
fn quarantine_corrupt_db(path: &Path) -> Result<()> {
|
||||
/// The queue goes with the node rows it names by local id: a create made here
|
||||
/// has no other record of its node, and a real node keeps its local id when its
|
||||
/// folder is listed again. Those rows come back unlisted, so their folders are
|
||||
/// fetched again rather than trusted. Claims are cleared by [`Db::open`] as
|
||||
/// after any other crash.
|
||||
const SALVAGE: &[(&str, &str)] = &[
|
||||
(
|
||||
"nodes",
|
||||
"lid IN (SELECT lid FROM old.pending_op WHERE lid IS NOT NULL
|
||||
UNION SELECT parent_lid FROM old.pending_op WHERE parent_lid IS NOT NULL)
|
||||
OR uid LIKE 'local~%'",
|
||||
),
|
||||
("pending_op", "1"),
|
||||
("sync_folder", "1"),
|
||||
("sync_entry", "1"),
|
||||
("pins", "1"),
|
||||
("own_sealed_rev", "1"),
|
||||
];
|
||||
|
||||
/// Copy what only this machine has out of the damaged database at `damaged`
|
||||
/// into the fresh one `conn` just created. Returns how each table went.
|
||||
///
|
||||
/// Each table is copied in its own transaction, so a table SQLite cannot read
|
||||
/// to the end is left out whole rather than half copied, and does not stop
|
||||
/// the others. Only the columns both files have are copied, which covers a
|
||||
/// file that broke in the middle of an upgrade. The damaged file is opened read
|
||||
/// only and left as it was.
|
||||
fn salvage(conn: &Connection, damaged: &Path) -> Vec<(&'static str, Result<usize>)> {
|
||||
let uri = format!(
|
||||
"file:{}?mode=ro",
|
||||
damaged
|
||||
.to_string_lossy()
|
||||
.replace('%', "%25")
|
||||
.replace('?', "%3f")
|
||||
.replace('#', "%23")
|
||||
);
|
||||
if let Err(e) = conn.execute("ATTACH DATABASE ?1 AS old", [&uri]) {
|
||||
let e = Error::Db(e);
|
||||
tracing::error!(error = %e, "could not open the damaged database to salvage it");
|
||||
return Vec::new();
|
||||
}
|
||||
let results = SALVAGE
|
||||
.iter()
|
||||
.map(|&(table, rows)| (table, salvage_table(conn, table, rows)))
|
||||
.collect();
|
||||
let _ = conn.execute("UPDATE nodes SET listed = 0", []);
|
||||
if let Err(e) = conn.execute("DETACH DATABASE old", []) {
|
||||
tracing::warn!(error = %e, "could not detach the damaged database");
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
fn salvage_table(conn: &Connection, table: &str, rows: &str) -> Result<usize> {
|
||||
let columns = |schema: &str| -> Result<Vec<String>> {
|
||||
let mut stmt = conn.prepare(&format!("PRAGMA {schema}.table_info({table})"))?;
|
||||
let names = stmt
|
||||
.query_map([], |r| r.get::<_, String>(1))?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
Ok(names)
|
||||
};
|
||||
let new = columns("main")?;
|
||||
let shared: Vec<String> = columns("old")?
|
||||
.into_iter()
|
||||
.filter(|c| new.contains(c))
|
||||
.map(|c| format!("\"{c}\""))
|
||||
.collect();
|
||||
if shared.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let shared = shared.join(", ");
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
let copied = tx.execute(
|
||||
&format!(
|
||||
"INSERT OR IGNORE INTO main.{table} ({shared}) SELECT {shared} FROM old.{table} WHERE {rows}"
|
||||
),
|
||||
[],
|
||||
)?;
|
||||
tx.commit()?;
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
/// Move a damaged database (and its WAL sidecars) out of the way so the next
|
||||
/// open starts from empty. Returns where the database itself went.
|
||||
///
|
||||
/// Renamed rather than deleted: it is the only evidence of what went wrong,
|
||||
/// and [`salvage`] reads what only this machine has back out of it.
|
||||
fn quarantine_corrupt_db(path: &Path) -> Result<PathBuf> {
|
||||
let stamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
@@ -457,7 +620,9 @@ fn quarantine_corrupt_db(path: &Path) -> Result<()> {
|
||||
Err(e) => tracing::warn!(file = %from.display(), error = %e, "could not quarantine"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
let mut kept = path.as_os_str().to_owned();
|
||||
kept.push(format!(".corrupt-{stamp}"));
|
||||
Ok(PathBuf::from(kept))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Unified mount/location presentation rows.
|
||||
//!
|
||||
//! A device row stores only its `sync_folder` identity. Its path, remote root,
|
||||
//! A device or attached row stores only its `sync_folder` identity. Its path, remote root,
|
||||
//! mode, state, pending mode, and last-sync timestamp are selected from
|
||||
//! `sync_folder`, which remains the authoritative sync-engine table.
|
||||
|
||||
@@ -72,7 +72,7 @@ impl Db {
|
||||
)? > 0)
|
||||
}
|
||||
|
||||
/// Every local location, with device fields joined from `sync_folder`.
|
||||
/// Every local location, with synced-folder fields joined from `sync_folder`.
|
||||
pub fn mount_list(&self) -> Result<Vec<MountSpec>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
@@ -81,14 +81,14 @@ impl Db {
|
||||
m.kind,
|
||||
m.sync_folder_id,
|
||||
m.share_root_uid,
|
||||
CASE WHEN m.kind = 'device' THEN sf.local_path ELSE m.local_path END,
|
||||
CASE WHEN m.kind = 'device' THEN sf.remote_uid ELSE m.root_uid END,
|
||||
CASE WHEN m.kind = 'device' THEN sf.remote_share_id ELSE m.root_share_id END,
|
||||
CASE WHEN m.kind = 'device' THEN sf.mode ELSE m.mode END,
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.local_path ELSE m.local_path END,
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.remote_uid ELSE m.root_uid END,
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.remote_share_id ELSE m.root_share_id END,
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.mode ELSE m.mode END,
|
||||
m.access,
|
||||
CASE WHEN m.kind = 'device' THEN sf.state ELSE 'idle' END,
|
||||
CASE WHEN m.kind = 'device' THEN sf.last_sync ELSE 0 END,
|
||||
CASE WHEN m.kind = 'device' THEN sf.pending_mode ELSE NULL END
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.state ELSE 'idle' END,
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.last_sync ELSE 0 END,
|
||||
CASE WHEN m.kind IN ('device', 'attached') THEN sf.pending_mode ELSE NULL END
|
||||
FROM mount AS m
|
||||
LEFT JOIN sync_folder AS sf ON sf.id = m.sync_folder_id
|
||||
ORDER BY CASE m.kind
|
||||
@@ -138,6 +138,11 @@ impl Db {
|
||||
Error::Other(format!("mount row {id} has no sync_folder_id"))
|
||||
})?,
|
||||
},
|
||||
"attached" => MountKind::Attached {
|
||||
sync_folder_id: sync_folder_id.ok_or_else(|| {
|
||||
Error::Other(format!("mount row {id} has no sync_folder_id"))
|
||||
})?,
|
||||
},
|
||||
"shared" => MountKind::Shared {
|
||||
share_root_uid: share_root_uid.ok_or_else(|| {
|
||||
Error::Other(format!("mount row {id} has no share_root_uid"))
|
||||
|
||||
+211
-44
@@ -252,6 +252,50 @@ const UID_NOW: &str = "COALESCE((SELECT uid FROM nodes WHERE lid = pending_op.li
|
||||
const PARENT_NOW: &str =
|
||||
"COALESCE((SELECT uid FROM nodes WHERE lid = pending_op.parent_lid), parent_uid)";
|
||||
|
||||
/// Whether any queued move takes a node out of the folder `source.uid`.
|
||||
/// A move updates the local node tree immediately, so its original parent
|
||||
/// comes from the queue, including while a worker is sending it to Drive.
|
||||
const MOVES_OUT_OF_FOLDER: &str = "
|
||||
COALESCE((SELECT is_dir FROM nodes WHERE uid = source.uid), 1) = 1
|
||||
AND EXISTS (
|
||||
WITH RECURSIVE folders(uid, lid) AS (
|
||||
SELECT source.uid, (SELECT lid FROM nodes WHERE uid = source.uid)
|
||||
UNION
|
||||
SELECT n.uid, n.lid FROM nodes n JOIN folders f
|
||||
ON n.parent_lid = f.lid OR (n.parent_lid IS NULL AND n.parent_uid = f.uid)
|
||||
WHERE n.is_dir = 1
|
||||
)
|
||||
SELECT 1 FROM pending_op AS move
|
||||
WHERE move.kind = 'rename'
|
||||
AND CASE WHEN json_valid(move.meta_json)
|
||||
THEN json_extract(move.meta_json, '$.original_parent_uid') END
|
||||
IN (SELECT uid FROM folders)
|
||||
AND move.parent_uid IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM folders f
|
||||
WHERE f.lid = move.parent_lid OR f.uid = move.parent_uid)
|
||||
)";
|
||||
|
||||
/// The `WHERE` terms for a row a drain worker may claim once it falls due: not
|
||||
/// claimed already, no other claimed op on the same node, no pending creation
|
||||
/// of its parent, and no moves that still need a folder we are about to trash.
|
||||
///
|
||||
/// One fragment for [`Db::claim_next_due_op`] and [`Db::earliest_due_at`]. A
|
||||
/// row the claim skips but the wait counts is due "now" for as long as it is
|
||||
/// held, and every idle worker spun on it (`docs/BUGS.md` B200).
|
||||
fn claimable() -> String {
|
||||
format!(
|
||||
"claimed_at = 0
|
||||
AND (parent_uid IS NULL OR substr({PARENT_NOW}, 1, {n}) <> '{v}~')
|
||||
AND uid NOT IN (SELECT uid FROM pending_op WHERE claimed_at <> 0)
|
||||
AND (lid IS NULL OR lid NOT IN (
|
||||
SELECT lid FROM pending_op WHERE claimed_at <> 0 AND lid IS NOT NULL))
|
||||
AND (kind <> 'trash' OR NOT EXISTS (
|
||||
SELECT 1 FROM (SELECT {UID_NOW} AS uid) AS source WHERE {MOVES_OUT_OF_FOLDER}))",
|
||||
v = LOCAL_VOLUME,
|
||||
n = LOCAL_VOLUME.len() + 1,
|
||||
)
|
||||
}
|
||||
|
||||
/// The volume id given to a node that exists only on this machine, so far. A
|
||||
/// real [`NodeUid`] is `{volume}~{link}`, so a placeholder is `local~<lid>` and
|
||||
/// round-trips through the same `Display`/parse path as any other uid.
|
||||
@@ -260,6 +304,45 @@ const PARENT_NOW: &str =
|
||||
/// drain replaces it with the uid the server assigns.
|
||||
pub const LOCAL_VOLUME: &str = "local";
|
||||
|
||||
/// One place a queued create was sent, with when it first was.
|
||||
struct SentTarget {
|
||||
parent: String,
|
||||
name: String,
|
||||
at_ms: Option<i64>,
|
||||
}
|
||||
|
||||
/// The `sent_to` column: `[[parent, name, at_ms], …]`, or `[[parent, name], …]`
|
||||
/// as written before the time was kept. `None` when it does not parse.
|
||||
fn parse_sent_to(json: Option<&str>) -> Option<Vec<SentTarget>> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Entry {
|
||||
At(String, String, i64),
|
||||
Bare(String, String),
|
||||
}
|
||||
let Some(json) = json else {
|
||||
return Some(Vec::new());
|
||||
};
|
||||
let entries: Vec<Entry> = serde_json::from_str(json).ok()?;
|
||||
Some(
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| match entry {
|
||||
Entry::At(parent, name, at) => SentTarget {
|
||||
parent,
|
||||
name,
|
||||
at_ms: Some(at),
|
||||
},
|
||||
Entry::Bare(parent, name) => SentTarget {
|
||||
parent,
|
||||
name,
|
||||
at_ms: None,
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
/// The uid a node made on this machine goes by until Drive gives it one: its
|
||||
/// row's local id under [`LOCAL_VOLUME`].
|
||||
///
|
||||
@@ -417,19 +500,40 @@ pub struct AttachedBlob {
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Whether a queued trash still holds `name` somewhere.
|
||||
/// A folder can look empty locally while Drive still holds the queued
|
||||
/// moves' source files. A trash has to go through the drain in that case.
|
||||
pub fn has_pending_moves_out_of(&self, uid: &str) -> Result<bool> {
|
||||
let conn = self.read();
|
||||
Ok(conn
|
||||
.prepare_cached(&format!(
|
||||
"SELECT EXISTS (SELECT 1 FROM (SELECT ?1 AS uid) AS source
|
||||
WHERE {MOVES_OUT_OF_FOLDER})"
|
||||
))?
|
||||
.query_row(params![uid], |row| row.get(0))?)
|
||||
}
|
||||
|
||||
/// Whether a queued trash still holds `name` in `parent_uid`.
|
||||
///
|
||||
/// A replacing rename or a delete-then-recreate queues the trash of the old
|
||||
/// node ahead of the op that wants its name. With several drain workers the
|
||||
/// later op can run first and find the name still taken; this tells it to
|
||||
/// wait for the trash rather than land under a conflict name.
|
||||
pub fn has_pending_trash_named(&self, name: &str) -> Result<bool> {
|
||||
/// wait for the trash rather than land under a conflict name. Older rows
|
||||
/// whose node and parent are unknown are matched by name alone.
|
||||
pub fn has_pending_trash_named(&self, name: &str, parent_uid: Option<&str>) -> Result<bool> {
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT EXISTS(
|
||||
SELECT 1 FROM pending_op WHERE kind = ?1 AND name = ?2
|
||||
)",
|
||||
params![OP_TRASH, name],
|
||||
SELECT 1 FROM pending_op AS p
|
||||
WHERE p.kind = ?1 AND p.name = ?2
|
||||
AND (?3 IS NULL OR ?3 = COALESCE(
|
||||
(SELECT uid FROM nodes WHERE lid = p.parent_lid),
|
||||
p.parent_uid,
|
||||
(SELECT COALESCE(parent.uid, n.parent_uid)
|
||||
FROM nodes n LEFT JOIN nodes parent ON parent.lid = n.parent_lid
|
||||
WHERE n.uid = p.uid),
|
||||
?3))
|
||||
)",
|
||||
params![OP_TRASH, name, parent_uid],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map_err(Into::into)
|
||||
@@ -522,7 +626,7 @@ impl Db {
|
||||
/// land in the order they were made (`docs/BUGS.md` B141). Should it fail,
|
||||
/// [`Db::record_op_failure`] or [`Db::release_op_claim`] drops it.
|
||||
pub fn enqueue_op(&self, op: &PendingOp) -> Result<(i64, Option<String>)> {
|
||||
let mut conn = self.conn.lock();
|
||||
let mut conn = self.durable()?;
|
||||
let tx = conn.transaction()?;
|
||||
let superseded: Option<String> = if op_supersedes(&op.kind) {
|
||||
let lid = node_lid(&tx, &op.uid)?;
|
||||
@@ -579,7 +683,7 @@ impl Db {
|
||||
name: &str,
|
||||
created_at: i64,
|
||||
) -> Result<(i64, Vec<String>)> {
|
||||
let mut conn = self.conn.lock();
|
||||
let mut conn = self.durable()?;
|
||||
let tx = conn.transaction()?;
|
||||
let blobs = drop_doomed_ops(&tx, uid)?;
|
||||
tx.execute(
|
||||
@@ -609,7 +713,7 @@ impl Db {
|
||||
blob_path: &str,
|
||||
meta_json: &str,
|
||||
) -> Result<Option<AttachedBlob>> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.durable()?;
|
||||
let lid = node_lid(&conn, uid)?;
|
||||
let existing: Option<(i64, Option<String>)> = conn
|
||||
.query_row(
|
||||
@@ -672,7 +776,7 @@ impl Db {
|
||||
rewrite: impl FnOnce(&str) -> Option<String>,
|
||||
) -> Result<CreateRetired> {
|
||||
let (local, real) = (landing.local, landing.real);
|
||||
let mut conn = self.conn.lock();
|
||||
let mut conn = self.durable()?;
|
||||
let tx = conn.transaction()?;
|
||||
let row: Option<CreateRow> = tx
|
||||
.query_row(
|
||||
@@ -789,12 +893,19 @@ impl Db {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record that queued create `id` is being sent to `parent` under `name`.
|
||||
/// Record that queued create `id` is being sent to `parent` under `name`
|
||||
/// at `at_ms`.
|
||||
///
|
||||
/// A rename while the op waits rewrites its target, but the file a lost
|
||||
/// answer left stays where it was sent. The retry and the trash look
|
||||
/// there too (`docs/BUGS.md` B184).
|
||||
pub fn note_create_target(&self, id: i64, parent: &str, name: &str) -> Result<()> {
|
||||
/// there too (`docs/BUGS.md` B184). A target sent before keeps its first
|
||||
/// time.
|
||||
///
|
||||
/// A record that does not parse is left as it is rather than replaced
|
||||
/// with this one target: the earlier ones are what the duplicate checks
|
||||
/// look for. Without them a lost answer forks a conflict copy, which
|
||||
/// loses nothing.
|
||||
pub fn note_create_target(&self, id: i64, parent: &str, name: &str, at_ms: i64) -> Result<()> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let sent: Option<String> = tx
|
||||
@@ -803,15 +914,26 @@ impl Db {
|
||||
})
|
||||
.optional()?
|
||||
.flatten();
|
||||
let mut targets: Vec<(String, String)> = sent
|
||||
.and_then(|json| serde_json::from_str(&json).ok())
|
||||
.unwrap_or_default();
|
||||
let target = (parent.to_string(), name.to_string());
|
||||
if !targets.contains(&target) {
|
||||
targets.push(target);
|
||||
let Some(mut targets) = parse_sent_to(sent.as_deref()) else {
|
||||
tracing::warn!(
|
||||
id,
|
||||
"a queued create's send record does not parse; keeping it"
|
||||
);
|
||||
return Ok(());
|
||||
};
|
||||
if !targets.iter().any(|t| t.parent == parent && t.name == name) {
|
||||
targets.push(SentTarget {
|
||||
parent: parent.to_string(),
|
||||
name: name.to_string(),
|
||||
at_ms: Some(at_ms),
|
||||
});
|
||||
let json: Vec<(&str, &str, i64)> = targets
|
||||
.iter()
|
||||
.map(|t| (t.parent.as_str(), t.name.as_str(), t.at_ms.unwrap_or(0)))
|
||||
.collect();
|
||||
tx.execute(
|
||||
"UPDATE pending_op SET sent_to = ?2 WHERE id = ?1",
|
||||
params![id, serde_json::to_string(&targets)?],
|
||||
params![id, serde_json::to_string(&json)?],
|
||||
)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
@@ -821,6 +943,24 @@ impl Db {
|
||||
/// The parent and name queued op `id` was sent under as a create, in the
|
||||
/// order it was ([`Db::note_create_target`]).
|
||||
pub fn create_targets(&self, id: i64) -> Result<Vec<(String, String)>> {
|
||||
Ok(self
|
||||
.sent_targets(id)?
|
||||
.into_iter()
|
||||
.map(|t| (t.parent, t.name))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// When queued create `id` was first sent, if that was recorded. Rows
|
||||
/// written before the time was kept have none.
|
||||
pub fn create_first_sent(&self, id: i64) -> Result<Option<i64>> {
|
||||
Ok(self
|
||||
.sent_targets(id)?
|
||||
.into_iter()
|
||||
.filter_map(|t| t.at_ms.filter(|&at| at > 0))
|
||||
.min())
|
||||
}
|
||||
|
||||
fn sent_targets(&self, id: i64) -> Result<Vec<SentTarget>> {
|
||||
let sent: Option<String> = self
|
||||
.conn
|
||||
.lock()
|
||||
@@ -829,9 +969,7 @@ impl Db {
|
||||
})
|
||||
.optional()?
|
||||
.flatten();
|
||||
Ok(sent
|
||||
.and_then(|json| serde_json::from_str(&json).ok())
|
||||
.unwrap_or_default())
|
||||
Ok(parse_sent_to(sent.as_deref()).unwrap_or_default())
|
||||
}
|
||||
|
||||
/// The SHA-1 of each blob queued op `id` sent as a create
|
||||
@@ -879,6 +1017,27 @@ impl Db {
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Replace the sidecar of op `id`, but only while it still holds the blob
|
||||
/// at `blob_path`.
|
||||
///
|
||||
/// The drain fills an incomplete blob in place and then records it as
|
||||
/// whole, so a retry after a lost answer can recognise its own bytes on the
|
||||
/// server. A write that closed meanwhile has pointed the row at a newer
|
||||
/// blob with its own sidecar; that one must not be overwritten. Returns
|
||||
/// false when the row is gone or holds another blob.
|
||||
pub fn update_op_meta_for_blob(
|
||||
&self,
|
||||
id: i64,
|
||||
blob_path: &str,
|
||||
meta_json: &str,
|
||||
) -> Result<bool> {
|
||||
let n = self.conn.lock().execute(
|
||||
"UPDATE pending_op SET meta_json = ?3 WHERE id = ?1 AND blob_path = ?2",
|
||||
params![id, blob_path, meta_json],
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Point a queued create at a new parent and name, for a node renamed or
|
||||
/// moved before it ever reached the server.
|
||||
///
|
||||
@@ -892,7 +1051,7 @@ impl Db {
|
||||
/// name it no longer wants, such as one held by a queued change. A parked
|
||||
/// create stays parked.
|
||||
pub fn rewrite_op_target(&self, uid: &str, parent_uid: &str, name: &str) -> Result<bool> {
|
||||
let conn = self.conn.lock();
|
||||
let conn = self.durable()?;
|
||||
let lid = node_lid(&conn, uid)?;
|
||||
let parent_lid = node_lid(&conn, parent_uid)?;
|
||||
let n = conn.execute(
|
||||
@@ -1106,7 +1265,7 @@ impl Db {
|
||||
/// the claim is a column: a row is offered to exactly one worker, and the
|
||||
/// worker that took it is the only one that may retire, defer or fail it.
|
||||
///
|
||||
/// Two exclusions on top of [`next_due_op`](Self::next_due_op)'s:
|
||||
/// The selected row must also satisfy [`claimable`]:
|
||||
///
|
||||
/// * `claimed_at = 0` — not already in flight somewhere else.
|
||||
/// * no *other* claimed op shares this `uid`. Ordering only has to hold per
|
||||
@@ -1116,6 +1275,8 @@ impl Db {
|
||||
/// pending entry. `ORDER BY id` plus this exclusion gives per-uid serial
|
||||
/// order for free — the older op is claimed first and the newer one is
|
||||
/// invisible until it retires.
|
||||
/// * its destination folder already exists remotely.
|
||||
/// * a folder trash waits for pending moves out of that folder's subtree.
|
||||
///
|
||||
/// Select and mark are one transaction, so two workers cannot both see the
|
||||
/// row as unclaimed.
|
||||
@@ -1127,15 +1288,9 @@ impl Db {
|
||||
"SELECT id, kind, {UID_NOW}, {PARENT_NOW}, name, blob_path, meta_json, created_at,
|
||||
attempts, last_error, next_attempt_at
|
||||
FROM pending_op
|
||||
WHERE next_attempt_at <= ?1
|
||||
AND claimed_at = 0
|
||||
AND (parent_uid IS NULL OR substr({PARENT_NOW}, 1, {n}) <> '{v}~')
|
||||
AND uid NOT IN (SELECT uid FROM pending_op WHERE claimed_at <> 0)
|
||||
AND (lid IS NULL OR lid NOT IN (
|
||||
SELECT lid FROM pending_op WHERE claimed_at <> 0 AND lid IS NOT NULL))
|
||||
WHERE next_attempt_at <= ?1 AND {claimable}
|
||||
ORDER BY id LIMIT 1",
|
||||
v = LOCAL_VOLUME,
|
||||
n = LOCAL_VOLUME.len() + 1,
|
||||
claimable = claimable(),
|
||||
))?;
|
||||
stmt.query_row(params![now], |r| {
|
||||
Ok(PendingOp {
|
||||
@@ -1208,19 +1363,17 @@ impl Db {
|
||||
/// debounced or backed-off op becomes eligible rather than waiting the full
|
||||
/// idle-poll interval.
|
||||
///
|
||||
/// Claimed ops are excluded: another worker is already on them, so they are
|
||||
/// not work this one is waiting for, and counting them would have an idle
|
||||
/// worker spin on a row it cannot have.
|
||||
/// Only rows [`claim_next_due_op`](Self::claim_next_due_op) could take
|
||||
/// count. A claimed op, or one held behind a claimed op on the same node,
|
||||
/// is not work this worker is waiting for, and counting it would have an
|
||||
/// idle worker spin on a row it cannot have.
|
||||
pub fn earliest_due_at(&self) -> Result<Option<i64>> {
|
||||
let conn = self.read();
|
||||
let ts: Option<i64> = conn
|
||||
.query_row(
|
||||
&format!(
|
||||
"SELECT MIN(next_attempt_at) FROM pending_op \
|
||||
WHERE claimed_at = 0 \
|
||||
AND (parent_uid IS NULL OR substr({PARENT_NOW}, 1, {n}) <> '{v}~')",
|
||||
v = LOCAL_VOLUME,
|
||||
n = LOCAL_VOLUME.len() + 1,
|
||||
"SELECT MIN(next_attempt_at) FROM pending_op WHERE {claimable}",
|
||||
claimable = claimable(),
|
||||
),
|
||||
[],
|
||||
|r| r.get(0),
|
||||
@@ -1378,12 +1531,26 @@ impl Db {
|
||||
}
|
||||
|
||||
/// [`retry_op_now`](Self::retry_op_now) for every op that has failed at
|
||||
/// least once. Returns how many rows changed.
|
||||
/// least once, for the user's "Retry all". Returns how many rows changed.
|
||||
pub fn retry_failed_ops_now(&self, now: i64) -> Result<usize> {
|
||||
self.retry_ops_now(now, "")
|
||||
}
|
||||
|
||||
/// [`retry_failed_ops_now`](Self::retry_failed_ops_now) for the ops Drive
|
||||
/// did not refuse, for a reconnect. A link coming back changes nothing
|
||||
/// about a full quota or a missing permission, so those keep their backoff
|
||||
/// instead of all being sent again at once.
|
||||
pub fn retry_unrefused_ops_now(&self, now: i64) -> Result<usize> {
|
||||
self.retry_ops_now(now, "AND issue IS NULL")
|
||||
}
|
||||
|
||||
fn retry_ops_now(&self, now: i64, filter: &str) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
let changed = conn.execute(
|
||||
"UPDATE pending_op SET next_attempt_at = ?1
|
||||
WHERE attempts > 0 AND next_attempt_at > ?1 AND next_attempt_at < ?2",
|
||||
&format!(
|
||||
"UPDATE pending_op SET next_attempt_at = ?1
|
||||
WHERE attempts > 0 AND next_attempt_at > ?1 AND next_attempt_at < ?2 {filter}"
|
||||
),
|
||||
params![now, PARK_UNTIL],
|
||||
)?;
|
||||
Ok(changed)
|
||||
|
||||
+404
-174
@@ -1,10 +1,12 @@
|
||||
//! The photos timeline: a flat, date-ordered projection of the photo share,
|
||||
//! persisted so the gallery opens instantly instead of re-enumerating on launch.
|
||||
|
||||
use rusqlite::params;
|
||||
use rusqlite::{Connection, OptionalExtension, Transaction, params};
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use proton_gallery_rs::grouping::{Grouped, group_photos, name_stem};
|
||||
|
||||
use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
@@ -40,6 +42,8 @@ pub struct StoredPhoto {
|
||||
/// Whether any member of the group is a camera raw file. The grid badges
|
||||
/// that, because the JPEG it shows is not the whole of what is stored.
|
||||
pub has_raw: bool,
|
||||
/// The town the photo was taken in, once its location has been read.
|
||||
pub place_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// A town photos were taken in: its GeoNames id (see [`crate::places`]), how
|
||||
@@ -119,135 +123,6 @@ fn capture_day(conn: &rusqlite::Connection, capture_time: i64) -> Option<String>
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// The file name without its extension, lowercased — `IMG_1234.CR2` and
|
||||
/// `IMG_1234.JPG` share `img_1234` — with the camera's own part-of-a-shot
|
||||
/// marker removed, so the two files a raw capture writes share a stem too.
|
||||
fn name_stem(name: Option<&str>) -> Option<String> {
|
||||
let name = name?;
|
||||
let stem = match name.rsplit_once('.') {
|
||||
Some((stem, _)) if !stem.is_empty() => stem,
|
||||
_ => name,
|
||||
};
|
||||
let stem = stem.to_ascii_lowercase();
|
||||
Some(strip_shot_marker(&stem).to_string())
|
||||
}
|
||||
|
||||
/// Drop a `.RAW-NN.ROLE` marker from a stem.
|
||||
///
|
||||
/// Pixel stores one raw capture as `PXL_20260919_000625670.RAW-01.COVER.jpg`
|
||||
/// and `PXL_20260919_000625670.RAW-02.ORIGINAL.dng`: one shot, two files, two
|
||||
/// stems that differ in exactly the part naming which file of the shot this is.
|
||||
/// Cutting the marker off is what lets rule 2 see the pair. The check is narrow
|
||||
/// on purpose — digits, then a single role word — so an ordinary name that
|
||||
/// happens to contain `.raw-` keeps its stem.
|
||||
fn strip_shot_marker(stem: &str) -> &str {
|
||||
let Some(at) = stem.rfind(".raw-") else {
|
||||
return stem;
|
||||
};
|
||||
let rest = &stem[at + ".raw-".len()..];
|
||||
let Some((index, role)) = rest.split_once('.') else {
|
||||
return stem;
|
||||
};
|
||||
let marked = !index.is_empty()
|
||||
&& index.bytes().all(|b| b.is_ascii_digit())
|
||||
&& !role.is_empty()
|
||||
&& role.bytes().all(|b| b.is_ascii_alphabetic());
|
||||
if marked { &stem[..at] } else { stem }
|
||||
}
|
||||
|
||||
/// One entry of the grouping pass: what rule 2 needs, plus what choosing a
|
||||
/// representative needs.
|
||||
struct Grouped {
|
||||
uid: String,
|
||||
kind: crate::control::PhotoKind,
|
||||
main_uid: Option<String>,
|
||||
day: Option<String>,
|
||||
stem: Option<String>,
|
||||
}
|
||||
|
||||
/// Which group each photo belongs to, and which member of it the gallery shows.
|
||||
///
|
||||
/// Precedence, as agreed:
|
||||
///
|
||||
/// 1. The server's own relation — a photo with a `main_photo_uid` joins that
|
||||
/// photo's group. The server knows about live photos and bursts; we do not
|
||||
/// have to guess at them.
|
||||
/// 2. Otherwise, same capture day **and** same file-name stem **and** one of the
|
||||
/// two is a camera raw while the other is not. That is the RAW+JPEG pair a
|
||||
/// camera writes, and the differing-kind clause keeps two JPEGs that happen
|
||||
/// to share a name apart.
|
||||
/// 3. Otherwise the photo is its own group.
|
||||
///
|
||||
/// The representative is the non-raw member when the group has one — a JPEG
|
||||
/// decodes in milliseconds and is what the person expects to see — and the
|
||||
/// earliest member in server order otherwise.
|
||||
fn group_photos(entries: &[Grouped]) -> Vec<String> {
|
||||
let mut parent: Vec<usize> = (0..entries.len()).collect();
|
||||
fn find(parent: &mut [usize], mut i: usize) -> usize {
|
||||
while parent[i] != i {
|
||||
parent[i] = parent[parent[i]];
|
||||
i = parent[i];
|
||||
}
|
||||
i
|
||||
}
|
||||
let union = |parent: &mut [usize], a: usize, b: usize| {
|
||||
let (a, b) = (find(parent, a), find(parent, b));
|
||||
if a != b {
|
||||
parent[b] = a;
|
||||
}
|
||||
};
|
||||
|
||||
let index: HashMap<&str, usize> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, e)| (e.uid.as_str(), i))
|
||||
.collect();
|
||||
for (i, entry) in entries.iter().enumerate() {
|
||||
if let Some(main) = entry.main_uid.as_deref()
|
||||
&& let Some(&j) = index.get(main)
|
||||
{
|
||||
union(&mut parent, j, i);
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 2 needs only one pass: everything sharing a day and a stem lands in
|
||||
// the same bucket, and the raw / non-raw split is checked pairwise there.
|
||||
let mut by_name: HashMap<(&str, &str), Vec<usize>> = HashMap::new();
|
||||
for (i, entry) in entries.iter().enumerate() {
|
||||
if let (Some(day), Some(stem)) = (entry.day.as_deref(), entry.stem.as_deref()) {
|
||||
by_name.entry((day, stem)).or_default().push(i);
|
||||
}
|
||||
}
|
||||
let is_raw = |i: usize| entries[i].kind == crate::control::PhotoKind::Raw;
|
||||
for bucket in by_name.values() {
|
||||
for (n, &i) in bucket.iter().enumerate() {
|
||||
for &j in &bucket[n + 1..] {
|
||||
if is_raw(i) != is_raw(j) {
|
||||
union(&mut parent, i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Representative per root, then the key every member carries.
|
||||
let mut representative: HashMap<usize, usize> = HashMap::new();
|
||||
for i in 0..entries.len() {
|
||||
let root = find(&mut parent, i);
|
||||
match representative.get(&root).copied() {
|
||||
Some(current) if !is_raw(current) || is_raw(i) => {}
|
||||
_ => {
|
||||
representative.insert(root, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
(0..entries.len())
|
||||
.map(|i| {
|
||||
let root = find(&mut parent, i);
|
||||
entries[representative[&root]].uid.clone()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The columns every photo read selects, with the group aggregates joined in.
|
||||
/// `g.n` is how many photos the group holds and `g.raw` whether any of them is a
|
||||
/// camera raw file — one query rather than a second lookup per tile.
|
||||
@@ -256,7 +131,7 @@ fn group_photos(entries: &[Grouped]) -> Vec<String> {
|
||||
/// own group: the column is `NULL` until the next refresh computes it, and a
|
||||
/// `NULL` would otherwise match nothing and drop the photo off the page.
|
||||
const PHOTO_SELECT: &str = "SELECT p.uid, p.capture_time, p.name, p.ratio, p.thumb_state, \
|
||||
p.kind, p.favorite, g.n, g.raw \
|
||||
p.kind, p.favorite, g.n, g.raw, p.place_id \
|
||||
FROM photos p JOIN (SELECT COALESCE(group_key, uid) AS k, COUNT(*) AS n, \
|
||||
MAX(kind = 2) AS raw FROM photos GROUP BY k) g ON g.k = COALESCE(p.group_key, p.uid)";
|
||||
|
||||
@@ -272,18 +147,290 @@ fn stored_photo(r: &rusqlite::Row<'_>) -> rusqlite::Result<StoredPhoto> {
|
||||
favorite: r.get::<_, i64>(6)? != 0,
|
||||
group_size: r.get::<_, i64>(7)?.max(1) as usize,
|
||||
has_raw: r.get::<_, i64>(8)? != 0,
|
||||
place_id: r.get(9)?,
|
||||
})
|
||||
}
|
||||
|
||||
const PHOTOS_GENERATION: &str = "photos_projection_generation";
|
||||
const PHOTOS_SYNCED: &str = "photos_synced_ms";
|
||||
const PHOTOS_AVAILABLE: &str = "photos_available";
|
||||
const PHOTOS_RECONCILE: &str = "photos_reconcile";
|
||||
|
||||
pub(super) fn photos_generation(conn: &Connection) -> rusqlite::Result<i64> {
|
||||
conn.query_row(
|
||||
"SELECT coalesce((SELECT CAST(value AS INTEGER) FROM sync_state WHERE key=?1),0)",
|
||||
[PHOTOS_GENERATION],
|
||||
|row| row.get(0),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn gallery_state(
|
||||
conn: &Connection,
|
||||
key: &str,
|
||||
value: impl rusqlite::ToSql,
|
||||
) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO sync_state(key,value) VALUES (?1,?2)
|
||||
ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
||||
params![key, value],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn photos_delete_tx(tx: &Transaction<'_>, uids: &[String]) -> Result<usize> {
|
||||
let mut removed = 0;
|
||||
for uid in uids {
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO photo_tombstones(uid) VALUES (?1)",
|
||||
[uid],
|
||||
)?;
|
||||
removed += tx.execute("DELETE FROM photos WHERE uid=?1", [uid])?;
|
||||
tx.execute("DELETE FROM albums WHERE uid=?1", [uid])?;
|
||||
tx.execute("DELETE FROM album_photos WHERE album_uid=?1", [uid])?;
|
||||
tx.execute(
|
||||
"UPDATE albums SET photo_count=max(photo_count-1,0)
|
||||
WHERE cover_uid=?1 OR uid IN (SELECT album_uid FROM album_photos WHERE uid=?1)",
|
||||
[uid],
|
||||
)?;
|
||||
tx.execute("DELETE FROM album_photos WHERE uid=?1", [uid])?;
|
||||
tx.execute(
|
||||
"UPDATE albums SET cover_uid=(SELECT uid FROM album_photos
|
||||
WHERE album_uid=albums.uid ORDER BY seq LIMIT 1) WHERE cover_uid=?1",
|
||||
[uid],
|
||||
)?;
|
||||
}
|
||||
tx.execute(
|
||||
"UPDATE photos SET group_key=NULL WHERE group_key IS NOT NULL
|
||||
AND group_key NOT IN (SELECT uid FROM photos)",
|
||||
[],
|
||||
)?;
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
impl Db {
|
||||
pub fn photo_projection_generation(&self) -> Result<i64> {
|
||||
Ok(photos_generation(&self.read())?)
|
||||
}
|
||||
|
||||
/// One transaction records the deletion, stale metadata, listing work and feed position.
|
||||
pub fn photos_apply_events(
|
||||
&self,
|
||||
cursor_key: &str,
|
||||
cursor: &str,
|
||||
gone: &[String],
|
||||
touched: &[String],
|
||||
gap: bool,
|
||||
seed: bool,
|
||||
) -> Result<()> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
if !gone.is_empty() {
|
||||
photos_delete_tx(&tx, gone)?;
|
||||
}
|
||||
for uid in touched {
|
||||
tx.execute("DELETE FROM photo_tombstones WHERE uid=?1", [uid])?;
|
||||
tx.execute("UPDATE photos SET resolved_at=NULL WHERE uid=?1", [uid])?;
|
||||
tx.execute("UPDATE album_photos SET resolved=0 WHERE uid=?1", [uid])?;
|
||||
}
|
||||
if gap {
|
||||
tx.execute("UPDATE photos SET resolved_at=NULL", [])?;
|
||||
tx.execute("UPDATE album_photos SET resolved=0", [])?;
|
||||
gallery_state(&tx, PHOTOS_RECONCILE, 1)?;
|
||||
}
|
||||
if seed || gap || !gone.is_empty() || !touched.is_empty() {
|
||||
let generation = photos_generation(&tx)?;
|
||||
gallery_state(&tx, PHOTOS_GENERATION, generation + 1)?;
|
||||
tx.execute(
|
||||
"DELETE FROM sync_state WHERE key='albums_synced_ms'
|
||||
OR key GLOB 'album_synced_ms:*'",
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
if seed || gap || !touched.is_empty() {
|
||||
tx.execute("DELETE FROM sync_state WHERE key=?1", [PHOTOS_SYNCED])?;
|
||||
}
|
||||
gallery_state(&tx, cursor_key, cursor)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn photos_invalidate(&self) -> Result<()> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let generation = photos_generation(&tx)?;
|
||||
gallery_state(&tx, PHOTOS_GENERATION, generation + 1)?;
|
||||
tx.execute("DELETE FROM sync_state WHERE key=?1", [PHOTOS_SYNCED])?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn photos_need_work(&self) -> Result<bool> {
|
||||
let conn = self.read();
|
||||
Ok(conn.query_row(
|
||||
"SELECT NOT EXISTS(SELECT 1 FROM sync_state WHERE key=?1)
|
||||
OR EXISTS(SELECT 1 FROM photos WHERE resolved_at IS NULL)",
|
||||
[PHOTOS_SYNCED],
|
||||
|r| r.get(0),
|
||||
)?)
|
||||
}
|
||||
|
||||
pub fn photos_pending(&self) -> Result<Vec<String>> {
|
||||
let conn = self.read();
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT uid FROM photos WHERE resolved_at IS NULL ORDER BY seq")?;
|
||||
Ok(stmt
|
||||
.query_map([], |r| r.get(0))?
|
||||
.collect::<rusqlite::Result<_>>()?)
|
||||
}
|
||||
|
||||
pub fn photo_removed(&self, uid: &str) -> Result<bool> {
|
||||
Ok(self.read().query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM photo_tombstones WHERE uid=?1)",
|
||||
[uid],
|
||||
|r| r.get(0),
|
||||
)?)
|
||||
}
|
||||
|
||||
pub fn photos_contains(&self, uid: &str, album: Option<&str>) -> Result<bool> {
|
||||
let conn = self.read();
|
||||
Ok(match album {
|
||||
Some(album) => conn.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM album_photos WHERE album_uid=?1 AND uid=?2)",
|
||||
params![album, uid],
|
||||
|r| r.get(0),
|
||||
)?,
|
||||
None => conn.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM photos WHERE uid=?1)
|
||||
OR EXISTS(SELECT 1 FROM album_photos WHERE uid=?1)",
|
||||
[uid],
|
||||
|r| r.get(0),
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Persist one enrichment batch. The generation check and UPDATE share the
|
||||
/// writer transaction with deletion, so a stale batch cannot restore a row.
|
||||
pub fn photos_resolve_at(&self, generation: i64, rows: &[TimelineRow]) -> Result<bool> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
if photos_generation(&tx)? != generation {
|
||||
return Ok(false);
|
||||
}
|
||||
{
|
||||
let mut update = tx.prepare(
|
||||
"UPDATE photos SET name=?2, media_type=?3, kind=?4, favorite=?5,
|
||||
similar_hash=CASE WHEN content_hash IS ?6 THEN similar_hash ELSE NULL END,
|
||||
content_hash=?6, main_uid=?7, latitude=?8, longitude=?9, place_id=?10,
|
||||
resolved_at=?11 WHERE uid=?1",
|
||||
)?;
|
||||
for row in rows.iter().filter(|row| row.resolved_at.is_some()) {
|
||||
let kind = crate::control::PhotoKind::classify(
|
||||
row.name.as_deref(),
|
||||
row.media_type.as_deref(),
|
||||
);
|
||||
update.execute(params![
|
||||
row.uid,
|
||||
row.name,
|
||||
row.media_type,
|
||||
kind.as_i64(),
|
||||
row.favorite.unwrap_or(false),
|
||||
row.content_hash,
|
||||
row.main_uid,
|
||||
row.location.map(|p| p.0),
|
||||
row.location.map(|p| p.1),
|
||||
row.location
|
||||
.and_then(|(lat, lon)| crate::places::nearest(lat, lon))
|
||||
.map(|city| city.id),
|
||||
row.resolved_at
|
||||
])?;
|
||||
}
|
||||
}
|
||||
if !rows.is_empty() {
|
||||
let (entries, existing): (Vec<_>, Vec<Option<String>>) = {
|
||||
let mut stmt = tx.prepare(
|
||||
"SELECT uid,kind,main_uid,strftime('%Y-%m-%d',capture_time,'unixepoch','localtime'),
|
||||
name,group_key FROM photos ORDER BY seq",
|
||||
)?;
|
||||
stmt.query_map([], |r| {
|
||||
let name: Option<String> = r.get(4)?;
|
||||
Ok((
|
||||
Grouped {
|
||||
uid: r.get(0)?,
|
||||
raw: crate::control::PhotoKind::from_i64(r.get(1)?)
|
||||
== crate::control::PhotoKind::Raw,
|
||||
main_uid: r.get(2)?,
|
||||
day: r.get(3)?,
|
||||
stem: name_stem(name.as_deref()),
|
||||
},
|
||||
r.get(5)?,
|
||||
))
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?
|
||||
.into_iter()
|
||||
.unzip()
|
||||
};
|
||||
let mut update = tx.prepare("UPDATE photos SET group_key=?2 WHERE uid=?1")?;
|
||||
for ((entry, old), group) in entries.iter().zip(existing).zip(group_photos(&entries)) {
|
||||
if old.as_deref() != Some(group.as_str()) {
|
||||
update.execute(params![entry.uid, group])?;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Replace the timeline wholesale. A row whose `resolved_at` is `Some` was
|
||||
/// read from the server in this pass, so its fields are taken as they are —
|
||||
/// a `None` there means the server no longer reports that value. A row whose
|
||||
/// `resolved_at` is `None` was skipped or failed to resolve, and keeps
|
||||
/// whatever is already stored rather than having it silently cleared.
|
||||
pub fn photos_replace(&self, items: &[TimelineRow]) -> Result<()> {
|
||||
self.photos_replace_impl(items, None).map(|_| ())
|
||||
}
|
||||
|
||||
pub fn photos_replace_at(
|
||||
&self,
|
||||
items: &[TimelineRow],
|
||||
generation: i64,
|
||||
available: bool,
|
||||
now: i64,
|
||||
) -> Result<bool> {
|
||||
self.photos_replace_impl(items, Some((generation, available, now)))
|
||||
}
|
||||
|
||||
fn photos_replace_impl(
|
||||
&self,
|
||||
items: &[TimelineRow],
|
||||
stamp: Option<(i64, bool, i64)>,
|
||||
) -> Result<bool> {
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let mut tombstones = HashSet::new();
|
||||
if let Some((generation, _, _)) = stamp {
|
||||
if photos_generation(&tx)? != generation {
|
||||
return Ok(false);
|
||||
}
|
||||
let repairing = tx
|
||||
.query_row(
|
||||
"SELECT 1 FROM sync_state WHERE key=?1",
|
||||
[PHOTOS_RECONCILE],
|
||||
|_| Ok(()),
|
||||
)
|
||||
.optional()?
|
||||
.is_some();
|
||||
if repairing {
|
||||
let mut remove = tx.prepare("DELETE FROM photo_tombstones WHERE uid=?1")?;
|
||||
for row in items {
|
||||
remove.execute([&row.uid])?;
|
||||
}
|
||||
tx.execute("DELETE FROM sync_state WHERE key=?1", [PHOTOS_RECONCILE])?;
|
||||
}
|
||||
let mut stmt = tx.prepare("SELECT uid FROM photo_tombstones")?;
|
||||
tombstones = stmt
|
||||
.query_map([], |r| r.get::<_, String>(0))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
}
|
||||
// `name` and `media_type` are learned-and-kept like the ratio and thumb
|
||||
// verdict: the timeline DTO carries only the uid and capture time, so the
|
||||
// daemon may not know them when it replaces the timeline — because the
|
||||
@@ -322,6 +469,7 @@ impl Db {
|
||||
// Everything the insert and the grouping need, resolved once per photo.
|
||||
let resolved: Vec<(&TimelineRow, Learned, crate::control::PhotoKind)> = items
|
||||
.iter()
|
||||
.filter(|row| !tombstones.contains(&row.uid))
|
||||
.map(|row| {
|
||||
let mut carried = learned.get(&row.uid).cloned().unwrap_or(Learned {
|
||||
thumb_state: THUMB_UNKNOWN,
|
||||
@@ -366,7 +514,7 @@ impl Db {
|
||||
.iter()
|
||||
.map(|(row, carried, kind)| Grouped {
|
||||
uid: row.uid.clone(),
|
||||
kind: *kind,
|
||||
raw: *kind == crate::control::PhotoKind::Raw,
|
||||
main_uid: carried.main_uid.clone(),
|
||||
day: capture_day(&tx, row.capture_time),
|
||||
stem: name_stem(carried.name.as_deref()),
|
||||
@@ -411,8 +559,12 @@ impl Db {
|
||||
])?;
|
||||
}
|
||||
}
|
||||
if let Some((_, available, now)) = stamp {
|
||||
gallery_state(&tx, PHOTOS_AVAILABLE, available as i64)?;
|
||||
gallery_state(&tx, PHOTOS_SYNCED, now)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// The uids whose node metadata is already resolved, so a refresh can skip
|
||||
@@ -455,8 +607,16 @@ impl Db {
|
||||
/// refresh. For a continuity loss, where the daemon cannot know what it
|
||||
/// missed, and for an explicit `refresh photos --full`.
|
||||
pub fn photos_mark_all_unresolved(&self) -> Result<usize> {
|
||||
let conn = self.conn.lock();
|
||||
Ok(conn.execute("UPDATE photos SET resolved_at = NULL", [])?)
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let marked = tx.execute("UPDATE photos SET resolved_at=NULL", [])?;
|
||||
tx.execute("UPDATE album_photos SET resolved=0", [])?;
|
||||
let generation = photos_generation(&tx)?;
|
||||
gallery_state(&tx, PHOTOS_GENERATION, generation + 1)?;
|
||||
gallery_state(&tx, PHOTOS_RECONCILE, 1)?;
|
||||
tx.execute("DELETE FROM sync_state WHERE key=?1", [PHOTOS_SYNCED])?;
|
||||
tx.commit()?;
|
||||
Ok(marked)
|
||||
}
|
||||
|
||||
/// Forget photos that have been trashed, so the gallery does not show them
|
||||
@@ -471,23 +631,9 @@ impl Db {
|
||||
}
|
||||
let mut conn = self.conn.lock();
|
||||
let tx = conn.transaction()?;
|
||||
let mut removed = 0;
|
||||
{
|
||||
let mut photos = tx.prepare("DELETE FROM photos WHERE uid = ?1")?;
|
||||
let mut albums = tx.prepare("DELETE FROM album_photos WHERE uid = ?1")?;
|
||||
for uid in uids {
|
||||
removed += photos.execute([uid])?;
|
||||
albums.execute([uid])?;
|
||||
}
|
||||
}
|
||||
// A group whose representative went is left pointing at nothing, which
|
||||
// drops its other files off every page. Make them stand alone until the
|
||||
// next refresh groups them again.
|
||||
tx.execute(
|
||||
"UPDATE photos SET group_key = NULL WHERE group_key IS NOT NULL \
|
||||
AND group_key NOT IN (SELECT uid FROM photos)",
|
||||
[],
|
||||
)?;
|
||||
let removed = photos_delete_tx(&tx, uids)?;
|
||||
let generation = photos_generation(&tx)?;
|
||||
gallery_state(&tx, PHOTOS_GENERATION, generation + 1)?;
|
||||
tx.commit()?;
|
||||
Ok(removed)
|
||||
}
|
||||
@@ -510,24 +656,42 @@ impl Db {
|
||||
range: Option<(i64, i64)>,
|
||||
favorites: bool,
|
||||
not_in_album: bool,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
self.photos_page_visible(offset, limit, kind, range, favorites, not_in_album, "[]")
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn photos_page_visible(
|
||||
&self,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
kind: Option<crate::control::PhotoKind>,
|
||||
range: Option<(i64, i64)>,
|
||||
favorites: bool,
|
||||
not_in_album: bool,
|
||||
excluded: &str,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.read();
|
||||
// Built up so any combination of the optional filters is one indexed
|
||||
// query rather than a statement per case.
|
||||
let mut sql = String::from(PHOTO_SELECT);
|
||||
let mut binds: Vec<i64> = Vec::new();
|
||||
let mut binds: Vec<rusqlite::types::Value> = Vec::new();
|
||||
let mut conds: Vec<String> = Vec::new();
|
||||
if excluded != "[]" {
|
||||
binds.push(excluded.to_owned().into());
|
||||
conds.push("p.uid NOT IN (SELECT value FROM json_each(?1))".to_owned());
|
||||
}
|
||||
if kind != Some(crate::control::PhotoKind::Raw) {
|
||||
conds.push("p.uid = COALESCE(p.group_key, p.uid)".to_string());
|
||||
}
|
||||
if let Some(k) = kind {
|
||||
binds.push(k.as_i64());
|
||||
binds.push((k.as_i64()).into());
|
||||
conds.push(format!("p.kind = ?{}", binds.len()));
|
||||
}
|
||||
if let Some((from, to)) = range {
|
||||
binds.push(from);
|
||||
binds.push((from).into());
|
||||
conds.push(format!("p.capture_time >= ?{}", binds.len()));
|
||||
binds.push(to);
|
||||
binds.push((to).into());
|
||||
conds.push(format!("p.capture_time < ?{}", binds.len()));
|
||||
}
|
||||
if favorites {
|
||||
@@ -546,9 +710,9 @@ impl Db {
|
||||
sql.push_str(" WHERE ");
|
||||
sql.push_str(&conds.join(" AND "));
|
||||
}
|
||||
binds.push(limit as i64);
|
||||
binds.push((limit as i64).into());
|
||||
let limit_pos = binds.len();
|
||||
binds.push(offset as i64);
|
||||
binds.push((offset as i64).into());
|
||||
let offset_pos = binds.len();
|
||||
sql.push_str(&format!(
|
||||
" ORDER BY p.seq LIMIT ?{limit_pos} OFFSET ?{offset_pos}"
|
||||
@@ -577,6 +741,25 @@ impl Db {
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Include associated files so a public preview cannot reveal a hidden
|
||||
/// RAW, JPEG or Live Photo through another part of the same shot.
|
||||
pub fn photo_group_members_for(&self, uids: &HashSet<String>) -> Result<HashSet<String>> {
|
||||
if uids.is_empty() {
|
||||
return Ok(HashSet::new());
|
||||
}
|
||||
let ids = serde_json::to_string(uids).expect("file identifiers serialize");
|
||||
let conn = self.read();
|
||||
let mut query = conn.prepare(
|
||||
"SELECT uid FROM photos WHERE COALESCE(group_key, uid) IN (
|
||||
SELECT COALESCE(group_key, uid) FROM photos
|
||||
WHERE uid IN (SELECT value FROM json_each(?1))
|
||||
)",
|
||||
)?;
|
||||
Ok(query
|
||||
.query_map([ids], |row| row.get(0))?
|
||||
.collect::<rusqlite::Result<_>>()?)
|
||||
}
|
||||
|
||||
/// The whole persisted timeline, in stored order — every file, not one row
|
||||
/// per group. Only for whole-library passes (the photo re-date); the gallery
|
||||
/// pages with [`Db::photos_page`].
|
||||
@@ -598,27 +781,38 @@ impl Db {
|
||||
pub fn photos_months(
|
||||
&self,
|
||||
kind: Option<crate::control::PhotoKind>,
|
||||
) -> Result<Vec<(i32, i32, usize)>> {
|
||||
self.photos_months_visible(kind, "[]")
|
||||
}
|
||||
|
||||
pub fn photos_months_visible(
|
||||
&self,
|
||||
kind: Option<crate::control::PhotoKind>,
|
||||
excluded: &str,
|
||||
) -> Result<Vec<(i32, i32, usize)>> {
|
||||
let conn = self.read();
|
||||
// The scrubber has to agree with the page it scrubs, so it counts what
|
||||
// the page shows: groups everywhere except on the Raw tab.
|
||||
let (filter, binds): (&str, Vec<i64>) = match kind {
|
||||
Some(crate::control::PhotoKind::Raw) => (" WHERE kind = 2", Vec::new()),
|
||||
Some(k) => (
|
||||
" WHERE uid = COALESCE(group_key, uid) AND kind = ?1",
|
||||
vec![k.as_i64()],
|
||||
),
|
||||
None => (" WHERE uid = COALESCE(group_key, uid)", Vec::new()),
|
||||
let filter = match kind {
|
||||
Some(crate::control::PhotoKind::Raw) => "kind = ?2",
|
||||
Some(_) => "uid = COALESCE(group_key, uid) AND kind = ?2",
|
||||
None => "uid = COALESCE(group_key, uid) AND ?2 IS NULL",
|
||||
};
|
||||
let sql = format!(
|
||||
"SELECT CAST(strftime('%Y', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS y, \
|
||||
"WITH hidden_groups AS (
|
||||
SELECT COALESCE(group_key, uid) FROM photos
|
||||
WHERE uid IN (SELECT value FROM json_each(?1))
|
||||
)
|
||||
SELECT CAST(strftime('%Y', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS y, \
|
||||
CAST(strftime('%m', capture_time, 'unixepoch', 'localtime') AS INTEGER) AS m, \
|
||||
COUNT(*) \
|
||||
FROM photos{filter} GROUP BY y, m ORDER BY y DESC, m DESC"
|
||||
FROM photos WHERE {filter}
|
||||
AND COALESCE(group_key, uid) NOT IN (SELECT * FROM hidden_groups)
|
||||
GROUP BY y, m ORDER BY y DESC, m DESC"
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params_from_iter(binds), |r| {
|
||||
.query_map(params![excluded, kind.map(|kind| kind.as_i64())], |r| {
|
||||
Ok((
|
||||
r.get::<_, i32>(0)?,
|
||||
r.get::<_, i32>(1)?,
|
||||
@@ -634,6 +828,15 @@ impl Db {
|
||||
/// pick the date, and its own year is left out, because today's photos are
|
||||
/// what the timeline already opens on. One row per group, like the timeline.
|
||||
pub fn photos_on_this_day(&self, today: i64, limit: usize) -> Result<Vec<StoredPhoto>> {
|
||||
self.photos_on_this_day_visible(today, limit, "[]")
|
||||
}
|
||||
|
||||
pub fn photos_on_this_day_visible(
|
||||
&self,
|
||||
today: i64,
|
||||
limit: usize,
|
||||
excluded: &str,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.read();
|
||||
let sql = format!(
|
||||
"{PHOTO_SELECT} WHERE p.uid = COALESCE(p.group_key, p.uid) \
|
||||
@@ -641,11 +844,12 @@ impl Db {
|
||||
= strftime('%m-%d', ?1, 'unixepoch', 'localtime') \
|
||||
AND strftime('%Y', p.capture_time, 'unixepoch', 'localtime') \
|
||||
< strftime('%Y', ?1, 'unixepoch', 'localtime') \
|
||||
AND p.uid NOT IN (SELECT value FROM json_each(?3))
|
||||
ORDER BY p.seq LIMIT ?2"
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(params![today, limit as i64], stored_photo)?
|
||||
.query_map(params![today, limit as i64, excluded], stored_photo)?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
@@ -654,6 +858,10 @@ impl Db {
|
||||
/// cover, most photographed first. Counted per group, like the timeline;
|
||||
/// photos without a location, or taken far from any town, are in none.
|
||||
pub fn photo_places(&self) -> Result<Vec<StoredPlace>> {
|
||||
self.photo_places_visible("[]")
|
||||
}
|
||||
|
||||
pub fn photo_places_visible(&self, excluded: &str) -> Result<Vec<StoredPlace>> {
|
||||
let rows: Vec<(u32, usize, String)> = {
|
||||
let conn = self.read();
|
||||
// SQLite fills the bare `uid` from the row `MIN(seq)` picked, which
|
||||
@@ -661,9 +869,10 @@ impl Db {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT place_id, COUNT(*), uid, MIN(seq) FROM photos \
|
||||
WHERE uid = COALESCE(group_key, uid) AND place_id IS NOT NULL \
|
||||
AND uid NOT IN (SELECT value FROM json_each(?1))
|
||||
GROUP BY place_id ORDER BY COUNT(*) DESC, MIN(seq)",
|
||||
)?;
|
||||
stmt.query_map([], |r| {
|
||||
stmt.query_map([excluded], |r| {
|
||||
Ok((r.get(0)?, r.get::<_, i64>(1)? as usize, r.get(2)?))
|
||||
})?
|
||||
.collect::<rusqlite::Result<_>>()?
|
||||
@@ -692,15 +901,29 @@ impl Db {
|
||||
id: u32,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
self.photos_at_place_visible(id, offset, limit, "[]")
|
||||
}
|
||||
|
||||
pub fn photos_at_place_visible(
|
||||
&self,
|
||||
id: u32,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
excluded: &str,
|
||||
) -> Result<Vec<StoredPhoto>> {
|
||||
let conn = self.read();
|
||||
let sql = format!(
|
||||
"{PHOTO_SELECT} WHERE p.uid = COALESCE(p.group_key, p.uid) AND p.place_id = ?1 \
|
||||
"{PHOTO_SELECT} WHERE p.uid = COALESCE(p.group_key, p.uid) AND p.place_id = ?1
|
||||
AND p.uid NOT IN (SELECT value FROM json_each(?4))
|
||||
ORDER BY p.seq LIMIT ?2 OFFSET ?3"
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(params![id, limit as i64, offset as i64], stored_photo)?
|
||||
.query_map(
|
||||
params![id, limit as i64, offset as i64, excluded],
|
||||
stored_photo,
|
||||
)?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
@@ -881,6 +1104,10 @@ impl Db {
|
||||
|
||||
/// Per-tab counts for the Photos page subtitle: `(photos, videos, raw)`.
|
||||
pub fn photos_counts(&self) -> Result<(usize, usize, usize)> {
|
||||
self.photos_counts_visible("[]")
|
||||
}
|
||||
|
||||
pub fn photos_counts_visible(&self, excluded: &str) -> Result<(usize, usize, usize)> {
|
||||
use crate::control::PhotoKind;
|
||||
let conn = self.read();
|
||||
// Photos and Videos count groups — one shot is one entry, however many
|
||||
@@ -888,9 +1115,12 @@ impl Db {
|
||||
// them individually.
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT kind, COUNT(*) FROM photos \
|
||||
WHERE uid = COALESCE(group_key, uid) OR kind = 2 GROUP BY kind",
|
||||
WHERE (uid = COALESCE(group_key, uid) OR kind = 2)
|
||||
AND uid NOT IN (SELECT value FROM json_each(?1)) GROUP BY kind",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?)))?;
|
||||
let rows = stmt.query_map([excluded], |r| {
|
||||
Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?))
|
||||
})?;
|
||||
let (mut photos, mut videos, mut raw) = (0usize, 0usize, 0usize);
|
||||
for row in rows {
|
||||
let (kind, n) = row?;
|
||||
|
||||
@@ -9,6 +9,8 @@ use std::collections::HashMap;
|
||||
|
||||
pub struct StoredSyncFolder {
|
||||
pub id: i64,
|
||||
/// `device` backup or `attached` My files mirror.
|
||||
pub kind: String,
|
||||
pub local_path: String,
|
||||
pub remote_uid: String,
|
||||
pub remote_share_id: String,
|
||||
@@ -45,6 +47,12 @@ pub struct StoredSyncEntry {
|
||||
pub local_mtime_ns: Option<i64>,
|
||||
pub remote_rev: Option<String>,
|
||||
pub remote_hash: Option<String>,
|
||||
/// The id of the remote revision this row was synced at. A mirror upload
|
||||
/// checks it before it supersedes what Drive holds, because another client
|
||||
/// may have committed a revision since (`docs/BUGS.md` B249). `None` on a
|
||||
/// row written before schema 44, or by a pass that did not learn it; the
|
||||
/// check then falls back to `remote_rev` and `remote_hash`.
|
||||
pub remote_revision_id: Option<String>,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
@@ -53,12 +61,32 @@ impl Db {
|
||||
local_path: &str,
|
||||
remote_uid: &str,
|
||||
remote_share_id: &str,
|
||||
) -> Result<i64> {
|
||||
self.sync_folder_insert(local_path, remote_uid, remote_share_id, "device")
|
||||
}
|
||||
|
||||
/// Attach an existing My files folder, without creating a device backup.
|
||||
pub fn sync_folder_attach(
|
||||
&self,
|
||||
local_path: &str,
|
||||
remote_uid: &str,
|
||||
remote_share_id: &str,
|
||||
) -> Result<i64> {
|
||||
self.sync_folder_insert(local_path, remote_uid, remote_share_id, "attached")
|
||||
}
|
||||
|
||||
fn sync_folder_insert(
|
||||
&self,
|
||||
local_path: &str,
|
||||
remote_uid: &str,
|
||||
remote_share_id: &str,
|
||||
kind: &str,
|
||||
) -> Result<i64> {
|
||||
let conn = self.conn.lock();
|
||||
conn.execute(
|
||||
"INSERT INTO sync_folder (local_path, remote_uid, remote_share_id)
|
||||
VALUES (?1, ?2, ?3)",
|
||||
params![local_path, remote_uid, remote_share_id],
|
||||
"INSERT INTO sync_folder (local_path, remote_uid, remote_share_id, kind)
|
||||
VALUES (?1, ?2, ?3, ?4)",
|
||||
params![local_path, remote_uid, remote_share_id, kind],
|
||||
)?;
|
||||
Ok(conn.last_insert_rowid())
|
||||
}
|
||||
@@ -67,7 +95,7 @@ impl Db {
|
||||
pub fn sync_folder_list(&self) -> Result<Vec<StoredSyncFolder>> {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync
|
||||
"SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync, kind
|
||||
FROM sync_folder ORDER BY id",
|
||||
)?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
@@ -80,6 +108,7 @@ impl Db {
|
||||
pending_mode: r.get(5)?,
|
||||
state: r.get(6)?,
|
||||
last_sync: r.get(7)?,
|
||||
kind: r.get(8)?,
|
||||
})
|
||||
})?;
|
||||
let mut items = Vec::new();
|
||||
@@ -93,7 +122,7 @@ impl Db {
|
||||
pub fn sync_folder_get(&self, id: i64) -> Result<Option<StoredSyncFolder>> {
|
||||
let conn = self.read();
|
||||
conn.query_row(
|
||||
"SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync
|
||||
"SELECT id, local_path, remote_uid, remote_share_id, mode, pending_mode, state, last_sync, kind
|
||||
FROM sync_folder WHERE id = ?1",
|
||||
params![id],
|
||||
|r| {
|
||||
@@ -106,6 +135,7 @@ impl Db {
|
||||
pending_mode: r.get(5)?,
|
||||
state: r.get(6)?,
|
||||
last_sync: r.get(7)?,
|
||||
kind: r.get(8)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
@@ -188,7 +218,7 @@ impl Db {
|
||||
let conn = self.read();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash,
|
||||
local_mtime_ns
|
||||
local_mtime_ns, remote_revision_id
|
||||
FROM sync_entry WHERE folder_id = ?1",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![folder_id], |r| {
|
||||
@@ -200,6 +230,7 @@ impl Db {
|
||||
remote_rev: r.get(4)?,
|
||||
remote_hash: r.get(5)?,
|
||||
local_mtime_ns: r.get(6)?,
|
||||
remote_revision_id: r.get(7)?,
|
||||
})
|
||||
})?;
|
||||
let mut map = HashMap::new();
|
||||
@@ -238,15 +269,16 @@ impl Db {
|
||||
conn.execute(
|
||||
"INSERT INTO sync_entry
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev,
|
||||
remote_hash, local_mtime_ns)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
||||
remote_hash, local_mtime_ns, remote_revision_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
||||
ON CONFLICT(folder_id, rel_path) DO UPDATE SET
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash,
|
||||
local_mtime_ns = excluded.local_mtime_ns",
|
||||
remote_uid = excluded.remote_uid,
|
||||
local_mtime = excluded.local_mtime,
|
||||
local_size = excluded.local_size,
|
||||
remote_rev = excluded.remote_rev,
|
||||
remote_hash = excluded.remote_hash,
|
||||
local_mtime_ns = excluded.local_mtime_ns,
|
||||
remote_revision_id = excluded.remote_revision_id",
|
||||
params![
|
||||
folder_id,
|
||||
e.rel_path,
|
||||
@@ -256,6 +288,7 @@ impl Db {
|
||||
e.remote_rev,
|
||||
e.remote_hash,
|
||||
e.local_mtime_ns,
|
||||
e.remote_revision_id,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
@@ -306,7 +339,7 @@ impl Db {
|
||||
{
|
||||
let mut select = tx.prepare(
|
||||
"SELECT rel_path, remote_uid, local_mtime, local_size, remote_rev, remote_hash,
|
||||
local_mtime_ns
|
||||
local_mtime_ns, remote_revision_id
|
||||
FROM sync_entry
|
||||
WHERE folder_id = ?1
|
||||
AND (rel_path = ?2 OR substr(rel_path, 1, length(?2) + 1) = ?2 || '/')",
|
||||
@@ -321,6 +354,7 @@ impl Db {
|
||||
remote_rev: r.get(4)?,
|
||||
remote_hash: r.get(5)?,
|
||||
local_mtime_ns: r.get(6)?,
|
||||
remote_revision_id: r.get(7)?,
|
||||
})
|
||||
})?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
@@ -332,8 +366,8 @@ impl Db {
|
||||
let mut insert = tx.prepare(
|
||||
"INSERT OR REPLACE INTO sync_entry
|
||||
(folder_id, rel_path, remote_uid, local_mtime, local_size, remote_rev,
|
||||
remote_hash, local_mtime_ns)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
||||
remote_hash, local_mtime_ns, remote_revision_id)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
|
||||
)?;
|
||||
for row in rows {
|
||||
let rel = format!("{to_rel}{}", &row.rel_path[from_rel.len()..]);
|
||||
@@ -346,6 +380,7 @@ impl Db {
|
||||
row.remote_rev,
|
||||
row.remote_hash,
|
||||
row.local_mtime_ns,
|
||||
row.remote_revision_id,
|
||||
])?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -337,6 +337,10 @@ fn places_group_photos_by_the_nearest_town() {
|
||||
.map(|p| p.uid)
|
||||
.collect();
|
||||
assert_eq!(berlin, ["b2", "b1-jpeg"]);
|
||||
let visible = db.photo_places_visible(r#"["b2","p1"]"#).unwrap();
|
||||
assert_eq!(visible.len(), 1);
|
||||
assert_eq!(visible[0].count, 1);
|
||||
assert_eq!(visible[0].cover.uid, "b1-jpeg");
|
||||
}
|
||||
|
||||
/// A town's photos come with where each was taken, one per RAW+JPEG shot.
|
||||
@@ -363,6 +367,23 @@ fn place_photos_carry_their_own_location() {
|
||||
);
|
||||
}
|
||||
|
||||
/// A timeline photo names the town it was taken in, so the viewer can show it;
|
||||
/// one without a location names none.
|
||||
#[test]
|
||||
fn timeline_photos_name_their_town() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.photos_replace(&[
|
||||
located("b", 500, "IMG_5.JPG", BERLIN),
|
||||
TimelineRow::new("n", 400),
|
||||
])
|
||||
.unwrap();
|
||||
let berlin = db.photo_places().unwrap()[0].id;
|
||||
let mut photos = db.photos_by_uid(&["b".into(), "n".into()]).unwrap();
|
||||
photos.sort_by(|a, b| a.uid.cmp(&b.uid));
|
||||
let towns: Vec<Option<u32>> = photos.iter().map(|photo| photo.place_id).collect();
|
||||
assert_eq!(towns, [Some(berlin), None]);
|
||||
}
|
||||
|
||||
/// A refresh that skips an already-resolved photo keeps where it was taken; one
|
||||
/// that reads the node again takes what the node now says.
|
||||
#[test]
|
||||
@@ -1283,6 +1304,90 @@ fn a_corrupt_database_is_moved_aside_and_rebuilt() {
|
||||
let _ = std::fs::remove_file(path.with_extension("db.lock"));
|
||||
}
|
||||
|
||||
/// A rebuilt database must not lose what only this machine has: writes Drive
|
||||
/// has not received, the mirror baselines and the pins (`docs/BUGS.md` B216).
|
||||
#[test]
|
||||
fn a_rebuild_keeps_the_queue_baselines_and_pins_of_the_damaged_file() {
|
||||
let id = now_test_id();
|
||||
let damaged =
|
||||
std::env::temp_dir().join(format!("pdfs-salvage-old-{}-{id}.db", std::process::id()));
|
||||
let fresh =
|
||||
std::env::temp_dir().join(format!("pdfs-salvage-new-{}-{id}.db", std::process::id()));
|
||||
|
||||
let db = Db::open(&damaged).unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&file("listed", "root", "listed.txt", 1))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("queued", "root", "queued.txt", 2))
|
||||
.unwrap();
|
||||
db.set_listed(&uid("root"), true).unwrap();
|
||||
db.enqueue_op(&PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("queued").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some("/staging/queued".into()),
|
||||
meta_json: None,
|
||||
created_at: 1,
|
||||
attempts: 3,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let folder_id = db
|
||||
.sync_folder_add("/home/me/Docs", "vol~docs", "share")
|
||||
.unwrap();
|
||||
db.sync_entry_upsert(
|
||||
folder_id,
|
||||
&StoredSyncEntry {
|
||||
rel_path: "a.txt".into(),
|
||||
remote_uid: Some("vol~a".into()),
|
||||
local_mtime: 5,
|
||||
local_mtime_ns: None,
|
||||
local_size: 6,
|
||||
remote_rev: Some("7".into()),
|
||||
remote_hash: Some("8".into()),
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
db.pin_add("vol~pinned", "/pinned", true).unwrap();
|
||||
drop(db);
|
||||
|
||||
let db = Db::open(&fresh).unwrap();
|
||||
let results = salvage(&db.conn.lock(), &damaged);
|
||||
assert!(results.iter().all(|(_, r)| r.is_ok()), "{results:?}");
|
||||
|
||||
let ops = db.pending_ops().unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(ops[0].uid, uid("queued").to_string());
|
||||
assert_eq!(ops[0].blob_path.as_deref(), Some("/staging/queued"));
|
||||
// The op's node came with it; the rest of the tree is fetched again.
|
||||
assert!(
|
||||
db.node_by_uid(&uid("queued").to_string())
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert!(
|
||||
db.node_by_uid(&uid("listed").to_string())
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(db.sync_entries(folder_id).unwrap().len(), 1);
|
||||
assert!(db.is_pinned("vol~pinned").unwrap());
|
||||
drop(db);
|
||||
|
||||
for path in [&damaged, &fresh] {
|
||||
for suffix in ["", "-wal", "-shm"] {
|
||||
let mut file = path.as_os_str().to_owned();
|
||||
file.push(suffix);
|
||||
let _ = std::fs::remove_file(std::path::PathBuf::from(file));
|
||||
}
|
||||
let _ = std::fs::remove_file(path.with_extension("db.lock"));
|
||||
}
|
||||
}
|
||||
|
||||
fn local(path: &str, name: &str, is_dir: bool) -> LocalEntry {
|
||||
LocalEntry {
|
||||
path: path.into(),
|
||||
@@ -1346,6 +1451,77 @@ fn rescanning_an_unchanged_file_keeps_exactly_one_index_entry() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A scan that stopped early or lost a batch did not see every file, so it
|
||||
/// must not prune the ones it missed (`docs/BUGS.md` B217).
|
||||
#[test]
|
||||
fn a_partial_local_scan_prunes_nothing() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let g = db.local_begin_scan().unwrap();
|
||||
db.local_upsert_batch(
|
||||
g,
|
||||
&[
|
||||
local("/home/u/a.txt", "a.txt", false),
|
||||
local("/home/u/b.txt", "b.txt", false),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(db.local_finish_scan(g, 1).unwrap(), 2);
|
||||
|
||||
let g = db.local_begin_scan().unwrap();
|
||||
db.local_upsert_batch(g, &[local("/home/u/a.txt", "a.txt", false)])
|
||||
.unwrap();
|
||||
assert_eq!(db.local_finish_partial_scan(2).unwrap(), 2);
|
||||
assert_eq!(db.search_local("b.txt", 10).unwrap().len(), 1);
|
||||
assert_eq!(db.local_indexed_at().unwrap(), Some(2));
|
||||
}
|
||||
|
||||
/// Deleted entries pile up in an FTS5 index until it is compacted. The daily
|
||||
/// compaction must leave both indexes whole and searchable, and must not run
|
||||
/// again the same day (`docs/BUGS.md` B217).
|
||||
#[test]
|
||||
fn compacting_the_search_indexes_keeps_them_whole_and_runs_once_a_day() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&file("f1", "root", "quarterly-report.pdf", 1))
|
||||
.unwrap();
|
||||
for round in 0..20 {
|
||||
let g = db.local_begin_scan().unwrap();
|
||||
let name = format!("churn-{round}.txt");
|
||||
db.local_upsert_batch(
|
||||
g,
|
||||
&[
|
||||
local("/home/u/keep.txt", "keep.txt", false),
|
||||
local(&format!("/home/u/{name}"), &name, false),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
// Far in the past, so no scan compacts on its own.
|
||||
db.local_finish_scan(g, 0).unwrap();
|
||||
}
|
||||
|
||||
let day = 24 * 60 * 60;
|
||||
db.compact_search_indexes_if_due(day).unwrap();
|
||||
{
|
||||
let conn = db.conn.lock();
|
||||
for table in ["local_fts", "nodes_fts"] {
|
||||
conn.execute(
|
||||
&format!("INSERT INTO {table}({table}) VALUES('integrity-check')"),
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
assert_eq!(db.search_local("keep", 10).unwrap().len(), 1);
|
||||
assert_eq!(db.search_local("churn-19", 10).unwrap().len(), 1);
|
||||
assert!(db.search_local("churn-3.txt", 10).unwrap().is_empty());
|
||||
assert_eq!(db.search("quarterly", 10).unwrap().len(), 1);
|
||||
let stamp = |db: &Db| db.state_str("fts_compacted_at").unwrap();
|
||||
assert_eq!(stamp(&db).as_deref(), Some(day.to_string().as_str()));
|
||||
|
||||
db.compact_search_indexes_if_due(day + 60).unwrap();
|
||||
assert_eq!(stamp(&db).as_deref(), Some(day.to_string().as_str()));
|
||||
}
|
||||
|
||||
/// Queries below the trigram minimum still match, via the `LIKE` fallback.
|
||||
#[test]
|
||||
fn local_search_short_query_like_fallback() {
|
||||
@@ -1689,9 +1865,16 @@ fn writing_nodes_does_not_scale_with_the_size_of_the_index() {
|
||||
.collect();
|
||||
db.upsert_nodes(&batch).unwrap();
|
||||
|
||||
let t = Instant::now();
|
||||
db.upsert_nodes(&batch).unwrap();
|
||||
t.elapsed()
|
||||
// The fastest of a few runs: the rest of the suite runs alongside, and
|
||||
// one run that lost the CPU to it read as the regression.
|
||||
(0..5)
|
||||
.map(|_| {
|
||||
let t = Instant::now();
|
||||
db.upsert_nodes(&batch).unwrap();
|
||||
t.elapsed()
|
||||
})
|
||||
.min()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
let small = cost_against_index_of(200);
|
||||
@@ -1914,7 +2097,11 @@ fn migration_v17_adds_share_access_to_a_v16_fixture() {
|
||||
{
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"DROP TRIGGER mount_sync_folder_insert;
|
||||
"DROP TRIGGER IF EXISTS mount_sync_folder_kind_update;
|
||||
DROP TRIGGER mount_sync_folder_insert;
|
||||
DROP TRIGGER sync_folder_attached_insert;
|
||||
DROP TRIGGER sync_folder_attached_update;
|
||||
ALTER TABLE sync_folder DROP COLUMN kind;
|
||||
DROP TABLE mount;
|
||||
DROP TABLE share_access;
|
||||
DROP TABLE album_photos;
|
||||
@@ -2191,6 +2378,100 @@ fn refuses_a_schema_newer_than_the_running_build() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
/// Queueing a write syncs it to disk, and leaves the writer at NORMAL after.
|
||||
///
|
||||
/// The op row is what makes a staged blob reachable once `close()` has
|
||||
/// reported success, so its commit must survive a power loss. Everything else
|
||||
/// the writer commits is metadata the next listing re-fetches, and pays only
|
||||
/// NORMAL's cost.
|
||||
#[test]
|
||||
fn queueing_a_write_commits_at_full_sync_and_restores_normal() {
|
||||
let path = std::env::temp_dir().join(format!("pdfs-db-sync-{}.db", std::process::id()));
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let db = Db::open(&path).unwrap();
|
||||
let level = |db: &Db| -> i64 {
|
||||
db.conn
|
||||
.lock()
|
||||
.query_row("PRAGMA synchronous", [], |r| r.get(0))
|
||||
.unwrap()
|
||||
};
|
||||
// 1 = NORMAL, 2 = FULL.
|
||||
assert_eq!(level(&db), 1);
|
||||
{
|
||||
let durable = db.durable().unwrap();
|
||||
let during: i64 = durable
|
||||
.query_row("PRAGMA synchronous", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(during, 2);
|
||||
}
|
||||
assert_eq!(level(&db), 1);
|
||||
|
||||
db.enqueue_op(&PendingOp {
|
||||
id: 0,
|
||||
kind: OP_TRASH.to_string(),
|
||||
uid: uid("a").to_string(),
|
||||
parent_uid: None,
|
||||
name: Some("a".to_string()),
|
||||
blob_path: None,
|
||||
meta_json: None,
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(level(&db), 1);
|
||||
assert_eq!(db.pending_ops().unwrap().len(), 1);
|
||||
drop(db);
|
||||
for suffix in ["", "-wal", "-shm"] {
|
||||
let _ = std::fs::remove_file(format!("{}{suffix}", path.display()));
|
||||
}
|
||||
}
|
||||
|
||||
/// A gap-filled blob is recorded as whole only while the op still holds it.
|
||||
///
|
||||
/// A write that closes during the drain points the row at a newer blob with
|
||||
/// its own sidecar. Overwriting that sidecar with the older blob's would
|
||||
/// describe bytes the row no longer holds.
|
||||
#[test]
|
||||
fn marking_a_blob_whole_leaves_a_newer_blob_alone() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let op = PendingOp {
|
||||
id: 0,
|
||||
kind: OP_REVISION.to_string(),
|
||||
uid: uid("a").to_string(),
|
||||
parent_uid: None,
|
||||
name: None,
|
||||
blob_path: Some("/staging/first".to_string()),
|
||||
meta_json: Some(r#"{"complete":false}"#.to_string()),
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
};
|
||||
db.enqueue_op(&op).unwrap();
|
||||
let id = db.pending_ops().unwrap()[0].id;
|
||||
|
||||
assert!(
|
||||
db.update_op_meta_for_blob(id, "/staging/first", r#"{"complete":true}"#)
|
||||
.unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
db.pending_ops().unwrap()[0].meta_json.as_deref(),
|
||||
Some(r#"{"complete":true}"#)
|
||||
);
|
||||
|
||||
assert!(
|
||||
!db.update_op_meta_for_blob(id, "/staging/older", r#"{"stale":true}"#)
|
||||
.unwrap(),
|
||||
"the row holds another blob"
|
||||
);
|
||||
assert_eq!(
|
||||
db.pending_ops().unwrap()[0].meta_json.as_deref(),
|
||||
Some(r#"{"complete":true}"#)
|
||||
);
|
||||
}
|
||||
|
||||
/// A queued write whose baseline is restamped must keep everything else.
|
||||
///
|
||||
/// The restamp happens after our *own* upload seals a new revision under a
|
||||
@@ -2463,15 +2744,34 @@ fn atomic_trash_replacement_returns_owned_blobs_after_commit() {
|
||||
#[test]
|
||||
fn a_queued_trash_holds_its_name_until_it_lands() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&file("held-name", "root", "report.txt", 1))
|
||||
.unwrap();
|
||||
let node = uid("held-name").to_string();
|
||||
assert!(!db.has_pending_trash_named("report.txt").unwrap());
|
||||
let root = uid("root").to_string();
|
||||
assert!(
|
||||
!db.has_pending_trash_named("report.txt", Some(&root))
|
||||
.unwrap()
|
||||
);
|
||||
|
||||
let (trash_id, _) = db.replace_ops_with_trash(&node, "report.txt", 1).unwrap();
|
||||
assert!(db.has_pending_trash_named("report.txt").unwrap());
|
||||
assert!(!db.has_pending_trash_named("other.txt").unwrap());
|
||||
assert!(
|
||||
db.has_pending_trash_named("report.txt", Some(&root))
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
!db.has_pending_trash_named("other.txt", Some(&root))
|
||||
.unwrap()
|
||||
);
|
||||
assert!(
|
||||
!db.has_pending_trash_named("report.txt", Some(&uid("other").to_string()))
|
||||
.unwrap()
|
||||
);
|
||||
|
||||
db.delete_op(trash_id).unwrap();
|
||||
assert!(!db.has_pending_trash_named("report.txt").unwrap());
|
||||
assert!(
|
||||
!db.has_pending_trash_named("report.txt", Some(&root))
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
/// A create that found its name held by a queued trash backed off. The trash
|
||||
@@ -3920,15 +4220,15 @@ fn what_a_create_sent_outlives_a_newer_blob_its_trash_and_a_restart() {
|
||||
.unwrap();
|
||||
assert!(db.create_sent(id).unwrap().is_empty());
|
||||
assert!(db.create_targets(id).unwrap().is_empty());
|
||||
db.note_create_target(id, &parent, "c").unwrap();
|
||||
db.note_create_target(id, &parent, "c").unwrap();
|
||||
db.note_create_target(id, &parent, "c", 10).unwrap();
|
||||
db.note_create_target(id, &parent, "c", 20).unwrap();
|
||||
db.note_create_sent(id, "aaaa").unwrap();
|
||||
db.note_create_sent(id, "aaaa").unwrap();
|
||||
db.record_op_failure(id, "no answer", 0).unwrap();
|
||||
db.attach_blob_to_create("local~sent", "/staging/second", "{}")
|
||||
.unwrap()
|
||||
.expect("the create is still queued");
|
||||
db.note_create_target(id, &parent, "d").unwrap();
|
||||
db.note_create_target(id, &parent, "d", 30).unwrap();
|
||||
db.note_create_sent(id, "bbbb").unwrap();
|
||||
db.record_op_failure(id, "no answer", 0).unwrap();
|
||||
db.delete_ops_for_uid("local~sent").unwrap();
|
||||
@@ -3951,6 +4251,73 @@ fn what_a_create_sent_outlives_a_newer_blob_its_trash_and_a_restart() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
/// A queued create knows when it first went out, which is where the window
|
||||
/// for adopting a node its lost answer left starts. Records written before
|
||||
/// the time was kept still list their targets, and one that does not parse is
|
||||
/// not replaced by the next target.
|
||||
#[test]
|
||||
fn a_create_remembers_when_it_was_first_sent() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let parent = uid("parent").to_string();
|
||||
let (id, _) = db
|
||||
.enqueue_op(&PendingOp {
|
||||
id: 0,
|
||||
kind: OP_CREATE.to_string(),
|
||||
uid: "local~first".to_string(),
|
||||
parent_uid: Some(parent.clone()),
|
||||
name: Some("a.txt".to_string()),
|
||||
blob_path: None,
|
||||
meta_json: None,
|
||||
created_at: 1,
|
||||
attempts: 0,
|
||||
last_error: None,
|
||||
next_attempt_at: 0,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(db.create_first_sent(id).unwrap(), None, "never sent");
|
||||
db.note_create_target(id, &parent, "a.txt", 500).unwrap();
|
||||
db.note_create_target(id, &parent, "a.txt", 900).unwrap();
|
||||
db.note_create_target(id, &parent, "b.txt", 700).unwrap();
|
||||
assert_eq!(db.create_first_sent(id).unwrap(), Some(500));
|
||||
|
||||
let set = |json: &str| {
|
||||
db.conn
|
||||
.lock()
|
||||
.execute(
|
||||
"UPDATE pending_op SET sent_to = ?2 WHERE id = ?1",
|
||||
rusqlite::params![id, json],
|
||||
)
|
||||
.unwrap();
|
||||
};
|
||||
set(&format!(r#"[["{parent}","old.txt"]]"#));
|
||||
assert_eq!(
|
||||
db.create_targets(id).unwrap(),
|
||||
vec![(parent.clone(), "old.txt".to_string())]
|
||||
);
|
||||
assert_eq!(db.create_first_sent(id).unwrap(), None);
|
||||
db.note_create_target(id, &parent, "a.txt", 900).unwrap();
|
||||
assert_eq!(
|
||||
db.create_targets(id).unwrap().len(),
|
||||
2,
|
||||
"the old target stays"
|
||||
);
|
||||
assert_eq!(db.create_first_sent(id).unwrap(), Some(900));
|
||||
|
||||
set("not json");
|
||||
db.note_create_target(id, &parent, "c.txt", 1000).unwrap();
|
||||
let raw: String = db
|
||||
.conn
|
||||
.lock()
|
||||
.query_row("SELECT sent_to FROM pending_op WHERE id = ?1", [id], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
raw, "not json",
|
||||
"an unreadable record is kept, not replaced"
|
||||
);
|
||||
}
|
||||
|
||||
/// A write after a create failed clears its attempts, so the next try goes
|
||||
/// out at once. The create was still sent: deleting the file then left no
|
||||
/// trash, and the file a lost answer made stayed on Drive under the name it
|
||||
@@ -3974,7 +4341,7 @@ fn a_create_rewritten_after_it_failed_still_leaves_a_trash_when_deleted() {
|
||||
next_attempt_at: 0,
|
||||
})
|
||||
.unwrap();
|
||||
db.note_create_target(id, &parent, "a.txt").unwrap();
|
||||
db.note_create_target(id, &parent, "a.txt", 10).unwrap();
|
||||
db.note_create_sent(id, "aaaa").unwrap();
|
||||
db.record_op_failure(id, "no answer", 0).unwrap();
|
||||
db.attach_blob_to_create("local~sent", "/staging/second", "{}")
|
||||
@@ -5021,6 +5388,7 @@ fn baseline_row(rel: &str) -> StoredSyncEntry {
|
||||
local_size: 7,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("7".into()),
|
||||
remote_revision_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5115,6 +5483,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
remote_revision_id: None,
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
@@ -5132,6 +5501,7 @@ fn mirror_uid_lookup_covers_the_root_and_synced_descendants_only() {
|
||||
local_size: 2,
|
||||
remote_rev: Some("3".into()),
|
||||
remote_hash: Some("2".into()),
|
||||
remote_revision_id: None,
|
||||
local_mtime_ns: None,
|
||||
},
|
||||
)
|
||||
@@ -5747,7 +6117,7 @@ fn cache_eviction_candidates_are_lru_ordered_and_limited() {
|
||||
}
|
||||
db.cache_touch("other", "block", 100, 0).unwrap();
|
||||
|
||||
let batch = db.cache_eviction_candidates("blob", 3).unwrap();
|
||||
let batch = db.cache_eviction_candidates("blob", 3, 0).unwrap();
|
||||
assert_eq!(batch.len(), 3, "honours the limit");
|
||||
let keys: Vec<&str> = batch.iter().map(|(k, _)| k.as_str()).collect();
|
||||
assert_eq!(keys, ["k9", "k8", "k7"], "least-recently-accessed first");
|
||||
@@ -6309,6 +6679,71 @@ fn migration_v27_adds_the_access_deferral_column_to_a_v26_queue() {
|
||||
let _ = std::fs::remove_file(format!("{}.lock", path.display()));
|
||||
}
|
||||
|
||||
/// V44 adds the remote revision a baseline row was synced at. A row carried
|
||||
/// over from V43 has none, and the mirror then compares the modification time
|
||||
/// and size as before, so upgrading changes nothing about what is in step
|
||||
/// (`docs/BUGS.md` B249).
|
||||
#[test]
|
||||
fn migration_v44_adds_the_synced_revision_without_disturbing_a_v43_baseline() {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-db-v43-fixture-{}-{}.db",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let row = StoredSyncEntry {
|
||||
rel_path: "notes.txt".into(),
|
||||
remote_uid: Some("vol~notes".into()),
|
||||
local_mtime: 1_700_000_000,
|
||||
local_mtime_ns: Some(1_700_000_000_123_456_789),
|
||||
local_size: 42,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("42".into()),
|
||||
remote_revision_id: Some("rev-1".into()),
|
||||
};
|
||||
{
|
||||
let db = Db::open(&path).unwrap();
|
||||
let folder = db
|
||||
.sync_folder_add("/home/me/Mirror", "vol~root", "share")
|
||||
.unwrap();
|
||||
db.sync_entry_upsert(folder, &row).unwrap();
|
||||
}
|
||||
{
|
||||
// Put the file back in the state a released V43 database was in.
|
||||
let conn = rusqlite::Connection::open(&path).unwrap();
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE sync_entry DROP COLUMN remote_revision_id;
|
||||
UPDATE sync_state SET value = '43' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let db = Db::open(&path).unwrap();
|
||||
let entries = db.sync_entries(1).unwrap();
|
||||
let entry = entries.get("notes.txt").expect("the baseline row survives");
|
||||
assert_eq!(
|
||||
entry,
|
||||
&StoredSyncEntry {
|
||||
remote_revision_id: None,
|
||||
..row.clone()
|
||||
}
|
||||
);
|
||||
|
||||
// And the column is live, including through a move of the row.
|
||||
db.sync_entry_upsert(1, &row).unwrap();
|
||||
db.sync_entries_move(1, "notes.txt", 1, "moved.txt")
|
||||
.unwrap();
|
||||
let entries = db.sync_entries(1).unwrap();
|
||||
assert_eq!(
|
||||
entries["moved.txt"].remote_revision_id.as_deref(),
|
||||
Some("rev-1")
|
||||
);
|
||||
drop(db);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
/// A mirror baseline carried over from V27 keeps its rows, and each one reads
|
||||
/// back with no nanosecond time. That `None` is the whole point: it makes the
|
||||
/// comparison fall back to whole seconds for rows written before the column
|
||||
@@ -6339,6 +6774,7 @@ fn migration_v28_adds_sub_second_local_times_without_disturbing_a_v27_baseline()
|
||||
local_size: 42,
|
||||
remote_rev: Some("1700000000".into()),
|
||||
remote_hash: Some("42".into()),
|
||||
remote_revision_id: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -6512,6 +6948,54 @@ fn two_workers_never_share_a_node() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trashing_a_folder_waits_for_moves_out_of_its_subtree() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_nodes(&[
|
||||
folder("root", None, "My Files"),
|
||||
folder("source", Some("root"), "source"),
|
||||
folder("nested", Some("source"), "nested"),
|
||||
folder("destination", Some("root"), "destination"),
|
||||
file("photo", "destination", "photo.jpg", 1),
|
||||
])
|
||||
.unwrap();
|
||||
let source = uid("source").to_string();
|
||||
let destination = uid("destination").to_string();
|
||||
let (move_id, _) = db
|
||||
.enqueue_op(&PendingOp {
|
||||
kind: OP_RENAME.to_string(),
|
||||
uid: uid("photo").to_string(),
|
||||
parent_uid: Some(destination.clone()),
|
||||
name: Some("photo.jpg".into()),
|
||||
blob_path: None,
|
||||
meta_json: Some(
|
||||
serde_json::to_string(&RenameMeta {
|
||||
original_parent_uid: uid("nested").to_string(),
|
||||
original_name: Some("photo.jpg".into()),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
next_attempt_at: 20,
|
||||
..bulk_op(0, &destination)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(db.has_pending_moves_out_of(&source).unwrap());
|
||||
assert!(!db.has_pending_moves_out_of(&destination).unwrap());
|
||||
assert!(
|
||||
!db.has_pending_moves_out_of(&uid("root").to_string())
|
||||
.unwrap()
|
||||
);
|
||||
let (trash_id, _) = db.replace_ops_with_trash(&source, "source", 2).unwrap();
|
||||
|
||||
assert!(db.claim_next_due_op(10).unwrap().is_none());
|
||||
assert_eq!(db.earliest_due_at().unwrap(), Some(20));
|
||||
assert_eq!(db.claim_next_due_op(20).unwrap().unwrap().id, move_id);
|
||||
assert!(db.claim_next_due_op(20).unwrap().is_none());
|
||||
assert_eq!(db.earliest_due_at().unwrap(), None);
|
||||
db.delete_op(move_id).unwrap();
|
||||
assert_eq!(db.claim_next_due_op(20).unwrap().unwrap().id, trash_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claims_do_not_survive_the_process_that_took_them() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -6547,6 +7031,36 @@ fn an_idle_worker_does_not_wait_on_an_op_another_worker_has() {
|
||||
);
|
||||
}
|
||||
|
||||
/// B200: a rename queued behind a long upload of the same node is due, but no
|
||||
/// worker may claim it until the upload retires. Counted as due, it had every
|
||||
/// idle worker wake at once, find nothing, and go round again.
|
||||
#[test]
|
||||
fn earliest_due_at_skips_an_op_held_behind_a_claimed_one() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let root = uid("root").to_string();
|
||||
let mut revision = bulk_op(0, &root);
|
||||
revision.parent_uid = None;
|
||||
db.enqueue_op(&revision).unwrap();
|
||||
db.enqueue_op(&PendingOp {
|
||||
kind: OP_RENAME.to_string(),
|
||||
parent_uid: Some(root.clone()),
|
||||
blob_path: None,
|
||||
meta_json: None,
|
||||
..bulk_op(0, &root)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
db.claim_next_due_op(10)
|
||||
.unwrap()
|
||||
.expect("the revision is due");
|
||||
assert!(db.claim_next_due_op(10).unwrap().is_none());
|
||||
assert_eq!(
|
||||
db.earliest_due_at().unwrap(),
|
||||
None,
|
||||
"a row no worker can claim is not work an idle worker is waiting for"
|
||||
);
|
||||
}
|
||||
|
||||
/// Trashing a photo from the gallery must not leave it behind on an album page:
|
||||
/// the row goes, its album membership goes with it, and nothing else moves.
|
||||
#[test]
|
||||
@@ -6655,6 +7169,29 @@ fn a_raw_and_its_jpeg_are_one_entry_in_the_grid() {
|
||||
["jpeg", "raw"]
|
||||
);
|
||||
assert_eq!(db.photos_group("alone").unwrap().len(), 1);
|
||||
|
||||
let hidden = db
|
||||
.photo_group_members_for(&["raw".to_owned()].into_iter().collect())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
hidden,
|
||||
["raw".to_owned(), "jpeg".to_owned()].into_iter().collect()
|
||||
);
|
||||
let excluded = serde_json::to_string(&hidden).unwrap();
|
||||
assert_eq!(db.photos_counts_visible(&excluded).unwrap(), (1, 0, 0));
|
||||
let months = db.photos_months_visible(None, &excluded).unwrap();
|
||||
assert_eq!(months.len(), 1);
|
||||
assert_eq!(months[0].2, 1);
|
||||
let public = db
|
||||
.photos_page_visible(0, 1, None, None, false, false, &excluded)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
public
|
||||
.iter()
|
||||
.map(|photo| photo.uid.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["alone"]
|
||||
);
|
||||
}
|
||||
|
||||
/// Pixel names the two files of one raw capture after the role each plays, so
|
||||
@@ -7179,6 +7716,56 @@ fn a_refused_change_names_its_node_by_local_id() {
|
||||
);
|
||||
}
|
||||
|
||||
/// The name checks the drain makes before each op look the name up in an
|
||||
/// index. Scanning the queue instead made a large offline batch quadratic.
|
||||
#[test]
|
||||
fn queue_name_checks_use_an_index() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let conn = db.conn.lock();
|
||||
let plan = |sql: &str| -> String {
|
||||
let mut stmt = conn.prepare(&format!("EXPLAIN QUERY PLAN {sql}")).unwrap();
|
||||
stmt.query_map([], |r| r.get::<_, String>(3))
|
||||
.unwrap()
|
||||
.map(|row| row.unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ")
|
||||
};
|
||||
for sql in [
|
||||
"SELECT 1 FROM pending_op WHERE kind = 'trash' AND name = 'a'",
|
||||
"SELECT 1 FROM pending_op WHERE name = 'a' AND kind IN ('create', 'mkdir', 'rename')",
|
||||
"SELECT 1 FROM pending_op WHERE kind = 'rename'",
|
||||
] {
|
||||
let plan = plan(sql);
|
||||
assert!(plan.contains("pending_op_kind_name"), "{sql}: {plan}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A reconnect moves up the ops that failed for want of a link. One Drive
|
||||
/// refused, for quota or access, keeps its backoff: the link coming back
|
||||
/// changes nothing about it. "Retry all" still moves both.
|
||||
#[test]
|
||||
fn a_reconnect_leaves_refused_ops_to_their_backoff() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let offline = queued_op(&db, "vol~a", 2, 90_000);
|
||||
let refused = queued_op(&db, "vol~b", 2, 90_000);
|
||||
db.set_op_issue(refused, Some("quota")).unwrap();
|
||||
|
||||
assert_eq!(db.retry_unrefused_ops_now(5_000).unwrap(), 1);
|
||||
let due = |id: i64| {
|
||||
db.pending_ops()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find(|op| op.id == id)
|
||||
.unwrap()
|
||||
.next_attempt_at
|
||||
};
|
||||
assert_eq!(due(offline), 5_000);
|
||||
assert_eq!(due(refused), 90_000);
|
||||
|
||||
assert_eq!(db.retry_failed_ops_now(5_000).unwrap(), 1);
|
||||
assert_eq!(due(refused), 5_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_all_touches_only_ops_that_have_failed() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
@@ -7219,3 +7806,153 @@ fn a_queued_op_stops_existing_once_the_node_is_deleted() {
|
||||
assert_eq!(blobs, vec!["/staging/journal"]);
|
||||
assert!(!db.op_exists(id).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attachments_have_their_own_location_and_cannot_evict_local_files() {
|
||||
use crate::mounts::MountKind;
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let id = db
|
||||
.sync_folder_attach("/work", "vol~work", "myfiles-share")
|
||||
.unwrap();
|
||||
let folder = db.sync_folder_get(id).unwrap().unwrap();
|
||||
assert_eq!(folder.kind, "attached");
|
||||
assert_eq!(folder.mode, "mirror");
|
||||
assert_eq!(db.sync_folder_list().unwrap()[0].kind, "attached");
|
||||
let locations = db.mount_list().unwrap();
|
||||
assert!(
|
||||
matches!(locations[0].kind, MountKind::Attached { sync_folder_id } if sync_folder_id == id)
|
||||
);
|
||||
assert_eq!(locations[0].local_path, "/work");
|
||||
assert_eq!(locations[0].root_share_id, "myfiles-share");
|
||||
assert!(db.sync_folder_set_mode(id, "ondemand").is_err());
|
||||
assert!(
|
||||
db.sync_folder_set_pending_mode(id, Some("ondemand"))
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(db.sync_folder_get(id).unwrap().unwrap().mode, "mirror");
|
||||
db.sync_folder_remove(id).unwrap();
|
||||
assert!(db.mount_list().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_v45_keeps_existing_device_folders_and_location_ids() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let id = db
|
||||
.sync_folder_add("/backup", "vol~backup", "share")
|
||||
.unwrap();
|
||||
db.sync_folder_set_mode(id, "ondemand").unwrap();
|
||||
db.sync_folder_set_pending_mode(id, Some("mirror")).unwrap();
|
||||
let location_id = db.mount_list().unwrap()[0].id;
|
||||
db.conn
|
||||
.lock()
|
||||
.execute_batch(
|
||||
"DROP TRIGGER sync_folder_attached_insert;
|
||||
DROP TRIGGER sync_folder_attached_update;
|
||||
DROP TRIGGER IF EXISTS mount_sync_folder_kind_update;
|
||||
DROP TRIGGER mount_sync_folder_insert;
|
||||
ALTER TABLE sync_folder DROP COLUMN kind;
|
||||
CREATE TRIGGER mount_sync_folder_insert AFTER INSERT ON sync_folder BEGIN
|
||||
INSERT INTO mount (kind, sync_folder_id) VALUES ('device', NEW.id); END;
|
||||
UPDATE sync_state SET value = '44' WHERE key = 'schema_version';",
|
||||
)
|
||||
.unwrap();
|
||||
db.migrate().unwrap();
|
||||
let folder = db.sync_folder_get(id).unwrap().unwrap();
|
||||
assert_eq!(folder.kind, "device");
|
||||
assert_eq!(folder.mode, "ondemand");
|
||||
assert_eq!(folder.pending_mode.as_deref(), Some("mirror"));
|
||||
assert_eq!(db.mount_list().unwrap()[0].id, location_id);
|
||||
let attached = db
|
||||
.sync_folder_attach("/work", "vol~work", "myfiles")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.sync_folder_get(attached).unwrap().unwrap().kind,
|
||||
"attached"
|
||||
);
|
||||
db.migrate().unwrap();
|
||||
assert_eq!(db.mount_list().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn photo_feed_failure_keeps_the_cursor_and_a_stale_listing_cannot_restore_deleted_rows() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let rows = vec![TimelineRow::new("gone", 200), TimelineRow::new("keep", 100)];
|
||||
db.photos_replace(&rows).unwrap();
|
||||
db.albums_replace(&[StoredAlbum {
|
||||
uid: "album".into(),
|
||||
name: "Test album".into(),
|
||||
photo_count: 2,
|
||||
cover_uid: Some("gone".into()),
|
||||
last_activity: None,
|
||||
shared: false,
|
||||
}])
|
||||
.unwrap();
|
||||
db.album_photos_replace(
|
||||
"album",
|
||||
&[
|
||||
("gone".into(), 200, None, None),
|
||||
("keep".into(), 100, None, None),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
db.set_state_str("photos-cursor-test", "old").unwrap();
|
||||
let stale = db.photo_projection_generation().unwrap();
|
||||
db.conn.lock().execute_batch(
|
||||
"CREATE TEMP TRIGGER reject_cursor BEFORE INSERT ON sync_state WHEN NEW.key='photos-cursor-test'
|
||||
BEGIN SELECT RAISE(ABORT, 'simulated cursor failure'); END;",
|
||||
).unwrap();
|
||||
|
||||
let gone = ["gone".into()];
|
||||
assert!(
|
||||
db.photos_apply_events("photos-cursor-test", "next", &gone, &[], false, false)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
db.state_str("photos-cursor-test").unwrap().as_deref(),
|
||||
Some("old")
|
||||
);
|
||||
assert_eq!(db.photos_count().unwrap(), 2);
|
||||
assert!(db.photos_contains("gone", Some("album")).unwrap());
|
||||
db.conn
|
||||
.lock()
|
||||
.execute("DROP TRIGGER reject_cursor", [])
|
||||
.unwrap();
|
||||
|
||||
db.photos_apply_events("photos-cursor-test", "next", &gone, &[], false, false)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
db.state_str("photos-cursor-test").unwrap().as_deref(),
|
||||
Some("next")
|
||||
);
|
||||
assert!(!db.photos_contains("gone", None).unwrap());
|
||||
assert_eq!(
|
||||
db.albums_list().unwrap()[0].cover_uid.as_deref(),
|
||||
Some("keep")
|
||||
);
|
||||
assert!(!db.photos_replace_at(&rows, stale, true, 1).unwrap());
|
||||
assert!(
|
||||
!db.album_photos_replace_at("album", &[("gone".into(), 200, None, None)], stale, 1)
|
||||
.unwrap()
|
||||
);
|
||||
assert!(!db.photos_contains("gone", None).unwrap());
|
||||
|
||||
db.photos_apply_events("photos-cursor-test", "restore", &[], &gone, false, false)
|
||||
.unwrap();
|
||||
let current = db.photo_projection_generation().unwrap();
|
||||
assert!(db.photos_replace_at(&rows, current, true, 2).unwrap());
|
||||
assert!(db.photos_contains("gone", None).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warm_photo_months_exclude_hidden_groups_with_thirteen_thousand_rows() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let mut rows: Vec<_> = (0..13_000)
|
||||
.map(|i| TimelineRow::new(format!("volume~{i}"), 1_700_000_000 + i))
|
||||
.collect();
|
||||
rows[1].main_uid = Some(rows[0].uid.clone());
|
||||
db.photos_replace(&rows).unwrap();
|
||||
let started = std::time::Instant::now();
|
||||
let months = db.photos_months_visible(None, r#"["volume~1"]"#).unwrap();
|
||||
assert_eq!(months.iter().map(|month| month.2).sum::<usize>(), 12_998);
|
||||
assert!(started.elapsed() < std::time::Duration::from_secs(1));
|
||||
}
|
||||
+144
-24
@@ -154,19 +154,20 @@ impl CoreError {
|
||||
/// through the boxes the drain deals in, where the concrete type survives
|
||||
/// but the static one does not.
|
||||
///
|
||||
/// A failure that is not an API response at all — a transport error, a
|
||||
/// timeout, a TLS failure — is what being offline looks like from here, so
|
||||
/// it classifies as [`ErrorKind::Offline`] rather than
|
||||
/// [`ErrorKind::Remote`]: the request never reached Proton, and telling the
|
||||
/// user their connection is down beats telling them the server refused.
|
||||
/// A transport error, a timeout or a TLS failure is what being offline
|
||||
/// looks like from here, so it classifies as [`ErrorKind::Offline`] rather
|
||||
/// than [`ErrorKind::Remote`]: the request never reached Proton, and
|
||||
/// telling the user their connection is down beats telling them the
|
||||
/// server refused. The SDK's other failures say what went wrong instead
|
||||
/// (see [`proton_kind`]).
|
||||
///
|
||||
/// Callers also hand it this crate's [`Error`] from the cache and the
|
||||
/// database, which is not a transport error at all. Taking those for
|
||||
/// "offline" told the user their connection was down when a pin failed on
|
||||
/// the database (`docs/BUGS.md` B266), so the source chain is searched for
|
||||
/// either type before falling back.
|
||||
pub fn from_api(e: &(dyn std::error::Error + 'static), what: &str) -> Self {
|
||||
let kind = match e.downcast_ref::<proton_sdk::ProtonError>() {
|
||||
Some(proton_sdk::ProtonError::Api(api)) => api_kind(api.code),
|
||||
// A `ProtonError` that is not an `Api` variant never reached the
|
||||
// API; so does anything that is not a `ProtonError` at all.
|
||||
_ => ErrorKind::Offline,
|
||||
};
|
||||
Self::new(kind, format!("{what}: {e}"))
|
||||
Self::new(api_kind_of(e), format!("{what}: {e}"))
|
||||
}
|
||||
|
||||
/// Prefix the message while keeping the classification.
|
||||
@@ -208,18 +209,57 @@ impl From<serde_json::Error> for CoreError {
|
||||
|
||||
impl From<Error> for CoreError {
|
||||
fn from(e: Error) -> Self {
|
||||
let kind = match &e {
|
||||
Error::NotLoggedIn
|
||||
| Error::ReloginRequired
|
||||
| Error::Locked
|
||||
| Error::WrongMailboxPassword
|
||||
| Error::HumanVerificationRequired(_) => ErrorKind::Denied,
|
||||
Error::Proton(proton_sdk::ProtonError::Api(api)) => api_kind(api.code),
|
||||
// A `ProtonError` that never reached the API is a transport failure.
|
||||
Error::Proton(_) => ErrorKind::Offline,
|
||||
_ => ErrorKind::Internal,
|
||||
};
|
||||
Self::new(kind, e.to_string())
|
||||
Self::new(core_kind(&e), e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// What this crate's [`Error`] means to a front-end.
|
||||
fn core_kind(e: &Error) -> ErrorKind {
|
||||
match e {
|
||||
Error::NotLoggedIn
|
||||
| Error::ReloginRequired
|
||||
| Error::Locked
|
||||
| Error::WrongMailboxPassword
|
||||
| Error::HumanVerificationRequired(_) => ErrorKind::Denied,
|
||||
Error::Proton(proton) => proton_kind(proton),
|
||||
_ => ErrorKind::Internal,
|
||||
}
|
||||
}
|
||||
|
||||
/// The kind for [`CoreError::from_api`]: the first SDK or crate error in `e`'s
|
||||
/// source chain decides it.
|
||||
fn api_kind_of(e: &(dyn std::error::Error + 'static)) -> ErrorKind {
|
||||
let mut cause = Some(e);
|
||||
while let Some(err) = cause {
|
||||
if let Some(proton) = err.downcast_ref::<proton_sdk::ProtonError>() {
|
||||
return proton_kind(proton);
|
||||
}
|
||||
if let Some(core) = err.downcast_ref::<Error>() {
|
||||
return core_kind(core);
|
||||
}
|
||||
cause = err.source();
|
||||
}
|
||||
// Not the SDK's at all: a timeout around the call, or the socket.
|
||||
ErrorKind::Offline
|
||||
}
|
||||
|
||||
/// What a failed SDK call means to a front-end.
|
||||
///
|
||||
/// Only a transport failure is the connection. The rest happened on this
|
||||
/// machine, and calling them "offline" sent the user to check a network that
|
||||
/// worked (issue #38: a photo upload on an account without a photo library
|
||||
/// read "You're offline").
|
||||
fn proton_kind(e: &proton_sdk::ProtonError) -> ErrorKind {
|
||||
use proton_sdk::ProtonError as P;
|
||||
match e {
|
||||
P::Api(api) => api_kind(api.code),
|
||||
P::Transport(_) => ErrorKind::Offline,
|
||||
// Proton answered, but not with anything this client can read.
|
||||
P::Serialization(_) => ErrorKind::Remote,
|
||||
// The SDK refused the call before sending it, such as an upload to a
|
||||
// photo library the account does not have.
|
||||
P::InvalidOperation(_) => ErrorKind::Invalid,
|
||||
P::Crypto(_) | P::KeysLocked => ErrorKind::Internal,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -278,9 +318,44 @@ fn api_kind(code: proton_sdk::api::ResponseCode) -> ErrorKind {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `e` says the session itself is gone: its refresh token was refused,
|
||||
/// or the account was deleted or disabled.
|
||||
///
|
||||
/// Nothing but a new login helps then, and no retry will ever succeed, so a
|
||||
/// caller must not take it for a lost connection and wait for it to come back
|
||||
/// (issue #35: a revoked session was shown as "offline" for good).
|
||||
pub fn session_revoked(e: &proton_sdk::ProtonError) -> bool {
|
||||
use proton_sdk::api::ResponseCode as C;
|
||||
match e {
|
||||
proton_sdk::ProtonError::Api(api) => {
|
||||
api.is_invalid_refresh_token()
|
||||
|| matches!(api.code, C::AccountDeleted | C::AccountDisabled)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `e` is Proton's temporary block on a sign-in (code 2028).
|
||||
///
|
||||
/// Its message says the app version is no longer supported, which is not what
|
||||
/// happened: the release is current and only waiting helps (issues #28, #39).
|
||||
/// A front-end must say that instead of passing the message through.
|
||||
pub fn sign_in_blocked(e: &proton_sdk::ProtonError) -> bool {
|
||||
matches!(e, proton_sdk::ProtonError::Api(api) if api.code == proton_sdk::api::ResponseCode::Blocked)
|
||||
}
|
||||
|
||||
/// The result every request-serving method in the daemon returns.
|
||||
pub type CoreResult<T> = std::result::Result<T, CoreError>;
|
||||
|
||||
impl From<proton_auth_rs::Error> for Error {
|
||||
fn from(error: proton_auth_rs::Error) -> Self {
|
||||
match error {
|
||||
proton_auth_rs::Error::Keyring(error) => Self::Keyring(error),
|
||||
other => Self::Other(other.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -309,6 +384,25 @@ mod tests {
|
||||
assert_eq!(api_kind(C::ProtonDriveUnknown), ErrorKind::Remote);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_refused_refresh_or_a_closed_account_revokes_the_session() {
|
||||
assert!(session_revoked(&api(C::InvalidRefreshToken)));
|
||||
assert!(session_revoked(&api(C::AccountDeleted)));
|
||||
assert!(session_revoked(&api(C::AccountDisabled)));
|
||||
// One file the account may not touch says nothing about the session.
|
||||
assert!(!session_revoked(&api(C::Forbidden)));
|
||||
assert!(!session_revoked(&api(C::Timeout)));
|
||||
assert!(!session_revoked(&ProtonError::KeysLocked));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_code_2028_is_a_blocked_sign_in() {
|
||||
assert!(sign_in_blocked(&api(C::Blocked)));
|
||||
assert!(!sign_in_blocked(&api(C::OutdatedApp)));
|
||||
assert!(!sign_in_blocked(&api(C::TooManyRequests)));
|
||||
assert!(!sign_in_blocked(&ProtonError::KeysLocked));
|
||||
}
|
||||
|
||||
/// The regression this whole pass exists to prevent: before it, every SDK
|
||||
/// failure was hardcoded to `Remote`, so a 403 asked the user to retry a
|
||||
/// request that could never succeed.
|
||||
@@ -339,6 +433,32 @@ mod tests {
|
||||
assert_eq!(CoreError::from_api(&io, "list").kind, ErrorKind::Offline);
|
||||
}
|
||||
|
||||
/// Issue #38: the SDK refused a photo upload before sending it, and the
|
||||
/// app said "You're offline" while every other call went through.
|
||||
#[test]
|
||||
fn a_call_the_sdk_refuses_to_send_is_not_offline() {
|
||||
let refused = ProtonError::invalid_operation("account has no photos volume");
|
||||
let e = CoreError::from_api(&refused, "upload photo failed");
|
||||
assert_eq!(e.kind, ErrorKind::Invalid);
|
||||
assert!(e.message.ends_with("account has no photos volume"));
|
||||
let e: CoreError = Error::Proton(ProtonError::KeysLocked).into();
|
||||
assert_ne!(e.kind, ErrorKind::Offline);
|
||||
}
|
||||
|
||||
/// A pin or a search that fails on the database is not the connection, so
|
||||
/// it must not say "You're offline" (`docs/BUGS.md` B266). An SDK error
|
||||
/// wrapped in this crate's type classifies as the SDK error does.
|
||||
#[test]
|
||||
fn a_database_error_handed_to_from_api_is_not_offline() {
|
||||
let db = Error::Other("database is locked".into());
|
||||
assert_eq!(CoreError::from_api(&db, "pin").kind, ErrorKind::Internal);
|
||||
let wrapped = Error::Proton(api(C::DoesNotExist));
|
||||
assert_eq!(
|
||||
CoreError::from_api(&wrapped, "load node").kind,
|
||||
ErrorKind::NotFound
|
||||
);
|
||||
}
|
||||
|
||||
/// `Error` is the plumbing type; converting it must classify as sharply as
|
||||
/// the direct path, or the two disagree about the same failure.
|
||||
#[test]
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
//! kept in kernel memory — not on disk, not in the Secret Service — and vanish
|
||||
//! when the user's last session ends or the machine reboots. Both the daemon and
|
||||
//! `pdfs unlock` reach the same entry because it hangs off the *user* keyring.
|
||||
//! It is also where session tokens wait while the Secret Service refuses them
|
||||
//! ([`UNSAVED_TOKENS`]), so they outlive the process that holds them.
|
||||
//!
|
||||
//! Kernel keys are not paged to swap, are invisible to other users, and are
|
||||
//! wiped by [`clear`] (`KEYCTL_REVOKE`), which makes them a better home than a
|
||||
@@ -20,12 +22,34 @@ use std::io;
|
||||
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
/// Key description. Namespaced so it cannot collide with another program's key.
|
||||
/// One secret in the user keyring, by its description. Descriptions are
|
||||
/// namespaced so they cannot collide with another program's keys.
|
||||
pub struct Key {
|
||||
description: &'static [u8],
|
||||
}
|
||||
|
||||
/// The account's key passphrases (see the module docs).
|
||||
#[cfg(not(test))]
|
||||
const DESCRIPTION: &[u8] = b"pdfs:key-passphrases\0";
|
||||
pub const PASSPHRASES: Key = Key {
|
||||
description: b"pdfs:key-passphrases\0",
|
||||
};
|
||||
/// Tests must never touch a real unlock.
|
||||
#[cfg(test)]
|
||||
const DESCRIPTION: &[u8] = b"pdfs:key-passphrases:test\0";
|
||||
pub const PASSPHRASES: Key = Key {
|
||||
description: b"pdfs:key-passphrases:test\0",
|
||||
};
|
||||
|
||||
/// Session tokens the Secret Service refused to store, until it takes them.
|
||||
/// A refresh token held only in a process's memory dies with the process.
|
||||
#[cfg(not(test))]
|
||||
pub const UNSAVED_TOKENS: Key = Key {
|
||||
description: b"pdfs:unsaved-tokens\0",
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub const UNSAVED_TOKENS: Key = Key {
|
||||
description: b"pdfs:unsaved-tokens:test\0",
|
||||
};
|
||||
|
||||
const KEY_TYPE: &[u8] = b"user\0";
|
||||
|
||||
const KEY_SPEC_USER_KEYRING: i32 = -4;
|
||||
@@ -49,7 +73,7 @@ fn last_error<T>() -> io::Result<T> {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
|
||||
fn find() -> io::Result<Option<libc::c_long>> {
|
||||
fn find(description: &[u8]) -> io::Result<Option<libc::c_long>> {
|
||||
// SAFETY: both strings are NUL-terminated and outlive the call.
|
||||
let id = unsafe {
|
||||
libc::syscall(
|
||||
@@ -57,7 +81,7 @@ fn find() -> io::Result<Option<libc::c_long>> {
|
||||
KEYCTL_SEARCH,
|
||||
KEY_SPEC_USER_KEYRING,
|
||||
KEY_TYPE.as_ptr(),
|
||||
DESCRIPTION.as_ptr(),
|
||||
description.as_ptr(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
@@ -83,100 +107,102 @@ pub fn forbid_dumps() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Store `secret`, replacing any earlier one.
|
||||
pub fn store(secret: &[u8]) -> io::Result<()> {
|
||||
// SAFETY: strings are NUL-terminated; the payload pointer/length pair
|
||||
// describes `secret`, which outlives the call.
|
||||
let id = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_add_key,
|
||||
KEY_TYPE.as_ptr(),
|
||||
DESCRIPTION.as_ptr(),
|
||||
secret.as_ptr(),
|
||||
secret.len(),
|
||||
KEY_SPEC_USER_KEYRING,
|
||||
)
|
||||
};
|
||||
if id < 0 {
|
||||
return last_error();
|
||||
}
|
||||
// SAFETY: plain integer arguments.
|
||||
let rc = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_keyctl,
|
||||
KEYCTL_SETPERM,
|
||||
id,
|
||||
PERM_OWNER_AND_POSSESSOR,
|
||||
)
|
||||
};
|
||||
if rc < 0 {
|
||||
// A key whose permissions are unknown is not one to leave behind.
|
||||
let err = io::Error::last_os_error();
|
||||
let _ = clear();
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the stored secret, or `None` when there is none.
|
||||
pub fn load() -> io::Result<Option<Zeroizing<Vec<u8>>>> {
|
||||
let Some(id) = find()? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Size first, then read; retry if the key was replaced in between.
|
||||
for _ in 0..3 {
|
||||
// SAFETY: a null buffer asks the kernel only for the payload length.
|
||||
let len = unsafe {
|
||||
impl Key {
|
||||
/// Store `secret`, replacing any earlier one.
|
||||
pub fn store(&self, secret: &[u8]) -> io::Result<()> {
|
||||
// SAFETY: strings are NUL-terminated; the payload pointer/length pair
|
||||
// describes `secret`, which outlives the call.
|
||||
let id = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_keyctl,
|
||||
KEYCTL_READ,
|
||||
id,
|
||||
std::ptr::null_mut::<u8>(),
|
||||
0usize,
|
||||
libc::SYS_add_key,
|
||||
KEY_TYPE.as_ptr(),
|
||||
self.description.as_ptr(),
|
||||
secret.as_ptr(),
|
||||
secret.len(),
|
||||
KEY_SPEC_USER_KEYRING,
|
||||
)
|
||||
};
|
||||
if len < 0 {
|
||||
if id < 0 {
|
||||
return last_error();
|
||||
}
|
||||
// SAFETY: plain integer arguments.
|
||||
let rc = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_keyctl,
|
||||
KEYCTL_SETPERM,
|
||||
id,
|
||||
PERM_OWNER_AND_POSSESSOR,
|
||||
)
|
||||
};
|
||||
if rc < 0 {
|
||||
// A key whose permissions are unknown is not one to leave behind.
|
||||
let err = io::Error::last_os_error();
|
||||
let _ = self.clear();
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the stored secret, or `None` when there is none.
|
||||
pub fn load(&self) -> io::Result<Option<Zeroizing<Vec<u8>>>> {
|
||||
let Some(id) = find(self.description)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
// Size first, then read; retry if the key was replaced in between.
|
||||
for _ in 0..3 {
|
||||
// SAFETY: a null buffer asks the kernel only for the payload length.
|
||||
let len = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_keyctl,
|
||||
KEYCTL_READ,
|
||||
id,
|
||||
std::ptr::null_mut::<u8>(),
|
||||
0usize,
|
||||
)
|
||||
};
|
||||
if len < 0 {
|
||||
return match io::Error::last_os_error().raw_os_error() {
|
||||
Some(libc::ENOKEY | libc::EKEYREVOKED | libc::EKEYEXPIRED) => Ok(None),
|
||||
_ => last_error(),
|
||||
};
|
||||
}
|
||||
let mut buf = Zeroizing::new(vec![0u8; len as usize]);
|
||||
// SAFETY: `buf` is `len` writable bytes.
|
||||
let read = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_keyctl,
|
||||
KEYCTL_READ,
|
||||
id,
|
||||
buf.as_mut_ptr(),
|
||||
buf.len(),
|
||||
)
|
||||
};
|
||||
if read < 0 {
|
||||
return last_error();
|
||||
}
|
||||
if read as usize <= buf.len() {
|
||||
buf.truncate(read as usize);
|
||||
return Ok(Some(buf));
|
||||
}
|
||||
}
|
||||
Err(io::Error::other("the key changed while it was being read"))
|
||||
}
|
||||
|
||||
/// Remove the stored secret. Succeeds when there is none.
|
||||
pub fn clear(&self) -> io::Result<()> {
|
||||
let Some(id) = find(self.description)? else {
|
||||
return Ok(());
|
||||
};
|
||||
// SAFETY: plain integer arguments.
|
||||
let rc = unsafe { libc::syscall(libc::SYS_keyctl, KEYCTL_REVOKE, id) };
|
||||
if rc < 0 {
|
||||
return match io::Error::last_os_error().raw_os_error() {
|
||||
Some(libc::ENOKEY | libc::EKEYREVOKED | libc::EKEYEXPIRED) => Ok(None),
|
||||
Some(libc::ENOKEY | libc::EKEYREVOKED) => Ok(()),
|
||||
_ => last_error(),
|
||||
};
|
||||
}
|
||||
let mut buf = Zeroizing::new(vec![0u8; len as usize]);
|
||||
// SAFETY: `buf` is `len` writable bytes.
|
||||
let read = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_keyctl,
|
||||
KEYCTL_READ,
|
||||
id,
|
||||
buf.as_mut_ptr(),
|
||||
buf.len(),
|
||||
)
|
||||
};
|
||||
if read < 0 {
|
||||
return last_error();
|
||||
}
|
||||
if read as usize <= buf.len() {
|
||||
buf.truncate(read as usize);
|
||||
return Ok(Some(buf));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(io::Error::other("the key changed while it was being read"))
|
||||
}
|
||||
|
||||
/// Remove the stored secret. Succeeds when there is none.
|
||||
pub fn clear() -> io::Result<()> {
|
||||
let Some(id) = find()? else {
|
||||
return Ok(());
|
||||
};
|
||||
// SAFETY: plain integer arguments.
|
||||
let rc = unsafe { libc::syscall(libc::SYS_keyctl, KEYCTL_REVOKE, id) };
|
||||
if rc < 0 {
|
||||
return match io::Error::last_os_error().raw_os_error() {
|
||||
Some(libc::ENOKEY | libc::EKEYREVOKED) => Ok(()),
|
||||
_ => last_error(),
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -187,18 +213,21 @@ mod tests {
|
||||
// so one test covers the whole round trip and skips when it is unavailable.
|
||||
#[test]
|
||||
fn stored_secret_round_trips_and_clears() {
|
||||
if store(b"probe").is_err() {
|
||||
if PASSPHRASES.store(b"probe").is_err() {
|
||||
eprintln!("kernel keyring unavailable; skipping");
|
||||
return;
|
||||
}
|
||||
let saved = load().unwrap().expect("stored secret is readable");
|
||||
let saved = PASSPHRASES
|
||||
.load()
|
||||
.unwrap()
|
||||
.expect("stored secret is readable");
|
||||
assert_eq!(saved.as_slice(), b"probe");
|
||||
|
||||
store(b"replaced").unwrap();
|
||||
assert_eq!(load().unwrap().unwrap().as_slice(), b"replaced");
|
||||
PASSPHRASES.store(b"replaced").unwrap();
|
||||
assert_eq!(PASSPHRASES.load().unwrap().unwrap().as_slice(), b"replaced");
|
||||
|
||||
clear().unwrap();
|
||||
assert!(load().unwrap().is_none());
|
||||
clear().unwrap();
|
||||
PASSPHRASES.clear().unwrap();
|
||||
assert!(PASSPHRASES.load().unwrap().is_none());
|
||||
PASSPHRASES.clear().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
//! Trace the key chain that opens My Files, for `pdfs diagnose`.
|
||||
//!
|
||||
//! The daemon only says "missing key" when the My Files share does not open
|
||||
//! (#40). This walks the chain the other way and names the link that breaks:
|
||||
//! the share's passphrase is encrypted to an address key, whose token is
|
||||
//! encrypted to an account key, which opens with the stored passphrase. Every
|
||||
//! finding holds key IDs and flags only, never key material, so the output is
|
||||
//! safe to post in an issue.
|
||||
|
||||
use proton_sdk::account::{AddressKeyReport, KeyReport, UserKeyReport};
|
||||
use proton_sdk::ids::AddressId;
|
||||
|
||||
use crate::auth;
|
||||
use crate::error::Result;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Level {
|
||||
Ok,
|
||||
Warn,
|
||||
Fail,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Finding {
|
||||
pub level: Level,
|
||||
pub label: String,
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
/// The address and recipients of the My Files share passphrase, or why they
|
||||
/// could not be read.
|
||||
pub type ShareRecipients = std::result::Result<(AddressId, Vec<Option<String>>), String>;
|
||||
|
||||
/// Resume the stored session and ask Proton for the account's keys and the
|
||||
/// My Files share. Fails when there is no session, the keys are locked, or
|
||||
/// Proton cannot be reached; reading the share is allowed to fail on its own.
|
||||
///
|
||||
/// The session's tokens are the daemon's too. A refresh here spends the
|
||||
/// refresh token the daemon holds, which it recovers from by resuming the
|
||||
/// stored session (see [`auth::stored_session_is_newer`]).
|
||||
pub async fn run() -> Result<(KeyReport, ShareRecipients)> {
|
||||
let (client, _session) = auth::resume_client().await?;
|
||||
let report = client.account().key_report().await?;
|
||||
let share = client
|
||||
.my_files_share_recipients()
|
||||
.await
|
||||
.map_err(|e| e.to_string());
|
||||
Ok((report, share))
|
||||
}
|
||||
|
||||
/// One finding per key, then one for the My Files share.
|
||||
pub fn findings(report: &KeyReport, share: &ShareRecipients) -> Vec<Finding> {
|
||||
let mut findings = Vec::new();
|
||||
for key in &report.user_keys {
|
||||
findings.push(user_key_finding(key));
|
||||
}
|
||||
for address in &report.addresses {
|
||||
for key in &address.keys {
|
||||
findings.push(address_key_finding(report, &address.email, key));
|
||||
}
|
||||
}
|
||||
findings.push(share_finding(report, share));
|
||||
findings
|
||||
}
|
||||
|
||||
fn user_key_finding(key: &UserKeyReport) -> Finding {
|
||||
let label = format!("account key {}", short(key.id.as_str()));
|
||||
let (level, detail) = match (key.active, key.unlocks) {
|
||||
(true, true) => (Level::Ok, "opens".to_string()),
|
||||
(true, false) => (
|
||||
Level::Fail,
|
||||
format!(
|
||||
"does not open: {}",
|
||||
key.error.as_deref().unwrap_or("unknown")
|
||||
),
|
||||
),
|
||||
(false, true) => (Level::Ok, "inactive, opens with your password".to_string()),
|
||||
(false, false) => (Level::Ok, "inactive, does not open".to_string()),
|
||||
};
|
||||
Finding {
|
||||
level,
|
||||
label,
|
||||
detail: with_primary(key.primary, detail),
|
||||
}
|
||||
}
|
||||
|
||||
fn address_key_finding(report: &KeyReport, email: &str, key: &AddressKeyReport) -> Finding {
|
||||
let label = format!("{email} key {}", short(key.id.as_str()));
|
||||
let (level, detail) = if !key.active {
|
||||
(Level::Ok, "inactive, not used".to_string())
|
||||
} else if key.unlocks {
|
||||
(Level::Ok, "opens".to_string())
|
||||
} else {
|
||||
// A non-primary key matters only for what was encrypted to it; the
|
||||
// share finding says whether My Files is.
|
||||
let level = if key.primary {
|
||||
Level::Fail
|
||||
} else {
|
||||
Level::Warn
|
||||
};
|
||||
(level, format!("does not open: {}", why_closed(report, key)))
|
||||
};
|
||||
Finding {
|
||||
level,
|
||||
label,
|
||||
detail: with_primary(key.primary, detail),
|
||||
}
|
||||
}
|
||||
|
||||
/// Why an active address key does not open, named by the account key its
|
||||
/// token was meant for when the token has one.
|
||||
fn why_closed(report: &KeyReport, key: &AddressKeyReport) -> String {
|
||||
let error = key.error.as_deref().unwrap_or("unknown");
|
||||
let Some(recipients) = &key.token_recipients else {
|
||||
return error.to_string();
|
||||
};
|
||||
let named: Vec<&str> = recipients.iter().flatten().map(String::as_str).collect();
|
||||
if let Some(user_key) = named.iter().find_map(|r| report.user_key_named(r)) {
|
||||
let id = short(user_key.id.as_str());
|
||||
return match (user_key.active, user_key.unlocks) {
|
||||
(false, true) => {
|
||||
format!(
|
||||
"its token is for inactive account key {id}, which opens with your password"
|
||||
)
|
||||
}
|
||||
(false, false) => {
|
||||
format!("its token is for inactive account key {id}, which does not open either")
|
||||
}
|
||||
(true, false) => format!("its token is for account key {id}, which does not open"),
|
||||
// The token opened; the key itself refused its passphrase.
|
||||
(true, true) => error.to_string(),
|
||||
};
|
||||
}
|
||||
match named.first() {
|
||||
Some(recipient) => {
|
||||
format!("its token is for key {recipient}, which is none of your account keys")
|
||||
}
|
||||
None if recipients.is_empty() => error.to_string(),
|
||||
None => "its token does not name the key it is for".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn share_finding(report: &KeyReport, share: &ShareRecipients) -> Finding {
|
||||
let label = "My Files share".to_string();
|
||||
let (level, detail) = match share {
|
||||
Err(e) => (Level::Warn, format!("could not be read: {e}")),
|
||||
Ok((address_id, recipients)) => {
|
||||
let named: Vec<&str> = recipients.iter().flatten().map(String::as_str).collect();
|
||||
match named.iter().find_map(|r| report.address_key_named(r)) {
|
||||
Some((address, key)) => {
|
||||
let name = format!("{} key {}", address.email, short(key.id.as_str()));
|
||||
if key.unlocks {
|
||||
(Level::Ok, format!("encrypted to {name}, which opens"))
|
||||
} else {
|
||||
(
|
||||
Level::Fail,
|
||||
format!("encrypted to {name}, which does not open"),
|
||||
)
|
||||
}
|
||||
}
|
||||
None => (
|
||||
Level::Fail,
|
||||
match named.first() {
|
||||
Some(recipient) => format!(
|
||||
"encrypted to key {recipient}, which is none of the keys of address {address_id}"
|
||||
),
|
||||
None => "its passphrase does not name the key it is for".to_string(),
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
};
|
||||
Finding {
|
||||
level,
|
||||
label,
|
||||
detail,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_primary(primary: bool, detail: String) -> String {
|
||||
if primary {
|
||||
format!("primary, {detail}")
|
||||
} else {
|
||||
detail
|
||||
}
|
||||
}
|
||||
|
||||
/// The start of a Proton key ID: enough to tell keys apart and to find them
|
||||
/// in the daemon log, which prints the whole ID.
|
||||
fn short(id: &str) -> String {
|
||||
match id.char_indices().nth(8) {
|
||||
Some((end, _)) => format!("{}…", &id[..end]),
|
||||
None => id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use proton_sdk::account::AddressReport;
|
||||
use proton_sdk::ids::{AddressKeyId, UserKeyId};
|
||||
|
||||
fn user_key(id: &str, active: bool, unlocks: bool, pgp: &str) -> UserKeyReport {
|
||||
UserKeyReport {
|
||||
id: UserKeyId::new(id),
|
||||
primary: active,
|
||||
active,
|
||||
unlocks,
|
||||
error: (!unlocks).then(|| "unlock: wrong passphrase".to_string()),
|
||||
key_ids: vec![pgp.to_string()],
|
||||
}
|
||||
}
|
||||
|
||||
fn address_key(id: &str, primary: bool, token_for: &str, pgp: &str) -> AddressKeyReport {
|
||||
AddressKeyReport {
|
||||
id: AddressKeyId::new(id),
|
||||
primary,
|
||||
active: true,
|
||||
token_recipients: Some(vec![Some(token_for.to_string())]),
|
||||
unlocks: token_for == "u1",
|
||||
error: (token_for != "u1").then(|| "token: failed to decrypt: missing key".to_string()),
|
||||
key_ids: vec![pgp.to_string()],
|
||||
}
|
||||
}
|
||||
|
||||
/// The account in #40, as far as its log shows: the primary address key
|
||||
/// opens, an older one does not, and My Files is encrypted to the older one.
|
||||
fn report(stranded_token_for: &str) -> KeyReport {
|
||||
KeyReport {
|
||||
user_keys: vec![
|
||||
user_key("UserKeyOne-123", true, true, "u1"),
|
||||
user_key("UserKeyTwo-456", false, true, "u2"),
|
||||
],
|
||||
addresses: vec![AddressReport {
|
||||
id: AddressId::new("address-1"),
|
||||
email: "user@proton.me".to_string(),
|
||||
keys: vec![
|
||||
address_key("AddrKeyNew-789", true, "u1", "a1"),
|
||||
address_key("AddrKeyOld-000", false, stranded_token_for, "a2"),
|
||||
],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stranded_address_key_is_traced_to_the_inactive_account_key() {
|
||||
let report = report("u2");
|
||||
let share = Ok((AddressId::new("address-1"), vec![Some("a2".to_string())]));
|
||||
let findings = findings(&report, &share);
|
||||
|
||||
assert_eq!(findings[0].detail, "primary, opens");
|
||||
assert_eq!(findings[1].detail, "inactive, opens with your password");
|
||||
assert_eq!(findings[2].level, Level::Ok);
|
||||
assert_eq!(findings[3].level, Level::Warn);
|
||||
assert_eq!(findings[3].label, "user@proton.me key AddrKeyO…");
|
||||
assert_eq!(
|
||||
findings[3].detail,
|
||||
"does not open: its token is for inactive account key UserKeyT…, which opens with your password"
|
||||
);
|
||||
assert_eq!(findings[4].level, Level::Fail);
|
||||
assert_eq!(
|
||||
findings[4].detail,
|
||||
"encrypted to user@proton.me key AddrKeyO…, which does not open"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_for_a_key_the_account_does_not_list_says_so() {
|
||||
let report = report("ffff");
|
||||
let detail = &findings(&report, &Err("offline".to_string()))[3].detail;
|
||||
assert_eq!(
|
||||
detail,
|
||||
"does not open: its token is for key ffff, which is none of your account keys"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_share_that_opens_and_one_that_cannot_be_read() {
|
||||
let report = report("u2");
|
||||
let open = Ok((AddressId::new("address-1"), vec![Some("a1".to_string())]));
|
||||
let last = findings(&report, &open).pop().unwrap();
|
||||
assert_eq!(last.level, Level::Ok);
|
||||
assert_eq!(
|
||||
last.detail,
|
||||
"encrypted to user@proton.me key AddrKeyN…, which opens"
|
||||
);
|
||||
|
||||
let last = findings(&report, &Err("offline".to_string()))
|
||||
.pop()
|
||||
.unwrap();
|
||||
assert_eq!(last.level, Level::Warn);
|
||||
assert_eq!(last.detail, "could not be read: offline");
|
||||
}
|
||||
}
|
||||
@@ -11,20 +11,26 @@ pub mod control;
|
||||
pub mod db;
|
||||
pub mod error;
|
||||
pub mod kernelkey;
|
||||
pub mod keycheck;
|
||||
pub mod localindex;
|
||||
pub mod menu;
|
||||
pub mod mountfail;
|
||||
pub mod mountpoint;
|
||||
pub mod mounts;
|
||||
pub mod opener;
|
||||
pub mod places;
|
||||
pub mod profile;
|
||||
pub mod sdkcache;
|
||||
pub mod search;
|
||||
pub mod service;
|
||||
pub mod shell;
|
||||
pub mod similar;
|
||||
pub mod syncignore;
|
||||
pub mod takeout;
|
||||
pub mod syncroot;
|
||||
pub mod timestamp;
|
||||
pub mod tray;
|
||||
pub mod update;
|
||||
|
||||
// Photo logic shared with Proton Gallery through the proton-rs crates.
|
||||
pub use proton_gallery_rs::{exiftime, places, similar, takeout};
|
||||
|
||||
pub use access::{Access, access_for, perm_bits};
|
||||
pub use error::{CoreError, CoreResult, Error, Result};
|
||||
|
||||
@@ -42,7 +42,7 @@ const SKIP_PATH_SUFFIXES: &[&str] = &["go/pkg/mod"];
|
||||
|
||||
/// Hard cap on indexed entries, so a pathological home directory cannot grow the
|
||||
/// database without bound. Reaching it stops the walk early.
|
||||
const MAX_ENTRIES: usize = 400_000;
|
||||
pub const MAX_ENTRIES: usize = 400_000;
|
||||
|
||||
/// Rows handed to the sink at a time. Large enough that the per-transaction cost
|
||||
/// amortises, small enough that the writer lock is released often (FUSE
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
//! Why the daemon's mount keeps failing, for the front ends to read.
|
||||
//!
|
||||
//! The control socket opens only once a mount has succeeded, so a daemon whose
|
||||
//! mount fails on every try had no way to say why. The app and the tray showed
|
||||
//! "Connecting…" for days while the reason sat in the journal (`docs/BUGS.md`
|
||||
//! B263). The daemon now writes its last failure to a small file in the state
|
||||
//! directory, next to the control socket, and removes it once a mount is up.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// The file's name inside the state directory.
|
||||
const FILE_NAME: &str = "mount-failure.json";
|
||||
|
||||
/// The last failed mount attempt.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MountFailure {
|
||||
/// The error, in English, as the journal has it.
|
||||
pub error: String,
|
||||
/// Unix seconds of the first failure in this run of failures.
|
||||
pub since: i64,
|
||||
/// Failed attempts in a row.
|
||||
pub attempts: u32,
|
||||
/// Unix seconds of the next attempt.
|
||||
pub next_attempt: i64,
|
||||
/// Set when the mount was refused because the folder already has files in
|
||||
/// it, so the app can explain that in the user's language. Absent in a
|
||||
/// file an older daemon wrote.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub not_empty: Option<crate::mountpoint::NotEmpty>,
|
||||
}
|
||||
|
||||
/// Where the failure is kept for the state directory `state_dir`.
|
||||
pub fn path(state_dir: &Path) -> PathBuf {
|
||||
state_dir.join(FILE_NAME)
|
||||
}
|
||||
|
||||
/// Record `failure`, replacing the one before it. Written to a temporary file
|
||||
/// and renamed, so a reader never sees half of it.
|
||||
pub fn record(state_dir: &Path, failure: &MountFailure) -> std::io::Result<()> {
|
||||
let json = serde_json::to_vec(failure).map_err(std::io::Error::other)?;
|
||||
proton_base_rs::atomic::write_with_mode(&path(state_dir), &json, 0o644)
|
||||
}
|
||||
|
||||
/// Forget the failure: a mount is up, or a new daemon has started.
|
||||
pub fn clear(state_dir: &Path) {
|
||||
let _ = std::fs::remove_file(path(state_dir));
|
||||
}
|
||||
|
||||
/// The recorded failure, if there is one and it can be read.
|
||||
pub fn read(state_dir: &Path) -> Option<MountFailure> {
|
||||
let bytes = std::fs::read(path(state_dir)).ok()?;
|
||||
serde_json::from_slice(&bytes).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_recorded_failure_reads_back_until_cleared() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-mountfail-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
assert_eq!(read(&dir), None);
|
||||
let failure = MountFailure {
|
||||
error: "fetch My Files root: missing key".into(),
|
||||
since: 1_700_000_000,
|
||||
attempts: 3,
|
||||
next_attempt: 1_700_000_040,
|
||||
not_empty: None,
|
||||
};
|
||||
record(&dir, &failure).unwrap();
|
||||
assert_eq!(read(&dir), Some(failure));
|
||||
clear(&dir);
|
||||
assert_eq!(read(&dir), None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failure_an_older_daemon_wrote_still_reads() {
|
||||
let old = r#"{"error":"missing key","since":1,"attempts":2,"next_attempt":3}"#;
|
||||
let failure: MountFailure = serde_json::from_str(old).unwrap();
|
||||
assert_eq!(failure.not_empty, None);
|
||||
assert_eq!(failure.attempts, 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
//! Whether a folder can take the My files mount without hiding anything.
|
||||
//!
|
||||
//! FUSE mounts over a folder that already has files in it without a word. The
|
||||
//! files then sit under the mount, out of sight: not uploaded, and back only
|
||||
//! once the service stops, so they look deleted (`docs/BUGS.md` B277, #67).
|
||||
//! The daemon refuses such a folder, and the app refuses it when it is picked.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// The file system type the kernel reports for this client's own mounts.
|
||||
pub const OWN_FS_TYPE: &str = "fuse.protondrive";
|
||||
|
||||
/// How many of a folder's entries an error names.
|
||||
const NAMED_ENTRIES: usize = 3;
|
||||
|
||||
/// A folder that already has something in it.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
pub struct NotEmpty {
|
||||
/// The folder.
|
||||
pub path: PathBuf,
|
||||
/// The first few names in it, sorted.
|
||||
pub entries: Vec<String>,
|
||||
/// How many entries it has in all.
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for NotEmpty {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{} is not empty (holds ", self.path.display())?;
|
||||
write!(f, "{}", self.entries.join(", "))?;
|
||||
let rest = self.count.saturating_sub(self.entries.len());
|
||||
if rest > 0 {
|
||||
write!(f, " and {rest} more")?;
|
||||
}
|
||||
write!(
|
||||
f,
|
||||
"); My files would hide those files. Move them out, or choose another folder for My files"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for NotEmpty {}
|
||||
|
||||
/// What is in `dir`, if anything. A folder that does not exist yet is empty.
|
||||
/// Hidden entries count: a `.directory` or a `.Trash-1000` hidden under the
|
||||
/// mount is as lost as any other file.
|
||||
pub fn not_empty(dir: &Path) -> std::io::Result<Option<NotEmpty>> {
|
||||
let read = match std::fs::read_dir(dir) {
|
||||
Ok(read) => read,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let mut names = read
|
||||
.map(|entry| entry.map(|entry| entry.file_name().to_string_lossy().into_owned()))
|
||||
.collect::<std::io::Result<Vec<_>>>()?;
|
||||
if names.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
names.sort();
|
||||
let count = names.len();
|
||||
names.truncate(NAMED_ENTRIES);
|
||||
Ok(Some(NotEmpty {
|
||||
path: dir.to_path_buf(),
|
||||
entries: names,
|
||||
count,
|
||||
}))
|
||||
}
|
||||
|
||||
/// The file system type mounted at `path`, as `/proc/self/mountinfo` names it
|
||||
/// (`ext4`, `fuse.protondrive`, …), or `None` when nothing is mounted there.
|
||||
pub fn mounted_fs_type(path: &Path) -> Option<String> {
|
||||
let wanted = path.canonicalize().ok()?;
|
||||
let mounts = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
mounts
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let fields: Vec<_> = line.split_whitespace().collect();
|
||||
let separator = fields.iter().position(|field| *field == "-")?;
|
||||
let encoded = *fields.get(4)?;
|
||||
let decoded = encoded.replace("\\040", " ").replace("\\011", "\t");
|
||||
let mountpoint = PathBuf::from(decoded);
|
||||
if mountpoint.canonicalize().ok()? == wanted {
|
||||
Some(fields.get(separator + 1)?.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.next()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn temp_dir(name: &str) -> PathBuf {
|
||||
let dir =
|
||||
std::env::temp_dir().join(format!("pdfs-mountpoint-{name}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_or_missing_folder_takes_the_mount() {
|
||||
let dir = temp_dir("empty");
|
||||
assert_eq!(not_empty(&dir).unwrap(), None);
|
||||
assert_eq!(not_empty(&dir.join("missing")).unwrap(), None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_with_files_is_refused_and_named() {
|
||||
let dir = temp_dir("files");
|
||||
for name in ["report.pdf", "Notes", ".directory", "zeta.txt"] {
|
||||
std::fs::write(dir.join(name), b"x").unwrap();
|
||||
}
|
||||
let found = not_empty(&dir).unwrap().expect("must refuse");
|
||||
assert_eq!(found.count, 4);
|
||||
assert_eq!(found.entries, [".directory", "Notes", "report.pdf"]);
|
||||
let message = found.to_string();
|
||||
assert!(message.contains("is not empty"), "{message}");
|
||||
assert!(
|
||||
message.contains(".directory, Notes, report.pdf and 1 more"),
|
||||
"{message}"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hidden_file_alone_is_enough_to_refuse() {
|
||||
let dir = temp_dir("hidden");
|
||||
std::fs::write(dir.join(".Trash-1000"), b"").unwrap();
|
||||
let found = not_empty(&dir).unwrap().expect("must refuse");
|
||||
assert_eq!(found.count, 1);
|
||||
assert!(!found.to_string().contains("more"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_folder_is_not_a_mount() {
|
||||
let dir = temp_dir("plain");
|
||||
assert_eq!(mounted_fs_type(&dir), None);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,8 @@ pub enum MountKind {
|
||||
MyFiles,
|
||||
/// A folder owned by this machine's Proton Drive device.
|
||||
Device { sync_folder_id: i64 },
|
||||
/// A real local mirror of an existing My files folder.
|
||||
Attached { sync_folder_id: i64 },
|
||||
/// A future standalone mount rooted at a node shared with this account.
|
||||
Shared { share_root_uid: String },
|
||||
}
|
||||
@@ -88,8 +90,8 @@ pub enum MountKind {
|
||||
/// One local Proton Drive location as exposed over the control protocol.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct MountSpec {
|
||||
/// Presentation-row id. Device actions use the `sync_folder_id` carried by
|
||||
/// [`MountKind::Device`], not this id.
|
||||
/// Presentation-row id. Synced-folder actions use the `sync_folder_id`
|
||||
/// carried by [`MountKind::Device`] or [`MountKind::Attached`], not this id.
|
||||
pub id: i64,
|
||||
pub kind: MountKind,
|
||||
pub local_path: String,
|
||||
|
||||
+153
-21
@@ -27,6 +27,7 @@ use std::os::unix::process::CommandExt;
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use crate::control::is_video_file_name;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Environment override for the terminal used by `"terminal": true` rules.
|
||||
@@ -108,11 +109,12 @@ impl OpenWith {
|
||||
/// a Markdown file). The rules are about *what the user opened*, so the
|
||||
/// caller passes that name and the argv still gets the real path.
|
||||
pub fn command_for_named(&self, path: &Path, name: &str, is_dir: bool) -> Vec<String> {
|
||||
let kind = Kind::of(path, name, is_dir);
|
||||
for rule in &self.rules {
|
||||
if rule.command.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !rule.patterns.iter().any(|p| matches(p, name, is_dir)) {
|
||||
if !rule.patterns.iter().any(|p| matches(p, name, kind)) {
|
||||
continue;
|
||||
}
|
||||
let argv = expand(&rule.command, path);
|
||||
@@ -189,31 +191,38 @@ fn spawn(argv: &[String], path: &Path) {
|
||||
.stderr(Stdio::null())
|
||||
.process_group(0)
|
||||
.spawn();
|
||||
if let Err(e) = spawned {
|
||||
tracing::error!(command = ?argv, "open failed: {e}");
|
||||
match spawned {
|
||||
Ok(child) => reap(child),
|
||||
Err(e) => tracing::error!(command = ?argv, "open failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`open`] with the policy from the user's `config.json`, read once per
|
||||
/// process. For callers that open the occasional path and have no reason to
|
||||
/// carry a policy around; long-lived windows should hold their own copy.
|
||||
/// Wait for `child` on a thread of its own. A child nobody waits for stays a
|
||||
/// zombie for as long as its parent lives, which for the app and the tray is
|
||||
/// the whole session.
|
||||
pub fn reap(mut child: std::process::Child) {
|
||||
std::thread::spawn(move || {
|
||||
let _ = child.wait();
|
||||
});
|
||||
}
|
||||
|
||||
/// [`open`] with the policy from the user's `config.json`, read again on
|
||||
/// every call: the file is small, a path is opened by hand, and a rule changed
|
||||
/// in Preferences then applies to the next file opened, without a restart.
|
||||
pub fn open_default(path: &Path, is_dir: bool) {
|
||||
open(default_policy(), path, is_dir);
|
||||
open(&default_policy(), path, is_dir);
|
||||
}
|
||||
|
||||
/// [`open_default`] matching the rules against `name` — see
|
||||
/// [`OpenWith::command_for_named`].
|
||||
pub fn open_default_named(path: &Path, name: &str, is_dir: bool) {
|
||||
open_named(default_policy(), path, name, is_dir);
|
||||
open_named(&default_policy(), path, name, is_dir);
|
||||
}
|
||||
|
||||
fn default_policy() -> &'static OpenWith {
|
||||
static POLICY: std::sync::OnceLock<OpenWith> = std::sync::OnceLock::new();
|
||||
POLICY.get_or_init(|| {
|
||||
crate::config::AppDirs::new()
|
||||
.map(|dirs| dirs.load_config().resolved_open_with())
|
||||
.unwrap_or_default()
|
||||
})
|
||||
fn default_policy() -> OpenWith {
|
||||
crate::config::AppDirs::new()
|
||||
.map(|dirs| dirs.load_config().resolved_open_with())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// `$PDFS_TERMINAL`, then `$TERMINAL`, then the first [`KNOWN_TERMINALS`] entry
|
||||
@@ -254,6 +263,33 @@ fn with_exec_flag(program: &str) -> Vec<String> {
|
||||
argv
|
||||
}
|
||||
|
||||
/// Split a command line typed by the user into argv, the way a shell would:
|
||||
/// quotes keep spaces, nothing is expanded. `None` for an unclosed quote.
|
||||
pub fn parse_command(line: &str) -> Option<Vec<String>> {
|
||||
shell_words::split(line).ok()
|
||||
}
|
||||
|
||||
/// Show `argv` as one line that [`parse_command`] turns back into it.
|
||||
pub fn format_command(argv: &[String]) -> String {
|
||||
shell_words::join(argv)
|
||||
}
|
||||
|
||||
/// Whether the program `argv` starts with can be found: an absolute or
|
||||
/// relative path to a file, or a name on `PATH`. A `$VAR` program counts as
|
||||
/// found, since it is looked up only when a file is opened.
|
||||
pub fn command_available(argv: &[String]) -> bool {
|
||||
let Some(program) = argv.first() else {
|
||||
return false;
|
||||
};
|
||||
if program.starts_with('$') {
|
||||
return true;
|
||||
}
|
||||
if program.contains('/') {
|
||||
return Path::new(program).is_file();
|
||||
}
|
||||
on_path(program)
|
||||
}
|
||||
|
||||
fn on_path(program: &str) -> bool {
|
||||
let Some(path) = std::env::var_os("PATH") else {
|
||||
return false;
|
||||
@@ -315,16 +351,46 @@ fn expand_env(token: &str) -> Option<String> {
|
||||
value.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
/// What is being opened, as far as the `@class` patterns care.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Kind {
|
||||
Dir,
|
||||
Video,
|
||||
File,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
/// `.ts` names TypeScript and MPEG transport streams alike, so for that
|
||||
/// one extension the size on disk decides (see
|
||||
/// [`is_video_file_name`](crate::control::is_video_file_name)). No other
|
||||
/// name costs a `stat`.
|
||||
fn of(path: &Path, name: &str, is_dir: bool) -> Kind {
|
||||
if is_dir {
|
||||
return Kind::Dir;
|
||||
}
|
||||
let size = extension(name)
|
||||
.eq_ignore_ascii_case("ts")
|
||||
.then(|| std::fs::metadata(path).ok().map(|meta| meta.len()))
|
||||
.flatten();
|
||||
if is_video_file_name(name, size) {
|
||||
Kind::Video
|
||||
} else {
|
||||
Kind::File
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `pattern` — a glob or an `@class` — matches this name.
|
||||
fn matches(pattern: &str, name: &str, is_dir: bool) -> bool {
|
||||
fn matches(pattern: &str, name: &str, kind: Kind) -> bool {
|
||||
let name_lower = name.to_lowercase();
|
||||
let file = kind == Kind::File;
|
||||
match pattern.to_lowercase().as_str() {
|
||||
"@any" => true,
|
||||
"@dir" => is_dir,
|
||||
"@text" => !is_dir && class_text(&name_lower),
|
||||
"@document" => !is_dir && class_document(&name_lower),
|
||||
"@image" => !is_dir && class_image(&name_lower),
|
||||
"@media" => !is_dir && class_media(&name_lower),
|
||||
"@dir" => kind == Kind::Dir,
|
||||
"@text" => file && class_text(&name_lower),
|
||||
"@document" => file && class_document(&name_lower),
|
||||
"@image" => file && class_image(&name_lower),
|
||||
"@media" => kind == Kind::Video || (file && class_media(&name_lower)),
|
||||
other if other.starts_with('@') => false,
|
||||
other => glob_match(other, &name_lower),
|
||||
}
|
||||
@@ -472,6 +538,21 @@ fn glob_match(pattern: &str, name: &str) -> bool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_reaped_child_does_not_stay_a_zombie() {
|
||||
let child = std::process::Command::new("true").spawn().unwrap();
|
||||
let proc_entry = std::path::PathBuf::from(format!("/proc/{}", child.id()));
|
||||
reap(child);
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while proc_entry.exists() && std::time::Instant::now() < deadline {
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
assert!(
|
||||
!proc_entry.exists(),
|
||||
"the child is still in the process table"
|
||||
);
|
||||
}
|
||||
|
||||
fn rule(patterns: &[&str], command: &[&str], terminal: bool) -> OpenRule {
|
||||
OpenRule {
|
||||
patterns: patterns.iter().map(|p| (*p).to_string()).collect(),
|
||||
@@ -508,6 +589,37 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_ts_file_opens_as_media_or_text_by_its_size() {
|
||||
let policy = OpenWith {
|
||||
terminal: Some(vec!["alacritty".into(), "-e".into()]),
|
||||
default: None,
|
||||
rules: vec![
|
||||
rule(&["@text"], &["nvim"], true),
|
||||
rule(&["@media"], &["mpv"], false),
|
||||
],
|
||||
};
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-opener-ts-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let source = dir.join("blob-source");
|
||||
std::fs::write(&source, "export const answer = 42;\n").unwrap();
|
||||
let recording = dir.join("blob-recording");
|
||||
std::fs::File::create(&recording)
|
||||
.unwrap()
|
||||
.set_len(crate::control::TS_VIDEO_MIN_BYTES)
|
||||
.unwrap();
|
||||
|
||||
let source_argv = policy.command_for_named(&source, "app.ts", false);
|
||||
let recording_argv = policy.command_for_named(&recording, "recording.ts", false);
|
||||
// A file that is not on this machine yet has no size to go by.
|
||||
let unknown_argv = policy.command_for_named(&dir.join("missing"), "recording.ts", false);
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
|
||||
assert_eq!(source_argv[..3], ["alacritty", "-e", "nvim"]);
|
||||
assert_eq!(recording_argv[0], "mpv");
|
||||
assert_eq!(unknown_argv[0], "mpv");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_policy_is_xdg_open() {
|
||||
let policy = OpenWith::default();
|
||||
@@ -615,4 +727,24 @@ mod tests {
|
||||
// Unknown terminals get the near-universal `-e`.
|
||||
assert_eq!(with_exec_flag("mystery-term"), vec!["mystery-term", "-e"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_typed_command_line_round_trips() {
|
||||
let argv = parse_command("alacritty -e 'my editor' --flag").unwrap();
|
||||
assert_eq!(argv, ["alacritty", "-e", "my editor", "--flag"]);
|
||||
assert_eq!(parse_command(&format_command(&argv)).unwrap(), argv);
|
||||
assert_eq!(parse_command("nvim 'unclosed"), None);
|
||||
assert_eq!(parse_command(" ").unwrap(), Vec::<String>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_command_is_available_when_its_program_is() {
|
||||
assert!(command_available(&["sh".to_string()]));
|
||||
assert!(command_available(&["$EDITOR".to_string()]));
|
||||
assert!(!command_available(&[]));
|
||||
assert!(!command_available(
|
||||
&["pdfs-no-such-program-xyz".to_string()]
|
||||
));
|
||||
assert!(!command_available(&["/nonexistent/program".to_string()]));
|
||||
}
|
||||
}
|
||||
@@ -1,185 +0,0 @@
|
||||
//! Offline reverse geocoding: the town a photo was taken in, from its GPS
|
||||
//! position, without asking any service.
|
||||
//!
|
||||
//! The towns are GeoNames' `cities15000` dump (every place with more than
|
||||
//! 15,000 people), trimmed to id, name, position and country in
|
||||
//! `data/cities15000.tsv` by `scripts/geonames-cities.py`. The script drops the
|
||||
//! parts of a city, such as Kreuzberg or "Paris 15 Vaugirard", so that a photo
|
||||
//! lands in the town people would name. GeoNames publishes the dump under
|
||||
//! CC BY 4.0, which is why the About dialog credits it.
|
||||
//!
|
||||
//! `data/countries.tsv` maps each country code to its ISO 3166-1 English name,
|
||||
//! as the `iso-codes` project spells it, so a front end can translate it
|
||||
//! through that project's `iso_3166-1` gettext domain.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// How far from the nearest town a photo may be and still count as taken
|
||||
/// there, in km. Farther out (at sea, in the wilderness) it has no place.
|
||||
pub const MAX_DISTANCE_KM: f64 = 50.0;
|
||||
|
||||
/// Mean Earth radius in km, for the haversine distance.
|
||||
const EARTH_RADIUS_KM: f64 = 6371.0;
|
||||
|
||||
const CITIES: &str = include_str!("../data/cities15000.tsv");
|
||||
const COUNTRIES: &str = include_str!("../data/countries.tsv");
|
||||
|
||||
/// One town of the table.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct City {
|
||||
/// The GeoNames id, stable across dumps.
|
||||
pub id: u32,
|
||||
pub name: &'static str,
|
||||
pub latitude: f64,
|
||||
pub longitude: f64,
|
||||
/// ISO 3166-1 alpha-2 country code.
|
||||
pub country: &'static str,
|
||||
}
|
||||
|
||||
/// The towns, and an index of them by whole-degree cell.
|
||||
struct Table {
|
||||
cities: Vec<City>,
|
||||
cells: HashMap<(i32, i32), Vec<usize>>,
|
||||
}
|
||||
|
||||
fn table() -> &'static Table {
|
||||
static TABLE: OnceLock<Table> = OnceLock::new();
|
||||
TABLE.get_or_init(|| {
|
||||
let cities: Vec<City> = CITIES.lines().filter_map(parse_city).collect();
|
||||
let mut cells: HashMap<(i32, i32), Vec<usize>> = HashMap::new();
|
||||
for (index, city) in cities.iter().enumerate() {
|
||||
cells
|
||||
.entry(cell(city.latitude, city.longitude))
|
||||
.or_default()
|
||||
.push(index);
|
||||
}
|
||||
Table { cities, cells }
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_city(line: &'static str) -> Option<City> {
|
||||
let mut fields = line.split('\t');
|
||||
Some(City {
|
||||
id: fields.next()?.parse().ok()?,
|
||||
name: fields.next()?,
|
||||
latitude: fields.next()?.parse().ok()?,
|
||||
longitude: fields.next()?.parse().ok()?,
|
||||
country: fields.next()?,
|
||||
})
|
||||
}
|
||||
|
||||
fn cell(latitude: f64, longitude: f64) -> (i32, i32) {
|
||||
(latitude.floor() as i32, longitude.floor() as i32)
|
||||
}
|
||||
|
||||
/// Great-circle distance between two positions, in km.
|
||||
pub fn distance_km(a: (f64, f64), b: (f64, f64)) -> f64 {
|
||||
let (lat1, lat2) = (a.0.to_radians(), b.0.to_radians());
|
||||
let dlat = lat2 - lat1;
|
||||
let dlon = (b.1 - a.1).to_radians();
|
||||
let h = (dlat / 2.0).sin().powi(2) + lat1.cos() * lat2.cos() * (dlon / 2.0).sin().powi(2);
|
||||
2.0 * EARTH_RADIUS_KM * h.sqrt().min(1.0).asin()
|
||||
}
|
||||
|
||||
/// The town nearest to a position, if one lies within [`MAX_DISTANCE_KM`].
|
||||
pub fn nearest(latitude: f64, longitude: f64) -> Option<&'static City> {
|
||||
if !latitude.is_finite() || !longitude.is_finite() {
|
||||
return None;
|
||||
}
|
||||
let table = table();
|
||||
// A degree of latitude is about 111 km everywhere; a degree of longitude
|
||||
// shrinks towards the poles, so more cells are needed to cover the radius.
|
||||
let lat_span = (MAX_DISTANCE_KM / 111.0).ceil() as i32;
|
||||
let lon_span = {
|
||||
let scale = latitude.to_radians().cos().abs().max(0.01);
|
||||
((MAX_DISTANCE_KM / (111.0 * scale)).ceil() as i32).min(180)
|
||||
};
|
||||
let (row, column) = cell(latitude, longitude);
|
||||
let mut best: Option<(f64, &City)> = None;
|
||||
for dlat in -lat_span..=lat_span {
|
||||
for dlon in -lon_span..=lon_span {
|
||||
// Longitude wraps at the antimeridian: -181 is 179.
|
||||
let lon_cell = (column + dlon + 180).rem_euclid(360) - 180;
|
||||
let Some(indices) = table.cells.get(&(row + dlat, lon_cell)) else {
|
||||
continue;
|
||||
};
|
||||
for &index in indices {
|
||||
let city = &table.cities[index];
|
||||
let d = distance_km((latitude, longitude), (city.latitude, city.longitude));
|
||||
if d <= MAX_DISTANCE_KM && best.is_none_or(|(bd, _)| d < bd) {
|
||||
best = Some((d, city));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
best.map(|(_, city)| city)
|
||||
}
|
||||
|
||||
/// The town with this GeoNames id.
|
||||
pub fn city(id: u32) -> Option<&'static City> {
|
||||
let cities = &table().cities;
|
||||
// The table is sorted by id.
|
||||
cities
|
||||
.binary_search_by_key(&id, |city| city.id)
|
||||
.ok()
|
||||
.map(|index| &cities[index])
|
||||
}
|
||||
|
||||
/// The English ISO 3166-1 name of a country code, such as "Germany" for "DE".
|
||||
pub fn country_name(code: &str) -> Option<&'static str> {
|
||||
COUNTRIES.lines().find_map(|line| {
|
||||
let (c, name) = line.split_once('\t')?;
|
||||
(c == code).then_some(name)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_position_in_a_town_resolves_to_that_town() {
|
||||
let berlin = nearest(52.52, 13.405).expect("Berlin is a town");
|
||||
assert_eq!(berlin.name, "Berlin");
|
||||
assert_eq!(berlin.country, "DE");
|
||||
assert_eq!(city(berlin.id), Some(berlin));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_open_sea_has_no_town() {
|
||||
assert_eq!(nearest(0.0, -30.0), None);
|
||||
assert_eq!(nearest(f64::NAN, 0.0), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_town_across_the_antimeridian_is_still_found() {
|
||||
// Just east of 180°, the nearest towns in Fiji are west of it.
|
||||
let town = nearest(-16.8, 179.99);
|
||||
assert!(town.is_none_or(|t| t.country == "FJ"));
|
||||
let suva = nearest(-18.14, 178.44).expect("Suva is a town");
|
||||
assert_eq!(suva.country, "FJ");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_town_has_a_named_country() {
|
||||
for city in &table().cities {
|
||||
assert!(
|
||||
country_name(city.country).is_some(),
|
||||
"{} has no country name for {}",
|
||||
city.name,
|
||||
city.country
|
||||
);
|
||||
}
|
||||
assert_eq!(country_name("DE"), Some("Germany"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distances_are_great_circle_km() {
|
||||
let d = distance_km((52.52, 13.405), (48.137, 11.575));
|
||||
assert!(
|
||||
(d - 504.0).abs() < 5.0,
|
||||
"Berlin to Munich is about 504 km, got {d}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -19,9 +19,31 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::mounts::MountMode;
|
||||
|
||||
/// File name of the profile.
|
||||
/// File name of the profile clients before 3.5.0 wrote, one per device.
|
||||
///
|
||||
/// Read, never written: two computers sharing a device each overwrote the
|
||||
/// other's. See [`profile_file_name`].
|
||||
pub const PROFILE_FILE_NAME: &str = "profile.json";
|
||||
|
||||
/// File name of one machine's profile: `profile-<machine id>.json`.
|
||||
///
|
||||
/// One per machine, because several computers may sync into one device, and
|
||||
/// each describes its own arrangement. A single shared file went to whichever
|
||||
/// computer saved last.
|
||||
pub fn profile_file_name(machine_id: &str) -> String {
|
||||
format!("profile-{machine_id}.json")
|
||||
}
|
||||
|
||||
/// Whether `name` is a profile: [`PROFILE_FILE_NAME`] or one machine's
|
||||
/// [`profile_file_name`].
|
||||
pub fn is_profile_file_name(name: &str) -> bool {
|
||||
name == PROFILE_FILE_NAME
|
||||
|| name
|
||||
.strip_prefix("profile-")
|
||||
.and_then(|rest| rest.strip_suffix(".json"))
|
||||
.is_some_and(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric()))
|
||||
}
|
||||
|
||||
/// Folder, directly under the device root, that holds the profile.
|
||||
///
|
||||
/// The API refuses `Cannot create file at the root of a device` (HTTP 422), so
|
||||
@@ -74,6 +96,10 @@ pub struct Profile {
|
||||
/// Hostname of the machine that wrote it, so a restore prompt can say
|
||||
/// *"restore settings from laptop-xyz?"* rather than quoting a uid.
|
||||
pub hostname: String,
|
||||
/// The writing machine's id, as in its [`profile_file_name`]. `None` in a
|
||||
/// profile written before 3.5.0.
|
||||
#[serde(default)]
|
||||
pub machine_id: Option<String>,
|
||||
/// When it was written, epoch seconds.
|
||||
pub saved_at: i64,
|
||||
#[serde(default)]
|
||||
@@ -92,6 +118,25 @@ pub struct Profile {
|
||||
pub mountpoint: Option<String>,
|
||||
}
|
||||
|
||||
/// The saved arrangement of the folder with uid `remote_uid`, from the
|
||||
/// profiles of every computer that syncs the device.
|
||||
///
|
||||
/// The profile of the machine named `hostname` wins, since it describes this
|
||||
/// computer; otherwise the most recently saved one that has the folder.
|
||||
pub fn saved_folder<'a>(
|
||||
profiles: &'a [Profile],
|
||||
remote_uid: &str,
|
||||
hostname: &str,
|
||||
) -> Option<&'a ProfileFolder> {
|
||||
let holding = |p: &&'a Profile| p.folders.iter().any(|f| f.remote_uid == remote_uid);
|
||||
let profile = profiles
|
||||
.iter()
|
||||
.filter(holding)
|
||||
.find(|p| p.hostname == hostname)
|
||||
.or_else(|| profiles.iter().filter(holding).max_by_key(|p| p.saved_at))?;
|
||||
profile.folders.iter().find(|f| f.remote_uid == remote_uid)
|
||||
}
|
||||
|
||||
impl Profile {
|
||||
/// Parse a profile document, refusing one written by a newer client.
|
||||
pub fn parse(bytes: &[u8]) -> Result<Self, String> {
|
||||
@@ -122,6 +167,7 @@ mod tests {
|
||||
version: PROFILE_VERSION,
|
||||
device_uid: "dev-1".to_string(),
|
||||
hostname: "laptop-xyz".to_string(),
|
||||
machine_id: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
saved_at: 1_700_000_000,
|
||||
folders: vec![ProfileFolder {
|
||||
remote_uid: "vol~link".to_string(),
|
||||
@@ -219,6 +265,52 @@ mod tests {
|
||||
assert!(p.ignore_patterns.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_profile_name_is_the_legacy_one_or_one_per_machine() {
|
||||
assert!(is_profile_file_name(PROFILE_FILE_NAME));
|
||||
let name = profile_file_name("0123456789abcdef0123456789abcdef");
|
||||
assert_eq!(name, "profile-0123456789abcdef0123456789abcdef.json");
|
||||
assert!(is_profile_file_name(&name));
|
||||
assert!(!is_profile_file_name("profile-.json"));
|
||||
assert!(!is_profile_file_name("profile-a/b.json"));
|
||||
assert!(!is_profile_file_name("profile (1).json"));
|
||||
assert!(!is_profile_file_name("notes.json"));
|
||||
}
|
||||
|
||||
/// A profile written before 3.5.0 has no machine id and still parses.
|
||||
#[test]
|
||||
fn a_profile_without_a_machine_id_parses() {
|
||||
let json = br#"{"version":1,"device_uid":"d","hostname":"h","saved_at":0}"#;
|
||||
assert_eq!(Profile::parse(json).unwrap().machine_id, None);
|
||||
}
|
||||
|
||||
/// Several computers sync one device, each with its own profile. This
|
||||
/// computer's own wins, then the newest that has the folder.
|
||||
#[test]
|
||||
fn the_saved_folder_comes_from_this_computer_then_the_newest_profile() {
|
||||
let profile = |hostname: &str, saved_at: i64, path: &str| Profile {
|
||||
hostname: hostname.to_string(),
|
||||
saved_at,
|
||||
folders: vec![ProfileFolder {
|
||||
remote_uid: "vol~docs".to_string(),
|
||||
local_path: path.to_string(),
|
||||
mode: MountMode::Mirror,
|
||||
}],
|
||||
..sample()
|
||||
};
|
||||
let profiles = vec![
|
||||
profile("desktop", 10, "/home/a/Docs"),
|
||||
profile("laptop", 5, "/home/b/Docs"),
|
||||
profile("old", 1, "/home/c/Docs"),
|
||||
];
|
||||
let path =
|
||||
|host: &str| saved_folder(&profiles, "vol~docs", host).map(|f| f.local_path.as_str());
|
||||
assert_eq!(path("laptop"), Some("/home/b/Docs"));
|
||||
assert_eq!(path("new-machine"), Some("/home/a/Docs"));
|
||||
assert_eq!(saved_folder(&profiles, "vol~other", "laptop"), None);
|
||||
assert_eq!(saved_folder(&[], "vol~docs", "laptop"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_version_is_refused() {
|
||||
let mut p = sample();
|
||||
|
||||
@@ -23,6 +23,11 @@ const SCRIPTS: [(&str, &str); 2] = [
|
||||
/// `NAUTILUS_SCRIPT_SELECTED_FILE_PATHS`; the positional args carry the same
|
||||
/// selection when the script runs on a local (non-URI) folder, so we prefer the
|
||||
/// environment and fall back to `"$@"` for the Nemo/Caja variants.
|
||||
///
|
||||
/// The `--` keeps a file named like an option, such as `--json`, a path. A
|
||||
/// name with a newline in it still arrives split in two from Nautilus, whose
|
||||
/// list is newline-separated; `pdfs` then fails on both halves, which touches
|
||||
/// nothing.
|
||||
fn script_body(subcommand: &str) -> String {
|
||||
format!(
|
||||
"#!/bin/sh\n\
|
||||
@@ -41,7 +46,7 @@ fn script_body(subcommand: &str) -> String {
|
||||
\n\
|
||||
printf '%s' \"$paths\" | while IFS= read -r path; do\n\
|
||||
\x20 [ -n \"$path\" ] || continue\n\
|
||||
\x20 pdfs {subcommand} \"$path\" || exit 1\n\
|
||||
\x20 pdfs {subcommand} -- \"$path\" || exit 1\n\
|
||||
done\n"
|
||||
)
|
||||
}
|
||||
@@ -95,3 +100,13 @@ fn write_scripts(dir: &PathBuf) -> io::Result<()> {
|
||||
tracing::debug!("installed file-manager scripts in {dir:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::script_body;
|
||||
|
||||
#[test]
|
||||
fn the_script_ends_options_before_each_path() {
|
||||
assert!(script_body("pin").contains(r#"pdfs pin -- "$path""#));
|
||||
}
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
//! Telling photos apart by what they show rather than by their bytes.
|
||||
//!
|
||||
//! The server's content hash only matches byte-identical files. A photo that
|
||||
//! was resized, re-encoded or sent through a messenger is another file, so the
|
||||
//! similar-photo finder hashes each thumbnail instead: a 64-bit difference
|
||||
//! hash ("dHash"), which compares neighbouring brightness in a 9×8 grid. Two
|
||||
//! copies of one picture land a few bits apart, and different pictures about
|
||||
//! half of the bits apart.
|
||||
|
||||
/// Hashes this many bits apart or fewer show the same picture.
|
||||
pub const SIMILAR_BITS: u32 = 6;
|
||||
|
||||
/// Stored for a photo whose thumbnail was looked at and gave no usable hash
|
||||
/// (it couldn't be decoded, or is one flat colour), so it isn't looked at
|
||||
/// again. A real hash is never 0: a picture that would hash to 0 has no
|
||||
/// brightness changes, which [`difference_hash`] turns down as flat.
|
||||
pub const NO_HASH: u64 = 0;
|
||||
|
||||
/// Samples of the grid that differ by less than this in brightness, all of
|
||||
/// them, make a flat picture — a black frame, a blank page — whose hash is
|
||||
/// noise and would match every other flat picture.
|
||||
const FLAT: u8 = 8;
|
||||
|
||||
/// The difference hash of an 8-bit greyscale image `width` px wide, rows
|
||||
/// stored one after another in `luma`. `None` for an empty or flat image.
|
||||
pub fn difference_hash(width: usize, height: usize, luma: &[u8]) -> Option<u64> {
|
||||
if width == 0 || height == 0 || luma.len() < width * height {
|
||||
return None;
|
||||
}
|
||||
// Box-average down to 9×8, so every source pixel counts once and the hash
|
||||
// doesn't depend on which pixels a nearest-neighbour pick happens to hit.
|
||||
let mut grid = [[0u8; 9]; 8];
|
||||
for (gy, row) in grid.iter_mut().enumerate() {
|
||||
let (y0, y1) = span(gy, 8, height);
|
||||
for (gx, cell) in row.iter_mut().enumerate() {
|
||||
let (x0, x1) = span(gx, 9, width);
|
||||
let mut sum = 0u64;
|
||||
for y in y0..y1 {
|
||||
sum += luma[y * width + x0..y * width + x1]
|
||||
.iter()
|
||||
.map(|&v| u64::from(v))
|
||||
.sum::<u64>();
|
||||
}
|
||||
*cell = (sum / ((y1 - y0) * (x1 - x0)) as u64) as u8;
|
||||
}
|
||||
}
|
||||
let (min, max) = grid
|
||||
.iter()
|
||||
.flatten()
|
||||
.fold((u8::MAX, 0), |(min, max), &v| (min.min(v), max.max(v)));
|
||||
if max - min < FLAT {
|
||||
return None;
|
||||
}
|
||||
let mut hash = 0u64;
|
||||
for row in &grid {
|
||||
for pair in row.windows(2) {
|
||||
hash = hash << 1 | u64::from(pair[0] < pair[1]);
|
||||
}
|
||||
}
|
||||
(hash != NO_HASH).then_some(hash)
|
||||
}
|
||||
|
||||
/// The source pixels `from..to` that cell `i` of `cells` covers along an edge
|
||||
/// of `len` px; never empty, even for an edge shorter than the grid.
|
||||
fn span(i: usize, cells: usize, len: usize) -> (usize, usize) {
|
||||
let from = (i * len / cells).min(len - 1);
|
||||
let to = ((i + 1) * len / cells).max(from + 1);
|
||||
(from, to)
|
||||
}
|
||||
|
||||
/// Whether two hashes show the same picture.
|
||||
pub fn alike(a: u64, b: u64) -> bool {
|
||||
(a ^ b).count_ones() <= SIMILAR_BITS
|
||||
}
|
||||
|
||||
/// Group `hashes` into sets of alike ones, by index, each set in index order
|
||||
/// and the sets by their first index. A hash alike to none other is in no
|
||||
/// set. Alike is taken as catching: a burst of shots each a little different
|
||||
/// from the last is one set.
|
||||
pub fn sets(hashes: &[u64]) -> Vec<Vec<usize>> {
|
||||
// Two hashes at most SIMILAR_BITS apart agree on at least one of their
|
||||
// eight bytes, as long as SIMILAR_BITS < 8. So only hashes sharing some
|
||||
// byte are compared, not every pair of the library.
|
||||
const _: () = assert!(SIMILAR_BITS < 8);
|
||||
let mut parent: Vec<usize> = (0..hashes.len()).collect();
|
||||
fn root(parent: &mut [usize], mut i: usize) -> usize {
|
||||
while parent[i] != i {
|
||||
parent[i] = parent[parent[i]];
|
||||
i = parent[i];
|
||||
}
|
||||
i
|
||||
}
|
||||
for byte in 0..8 {
|
||||
let mut buckets: Vec<Vec<usize>> = vec![Vec::new(); 256];
|
||||
for (i, &hash) in hashes.iter().enumerate() {
|
||||
buckets[(hash >> (byte * 8)) as u8 as usize].push(i);
|
||||
}
|
||||
for bucket in &buckets {
|
||||
for (n, &a) in bucket.iter().enumerate() {
|
||||
for &b in &bucket[n + 1..] {
|
||||
if alike(hashes[a], hashes[b]) {
|
||||
let (ra, rb) = (root(&mut parent, a), root(&mut parent, b));
|
||||
if ra != rb {
|
||||
parent[ra.max(rb)] = ra.min(rb);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut by_root: std::collections::BTreeMap<usize, Vec<usize>> = Default::default();
|
||||
for i in 0..hashes.len() {
|
||||
let r = root(&mut parent, i);
|
||||
by_root.entry(r).or_default().push(i);
|
||||
}
|
||||
by_root.into_values().filter(|set| set.len() > 1).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A 90×80 picture: a bright disc on a gradient.
|
||||
fn picture(width: usize, height: usize) -> Vec<u8> {
|
||||
let mut luma = Vec::with_capacity(width * height);
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
let (fx, fy) = (x as f64 / width as f64, y as f64 / height as f64);
|
||||
let disc = ((fx - 0.3).powi(2) + (fy - 0.6).powi(2)).sqrt() < 0.2;
|
||||
let v = if disc { 240.0 } else { 40.0 + 120.0 * fx * fy };
|
||||
luma.push(v as u8);
|
||||
}
|
||||
}
|
||||
luma
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resized_copy_hashes_alike() {
|
||||
let big = difference_hash(900, 800, &picture(900, 800)).unwrap();
|
||||
let small = difference_hash(91, 79, &picture(91, 79)).unwrap();
|
||||
assert!(
|
||||
alike(big, small),
|
||||
"{} bits apart",
|
||||
(big ^ small).count_ones()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_picture_hashes_apart() {
|
||||
let one = difference_hash(90, 80, &picture(90, 80)).unwrap();
|
||||
let mirrored: Vec<u8> = picture(90, 80)
|
||||
.chunks(90)
|
||||
.flat_map(|row| row.iter().rev().copied().collect::<Vec<_>>())
|
||||
.collect();
|
||||
let other = difference_hash(90, 80, &mirrored).unwrap();
|
||||
assert!(!alike(one, other));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flat_picture_has_no_hash() {
|
||||
assert_eq!(difference_hash(64, 64, &[17; 64 * 64]), None);
|
||||
assert_eq!(difference_hash(0, 0, &[]), None);
|
||||
assert_eq!(difference_hash(4, 4, &[0; 3]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_picture_smaller_than_the_grid_still_hashes() {
|
||||
let luma: Vec<u8> = (0..4 * 3).map(|i| (i * 20) as u8).collect();
|
||||
assert!(difference_hash(4, 3, &luma).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alike_hashes_form_sets_and_catch_on() {
|
||||
let a = 0x0123_4567_89ab_cdef_u64;
|
||||
let hashes = [
|
||||
a,
|
||||
!a,
|
||||
a ^ 0b111,
|
||||
a ^ 0b111 ^ (0b1111 << 40),
|
||||
0x5555_0000_ffff_aaaa,
|
||||
];
|
||||
assert_eq!(sets(&hashes), vec![vec![0, 2, 3]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashes_apart_in_most_bytes_are_still_found() {
|
||||
// One bit apart in each of six bytes: six bits in all, and only the
|
||||
// top two bytes agree.
|
||||
let a = 1u64 << 63;
|
||||
let b = (0..6).fold(a, |h, byte| h | 1 << (byte * 8));
|
||||
assert_eq!(sets(&[a, b]), vec![vec![0, 1]]);
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,11 @@ use ignore::gitignore::{Gitignore, GitignoreBuilder};
|
||||
/// them. Version-control metadata, dependency and build trees, editor
|
||||
/// leftovers, and OS turds — none of which belong on Drive, all of which are
|
||||
/// large or rewritten constantly.
|
||||
///
|
||||
/// The second half is what Nextcloud, Dropbox and Syncthing leave out as well:
|
||||
/// files a desktop, a file system or a program in the middle of a write leaves
|
||||
/// behind, never anything a person made. Every one is in a test, next to a real
|
||||
/// file of a similar name that must stay synced.
|
||||
pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
|
||||
".git/",
|
||||
".hg/",
|
||||
@@ -43,6 +48,23 @@ pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
|
||||
"*.tmp",
|
||||
".DS_Store",
|
||||
"Thumbs.db",
|
||||
// Folder settings of Windows and KDE's Dolphin.
|
||||
"desktop.ini",
|
||||
".directory",
|
||||
// Trash folders and file-system leftovers: a file deleted while open on a
|
||||
// FUSE or NFS mount lives on under one of these names until it is closed.
|
||||
".Trash-*/",
|
||||
".Trashes/",
|
||||
".Spotlight-V100/",
|
||||
".fuse_hidden*",
|
||||
".nfs*",
|
||||
// Downloads and saves still being written.
|
||||
"*.part",
|
||||
"*.crdownload",
|
||||
"*.kate-swp",
|
||||
// Office lock files: LibreOffice's `.~lock.<name>#`, Microsoft Office's `~$<name>`.
|
||||
".~lock.*#",
|
||||
"~$*",
|
||||
];
|
||||
|
||||
/// Suffixes that mean a file is incomplete *by construction*: only the program
|
||||
@@ -119,6 +141,225 @@ pub fn is_scratch_name(name: &str) -> bool {
|
||||
/// Filenames accepted as a folder's ignore file, in precedence order.
|
||||
const IGNORE_FILE_NAMES: &[&str] = &[".pdfsignore", ".protonignore"];
|
||||
|
||||
/// Whether `line` is a rule [`IgnoreRules::load`] accepts: a pattern, a
|
||||
/// comment or a blank line. The error says what is wrong with it.
|
||||
pub fn check_pattern(line: &str) -> Result<(), String> {
|
||||
GitignoreBuilder::new("/")
|
||||
.add_line(None, line)
|
||||
.map(|_| ())
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// The lines of an ignore file that [`IgnoreRules::load`] would skip, by line
|
||||
/// number (from 1), with what is wrong with each.
|
||||
pub fn check_rules(text: &str) -> Vec<(usize, String)> {
|
||||
text.lines()
|
||||
.enumerate()
|
||||
.filter_map(|(i, line)| check_pattern(line).err().map(|e| (i + 1, e)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A synced folder's own ignore file, read so it can be edited and written
|
||||
/// back. See [`FolderRules::read`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FolderRules {
|
||||
/// The file the rules are in: the one in use, or `.pdfsignore` when there
|
||||
/// is none yet.
|
||||
pub path: std::path::PathBuf,
|
||||
/// The rules as read; empty when there is no file. `None` means there was
|
||||
/// no file, which [`write`](Self::write) checks is still so.
|
||||
pub text: Option<String>,
|
||||
}
|
||||
|
||||
/// Why [`FolderRules::write`] did not write.
|
||||
#[derive(Debug)]
|
||||
pub enum WriteRulesError {
|
||||
/// The file changed after it was read, for example by an edit on another
|
||||
/// computer that sync brought in. The rules on disk are the new ones.
|
||||
Changed,
|
||||
/// Reading or writing failed.
|
||||
Io(std::io::Error),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for WriteRulesError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Changed => f.write_str("the file changed since it was opened"),
|
||||
Self::Io(e) => e.fmt(f),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for WriteRulesError {}
|
||||
|
||||
impl From<std::io::Error> for WriteRulesError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Self::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl FolderRules {
|
||||
/// Read the ignore file of the folder at `root`, the same one
|
||||
/// [`IgnoreRules::load`] uses.
|
||||
pub fn read(root: &Path) -> std::io::Result<Self> {
|
||||
for name in IGNORE_FILE_NAMES {
|
||||
let path = root.join(name);
|
||||
match std::fs::read_to_string(&path) {
|
||||
Ok(text) => {
|
||||
return Ok(Self {
|
||||
path,
|
||||
text: Some(text),
|
||||
});
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
// Like `load`, a missing folder reads as one without rules; the write
|
||||
// is what fails then.
|
||||
Ok(Self {
|
||||
path: root.join(IGNORE_FILE_NAMES[0]),
|
||||
text: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Replace the rules with `text`, unless the file changed since it was
|
||||
/// read. Rules that are all blank remove the file, so a folder never keeps
|
||||
/// an empty one around.
|
||||
///
|
||||
/// Written to a temporary file and renamed over the old one, so the sync
|
||||
/// pass reading the rules at the same moment sees the old ones or the new
|
||||
/// ones, never half of them.
|
||||
pub fn write(&self, text: &str) -> Result<(), WriteRulesError> {
|
||||
let now = match std::fs::read_to_string(&self.path) {
|
||||
Ok(now) => Some(now),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
if now != self.text {
|
||||
return Err(WriteRulesError::Changed);
|
||||
}
|
||||
if text.trim().is_empty() {
|
||||
return match std::fs::remove_file(&self.path) {
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
|
||||
_ => Ok(()),
|
||||
};
|
||||
}
|
||||
let Some(name) = self.path.file_name() else {
|
||||
return Err(std::io::Error::other("the rules path has no file name").into());
|
||||
};
|
||||
let mut tmp_name = std::ffi::OsString::from(".");
|
||||
tmp_name.push(name);
|
||||
tmp_name.push(format!(".tmp-{}", std::process::id()));
|
||||
let tmp = self.path.with_file_name(tmp_name);
|
||||
let mut body = text.to_string();
|
||||
if !body.ends_with('\n') {
|
||||
body.push('\n');
|
||||
}
|
||||
let written = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(&tmp, &self.path));
|
||||
if written.is_err() {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
}
|
||||
Ok(written?)
|
||||
}
|
||||
|
||||
/// The patterns in the rules: every line that is not blank or a comment,
|
||||
/// trimmed of surrounding white space.
|
||||
pub fn patterns(&self) -> Vec<&str> {
|
||||
self.text
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|line| !line.is_empty() && !line.starts_with('#'))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// How often [`add_folder_pattern`] and [`remove_folder_pattern`] read the
|
||||
/// rules again when they changed between the read and the write.
|
||||
const RULES_RETRIES: usize = 3;
|
||||
|
||||
/// Add `pattern` at the end of the own rules of the folder at `root`, keeping
|
||||
/// every other line and comment. Says whether it added it: a pattern already
|
||||
/// there is left alone.
|
||||
pub fn add_folder_pattern(root: &Path, pattern: &str) -> std::io::Result<bool> {
|
||||
let pattern = pattern.trim();
|
||||
edit_folder_rules(root, |rules| {
|
||||
if rules.patterns().contains(&pattern) {
|
||||
return None;
|
||||
}
|
||||
let mut text = rules.text.clone().unwrap_or_default();
|
||||
if !text.is_empty() && !text.ends_with('\n') {
|
||||
text.push('\n');
|
||||
}
|
||||
text.push_str(pattern);
|
||||
text.push('\n');
|
||||
Some(text)
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove every line that is `pattern` from the own rules of the folder at
|
||||
/// `root`. Says whether there was one.
|
||||
pub fn remove_folder_pattern(root: &Path, pattern: &str) -> std::io::Result<bool> {
|
||||
let pattern = pattern.trim();
|
||||
edit_folder_rules(root, |rules| {
|
||||
if !rules.patterns().contains(&pattern) {
|
||||
return None;
|
||||
}
|
||||
let text = rules.text.as_deref().unwrap_or_default();
|
||||
Some(
|
||||
text.lines()
|
||||
.filter(|line| line.trim() != pattern)
|
||||
.map(|line| format!("{line}\n"))
|
||||
.collect(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the folder's rules, let `edit` return their new text (or `None` to
|
||||
/// leave them), and write it. Rules that changed in between, such as by sync
|
||||
/// bringing in an edit from another computer, are read again and edited anew.
|
||||
fn edit_folder_rules(
|
||||
root: &Path,
|
||||
edit: impl Fn(&FolderRules) -> Option<String>,
|
||||
) -> std::io::Result<bool> {
|
||||
for _ in 0..RULES_RETRIES {
|
||||
let rules = FolderRules::read(root)?;
|
||||
let Some(text) = edit(&rules) else {
|
||||
return Ok(false);
|
||||
};
|
||||
match rules.write(&text) {
|
||||
Ok(()) => return Ok(true),
|
||||
Err(WriteRulesError::Changed) => continue,
|
||||
Err(WriteRulesError::Io(e)) => return Err(e),
|
||||
}
|
||||
}
|
||||
Err(std::io::Error::other(
|
||||
"the rules kept changing while they were being saved",
|
||||
))
|
||||
}
|
||||
|
||||
/// Where a rule that decided about a path came from.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum RuleSource {
|
||||
/// The rules for every synced folder (`ignore_patterns`).
|
||||
AllFolders,
|
||||
/// The folder's own ignore file.
|
||||
ThisFolder,
|
||||
}
|
||||
|
||||
/// What the rules say about one path, and which rule says it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
/// No rule matches: the path is synced.
|
||||
Synced,
|
||||
/// Left out by `pattern`. `pattern` may name a folder the path is in.
|
||||
Excluded { pattern: String, source: RuleSource },
|
||||
/// A rule excluded it, and a later `!pattern` brought it back.
|
||||
Included { pattern: String, source: RuleSource },
|
||||
}
|
||||
|
||||
/// Compiled ignore rules for one synced folder.
|
||||
pub struct IgnoreRules {
|
||||
matcher: Option<Gitignore>,
|
||||
@@ -161,6 +402,27 @@ impl IgnoreRules {
|
||||
}
|
||||
}
|
||||
|
||||
/// [`load`](Self::load) with `text` standing in for the folder's ignore
|
||||
/// file, for showing what rules being edited would do before they are
|
||||
/// saved.
|
||||
pub fn with_folder_text(root: &Path, globals: &[String], text: &str) -> Self {
|
||||
let mut builder = GitignoreBuilder::new(root);
|
||||
let mut any = false;
|
||||
for pattern in globals {
|
||||
any |= builder.add_line(None, pattern).is_ok();
|
||||
}
|
||||
let file = root.join(IGNORE_FILE_NAMES[0]);
|
||||
for line in text.lines() {
|
||||
any |= builder.add_line(Some(file.clone()), line).is_ok();
|
||||
}
|
||||
if !any {
|
||||
return Self { matcher: None };
|
||||
}
|
||||
Self {
|
||||
matcher: builder.build().ok(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Rules matching nothing, for callers with no folder context.
|
||||
pub fn empty() -> Self {
|
||||
Self { matcher: None }
|
||||
@@ -207,6 +469,50 @@ impl IgnoreRules {
|
||||
}
|
||||
}
|
||||
|
||||
impl IgnoreRules {
|
||||
/// What the rules say about `rel`, and which rule decides, the way
|
||||
/// [`is_ignored`](Self::is_ignored) decides it.
|
||||
pub fn explain(&self, rel: &str, is_dir: bool) -> Verdict {
|
||||
let Some(matcher) = &self.matcher else {
|
||||
return Verdict::Synced;
|
||||
};
|
||||
let rel = rel.trim_matches('/');
|
||||
if rel.is_empty() {
|
||||
return Verdict::Synced;
|
||||
}
|
||||
let source = |glob: &ignore::gitignore::Glob| match glob.from() {
|
||||
Some(_) => RuleSource::ThisFolder,
|
||||
None => RuleSource::AllFolders,
|
||||
};
|
||||
let mut prefix = String::new();
|
||||
let mut parts = rel.split('/').peekable();
|
||||
while let Some(part) = parts.next() {
|
||||
if !prefix.is_empty() {
|
||||
prefix.push('/');
|
||||
}
|
||||
prefix.push_str(part);
|
||||
let last = parts.peek().is_none();
|
||||
let as_dir = if last { is_dir } else { true };
|
||||
match matcher.matched(&prefix, as_dir) {
|
||||
ignore::Match::Ignore(glob) => {
|
||||
return Verdict::Excluded {
|
||||
pattern: glob.original().to_string(),
|
||||
source: source(glob),
|
||||
};
|
||||
}
|
||||
ignore::Match::Whitelist(glob) if last => {
|
||||
return Verdict::Included {
|
||||
pattern: glob.original().to_string(),
|
||||
source: source(glob),
|
||||
};
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Verdict::Synced
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for IgnoreRules {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("IgnoreRules")
|
||||
@@ -255,6 +561,198 @@ mod tests {
|
||||
std::fs::write(root.join(name), body).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_added_defaults_leave_out_clutter_but_not_look_alikes() {
|
||||
let dir = TempDir::new();
|
||||
let rules = IgnoreRules::load(dir.path(), &globals());
|
||||
for (clutter, is_dir) in [
|
||||
("desktop.ini", false),
|
||||
("Photos/.directory", false),
|
||||
(".Trash-1000", true),
|
||||
(".Trashes", true),
|
||||
(".Spotlight-V100", true),
|
||||
(".fuse_hidden0000001a00000002", false),
|
||||
(".nfs000000000123abcd", false),
|
||||
("movie.mkv.part", false),
|
||||
("setup.exe.crdownload", false),
|
||||
("notes.txt.kate-swp", false),
|
||||
(".~lock.report.odt#", false),
|
||||
("~$budget.xlsx", false),
|
||||
] {
|
||||
assert!(rules.is_ignored(clutter, is_dir), "{clutter}");
|
||||
}
|
||||
for kept in [
|
||||
"Desktop/notes.txt",
|
||||
"directory.txt",
|
||||
"Trash-talk.md",
|
||||
"nfs-setup.md",
|
||||
"parts.csv",
|
||||
"part.txt",
|
||||
"report.odt",
|
||||
"budget.xlsx",
|
||||
] {
|
||||
assert!(!rules.is_ignored(kept, false), "{kept}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bad_pattern_is_reported_with_its_line() {
|
||||
assert!(check_pattern("*.db").is_ok());
|
||||
assert!(check_pattern("# a comment").is_ok());
|
||||
assert!(check_pattern("").is_ok());
|
||||
assert!(check_pattern("*.{jpg,png").is_err());
|
||||
assert!(check_pattern("[z-a]").is_err());
|
||||
let bad = check_rules("*.log\n{oops\nbuild/\n");
|
||||
assert_eq!(bad.len(), 1);
|
||||
assert_eq!(bad[0].0, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explain_names_the_rule_and_where_it_came_from() {
|
||||
let dir = TempDir::new();
|
||||
write(dir.path(), ".pdfsignore", "*.db\n!keep.tmp\n");
|
||||
let rules = IgnoreRules::load(dir.path(), &globals());
|
||||
assert_eq!(
|
||||
rules.explain("data/cache.db", false),
|
||||
Verdict::Excluded {
|
||||
pattern: "*.db".into(),
|
||||
source: RuleSource::ThisFolder
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
rules.explain("app/node_modules/x/index.js", false),
|
||||
Verdict::Excluded {
|
||||
pattern: "node_modules/".into(),
|
||||
source: RuleSource::AllFolders
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
rules.explain("keep.tmp", false),
|
||||
Verdict::Included {
|
||||
pattern: "!keep.tmp".into(),
|
||||
source: RuleSource::ThisFolder
|
||||
}
|
||||
);
|
||||
assert_eq!(rules.explain("src/main.rs", false), Verdict::Synced);
|
||||
// `explain` and `is_ignored` never disagree.
|
||||
for (path, is_dir) in [
|
||||
("data/cache.db", false),
|
||||
("keep.tmp", false),
|
||||
("x.tmp", false),
|
||||
] {
|
||||
assert_eq!(
|
||||
rules.is_ignored(path, is_dir),
|
||||
matches!(rules.explain(path, is_dir), Verdict::Excluded { .. }),
|
||||
"{path}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rules_being_edited_explain_like_the_saved_ones() {
|
||||
let dir = TempDir::new();
|
||||
let text = "*.db\n!keep.db\n";
|
||||
write(dir.path(), ".pdfsignore", text);
|
||||
let saved = IgnoreRules::load(dir.path(), &globals());
|
||||
let edited = IgnoreRules::with_folder_text(dir.path(), &globals(), text);
|
||||
for path in ["a.db", "keep.db", "node_modules/x", "src/x.rs"] {
|
||||
assert_eq!(
|
||||
saved.explain(path, false),
|
||||
edited.explain(path, false),
|
||||
"{path}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_rules_are_written_back_and_removed_when_blank() {
|
||||
let dir = TempDir::new();
|
||||
let rules = FolderRules::read(dir.path()).unwrap();
|
||||
assert_eq!(rules.text, None);
|
||||
assert_eq!(rules.path, dir.path().join(".pdfsignore"));
|
||||
rules.write("*.db").unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join(".pdfsignore")).unwrap(),
|
||||
"*.db\n"
|
||||
);
|
||||
assert!(IgnoreRules::load(dir.path(), &[]).is_ignored("x.db", false));
|
||||
|
||||
let rules = FolderRules::read(dir.path()).unwrap();
|
||||
rules.write(" \n").unwrap();
|
||||
assert!(!dir.path().join(".pdfsignore").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_rules_keep_the_file_name_in_use() {
|
||||
let dir = TempDir::new();
|
||||
write(dir.path(), ".protonignore", "old/\n");
|
||||
let rules = FolderRules::read(dir.path()).unwrap();
|
||||
assert_eq!(rules.path, dir.path().join(".protonignore"));
|
||||
rules.write("new/\n").unwrap();
|
||||
assert!(!dir.path().join(".pdfsignore").exists());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join(".protonignore")).unwrap(),
|
||||
"new/\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_rules_changed_elsewhere_are_not_written_over() {
|
||||
let dir = TempDir::new();
|
||||
write(dir.path(), ".pdfsignore", "a/\n");
|
||||
let rules = FolderRules::read(dir.path()).unwrap();
|
||||
write(dir.path(), ".pdfsignore", "b/\n");
|
||||
assert!(matches!(rules.write("c/"), Err(WriteRulesError::Changed)));
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join(".pdfsignore")).unwrap(),
|
||||
"b/\n"
|
||||
);
|
||||
|
||||
// A file that appeared after an empty read counts as a change too.
|
||||
let dir = TempDir::new();
|
||||
let rules = FolderRules::read(dir.path()).unwrap();
|
||||
write(dir.path(), ".pdfsignore", "b/\n");
|
||||
assert!(matches!(rules.write("c/"), Err(WriteRulesError::Changed)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pattern_added_to_a_folder_keeps_its_other_lines() {
|
||||
let dir = TempDir::new();
|
||||
assert!(add_folder_pattern(dir.path(), " *.db ").unwrap());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join(".pdfsignore")).unwrap(),
|
||||
"*.db\n"
|
||||
);
|
||||
|
||||
write(dir.path(), ".pdfsignore", "# my notes\nbuild/");
|
||||
assert!(add_folder_pattern(dir.path(), "*.db").unwrap());
|
||||
assert!(!add_folder_pattern(dir.path(), "build/").unwrap());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join(".pdfsignore")).unwrap(),
|
||||
"# my notes\nbuild/\n*.db\n"
|
||||
);
|
||||
assert_eq!(
|
||||
FolderRules::read(dir.path()).unwrap().patterns(),
|
||||
["build/", "*.db"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn removing_a_folder_pattern_keeps_comments_and_drops_an_empty_file() {
|
||||
let dir = TempDir::new();
|
||||
write(dir.path(), ".pdfsignore", "# keep\nbuild/\n*.db\n");
|
||||
assert!(remove_folder_pattern(dir.path(), "build/").unwrap());
|
||||
assert!(!remove_folder_pattern(dir.path(), "build/").unwrap());
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(dir.path().join(".pdfsignore")).unwrap(),
|
||||
"# keep\n*.db\n"
|
||||
);
|
||||
|
||||
write(dir.path(), ".pdfsignore", "*.db\n");
|
||||
assert!(remove_folder_pattern(dir.path(), "*.db").unwrap());
|
||||
assert!(!dir.path().join(".pdfsignore").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_rules_at_all_ignores_nothing() {
|
||||
let dir = TempDir::new();
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
//! Which local folders may become a sync root.
|
||||
//!
|
||||
//! A mirror uploads everything under its root and deletes what Drive deleted,
|
||||
//! so a root that holds more than the user meant is dangerous. `$HOME` takes
|
||||
//! the decrypted content cache, `cache.db` and `~/.ssh` up to Drive, and walks
|
||||
//! into the mount, which downloads the whole Drive only to upload it again.
|
||||
//! Every place a root is added or restored goes through [`problem`].
|
||||
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use crate::config::{AppConfig, AppDirs};
|
||||
|
||||
/// The paths a sync root must stay clear of.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Reserved {
|
||||
/// The user's home folder. A root may sit inside it, not be it or above it.
|
||||
pub home: Option<PathBuf>,
|
||||
/// The Proton Drive mount.
|
||||
pub mountpoint: Option<PathBuf>,
|
||||
/// This app's own configuration, state and cache folders.
|
||||
pub app_dirs: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
impl Reserved {
|
||||
/// The reserved paths of this user and `config`, resolved like a root.
|
||||
pub fn current(config: &AppConfig) -> Self {
|
||||
let Ok(dirs) = AppDirs::new() else {
|
||||
return Self::default();
|
||||
};
|
||||
let resolved = |path: PathBuf| resolve(&path).unwrap_or(path);
|
||||
let mut app_dirs = vec![dirs.state_dir(), dirs.cache_dir()];
|
||||
if let Some(config_dir) = dirs.config_path().parent() {
|
||||
app_dirs.push(config_dir.to_path_buf());
|
||||
}
|
||||
Self {
|
||||
home: dirs.home_dir().map(resolved),
|
||||
mountpoint: Some(resolved(dirs.resolved_mountpoint(config))),
|
||||
app_dirs: app_dirs.into_iter().map(resolved).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `path` as a root is compared: absolute, with symlinks resolved as far as the
|
||||
/// path exists. A folder a restore is about to create does not exist yet, so
|
||||
/// the part past the deepest existing folder is kept as written.
|
||||
///
|
||||
/// A `..` is refused rather than resolved: before a symlink it means something
|
||||
/// else than after it, and a path recorded on another machine has no business
|
||||
/// holding one.
|
||||
pub fn resolve(path: &Path) -> Result<PathBuf, String> {
|
||||
if !path.is_absolute() {
|
||||
return Err(format!("{} is not an absolute path", path.display()));
|
||||
}
|
||||
if path.components().any(|c| c == Component::ParentDir) {
|
||||
return Err(format!("{} goes up with `..`", path.display()));
|
||||
}
|
||||
let mut existing = path.to_path_buf();
|
||||
let mut rest = Vec::new();
|
||||
loop {
|
||||
match existing.canonicalize() {
|
||||
Ok(canonical) => {
|
||||
return Ok(rest
|
||||
.into_iter()
|
||||
.rev()
|
||||
.fold(canonical, |path: PathBuf, part| path.join(part)));
|
||||
}
|
||||
Err(_) => {
|
||||
let Some(name) = existing.file_name().map(|n| n.to_os_string()) else {
|
||||
return Ok(path.to_path_buf());
|
||||
};
|
||||
rest.push(name);
|
||||
if !existing.pop() {
|
||||
return Ok(path.to_path_buf());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why `root` cannot be synced, or `None` when it can. `root` is resolved
|
||||
/// ([`resolve`]); `others` are the roots already synced.
|
||||
pub fn problem(root: &Path, others: &[PathBuf], reserved: &Reserved) -> Option<String> {
|
||||
let shown = root.display();
|
||||
if root.parent().is_none() {
|
||||
return Some(format!("{shown} is the root of the file system"));
|
||||
}
|
||||
if let Some(home) = &reserved.home {
|
||||
if root == home {
|
||||
return Some(format!(
|
||||
"{shown} is your home folder; pick a folder inside it"
|
||||
));
|
||||
}
|
||||
if home.starts_with(root) {
|
||||
return Some(format!("{shown} holds your home folder"));
|
||||
}
|
||||
}
|
||||
if let Some(mountpoint) = &reserved.mountpoint
|
||||
&& overlaps(root, mountpoint)
|
||||
{
|
||||
return Some(format!(
|
||||
"{shown} overlaps the Proton Drive folder {}",
|
||||
mountpoint.display()
|
||||
));
|
||||
}
|
||||
if let Some(dir) = reserved.app_dirs.iter().find(|dir| overlaps(root, dir)) {
|
||||
return Some(format!(
|
||||
"{shown} overlaps this app's own data in {}",
|
||||
dir.display()
|
||||
));
|
||||
}
|
||||
if others.iter().any(|other| other == root) {
|
||||
return Some(format!("{shown} is already synced"));
|
||||
}
|
||||
if let Some(other) = others.iter().find(|other| overlaps(root, other)) {
|
||||
return Some(format!(
|
||||
"{shown} overlaps the synced folder {}",
|
||||
other.display()
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether one of `a` and `b` is the other or inside it.
|
||||
fn overlaps(a: &Path, b: &Path) -> bool {
|
||||
a.starts_with(b) || b.starts_with(a)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn reserved() -> Reserved {
|
||||
Reserved {
|
||||
home: Some("/home/u".into()),
|
||||
mountpoint: Some("/home/u/ProtonDrive".into()),
|
||||
app_dirs: vec![
|
||||
"/home/u/.local/state/proton-drive-linux".into(),
|
||||
"/home/u/.cache/proton-drive-linux".into(),
|
||||
"/home/u/.config/proton-drive-linux".into(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn refused(root: &str, others: &[&str]) -> bool {
|
||||
let others: Vec<PathBuf> = others.iter().map(PathBuf::from).collect();
|
||||
problem(Path::new(root), &others, &reserved()).is_some()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_inside_home_can_be_synced() {
|
||||
assert!(!refused("/home/u/Documents", &[]));
|
||||
assert!(!refused("/home/u/Pictures", &["/home/u/Documents"]));
|
||||
assert!(!refused("/mnt/data/photos", &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_file_system_root_home_and_its_parents_are_refused() {
|
||||
assert!(refused("/", &[]));
|
||||
assert!(refused("/home/u", &[]));
|
||||
assert!(refused("/home", &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_mount_and_the_apps_own_data_are_refused_either_way_round() {
|
||||
assert!(refused("/home/u/ProtonDrive", &[]));
|
||||
assert!(refused("/home/u/ProtonDrive/Documents", &[]));
|
||||
assert!(refused("/home/u/.cache", &[]));
|
||||
assert!(refused(
|
||||
"/home/u/.local/state/proton-drive-linux/staging",
|
||||
&[]
|
||||
));
|
||||
assert!(refused("/home/u/.config", &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_root_inside_or_around_another_root_is_refused() {
|
||||
assert!(refused("/home/u/Documents", &["/home/u/Documents"]));
|
||||
assert!(refused("/home/u/Documents/Work", &["/home/u/Documents"]));
|
||||
assert!(refused("/home/u/stuff", &["/home/u/stuff/Documents"]));
|
||||
// A shared prefix of characters is not a shared folder.
|
||||
assert!(!refused("/home/u/Documents2", &["/home/u/Documents"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_path_that_goes_up_or_is_relative_does_not_resolve() {
|
||||
assert!(resolve(Path::new("/home/u/../../etc")).is_err());
|
||||
assert!(resolve(Path::new("Documents")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_still_to_be_created_resolves_under_its_existing_parent() {
|
||||
let base = std::env::temp_dir().join(format!("pdfs-syncroot-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
let canonical = base.canonicalize().unwrap();
|
||||
assert_eq!(
|
||||
resolve(&base.join("new/inner")).unwrap(),
|
||||
canonical.join("new/inner")
|
||||
);
|
||||
let link = base.join("link");
|
||||
let _ = std::fs::remove_file(&link);
|
||||
std::os::unix::fs::symlink(&canonical, &link).unwrap();
|
||||
assert_eq!(
|
||||
resolve(&link.join("later")).unwrap(),
|
||||
canonical.join("later"),
|
||||
"a symlink is followed to where it points"
|
||||
);
|
||||
std::fs::remove_dir_all(&base).unwrap();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,33 @@
|
||||
//! Convert the client's stored epoch seconds at the SDK boundary.
|
||||
|
||||
use proton_drive_rs::ModificationTime;
|
||||
|
||||
/// A UTC modification time, or `None` when outside the SDK's supported range.
|
||||
pub fn modification_time(seconds: i64) -> Option<ModificationTime> {
|
||||
chrono::DateTime::from_timestamp(seconds, 0).map(|time| time.fixed_offset())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn epoch_seconds_preserve_the_instant_including_before_1970() {
|
||||
for seconds in [-1, 0, 1_709_208_000, 1_782_482_460] {
|
||||
let time = modification_time(seconds).unwrap();
|
||||
assert_eq!(time.timestamp(), seconds);
|
||||
assert_eq!(time.offset().local_minus_utc(), 0);
|
||||
assert_eq!(time.timestamp_subsec_nanos(), 0);
|
||||
}
|
||||
assert_eq!(
|
||||
modification_time(-1).unwrap().to_rfc3339(),
|
||||
"1969-12-31T23:59:59+00:00"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_range_seconds_omit_the_timestamp() {
|
||||
assert!(modification_time(i64::MIN).is_none());
|
||||
assert!(modification_time(i64::MAX).is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
//! The opt-in check for a newer release ([`crate::config::AppConfig::update_check`]).
|
||||
//!
|
||||
//! Only the decisions live here: what GitHub's answer says, which version is
|
||||
//! newer, when to ask again, and whether a package manager already owns
|
||||
//! updates. The desktop app does the request itself.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
/// GitHub's latest published release. Drafts and pre-releases never show up
|
||||
/// here, so a tagged release candidate does not reach anyone.
|
||||
pub const RELEASES_LATEST_API: &str =
|
||||
"https://api.github.com/repos/narrrl/proton-drive-linux/releases/latest";
|
||||
|
||||
/// How often the app asks, across restarts.
|
||||
pub const CHECK_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
|
||||
/// Installed next to the binaries' `share/` by packages whose package manager
|
||||
/// brings new releases (COPR, the AUR, Nix). The release `.deb`, `.rpm` and
|
||||
/// `.tar.gz` do not ship it, since nothing updates them.
|
||||
pub const MANAGED_MARKER: &str = "share/proton-drive-linux/managed-updates";
|
||||
|
||||
/// The version of the release in a `/releases/latest` answer, without the
|
||||
/// tag's `v`. `None` for an answer that does not parse or a tag that is not
|
||||
/// `X.Y.Z`, so a changed API can never claim an update.
|
||||
pub fn latest_version(body: &[u8]) -> Option<String> {
|
||||
let json: serde_json::Value = serde_json::from_slice(body).ok()?;
|
||||
let tag = json.get("tag_name")?.as_str()?;
|
||||
let version = tag.strip_prefix('v').unwrap_or(tag);
|
||||
parse(version).map(|_| version.to_owned())
|
||||
}
|
||||
|
||||
/// Whether `version` is a later release than `current`. Anything that is not
|
||||
/// `X.Y.Z` on either side is not.
|
||||
pub fn is_newer(version: &str, current: &str) -> bool {
|
||||
matches!((parse(version), parse(current)), (Some(v), Some(c)) if v > c)
|
||||
}
|
||||
|
||||
/// The release page of `version` on GitHub.
|
||||
pub fn release_url(version: &str) -> String {
|
||||
format!("https://github.com/narrrl/proton-drive-linux/releases/tag/v{version}")
|
||||
}
|
||||
|
||||
/// `X.Y.Z` as numbers, so `3.10.0` sorts after `3.9.0`.
|
||||
fn parse(version: &str) -> Option<(u64, u64, u64)> {
|
||||
let mut parts = version.split('.').map(|p| p.parse::<u64>().ok());
|
||||
let triple = (parts.next()??, parts.next()??, parts.next()??);
|
||||
parts.next().is_none().then_some(triple)
|
||||
}
|
||||
|
||||
/// How long until the next check is due, given the last one in Unix seconds.
|
||||
/// Zero when it is due now. A clock set back past the last check counts as
|
||||
/// due, so a wrong clock cannot stop checks.
|
||||
pub fn next_check_in(last_checked: Option<i64>, now: i64) -> Duration {
|
||||
let interval = CHECK_INTERVAL.as_secs() as i64;
|
||||
match last_checked {
|
||||
Some(last) if (0..interval).contains(&(now - last)) => {
|
||||
Duration::from_secs((interval - (now - last)) as u64)
|
||||
}
|
||||
_ => Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the binary at `exe` was installed by a package manager that updates
|
||||
/// it, which [`MANAGED_MARKER`] next to its `bin/` says.
|
||||
pub fn updates_managed(exe: &Path) -> bool {
|
||||
exe.parent()
|
||||
.and_then(Path::parent)
|
||||
.is_some_and(|prefix| prefix.join(MANAGED_MARKER).exists())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn latest(tag: &str) -> Vec<u8> {
|
||||
format!(
|
||||
r#"{{"tag_name":"{tag}","html_url":"https://github.com/narrrl/proton-drive-linux/releases/tag/{tag}","draft":false}}"#
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_latest_release_is_read_without_its_v() {
|
||||
assert_eq!(latest_version(&latest("v3.6.0")).as_deref(), Some("3.6.0"));
|
||||
assert_eq!(latest_version(&latest("3.6.0")).as_deref(), Some("3.6.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_that_does_not_parse_has_no_version() {
|
||||
assert_eq!(latest_version(b"not json"), None);
|
||||
assert_eq!(latest_version(b"{}"), None);
|
||||
assert_eq!(latest_version(br#"{"tag_name":7}"#), None);
|
||||
assert_eq!(latest_version(&latest("nightly")), None);
|
||||
assert_eq!(latest_version(&latest("v4.0.0-rc1")), None);
|
||||
assert_eq!(latest_version(&latest("v4.0")), None);
|
||||
assert_eq!(latest_version(&latest("v4.0.0.1")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_later_release_is_newer() {
|
||||
assert!(is_newer("3.6.0", "3.5.0"));
|
||||
assert!(!is_newer("3.5.0", "3.5.0"));
|
||||
assert!(!is_newer("3.4.1", "3.5.0"));
|
||||
assert!(!is_newer("2.99.99", "3.0.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn versions_compare_as_numbers() {
|
||||
assert!(is_newer("3.10.0", "3.9.0"));
|
||||
assert!(is_newer("3.5.10", "3.5.9"));
|
||||
assert!(is_newer("4.0.0", "3.99.99"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_version_that_is_not_x_y_z_is_never_newer() {
|
||||
assert!(!is_newer("4.0.0", "3.5.0-dev"));
|
||||
assert!(!is_newer("garbage", "3.5.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_release_page_uses_the_tag() {
|
||||
assert_eq!(
|
||||
release_url("3.6.0"),
|
||||
"https://github.com/narrrl/proton-drive-linux/releases/tag/v3.6.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_check_is_due_once_a_day() {
|
||||
let day = CHECK_INTERVAL.as_secs() as i64;
|
||||
assert_eq!(next_check_in(None, 1_000), Duration::ZERO);
|
||||
assert_eq!(next_check_in(Some(1_000), 1_000), CHECK_INTERVAL);
|
||||
assert_eq!(
|
||||
next_check_in(Some(1_000), 1_000 + day - 1),
|
||||
Duration::from_secs(1)
|
||||
);
|
||||
assert_eq!(next_check_in(Some(1_000), 1_000 + day), Duration::ZERO);
|
||||
assert_eq!(
|
||||
next_check_in(Some(1_000), 999),
|
||||
Duration::ZERO,
|
||||
"a clock set back must not stop checks"
|
||||
);
|
||||
}
|
||||
|
||||
/// A unique temp directory removed on drop; avoids a dev-dependency, as in
|
||||
/// [`crate::syncignore`]'s tests.
|
||||
struct TempDir(std::path::PathBuf);
|
||||
impl TempDir {
|
||||
fn new() -> Self {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static N: AtomicU64 = AtomicU64::new(0);
|
||||
let p = std::env::temp_dir().join(format!(
|
||||
"pdfs-update-{}-{}",
|
||||
std::process::id(),
|
||||
N.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
TempDir(p)
|
||||
}
|
||||
}
|
||||
impl Drop for TempDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_marker_next_to_bin_means_a_package_manager_updates() {
|
||||
let prefix = TempDir::new();
|
||||
let exe = prefix.0.join("bin/pdfs-app");
|
||||
assert!(!updates_managed(&exe));
|
||||
let marker = prefix.0.join(MANAGED_MARKER);
|
||||
std::fs::create_dir_all(marker.parent().unwrap()).unwrap();
|
||||
std::fs::write(&marker, "").unwrap();
|
||||
assert!(updates_managed(&exe));
|
||||
}
|
||||
}
|
||||
@@ -14,15 +14,25 @@ pdfs-core.workspace = true
|
||||
zip.workspace = true
|
||||
sha1.workspace = true
|
||||
proton-drive-rs.workspace = true
|
||||
proton-gallery-app-rs.workspace = true
|
||||
proton-gallery-rs.workspace = true
|
||||
hyper = { version = "1", features = ["http1", "server"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio"] }
|
||||
http-body-util = "0.1"
|
||||
getrandom = "0.4"
|
||||
zeroize.workspace = true
|
||||
futures.workspace = true
|
||||
async-trait.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
fuser.workspace = true
|
||||
libc.workspace = true
|
||||
rustix = { version = "1", features = ["net", "process"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
base64.workspace = true
|
||||
image.workspace = true
|
||||
lofty.workspace = true
|
||||
notify = "8"
|
||||
parking_lot.workspace = true
|
||||
|
||||
|
||||
+111
-92
@@ -12,7 +12,7 @@
|
||||
//! than looked up in our own timeline, where a shared album's photos never
|
||||
//! appear.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use pdfs_core::control::{AlbumInfo, PhotoItem, Topic};
|
||||
@@ -24,8 +24,8 @@ use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use tracing::warn;
|
||||
|
||||
use super::{
|
||||
ALBUM_SYNCED_PREFIX, ALBUMS_SYNCED_MS, Core, PHOTOS_AVAILABLE, PHOTOS_SYNCED_MS,
|
||||
TIMELINE_ENRICH_CHUNK, TIMELINE_TTL, now_ms, parse_uid,
|
||||
ALBUM_SYNCED_PREFIX, ALBUMS_SYNCED_MS, Core, PHOTOS_AVAILABLE, TIMELINE_ENRICH_CHUNK,
|
||||
TIMELINE_TTL, link, now_ms, parse_uid,
|
||||
};
|
||||
|
||||
impl Core {
|
||||
@@ -34,7 +34,13 @@ impl Core {
|
||||
/// [`Core::photos_timeline`] gives.
|
||||
pub(crate) fn albums(&self) -> CoreResult<Option<Vec<AlbumInfo>>> {
|
||||
let count = self.db.albums_count().map_err(CoreError::from)?;
|
||||
if count == 0 {
|
||||
if count == 0
|
||||
&& self
|
||||
.db
|
||||
.state_i64(ALBUMS_SYNCED_MS)
|
||||
.map_err(CoreError::from)?
|
||||
.is_none()
|
||||
{
|
||||
// Nothing on disk, so this one request waits for the fetch — unless
|
||||
// the account is already known to have no photos volume.
|
||||
let known_empty = self.db.state_i64(PHOTOS_AVAILABLE).ok().flatten() == Some(0);
|
||||
@@ -56,77 +62,71 @@ impl Core {
|
||||
/// the persisted listing. Returns false when the account has no photos
|
||||
/// volume.
|
||||
pub(crate) async fn refresh_albums(&self) -> CoreResult<bool> {
|
||||
let generation = self
|
||||
.db
|
||||
.photo_projection_generation()
|
||||
.map_err(CoreError::from)?;
|
||||
let photos = self.photos();
|
||||
if photos
|
||||
.get_photos_root()
|
||||
if link::bounded(photos.get_photos_root())
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "photos root"))?
|
||||
.is_none()
|
||||
{
|
||||
let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 0);
|
||||
let _ = self.db.set_state_i64(PHOTOS_SYNCED_MS, now_ms());
|
||||
let _ = self.db.set_state_i64(ALBUMS_SYNCED_MS, now_ms());
|
||||
self.db
|
||||
.albums_replace_at(&[], generation, now_ms())
|
||||
.map_err(CoreError::from)?;
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut uids = photos
|
||||
.enumerate_album_node_uids()
|
||||
let mut uids = link::bounded(photos.enumerate_album_node_uids())
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "albums"))?;
|
||||
let shared = link::bounded(photos.enumerate_shared_with_me_album_uids())
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "shared albums"))?;
|
||||
let shared_keys: HashSet<_> = shared.iter().map(ToString::to_string).collect();
|
||||
uids.extend(shared);
|
||||
let mut unique = HashSet::new();
|
||||
uids.retain(|uid| unique.insert(uid.clone()));
|
||||
|
||||
// Shared-with-me albums are a separate listing on the sharer's volume.
|
||||
// Best-effort: an account whose sharing listing fails still gets its own
|
||||
// albums rather than an error page.
|
||||
let shared: HashSet<String> = match photos.enumerate_shared_with_me_album_uids().await {
|
||||
Ok(shared) => {
|
||||
let keys = shared.iter().map(|u| u.to_string()).collect();
|
||||
uids.extend(shared);
|
||||
keys
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "enumerating shared-with-me albums failed");
|
||||
HashSet::new()
|
||||
}
|
||||
};
|
||||
|
||||
// The listings carry uids only; the name, photo count and cover come from
|
||||
// the album nodes themselves. A chunk that fails to resolve drops those
|
||||
// albums from this refresh rather than failing the whole listing.
|
||||
let mut albums: Vec<StoredAlbum> = Vec::with_capacity(uids.len());
|
||||
let mut albums = Vec::with_capacity(uids.len());
|
||||
for chunk in uids.chunks(TIMELINE_ENRICH_CHUNK) {
|
||||
match photos.enumerate_nodes(chunk).await {
|
||||
Ok(nodes) => {
|
||||
for node in nodes {
|
||||
let key = node.uid.to_string();
|
||||
let props = node.album.as_ref();
|
||||
albums.push(StoredAlbum {
|
||||
shared: shared.contains(&key),
|
||||
uid: key,
|
||||
name: node.name,
|
||||
photo_count: props.map_or(0, |a| a.photo_count.max(0) as usize),
|
||||
cover_uid: props
|
||||
.and_then(|a| a.cover_photo_uid.as_ref())
|
||||
.map(|u| u.to_string()),
|
||||
last_activity: props.and_then(|a| a.last_activity_time),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(error = %e, "resolving an album chunk failed"),
|
||||
let nodes = link::bounded(photos.enumerate_nodes(chunk))
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "album metadata"))?;
|
||||
if nodes.len() != chunk.len() {
|
||||
return Err(CoreError::remote(
|
||||
"Some album details are still unavailable.",
|
||||
));
|
||||
}
|
||||
for node in nodes.into_iter().filter(|node| !node.trashed) {
|
||||
let uid = node.uid.to_string();
|
||||
let props = node.album.as_ref();
|
||||
albums.push(StoredAlbum {
|
||||
shared: shared_keys.contains(&uid),
|
||||
uid,
|
||||
name: node.name,
|
||||
photo_count: props.map_or(0, |a| a.photo_count.max(0) as usize),
|
||||
cover_uid: props
|
||||
.and_then(|a| a.cover_photo_uid.as_ref())
|
||||
.map(ToString::to_string),
|
||||
last_activity: props.and_then(|a| a.last_activity_time),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Newest activity first, and albums the server gave no activity time for
|
||||
// last — an album nobody has touched belongs below the ones in use. Ties
|
||||
// fall back to the name so the order is stable between refreshes.
|
||||
albums.sort_by(|a, b| {
|
||||
b.last_activity
|
||||
.cmp(&a.last_activity)
|
||||
.then_with(|| a.name.to_lowercase().cmp(&b.name.to_lowercase()))
|
||||
});
|
||||
|
||||
self.db.albums_replace(&albums).map_err(CoreError::from)?;
|
||||
let _ = self.db.set_state_i64(PHOTOS_AVAILABLE, 1);
|
||||
let _ = self.db.set_state_i64(ALBUMS_SYNCED_MS, now_ms());
|
||||
if !self
|
||||
.db
|
||||
.albums_replace_at(&albums, generation, now_ms())
|
||||
.map_err(CoreError::from)?
|
||||
{
|
||||
return Err(CoreError::remote(
|
||||
"Photos changed during the album refresh. Try again shortly.",
|
||||
));
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -154,11 +154,21 @@ impl Core {
|
||||
album: &NodeUid,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
excluded: &str,
|
||||
) -> CoreResult<Vec<PhotoItem>> {
|
||||
let key = album.to_string();
|
||||
if self.db.photo_removed(&key).map_err(CoreError::from)? {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let stamp = format!("{ALBUM_SYNCED_PREFIX}{key}");
|
||||
let count = self.db.album_photos_count(&key).map_err(CoreError::from)?;
|
||||
if count == 0 {
|
||||
if count == 0
|
||||
&& self
|
||||
.db
|
||||
.state_i64(&stamp)
|
||||
.map_err(CoreError::from)?
|
||||
.is_none()
|
||||
{
|
||||
self.rt.block_on(self.refresh_album(album))?;
|
||||
} else if self.listing_stale(&stamp, TIMELINE_TTL) {
|
||||
let core = self.clone();
|
||||
@@ -172,7 +182,7 @@ impl Core {
|
||||
|
||||
let page = self
|
||||
.db
|
||||
.album_photos_page(&key, offset, limit)
|
||||
.album_photos_page_visible(&key, offset, limit, excluded)
|
||||
.map_err(CoreError::from)?;
|
||||
Ok(page.into_iter().map(|p| self.photo_item(p)).collect())
|
||||
}
|
||||
@@ -213,47 +223,56 @@ impl Core {
|
||||
|
||||
/// Re-enumerate one album's photos and replace its persisted contents.
|
||||
pub(crate) async fn refresh_album(&self, album: &NodeUid) -> CoreResult<()> {
|
||||
let generation = self
|
||||
.db
|
||||
.photo_projection_generation()
|
||||
.map_err(CoreError::from)?;
|
||||
let photos = self.photos();
|
||||
let items = photos
|
||||
.enumerate_album(album)
|
||||
let items = link::bounded(photos.enumerate_album(album))
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "album"))?;
|
||||
|
||||
// Same enrichment as the timeline: the listing carries a uid and a
|
||||
// capture time, and the gallery needs a name and media type to classify
|
||||
// each entry. Best-effort per chunk.
|
||||
let uids: Vec<NodeUid> = items.iter().map(|it| it.uid.clone()).collect();
|
||||
let mut meta: HashMap<String, (Option<String>, Option<String>)> = HashMap::new();
|
||||
let key = album.to_string();
|
||||
let mut meta = self
|
||||
.db
|
||||
.album_photo_metadata(&key)
|
||||
.map_err(CoreError::from)?;
|
||||
let uids: Vec<NodeUid> = items
|
||||
.iter()
|
||||
.filter(|item| !meta.contains_key(&item.uid.to_string()))
|
||||
.map(|item| item.uid.clone())
|
||||
.collect();
|
||||
for chunk in uids.chunks(TIMELINE_ENRICH_CHUNK) {
|
||||
match photos.enumerate_nodes(chunk).await {
|
||||
Ok(nodes) => {
|
||||
for node in nodes {
|
||||
let media_type = match &node.kind {
|
||||
NodeKind::File { media_type, .. } => Some(media_type.clone()),
|
||||
NodeKind::Folder => None,
|
||||
};
|
||||
meta.insert(node.uid.to_string(), (Some(node.name), media_type));
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(error = %e, "resolving album photo metadata failed"),
|
||||
let nodes = link::bounded(photos.enumerate_nodes(chunk))
|
||||
.await
|
||||
.map_err(|e| CoreError::from_api(&e, "album photo metadata"))?;
|
||||
for node in nodes.into_iter().filter(|node| !node.trashed) {
|
||||
let media_type = match node.kind {
|
||||
NodeKind::File { media_type, .. } => Some(media_type),
|
||||
NodeKind::Folder => None,
|
||||
};
|
||||
meta.insert(node.uid.to_string(), (node.name, media_type));
|
||||
}
|
||||
}
|
||||
|
||||
let rows: Vec<(String, i64, Option<String>, Option<String>)> = items
|
||||
.iter()
|
||||
.map(|it| {
|
||||
let key = it.uid.to_string();
|
||||
let (name, media_type) = meta.get(&key).cloned().unwrap_or((None, None));
|
||||
(key, it.capture_time, name, media_type)
|
||||
let rows: Vec<_> = items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let uid = item.uid.to_string();
|
||||
let (name, media_type) = meta
|
||||
.get(&uid)
|
||||
.map(|(name, mime)| (Some(name.clone()), mime.clone()))
|
||||
.unwrap_or_default();
|
||||
(uid, item.capture_time, name, media_type)
|
||||
})
|
||||
.collect();
|
||||
let key = album.to_string();
|
||||
self.db
|
||||
.album_photos_replace(&key, &rows)
|
||||
.map_err(CoreError::from)?;
|
||||
let _ = self
|
||||
if !self
|
||||
.db
|
||||
.set_state_i64(&format!("{ALBUM_SYNCED_PREFIX}{key}"), now_ms());
|
||||
.album_photos_replace_at(&key, &rows, generation, now_ms())
|
||||
.map_err(CoreError::from)?
|
||||
{
|
||||
return Err(CoreError::remote(
|
||||
"Photos changed during the album refresh. Try again shortly.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -283,7 +302,7 @@ impl Core {
|
||||
.block_on(self.photos().delete_album(album, delete_photos))
|
||||
.map_err(|e| CoreError::from_api(&e, "delete album"))?;
|
||||
let key = album.to_string();
|
||||
if let Err(error) = self.db.album_photos_replace(&key, &[]) {
|
||||
if let Err(error) = self.db.photos_delete(&[key]) {
|
||||
warn!(%album, %error, "a deleted album's photos could not be dropped");
|
||||
}
|
||||
self.after_album_change(None);
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Explicitly bind an existing My files folder to a real local mirror.
|
||||
//!
|
||||
//! Unlike a FUSE mount, this never covers or evicts the local directory.
|
||||
//! The existing three-way engine performs the initial, non-destructive union.
|
||||
|
||||
use super::{Core, parse_uid, sync};
|
||||
use pdfs_core::db::StoredSyncFolder;
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
impl Core {
|
||||
/// A bounded, server-verified ancestry walk. Do not decide ownership or
|
||||
/// remote-root overlap from a possibly incomplete cached folder listing.
|
||||
pub(super) fn attach_ancestors(&self, start: &NodeUid) -> CoreResult<Vec<NodeUid>> {
|
||||
let mut ancestors = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut next = Some(start.clone());
|
||||
while let Some(uid) = next {
|
||||
if !seen.insert(uid.clone()) || ancestors.len() >= 256 {
|
||||
return Err(CoreError::invalid(
|
||||
"invalid or excessively deep remote folder ancestry",
|
||||
));
|
||||
}
|
||||
ancestors.push(uid.clone());
|
||||
if uid == self.primary_root_uid {
|
||||
break;
|
||||
}
|
||||
let nodes = self
|
||||
.rt
|
||||
.block_on(self.drive.enumerate_nodes(std::slice::from_ref(&uid)))
|
||||
.map_err(|e| CoreError::from_api(&e, "verify sync folder ancestry"))?;
|
||||
let node = nodes
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| CoreError::not_found("remote sync folder is missing"))?;
|
||||
if node.trashed {
|
||||
return Err(CoreError::invalid("remote sync folder is in the trash"));
|
||||
}
|
||||
next = node.parent_uid;
|
||||
}
|
||||
Ok(ancestors)
|
||||
}
|
||||
|
||||
pub(crate) fn attach_sync_folder(
|
||||
&self,
|
||||
local: &Path,
|
||||
remote: &Path,
|
||||
mountpoint: &Path,
|
||||
) -> CoreResult<StoredSyncFolder> {
|
||||
let _registration = super::devices::SYNC_ROOT_REGISTRATION.lock();
|
||||
if !self.primary {
|
||||
return Err(CoreError::invalid("attach must use My files"));
|
||||
}
|
||||
if remote.is_absolute()
|
||||
|| remote
|
||||
.components()
|
||||
.any(|c| !matches!(c, Component::Normal(_) | Component::CurDir))
|
||||
{
|
||||
return Err(CoreError::invalid(
|
||||
"remote folder must stay inside My files",
|
||||
));
|
||||
}
|
||||
let local = local
|
||||
.canonicalize()
|
||||
.map_err(|e| CoreError::invalid(format!("resolve {}: {e}", local.display())))?;
|
||||
if !local.is_dir() {
|
||||
return Err(CoreError::invalid("local sync path is not a directory"));
|
||||
}
|
||||
let existing = self
|
||||
.db
|
||||
.sync_folder_list()
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
let others = existing
|
||||
.iter()
|
||||
.map(|f| PathBuf::from(&f.local_path))
|
||||
.collect::<Vec<_>>();
|
||||
let mut reserved = crate::devices::sync_root_reserved();
|
||||
reserved.mountpoint = Some(
|
||||
mountpoint
|
||||
.canonicalize()
|
||||
.map_err(|e| CoreError::internal(format!("resolve mountpoint: {e}")))?,
|
||||
);
|
||||
if let Some(problem) = pdfs_core::syncroot::problem(&local, &others, &reserved) {
|
||||
return Err(CoreError::invalid(problem));
|
||||
}
|
||||
let (ino, remote_uid) = self.resolve(remote)?;
|
||||
let is_folder = self
|
||||
.state()
|
||||
.entries
|
||||
.get(&ino)
|
||||
.is_some_and(|e| e.node.is_folder() && !e.node.trashed);
|
||||
if !is_folder {
|
||||
return Err(CoreError::invalid("remote sync path is not a folder"));
|
||||
}
|
||||
let ancestors = self.attach_ancestors(&remote_uid)?;
|
||||
if remote_uid == self.primary_root_uid || !ancestors.contains(&self.primary_root_uid) {
|
||||
return Err(CoreError::invalid(
|
||||
"choose an owned folder below My files, not its root or a shared folder",
|
||||
));
|
||||
}
|
||||
for other in existing.iter().filter(|f| f.kind == "attached") {
|
||||
let uid = parse_uid(&other.remote_uid)
|
||||
.ok_or_else(|| CoreError::internal("invalid existing remote sync root"))?;
|
||||
if uid == remote_uid
|
||||
|| ancestors.contains(&uid)
|
||||
|| self.attach_ancestors(&uid)?.contains(&remote_uid)
|
||||
{
|
||||
return Err(CoreError::invalid(
|
||||
"remote folder overlaps an existing synced folder",
|
||||
));
|
||||
}
|
||||
}
|
||||
let share_id = self
|
||||
.rt
|
||||
.block_on(self.drive.context_share_id(&self.primary_root_uid))
|
||||
.map_err(|e| CoreError::from_api(&e, "resolve My files share"))?
|
||||
.to_string();
|
||||
if share_id.is_empty() {
|
||||
return Err(CoreError::invalid("My files share is unavailable"));
|
||||
}
|
||||
let id = self
|
||||
.db
|
||||
.sync_folder_attach(&local.to_string_lossy(), &remote_uid.to_string(), &share_id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
// An empty baseline makes one-sided files new, never deletions. The
|
||||
// engine hashes matching-size files and preserves divergent versions.
|
||||
let _ = self.sync_tx.send(sync::SyncMsg::Rewatch);
|
||||
let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id));
|
||||
self.db
|
||||
.sync_folder_get(id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.ok_or_else(|| CoreError::not_found("new sync folder disappeared"))
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::shutdown::Shutdown;
|
||||
use pdfs_core::error::session_revoked;
|
||||
|
||||
/// Drop `uid` from one mount's tree and tell that mount's kernel session the
|
||||
/// entry is gone. Shared by the trash and delete paths, which differ only in
|
||||
@@ -156,6 +157,17 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
if ours {
|
||||
debug!(uid = %node_uid, "ignoring remote event for a node with a queued write");
|
||||
}
|
||||
// A file kept offline keeps its copy through a change, and the
|
||||
// keeper replaces it once this batch's listings are read again.
|
||||
// The copy is checked against the revision's `(mtime, size)`, so
|
||||
// no read takes it for a newer revision. Evicted, it was gone until
|
||||
// something read the file, which then downloaded it (B282). A
|
||||
// node new in, or moved into, a kept folder is fetched by checking
|
||||
// that folder.
|
||||
let kept = match !*is_trashed && !ours {
|
||||
true => kept_by_pin(core, node_uid, parent_node_uid.as_ref()),
|
||||
false => None,
|
||||
};
|
||||
let mut had_node = false;
|
||||
core.for_each_mount(|st, notify| {
|
||||
if *is_trashed {
|
||||
@@ -171,9 +183,12 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
}
|
||||
});
|
||||
// Once, not once per mount: the content cache is per-daemon.
|
||||
if had_node {
|
||||
if had_node && kept.is_none() {
|
||||
core.cache.evict(node_uid);
|
||||
}
|
||||
if let Some(kept) = kept {
|
||||
dirty.kept.insert(kept);
|
||||
}
|
||||
// A create, rename or move-in shows up as a change to the parent
|
||||
// listing too, which is recorded for the end of the batch rather
|
||||
// than acted on here — see [`DirtyParents`].
|
||||
@@ -204,6 +219,8 @@ fn apply_event(core: &Core, event: &DriveEvent, dirty: &mut DirtyParents) -> pdf
|
||||
| DriveEvent::ScopeAccessLost { .. }
|
||||
| DriveEvent::SharedWithMeUpdated { .. } => {
|
||||
warn!("event access continuity lost; downgrading shared trees and resyncing lazily");
|
||||
// Changes went by unseen, kept files among them (B282).
|
||||
core.recheck_all_pins();
|
||||
apply_result = apply_result.and(core.db.clear_state(SHARED_WITH_ME_SYNCED_MS));
|
||||
core.for_each_mount(|st, notify| {
|
||||
let dirs: Vec<u64> = st.children.keys().copied().collect();
|
||||
@@ -295,6 +312,21 @@ struct DirtyParents {
|
||||
/// Checked per mount — a mount that already lists every one of those
|
||||
/// children under that parent keeps its listing.
|
||||
ours: HashMap<NodeUid, HashSet<NodeUid>>,
|
||||
/// Pinned files and folders the batch changed, for the pin keeper to
|
||||
/// check once the listings above are dropped (`keeper`).
|
||||
kept: HashSet<NodeUid>,
|
||||
}
|
||||
|
||||
/// What the pin keeper has to check after a remote change to `node_uid`,
|
||||
/// whose parent is now `parent_uid`: the node itself when a pin keeps it, or
|
||||
/// else the parent, when a pinned folder holds it and the node is new there.
|
||||
fn kept_by_pin(core: &Core, node_uid: &NodeUid, parent_uid: Option<&NodeUid>) -> Option<NodeUid> {
|
||||
if core.cache.is_pinned(node_uid) {
|
||||
return Some(node_uid.clone());
|
||||
}
|
||||
parent_uid
|
||||
.filter(|parent| core.cache.is_pinned(parent))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
/// Whether this mount's cached listing of `parent_uid` already accounts for
|
||||
@@ -402,6 +434,12 @@ pub(super) async fn run_event_sync(
|
||||
}
|
||||
break Some(c.clone());
|
||||
}
|
||||
// Refused for good: retrying would be refused the same way
|
||||
// for the life of the daemon. The next login restarts it.
|
||||
Err(e) if session_revoked(&e) => {
|
||||
core.mark_session_expired("seed event cursor", &e);
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, ?delay, "seed event cursor failed; retrying");
|
||||
tokio::time::sleep(delay).await;
|
||||
@@ -469,6 +507,9 @@ pub(super) async fn run_event_sync(
|
||||
cursor = Some(applied);
|
||||
}
|
||||
flush_dirty_parents(&core, &dirty);
|
||||
for uid in &dirty.kept {
|
||||
core.recheck_pinned(uid);
|
||||
}
|
||||
// Another device changed the tree: whatever a front-end lists may have
|
||||
// moved, including what is shared and what is in the trash.
|
||||
core.events
|
||||
@@ -476,30 +517,17 @@ pub(super) async fn run_event_sync(
|
||||
}
|
||||
}
|
||||
|
||||
/// The state key holding the photos volume's event cursor. Separate from the
|
||||
/// Drive cursor because the two volumes have their own event streams, and one
|
||||
/// cursor cannot address both.
|
||||
const PHOTOS_EVENT_CURSOR: &str = "photos_event_cursor";
|
||||
|
||||
/// Poll the photos volume's event stream, so a photo deleted or added on another
|
||||
/// device shows up in the gallery within one poll instead of on the timeline's
|
||||
/// freshness window.
|
||||
///
|
||||
/// Deliberately much smaller than [`run_event_sync`]: the photos volume is not
|
||||
/// part of the FUSE mount, so there is no inode space to converge and nothing to
|
||||
/// invalidate in the kernel. A trash or a delete removes the rows behind the
|
||||
/// gallery; anything else only drops the timeline's freshness stamp, and the
|
||||
/// next read re-enumerates. Getting that wrong costs a stale tile, not a wrong
|
||||
/// file.
|
||||
/// Follow the Photos volume and commit changes with its durable event cursor.
|
||||
pub(super) async fn run_photos_event_sync(core: Core) {
|
||||
let photos = core.photos();
|
||||
// The photos volume only exists once the user has used Photos at all, and
|
||||
// the daemon may well start before the network does.
|
||||
let scope = loop {
|
||||
match photos.get_photos_root().await {
|
||||
match link::bounded(photos.get_photos_root()).await {
|
||||
Ok(Some(root)) => break root.tree_event_scope_id(),
|
||||
Ok(None) => {
|
||||
debug!("no photos volume; photo event sync not started");
|
||||
Ok(None) => core.next_poll(POLL_INTERVAL).await,
|
||||
Err(error) if session_revoked(&error) => {
|
||||
core.mark_session_expired("read the photos root", &error);
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
@@ -509,39 +537,71 @@ pub(super) async fn run_photos_event_sync(core: Core) {
|
||||
}
|
||||
};
|
||||
|
||||
let mut cursor: Option<DriveEventId> = core
|
||||
.db
|
||||
.state_str(PHOTOS_EVENT_CURSOR)
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(DriveEventId::from);
|
||||
info!(?cursor, "photo event sync started");
|
||||
let key = format!("photos_event_cursor:{}", scope.volume_id());
|
||||
let mut cursor = loop {
|
||||
match core.db.state_str(&key) {
|
||||
Ok(cursor) => break cursor.map(DriveEventId::from),
|
||||
Err(error) => {
|
||||
warn!(%error, "reading the photo event checkpoint failed");
|
||||
core.next_poll(POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
info!("photo event sync started");
|
||||
|
||||
loop {
|
||||
core.next_poll(POLL_INTERVAL).await;
|
||||
let events = match link::bounded(photos.enumerate_events(&scope, cursor.as_ref())).await {
|
||||
Ok(events) => events,
|
||||
Err(error) if core.lost_link(&error, "poll photo events") => continue,
|
||||
Err(error) => {
|
||||
warn!(%error, "photo event poll failed; retrying after interval");
|
||||
if !core.lost_link(&error, "poll photo events") {
|
||||
warn!(%error, "photo event poll failed; retrying after interval");
|
||||
}
|
||||
core.next_poll(POLL_INTERVAL).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(head) = events.last().map(|event| event.id().clone()) else {
|
||||
continue;
|
||||
};
|
||||
apply_photo_events(&core, &events);
|
||||
core.events.publish(&[Topic::Photos]);
|
||||
// One cursor write per batch, after the batch is applied: a crash in
|
||||
// between replays events whose effects are idempotent.
|
||||
if let Err(error) = core.db.set_state_str(PHOTOS_EVENT_CURSOR, head.as_str()) {
|
||||
warn!(%error, "persisting the photo event cursor failed");
|
||||
continue;
|
||||
if let Some(head) = events.last().map(|event| event.id().clone()) {
|
||||
if let Err(error) = invalidate_photo_events(&core, &events).await {
|
||||
warn!(%error, "invalidating photo event metadata failed");
|
||||
core.next_poll(POLL_INTERVAL).await;
|
||||
continue;
|
||||
}
|
||||
// A failed database write must replay all changes, including echoes.
|
||||
let plan = classify_photo_events(&events, |_| false);
|
||||
if let Err(error) = core.db.photos_apply_events(
|
||||
&key,
|
||||
head.as_str(),
|
||||
&plan.gone,
|
||||
&plan.touched,
|
||||
plan.full,
|
||||
cursor.is_none(),
|
||||
) {
|
||||
warn!(%error, "committing the photo event checkpoint failed");
|
||||
core.next_poll(POLL_INTERVAL).await;
|
||||
continue;
|
||||
}
|
||||
cursor = Some(head);
|
||||
if !plan.is_empty() {
|
||||
core.events.publish(&[Topic::Photos]);
|
||||
}
|
||||
}
|
||||
cursor = Some(head);
|
||||
if core.db.photos_need_work().unwrap_or(true) {
|
||||
core.spawn_timeline_refresh();
|
||||
}
|
||||
core.next_poll(POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn invalidate_photo_events(
|
||||
core: &Core,
|
||||
events: &[DriveEvent],
|
||||
) -> proton_drive_rs::proton_sdk::error::Result<()> {
|
||||
for event in events {
|
||||
core.client.invalidate_caches_for_event(event).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What one batch of photos-volume events asks the gallery's tables to do.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
pub(super) struct PhotoEventPlan {
|
||||
@@ -571,14 +631,22 @@ fn classify_photo_events(
|
||||
mut is_echo: impl FnMut(&NodeUid) -> bool,
|
||||
) -> PhotoEventPlan {
|
||||
let mut plan = PhotoEventPlan::default();
|
||||
for event in events {
|
||||
let mut seen = HashSet::new();
|
||||
for event in events.iter().rev() {
|
||||
match event {
|
||||
DriveEvent::NodeDeleted { node_uid, .. } => plan.gone.push(node_uid.to_string()),
|
||||
DriveEvent::NodeDeleted { node_uid, .. } => {
|
||||
if seen.insert(node_uid) {
|
||||
plan.gone.push(node_uid.to_string());
|
||||
}
|
||||
}
|
||||
DriveEvent::NodeUpdated {
|
||||
node_uid,
|
||||
is_trashed,
|
||||
..
|
||||
} => {
|
||||
if !seen.insert(node_uid) {
|
||||
continue;
|
||||
}
|
||||
if *is_trashed {
|
||||
plan.gone.push(node_uid.to_string());
|
||||
} else if !is_echo(node_uid) {
|
||||
@@ -596,47 +664,11 @@ fn classify_photo_events(
|
||||
DriveEvent::CursorAdvanced { .. } | DriveEvent::SharedWithMeUpdated { .. } => {}
|
||||
}
|
||||
}
|
||||
plan.gone.reverse();
|
||||
plan.touched.reverse();
|
||||
plan
|
||||
}
|
||||
|
||||
/// Apply one batch of photos-volume events to the gallery's tables.
|
||||
fn apply_photo_events(core: &Core, events: &[DriveEvent]) {
|
||||
let plan = classify_photo_events(events, |uid| core.take_self_change(uid));
|
||||
let PhotoEventPlan {
|
||||
gone,
|
||||
touched,
|
||||
full,
|
||||
} = &plan;
|
||||
if !gone.is_empty() {
|
||||
match core.db.photos_delete(gone) {
|
||||
Ok(removed) if removed > 0 => {
|
||||
debug!(removed, "photos removed by a remote event");
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(%error, "removing remotely deleted photos failed"),
|
||||
}
|
||||
}
|
||||
// A stale mark is what makes the next refresh cheap: it reads these photos'
|
||||
// nodes and carries every other photo's metadata as it stands.
|
||||
if !touched.is_empty() {
|
||||
match core.db.photos_mark_unresolved(touched) {
|
||||
Ok(marked) if marked > 0 => debug!(marked, "photos changed by a remote event"),
|
||||
Ok(_) => {}
|
||||
Err(error) => warn!(%error, "marking remotely changed photos failed"),
|
||||
}
|
||||
}
|
||||
if *full {
|
||||
match core.db.photos_mark_all_unresolved() {
|
||||
Ok(marked) => debug!(marked, "the photo event stream lost continuity"),
|
||||
Err(error) => warn!(%error, "marking the whole timeline unresolved failed"),
|
||||
}
|
||||
}
|
||||
if !plan.is_empty() {
|
||||
core.invalidate_photos();
|
||||
core.invalidate_albums();
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep the local-file index fresh for the launcher prompt's "This computer"
|
||||
/// results. Rebuilds the index whenever it is older than [`LOCAL_INDEX_TTL`],
|
||||
/// then sleeps; runs on its own thread for the life of the daemon.
|
||||
@@ -710,18 +742,26 @@ fn scan_local_once(
|
||||
|
||||
// The walk has no idea how many files it will find, so the job counts what it
|
||||
// has seen and stays indeterminate.
|
||||
let job = transfers.begin_job("Indexing this computer");
|
||||
let job = transfers.begin_background_job("Indexing this computer");
|
||||
job.detail("Scanning your files");
|
||||
let mut failed = false;
|
||||
let walked = localindex::scan(&[home], &excludes, |batch| {
|
||||
if let Err(e) = db.local_upsert_batch(generation, &batch) {
|
||||
warn!(error = %e, "local index: batch write failed");
|
||||
failed = true;
|
||||
}
|
||||
});
|
||||
|
||||
// Prune what this scan did not see and rebuild the FTS index over the rest,
|
||||
// even if some batches failed — a partial index still beats none.
|
||||
// Prune what this scan did not see, unless it did not see everything:
|
||||
// a path it never reached, or one whose batch failed to write, is not a
|
||||
// path that is gone (`docs/BUGS.md` B217).
|
||||
job.detail("Building the search index");
|
||||
match db.local_finish_scan(generation, now_secs()) {
|
||||
let finished = if failed || walked >= localindex::MAX_ENTRIES {
|
||||
db.local_finish_partial_scan(now_secs())
|
||||
} else {
|
||||
db.local_finish_scan(generation, now_secs())
|
||||
};
|
||||
match finished {
|
||||
Ok(indexed) => info!(
|
||||
walked,
|
||||
indexed,
|
||||
|
||||
+380
-64
@@ -29,6 +29,17 @@ use super::{Core, count_noun, human_bytes, human_duration, parse_uid};
|
||||
|
||||
const MAX_CONTROL_REQUEST_BYTES: u64 = 1024 * 1024;
|
||||
const CONTROL_READ_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
/// Answer a [`CtlRequest::FolderSize`].
|
||||
fn folder_size_reply(size: CoreResult<proton_drive_rs::FolderSize>) -> CtlResponse {
|
||||
match size {
|
||||
Ok(size) => CtlResponse::FolderSize {
|
||||
size: size.size,
|
||||
items: size.descendants,
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Answer a [`CtlRequest::StopSharing`]: log it, and turn a partial removal
|
||||
/// into an error that names what is still shared.
|
||||
fn stop_sharing_reply(
|
||||
@@ -99,7 +110,7 @@ struct ControlHandlerPermit;
|
||||
impl ControlHandlerPermit {
|
||||
fn acquire() -> Option<Self> {
|
||||
ACTIVE_CONTROL_HANDLERS
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| {
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |active| {
|
||||
(active < MAX_CONTROL_HANDLERS).then_some(active + 1)
|
||||
})
|
||||
.ok()
|
||||
@@ -193,18 +204,42 @@ fn route_to_mount(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Cor
|
||||
/// B99), so resolving one needs this too. A mirror folder has no mount to run
|
||||
/// against: a file there is an ordinary local file.
|
||||
fn route_change(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Core, PathBuf)> {
|
||||
route_within(core, mountpoint, path, |path| {
|
||||
format!("{path} is in a mirrored folder; change the local file there instead")
|
||||
})
|
||||
}
|
||||
|
||||
/// [`route_to_mount`] for keeping a node offline or releasing it. A file in an
|
||||
/// on-demand synced folder is a different Drive node from any file of the
|
||||
/// same name under the primary mount, and keeping one offline did nothing for
|
||||
/// the other: a program reading the synced folder's copy downloaded it
|
||||
/// (`docs/BUGS.md` B281). A mirror folder already holds every file on disk.
|
||||
fn route_keep(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Core, PathBuf)> {
|
||||
route_within(core, mountpoint, path, |path| {
|
||||
format!(
|
||||
"{path} is in a mirrored folder, which keeps all its files on this computer already"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// [`route_to_mount`], refusing a path that leaves the mount it routed to and
|
||||
/// answering a path in a mirror folder, which has no mount, with `mirrored`.
|
||||
fn route_within(
|
||||
core: &Core,
|
||||
mountpoint: &Path,
|
||||
path: &str,
|
||||
mirrored: impl FnOnce(&str) -> String,
|
||||
) -> CoreResult<(Core, PathBuf)> {
|
||||
let routed = route_to_mount(core, mountpoint, path).map_err(|e| {
|
||||
let p = Path::new(path);
|
||||
let mirrored = core
|
||||
let in_mirror = core
|
||||
.db
|
||||
.sync_folder_list()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.any(|folder| folder.mode == "mirror" && p.starts_with(&folder.local_path));
|
||||
if mirrored {
|
||||
CoreError::invalid(format!(
|
||||
"{path} is in a mirrored folder; change the local file there instead"
|
||||
))
|
||||
if in_mirror {
|
||||
CoreError::invalid(mirrored(path))
|
||||
} else {
|
||||
e
|
||||
}
|
||||
@@ -220,6 +255,18 @@ fn route_change(core: &Core, mountpoint: &Path, path: &str) -> CoreResult<(Core,
|
||||
}
|
||||
}
|
||||
|
||||
/// The path the pin registry keeps for a pin made through `routed`: the
|
||||
/// mount-relative path on the primary mount, as it always was, and the
|
||||
/// absolute path the request named on an on-demand folder's mount, since
|
||||
/// that node is not under the primary mountpoint (`docs/BUGS.md` B281).
|
||||
fn pin_record(routed: &Core, rel: &Path, path: &str) -> PathBuf {
|
||||
if routed.primary {
|
||||
rel.to_path_buf()
|
||||
} else {
|
||||
Path::new(path).components().collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// [`route_change`] for a request that trashes or renames the node it names.
|
||||
/// A mount's root is My files or a whole synced folder. An on-demand folder's
|
||||
/// root has a parent on Drive, so nothing further down would stop `pdfs rm
|
||||
@@ -334,6 +381,9 @@ pub(crate) fn status_response(core: &Core, username: &str, mountpoint: &Path) ->
|
||||
budget: core.cache.budget(),
|
||||
pins,
|
||||
online: core.online.load(Ordering::Relaxed),
|
||||
session_expired: core.session_expired(),
|
||||
tokens_unsaved: pdfs_core::auth::tokens_unsaved(),
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
pending_uploads: queued.uploads.max(0) as u64 + landing,
|
||||
pending_changes: queued.changes.max(0) as u64,
|
||||
}
|
||||
@@ -346,6 +396,11 @@ fn touched_topics(request: &CtlRequest) -> &'static [Topic] {
|
||||
use CtlRequest as R;
|
||||
match request {
|
||||
R::Pin { .. } | R::Unpin { .. } => &[Topic::Files],
|
||||
// Only locking invalidates a hidden view. Authentication does not
|
||||
// change the public catalog and must not emit this notification.
|
||||
R::DriveGallery {
|
||||
action: pdfs_core::control::GalleryAction::Lock,
|
||||
} => &[Topic::GalleryVisibility],
|
||||
R::CreateAlbum { .. }
|
||||
| R::RenameAlbum { .. }
|
||||
| R::DeleteAlbum { .. }
|
||||
@@ -371,9 +426,11 @@ fn touched_topics(request: &CtlRequest) -> &'static [Topic] {
|
||||
R::RenameDevice { .. } | R::DeleteDevice { .. } => &[Topic::Devices],
|
||||
R::AdoptDevice { .. } | R::MigrateDevice { .. } => &[Topic::Devices, Topic::Locations],
|
||||
R::AddSyncFolder { .. }
|
||||
| R::AttachSyncFolder { .. }
|
||||
| R::RemoveSyncFolder { .. }
|
||||
| R::SetSyncFolderMode { .. }
|
||||
| R::SetSyncFolderPaused { .. }
|
||||
| R::ResetSyncFolder { .. }
|
||||
| R::RestoreSyncFolders { .. }
|
||||
| R::RestoreDeviceFolders { .. } => &[Topic::Locations, Topic::Devices],
|
||||
R::RetryPendingOp { .. } => &[Topic::Queue],
|
||||
@@ -389,6 +446,8 @@ fn touched_topics(request: &CtlRequest) -> &'static [Topic] {
|
||||
| R::CreatePublicLinkByUid { .. }
|
||||
| R::RemovePublicLink { .. }
|
||||
| R::RemovePublicLinkByUid { .. }
|
||||
| R::UpdatePublicLink { .. }
|
||||
| R::UpdatePublicLinkByUid { .. }
|
||||
| R::StopSharing { .. }
|
||||
| R::StopSharingByUid { .. } => &[Topic::Shares, Topic::Files],
|
||||
R::LeaveShared { .. }
|
||||
@@ -498,13 +557,75 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// Registered before the request is even parsed, so a handler that wedges
|
||||
// inside the daemon shows up by name in `pdfs diagnostics`.
|
||||
let _inflight = super::diagnostics::InflightGuard::new(super::diagnostics::request_kind(&line));
|
||||
if let Ok(CtlRequest::PhotoMonths { kind }) = &request {
|
||||
let started = std::time::Instant::now();
|
||||
let visibility = core.cached_gallery_visibility();
|
||||
let visibility_ms = started.elapsed().as_millis() as u64;
|
||||
let mut database_ms = None;
|
||||
let mut response_started = std::time::Instant::now();
|
||||
let response = match visibility {
|
||||
Ok(visibility) => {
|
||||
let excluded = serde_json::to_string(&visibility.media)
|
||||
.expect("a set of file identifiers serializes");
|
||||
let query = std::time::Instant::now();
|
||||
let months = core.db.photos_months_visible(*kind, &excluded);
|
||||
response_started = std::time::Instant::now();
|
||||
database_ms = Some(response_started.duration_since(query).as_millis() as u64);
|
||||
match months {
|
||||
Ok(months) => CtlResponse::PhotoMonths {
|
||||
months: months
|
||||
.into_iter()
|
||||
.map(|(year, month, count)| PhotoMonth { year, month, count })
|
||||
.collect(),
|
||||
},
|
||||
Err(error) => CtlResponse::error(error.into()),
|
||||
}
|
||||
}
|
||||
Err(error) => CtlResponse::error(error),
|
||||
};
|
||||
write_response(stream, response);
|
||||
tracing::info!(
|
||||
visibility_ms,
|
||||
database_ms,
|
||||
response_ms = response_started.elapsed().as_millis() as u64,
|
||||
total_ms = started.elapsed().as_millis() as u64,
|
||||
"PhotoMonths timings"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let visibility = if matches!(
|
||||
&request,
|
||||
Ok(CtlRequest::PhotosTimeline { .. }
|
||||
| CtlRequest::PhotoDuplicates
|
||||
| CtlRequest::PhotoSimilar
|
||||
| CtlRequest::PhotoPlaces
|
||||
| CtlRequest::PlacePhotos { .. }
|
||||
| CtlRequest::PlaceLocations { .. }
|
||||
| CtlRequest::PhotosOnThisDay { .. }
|
||||
| CtlRequest::PhotoGroup { .. }
|
||||
| CtlRequest::PhotoAlbums
|
||||
| CtlRequest::AlbumPhotos { .. }
|
||||
| CtlRequest::PhotoThumbs { .. }
|
||||
| CtlRequest::OpenGalleryImage { private: false, .. }
|
||||
| CtlRequest::OpenPhoto { .. })
|
||||
) {
|
||||
core.gallery_visibility()
|
||||
} else {
|
||||
Ok(std::sync::Arc::default())
|
||||
};
|
||||
let visibility = match visibility {
|
||||
Ok(visibility) => visibility,
|
||||
Err(error) => return write_response(stream, CtlResponse::error(error)),
|
||||
};
|
||||
let excluded =
|
||||
serde_json::to_string(&visibility.media).expect("a set of file identifiers serializes");
|
||||
let response = match request {
|
||||
Ok(CtlRequest::Status) => status_response(core, username, mountpoint),
|
||||
Ok(CtlRequest::Subscribe) => {
|
||||
CtlResponse::error(CoreError::invalid("a subscription is served on its own"))
|
||||
}
|
||||
Ok(CtlRequest::Pin { path }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => match core.pin(&rel) {
|
||||
Ok(CtlRequest::Pin { path }) => match route_keep(core, mountpoint, &path) {
|
||||
Ok((routed, rel)) => match routed.pin(&rel, &pin_record(&routed, &rel, &path)) {
|
||||
Ok(name) => {
|
||||
core.touch_profile();
|
||||
CtlResponse::Ok {
|
||||
@@ -515,8 +636,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::Unpin { path }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => match core.unpin(&rel) {
|
||||
Ok(CtlRequest::Unpin { path }) => match route_keep(core, mountpoint, &path) {
|
||||
Ok((routed, rel)) => match routed.unpin(&rel) {
|
||||
Ok(name) => {
|
||||
core.touch_profile();
|
||||
CtlResponse::Ok {
|
||||
@@ -573,7 +694,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// A full refresh is "forget what you know", not a second code
|
||||
// path: the ordinary refresh then finds every photo stale.
|
||||
if let Some(Err(e)) = full.then(|| core.db.photos_mark_all_unresolved()) {
|
||||
warn!(error = %e, "marking the whole timeline unresolved failed");
|
||||
return write_response(stream, CtlResponse::error(e.into()));
|
||||
}
|
||||
// Whatever changed the timeline can just as easily have
|
||||
// changed an album; one scope covers the whole photos view.
|
||||
@@ -614,11 +735,12 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
range,
|
||||
favorites,
|
||||
not_in_album,
|
||||
&excluded,
|
||||
) {
|
||||
Ok(Some(items)) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
counts: core.db.photos_counts().ok(),
|
||||
counts: core.db.photos_counts_visible(&excluded).ok(),
|
||||
},
|
||||
Ok(None) => CtlResponse::Photos {
|
||||
available: false,
|
||||
@@ -639,7 +761,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::PhotoPlaces) => match core.photo_places() {
|
||||
Ok(CtlRequest::PhotoPlaces) => match core.photo_places(&excluded) {
|
||||
Ok(items) => CtlResponse::Places {
|
||||
items,
|
||||
mapping: core.timeline_progress(),
|
||||
@@ -647,7 +769,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::PlacePhotos { id, offset, limit }) => {
|
||||
match core.place_photos(id, offset, clamp_limit(limit)) {
|
||||
match core.place_photos(id, offset, clamp_limit(limit), &excluded) {
|
||||
Ok(items) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
@@ -661,7 +783,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::PhotosOnThisDay { limit }) => {
|
||||
match core.photos_on_this_day(clamp_limit(limit)) {
|
||||
match core.photos_on_this_day(clamp_limit(limit), &excluded) {
|
||||
Ok(items) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
@@ -706,7 +828,10 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// An album page is a timeline page as far as the front-end is
|
||||
// concerned, so it comes back in the same reply — with `counts`
|
||||
// unset, which describe the whole timeline rather than this album.
|
||||
Some(album) => match core.album_photos(&album, offset, clamp_limit(limit)) {
|
||||
Some(album) => match visibility
|
||||
.permits(&uid)
|
||||
.and_then(|()| core.album_photos(&album, offset, clamp_limit(limit), &excluded))
|
||||
{
|
||||
Ok(items) => CtlResponse::Photos {
|
||||
available: true,
|
||||
items,
|
||||
@@ -748,17 +873,28 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
core.remove_from_album(album, photos)
|
||||
})
|
||||
}
|
||||
Ok(CtlRequest::PhotoMonths { kind }) => match core.db.photos_months(kind) {
|
||||
Ok(months) => CtlResponse::PhotoMonths {
|
||||
months: months
|
||||
.into_iter()
|
||||
.map(|(year, month, count)| PhotoMonth { year, month, count })
|
||||
.collect(),
|
||||
Ok(CtlRequest::PhotoPresence { uid, album }) => match core.cached_gallery_visibility() {
|
||||
Ok(visibility)
|
||||
if visibility.media.contains(&uid)
|
||||
|| album
|
||||
.as_ref()
|
||||
.is_some_and(|uid| visibility.albums.contains(uid)) =>
|
||||
{
|
||||
CtlResponse::PhotoPresence { present: false }
|
||||
}
|
||||
Ok(_) => match core.db.photos_contains(&uid, album.as_deref()) {
|
||||
Ok(present) => CtlResponse::PhotoPresence { present },
|
||||
Err(error) => CtlResponse::error(error.into()),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e.into()),
|
||||
Err(error) => CtlResponse::error(error),
|
||||
},
|
||||
Ok(CtlRequest::PhotoMonths { .. }) => unreachable!("handled before visibility requests"),
|
||||
Ok(CtlRequest::PhotoThumbs { uids }) => {
|
||||
let parsed: Vec<NodeUid> = uids.iter().filter_map(|u| parse_uid(u)).collect();
|
||||
let parsed: Vec<NodeUid> = uids
|
||||
.iter()
|
||||
.filter(|uid| !visibility.media.contains(*uid))
|
||||
.filter_map(|u| parse_uid(u))
|
||||
.collect();
|
||||
CtlResponse::Thumbs {
|
||||
items: core.photo_thumbs(&parsed),
|
||||
}
|
||||
@@ -808,7 +944,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
running: core.timeline_refreshing(),
|
||||
},
|
||||
Ok(CtlRequest::OpenPhoto { uid }) => match parse_uid(&uid) {
|
||||
Some(u) => match core.open_photo(&u) {
|
||||
Some(u) => match visibility.permits(&uid).and_then(|()| core.open_photo(&u)) {
|
||||
Ok(p) => CtlResponse::FilePath {
|
||||
path: p.display().to_string(),
|
||||
},
|
||||
@@ -816,6 +952,38 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
},
|
||||
None => CtlResponse::error(CoreError::invalid(format!("bad uid: {uid}"))),
|
||||
},
|
||||
Ok(CtlRequest::DriveGallery {
|
||||
action: pdfs_core::control::GalleryAction::AuthenticateHidden,
|
||||
}) => core
|
||||
.authenticate_hidden_gallery(&stream)
|
||||
.unwrap_or_else(CtlResponse::error),
|
||||
Ok(CtlRequest::DriveGallery { action }) => core
|
||||
.drive_gallery(action)
|
||||
.unwrap_or_else(CtlResponse::error),
|
||||
Ok(CtlRequest::OpenGalleryVideo { uid, private }) => core
|
||||
.open_gallery_video(&uid, private)
|
||||
.unwrap_or_else(CtlResponse::error),
|
||||
Ok(CtlRequest::OpenGalleryImage { uid, private: true }) => {
|
||||
core.gallery_image(&uid).unwrap_or_else(CtlResponse::error)
|
||||
}
|
||||
Ok(CtlRequest::OpenGalleryImage {
|
||||
uid,
|
||||
private: false,
|
||||
}) => match visibility
|
||||
.permits(&uid)
|
||||
.and_then(|()| core.open_shared_file(&uid))
|
||||
{
|
||||
Ok(path) => CtlResponse::FilePath {
|
||||
path: path.display().to_string(),
|
||||
},
|
||||
Err(error) => CtlResponse::error(error),
|
||||
},
|
||||
Ok(CtlRequest::CloseGalleryVideo { token }) => core
|
||||
.close_gallery_video(&token)
|
||||
.unwrap_or_else(CtlResponse::error),
|
||||
Ok(CtlRequest::GalleryThumbnail { uid, private }) => core
|
||||
.gallery_thumbnail(&uid, private)
|
||||
.unwrap_or_else(CtlResponse::error),
|
||||
// The scan alone walks every archive's central directory, so even the
|
||||
// dry run goes to a background thread and is read back through
|
||||
// `ImportStatus` rather than answered inline.
|
||||
@@ -823,35 +991,23 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
if archives.is_empty() {
|
||||
CtlResponse::error(CoreError::invalid("no Takeout archives given"))
|
||||
} else {
|
||||
let paths: Vec<PathBuf> = archives.into_iter().map(PathBuf::from).collect();
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)))
|
||||
} else {
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
let paths = archives.into_iter().map(PathBuf::from).collect();
|
||||
start_takeout_import(core, paths, dry_run)
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ImportStatus) => {
|
||||
let (running, summary) = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus { running, summary }
|
||||
let report = crate::takeout::import_status();
|
||||
CtlResponse::ImportStatus {
|
||||
running: report.running,
|
||||
summary: report.summary,
|
||||
progress: report.progress,
|
||||
resumable: report.resumable,
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ResumeImport) => match crate::takeout::resumable_archives() {
|
||||
None => CtlResponse::error(CoreError::not_found("there is no import to resume")),
|
||||
Some(paths) => start_takeout_import(core, paths, false),
|
||||
},
|
||||
Ok(CtlRequest::CancelImport) => {
|
||||
if crate::takeout::cancel_import() {
|
||||
CtlResponse::Ok {
|
||||
@@ -908,10 +1064,24 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
))),
|
||||
Ok((file, len)) => {
|
||||
let photos = core.photos();
|
||||
// Unless the caller knows better: the photo's own EXIF,
|
||||
// then the file's modification time, which a copy resets.
|
||||
let capture_time = capture_time.or_else(|| {
|
||||
let path = Path::new(&source_path);
|
||||
pdfs_core::exiftime::capture_time_of(path).or_else(|| {
|
||||
let modified =
|
||||
std::fs::metadata(path).and_then(|m| m.modified()).ok()?;
|
||||
let since = modified.duration_since(std::time::UNIX_EPOCH).ok()?;
|
||||
Some(since.as_secs() as i64)
|
||||
})
|
||||
});
|
||||
let metadata = proton_drive_rs::PhotoUploadMetadata {
|
||||
capture_time,
|
||||
last_modification_time: capture_time
|
||||
.and_then(pdfs_core::timestamp::modification_time),
|
||||
..Default::default()
|
||||
};
|
||||
let thumbnails = core.upload_thumbnails(Path::new(&source_path), &name);
|
||||
let guard =
|
||||
core.transfers
|
||||
.begin(name.clone(), "", TransferDirection::Upload, len);
|
||||
@@ -921,7 +1091,7 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
&media_type,
|
||||
reader,
|
||||
len as i64,
|
||||
Vec::new(),
|
||||
thumbnails,
|
||||
metadata,
|
||||
false,
|
||||
)) {
|
||||
@@ -930,7 +1100,8 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// timeline, and the GUI reloads the gallery the moment this
|
||||
// reply lands — so refresh now rather than leaving it to a
|
||||
// background pass that would land just after that reload.
|
||||
if let Err(e) = core.rt.block_on(core.refresh_timeline()) {
|
||||
core.invalidate_photos();
|
||||
if let Err(e) = core.refresh_timeline_joined() {
|
||||
warn!(error = %e, "timeline refresh after upload failed");
|
||||
}
|
||||
CtlResponse::Ok {
|
||||
@@ -938,6 +1109,9 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
// The reply goes to the app's toast only; without
|
||||
// this the journal shows a failure and no reason.
|
||||
warn!(error = %e, "photo upload failed");
|
||||
CtlResponse::error(CoreError::from_api(&e, "upload photo failed"))
|
||||
}
|
||||
}
|
||||
@@ -1045,6 +1219,36 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
(Err(e), _) | (_, Err(e)) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::Copy { path, new_parent }) => {
|
||||
match (
|
||||
rel_to_mount(mountpoint, &path),
|
||||
rel_to_mount(mountpoint, &new_parent),
|
||||
) {
|
||||
(Ok(rel), Ok(parent_rel)) => match core.copy_to(&rel, &parent_rel) {
|
||||
Ok(name) => {
|
||||
let dest = if new_parent.is_empty() {
|
||||
"My files".to_string()
|
||||
} else {
|
||||
new_parent.clone()
|
||||
};
|
||||
core.log_activity(ActivityKind::Copy, &name, format!("to {dest}"), true);
|
||||
CtlResponse::Ok {
|
||||
message: format!("copied to {name}"),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
core.log_activity(ActivityKind::Copy, &path, &e, false);
|
||||
CtlResponse::error(e)
|
||||
}
|
||||
},
|
||||
(Err(e), _) | (_, Err(e)) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::FolderSize { path }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => folder_size_reply(core.folder_size(&rel)),
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::FolderSizeByUid { uid }) => folder_size_reply(core.folder_size_by_uid(&uid)),
|
||||
Ok(CtlRequest::Delete { path }) => match route_node(core, mountpoint, &path) {
|
||||
Ok((core, rel)) => match core.delete(&rel) {
|
||||
Ok(name) => {
|
||||
@@ -1231,11 +1435,9 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
core.cache.set_budget(bytes);
|
||||
// Persist so the next mount keeps the new cap. Best-effort: a config
|
||||
// write failure is reported but the live cap is already applied.
|
||||
match AppDirs::new().map(|dirs| {
|
||||
let mut cfg = dirs.load_config();
|
||||
cfg.cache_budget = Some(bytes);
|
||||
dirs.save_config(&cfg)
|
||||
}) {
|
||||
match AppDirs::new()
|
||||
.map(|dirs| dirs.update_config(|cfg| cfg.cache_budget = Some(bytes)))
|
||||
{
|
||||
Ok(Ok(())) => {
|
||||
core.touch_profile();
|
||||
CtlResponse::Ok {
|
||||
@@ -1255,10 +1457,10 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// Persisted like the cache budget: best-effort, the live caps are
|
||||
// already applied.
|
||||
match AppDirs::new().map(|dirs| {
|
||||
let mut cfg = dirs.load_config();
|
||||
cfg.upload_limit = Some(upload);
|
||||
cfg.download_limit = Some(download);
|
||||
dirs.save_config(&cfg)
|
||||
dirs.update_config(|cfg| {
|
||||
cfg.upload_limit = Some(upload);
|
||||
cfg.download_limit = Some(download);
|
||||
})
|
||||
}) {
|
||||
Ok(Ok(())) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
@@ -1335,6 +1537,20 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
message: format!("adding {local_path}"),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::AttachSyncFolder {
|
||||
local_path,
|
||||
remote_path,
|
||||
}) => {
|
||||
let result = rel_to_mount(mountpoint, &remote_path).and_then(|remote| {
|
||||
core.attach_sync_folder(Path::new(&local_path), &remote, mountpoint)
|
||||
});
|
||||
match result {
|
||||
Ok(folder) => CtlResponse::Ok {
|
||||
message: format!("sync folder {} attached: {}", folder.id, folder.local_path),
|
||||
},
|
||||
Err(error) => CtlResponse::error(error),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ListSyncFolders) => match core.list_sync_folders() {
|
||||
Ok(items) => CtlResponse::SyncFolders { items },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
@@ -1371,6 +1587,13 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Ok(CtlRequest::ResetSyncFolder { id }) => match core.reset_sync_folder(id) {
|
||||
Ok(message) => {
|
||||
core.log_activity(ActivityKind::Sync, &message, "", true);
|
||||
CtlResponse::Ok { message }
|
||||
}
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::SyncNow { id }) => {
|
||||
core.sync_now(id);
|
||||
CtlResponse::Ok {
|
||||
@@ -1457,9 +1680,17 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
let core = core.clone();
|
||||
std::thread::spawn(move || {
|
||||
match core.restore_sync_folders(&items, device.as_deref()) {
|
||||
Ok(message) => {
|
||||
Ok(outcome) => {
|
||||
core.touch_profile();
|
||||
core.log_activity(ActivityKind::Download, &message, "", true);
|
||||
if !outcome.complete() {
|
||||
warn!(skipped = ?outcome.skipped, "restore skipped folders");
|
||||
}
|
||||
core.log_activity(
|
||||
ActivityKind::Download,
|
||||
outcome.summary(),
|
||||
outcome.skipped.join("; "),
|
||||
outcome.complete(),
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "restore failed");
|
||||
@@ -1612,6 +1843,31 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
CtlResponse::error(e)
|
||||
}
|
||||
},
|
||||
Ok(CtlRequest::UpdatePublicLink {
|
||||
path,
|
||||
id,
|
||||
role,
|
||||
password,
|
||||
expires,
|
||||
}) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => {
|
||||
match core.update_public_link(&rel, &id, role.as_deref(), password, expires) {
|
||||
Ok(link) => CtlResponse::PublicLink { link },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::UpdatePublicLinkByUid {
|
||||
uid,
|
||||
id,
|
||||
role,
|
||||
password,
|
||||
expires,
|
||||
}) => match core.update_public_link_by_uid(&uid, &id, role.as_deref(), password, expires) {
|
||||
Ok(link) => CtlResponse::PublicLink { link },
|
||||
Err(e) => CtlResponse::error(e),
|
||||
},
|
||||
Ok(CtlRequest::RemovePublicLink { path, id }) => match rel_to_mount(mountpoint, &path) {
|
||||
Ok(rel) => match core.remove_public_link(&rel, &id) {
|
||||
Ok(()) => {
|
||||
@@ -1788,9 +2044,14 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
// The request did not parse: the caller sent something malformed.
|
||||
Err(e) => CtlResponse::error(CoreError::invalid(format!("bad request: {e}"))),
|
||||
};
|
||||
let response = filter_gallery_response(response, &visibility);
|
||||
if !touched.is_empty() && !matches!(response, CtlResponse::Error { .. }) {
|
||||
core.events.publish(touched);
|
||||
}
|
||||
write_response(stream, response);
|
||||
}
|
||||
|
||||
fn write_response(mut stream: UnixStream, response: CtlResponse) {
|
||||
let mut out = match serde_json::to_vec(&response) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
@@ -1799,10 +2060,40 @@ fn handle_control_conn(core: &Core, username: &str, mountpoint: &Path, stream: U
|
||||
}
|
||||
};
|
||||
out.push(b'\n');
|
||||
let mut stream = stream;
|
||||
let _ = stream.write_all(&out);
|
||||
}
|
||||
|
||||
fn filter_gallery_response(
|
||||
mut response: CtlResponse,
|
||||
visibility: &super::gallery::Visibility,
|
||||
) -> CtlResponse {
|
||||
let visible = |item: &pdfs_core::control::PhotoItem| !visibility.media.contains(&item.uid);
|
||||
match &mut response {
|
||||
CtlResponse::Photos { items, .. } => items.retain(visible),
|
||||
CtlResponse::Albums { items, .. } => {
|
||||
items.retain(|item| !visibility.albums.contains(&item.uid));
|
||||
for album in items {
|
||||
if album
|
||||
.cover_uid
|
||||
.as_ref()
|
||||
.is_some_and(|uid| visibility.media.contains(uid))
|
||||
{
|
||||
album.cover_uid = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
CtlResponse::PhotoDuplicates { sets } | CtlResponse::PhotoSimilar { sets, .. } => {
|
||||
for set in sets.iter_mut() {
|
||||
set.retain(visible);
|
||||
}
|
||||
sets.retain(|set| set.len() > 1);
|
||||
}
|
||||
CtlResponse::PhotoLocations { items } => items.retain(|located| visible(&located.photo)),
|
||||
_ => {}
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
/// Listen on the control socket, serving one command per connection, each on its
|
||||
/// own thread. Runs on its own thread; returns only if the listener itself fails.
|
||||
///
|
||||
@@ -1854,6 +2145,31 @@ pub(crate) fn run_control_socket(
|
||||
}
|
||||
}
|
||||
|
||||
/// Check that every archive is there, then start the import in the background.
|
||||
fn start_takeout_import(core: &Core, paths: Vec<PathBuf>, dry_run: bool) -> CtlResponse {
|
||||
let missing: Vec<String> = paths
|
||||
.iter()
|
||||
.filter(|path| !path.is_file())
|
||||
.map(|path| path.display().to_string())
|
||||
.collect();
|
||||
if !missing.is_empty() {
|
||||
return CtlResponse::error(CoreError::not_found(format!(
|
||||
"cannot read: {}",
|
||||
missing.join(", ")
|
||||
)));
|
||||
}
|
||||
let count = paths.len();
|
||||
match core.spawn_takeout_import(paths, dry_run) {
|
||||
Ok(()) => CtlResponse::Ok {
|
||||
message: format!(
|
||||
"{} {count} archive(s)",
|
||||
if dry_run { "scanning" } else { "importing" }
|
||||
),
|
||||
},
|
||||
Err(e) => CtlResponse::error(e),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_limit_tests {
|
||||
use super::*;
|
||||
|
||||
+109
-22
@@ -33,6 +33,11 @@ use super::{
|
||||
now_secs, parse_uid, spawn_session, sync_folder_info, this_hostname,
|
||||
};
|
||||
|
||||
/// Registration validates local and remote root overlap before inserting a
|
||||
/// mapping. Serialize all registration paths so concurrent control requests
|
||||
/// cannot both validate against the same old set of roots.
|
||||
pub(super) static SYNC_ROOT_REGISTRATION: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// A mode that this daemon is allowed to apply.
|
||||
///
|
||||
/// Persisted and control-protocol values first pass through [`MountMode`] so
|
||||
@@ -105,13 +110,20 @@ pub(crate) fn adopted_device_uid() -> Option<String> {
|
||||
(!uid.is_empty()).then_some(uid)
|
||||
}
|
||||
|
||||
/// The paths no sync root may overlap, with the mountpoint as the config says
|
||||
/// now.
|
||||
pub(crate) fn sync_root_reserved() -> pdfs_core::syncroot::Reserved {
|
||||
let config = pdfs_core::config::AppDirs::new()
|
||||
.map(|dirs| dirs.load_config())
|
||||
.unwrap_or_default();
|
||||
pdfs_core::syncroot::Reserved::current(&config)
|
||||
}
|
||||
|
||||
/// Pin (or, with `None`, unpin) the adopted device uid in `config.json`.
|
||||
pub(crate) fn set_adopted_device_uid(uid: Option<&str>) -> CoreResult<()> {
|
||||
let dirs = pdfs_core::config::AppDirs::new()
|
||||
.map_err(|e| CoreError::internal(format!("config dirs: {e}")))?;
|
||||
let mut cfg = dirs.load_config();
|
||||
cfg.device_uid = uid.map(|u| u.to_string());
|
||||
dirs.save_config(&cfg)
|
||||
dirs.update_config(|cfg| cfg.device_uid = uid.map(|u| u.to_string()))
|
||||
.map_err(|e| CoreError::internal(format!("write config: {e}")))
|
||||
}
|
||||
|
||||
@@ -143,7 +155,9 @@ impl Core {
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
for location in &mut locations {
|
||||
location.mounted = self.states.is_mounted_at(Path::new(&location.local_path));
|
||||
if let MountKind::Device { sync_folder_id } = &location.kind {
|
||||
if let MountKind::Device { sync_folder_id } | MountKind::Attached { sync_folder_id } =
|
||||
&location.kind
|
||||
{
|
||||
location.progress = progress.get(sync_folder_id).cloned();
|
||||
location.paused = self.sync_folder_paused(*sync_folder_id);
|
||||
}
|
||||
@@ -157,7 +171,7 @@ impl Core {
|
||||
pub(crate) fn list_devices(&self) -> CoreResult<Vec<DeviceInfo>> {
|
||||
let devices = self
|
||||
.rt
|
||||
.block_on(self.client.enumerate_devices())
|
||||
.block_on(self.drive.enumerate_devices())
|
||||
.map_err(|e| CoreError::from_api(&e, "list devices"))?;
|
||||
// No cached device row yet means this machine syncs nothing, so none of
|
||||
// the listed devices is ours.
|
||||
@@ -214,7 +228,7 @@ impl Core {
|
||||
};
|
||||
let remote = self
|
||||
.rt
|
||||
.block_on(self.client.enumerate_devices())
|
||||
.block_on(self.drive.enumerate_devices())
|
||||
.map_err(|e| CoreError::from_api(&e, "enumerate devices"))?;
|
||||
let dev = remote
|
||||
.into_iter()
|
||||
@@ -257,7 +271,7 @@ impl Core {
|
||||
};
|
||||
let remote = self
|
||||
.rt
|
||||
.block_on(self.client.enumerate_devices())
|
||||
.block_on(self.drive.enumerate_devices())
|
||||
.map_err(|e| CoreError::from_api(&e, "enumerate devices"))?;
|
||||
let d = remote
|
||||
.into_iter()
|
||||
@@ -395,7 +409,7 @@ impl Core {
|
||||
// any cached record, and recover an existing device by name.
|
||||
let remote = self
|
||||
.rt
|
||||
.block_on(self.client.enumerate_devices())
|
||||
.block_on(self.drive.enumerate_devices())
|
||||
.map_err(|e| CoreError::from_api(&e, "enumerate devices"))?;
|
||||
|
||||
// An adopted uid is an explicit instruction, so it outranks everything
|
||||
@@ -454,7 +468,7 @@ impl Core {
|
||||
None => {
|
||||
let d = self
|
||||
.rt
|
||||
.block_on(self.client.create_device(&name, DeviceType::Linux))
|
||||
.block_on(self.drive.create_device(&name, DeviceType::Linux))
|
||||
.map_err(|e| CoreError::from_api(&e, "create device"))?;
|
||||
StoredDevice {
|
||||
uid: d.uid.to_string(),
|
||||
@@ -486,7 +500,7 @@ impl Core {
|
||||
}
|
||||
|
||||
/// Everything directly under a device root, trashed nodes included.
|
||||
fn device_root_nodes(&self, root_uid: &NodeUid) -> CoreResult<Vec<Node>> {
|
||||
pub(super) fn device_root_nodes(&self, root_uid: &NodeUid) -> CoreResult<Vec<Node>> {
|
||||
let uids = self
|
||||
.rt
|
||||
.block_on(self.drive.enumerate_folder_children_node_uids(root_uid))
|
||||
@@ -504,6 +518,7 @@ impl Core {
|
||||
/// tree into it once, and record the mapping. Phase 1 is a one-shot upload —
|
||||
/// the two-way engine (Phase 2) reconciles later changes.
|
||||
pub(crate) fn add_sync_folder(&self, local: &Path) -> CoreResult<StoredSyncFolder> {
|
||||
let _registration = SYNC_ROOT_REGISTRATION.lock();
|
||||
let meta = std::fs::metadata(local)
|
||||
.map_err(|e| CoreError::internal(format!("stat {}: {e}", local.display())))?;
|
||||
if !meta.is_dir() {
|
||||
@@ -517,18 +532,20 @@ impl Core {
|
||||
.map_err(|e| CoreError::internal(format!("canonicalize {}: {e}", local.display())))?;
|
||||
let local_str = local.to_string_lossy().to_string();
|
||||
|
||||
// Reject duplicates up front for a clear error (UNIQUE would also catch it).
|
||||
if self
|
||||
// `/`, the home folder, the mount, this app's own data, and a folder
|
||||
// inside or around another root are refused (`syncroot::problem`). A
|
||||
// duplicate is one of those, refused here for a clear error before
|
||||
// the UNIQUE constraint would catch it.
|
||||
let others = self
|
||||
.db
|
||||
.sync_folder_list()
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.iter()
|
||||
.any(|f| f.local_path == local_str)
|
||||
{
|
||||
return Err(CoreError::invalid(format!(
|
||||
"{} is already synced",
|
||||
local.display()
|
||||
)));
|
||||
.map(|f| PathBuf::from(&f.local_path))
|
||||
.collect::<Vec<_>>();
|
||||
let reserved = sync_root_reserved();
|
||||
if let Some(problem) = pdfs_core::syncroot::problem(&local, &others, &reserved) {
|
||||
return Err(CoreError::invalid(problem));
|
||||
}
|
||||
|
||||
let name = local
|
||||
@@ -657,12 +674,16 @@ impl Core {
|
||||
} else {
|
||||
p.total.max(p.done) as u64
|
||||
},
|
||||
// Most passes find nothing; only one that queues work is
|
||||
// the drive syncing.
|
||||
background: true,
|
||||
},
|
||||
SyncPhase::Applying => JobItem {
|
||||
title: format!("Syncing {folder}"),
|
||||
detail: p.current.clone(),
|
||||
done: p.done as u64,
|
||||
total: p.total.max(p.done) as u64,
|
||||
background: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -684,6 +705,11 @@ impl Core {
|
||||
.sync_folder_get(id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.ok_or_else(|| CoreError::not_found(format!("no synced folder with id {id}")))?;
|
||||
if folder.kind == "attached" && delete_remote {
|
||||
return Err(CoreError::invalid(
|
||||
"attached My files folders cannot be deleted by stopping sync; omit --delete-remote",
|
||||
));
|
||||
}
|
||||
// An `ondemand` folder *is* a FUSE mount over its local path, so dropping
|
||||
// only the row would strand the mount: the path would keep serving a
|
||||
// folder the daemon no longer tracks, and nothing would ever unmount it.
|
||||
@@ -770,6 +796,11 @@ impl Core {
|
||||
.sync_folder_get(id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.ok_or_else(|| CoreError::not_found(format!("no synced folder with id {id}")))?;
|
||||
if folder.kind == "attached" && target_mode != ApplicableMountMode::Mirror {
|
||||
return Err(CoreError::invalid(
|
||||
"attached My files folders only support mirror mode; use the My files mount for online-only access",
|
||||
));
|
||||
}
|
||||
let current_mode = ApplicableMountMode::parse(&folder.mode)
|
||||
.map_err(|error| CoreError::invalid(format!("unsupported current mode: {error}")))?;
|
||||
// A switch waits on the folder's reconcile, which a pause skips; refuse
|
||||
@@ -945,6 +976,11 @@ impl Core {
|
||||
.sync_folder_get(id)
|
||||
.map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?
|
||||
.ok_or_else(|| SwitchBlocked::Failed(format!("no synced folder with id {id}")))?;
|
||||
if folder.kind == "attached" && mode != ApplicableMountMode::Mirror {
|
||||
return Err(SwitchBlocked::Failed(
|
||||
"attached My files folders only support mirror mode".into(),
|
||||
));
|
||||
}
|
||||
if !pending_intent_matches(folder.pending_mode.as_deref(), expected_pending) {
|
||||
return Err(SwitchBlocked::Superseded);
|
||||
}
|
||||
@@ -974,9 +1010,15 @@ impl Core {
|
||||
.db
|
||||
.sync_entries(id)
|
||||
.map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?;
|
||||
if let Some(reason) =
|
||||
sync::mirror_tree_unsynced(&local, &baseline, &self.ignore_rules(&local))
|
||||
{
|
||||
let rules = self.ignore_rules(&local);
|
||||
// Emptied before any pass saw it, the folder is still `idle`, and
|
||||
// the check below would refuse the switch for good. The push pass
|
||||
// of a queued switch forgets those paths instead (`docs/BUGS.md`
|
||||
// B255).
|
||||
if sync::mirror_tree_gone(&local, &baseline, &rules) {
|
||||
return Err(SwitchBlocked::NotNow);
|
||||
}
|
||||
if let Some(reason) = sync::mirror_tree_unsynced(&local, &baseline, &rules) {
|
||||
return Err(SwitchBlocked::Failed(format!(
|
||||
"going online-only would delete what is only on this computer: {reason}"
|
||||
)));
|
||||
@@ -1065,6 +1107,50 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Forget what a mirror folder synced before, so its next pass compares the
|
||||
/// two sides as on the first sync: what one side has alone is copied to the
|
||||
/// other, and a file that differs is kept twice as a conflict. Nothing is
|
||||
/// deleted on either side.
|
||||
///
|
||||
/// This is the way out for a folder whose passes refuse to run because its
|
||||
/// whole local copy, or its whole copy on Drive, is gone (#46). The baseline
|
||||
/// is what makes that read as "delete everything", and it is what this
|
||||
/// drops. A queued switch to on-demand is dropped with it: the push pass
|
||||
/// that switch runs reads an unrecorded local file as new and would upload
|
||||
/// a second copy of every file Drive already has.
|
||||
pub(crate) fn reset_sync_folder(&self, id: i64) -> CoreResult<String> {
|
||||
let lock = self.sync_lock(id);
|
||||
let Some(_guard) = lock.try_lock() else {
|
||||
return Err(CoreError::conflict(
|
||||
"this folder is syncing; try again when the pass ends",
|
||||
));
|
||||
};
|
||||
let folder = self
|
||||
.db
|
||||
.sync_folder_get(id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.ok_or_else(|| CoreError::not_found(format!("no synced folder with id {id}")))?;
|
||||
if folder.mode != "mirror" {
|
||||
return Err(CoreError::invalid(
|
||||
"an online-only folder keeps nothing on this computer to sync from scratch",
|
||||
));
|
||||
}
|
||||
self.db
|
||||
.sync_entries_clear(id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
if folder.pending_mode.as_deref() == Some("ondemand") {
|
||||
self.db
|
||||
.sync_folder_set_pending_mode(id, None)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
}
|
||||
let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id));
|
||||
info!(id, path = %folder.local_path, "sync folder reset");
|
||||
Ok(format!(
|
||||
"{} will sync from scratch",
|
||||
base_name(&folder.local_path)
|
||||
))
|
||||
}
|
||||
|
||||
/// Spawn a secondary FUSE session rooted at `root` over `local` from a
|
||||
/// [`Core::fork_state`] sibling. Clears any stale kernel mount first (a crashed run can leave
|
||||
/// one, which would fail the fresh mount with EBUSY).
|
||||
@@ -1134,7 +1220,7 @@ impl Core {
|
||||
}
|
||||
};
|
||||
for folder in folders {
|
||||
if folder.mode != "ondemand" {
|
||||
if folder.kind != "device" || folder.mode != "ondemand" {
|
||||
continue;
|
||||
}
|
||||
let Some(root_uid) = parse_uid(&folder.remote_uid) else {
|
||||
@@ -1330,6 +1416,7 @@ mod tests {
|
||||
|
||||
fn folder(local_path: &str, remote_uid: &str, mode: &str) -> StoredSyncFolder {
|
||||
StoredSyncFolder {
|
||||
kind: "device".into(),
|
||||
id: 7,
|
||||
local_path: local_path.to_string(),
|
||||
remote_uid: remote_uid.to_string(),
|
||||
|
||||
@@ -163,6 +163,7 @@ impl Core {
|
||||
control_handlers: super::control::active_handlers(),
|
||||
control_handler_limit: super::control::handler_limit(),
|
||||
pending_ops: self.pending_ops_quickly(),
|
||||
sync: Some(self.sync_engine.report()),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+241
-58
@@ -20,7 +20,7 @@
|
||||
//! are kept as a conflict copy rather than discarded — see
|
||||
//! [`Core::revision_conflict`].
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
@@ -53,6 +53,21 @@ use super::{
|
||||
use proton_drive_rs::proton_sdk::api::ResponseCode;
|
||||
use proton_drive_rs::proton_sdk::error::ProtonError;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TrashedMove(String);
|
||||
|
||||
impl std::fmt::Display for TrashedMove {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"‘{}’ is in Proton Drive Trash. Restore it before retrying this move.",
|
||||
self.0
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TrashedMove {}
|
||||
|
||||
const DRAIN_ACCESS_RECHECK: Duration = Duration::from_secs(5);
|
||||
|
||||
/// How long an op may stay access-deferred before the deferral is reported as a
|
||||
@@ -347,17 +362,7 @@ impl Core {
|
||||
// its writer has to be noticed.
|
||||
self.sweep_parked_creates();
|
||||
}
|
||||
// Paused, nothing due changes that: sleep until a resume
|
||||
// wakes the worker (or the idle poll notices a timed pause
|
||||
// ran out), not until the next op falls due — that is
|
||||
// "now" for everything already queued.
|
||||
if paused {
|
||||
self.wait_for_drain_work();
|
||||
continue;
|
||||
}
|
||||
// A debounced or backed-off op may be waiting: sleep only
|
||||
// until it becomes due rather than the full idle-poll.
|
||||
self.wait_for_drain_work_or_due();
|
||||
self.wait_for_drain_work_or_due(online && !paused);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -399,13 +404,7 @@ impl Core {
|
||||
// A name a change of ours is about to free is a wait, not a
|
||||
// failure: the trash or rename holding it lands in moments.
|
||||
let held = waits_for_name(e.as_ref());
|
||||
let backoff = if held {
|
||||
DRAIN_BACKOFF_MIN
|
||||
} else {
|
||||
DRAIN_BACKOFF_MIN
|
||||
.saturating_mul(1u32 << attempts.min(6))
|
||||
.min(DRAIN_BACKOFF_MAX)
|
||||
};
|
||||
let backoff = drain_backoff(attempts, held, op.id);
|
||||
warn!(uid = %op.uid, attempts, error = %err_str, "pending upload failed; will retry");
|
||||
if let Err(e) = self.db.record_attempt_failure(
|
||||
op.id,
|
||||
@@ -732,14 +731,16 @@ impl Core {
|
||||
/// Like [`wait_for_drain_work`](Self::wait_for_drain_work), but first
|
||||
/// checks whether a debounced or backed-off op is due before the idle-poll
|
||||
/// ceiling. A 2-second debounce must not sleep 30 seconds.
|
||||
fn wait_for_drain_work_or_due(&self) {
|
||||
let timeout = match self.db.earliest_due_at() {
|
||||
Ok(Some(ts)) => {
|
||||
let remaining = (ts - now_millis()).max(0) as u64;
|
||||
Duration::from_millis(remaining).min(DRAIN_IDLE_POLL)
|
||||
}
|
||||
_ => DRAIN_IDLE_POLL,
|
||||
///
|
||||
/// `claiming` is false while the worker claims nothing (offline or
|
||||
/// paused); see [`idle_wait`].
|
||||
fn wait_for_drain_work_or_due(&self, claiming: bool) {
|
||||
let earliest = if claiming {
|
||||
self.db.earliest_due_at().ok().flatten()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let timeout = idle_wait(claiming, earliest, now_millis());
|
||||
sleep_for_drain_work(&self.drain_wake, || self.shutdown.is_stopping(), timeout);
|
||||
}
|
||||
|
||||
@@ -809,13 +810,13 @@ impl Core {
|
||||
let parent = parse_node_uid(parent_str).ok_or("rename op has an unparseable parent")?;
|
||||
let name = op.name.clone().ok_or("rename op has no name")?;
|
||||
|
||||
// The node we were asked to rename may be gone or trashed by now. Either
|
||||
// way there is nothing to rename and nothing to lose — a rename holds no
|
||||
// bytes — so the op is satisfied rather than retried forever.
|
||||
// A trashed node can be restored. Only retire the operation if the node
|
||||
// is gone entirely.
|
||||
let node = match self.fetch_node_remote(&uid)? {
|
||||
Some(n) if !n.trashed => n,
|
||||
_ => {
|
||||
warn!(%uid, name, "renamed node is gone or trashed remotely; dropping the rename");
|
||||
Some(n) if n.trashed => return Err(TrashedMove(n.name).into()),
|
||||
Some(n) => n,
|
||||
None => {
|
||||
warn!(%uid, name, "renamed node is gone remotely; dropping the rename");
|
||||
self.db.delete_op(op.id)?;
|
||||
return Ok(());
|
||||
}
|
||||
@@ -841,23 +842,38 @@ impl Core {
|
||||
// no window for the requirements to go stale between two calls —
|
||||
// the race B46 queued this op to get away from.
|
||||
let target = (node.name != name).then_some(name.as_str());
|
||||
match self.move_rename_remote(&uid, &parent, target) {
|
||||
let try_move = |target| {
|
||||
self.move_rename_remote(&uid, &parent, target).map_err(
|
||||
|error| -> Box<dyn std::error::Error> {
|
||||
if (is_gone(&error) || refusal(&error) == Some(Some(SyncIssue::Access)))
|
||||
&& let Some(name) = trashed_move_folder(
|
||||
node.parent_uid.iter().chain(std::iter::once(&parent)),
|
||||
|uid| self.fetch_node_remote(uid).ok().flatten(),
|
||||
)
|
||||
{
|
||||
return TrashedMove(name).into();
|
||||
}
|
||||
error.into()
|
||||
},
|
||||
)
|
||||
};
|
||||
match try_move(target) {
|
||||
Ok(()) => name.clone(),
|
||||
// The destination holds that name already. Landing under a
|
||||
// *different* name is the non-destructive resolution: it neither
|
||||
// clobbers their file nor drops ours, and it is visible.
|
||||
Err(e) if is_already_exists(&e) => {
|
||||
Err(e) if is_already_exists(e.as_ref()) => {
|
||||
if self.name_is_held(op, Some(&parent), &name)? {
|
||||
return Err(held_by_queued_change(&name));
|
||||
}
|
||||
// What held it may have let go since Drive answered
|
||||
// (docs/BUGS.md B144).
|
||||
match self.move_rename_remote(&uid, &parent, target) {
|
||||
match try_move(target) {
|
||||
Ok(()) => name.clone(),
|
||||
Err(e) if is_already_exists(&e) => {
|
||||
Err(e) if is_already_exists(e.as_ref()) => {
|
||||
let alt = conflict_name(&name, now_secs());
|
||||
warn!(%uid, name, alt, "destination already holds that name; using a conflict name");
|
||||
self.move_rename_remote(&uid, &parent, Some(&alt))?;
|
||||
try_move(Some(&alt))?;
|
||||
self.adopt_drained_name(&uid, &alt);
|
||||
self.log_activity(
|
||||
ActivityKind::Rename,
|
||||
@@ -867,7 +883,7 @@ impl Core {
|
||||
);
|
||||
alt
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
// The destination folder is gone. Leaving the node in its current
|
||||
@@ -876,7 +892,7 @@ impl Core {
|
||||
// the user chose still applies there. Retrying the move could
|
||||
// only fail again — the folder is not coming back — and would
|
||||
// wedge the queue.
|
||||
Err(e) if is_gone(&e) => {
|
||||
Err(e) if is_gone(e.as_ref()) => {
|
||||
warn!(%uid, name, %parent, "move destination is gone; leaving the node where it is");
|
||||
let landed = match self.drain_rename_in_place(
|
||||
op,
|
||||
@@ -899,7 +915,7 @@ impl Core {
|
||||
);
|
||||
landed
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
};
|
||||
self.db.delete_op(op.id)?;
|
||||
@@ -1126,7 +1142,7 @@ impl Core {
|
||||
_ => None,
|
||||
};
|
||||
let sent = self.create_sent_digests(op.id, current);
|
||||
Ok(adoptable(is_dir, &twin, op.created_at, &sent).then_some(twin.uid))
|
||||
Ok(adoptable(is_dir, &twin, self.first_sent(op), &sent).then_some(twin.uid))
|
||||
}
|
||||
|
||||
/// Record the name the remote actually gave a node, after a conflict forced
|
||||
@@ -1295,6 +1311,13 @@ impl Core {
|
||||
sent: (&sent.0, &sent.1),
|
||||
landed: (&home.to_string(), &name),
|
||||
};
|
||||
// Counted from the op's retirement until the bytes are cached and the
|
||||
// tree has the real uid, as an upload is. Uncounted, the queue read
|
||||
// empty while that still ran: a delete in between took an open file
|
||||
// for closed and dropped what its handle reads, and a script that
|
||||
// waited for an empty queue read the file before it was cached
|
||||
// (`docs/BUGS.md` B257).
|
||||
let _landing = LandingUpload::begin(&self.landing_uploads);
|
||||
let newer = match self.retire_create(op, uploaded, &landing, &local, &real)? {
|
||||
CreateRetired::Withdrawn { trash } => {
|
||||
return self.trash_withdrawn_create(op, trash, &local, &real, &name);
|
||||
@@ -1356,7 +1379,11 @@ impl Core {
|
||||
parent: Option<&NodeUid>,
|
||||
name: &str,
|
||||
) -> Result<bool, Box<dyn std::error::Error>> {
|
||||
if self.db.has_pending_trash_named(name)? {
|
||||
let parent_uid = parent.map(NodeUid::to_string);
|
||||
if self
|
||||
.db
|
||||
.has_pending_trash_named(name, parent_uid.as_deref())?
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
if op.attempts >= NAME_HOLD_ATTEMPTS {
|
||||
@@ -1521,8 +1548,10 @@ impl Core {
|
||||
.as_deref()
|
||||
.is_some_and(|sha| holds_bytes(&twin, sha));
|
||||
let sent = self.create_sent_digests(op.id, current);
|
||||
Ok(adoptable(op.kind == OP_MKDIR, &twin, op.created_at, &sent)
|
||||
.then_some((twin.uid, holds_blob)))
|
||||
Ok(
|
||||
adoptable(op.kind == OP_MKDIR, &twin, self.first_sent(op), &sent)
|
||||
.then_some((twin.uid, holds_blob)),
|
||||
)
|
||||
}
|
||||
|
||||
/// The SHA-1s of the bytes queued create `id` may have made its file with:
|
||||
@@ -1537,6 +1566,22 @@ impl Core {
|
||||
sent
|
||||
}
|
||||
|
||||
/// When create `op` first went out, which is the earliest a node its lost
|
||||
/// answer left can have been made. An op queued offline may wait for days
|
||||
/// before that; measured from when it was queued, the adoption window let
|
||||
/// it take a node another device made in the meantime. Ops sent before the
|
||||
/// time was recorded fall back to when they were queued.
|
||||
fn first_sent(&self, op: &PendingOp) -> i64 {
|
||||
match self.db.create_first_sent(op.id) {
|
||||
Ok(Some(at)) => at.max(op.created_at),
|
||||
Ok(None) => op.created_at,
|
||||
Err(e) => {
|
||||
warn!(id = op.id, error = %e, "reading when a queued create was sent failed");
|
||||
op.created_at
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Make one queued `create`/`mkdir` real under a given name, and hand back
|
||||
/// the API's own error so the caller can tell a name clash from a failure.
|
||||
///
|
||||
@@ -1549,7 +1594,7 @@ impl Core {
|
||||
name: &str,
|
||||
) -> Result<NodeUid, Box<dyn std::error::Error>> {
|
||||
self.db
|
||||
.note_create_target(op.id, &parent.to_string(), name)?;
|
||||
.note_create_target(op.id, &parent.to_string(), name, now_millis())?;
|
||||
match op.kind == OP_MKDIR {
|
||||
true => Ok(self.block_on_bounded(self.drive.create_folder(
|
||||
parent,
|
||||
@@ -1594,7 +1639,7 @@ impl Core {
|
||||
.begin(name, op.uid.clone(), TransferDirection::Upload, meta.len);
|
||||
let reader = CountingReader::new(File::open(blob)?, &guard);
|
||||
let uid = self.block_on_within(
|
||||
link::upload_deadline(meta.len),
|
||||
link::upload_deadline(meta.len, self.transfers.upload_cap()),
|
||||
self.drive.upload_file_from(
|
||||
parent,
|
||||
name,
|
||||
@@ -1991,7 +2036,7 @@ impl Core {
|
||||
.begin(&alt, meta.uid.clone(), TransferDirection::Upload, meta.len);
|
||||
let reader = CountingReader::new(File::open(blob)?, &guard);
|
||||
self.block_on_within(
|
||||
link::upload_deadline(meta.len),
|
||||
link::upload_deadline(meta.len, self.transfers.upload_cap()),
|
||||
self.drive.upload_file_from(
|
||||
&parent,
|
||||
&alt,
|
||||
@@ -2244,7 +2289,7 @@ impl Core {
|
||||
.clone()
|
||||
.ok_or("pending op has no staged blob")?;
|
||||
let blob = PathBuf::from(blob);
|
||||
let meta: StagedWrite = serde_json::from_str(op.meta_json.as_deref().unwrap_or(""))?;
|
||||
let mut meta: StagedWrite = serde_json::from_str(op.meta_json.as_deref().unwrap_or(""))?;
|
||||
let uid = parse_node_uid(&meta.uid).ok_or("staged write has an unparseable uid")?;
|
||||
|
||||
if let Some(reason) = self.revision_conflict(&uid, &meta, &blob)? {
|
||||
@@ -2283,6 +2328,22 @@ impl Core {
|
||||
&written,
|
||||
)
|
||||
.map_err(|e| self.errno_error(e, "gap-fill from base failed"))?;
|
||||
// The blob is whole now. Recorded, so that a retry after a lost
|
||||
// answer can match these bytes against the server's revision
|
||||
// (`revision_conflict`) instead of forking a conflict copy of the
|
||||
// user's own write.
|
||||
meta.complete = true;
|
||||
match serde_json::to_string(&meta) {
|
||||
Ok(json) => {
|
||||
let path = op.blob_path.as_deref().unwrap_or_default();
|
||||
if let Err(e) = self.db.update_op_meta_for_blob(op.id, path, &json) {
|
||||
warn!(%uid, error = %e,
|
||||
"recording a gap-filled blob as whole failed; \
|
||||
a lost answer may land it as a conflict copy");
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(%uid, error = %e, "serializing a gap-filled sidecar failed"),
|
||||
}
|
||||
}
|
||||
|
||||
let name = self.node_name(&uid);
|
||||
@@ -2303,7 +2364,7 @@ impl Core {
|
||||
let reader = CountingReader::new(File::open(&blob)?, &guard).with_cancel(cancel.flag());
|
||||
let started = Instant::now();
|
||||
let sent = self.block_on_within(
|
||||
link::upload_deadline(meta.len),
|
||||
link::upload_deadline(meta.len, self.transfers.upload_cap()),
|
||||
self.drive
|
||||
.upload_new_revision_from(&uid, reader, meta.len as i64, thumbnails, None),
|
||||
);
|
||||
@@ -2668,6 +2729,29 @@ fn moved_elsewhere(meta: &RenameMeta, node: &Node, parent: &NodeUid, name: &str)
|
||||
!at(&meta.original_parent_uid, original_name) && !at(&parent.to_string(), name)
|
||||
}
|
||||
|
||||
/// A child can have a normal metadata record even when an ancestor is trashed.
|
||||
/// Check both live parent chains before interpreting a refused move as an
|
||||
/// access problem. Shared ancestors only need to be fetched once.
|
||||
fn trashed_move_folder<'a>(
|
||||
parents: impl IntoIterator<Item = &'a NodeUid>,
|
||||
mut fetch: impl FnMut(&NodeUid) -> Option<Node>,
|
||||
) -> Option<String> {
|
||||
let mut seen = HashSet::new();
|
||||
for parent in parents {
|
||||
let mut next = Some(parent.clone());
|
||||
while let Some(uid) = next.take()
|
||||
&& seen.insert(uid.clone())
|
||||
&& let Some(node) = fetch(&uid)
|
||||
{
|
||||
if node.trashed {
|
||||
return Some(node.name);
|
||||
}
|
||||
next = node.parent_uid;
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The answer from Drive somewhere in `e`'s chain, if it got that far.
|
||||
fn proton_error<'e>(e: &'e (dyn std::error::Error + 'static)) -> Option<&'e ProtonError> {
|
||||
std::iter::successors(Some(e), |e| e.source()).find_map(|e| e.downcast_ref::<ProtonError>())
|
||||
@@ -2677,8 +2761,13 @@ fn proton_error<'e>(e: &'e (dyn std::error::Error + 'static)) -> Option<&'e Prot
|
||||
/// could not be reached, which says nothing; otherwise the issue Drive's answer
|
||||
/// is, or `Some(None)` for a failure that is not a refusal.
|
||||
fn refusal(e: &(dyn std::error::Error + 'static)) -> Option<Option<SyncIssue>> {
|
||||
if e.is::<TrashedMove>() {
|
||||
return Some(Some(SyncIssue::Trashed));
|
||||
}
|
||||
let proton = proton_error(e)?;
|
||||
if is_network_error(proton) {
|
||||
// A refused session refuses everything, so it says nothing about this
|
||||
// op either; the mount is offline for it (`Core::lost_link`).
|
||||
if is_network_error(proton) || pdfs_core::error::session_revoked(proton) {
|
||||
return None;
|
||||
}
|
||||
let ProtonError::Api(api) = proton else {
|
||||
@@ -2759,24 +2848,43 @@ fn waits_for_name(e: &(dyn std::error::Error + 'static)) -> bool {
|
||||
std::iter::successors(Some(e), |e| e.source()).any(|e| e.is::<NameHeld>())
|
||||
}
|
||||
|
||||
/// How long an op waits after its `attempts`-th failure: doubling from
|
||||
/// [`DRAIN_BACKOFF_MIN`] up to [`DRAIN_BACKOFF_MAX`], plus up to a fifth more,
|
||||
/// different for each op (`seed`), so that ops that failed together are not
|
||||
/// all retried in the same second. One that waits for a name (`held`) waits
|
||||
/// the minimum.
|
||||
///
|
||||
/// The doubling stopped at 2^6, 128 s, so the maximum was never reached.
|
||||
fn drain_backoff(attempts: i64, held: bool, seed: i64) -> Duration {
|
||||
if held {
|
||||
return DRAIN_BACKOFF_MIN;
|
||||
}
|
||||
let base = DRAIN_BACKOFF_MIN
|
||||
.saturating_mul(1u32 << attempts.clamp(0, 8))
|
||||
.min(DRAIN_BACKOFF_MAX);
|
||||
// A cheap, stable spread: the same op and attempt always wait the same.
|
||||
let spread = (seed.wrapping_mul(0x9E37_79B9).wrapping_add(attempts) as u64 % 21) as u32;
|
||||
base + base * spread / 100
|
||||
}
|
||||
|
||||
/// How many attempts an op waits for a rename or a create on the wire to free
|
||||
/// the name it wants. Two files swapping names each wait for the other, so the
|
||||
/// wait has to end; a conflict name ends it.
|
||||
const NAME_HOLD_ATTEMPTS: i64 = 4;
|
||||
|
||||
/// How much older than its op a remote node may be and still be the one an
|
||||
/// unanswered call made. The op is queued the moment the call gives up, so the
|
||||
/// node is at most one call's deadline older; the rest is clock skew between
|
||||
/// this machine and Proton.
|
||||
/// How much older than its op's first send a remote node may be and still be
|
||||
/// the one an unanswered call made. The node is made during that call, so it
|
||||
/// is at most one call's deadline older; the rest is clock skew between this
|
||||
/// machine and Proton.
|
||||
const ADOPT_WINDOW_MS: i64 = 2 * 60 * 1000;
|
||||
|
||||
/// Whether `twin`, found holding the name a queued create wants, is the node an
|
||||
/// earlier, unanswered attempt at that create made: the same kind, created
|
||||
/// around when the op was queued, and for a file still empty, as the mount's
|
||||
/// earlier, unanswered attempt at that create made: the same kind, created no
|
||||
/// earlier than around when the op was first sent (`first_sent_ms`), and for a file still empty, as the mount's
|
||||
/// create leaves it, or holding exactly bytes the op uploaded, whose SHA-1 is
|
||||
/// one of `sent`.
|
||||
fn adoptable(is_dir: bool, twin: &Node, op_created_ms: i64, sent: &[String]) -> bool {
|
||||
if twin.creation_time.saturating_mul(1000) < op_created_ms - ADOPT_WINDOW_MS {
|
||||
fn adoptable(is_dir: bool, twin: &Node, first_sent_ms: i64, sent: &[String]) -> bool {
|
||||
if twin.creation_time.saturating_mul(1000) < first_sent_ms - ADOPT_WINDOW_MS {
|
||||
return false;
|
||||
}
|
||||
match &twin.kind {
|
||||
@@ -2833,6 +2941,23 @@ fn sleep_for_drain_work(
|
||||
*woken = false;
|
||||
}
|
||||
|
||||
/// How long an idle drain worker sleeps: until the earliest queued op falls
|
||||
/// due, at most the idle poll.
|
||||
///
|
||||
/// A worker that claims nothing — offline or paused — sleeps the full idle
|
||||
/// poll whatever is due. Every op already queued is due "now" for it, so
|
||||
/// sleeping until then was a zero wait, and all the workers spun on the queue
|
||||
/// for as long as the outage lasted (`docs/BUGS.md` B200). A reconnect or a
|
||||
/// resume wakes the drain, so nothing waits the full poll for it.
|
||||
fn idle_wait(claiming: bool, earliest_due: Option<i64>, now: i64) -> Duration {
|
||||
match earliest_due {
|
||||
Some(ts) if claiming => {
|
||||
Duration::from_millis((ts - now).max(0) as u64).min(DRAIN_IDLE_POLL)
|
||||
}
|
||||
_ => DRAIN_IDLE_POLL,
|
||||
}
|
||||
}
|
||||
|
||||
fn placed_in(st: &crate::state::State, uid: &NodeUid) -> Option<(String, Option<NodeUid>)> {
|
||||
let entry = st.by_uid.get(uid).and_then(|ino| st.entries.get(ino))?;
|
||||
Some((entry.node.name.clone(), entry.node.parent_uid.clone()))
|
||||
@@ -2858,6 +2983,23 @@ mod tests {
|
||||
assert!(started.elapsed() < Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_offline_worker_waits_the_idle_poll_for_a_past_due_op() {
|
||||
// The op fell due long ago, and offline nobody attempts it, so it
|
||||
// stays due: waiting for it was a zero wait, forever (B200).
|
||||
assert_eq!(idle_wait(false, Some(0), 10_000), DRAIN_IDLE_POLL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_claiming_worker_waits_only_until_the_next_op_falls_due() {
|
||||
assert_eq!(
|
||||
idle_wait(true, Some(12_000), 10_000),
|
||||
Duration::from_secs(2)
|
||||
);
|
||||
assert_eq!(idle_wait(true, Some(0), 10_000), Duration::ZERO);
|
||||
assert_eq!(idle_wait(true, None, 10_000), DRAIN_IDLE_POLL);
|
||||
}
|
||||
|
||||
fn pending(kind: &str) -> PendingOp {
|
||||
PendingOp {
|
||||
id: 0,
|
||||
@@ -3029,6 +3171,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_move_identifies_a_trashed_ancestor_on_either_side() {
|
||||
let source = folder_node("source", Some("old"));
|
||||
let destination = folder_node("destination", Some("new"));
|
||||
let mut old = folder_node("old", None);
|
||||
let mut new = folder_node("new", None);
|
||||
let find = |old: &Node, new: &Node| {
|
||||
trashed_move_folder([&source.uid, &destination.uid], |uid| {
|
||||
[&source, &destination, old, new]
|
||||
.into_iter()
|
||||
.find(|node| node.uid == *uid)
|
||||
.cloned()
|
||||
})
|
||||
};
|
||||
old.trashed = true;
|
||||
assert_eq!(find(&old, &new).as_deref(), Some("old"));
|
||||
old.trashed = false;
|
||||
new.trashed = true;
|
||||
let error = TrashedMove(find(&old, &new).unwrap());
|
||||
assert_eq!(refusal(&error), Some(Some(SyncIssue::Trashed)));
|
||||
assert!(error.to_string().contains("‘new’"));
|
||||
new.trashed = false;
|
||||
assert_eq!(find(&old, &new), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_cross_parent_rename_retains_viewer_source_authority_after_local_move() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
@@ -3324,6 +3491,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failing_op_backs_off_to_the_maximum_with_a_spread() {
|
||||
let at = |attempts| drain_backoff(attempts, false, 7);
|
||||
assert!(at(1) >= DRAIN_BACKOFF_MIN * 2);
|
||||
assert!(at(2) > at(1));
|
||||
// Reached, which the doubling capped at 2^6 never did.
|
||||
assert!(at(8) >= DRAIN_BACKOFF_MAX);
|
||||
assert!(at(30) <= DRAIN_BACKOFF_MAX + DRAIN_BACKOFF_MAX / 5);
|
||||
// Ops that failed together do not all come back together.
|
||||
let waits: std::collections::HashSet<_> =
|
||||
(1..=20).map(|id| drain_backoff(8, false, id)).collect();
|
||||
assert!(waits.len() > 5, "{waits:?}");
|
||||
// A wait for a name stays short however often it repeats.
|
||||
assert_eq!(drain_backoff(30, true, 7), DRAIN_BACKOFF_MIN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_fresh_empty_twin_of_the_same_kind_is_adopted() {
|
||||
// The op was queued at t=1000 s, after a create that got no answer.
|
||||
|
||||
@@ -8,8 +8,10 @@
|
||||
//! real client implements it by forwarding, and the simulation tests implement
|
||||
//! it with an in-memory Drive that injects Drive's known faults.
|
||||
//!
|
||||
//! Sharing, Photos, devices, invitations and public links stay on the concrete
|
||||
//! client (`Core::client`) until a test needs them faked.
|
||||
//! Sharing, Photos, invitations, public links and the rest of the device calls
|
||||
//! stay on the concrete client (`Core::client`) until a test needs them faked.
|
||||
//! Listing and registering devices are here, so a simulated daemon can sync a
|
||||
//! local folder.
|
||||
//!
|
||||
//! The method names match the client's own, so a call site reads the same
|
||||
//! whichever it goes through. The client's generic methods (a reader or writer
|
||||
@@ -24,9 +26,10 @@ use futures::StreamExt as _;
|
||||
use futures::stream::BoxStream;
|
||||
use proton_drive_rs::proton_sdk::account::Quota;
|
||||
use proton_drive_rs::proton_sdk::error::Result;
|
||||
use proton_drive_rs::proton_sdk::ids::{DriveEventId, NodeUid};
|
||||
use proton_drive_rs::proton_sdk::ids::{DriveEventId, NodeUid, ShareId};
|
||||
use proton_drive_rs::{
|
||||
DriveEvent, DriveEventScopeId, Node, NodeMoveItem, ProtonDriveClient, RevisionReader, Thumbnail,
|
||||
AvailableNameTarget, Device, DeviceType, DriveEvent, DriveEventScopeId, Node, NodeMoveItem,
|
||||
ProtonDriveClient, RevisionReader, Thumbnail, ThumbnailType,
|
||||
};
|
||||
|
||||
/// One outcome per node of a batch request; see [`ProtonDriveClient::trash_nodes`].
|
||||
@@ -40,6 +43,8 @@ pub(crate) type OutcomeStream<'a> = BoxStream<'a, Result<(NodeUid, Result<()>)>>
|
||||
pub(crate) trait DriveApi: Send + Sync {
|
||||
async fn get_my_files_folder(&self) -> Result<Node>;
|
||||
|
||||
async fn context_share_id(&self, uid: &NodeUid) -> Result<ShareId>;
|
||||
|
||||
async fn get_node(&self, uid: &NodeUid) -> Result<Option<Node>>;
|
||||
|
||||
async fn enumerate_nodes(&self, uids: &[NodeUid]) -> Result<Vec<Node>>;
|
||||
@@ -113,6 +118,14 @@ pub(crate) trait DriveApi: Send + Sync {
|
||||
|
||||
async fn open_revision(&self, uid: &NodeUid) -> Result<Arc<dyn RevisionRead>>;
|
||||
|
||||
/// The file's thumbnail of `thumbnail_type`, or `None` when its revision
|
||||
/// carries none.
|
||||
async fn download_thumbnail(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
thumbnail_type: ThumbnailType,
|
||||
) -> Result<Option<Vec<u8>>>;
|
||||
|
||||
async fn download_file_to_dyn(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
@@ -135,6 +148,22 @@ pub(crate) trait DriveApi: Send + Sync {
|
||||
|
||||
async fn move_node(&self, uid: &NodeUid, new_parent: &NodeUid) -> Result<()>;
|
||||
|
||||
/// See [`ProtonDriveClient::copy_node`].
|
||||
async fn copy_node(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
new_parent: &NodeUid,
|
||||
name: Option<&str>,
|
||||
) -> Result<NodeUid>;
|
||||
|
||||
/// See [`ProtonDriveClient::get_available_name`].
|
||||
async fn get_available_name(
|
||||
&self,
|
||||
parent_uid: &NodeUid,
|
||||
name: &str,
|
||||
target: AvailableNameTarget,
|
||||
) -> Result<String>;
|
||||
|
||||
fn move_nodes_streaming<'a>(
|
||||
&'a self,
|
||||
items: Vec<NodeMoveItem>,
|
||||
@@ -154,6 +183,12 @@ pub(crate) trait DriveApi: Send + Sync {
|
||||
/// The account's storage, across every Proton product.
|
||||
async fn quota(&self) -> Result<Quota>;
|
||||
|
||||
/// Every computer that backs up to the account.
|
||||
async fn enumerate_devices(&self) -> Result<Vec<Device>>;
|
||||
|
||||
/// Register a computer, with a root folder of its own named `name`.
|
||||
async fn create_device(&self, name: &str, device_type: DeviceType) -> Result<Device>;
|
||||
|
||||
/// Called by the drain once it has let go of the blob of an upload to
|
||||
/// `uid` that landed, or retired the create that made `uid`, before the
|
||||
/// tree has the node, so a simulation can hold it there. Blocks the drain
|
||||
@@ -280,6 +315,10 @@ impl RevisionRead for RevisionReader {
|
||||
|
||||
#[async_trait]
|
||||
impl DriveApi for ProtonDriveClient {
|
||||
async fn context_share_id(&self, uid: &NodeUid) -> Result<ShareId> {
|
||||
ProtonDriveClient::context_share_id(self, uid).await
|
||||
}
|
||||
|
||||
async fn get_my_files_folder(&self) -> Result<Node> {
|
||||
ProtonDriveClient::get_my_files_folder(self).await
|
||||
}
|
||||
@@ -321,7 +360,13 @@ impl DriveApi for ProtonDriveClient {
|
||||
name: &str,
|
||||
last_modification_time: Option<i64>,
|
||||
) -> Result<NodeUid> {
|
||||
ProtonDriveClient::create_folder(self, parent_uid, name, last_modification_time).await
|
||||
ProtonDriveClient::create_folder(
|
||||
self,
|
||||
parent_uid,
|
||||
name,
|
||||
last_modification_time.and_then(pdfs_core::timestamp::modification_time),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn upload_file(
|
||||
@@ -353,7 +398,7 @@ impl DriveApi for ProtonDriveClient {
|
||||
reader,
|
||||
intended_size,
|
||||
thumbnails,
|
||||
last_modification_time,
|
||||
last_modification_time.and_then(pdfs_core::timestamp::modification_time),
|
||||
aead,
|
||||
)
|
||||
.await
|
||||
@@ -378,7 +423,7 @@ impl DriveApi for ProtonDriveClient {
|
||||
reader,
|
||||
intended_size,
|
||||
thumbnails,
|
||||
last_modification_time,
|
||||
last_modification_time.and_then(pdfs_core::timestamp::modification_time),
|
||||
aead,
|
||||
)
|
||||
.await
|
||||
@@ -398,7 +443,7 @@ impl DriveApi for ProtonDriveClient {
|
||||
reader,
|
||||
intended_size,
|
||||
thumbnails,
|
||||
last_modification_time,
|
||||
last_modification_time.and_then(pdfs_core::timestamp::modification_time),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -408,6 +453,14 @@ impl DriveApi for ProtonDriveClient {
|
||||
Ok(Arc::new(reader))
|
||||
}
|
||||
|
||||
async fn download_thumbnail(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
thumbnail_type: ThumbnailType,
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
ProtonDriveClient::download_thumbnail(self, uid, thumbnail_type).await
|
||||
}
|
||||
|
||||
async fn download_file_to_dyn(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
@@ -439,6 +492,24 @@ impl DriveApi for ProtonDriveClient {
|
||||
ProtonDriveClient::move_node(self, uid, new_parent).await
|
||||
}
|
||||
|
||||
async fn copy_node(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
new_parent: &NodeUid,
|
||||
name: Option<&str>,
|
||||
) -> Result<NodeUid> {
|
||||
ProtonDriveClient::copy_node(self, uid, new_parent, name).await
|
||||
}
|
||||
|
||||
async fn get_available_name(
|
||||
&self,
|
||||
parent_uid: &NodeUid,
|
||||
name: &str,
|
||||
target: AvailableNameTarget,
|
||||
) -> Result<String> {
|
||||
ProtonDriveClient::get_available_name(self, parent_uid, name, target).await
|
||||
}
|
||||
|
||||
fn move_nodes_streaming<'a>(
|
||||
&'a self,
|
||||
items: Vec<NodeMoveItem>,
|
||||
@@ -470,4 +541,12 @@ impl DriveApi for ProtonDriveClient {
|
||||
async fn quota(&self) -> Result<Quota> {
|
||||
ProtonDriveClient::quota(self).await
|
||||
}
|
||||
|
||||
async fn enumerate_devices(&self) -> Result<Vec<Device>> {
|
||||
ProtonDriveClient::enumerate_devices(self).await
|
||||
}
|
||||
|
||||
async fn create_device(&self, name: &str, device_type: DeviceType) -> Result<Device> {
|
||||
ProtonDriveClient::create_device(self, name, device_type).await
|
||||
}
|
||||
}
|
||||
@@ -235,6 +235,8 @@ fn status_keys(status: &CtlResponse) -> (String, String) {
|
||||
used,
|
||||
budget,
|
||||
online,
|
||||
session_expired,
|
||||
tokens_unsaved,
|
||||
pending_uploads,
|
||||
pending_changes,
|
||||
..
|
||||
@@ -247,7 +249,7 @@ fn status_keys(status: &CtlResponse) -> (String, String) {
|
||||
);
|
||||
let key = format!(
|
||||
"{queue}/{staged_bytes}/{paused}/{paused_until:?}/{username}/{mountpoint}/{pinned}/\
|
||||
{used}/{budget}/{online}"
|
||||
{used}/{budget}/{online}/{session_expired}/{tokens_unsaved}"
|
||||
);
|
||||
(key, queue)
|
||||
}
|
||||
@@ -325,6 +327,9 @@ mod tests {
|
||||
budget: 0,
|
||||
pins: Vec::new(),
|
||||
online: true,
|
||||
session_expired: false,
|
||||
tokens_unsaved: false,
|
||||
version: "3.3.0".into(),
|
||||
pending_uploads: 1,
|
||||
pending_changes: 0,
|
||||
};
|
||||
|
||||
+382
-133
@@ -261,10 +261,13 @@ impl Filesystem for ProtonFs {
|
||||
// in-flight (possibly unsaved) content: authored bytes come from the
|
||||
// scratch file, untouched bytes from the base.
|
||||
//
|
||||
// This path stays on the dispatch loop. `write` runs there too, so
|
||||
// keeping its reads there as well preserves the ordering the write path
|
||||
// has always had between a write and a read of the same handle; a mostly
|
||||
// local read is not worth reasoning about that for.
|
||||
// Answered on the dispatch loop when everything it needs is on this
|
||||
// machine, which keeps the order `write` has with reads of the same
|
||||
// handle. A base range that has to be downloaded goes to a worker
|
||||
// instead, with the handle as it was when the read came in: the kernel
|
||||
// issued the read before any write that comes after it, so that is
|
||||
// the content it asked for. On the loop, the download held every
|
||||
// request of the mount (`docs/BUGS.md` B227).
|
||||
let handle = {
|
||||
let st = self.core.state();
|
||||
st.active_writes.get(&ino.0).map(|h| {
|
||||
@@ -279,7 +282,7 @@ impl Filesystem for ProtonFs {
|
||||
})
|
||||
};
|
||||
if let Some((file, len, uid, base_mtime, base_size, written)) = handle {
|
||||
match self.core.serve_open_read(
|
||||
match self.core.serve_open_read_local(
|
||||
&file,
|
||||
len,
|
||||
&uid,
|
||||
@@ -289,7 +292,25 @@ impl Filesystem for ProtonFs {
|
||||
offset,
|
||||
size as u64,
|
||||
) {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Ok(Some(bytes)) => reply.data(&bytes),
|
||||
Ok(None) => {
|
||||
let fs = self.clone();
|
||||
self.core.workers.run(Lane::Transfer, "read", move || {
|
||||
match fs.core.serve_open_read(
|
||||
&file,
|
||||
len,
|
||||
&uid,
|
||||
base_mtime,
|
||||
base_size,
|
||||
&written,
|
||||
offset,
|
||||
size as u64,
|
||||
) {
|
||||
Ok(bytes) => reply.data(&bytes),
|
||||
Err(e) => reply.error(e),
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(e) => reply.error(e),
|
||||
}
|
||||
return;
|
||||
@@ -409,58 +430,10 @@ impl Filesystem for ProtonFs {
|
||||
reply.error(error);
|
||||
return;
|
||||
}
|
||||
let fh = fh.0;
|
||||
// Stage the bytes straight into the scratch file (no base download): only
|
||||
// the untouched remainder is pulled from the remote, and only at commit.
|
||||
let file = {
|
||||
let st = self.core.state();
|
||||
match st.handles.get(&fh).and_then(|&i| st.active_writes.get(&i)) {
|
||||
Some(aw) => aw.file.clone(),
|
||||
None => {
|
||||
reply.error(Errno::EBADF);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Err(e) = file.write_all_at(data, offset) {
|
||||
error!(ino = ino.0, fh, error = %e, "scratch write failed");
|
||||
// A full local disk is not a broken file: answer `ENOSPC` so the
|
||||
// application can say so (and so `cp`/editors take their own
|
||||
// out-of-space path), and free what the cache is holding — the
|
||||
// scratch file the next write goes to shares that filesystem.
|
||||
if e.raw_os_error() == Some(libc::ENOSPC) {
|
||||
self.core.cache.emergency_evict();
|
||||
reply.error(Errno::ENOSPC);
|
||||
} else {
|
||||
reply.error(Errno::EIO);
|
||||
}
|
||||
return;
|
||||
match self.stage_write(ino.0, fh.0, offset, data) {
|
||||
Ok(()) => reply.written(data.len() as u32),
|
||||
Err(error) => reply.error(error),
|
||||
}
|
||||
let new_len = {
|
||||
let mut st = self.core.state();
|
||||
let Some(aw) = st
|
||||
.handles
|
||||
.get(&fh)
|
||||
.copied()
|
||||
.and_then(|i| st.active_writes.get_mut(&i))
|
||||
else {
|
||||
reply.error(Errno::EBADF);
|
||||
return;
|
||||
};
|
||||
aw.record_write(offset, offset + data.len() as u64);
|
||||
let len = aw.len;
|
||||
st.set_size(ino.0, len);
|
||||
len
|
||||
};
|
||||
debug!(
|
||||
ino = ino.0,
|
||||
fh,
|
||||
offset,
|
||||
len = data.len(),
|
||||
new_len,
|
||||
"staged write"
|
||||
);
|
||||
reply.written(data.len() as u32);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -556,48 +529,62 @@ impl Filesystem for ProtonFs {
|
||||
return;
|
||||
}
|
||||
let new_end = offset.saturating_add(length);
|
||||
let keep_size = (mode & 1) != 0; // FALLOC_FL_KEEP_SIZE
|
||||
|
||||
let is_punch_hole = (mode & 0x02) != 0; // FALLOC_FL_PUNCH_HOLE
|
||||
|
||||
let res = {
|
||||
// Only the modes whose effect on the scratch file this handle can
|
||||
// record. Anything else went through to the scratch file untracked:
|
||||
// a zeroed range was refilled from the base at commit, and a collapse
|
||||
// or insert shifted bytes that `len` and `written` did not follow
|
||||
// (`docs/BUGS.md` B224). `EOPNOTSUPP` is also what makes glibc's
|
||||
// `posix_fallocate` fall back to writing zeros.
|
||||
let Some(kind) = falloc_kind(mode) else {
|
||||
reply.error(Errno::EOPNOTSUPP);
|
||||
return;
|
||||
};
|
||||
let file = {
|
||||
let st = self.core.state();
|
||||
match st.handles.get(&_fh.0).and_then(|i| st.active_writes.get(i)) {
|
||||
Some(aw) => aw.file.clone(),
|
||||
None => {
|
||||
reply.error(if st.entries.contains_key(&ino.0) {
|
||||
Errno::EBADF
|
||||
} else {
|
||||
Errno::ENOENT
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
// The syscall can take a while on a large range; the state lock is
|
||||
// not held across it, as `write` does not hold it across its I/O.
|
||||
let ret = {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
unsafe { libc::fallocate(file.as_raw_fd(), mode, offset as i64, length as i64) }
|
||||
};
|
||||
let res = if ret != 0 {
|
||||
Err(Errno::from(std::io::Error::last_os_error()))
|
||||
} else {
|
||||
let mut st = self.core.state();
|
||||
match st
|
||||
if let Some(aw) = st
|
||||
.handles
|
||||
.get(&_fh.0)
|
||||
.copied()
|
||||
.and_then(|i| st.active_writes.get_mut(&i))
|
||||
{
|
||||
Some(aw) => {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
let fd = aw.file.as_raw_fd();
|
||||
let ret = unsafe { libc::fallocate(fd, mode, offset as i64, length as i64) };
|
||||
if ret == 0 {
|
||||
if is_punch_hole {
|
||||
// A punched range reads as zero. Mark it authored so
|
||||
// queue_revision does not refill it from the remote
|
||||
// baseline and silently undo the hole at commit.
|
||||
aw.written.add(offset, new_end.min(aw.len));
|
||||
aw.dirty = true;
|
||||
} else if !keep_size && new_end > aw.len {
|
||||
aw.written.add(aw.len, new_end);
|
||||
aw.len = new_end;
|
||||
aw.dirty = true;
|
||||
st.set_size(ino.0, new_end);
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Errno::EIO)
|
||||
}
|
||||
let len = aw.len;
|
||||
let (authored, grown) = kind.effect(offset, new_end, len);
|
||||
if let Some((from, to)) = authored {
|
||||
// Zeroed or punched bytes read as zero. Marked authored so
|
||||
// queue_revision does not refill them from the remote
|
||||
// baseline and silently undo them at commit.
|
||||
aw.written.add(from, to);
|
||||
aw.dirty = true;
|
||||
}
|
||||
None => {
|
||||
if st.entries.contains_key(&ino.0) {
|
||||
Err(Errno::EBADF)
|
||||
} else {
|
||||
Err(Errno::ENOENT)
|
||||
}
|
||||
if let Some(end) = grown {
|
||||
aw.len = end;
|
||||
aw.dirty = true;
|
||||
st.set_size(ino.0, end);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
match res {
|
||||
@@ -606,18 +593,47 @@ impl Filesystem for ProtonFs {
|
||||
}
|
||||
}
|
||||
|
||||
/// `close(2)` calls flush before release. The upload is queued in `release`
|
||||
/// and performed in the background, so there is nothing
|
||||
/// to push here — the written bytes are already in the scratch file.
|
||||
/// `close(2)` calls flush before release, and flush is the only one of the
|
||||
/// two whose answer reaches the program: the kernel drops `release`'s. So
|
||||
/// the scratch file of a handle with writes on it is synced here, on a
|
||||
/// worker, and a full disk or a failing one is what `close` returns.
|
||||
///
|
||||
/// Before, flush answered `ok` at once and `release` synced the file, so an
|
||||
/// `ENOSPC` that delayed allocation held back until then was reported to
|
||||
/// nobody, and the program took the save as done (`docs/BUGS.md` B222).
|
||||
/// It also means `release`, which stays on the dispatch loop for ordering,
|
||||
/// finds the pages already on disk instead of writing back a whole file
|
||||
/// while every other request on the mount waits.
|
||||
fn flush(
|
||||
&self,
|
||||
_req: &Request,
|
||||
_ino: INodeNo,
|
||||
_fh: FileHandle,
|
||||
fh: FileHandle,
|
||||
_lock_owner: LockOwner,
|
||||
reply: ReplyEmpty,
|
||||
) {
|
||||
reply.ok();
|
||||
let file = {
|
||||
let st = self.core.state();
|
||||
st.handles
|
||||
.get(&fh.0)
|
||||
.and_then(|ino| st.active_writes.get(ino))
|
||||
.filter(|aw| aw.dirty)
|
||||
.map(|aw| aw.file.clone())
|
||||
};
|
||||
let Some(file) = file else {
|
||||
reply.ok();
|
||||
return;
|
||||
};
|
||||
let fs = self.clone();
|
||||
self.core
|
||||
.workers
|
||||
.run(Lane::Transfer, "flush", move || match file.sync_data() {
|
||||
Ok(()) => reply.ok(),
|
||||
Err(e) => {
|
||||
error!(fh = fh.0, error = %e, "flush of scratch file failed");
|
||||
reply.error(fs.scratch_sync_errno(&e));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Durability here means the scratch file, not the remote: a caller asking
|
||||
@@ -655,7 +671,7 @@ impl Filesystem for ProtonFs {
|
||||
reply: ReplyEmpty,
|
||||
) {
|
||||
let ino = self.tree(ino);
|
||||
let (handle, unlinked_uid) = {
|
||||
let (handle, unlinked_uid, releasing) = {
|
||||
let mut st = self.core.state();
|
||||
let unlinked_uid = release_unlinked_entry(&mut st, ino.0);
|
||||
let ino = st.handles.remove(&fh.0);
|
||||
@@ -668,7 +684,11 @@ impl Filesystem for ProtonFs {
|
||||
None
|
||||
}
|
||||
});
|
||||
(h, unlinked_uid)
|
||||
let releasing = ino.filter(|_| h.is_some());
|
||||
if let Some(ino) = releasing {
|
||||
st.releasing.insert(ino);
|
||||
}
|
||||
(h, unlinked_uid, releasing)
|
||||
};
|
||||
// Deliberately **not** handed to a worker, unlike the other slow
|
||||
// handlers here. The kernel does not wait for a `release` reply before
|
||||
@@ -685,6 +705,9 @@ impl Filesystem for ProtonFs {
|
||||
// finish it (audit F1). What is left is a rename, two fsyncs, a database
|
||||
// insert, and at most a bounded copy out of the local cache.
|
||||
self.finish_release(handle, unlinked_uid, reply);
|
||||
if let Some(ino) = releasing {
|
||||
self.core.state().releasing.remove(&ino);
|
||||
}
|
||||
}
|
||||
|
||||
fn unlink(&self, _req: &Request, parent: INodeNo, name: &OsStr, reply: ReplyEmpty) {
|
||||
@@ -952,20 +975,59 @@ impl Filesystem for ProtonFs {
|
||||
fn copy_file_range(
|
||||
&self,
|
||||
_req: &Request,
|
||||
_ino_in: INodeNo,
|
||||
ino_in: INodeNo,
|
||||
_fh_in: FileHandle,
|
||||
_offset_in: u64,
|
||||
_ino_out: INodeNo,
|
||||
_fh_out: FileHandle,
|
||||
_offset_out: u64,
|
||||
_len: u64,
|
||||
_flags: CopyFileRangeFlags,
|
||||
offset_in: u64,
|
||||
ino_out: INodeNo,
|
||||
fh_out: FileHandle,
|
||||
offset_out: u64,
|
||||
len: u64,
|
||||
flags: CopyFileRangeFlags,
|
||||
reply: ReplyWrite,
|
||||
) {
|
||||
// Also must stay ENOSYS: there is no server-side copy, and this is the
|
||||
// errno that sends coreutils (and the kernel) down the read/write path
|
||||
// instead of failing the copy outright.
|
||||
reply.error(Errno::ENOSYS);
|
||||
// Copied here when the source bytes are on this machine, as `write`
|
||||
// would stage them, minus a FUSE round trip per page: `cp` of a cached
|
||||
// 256 MiB file ran at about 100 MiB/s through read and write. Anything
|
||||
// that would have to be downloaded gets `EOPNOTSUPP`, which makes the
|
||||
// kernel copy that one call through read and write itself; `ENOSYS`
|
||||
// would turn the op off for the whole mount. There is no server-side
|
||||
// copy into a file `cp` has already created, so the copy still uploads.
|
||||
//
|
||||
// On the dispatch loop, like `write`, so it keeps its order with the
|
||||
// reads and writes of the same handle. [`COPY_RANGE_MAX`] bounds how
|
||||
// long one call holds the loop.
|
||||
let (ino_in, ino_out) = (self.tree(ino_in), self.tree(ino_out));
|
||||
if !flags.is_empty() || ino_in == ino_out {
|
||||
reply.error(Errno::EOPNOTSUPP);
|
||||
return;
|
||||
}
|
||||
if let Err(error) = self.core.require_writable(ino_out.0) {
|
||||
reply.error(error);
|
||||
return;
|
||||
}
|
||||
let bytes = match self
|
||||
.core
|
||||
.read_local(ino_in.0, offset_in, len.min(COPY_RANGE_MAX))
|
||||
{
|
||||
Ok(Some(bytes)) => bytes,
|
||||
Ok(None) => {
|
||||
debug!(ino_in = ino_in.0, offset_in, "copy range not local");
|
||||
reply.error(Errno::EOPNOTSUPP);
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
reply.error(error);
|
||||
return;
|
||||
}
|
||||
};
|
||||
if bytes.is_empty() {
|
||||
reply.written(0);
|
||||
return;
|
||||
}
|
||||
match self.stage_write(ino_out.0, fh_out.0, offset_out, &bytes) {
|
||||
Ok(()) => reply.written(bytes.len() as u32),
|
||||
Err(error) => reply.error(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -993,6 +1055,11 @@ pub struct ProtonFs {
|
||||
/// worker busy indefinitely.
|
||||
const OPEN_BASE_ATTEMPTS: u32 = 3;
|
||||
|
||||
/// The most one `copy_file_range` call copies; the caller loops for the rest.
|
||||
/// A local copy of this much takes milliseconds, which is how long it holds
|
||||
/// the dispatch loop.
|
||||
const COPY_RANGE_MAX: u64 = 16 * 1024 * 1024;
|
||||
|
||||
/// One directory's entries as `readdir` reports them: `.` and `..` first, then
|
||||
/// the interned children in `children` order.
|
||||
///
|
||||
@@ -1022,6 +1089,52 @@ fn build_listing(st: &State, ino: u64) -> DirListing {
|
||||
/// The cookie handed back is the index of the *next* entry, so a resumed call
|
||||
/// continues exactly where the last one stopped — which is only true because
|
||||
/// the listing behind those indexes is a snapshot (`docs/BUGS.md` B43).
|
||||
/// What an `fallocate` mode does to a write handle, for the modes it can
|
||||
/// record (`docs/BUGS.md` B224).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum FallocKind {
|
||||
/// Reserve space; past the end it grows the file unless `keep_size`.
|
||||
Allocate { keep_size: bool },
|
||||
/// Deallocate a range inside the file, which then reads as zero.
|
||||
Punch,
|
||||
/// Zero a range; past the end it grows the file unless `keep_size`.
|
||||
Zero { keep_size: bool },
|
||||
}
|
||||
|
||||
fn falloc_kind(mode: i32) -> Option<FallocKind> {
|
||||
const KEEP: i32 = libc::FALLOC_FL_KEEP_SIZE;
|
||||
const PUNCH: i32 = libc::FALLOC_FL_PUNCH_HOLE;
|
||||
const ZERO: i32 = libc::FALLOC_FL_ZERO_RANGE;
|
||||
match mode {
|
||||
0 => Some(FallocKind::Allocate { keep_size: false }),
|
||||
KEEP => Some(FallocKind::Allocate { keep_size: true }),
|
||||
m if m == PUNCH | KEEP => Some(FallocKind::Punch),
|
||||
ZERO => Some(FallocKind::Zero { keep_size: false }),
|
||||
m if m == ZERO | KEEP => Some(FallocKind::Zero { keep_size: true }),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl FallocKind {
|
||||
/// For `[offset, end)` on a file `len` bytes long: the range that now
|
||||
/// reads as zero and must count as written, and the new length if it grew.
|
||||
fn effect(self, offset: u64, end: u64, len: u64) -> (Option<(u64, u64)>, Option<u64>) {
|
||||
let grow = |keep_size: bool| (!keep_size && end > len).then_some(end);
|
||||
let zeroed = |to: u64| (offset < to).then_some((offset, to));
|
||||
match self {
|
||||
// Space past the old end reads as zero, and nothing before it moved.
|
||||
Self::Allocate { keep_size } => {
|
||||
(grow(keep_size).map(|end| (len, end)), grow(keep_size))
|
||||
}
|
||||
Self::Punch => (zeroed(end.min(len)), None),
|
||||
Self::Zero { keep_size } => {
|
||||
let to = if keep_size { end.min(len) } else { end };
|
||||
(zeroed(to), grow(keep_size))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reply_listing(listing: &DirListing, offset: u64, root: u64, mut reply: ReplyDirectory) {
|
||||
for (i, (ino, is_dir, name)) in listing.iter().enumerate().skip(offset as usize) {
|
||||
let ft = if *is_dir {
|
||||
@@ -1267,8 +1380,12 @@ impl ProtonFs {
|
||||
//
|
||||
// Complete pending blobs are the common case and can be copied without
|
||||
// any network access. An incomplete blob still has gaps referring to
|
||||
// the remote base; stacking another write on it cannot be represented
|
||||
// safely by WriteHandle, so fail the open rather than risk corruption.
|
||||
// the remote base, and stacking another write on it cannot be
|
||||
// represented by WriteHandle, so the copy's gaps are filled from that
|
||||
// base first, as the drain fills them before an upload. Only when that
|
||||
// fails (offline, or the base is gone) does the open fail: refusing
|
||||
// every such open failed a create over a file whose partial edit had
|
||||
// not drained yet (`docs/BUGS.md` B204).
|
||||
//
|
||||
// Reading that blob and installing the handle cannot be done under one
|
||||
// lock — the copy is the size of the file — so the base is re-checked at
|
||||
@@ -1302,10 +1419,6 @@ impl ProtonFs {
|
||||
}
|
||||
}
|
||||
let pending_base = self.core.pending.lock().get(&uid).cloned();
|
||||
if pending_base.as_ref().is_some_and(|p| !p.meta.complete) {
|
||||
error!(%uid, "refusing write over incomplete queued revision");
|
||||
break 'install Err(Errno::EIO);
|
||||
}
|
||||
let (file, path) = match self.core.cache.create_scratch() {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
@@ -1321,10 +1434,37 @@ impl ProtonFs {
|
||||
// (`ContentCache::clone_or_copy`).
|
||||
if let Err(e) = ContentCache::clone_or_copy(&pending.path, &path) {
|
||||
let _ = std::fs::remove_file(&path);
|
||||
// The drain lets go of the pending entry before it
|
||||
// discards the blob, so a blob that went away under the
|
||||
// copy is a revision that landed or was replaced: the
|
||||
// same race the re-check below catches, only earlier.
|
||||
if self.core.pending_blob(&uid).as_ref() != Some(&pending.path) {
|
||||
warn!(%uid, attempt, "queued revision went away while opening for write; retrying");
|
||||
continue;
|
||||
}
|
||||
error!(%uid, source = %pending.path.display(), error = %e,
|
||||
"copy queued revision into write scratch failed");
|
||||
break 'install Err(Errno::EIO);
|
||||
}
|
||||
if !pending.meta.complete {
|
||||
let mut authored = Intervals::default();
|
||||
for &(s, e) in &pending.meta.authored {
|
||||
authored.add(s, e);
|
||||
}
|
||||
if let Err(error) = self.core.fill_gaps(
|
||||
&uid,
|
||||
&file,
|
||||
pending.meta.len,
|
||||
pending.meta.base_mtime,
|
||||
pending.meta.base_size,
|
||||
&authored,
|
||||
) {
|
||||
let _ = std::fs::remove_file(&path);
|
||||
error!(%uid, ?error, "cannot complete the queued revision a write open is based on");
|
||||
break 'install Err(error);
|
||||
}
|
||||
debug!(%uid, "write open completed an incomplete queued revision");
|
||||
}
|
||||
initial_written.add(0, pending.meta.len);
|
||||
}
|
||||
// Sampled *before* the state lock, never under it: no site in
|
||||
@@ -1416,6 +1556,20 @@ impl ProtonFs {
|
||||
}
|
||||
|
||||
/// The body of [`Filesystem::fsync`], run off the dispatch loop.
|
||||
/// The errno to give a program for a failed sync of a scratch file.
|
||||
/// Delayed allocation means a full disk usually surfaces at the sync
|
||||
/// rather than at `write(2)`, and `ENOSPC` is what a program checks for.
|
||||
fn scratch_sync_errno(&self, e: &std::io::Error) -> Errno {
|
||||
match e.raw_os_error() {
|
||||
Some(libc::ENOSPC) => {
|
||||
self.core.cache.emergency_evict();
|
||||
Errno::ENOSPC
|
||||
}
|
||||
Some(libc::EDQUOT) => Errno::EDQUOT,
|
||||
_ => Errno::EIO,
|
||||
}
|
||||
}
|
||||
|
||||
fn serve_fsync(&self, fh: FileHandle, reply: ReplyEmpty) {
|
||||
// Everything the durability sidecar needs, copied out so no lock is held
|
||||
// across the sync.
|
||||
@@ -1436,22 +1590,18 @@ impl ProtonFs {
|
||||
)
|
||||
})
|
||||
};
|
||||
// A handle opened for reading has nothing of its own to make durable,
|
||||
// and POSIX allows `fsync` on it; `EBADF` failed `sync FILE` and
|
||||
// backup tools that sync what they read (`docs/BUGS.md` B222).
|
||||
let Some((f, path, uid, dirty, written, len, base_size, base_mtime, base_revision_id)) =
|
||||
handle
|
||||
else {
|
||||
reply.error(Errno::EBADF);
|
||||
reply.ok();
|
||||
return;
|
||||
};
|
||||
if let Err(e) = f.sync_all() {
|
||||
error!(fh = fh.0, error = %e, "fsync of scratch file failed");
|
||||
// Delayed allocation means a full disk usually surfaces here rather
|
||||
// than at `write(2)`; `ENOSPC` is what an application checks for.
|
||||
if e.raw_os_error() == Some(libc::ENOSPC) {
|
||||
self.core.cache.emergency_evict();
|
||||
reply.error(Errno::ENOSPC);
|
||||
} else {
|
||||
reply.error(Errno::EIO);
|
||||
}
|
||||
reply.error(self.scratch_sync_errno(&e));
|
||||
return;
|
||||
}
|
||||
// A clean handle has nothing on it that the remote does not already
|
||||
@@ -1502,6 +1652,48 @@ impl ProtonFs {
|
||||
}
|
||||
|
||||
/// The body of [`Filesystem::create`], run off the dispatch loop.
|
||||
/// Stage `data` at `offset` of the write handle `fh` on `ino`, straight into
|
||||
/// its scratch file (no base download): only the untouched remainder is
|
||||
/// pulled from the remote, and only at commit.
|
||||
fn stage_write(&self, ino: u64, fh: u64, offset: u64, data: &[u8]) -> Result<(), Errno> {
|
||||
let file = {
|
||||
let st = self.core.state();
|
||||
match st.handles.get(&fh).and_then(|&i| st.active_writes.get(&i)) {
|
||||
Some(aw) => aw.file.clone(),
|
||||
None => return Err(Errno::EBADF),
|
||||
}
|
||||
};
|
||||
if let Err(e) = file.write_all_at(data, offset) {
|
||||
error!(ino, fh, error = %e, "scratch write failed");
|
||||
// A full local disk is not a broken file: answer `ENOSPC` so the
|
||||
// application can say so (and so `cp`/editors take their own
|
||||
// out-of-space path), and free what the cache is holding — the
|
||||
// scratch file the next write goes to shares that filesystem.
|
||||
if e.raw_os_error() == Some(libc::ENOSPC) {
|
||||
self.core.cache.emergency_evict();
|
||||
return Err(Errno::ENOSPC);
|
||||
}
|
||||
return Err(Errno::EIO);
|
||||
}
|
||||
let new_len = {
|
||||
let mut st = self.core.state();
|
||||
let Some(aw) = st
|
||||
.handles
|
||||
.get(&fh)
|
||||
.copied()
|
||||
.and_then(|i| st.active_writes.get_mut(&i))
|
||||
else {
|
||||
return Err(Errno::EBADF);
|
||||
};
|
||||
aw.record_write(offset, offset + data.len() as u64);
|
||||
let len = aw.len;
|
||||
st.set_size(ino, len);
|
||||
len
|
||||
};
|
||||
debug!(ino, fh, offset, len = data.len(), new_len, "staged write");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn serve_create(&self, parent: u64, name: &str, reply: ReplyCreate) {
|
||||
if let Err(error) = self.core.require_writable(parent) {
|
||||
reply.error(error);
|
||||
@@ -1698,9 +1890,7 @@ impl ProtonFs {
|
||||
// empty, which it is, so reads stay offline-clean across a restart too.
|
||||
if local {
|
||||
st.children.insert(ino, Vec::new());
|
||||
if let Err(e) = self.core.db.set_listed(&uid, true) {
|
||||
warn!(%uid, error = %e, "db set_listed(true) failed for local folder");
|
||||
}
|
||||
st.record_listed(uid);
|
||||
}
|
||||
let entry = st.entries.get(&ino).unwrap();
|
||||
let attr = self.attr(ino, &entry.node, entry.access);
|
||||
@@ -2182,17 +2372,19 @@ impl ProtonFs {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Offline: queue it. Trashing is the one mutation a user expects to work
|
||||
// regardless — the file is gone from their point of view the moment the
|
||||
// command returns. The same when the network goes
|
||||
// away under the remote call below.
|
||||
let queue_it = |reply: ReplyEmpty| match self.core.queue_trash(&uid, name) {
|
||||
// Queue a folder trash while Drive still holds children that were moved
|
||||
// out locally. The drain waits for those moves before sending it.
|
||||
let queue_it = |reply: ReplyEmpty| match self.core.queue_trash(&uid, name, open_now) {
|
||||
Ok(()) => reply.ok(),
|
||||
Err(e) => reply.error(e),
|
||||
};
|
||||
if !self.core.sends_inline() {
|
||||
queue_it(reply);
|
||||
return;
|
||||
return queue_it(reply);
|
||||
}
|
||||
match self.core.has_pending_moves_out_of(&uid) {
|
||||
Ok(false) => {}
|
||||
Ok(true) => return queue_it(reply),
|
||||
Err(error) => return reply.error(error),
|
||||
}
|
||||
if let Err(e) = self.core.require_uid_writable(&uid) {
|
||||
reply.error(e);
|
||||
@@ -2318,3 +2510,60 @@ impl ProtonFs {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod falloc_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_the_modes_a_handle_can_record_are_accepted() {
|
||||
let keep = libc::FALLOC_FL_KEEP_SIZE;
|
||||
assert_eq!(
|
||||
falloc_kind(0),
|
||||
Some(FallocKind::Allocate { keep_size: false })
|
||||
);
|
||||
assert_eq!(
|
||||
falloc_kind(keep),
|
||||
Some(FallocKind::Allocate { keep_size: true })
|
||||
);
|
||||
assert_eq!(
|
||||
falloc_kind(libc::FALLOC_FL_PUNCH_HOLE | keep),
|
||||
Some(FallocKind::Punch)
|
||||
);
|
||||
assert_eq!(
|
||||
falloc_kind(libc::FALLOC_FL_ZERO_RANGE),
|
||||
Some(FallocKind::Zero { keep_size: false })
|
||||
);
|
||||
for mode in [
|
||||
libc::FALLOC_FL_PUNCH_HOLE,
|
||||
libc::FALLOC_FL_COLLAPSE_RANGE,
|
||||
libc::FALLOC_FL_INSERT_RANGE,
|
||||
libc::FALLOC_FL_UNSHARE_RANGE,
|
||||
] {
|
||||
assert_eq!(falloc_kind(mode), None, "{mode:#x}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A zeroed range was left unwritten, so the commit filled it back in
|
||||
/// from the remote base (`docs/BUGS.md` B224).
|
||||
#[test]
|
||||
fn a_zeroed_or_punched_range_counts_as_written() {
|
||||
let zero = FallocKind::Zero { keep_size: false };
|
||||
assert_eq!(zero.effect(10, 20, 100), (Some((10, 20)), None));
|
||||
assert_eq!(zero.effect(90, 120, 100), (Some((90, 120)), Some(120)));
|
||||
let zero_keep = FallocKind::Zero { keep_size: true };
|
||||
assert_eq!(zero_keep.effect(90, 120, 100), (Some((90, 100)), None));
|
||||
assert_eq!(
|
||||
FallocKind::Punch.effect(90, 120, 100),
|
||||
(Some((90, 100)), None)
|
||||
);
|
||||
assert_eq!(FallocKind::Punch.effect(120, 130, 100), (None, None));
|
||||
let alloc = FallocKind::Allocate { keep_size: false };
|
||||
assert_eq!(alloc.effect(0, 50, 100), (None, None));
|
||||
assert_eq!(alloc.effect(90, 120, 100), (Some((100, 120)), Some(120)));
|
||||
assert_eq!(
|
||||
FallocKind::Allocate { keep_size: true }.effect(90, 120, 100),
|
||||
(None, None)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,653 @@
|
||||
//! Host Proton Gallery's account and catalog services in the Drive daemon.
|
||||
|
||||
mod video;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{
|
||||
Arc, RwLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use pdfs_core::control::{
|
||||
GalleryAction, GalleryAlbum, GalleryFolder, GalleryPlace, PhotoItem, PhotoKind, Response,
|
||||
};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use proton_drive_rs::proton_sdk::session::ProtonApiSession;
|
||||
use proton_gallery_app_rs::{App, View};
|
||||
use proton_gallery_rs::CancellationToken;
|
||||
use proton_gallery_rs::model::{AssetKey, SourceId};
|
||||
use tokio::sync::{Mutex, Notify, OnceCell};
|
||||
|
||||
use super::{Core, NodeUid, parse_uid};
|
||||
|
||||
pub(super) struct Gallery {
|
||||
root: PathBuf,
|
||||
session: ProtonApiSession,
|
||||
cancel: CancellationToken,
|
||||
app: OnceCell<App>,
|
||||
checked: Mutex<Option<Instant>>,
|
||||
visibility: RwLock<Option<Arc<Visibility>>>,
|
||||
refresh: Notify,
|
||||
access: AtomicU64,
|
||||
video: video::Server,
|
||||
}
|
||||
|
||||
#[derive(Default, PartialEq, Eq)]
|
||||
pub(super) struct Visibility {
|
||||
pub(super) media: HashSet<String>,
|
||||
pub(super) albums: HashSet<String>,
|
||||
}
|
||||
|
||||
impl Visibility {
|
||||
pub(super) fn permits(&self, uid: &str) -> CoreResult<()> {
|
||||
if self.media.contains(uid) || self.albums.contains(uid) {
|
||||
return Err(CoreError::invalid("Unlock hidden albums to open this item"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Gallery {
|
||||
pub(super) fn new(root: PathBuf, session: ProtonApiSession) -> Self {
|
||||
Self {
|
||||
root,
|
||||
session,
|
||||
cancel: CancellationToken::new(),
|
||||
app: OnceCell::new(),
|
||||
checked: Mutex::new(None),
|
||||
visibility: RwLock::default(),
|
||||
refresh: Notify::new(),
|
||||
access: AtomicU64::new(0),
|
||||
video: video::Server::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn cancel(&self) {
|
||||
self.cancel.cancel();
|
||||
self.video.close_all();
|
||||
}
|
||||
|
||||
async fn app(&self, core: &Core) -> CoreResult<&App> {
|
||||
self.app
|
||||
.get_or_try_init(|| async {
|
||||
App::open_hosted(
|
||||
self.root.clone(),
|
||||
self.session.clone(),
|
||||
core.client.clone(),
|
||||
self.cancel.child_token(),
|
||||
)
|
||||
.await
|
||||
.map_err(gallery_error)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn ensure_visibility(
|
||||
&self,
|
||||
core: &Core,
|
||||
app: &App,
|
||||
force: bool,
|
||||
) -> CoreResult<Arc<Visibility>> {
|
||||
if self.cancel.is_cancelled() {
|
||||
return Err(CoreError::remote("The Gallery session expired"));
|
||||
}
|
||||
let mut checked = self.checked.lock().await;
|
||||
if !force
|
||||
&& let Some(instant) = *checked
|
||||
&& instant.elapsed()
|
||||
< if self.snapshot().is_some() {
|
||||
Duration::from_secs(60)
|
||||
} else {
|
||||
Duration::from_secs(5)
|
||||
}
|
||||
{
|
||||
return self.snapshot().ok_or_else(visibility_loading);
|
||||
}
|
||||
if self.snapshot().is_none()
|
||||
&& app
|
||||
.gallery_visibility_cached()
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
{
|
||||
self.publish_visibility(core, self.read_visibility(core, app).await?);
|
||||
}
|
||||
let started = Instant::now();
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
_ = self.cancel.cancelled() => Err(CoreError::remote("The Gallery session expired")),
|
||||
result = tokio::time::timeout(Duration::from_secs(20), async {
|
||||
app.refresh_gallery_configuration(false).await.map_err(gallery_error)?;
|
||||
self.read_visibility(core, app).await
|
||||
}) => result.unwrap_or_else(|_| Err(CoreError::remote("Gallery visibility check timed out"))),
|
||||
};
|
||||
tracing::debug!(
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
success = result.is_ok(),
|
||||
"Gallery visibility refresh"
|
||||
);
|
||||
*checked = Some(Instant::now());
|
||||
if let Ok(visibility) = result {
|
||||
self.publish_visibility(core, visibility.clone());
|
||||
return Ok(visibility);
|
||||
}
|
||||
if !force && let Some(visibility) = self.snapshot() {
|
||||
return Ok(visibility);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> Option<Arc<Visibility>> {
|
||||
self.visibility
|
||||
.read()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn publish_visibility(&self, core: &Core, visibility: Arc<Visibility>) {
|
||||
let mut snapshot = self.visibility.write().unwrap_or_else(|e| e.into_inner());
|
||||
if snapshot.as_ref() != Some(&visibility) {
|
||||
*snapshot = Some(visibility);
|
||||
drop(snapshot);
|
||||
self.video.close_all();
|
||||
core.events
|
||||
.publish(&[pdfs_core::control::Topic::GalleryVisibilitySynced]);
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_visibility(&self, core: &Core, app: &App) -> CoreResult<Arc<Visibility>> {
|
||||
let account = self.account()?;
|
||||
let mut visibility = Visibility::default();
|
||||
for (volume, node) in app
|
||||
.catalog
|
||||
.hidden_assets(account.clone())
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
{
|
||||
visibility
|
||||
.media
|
||||
.insert(NodeUid::new(volume.into(), node.into()).to_string());
|
||||
}
|
||||
visibility.media.extend(
|
||||
core.db
|
||||
.photo_group_members_for(&visibility.media)
|
||||
.map_err(CoreError::from)?,
|
||||
);
|
||||
for album in app
|
||||
.catalog
|
||||
.hidden_configuration_albums(account)
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
{
|
||||
visibility.albums.insert(
|
||||
NodeUid::new(album.volume.as_str().into(), album.node.as_str().into()).to_string(),
|
||||
);
|
||||
}
|
||||
Ok(Arc::new(visibility))
|
||||
}
|
||||
|
||||
fn account(&self) -> CoreResult<proton_gallery_rs::model::AccountId> {
|
||||
proton_gallery_rs::model::AccountId::new(self.session.user_id().as_str())
|
||||
.map_err(gallery_error)
|
||||
}
|
||||
|
||||
async fn page(
|
||||
&self,
|
||||
app: &App,
|
||||
album: Option<GalleryPlace>,
|
||||
proton: bool,
|
||||
access: Option<u64>,
|
||||
offset: usize,
|
||||
limit: usize,
|
||||
) -> CoreResult<Response> {
|
||||
let private = access.is_some();
|
||||
let parent = album
|
||||
.as_ref()
|
||||
.map(|a| (a.gallery.as_str(), a.album.as_str()));
|
||||
let mut albums = Vec::new();
|
||||
if offset == 0 && !proton {
|
||||
if private && parent.is_none() {
|
||||
albums.extend(
|
||||
app.photo_albums_with_hidden(true)
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.settings.hidden)
|
||||
.map(|entry| gallery_album(entry, true)),
|
||||
);
|
||||
albums.extend(
|
||||
app.hidden_drive_albums()
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
.into_iter()
|
||||
.map(|entry| gallery_album(entry, false)),
|
||||
);
|
||||
} else {
|
||||
albums.extend(
|
||||
app.drive_albums(parent, private)
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
.into_iter()
|
||||
.map(|entry| gallery_album(entry, false)),
|
||||
);
|
||||
}
|
||||
}
|
||||
let (items, total) = if let Some(GalleryPlace { gallery, album }) = album {
|
||||
let view = if proton && private {
|
||||
let id = SourceId::new(&album).map_err(gallery_error)?;
|
||||
if !app
|
||||
.catalog
|
||||
.album_settings(id)
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
.hidden
|
||||
{
|
||||
return Err(CoreError::not_found("Hidden album"));
|
||||
}
|
||||
View::PhotoAlbum(album)
|
||||
} else {
|
||||
if private
|
||||
&& !app
|
||||
.catalog
|
||||
.drive_album_is_private(
|
||||
SourceId::new(&gallery).map_err(gallery_error)?,
|
||||
SourceId::new(&album).map_err(gallery_error)?,
|
||||
)
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
{
|
||||
return Err(CoreError::not_found("Hidden gallery folder"));
|
||||
}
|
||||
View::DriveAlbum { gallery, album }
|
||||
};
|
||||
let outline = app
|
||||
.outline_with_visibility(view, false, private)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
let total = outline.len();
|
||||
let items = app
|
||||
.outline_items(&outline, offset.min(total), limit.clamp(1, 250))
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|item| PhotoItem {
|
||||
uid: asset_uid(&item.key),
|
||||
capture_time: item.sort_time,
|
||||
thumb_path: None,
|
||||
name: Some(item.name),
|
||||
ratio: item.aspect_ratio,
|
||||
no_thumb: false,
|
||||
kind: if item.is_video {
|
||||
PhotoKind::Video
|
||||
} else {
|
||||
PhotoKind::Photo
|
||||
},
|
||||
favorite: item.favorite,
|
||||
group_size: 1,
|
||||
has_raw: false,
|
||||
place_id: None,
|
||||
})
|
||||
.collect();
|
||||
(items, total)
|
||||
} else {
|
||||
(Vec::new(), 0)
|
||||
};
|
||||
self.require_access(access)?;
|
||||
Ok(Response::GalleryPage {
|
||||
albums,
|
||||
items,
|
||||
total,
|
||||
})
|
||||
}
|
||||
|
||||
fn lock(&self) {
|
||||
let _ = self
|
||||
.access
|
||||
.try_update(Ordering::SeqCst, Ordering::SeqCst, |value| {
|
||||
Some((value | 1).wrapping_add(1))
|
||||
});
|
||||
self.video.close_all();
|
||||
}
|
||||
|
||||
fn private_access(&self) -> CoreResult<u64> {
|
||||
let access = self.access.load(Ordering::SeqCst);
|
||||
if access & 1 == 0 {
|
||||
return Err(CoreError::invalid("Unlock hidden albums to open this item"));
|
||||
}
|
||||
Ok(access)
|
||||
}
|
||||
|
||||
fn require_access(&self, access: Option<u64>) -> CoreResult<()> {
|
||||
if let Some(access) = access
|
||||
&& self.private_access()? != access
|
||||
{
|
||||
return Err(CoreError::invalid("Hidden albums were locked"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn poll(self: Arc<Self>, core: Core) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = self.cancel.cancelled() => break,
|
||||
_ = self.refresh.notified() => {},
|
||||
_ = tokio::time::sleep(Duration::from_secs(60)) => {}
|
||||
}
|
||||
if let Ok(Ok(app)) = tokio::time::timeout(Duration::from_secs(5), self.app(&core)).await
|
||||
{
|
||||
let _ = self.ensure_visibility(&core, app, false).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn asset(&self, uid: &NodeUid) -> CoreResult<AssetKey> {
|
||||
Ok(AssetKey {
|
||||
account: self.account()?,
|
||||
volume: proton_gallery_rs::model::VolumeId::new(uid.volume_id.as_str())
|
||||
.map_err(gallery_error)?,
|
||||
node: proton_gallery_rs::model::NodeId::new(uid.link_id.as_str())
|
||||
.map_err(gallery_error)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn gallery_album(entry: proton_gallery_rs::library::Album, proton: bool) -> GalleryAlbum {
|
||||
GalleryAlbum {
|
||||
place: GalleryPlace {
|
||||
gallery: entry.gallery.to_string(),
|
||||
album: entry.source.id.to_string(),
|
||||
},
|
||||
proton,
|
||||
folder: GalleryFolder {
|
||||
uid: NodeUid::new(
|
||||
entry.source.volume.as_str().into(),
|
||||
entry.source.root.as_str().into(),
|
||||
)
|
||||
.to_string(),
|
||||
name: entry.settings.title.unwrap_or(entry.source.label),
|
||||
},
|
||||
cover: entry.cover.map(|asset| asset_uid(&asset.key)),
|
||||
items: entry.direct_items,
|
||||
folders: entry.child_albums,
|
||||
hidden: entry.settings.hidden,
|
||||
}
|
||||
}
|
||||
|
||||
fn gallery_error(error: impl std::fmt::Display) -> CoreError {
|
||||
CoreError::remote(error.to_string())
|
||||
}
|
||||
|
||||
fn uid(value: &str) -> CoreResult<NodeUid> {
|
||||
parse_uid(value).ok_or_else(|| CoreError::invalid("Invalid gallery node"))
|
||||
}
|
||||
|
||||
fn done() -> Response {
|
||||
Response::Ok {
|
||||
message: "Gallery updated".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn asset_uid(asset: &AssetKey) -> String {
|
||||
NodeUid::new(asset.volume.as_str().into(), asset.node.as_str().into()).to_string()
|
||||
}
|
||||
|
||||
fn visibility_loading() -> CoreError {
|
||||
CoreError::new(
|
||||
pdfs_core::control::ErrorKind::Offline,
|
||||
"Gallery visibility is loading. Try again shortly.",
|
||||
)
|
||||
}
|
||||
|
||||
impl Core {
|
||||
pub(super) fn cached_gallery_visibility(&self) -> CoreResult<Arc<Visibility>> {
|
||||
if self.gallery.is_none() {
|
||||
return Ok(Arc::default());
|
||||
}
|
||||
let gallery = self.gallery_host()?;
|
||||
match gallery.snapshot() {
|
||||
Some(snapshot) => Ok(snapshot),
|
||||
None => {
|
||||
gallery.refresh.notify_one();
|
||||
Err(visibility_loading())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn gallery_visibility(&self) -> CoreResult<Arc<Visibility>> {
|
||||
if self.gallery.is_none() {
|
||||
return Ok(Arc::default());
|
||||
}
|
||||
let gallery = self.gallery_host()?;
|
||||
self.rt.block_on(async {
|
||||
let app = gallery.app(self).await?;
|
||||
gallery.ensure_visibility(self, app, false).await
|
||||
})
|
||||
}
|
||||
|
||||
fn gallery_host(&self) -> CoreResult<&Gallery> {
|
||||
self.gallery
|
||||
.as_deref()
|
||||
.filter(|gallery| !gallery.cancel.is_cancelled())
|
||||
.ok_or_else(|| CoreError::remote("The Gallery session is unavailable"))
|
||||
}
|
||||
|
||||
pub(super) fn authenticate_hidden_gallery(
|
||||
&self,
|
||||
stream: &std::os::unix::net::UnixStream,
|
||||
) -> CoreResult<Response> {
|
||||
let gallery = self.gallery_host()?;
|
||||
let access = gallery.access.load(Ordering::SeqCst);
|
||||
let peer = rustix::net::sockopt::socket_peercred(stream)
|
||||
.map_err(|_| CoreError::invalid("Couldn’t identify the authentication requester."))?;
|
||||
if peer.uid != rustix::process::geteuid() {
|
||||
return Err(CoreError::invalid(
|
||||
"Use your Linux sign-in to unlock Hidden albums.",
|
||||
));
|
||||
}
|
||||
let subject = proton_gallery_app_rs::desktop_auth::Subject::new(
|
||||
peer.pid.as_raw_nonzero().get() as u32,
|
||||
peer.uid.as_raw(),
|
||||
)
|
||||
.map_err(gallery_error)?;
|
||||
self.rt
|
||||
.block_on(subject.authenticate("io.narl.proton-drive-linux.unlock-hidden-albums"))
|
||||
.map_err(gallery_error)?;
|
||||
if gallery.cancel.is_cancelled() {
|
||||
return Err(CoreError::invalid("The Gallery session has ended."));
|
||||
}
|
||||
gallery
|
||||
.access
|
||||
.compare_exchange(access, access | 1, Ordering::SeqCst, Ordering::SeqCst)
|
||||
.map_err(|_| CoreError::invalid("Hidden albums were locked"))?;
|
||||
Ok(done())
|
||||
}
|
||||
|
||||
pub(super) fn drive_gallery(&self, action: GalleryAction) -> CoreResult<Response> {
|
||||
let gallery = self.gallery_host()?;
|
||||
let action = match action {
|
||||
GalleryAction::Lock => {
|
||||
gallery.lock();
|
||||
return Ok(done());
|
||||
}
|
||||
other => other,
|
||||
};
|
||||
self.rt.block_on(async {
|
||||
let app = gallery.app(self).await?;
|
||||
gallery
|
||||
.ensure_visibility(self, app, matches!(action, GalleryAction::Refresh))
|
||||
.await?;
|
||||
match action {
|
||||
GalleryAction::AuthenticateHidden | GalleryAction::Lock => unreachable!(),
|
||||
GalleryAction::List {
|
||||
album,
|
||||
offset,
|
||||
limit,
|
||||
} => gallery.page(app, album, false, None, offset, limit).await,
|
||||
GalleryAction::PrivateList {
|
||||
album,
|
||||
proton,
|
||||
offset,
|
||||
limit,
|
||||
} => {
|
||||
let access = gallery.private_access()?;
|
||||
gallery
|
||||
.page(app, album, proton, Some(access), offset, limit)
|
||||
.await
|
||||
}
|
||||
GalleryAction::Folders { parent } => {
|
||||
let list = app
|
||||
.browse_drive_folders(parent.as_deref().map(uid).transpose()?)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
Ok(Response::GalleryFolders {
|
||||
folders: list
|
||||
.children
|
||||
.into_iter()
|
||||
.map(|folder| GalleryFolder {
|
||||
uid: folder.uid.to_string(),
|
||||
name: folder.name,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
GalleryAction::Add { uid: node } => {
|
||||
app.add_drive_folder(uid(&node)?)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
gallery.ensure_visibility(self, app, true).await?;
|
||||
Ok(done())
|
||||
}
|
||||
GalleryAction::Remove { gallery: id } => {
|
||||
app.forget_drive_gallery(SourceId::new(id).map_err(gallery_error)?)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
gallery.ensure_visibility(self, app, true).await?;
|
||||
Ok(done())
|
||||
}
|
||||
GalleryAction::Refresh => {
|
||||
for album in app.drive_albums(None, true).await.map_err(gallery_error)? {
|
||||
let root = NodeUid::new(
|
||||
album.source.volume.as_str().into(),
|
||||
album.source.root.as_str().into(),
|
||||
);
|
||||
app.scan_drive_folder(root).await.map_err(gallery_error)?;
|
||||
}
|
||||
gallery.ensure_visibility(self, app, true).await?;
|
||||
Ok(done())
|
||||
}
|
||||
GalleryAction::Cover { album, uid: node } => {
|
||||
let id = SourceId::new(&album.album).map_err(gallery_error)?;
|
||||
let root = SourceId::new(&album.gallery).map_err(gallery_error)?;
|
||||
app.checked_album(album.gallery, album.album)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
let access = if app
|
||||
.catalog
|
||||
.drive_album_is_private(root.clone(), id.clone())
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
{
|
||||
Some(gallery.private_access()?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut settings = app
|
||||
.catalog
|
||||
.album_settings(id.clone())
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
gallery.require_access(access)?;
|
||||
settings.cover = Some(gallery.asset(&uid(&node)?)?);
|
||||
app.catalog
|
||||
.set_album_settings_at(Some(root), id, settings)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
gallery.ensure_visibility(self, app, true).await?;
|
||||
Ok(done())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn open_gallery_video(&self, node: &str, private: bool) -> CoreResult<Response> {
|
||||
let gallery = self.gallery_host()?;
|
||||
let uid = uid(node)?;
|
||||
self.rt.block_on(async {
|
||||
let app = gallery.app(self).await?;
|
||||
let visibility = gallery.ensure_visibility(self, app, false).await?;
|
||||
let access = if private {
|
||||
Some(gallery.private_access()?)
|
||||
} else {
|
||||
visibility.permits(node)?;
|
||||
None
|
||||
};
|
||||
let stream = app
|
||||
.stream_video(&gallery.asset(&uid)?)
|
||||
.await
|
||||
.map_err(gallery_error)?;
|
||||
let response = gallery.video.open(stream, &gallery.cancel).await?;
|
||||
if let Err(error) = gallery.require_access(access) {
|
||||
if let Response::GalleryVideo { token, .. } = &response {
|
||||
gallery.video.close(token);
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn gallery_image(&self, node: &str) -> CoreResult<Response> {
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
let gallery = self.gallery_host()?;
|
||||
let access = gallery.private_access()?;
|
||||
let asset = gallery.asset(&uid(node)?)?;
|
||||
self.rt.block_on(async {
|
||||
let app = gallery.app(self).await?;
|
||||
let bytes = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
if let Some(bytes) = app.preview(&asset).await? {
|
||||
Ok(bytes)
|
||||
} else {
|
||||
app.original_in_memory(&asset, 8 * 1024 * 1024).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| CoreError::remote("Proton took too long to open this photo"))?
|
||||
.map_err(gallery_error)?;
|
||||
gallery.require_access(Some(access))?;
|
||||
Ok(Response::GalleryThumbnail {
|
||||
data: Some(STANDARD.encode(bytes)),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn close_gallery_video(&self, token: &str) -> CoreResult<Response> {
|
||||
self.gallery_host()?.video.close(token);
|
||||
Ok(done())
|
||||
}
|
||||
|
||||
pub(super) fn gallery_thumbnail(&self, node: &str, private: bool) -> CoreResult<Response> {
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
let gallery = self.gallery_host()?;
|
||||
let asset = gallery.asset(&uid(node)?)?;
|
||||
self.rt.block_on(async {
|
||||
let app = gallery.app(self).await?;
|
||||
let visibility = gallery.ensure_visibility(self, app, false).await?;
|
||||
let access = if private {
|
||||
Some(gallery.private_access()?)
|
||||
} else {
|
||||
visibility.permits(node)?;
|
||||
None
|
||||
};
|
||||
let data = app
|
||||
.thumbnail(&asset)
|
||||
.await
|
||||
.map_err(gallery_error)?
|
||||
.map(|bytes| STANDARD.encode(bytes));
|
||||
gallery.require_access(access)?;
|
||||
Ok(Response::GalleryThumbnail { data })
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
use std::{collections::VecDeque, io, sync::Arc, time::Duration};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use http_body_util::{BodyExt, Empty, StreamBody, combinators::UnsyncBoxBody};
|
||||
use hyper::{
|
||||
Method, Request, Response, StatusCode,
|
||||
body::{Bytes, Frame, Incoming},
|
||||
header,
|
||||
server::conn::http1,
|
||||
service::service_fn,
|
||||
};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use pdfs_core::{CoreResult, control::Response as ControlResponse};
|
||||
use proton_gallery_app_rs::video::VideoStream;
|
||||
use proton_gallery_rs::CancellationToken;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
type Body = UnsyncBoxBody<Bytes, io::Error>;
|
||||
|
||||
const LEASE: Duration = Duration::from_secs(4 * 60 * 60);
|
||||
const MAX_STREAMS: usize = 4;
|
||||
const CHUNK: u32 = 256 * 1024;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct Server {
|
||||
active: parking_lot::Mutex<VecDeque<(String, CancellationToken)>>,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
pub(super) fn close(&self, token: &str) {
|
||||
let mut active = self.active.lock();
|
||||
if let Some(index) = active.iter().position(|(id, _)| id == token)
|
||||
&& let Some((_, cancel)) = active.remove(index)
|
||||
{
|
||||
cancel.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn close_all(&self) {
|
||||
for (_, cancel) in self.active.lock().drain(..) {
|
||||
cancel.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn open(
|
||||
&self,
|
||||
stream: Arc<VideoStream>,
|
||||
account: &CancellationToken,
|
||||
) -> CoreResult<ControlResponse> {
|
||||
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))
|
||||
.await
|
||||
.map_err(super::gallery_error)?;
|
||||
let address = listener.local_addr().map_err(super::gallery_error)?;
|
||||
let mut nonce = [0; 32];
|
||||
getrandom::fill(&mut nonce).map_err(super::gallery_error)?;
|
||||
let token = URL_SAFE_NO_PAD.encode(nonce);
|
||||
let path: Arc<str> = format!("/{token}").into();
|
||||
let cancel = account.child_token();
|
||||
{
|
||||
let mut active = self.active.lock();
|
||||
active.retain(|(_, cancel)| !cancel.is_cancelled());
|
||||
if active.len() >= MAX_STREAMS
|
||||
&& let Some((_, oldest)) = active.pop_front()
|
||||
{
|
||||
oldest.cancel();
|
||||
}
|
||||
active.push_back((token.clone(), cancel.clone()));
|
||||
}
|
||||
let size = stream.length();
|
||||
tokio::spawn(async move {
|
||||
let mut connections = tokio::task::JoinSet::new();
|
||||
let lifetime = tokio::time::sleep(LEASE);
|
||||
tokio::pin!(lifetime);
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => break,
|
||||
_ = &mut lifetime => break,
|
||||
_ = connections.join_next(), if !connections.is_empty() => {},
|
||||
incoming = listener.accept(), if connections.len() < 8 => {
|
||||
let Ok((socket, _)) = incoming else { break };
|
||||
let stream = stream.clone();
|
||||
let path = path.clone();
|
||||
connections.spawn(async move {
|
||||
let service = service_fn(move |request| reply(request, stream.clone(), path.clone()));
|
||||
let _ = http1::Builder::new()
|
||||
.timer(hyper_util::rt::TokioTimer::new())
|
||||
.header_read_timeout(Duration::from_secs(10))
|
||||
.keep_alive(false)
|
||||
.serve_connection(TokioIo::new(socket), service).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.stop();
|
||||
cancel.cancel();
|
||||
connections.abort_all();
|
||||
});
|
||||
Ok(ControlResponse::GalleryVideo {
|
||||
url: format!("http://{address}/{token}"),
|
||||
token,
|
||||
size,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn range(value: Option<&str>, size: u64) -> Option<(u64, u64)> {
|
||||
let Some(value) = value else {
|
||||
return Some((0, size));
|
||||
};
|
||||
let (first, last) = value.strip_prefix("bytes=")?.split_once('-')?;
|
||||
if first.is_empty() {
|
||||
let suffix = last.parse::<u64>().ok()?.min(size);
|
||||
return (suffix > 0).then_some((size - suffix, size));
|
||||
}
|
||||
let start = first.parse::<u64>().ok()?;
|
||||
let end = if last.is_empty() {
|
||||
size
|
||||
} else {
|
||||
last.parse::<u64>().ok()?.saturating_add(1).min(size)
|
||||
};
|
||||
(start < end).then_some((start, end))
|
||||
}
|
||||
|
||||
fn empty() -> Body {
|
||||
Empty::new().map_err(|never| match never {}).boxed_unsync()
|
||||
}
|
||||
|
||||
async fn reply(
|
||||
request: Request<Incoming>,
|
||||
stream: Arc<VideoStream>,
|
||||
path: Arc<str>,
|
||||
) -> Result<Response<Body>, hyper::http::Error> {
|
||||
let response = Response::builder().header(header::CACHE_CONTROL, "no-store");
|
||||
if request
|
||||
.uri()
|
||||
.path_and_query()
|
||||
.is_none_or(|actual| actual.as_str() != path.as_ref())
|
||||
{
|
||||
return response.status(StatusCode::NOT_FOUND).body(empty());
|
||||
}
|
||||
if !matches!(*request.method(), Method::HEAD | Method::GET) {
|
||||
return response
|
||||
.status(StatusCode::METHOD_NOT_ALLOWED)
|
||||
.body(empty());
|
||||
}
|
||||
let size = stream.length();
|
||||
let requested = request.headers().get(header::RANGE);
|
||||
let parsed = match requested {
|
||||
Some(value) => value
|
||||
.to_str()
|
||||
.ok()
|
||||
.and_then(|value| range(Some(value), size)),
|
||||
None => range(None, size),
|
||||
};
|
||||
let Some((start, end)) = parsed else {
|
||||
return response
|
||||
.status(StatusCode::RANGE_NOT_SATISFIABLE)
|
||||
.header(header::CONTENT_RANGE, format!("bytes */{size}"))
|
||||
.body(empty());
|
||||
};
|
||||
let response = response
|
||||
.header(header::ACCEPT_RANGES, "bytes")
|
||||
.header(header::CONTENT_LENGTH, end - start)
|
||||
.header(header::CONTENT_TYPE, "application/octet-stream");
|
||||
let response = if requested.is_some() {
|
||||
response.status(StatusCode::PARTIAL_CONTENT).header(
|
||||
header::CONTENT_RANGE,
|
||||
format!("bytes {start}-{}/{size}", end - 1),
|
||||
)
|
||||
} else {
|
||||
response
|
||||
};
|
||||
if request.method() == Method::HEAD {
|
||||
return response.body(empty());
|
||||
}
|
||||
let body = futures::stream::try_unfold((stream, start), move |(stream, at)| async move {
|
||||
if at == end {
|
||||
return Ok(None);
|
||||
}
|
||||
let read = (end - at).min(u64::from(CHUNK)) as u32;
|
||||
let bytes = stream.read(at, read).await.map_err(io::Error::other)?;
|
||||
if bytes.is_empty() {
|
||||
return Err(io::Error::from(io::ErrorKind::UnexpectedEof));
|
||||
}
|
||||
let next = at + bytes.len() as u64;
|
||||
Ok(Some((Frame::data(Bytes::from(bytes)), (stream, next))))
|
||||
});
|
||||
response.body(BodyExt::boxed_unsync(StreamBody::new(body)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::range;
|
||||
|
||||
#[test]
|
||||
fn resolves_seek_ranges_and_rejects_invalid_ranges() {
|
||||
assert_eq!(range(None, 10), Some((0, 10)));
|
||||
assert_eq!(range(Some("bytes=3-5"), 10), Some((3, 6)));
|
||||
assert_eq!(range(Some("bytes=9-"), 10), Some((9, 10)));
|
||||
assert_eq!(range(Some("bytes=-3"), 10), Some((7, 10)));
|
||||
assert_eq!(range(Some("bytes=10-"), 10), None);
|
||||
assert_eq!(range(Some("bytes=0-1,3-4"), 10), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
//! Keeping pinned files on this computer after they were pinned.
|
||||
//!
|
||||
//! A pin downloads a file once. Nothing downloaded it again afterwards: a
|
||||
//! change from another device deleted the cached copy, a file added to a
|
||||
//! pinned folder elsewhere was never fetched, and a download that failed while
|
||||
//! the link was down stayed failed. Each of those left a file listed as kept
|
||||
//! offline that the next read downloaded, and that could not be read at all
|
||||
//! without a network (`docs/BUGS.md` B282).
|
||||
//!
|
||||
//! The keeper is one thread that checks pins again and downloads what is
|
||||
//! missing or stale: every pin when the daemon starts and when the link comes
|
||||
//! back, and the pins a batch of remote events touched once the batch is
|
||||
//! applied. A remote change no longer deletes a pinned file's copy. The copy
|
||||
//! is stored under the revision's `(mtime, size)`, so a read of a newer
|
||||
//! revision does not take it, and the keeper replaces it once the new one is
|
||||
//! downloaded.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use proton_drive_rs::NodeKind;
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::{Core, is_local_uid, node_size, parse_uid};
|
||||
|
||||
/// How often the keeper looks for work.
|
||||
const KEEPER_TICK: Duration = Duration::from_secs(1);
|
||||
/// The first wait before a check that could not download everything is tried
|
||||
/// again. It doubles on every failure, up to [`RETRY_MAX`].
|
||||
const RETRY_FIRST: Duration = Duration::from_secs(30);
|
||||
const RETRY_MAX: Duration = Duration::from_secs(30 * 60);
|
||||
|
||||
/// What the keeper owes a check.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct PinWork {
|
||||
/// Every pin.
|
||||
all: bool,
|
||||
/// The pinned files and folders named here. A folder is checked whole.
|
||||
nodes: HashSet<NodeUid>,
|
||||
}
|
||||
|
||||
impl PinWork {
|
||||
fn is_empty(&self) -> bool {
|
||||
!self.all && self.nodes.is_empty()
|
||||
}
|
||||
|
||||
fn merge(&mut self, other: PinWork) {
|
||||
self.all |= other.all;
|
||||
self.nodes.extend(other.nodes);
|
||||
}
|
||||
}
|
||||
|
||||
/// A file a pin keeps, as the tree knows it now.
|
||||
pub(crate) struct KeptFile {
|
||||
pub(crate) uid: NodeUid,
|
||||
pub(crate) name: String,
|
||||
pub(crate) mtime: i64,
|
||||
pub(crate) size: u64,
|
||||
}
|
||||
|
||||
impl Core {
|
||||
/// Have the keeper check every pin.
|
||||
pub(crate) fn recheck_all_pins(&self) {
|
||||
self.pin_work.lock().all = true;
|
||||
}
|
||||
|
||||
/// Have the keeper check the pinned file or folder `uid`.
|
||||
pub(crate) fn recheck_pinned(&self, uid: &NodeUid) {
|
||||
self.pin_work.lock().nodes.insert(uid.clone());
|
||||
}
|
||||
|
||||
/// The keeper's loop, until the daemon stops. Starts with a check of every
|
||||
/// pin, which catches up on whatever changed while the daemon was down.
|
||||
///
|
||||
/// Work that failed waits out a backoff of its own. New work does not wait
|
||||
/// for it, so a file that keeps failing does not hold up the next change.
|
||||
pub(crate) fn run_pin_keeper(&self) {
|
||||
self.recheck_all_pins();
|
||||
let mut backlog = PinWork::default();
|
||||
let mut backlog_due = Instant::now();
|
||||
let mut retry = RETRY_FIRST;
|
||||
while self.shutdown.sleep(KEEPER_TICK) {
|
||||
if !self.is_online() {
|
||||
continue;
|
||||
}
|
||||
let mut work = std::mem::take(&mut *self.pin_work.lock());
|
||||
let retrying = !backlog.is_empty() && Instant::now() >= backlog_due;
|
||||
if retrying {
|
||||
work.merge(std::mem::take(&mut backlog));
|
||||
}
|
||||
if work.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let failed = self.keep_pins(&work);
|
||||
if failed.is_empty() {
|
||||
if retrying {
|
||||
retry = RETRY_FIRST;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
backlog.merge(failed);
|
||||
backlog_due = Instant::now() + retry;
|
||||
retry = (retry * 2).min(RETRY_MAX);
|
||||
}
|
||||
}
|
||||
|
||||
/// Download every file `work` covers whose cached copy is missing or
|
||||
/// stale. Returns what could not be checked or downloaded.
|
||||
fn keep_pins(&self, work: &PinWork) -> PinWork {
|
||||
let mut failed = PinWork::default();
|
||||
let roots: Vec<NodeUid> = if work.all {
|
||||
self.cache
|
||||
.list_pins()
|
||||
.iter()
|
||||
.filter_map(|pin| parse_uid(&pin.uid))
|
||||
.chain(work.nodes.iter().cloned())
|
||||
.collect()
|
||||
} else {
|
||||
work.nodes.iter().cloned().collect()
|
||||
};
|
||||
let mut seen: HashSet<u64> = HashSet::new();
|
||||
let mut files: Vec<u64> = Vec::new();
|
||||
for uid in roots {
|
||||
if self.shutdown.is_stopping() {
|
||||
return failed;
|
||||
}
|
||||
if !self.cache.is_pinned(&uid) {
|
||||
continue;
|
||||
}
|
||||
let found = {
|
||||
let st = self.state();
|
||||
st.by_uid.get(&uid).and_then(|&ino| {
|
||||
let e = st.entries.get(&ino)?;
|
||||
Some((ino, e.parent, e.node.is_folder()))
|
||||
})
|
||||
};
|
||||
let Some((ino, parent, is_folder)) = found else {
|
||||
debug!(%uid, "pin keeper: pinned node is not in the tree");
|
||||
continue;
|
||||
};
|
||||
if is_folder {
|
||||
match self.pin_subtree_files(ino) {
|
||||
Ok(inos) => files.extend(inos.into_iter().filter(|i| seen.insert(*i))),
|
||||
Err(e) => {
|
||||
warn!(%uid, error = %e, "pin keeper: listing a kept folder failed");
|
||||
failed.nodes.insert(uid);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// An event about this file dropped its parent's listing. Listing
|
||||
// it again brings the file's entry up to the revision Drive has
|
||||
// now, which is what the cached copy is checked against.
|
||||
if !work.all
|
||||
&& parent != ino
|
||||
&& let Err(e) = self.ensure_children(parent)
|
||||
{
|
||||
warn!(%uid, error = ?e, "pin keeper: listing a kept file's folder failed");
|
||||
failed.nodes.insert(uid);
|
||||
continue;
|
||||
}
|
||||
if seen.insert(ino) {
|
||||
files.push(ino);
|
||||
}
|
||||
}
|
||||
|
||||
let stale: Vec<KeptFile> = files
|
||||
.into_iter()
|
||||
.filter_map(|ino| self.kept_file(ino))
|
||||
.filter(|f| !self.cache.is_cached(&f.uid, f.mtime, f.size))
|
||||
.collect();
|
||||
if stale.is_empty() {
|
||||
return failed;
|
||||
}
|
||||
let job = self.transfers.begin_job("Updating files kept offline");
|
||||
job.set_total(stale.len() as u64);
|
||||
let (mut fetched, mut failures) = (0usize, 0usize);
|
||||
for file in stale {
|
||||
if self.shutdown.is_stopping() {
|
||||
break;
|
||||
}
|
||||
// Unpinned while the check ran.
|
||||
if !self.cache.is_pinned(&file.uid) {
|
||||
job.step();
|
||||
continue;
|
||||
}
|
||||
job.detail(file.name.clone());
|
||||
match self.fetch_pinned(&file.uid, &file.name, file.mtime, file.size) {
|
||||
Ok(()) => fetched += 1,
|
||||
Err(e) => {
|
||||
warn!(uid = %file.uid, error = %e, "pin keeper: download failed");
|
||||
failed.nodes.insert(file.uid);
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
job.step();
|
||||
}
|
||||
info!(
|
||||
fetched,
|
||||
failed = failures,
|
||||
"pin keeper: brought files kept offline up to date"
|
||||
);
|
||||
failed
|
||||
}
|
||||
|
||||
/// The file at `ino`, as a pin keeps it. `None` for a folder, a file this
|
||||
/// daemon still owes an upload for, or a file whose real size cannot be
|
||||
/// read.
|
||||
///
|
||||
/// A queued write is ahead of Drive. Downloading Drive's revision would
|
||||
/// cache what the write replaces, and the drain keeps the uploaded bytes
|
||||
/// of a pinned file once they land.
|
||||
///
|
||||
/// A listing carries the ciphertext size until its size upgrade lands
|
||||
/// (B12). Stored under that size, the copy never matched the plaintext it
|
||||
/// holds, so every read downloaded the file again and every check
|
||||
/// downloaded it once more.
|
||||
pub(crate) fn kept_file(&self, ino: u64) -> Option<KeptFile> {
|
||||
let (uid, name, mtime, size, provisional) = {
|
||||
let st = self.state();
|
||||
let e = st.entries.get(&ino)?;
|
||||
let NodeKind::File { claimed_size, .. } = &e.node.kind else {
|
||||
return None;
|
||||
};
|
||||
(
|
||||
e.uid.clone(),
|
||||
e.node.name.clone(),
|
||||
e.node.modification_time,
|
||||
node_size(&e.node),
|
||||
claimed_size.is_none(),
|
||||
)
|
||||
};
|
||||
if is_local_uid(&uid) || self.pending.lock().contains_key(&uid) {
|
||||
return None;
|
||||
}
|
||||
let size = match provisional {
|
||||
true => self.resolve_provisional_size(ino, &uid)?,
|
||||
false => size,
|
||||
};
|
||||
Some(KeptFile {
|
||||
uid,
|
||||
name,
|
||||
mtime,
|
||||
size,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn work_that_failed_keeps_every_node_and_the_full_check() {
|
||||
let mut backlog = PinWork::default();
|
||||
assert!(backlog.is_empty());
|
||||
let uid = parse_uid("vol~a").unwrap();
|
||||
backlog.merge(PinWork {
|
||||
all: false,
|
||||
nodes: HashSet::from([uid.clone()]),
|
||||
});
|
||||
backlog.merge(PinWork {
|
||||
all: true,
|
||||
nodes: HashSet::from([uid.clone()]),
|
||||
});
|
||||
assert!(backlog.all);
|
||||
assert_eq!(backlog.nodes.len(), 1);
|
||||
assert!(!backlog.is_empty());
|
||||
}
|
||||
}
|
||||
+549
-94
@@ -74,11 +74,12 @@ use proton_drive_rs::proton_sdk::api::ResponseCode;
|
||||
use proton_drive_rs::proton_sdk::error::ProtonError;
|
||||
use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, ShareId, VolumeId};
|
||||
use proton_drive_rs::{
|
||||
DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, NodeMoveItem, ProtonDriveClient,
|
||||
ProtonPhotosClient, SharedWithMeItem, ThumbnailType,
|
||||
AvailableNameTarget, DriveEvent, DriveEventScopeId, MemberRole, Node, NodeKind, NodeMoveItem,
|
||||
ProtonDriveClient, ProtonPhotosClient, SharedWithMeItem, ThumbnailType,
|
||||
};
|
||||
|
||||
mod albums;
|
||||
mod attach;
|
||||
mod background;
|
||||
mod conflicts;
|
||||
mod control;
|
||||
@@ -90,6 +91,8 @@ mod drive;
|
||||
use drive::{DriveApi, RevisionRead};
|
||||
mod events;
|
||||
mod filesystem;
|
||||
mod gallery;
|
||||
mod keeper;
|
||||
mod link;
|
||||
pub use filesystem::ProtonFs;
|
||||
mod mount;
|
||||
@@ -116,14 +119,16 @@ mod upload;
|
||||
mod r#virtual;
|
||||
mod workers;
|
||||
use background::{forget_and_notify, run_event_sync, run_local_index, run_photos_event_sync};
|
||||
use link::BATCH_CALL_TIMEOUT;
|
||||
pub(crate) use mount::is_stale_mount;
|
||||
pub use mount::prepare_mountpoint;
|
||||
pub use mount::{MountOptions, MountOutcome, mount};
|
||||
use mount::{
|
||||
SecondaryInsertRejection, SecondaryMount, clear_stale_mount, fuse_connection_id, spawn_session,
|
||||
};
|
||||
use reads::{BlockFlight, BlockRing, PREFETCH_BUDGET, Prefetch, ReaderSlot, STREAM_BYPASS_MIN};
|
||||
use state::{Entry, Intervals, PendingRevision, State, StateGuard, WriteHandle};
|
||||
pub use systemd::idle_sleep;
|
||||
pub use systemd::{idle_sleep, status as notify_status, stretch_watchdog_for_startup};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use transfers::{CountingWriter, JobGuard, OwnedCountingReader, TransferRegistry};
|
||||
use r#virtual::{
|
||||
@@ -495,6 +500,7 @@ struct Core {
|
||||
/// The concrete client, for what [`DriveApi`] does not cover: sharing,
|
||||
/// Photos, devices, invitations and public links.
|
||||
client: ProtonDriveClient,
|
||||
gallery: Option<Arc<gallery::Gallery>>,
|
||||
rt: tokio::runtime::Handle,
|
||||
/// The daemon's My Files root. Forked on-demand sessions retain this value
|
||||
/// so ownership checks never mistake their own mount root for account scope.
|
||||
@@ -586,6 +592,8 @@ struct Core {
|
||||
/// Nudges the drain workers: set true and notify to have them re-examine
|
||||
/// the queue instead of waiting out their backoff.
|
||||
drain_wake: Arc<(Mutex<bool>, Condvar)>,
|
||||
/// The pins the keeper owes a check (`keeper`).
|
||||
pin_work: Arc<Mutex<keeper::PinWork>>,
|
||||
/// The daemon's stop signal, shared with every background loop this `Core`
|
||||
/// was cloned into so teardown can end and join them (`docs/BUGS.md` B44).
|
||||
shutdown: Arc<shutdown::Shutdown>,
|
||||
@@ -727,6 +735,8 @@ struct Core {
|
||||
/// can say what a pass is doing rather than just "syncing". An entry exists
|
||||
/// only while that folder's reconcile pass is running.
|
||||
sync_progress: Arc<Mutex<HashMap<i64, SyncProgress>>>,
|
||||
/// Whether the folder-sync engine is alive, and which pass it is on.
|
||||
sync_engine: Arc<sync::EngineHealth>,
|
||||
/// Channel to the folder-sync engine: nudges it to
|
||||
/// reconcile a folder, reconcile everything, or re-scan its watch set.
|
||||
sync_tx: std::sync::mpsc::Sender<sync::SyncMsg>,
|
||||
@@ -1034,7 +1044,9 @@ impl Core {
|
||||
|
||||
let core = self.clone();
|
||||
std::thread::spawn(move || {
|
||||
let job = core.transfers.begin_job("Checking the cache database");
|
||||
let job = core
|
||||
.transfers
|
||||
.begin_background_job("Checking the cache database");
|
||||
job.detail("Verifying every page");
|
||||
let problems = match core.db.integrity_check() {
|
||||
Ok(problems) => problems,
|
||||
@@ -2118,12 +2130,39 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `ino`'s listing is already in memory, i.e. whether
|
||||
/// [`Core::ensure_children`] would return without touching the network.
|
||||
/// Lets a handler decide between answering inline and handing off to a
|
||||
/// worker, at the cost of one uncontended map lookup.
|
||||
/// Whether [`Core::ensure_children`] would answer for `ino` without
|
||||
/// touching the network. Lets a handler decide between answering inline
|
||||
/// and handing off to a worker.
|
||||
///
|
||||
/// A resident listing is not enough on its own. A shared folder's, or the
|
||||
/// Shared with me root's, is fetched again once it is a minute old, and
|
||||
/// that fetch ran on the dispatch loop, so every request on the mount
|
||||
/// waited for Drive to answer (`docs/BUGS.md` B219).
|
||||
fn children_cached(&self, ino: u64) -> bool {
|
||||
self.state().children.contains_key(&ino)
|
||||
let (uid, resident) = {
|
||||
let st = self.state();
|
||||
match st.entries.get(&ino) {
|
||||
Some(e) => (e.uid.clone(), st.children.contains_key(&ino)),
|
||||
// `list_children` answers ENOENT without the network.
|
||||
None => return true,
|
||||
}
|
||||
};
|
||||
if !resident {
|
||||
return false;
|
||||
}
|
||||
let freshness = if is_virtual_uid(&uid) {
|
||||
Some(SHARED_WITH_ME_SYNCED_MS.to_string())
|
||||
} else if !self.is_own_or_virtual(&uid) && !is_local_uid(&uid) {
|
||||
Some(Self::shared_folder_freshness_key(&uid))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
freshness.is_none_or(|key| {
|
||||
!listing_needs_refresh(
|
||||
self.online.load(Ordering::Relaxed),
|
||||
self.shared_listing_stale(&key),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Re-apply the optimistic size of any queued write to `nodes`.
|
||||
@@ -2291,8 +2330,7 @@ impl Core {
|
||||
|
||||
let generation = self.shared_generation.load(Ordering::SeqCst);
|
||||
let accepted = accepted_share_provenance(
|
||||
self.rt
|
||||
.block_on(self.client.enumerate_shared_with_me())
|
||||
self.block_on_within(BATCH_CALL_TIMEOUT, self.client.enumerate_shared_with_me())
|
||||
.map_err(|error| {
|
||||
error!(%error, "enumerating shared roots failed");
|
||||
Errno::EIO
|
||||
@@ -2301,12 +2339,14 @@ impl Core {
|
||||
let nodes = if accepted.uids.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
self.rt
|
||||
.block_on(self.drive.enumerate_nodes_light(&accepted.uids))
|
||||
.map_err(|error| {
|
||||
error!(%error, "materializing shared roots failed");
|
||||
Errno::EIO
|
||||
})?
|
||||
self.block_on_within(
|
||||
BATCH_CALL_TIMEOUT,
|
||||
self.drive.enumerate_nodes_light(&accepted.uids),
|
||||
)
|
||||
.map_err(|error| {
|
||||
error!(%error, "materializing shared roots failed");
|
||||
Errno::EIO
|
||||
})?
|
||||
};
|
||||
let virtual_uid = shared_with_me_uid();
|
||||
let published = prepare_shared_roots(&accepted, nodes, &virtual_uid);
|
||||
@@ -2685,6 +2725,7 @@ impl Core {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let preassigned = state::inos_for(&self.db, &filtered_nodes);
|
||||
let mut st = self.state();
|
||||
// Lost the race? Another thread already populated it.
|
||||
if st.children.contains_key(&ino) {
|
||||
@@ -2706,15 +2747,14 @@ impl Core {
|
||||
})
|
||||
.map(|n| n.uid.clone())
|
||||
.collect();
|
||||
let inos = st.intern_batch(ino, filtered_nodes);
|
||||
let inos = st.intern_batch_with_inos(ino, filtered_nodes, preassigned);
|
||||
child_inos.extend(inos);
|
||||
st.children.insert(ino, child_inos);
|
||||
let root_snapshot = primary_root.then(|| RootListingSnapshot::capture(&st, ino).unwrap());
|
||||
// Record the listing as complete so a later restart (or a trimmed hot
|
||||
// cache) can rebuild it from the DB without the API.
|
||||
if let Err(e) = self.db.set_listed(&folder_uid, true) {
|
||||
warn!(%folder_uid, error = %e, "db set_listed(true) failed");
|
||||
}
|
||||
// cache) can rebuild it from the DB without the API. Written with the
|
||||
// children's rows when the lock goes, not under it.
|
||||
st.record_listed(folder_uid.clone());
|
||||
drop(st);
|
||||
self.flush_access_changes();
|
||||
if let Some(snapshot) = root_snapshot {
|
||||
@@ -2877,7 +2917,8 @@ impl Core {
|
||||
let (mut key, mut uids, mut slot) = (key, uids, slot);
|
||||
loop {
|
||||
for chunk in uids.chunks(SIZE_UPGRADE_CHUNK) {
|
||||
let result = self.rt.block_on(self.drive.enumerate_nodes(chunk));
|
||||
let result =
|
||||
self.block_on_within(BATCH_CALL_TIMEOUT, self.drive.enumerate_nodes(chunk));
|
||||
self.apply_size_upgrade(key, result);
|
||||
slot.chunk_done();
|
||||
}
|
||||
@@ -3129,6 +3170,28 @@ impl Core {
|
||||
)
|
||||
}
|
||||
|
||||
/// [`Core::resolve_anywhere`], widened to the Photos timeline for sharing.
|
||||
///
|
||||
/// A photo lives on the photos volume, which no location shows, so the
|
||||
/// residency proof above never finds it. The timeline in the `photos` table
|
||||
/// is this daemon's proof instead: it lists only photos on the user's own
|
||||
/// volume, which are theirs to share. A photo in an album someone else
|
||||
/// shared is not in it and stays refused.
|
||||
pub(crate) fn resolve_shareable(&self, uid: &str) -> CoreResult<NodeUid> {
|
||||
resolve_anywhere_with(
|
||||
uid,
|
||||
|uid| self.owns_visible_uid(uid),
|
||||
|uid| {
|
||||
let uid = uid.to_string();
|
||||
Ok(self.db.mirror_contains_uid(&uid)?
|
||||
|| !self
|
||||
.db
|
||||
.photos_by_uid(std::slice::from_ref(&uid))?
|
||||
.is_empty())
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn source_parent_uid(&self, ino: u64, rel: &Path) -> CoreResult<NodeUid> {
|
||||
let state = self.state();
|
||||
let entry = state
|
||||
@@ -3951,6 +4014,15 @@ impl Core {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn has_pending_moves_out_of(&self, uid: &NodeUid) -> Result<bool, Errno> {
|
||||
self.db
|
||||
.has_pending_moves_out_of(&uid.to_string())
|
||||
.map_err(|error| {
|
||||
error!(%uid, %error, "checking for moves out of a trash target failed");
|
||||
Errno::EIO
|
||||
})
|
||||
}
|
||||
|
||||
/// Queue trashing a node the server knows about, and drop it from the tree
|
||||
/// now.
|
||||
///
|
||||
@@ -3965,7 +4037,12 @@ impl Core {
|
||||
/// the cached ones, as when the trash is sent at once: a partial write
|
||||
/// fills its gaps from them, offline too (`docs/BUGS.md` B186). The final
|
||||
/// close or the trash landing, whichever comes last, lets go of them.
|
||||
fn queue_trash(&self, uid: &NodeUid, name: &str) -> Result<(), Errno> {
|
||||
///
|
||||
/// `open` is whether the caller found the file open. A create that landed
|
||||
/// under a delete is trashed by its real uid before the tree has it, so
|
||||
/// asking the tree alone took an open file for closed (`docs/BUGS.md`
|
||||
/// B257).
|
||||
fn queue_trash(&self, uid: &NodeUid, name: &str, open: bool) -> Result<(), Errno> {
|
||||
self.require_uid_writable(uid)?;
|
||||
// The bytes below are about to be discarded, and a drain worker may be
|
||||
// reading them onto the wire right now. Stop it first: with several
|
||||
@@ -3980,7 +4057,7 @@ impl Core {
|
||||
error!(%uid, error = %e, "queueing trash failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
let open_now = self.is_open_anywhere(uid);
|
||||
let open_now = open || self.is_open_anywhere(uid);
|
||||
if open_now {
|
||||
let read_from = self.pending_blob(uid);
|
||||
self.release_dropped_below(
|
||||
@@ -4225,9 +4302,16 @@ impl Core {
|
||||
}
|
||||
|
||||
/// Pin the node at mountpoint-relative `rel`. A file downloads its full
|
||||
/// plaintext into the content cache; a folder records a recursive pin and
|
||||
/// downloads every descendant file (selective sync). Returns a human message.
|
||||
fn pin(&self, rel: &Path) -> CoreResult<String> {
|
||||
/// plaintext into the content cache before this returns. A folder records a
|
||||
/// recursive pin and downloads every descendant file on a background thread,
|
||||
/// reported as a job: a big folder far outlasts the socket read timeout.
|
||||
/// Returns a human message.
|
||||
///
|
||||
/// `recorded` is the path the pin registry keeps for the front ends to
|
||||
/// open the pin by: `rel` itself on the primary mount, which they join to
|
||||
/// the mountpoint, and the absolute local path on an on-demand folder's
|
||||
/// mount, whose node is not reachable under the primary mountpoint at all.
|
||||
fn pin(&self, rel: &Path, recorded: &Path) -> CoreResult<String> {
|
||||
let (ino, uid) = self.resolve(rel)?;
|
||||
let (name, is_folder, mtime, size) = {
|
||||
let st = self.state();
|
||||
@@ -4243,34 +4327,105 @@ impl Core {
|
||||
)
|
||||
};
|
||||
if is_folder {
|
||||
// Offline, every download would fail and the folder would only look
|
||||
// kept. Refuse instead, as a file pin does.
|
||||
if !self.is_online() {
|
||||
return Err(CoreError::offline());
|
||||
}
|
||||
// Record the recursive pin first so every descendant is eviction-
|
||||
// exempt before we start filling the cache with the subtree.
|
||||
self.cache
|
||||
.add_pin(&uid, rel, true)
|
||||
.add_pin(&uid, recorded, true)
|
||||
.map_err(|e| CoreError::from_api(&e, "pin"))?;
|
||||
let n = self.pin_subtree(ino)?;
|
||||
return Ok(format!("{name} ({n} files)"));
|
||||
let core = self.clone();
|
||||
let folder = name.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("pin-folder".into())
|
||||
.spawn(move || core.pin_subtree(ino, &uid, &folder))
|
||||
.map_err(|e| CoreError::internal(format!("spawn pin: {e}")))?;
|
||||
return Ok(format!("{name}: downloading its files in the background"));
|
||||
}
|
||||
let bytes = self
|
||||
.download_file_tracked(&uid, &name, size)
|
||||
.map_err(|e| CoreError::from_api(&e, "download"))?;
|
||||
// Stored under a provisional size, the copy would never match the
|
||||
// plaintext it holds (`docs/BUGS.md` B282).
|
||||
let size = match self.kept_file(ino) {
|
||||
Some(file) if file.uid == uid => file.size,
|
||||
_ => size,
|
||||
};
|
||||
self.fetch_content(&uid, &name, mtime, size)?;
|
||||
self.cache
|
||||
.store(&uid, mtime, size, &bytes)
|
||||
.map_err(|e| CoreError::internal(format!("cache store: {e}")))?;
|
||||
self.cache
|
||||
.add_pin(&uid, rel, false)
|
||||
.add_pin(&uid, recorded, false)
|
||||
.map_err(|e| CoreError::from_api(&e, "pin"))?;
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
/// Download and cache every file in the subtree rooted at folder `ino`,
|
||||
/// returning the count cached (already-fresh blobs counted, not re-fetched).
|
||||
/// Walks the tree depth-first, enumerating each folder so a cold subtree is
|
||||
/// fully discovered; the lock is dropped before each network download.
|
||||
fn pin_subtree(&self, ino: u64) -> CoreResult<usize> {
|
||||
let mut files: Vec<(NodeUid, String, i64, u64)> = Vec::new();
|
||||
/// Download and cache every file in the subtree rooted at the pinned folder
|
||||
/// `ino` (`folder`, named `name`), as a job the front ends show. Walks the
|
||||
/// tree depth-first, enumerating each folder so a cold subtree is fully
|
||||
/// discovered; the lock is dropped before each network download. Stops when
|
||||
/// the folder is unpinned or the daemon stops.
|
||||
fn pin_subtree(&self, ino: u64, folder: &NodeUid, name: &str) {
|
||||
let job = self
|
||||
.transfers
|
||||
.begin_job(format!("Making {name} available offline"));
|
||||
match self.pin_subtree_files(ino) {
|
||||
Ok(files) => {
|
||||
job.set_total(files.len() as u64);
|
||||
let mut failed = 0usize;
|
||||
for ino in files {
|
||||
if self.shutdown.is_stopping() || !self.cache.is_pinned(folder) {
|
||||
return;
|
||||
}
|
||||
// A file with a queued write, or no readable size, is
|
||||
// left to the drain and the keeper (`keeper::kept_file`).
|
||||
if let Some(file) = self.kept_file(ino) {
|
||||
job.detail(file.name.clone());
|
||||
if let Err(e) =
|
||||
self.fetch_pinned(&file.uid, &file.name, file.mtime, file.size)
|
||||
{
|
||||
warn!(uid = %file.uid, error = %e, "pin folder: download failed");
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
job.step();
|
||||
}
|
||||
info!(folder = %name, failed, "pin folder: done");
|
||||
}
|
||||
Err(e) => warn!(folder = %name, error = %e, "pin folder: enumerate failed"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache the file `uid` for a folder pin. Its `mtime` and `size` were read
|
||||
/// before the download, and a create landing or a remote edit can change
|
||||
/// them while it runs. The blob is then stored under the old ones and reads
|
||||
/// as stale, so the file is listed as kept but not available offline. Fetch
|
||||
/// it again under the new ones, a few times at most.
|
||||
fn fetch_pinned(&self, uid: &NodeUid, name: &str, mtime: i64, size: u64) -> CoreResult<()> {
|
||||
let mut fetched = (mtime, size);
|
||||
for _ in 0..3 {
|
||||
self.fetch_content(uid, name, fetched.0, fetched.1)?;
|
||||
let current = {
|
||||
let st = self.state();
|
||||
st.by_uid
|
||||
.get(uid)
|
||||
.and_then(|ino| st.entries.get(ino))
|
||||
.map(|e| (e.node.modification_time, node_size(&e.node)))
|
||||
};
|
||||
match current {
|
||||
Some(now) if now != fetched => fetched = now,
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The inode of every file in the subtree rooted at folder `ino`.
|
||||
fn pin_subtree_files(&self, ino: u64) -> CoreResult<Vec<u64>> {
|
||||
let mut files: Vec<u64> = Vec::new();
|
||||
let mut stack = vec![ino];
|
||||
while let Some(dir) = stack.pop() {
|
||||
if self.shutdown.is_stopping() {
|
||||
break;
|
||||
}
|
||||
self.ensure_children(dir)
|
||||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||||
let st = self.state();
|
||||
@@ -4280,33 +4435,13 @@ impl Core {
|
||||
if e.node.is_folder() {
|
||||
stack.push(k);
|
||||
} else {
|
||||
files.push((
|
||||
e.uid.clone(),
|
||||
e.node.name.clone(),
|
||||
e.node.modification_time,
|
||||
node_size(&e.node),
|
||||
));
|
||||
files.push(k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut count = 0;
|
||||
for (uid, name, mtime, size) in files {
|
||||
if self.cache.is_cached(&uid, mtime, size) {
|
||||
count += 1;
|
||||
continue;
|
||||
}
|
||||
match self.download_file_tracked(&uid, &name, size) {
|
||||
Ok(bytes) => {
|
||||
if self.cache.store(&uid, mtime, size, &bytes).is_ok() {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Err(e) => warn!(%uid, error = %e, "pin subtree: download failed"),
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Fetch a thumbnail of `ttype` for the file at `ino`, served from the cache
|
||||
@@ -4330,9 +4465,15 @@ impl Core {
|
||||
if self.thumbnail_misses.lock().remote_contains(&key, mtime) {
|
||||
return Ok(None);
|
||||
}
|
||||
// A file manager asks for one per file in a folder, on the metadata
|
||||
// workers. Offline, or on a stalled link, each one held a worker for
|
||||
// the SDK's whole retry budget, and every cold lookup, create and
|
||||
// rename on the mount queued behind them (`docs/BUGS.md` B220).
|
||||
if !self.is_online() {
|
||||
return Ok(None);
|
||||
}
|
||||
let bytes = self
|
||||
.rt
|
||||
.block_on(self.client.download_thumbnail(&uid, ttype))
|
||||
.block_on_bounded(self.client.download_thumbnail(&uid, ttype))
|
||||
.map_err(|e| {
|
||||
warn!(%uid, error = %e, "download thumbnail failed");
|
||||
Errno::EIO
|
||||
@@ -4352,6 +4493,8 @@ impl Core {
|
||||
|
||||
/// Unpin the node at `rel`, evicting its cached content. For a folder, also
|
||||
/// evicts every descendant's cached blob (the subtree is no longer kept).
|
||||
/// A node kept only because a folder above it is pinned is refused: there
|
||||
/// is no pin of its own to remove, and it would stay kept all the same.
|
||||
fn unpin(&self, rel: &Path) -> CoreResult<String> {
|
||||
let (ino, uid) = self.resolve(rel)?;
|
||||
let (name, is_folder) = {
|
||||
@@ -4361,14 +4504,24 @@ impl Core {
|
||||
.map(|e| (e.node.name.clone(), e.node.is_folder()))
|
||||
.unwrap_or_default()
|
||||
};
|
||||
if self.pinned_by_folder(ino) {
|
||||
return Err(CoreError::conflict(format!(
|
||||
"{name} is kept offline because a folder above it is; unpin that folder instead"
|
||||
)));
|
||||
}
|
||||
self.cache
|
||||
.remove_pin(&uid)
|
||||
.map_err(|e| CoreError::from_api(&e, "unpin"))?;
|
||||
// A recursively-pinned folder's descendants were eviction-exempt; now
|
||||
// that the pin is gone, reclaim their blobs eagerly instead of waiting
|
||||
// for budget pressure. Descendants come from the DB node tree.
|
||||
// for budget pressure. Descendants come from the DB node tree. One
|
||||
// pinned on its own, or under a pinned subfolder, is still kept.
|
||||
if is_folder && let Ok(uids) = self.db.descendants(&uid.to_string()) {
|
||||
let still_pinned = self.pinned_set();
|
||||
for s in uids {
|
||||
if still_pinned.contains(&s) {
|
||||
continue;
|
||||
}
|
||||
if let Some(u) = parse_uid(&s) {
|
||||
self.cache.evict(&u);
|
||||
self.evict_reader(&u);
|
||||
@@ -4378,6 +4531,20 @@ impl Core {
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
/// Whether the node at `ino` is kept offline because a folder above it is
|
||||
/// pinned, whether or not it is also pinned on its own.
|
||||
fn pinned_by_folder(&self, ino: u64) -> bool {
|
||||
let parent = {
|
||||
let st = self.state();
|
||||
st.entries
|
||||
.get(&ino)
|
||||
.filter(|e| e.parent != ino)
|
||||
.and_then(|e| st.entries.get(&e.parent))
|
||||
.map(|p| p.uid.clone())
|
||||
};
|
||||
parent.is_some_and(|uid| self.cache.is_pinned(&uid))
|
||||
}
|
||||
|
||||
/// A Photos API handle sharing this Core's Drive client and session, so it
|
||||
/// reuses the daemon's single authenticated session rather than logging in
|
||||
/// again (Proton refresh tokens are single-use). Cheap — the Drive client
|
||||
@@ -4390,10 +4557,13 @@ impl Core {
|
||||
/// the same lazy remote enumeration `readdir` uses, projected into
|
||||
/// serializable [`DirEntry`]s (with per-file pin state).
|
||||
fn list_dir(&self, rel: &Path) -> CoreResult<Vec<DirEntry>> {
|
||||
let (ino, _uid) = self.resolve(rel)?;
|
||||
let (ino, uid) = self.resolve(rel)?;
|
||||
self.ensure_children(ino)
|
||||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||||
self.settle_listing_sizes(ino);
|
||||
// Every entry shares this folder as its parent, so one lookup says
|
||||
// whether they are all kept through it.
|
||||
let pinned_by_folder = self.cache.is_pinned(&uid);
|
||||
// Snapshot the listing, then drop the lock before touching the on-disk
|
||||
// pin registry so a slow disk read doesn't block FUSE metadata ops.
|
||||
let rows: Vec<(String, bool, u64, i64, NodeUid, u64)> = {
|
||||
@@ -4425,7 +4595,8 @@ impl Core {
|
||||
is_dir,
|
||||
size,
|
||||
modified,
|
||||
pinned: self.cache.is_pinned(&uid),
|
||||
pinned: pinned_by_folder || self.cache.is_pinned(&uid),
|
||||
pinned_by_folder,
|
||||
cached: !is_dir && self.cache.is_cached(&uid, modified, size),
|
||||
uid: uid.to_string(),
|
||||
// Listing entries live in the requested dir; the caller derives
|
||||
@@ -5092,6 +5263,133 @@ impl Core {
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
/// Copy a file or folder into `new_parent_rel` on Proton Drive, which copies
|
||||
/// the content there: nothing is downloaded or uploaded. Both paths are
|
||||
/// mountpoint-relative. A name already taken in the folder gets the next
|
||||
/// free ` (n)` suffix. Returns the copy's name.
|
||||
///
|
||||
/// Drive copies what it holds, so the copy is refused offline and while the
|
||||
/// mount still owes Drive part of it: a node or folder only queued locally,
|
||||
/// a file with a queued upload or unsaved writes, or — for a folder, whose
|
||||
/// queued changes could sit anywhere below it — any queued op at all.
|
||||
fn copy_to(&self, rel: &Path, new_parent_rel: &Path) -> CoreResult<String> {
|
||||
let (ino, uid) = self.resolve(rel)?;
|
||||
let (pino, new_parent_uid) = self
|
||||
.resolve_path(new_parent_rel)
|
||||
.map_err(|e| self.errno_error(e, "resolve new parent"))?;
|
||||
self.ensure_children(pino)
|
||||
.map_err(|e| self.errno_error(e, "enumerate"))?;
|
||||
self.require_uid_writable(&new_parent_uid)
|
||||
.map_err(|error| self.errno_error(error, "copy access"))?;
|
||||
let (name, is_folder, unsaved) = {
|
||||
let st = self.state();
|
||||
let entry = st
|
||||
.entries
|
||||
.get(&ino)
|
||||
.ok_or_else(|| CoreError::not_found(format!("{} is gone", rel.display())))?;
|
||||
// Unsaved writes, or a release still queueing them, to the file or,
|
||||
// for a folder, to any file below it.
|
||||
let unsaved = st
|
||||
.active_writes
|
||||
.iter()
|
||||
.filter(|(_, w)| w.dirty)
|
||||
.map(|(&open, _)| open)
|
||||
.chain(st.releasing.iter().copied())
|
||||
.any(|open| relocate::ino_is_under(&st, open, ino));
|
||||
(entry.node.name.clone(), entry.node.is_folder(), unsaved)
|
||||
};
|
||||
// Unlike a move, a copy cannot wait in the queue: it needs the node as
|
||||
// Drive has it, so it goes now or not at all, local-first or not.
|
||||
if !self.is_online() {
|
||||
return Err(CoreError::offline());
|
||||
}
|
||||
let owed = unsaved
|
||||
|| is_local_uid(&uid)
|
||||
|| is_local_uid(&new_parent_uid)
|
||||
|| self.db.has_any_op(&uid.to_string()).unwrap_or(true)
|
||||
|| (is_folder
|
||||
&& self
|
||||
.db
|
||||
.pending_op_counts()
|
||||
.map(|counts| counts.uploads + counts.changes > 0)
|
||||
.unwrap_or(true));
|
||||
if owed {
|
||||
return Err(CoreError::conflict(format!(
|
||||
"{name} has changes that have not reached Proton Drive yet; \
|
||||
copy it once they have synced"
|
||||
)));
|
||||
}
|
||||
let target = if is_folder {
|
||||
AvailableNameTarget::Folder
|
||||
} else {
|
||||
AvailableNameTarget::File
|
||||
};
|
||||
let copy_name = self
|
||||
.rt
|
||||
.block_on(
|
||||
self.drive
|
||||
.get_available_name(&new_parent_uid, &name, target),
|
||||
)
|
||||
.map_err(|e| CoreError::from_api(&e, "copy"))?;
|
||||
let new_uid = self
|
||||
.rt
|
||||
.block_on(
|
||||
self.drive
|
||||
.copy_node(&uid, &new_parent_uid, Some(©_name)),
|
||||
)
|
||||
.map_err(|e| CoreError::from_api(&e, "copy"))?;
|
||||
let node = self
|
||||
.fetch_minted_node(&new_uid, ©_name)
|
||||
.ok_or_else(|| minted_but_unlisted(©_name))?;
|
||||
let mut st = self.state();
|
||||
let copy_ino = st.intern(pino, node);
|
||||
if let Some(kids) = st.children.get_mut(&pino)
|
||||
&& !kids.contains(©_ino)
|
||||
{
|
||||
kids.push(copy_ino);
|
||||
}
|
||||
drop(st);
|
||||
self.flush_access_changes();
|
||||
Ok(copy_name)
|
||||
}
|
||||
|
||||
/// What the folder at `rel` holds on Proton Drive: bytes on storage and the
|
||||
/// number of items below it. Drive adds it up, so nothing is listed here;
|
||||
/// trashed items count, and the bytes are encrypted ones.
|
||||
pub(crate) fn folder_size(&self, rel: &Path) -> CoreResult<proton_drive_rs::FolderSize> {
|
||||
let (ino, uid) = self.resolve(rel)?;
|
||||
let is_folder = self
|
||||
.state()
|
||||
.entries
|
||||
.get(&ino)
|
||||
.is_some_and(|entry| entry.node.is_folder());
|
||||
if !is_folder {
|
||||
return Err(CoreError::invalid(format!(
|
||||
"{} is not a folder",
|
||||
rel.display()
|
||||
)));
|
||||
}
|
||||
self.folder_size_for_uid(&uid)
|
||||
}
|
||||
|
||||
/// [`Core::folder_size`] for a folder addressed through any daemon location.
|
||||
pub(crate) fn folder_size_by_uid(&self, uid: &str) -> CoreResult<proton_drive_rs::FolderSize> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
self.folder_size_for_uid(&uid)
|
||||
}
|
||||
|
||||
fn folder_size_for_uid(&self, uid: &NodeUid) -> CoreResult<proton_drive_rs::FolderSize> {
|
||||
if is_local_uid(uid) {
|
||||
return Err(CoreError::conflict("the folder is not on Proton Drive yet"));
|
||||
}
|
||||
if !self.is_online() {
|
||||
return Err(CoreError::offline());
|
||||
}
|
||||
self.rt
|
||||
.block_on(self.client.get_folder_size(uid))
|
||||
.map_err(|e| CoreError::from_api(&e, "folder size"))
|
||||
}
|
||||
|
||||
/// Trash a file or folder. `rel` is mountpoint-relative. Forgets the node,
|
||||
/// evicts any cached content, and invalidates the parent listing.
|
||||
fn delete(&self, rel: &Path) -> CoreResult<String> {
|
||||
@@ -5103,6 +5401,11 @@ impl Core {
|
||||
&parent_uid,
|
||||
)
|
||||
.map_err(|error| self.errno_error(error, "trash access"))?;
|
||||
if self.db.has_pending_moves_out_of(&uid.to_string())? {
|
||||
return Err(CoreError::conflict(
|
||||
"wait for pending moves out of this folder to sync before trashing it",
|
||||
));
|
||||
}
|
||||
self.rt
|
||||
.block_on(self.drive.trash_nodes(std::slice::from_ref(&uid)))
|
||||
.and_then(batch::into_unit)
|
||||
@@ -5140,9 +5443,12 @@ impl Core {
|
||||
/// `unlink` under an open handle. Its queued ops go now, or the drain would
|
||||
/// upload to the node after it was trashed (`docs/BUGS.md` B106).
|
||||
fn remove_replaced(&self, ino: u64, uid: &NodeUid, name: &str) -> Result<(), Errno> {
|
||||
// Asked under the uid the tree has now: a landed create's real uid is
|
||||
// not in the tree yet (`docs/BUGS.md` B257).
|
||||
let open = self.is_open_anywhere(uid);
|
||||
let landed;
|
||||
let uid = if is_local_uid(uid) {
|
||||
if self.is_open_anywhere(uid) {
|
||||
if open {
|
||||
self.withdraw_queued_ops(uid)?;
|
||||
} else {
|
||||
self.discard_queued_ops(uid)?;
|
||||
@@ -5164,10 +5470,12 @@ impl Core {
|
||||
} else {
|
||||
uid
|
||||
};
|
||||
// Local-first or offline, or the network goes away under the call:
|
||||
// queue the trash. The rename that follows queues as well, behind it.
|
||||
// Local-first or offline: queue the trash and the rename that follows.
|
||||
if !self.sends_inline() {
|
||||
return self.queue_trash(uid, name);
|
||||
return self.queue_trash(uid, name, open);
|
||||
}
|
||||
if self.has_pending_moves_out_of(uid)? {
|
||||
return Err(Errno::EBUSY);
|
||||
}
|
||||
self.require_uid_writable(uid)?;
|
||||
if let Err(e) = self
|
||||
@@ -5175,7 +5483,7 @@ impl Core {
|
||||
.and_then(batch::into_unit)
|
||||
{
|
||||
if self.lost_link(&e, "trash a replaced node") {
|
||||
return self.queue_trash(uid, name);
|
||||
return self.queue_trash(uid, name, open);
|
||||
}
|
||||
error!(%uid, name, error = %e, "trashing the node a rename replaces failed");
|
||||
self.log_activity(ActivityKind::Trash, name, e.to_string(), false);
|
||||
@@ -5317,6 +5625,7 @@ impl Core {
|
||||
// A trashed node can't be pinned or served from the mount, so its
|
||||
// content is never current cache: report neither.
|
||||
pinned: false,
|
||||
pinned_by_folder: false,
|
||||
cached: false,
|
||||
uid: item.uid,
|
||||
path: String::new(),
|
||||
@@ -5463,7 +5772,7 @@ impl Core {
|
||||
/// Drop the persisted photos timeline's freshness stamp, so the next timeline
|
||||
/// read fetches rather than serving what it already has.
|
||||
fn invalidate_photos(&self) {
|
||||
let _ = self.db.clear_state(PHOTOS_SYNCED_MS);
|
||||
let _ = self.db.photos_invalidate();
|
||||
}
|
||||
|
||||
/// Drop one folder's cached child listing (`rel` is mountpoint-relative), so
|
||||
@@ -5747,6 +6056,7 @@ impl Core {
|
||||
if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) {
|
||||
f(p);
|
||||
}
|
||||
self.sync_engine.moved();
|
||||
}
|
||||
|
||||
/// Set the number of items the scan expects to check, from the size of the
|
||||
@@ -6050,11 +6360,61 @@ fn is_gone(e: &(dyn std::error::Error + 'static)) -> bool {
|
||||
/// (`sync.rs`, `conflict_path`), so the two halves of the product name the same
|
||||
/// situation the same way and a user only has to learn it once.
|
||||
fn conflict_name(name: &str, stamp: i64) -> String {
|
||||
conflict_name_as(name, conflict_label(), stamp)
|
||||
}
|
||||
|
||||
/// [`conflict_name`] with the machine's label given.
|
||||
fn conflict_name_as(name: &str, label: &str, stamp: i64) -> String {
|
||||
let path = Path::new(name);
|
||||
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(name);
|
||||
let tag = conflict_tag(label, stamp, 0);
|
||||
match path.extension().and_then(|s| s.to_str()) {
|
||||
Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {stamp})"),
|
||||
Some(ext) => format!("{stem} (sync-conflict {tag}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {tag})"),
|
||||
}
|
||||
}
|
||||
|
||||
/// This machine's name as its conflict copies carry it (`docs/BUGS.md` B251).
|
||||
///
|
||||
/// Two computers syncing one folder that each keep a copy of the same file in
|
||||
/// the same second would otherwise make the same name on Drive, and nobody
|
||||
/// could tell which computer a copy came from. Read once, from the hostname.
|
||||
fn conflict_label() -> &'static str {
|
||||
static LABEL: OnceLock<String> = OnceLock::new();
|
||||
LABEL.get_or_init(|| conflict_label_of(&this_hostname()))
|
||||
}
|
||||
|
||||
/// `host` as one token: anything but ASCII letters, digits, `-`, `_` and `.`
|
||||
/// becomes `-`, so the label holds no space for `conflict_base_name` to trip
|
||||
/// on and no `/` or `)` to break the name.
|
||||
fn conflict_label_of(host: &str) -> String {
|
||||
let label: String = host
|
||||
.chars()
|
||||
.take(32)
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') {
|
||||
c
|
||||
} else {
|
||||
'-'
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
label.trim_matches('-').to_string()
|
||||
}
|
||||
|
||||
/// The inside of a conflict copy's marker: `<label> <stamp>`, with `-<n>`
|
||||
/// when an earlier copy already holds that name. Without a label it is the
|
||||
/// bare stamp that copies had before B251.
|
||||
fn conflict_tag(label: &str, stamp: i64, suffix: u32) -> String {
|
||||
let suffix = if suffix == 0 {
|
||||
String::new()
|
||||
} else {
|
||||
format!("-{suffix}")
|
||||
};
|
||||
if label.is_empty() {
|
||||
format!("{stamp}{suffix}")
|
||||
} else {
|
||||
format!("{label} {stamp}{suffix}")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6138,6 +6498,7 @@ fn sync_folder_info(
|
||||
) -> SyncFolderInfo {
|
||||
SyncFolderInfo {
|
||||
id: f.id,
|
||||
kind: f.kind,
|
||||
local_path: f.local_path,
|
||||
remote_uid: f.remote_uid,
|
||||
mode: f.mode,
|
||||
@@ -6664,7 +7025,27 @@ fn fuse_name(name: &OsStr) -> Result<String, Errno> {
|
||||
name.to_str().map(str::to_owned).ok_or(Errno::EILSEQ)
|
||||
}
|
||||
|
||||
/// Whether a decrypted remote name can stand as one local path component.
|
||||
///
|
||||
/// Drive checks names only when a client creates or renames a node, and only
|
||||
/// for length, so a name made by another client, or by a server that should
|
||||
/// not be trusted, can hold anything. Joined onto a local path, a `/` or a `..`
|
||||
/// in it walks out of the folder it belongs to, and the mirror engine would
|
||||
/// then write or delete outside the sync root. In the mount, the kernel fails
|
||||
/// the whole listing with EIO on a single such name (`docs/BUGS.md` B215).
|
||||
pub(crate) fn remote_name_is_safe(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& name.len() <= 255
|
||||
&& !name.contains(['/', '\0'])
|
||||
}
|
||||
|
||||
fn node_visible(node: &Node, folder_uid: &NodeUid, hidden: &HashSet<NodeUid>) -> bool {
|
||||
if !remote_name_is_safe(&node.name) {
|
||||
warn!(uid = %node.uid, "a remote name cannot be a local file name; leaving it out");
|
||||
return false;
|
||||
}
|
||||
!node.trashed && node.uid != *folder_uid && !hidden.contains(&node.uid)
|
||||
}
|
||||
|
||||
@@ -7000,13 +7381,15 @@ mod size_upgrade_tests {
|
||||
fn a_listing_waits_for_the_whole_size_batch_but_not_forever() {
|
||||
let slot = Arc::new(SizeUpgrade::default());
|
||||
let worker = slot.clone();
|
||||
// Read before the spawn: the worker's sleeps can start before this
|
||||
// thread runs again, which made the bound below fail on a busy runner.
|
||||
let started = Instant::now();
|
||||
let t = std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(30));
|
||||
worker.chunk_done();
|
||||
std::thread::sleep(Duration::from_millis(60));
|
||||
worker.finish();
|
||||
});
|
||||
let started = Instant::now();
|
||||
slot.wait_finished(started + Duration::from_secs(5));
|
||||
// Past the first chunk, up to the end of the batch.
|
||||
assert!(started.elapsed() >= Duration::from_millis(90));
|
||||
@@ -7413,14 +7796,15 @@ mod tests {
|
||||
use super::{
|
||||
Access, AccessFlags, Errno, HashMap, Intervals, PendingRevision, RootListingSnapshot,
|
||||
SELF_CHANGE_TTL_MS, ShareId, SharedWithMeItem, StateRegistry, VirtualRootPlan,
|
||||
accepted_share_provenance, conflict_name, copy_pending_for_truncate, expand_restore,
|
||||
first_seen_in_trash, fuse_name, is_stale_mount, node_visible, note_self_change,
|
||||
off_the_runtime, orphan_past_retention, parse_node_uid, prepare_shared_roots,
|
||||
preserve_on_access_denied, publish_virtual_root_in_listing,
|
||||
reconcile_virtual_root_in_listing, release_can_discard_unlinked,
|
||||
release_must_retain_queued_trash, release_unlinked_entry, rename_needs_queue,
|
||||
require_node_parent_access, require_rename_access, resolve_anywhere_with,
|
||||
shared_with_me_uid, take_self_change, uid_write_authority, virtual_node,
|
||||
accepted_share_provenance, conflict_label, conflict_label_of, conflict_name,
|
||||
conflict_name_as, copy_pending_for_truncate, expand_restore, first_seen_in_trash,
|
||||
fuse_name, is_stale_mount, node_visible, note_self_change, off_the_runtime,
|
||||
orphan_past_retention, parse_node_uid, prepare_shared_roots, preserve_on_access_denied,
|
||||
publish_virtual_root_in_listing, reconcile_virtual_root_in_listing,
|
||||
release_can_discard_unlinked, release_must_retain_queued_trash, release_unlinked_entry,
|
||||
rename_needs_queue, require_node_parent_access, require_rename_access,
|
||||
resolve_anywhere_with, shared_with_me_uid, take_self_change, uid_write_authority,
|
||||
virtual_node,
|
||||
};
|
||||
use super::{Db, WriteAuthority};
|
||||
use std::time::Duration;
|
||||
@@ -8375,15 +8759,38 @@ mod tests {
|
||||
#[test]
|
||||
fn conflict_name_keeps_the_extension() {
|
||||
assert_eq!(
|
||||
conflict_name("notes.txt", 1700),
|
||||
"notes (sync-conflict 1700).txt"
|
||||
conflict_name_as("notes.txt", "laptop", 1700),
|
||||
"notes (sync-conflict laptop 1700).txt"
|
||||
);
|
||||
assert_eq!(conflict_name("README", 42), "README (sync-conflict 42)");
|
||||
assert_eq!(
|
||||
conflict_name("archive.tar.gz", 7),
|
||||
"archive.tar (sync-conflict 7).gz",
|
||||
conflict_name_as("README", "laptop", 42),
|
||||
"README (sync-conflict laptop 42)"
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_name_as("archive.tar.gz", "laptop", 7),
|
||||
"archive.tar (sync-conflict laptop 7).gz",
|
||||
"only the last extension is one, as everywhere else"
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_name_as("notes.txt", "", 1700),
|
||||
"notes (sync-conflict 1700).txt"
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_name("notes.txt", 1700),
|
||||
conflict_name_as("notes.txt", conflict_label(), 1700)
|
||||
);
|
||||
}
|
||||
|
||||
/// A hostname goes into a file name, so it is cut down to one safe token
|
||||
/// (B251).
|
||||
#[test]
|
||||
fn a_hostname_becomes_one_safe_token() {
|
||||
assert_eq!(conflict_label_of("laptop"), "laptop");
|
||||
assert_eq!(conflict_label_of("my-pc.local"), "my-pc.local");
|
||||
assert_eq!(conflict_label_of("Linux device"), "Linux-device");
|
||||
assert_eq!(conflict_label_of("a/b) c"), "a-b--c");
|
||||
assert_eq!(conflict_label_of(" "), "");
|
||||
assert_eq!(conflict_label_of(&"x".repeat(40)).len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -8575,6 +8982,28 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_remote_name_that_is_not_one_path_component_is_left_out() {
|
||||
let folder_uid = NodeUid::new(VolumeId::from("vol"), LinkId::from("folder"));
|
||||
let none = std::collections::HashSet::new();
|
||||
for name in [
|
||||
"a/b",
|
||||
"../../.bashrc",
|
||||
"..",
|
||||
".",
|
||||
"",
|
||||
"a\0b",
|
||||
&"x".repeat(256),
|
||||
] {
|
||||
let node = node_helper("child", "folder", name, false);
|
||||
assert!(!node_visible(&node, &folder_uid, &none), "{name:?}");
|
||||
}
|
||||
for name in [".hidden", "a..b", "unicodé-文件", &"x".repeat(255)] {
|
||||
let node = node_helper("child", "folder", name, false);
|
||||
assert!(node_visible(&node, &folder_uid, &none), "{name:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn denied_queue_guard_preserves_accepted_bytes_and_pending_count() {
|
||||
use std::cell::Cell;
|
||||
@@ -8895,7 +9324,7 @@ mod tests {
|
||||
assert!(!replaced.contains("st.forget(uid)"));
|
||||
assert_before(
|
||||
replaced,
|
||||
"if self.is_open_anywhere(uid)",
|
||||
"let open = self.is_open_anywhere(uid)",
|
||||
"self.discard_queued_ops(uid)",
|
||||
);
|
||||
assert_before(replaced, "if open_now", "self.cache.evict(uid)");
|
||||
@@ -9080,6 +9509,32 @@ mod tests {
|
||||
assert!(online.contains("self.relist_offline_listings();"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_shared_listing_is_fetched_off_the_dispatch_loop() {
|
||||
// A resident listing that `list_children` would fetch again anyway
|
||||
// must not be answered inline (B219).
|
||||
let cached = function_source(include_str!("lib.rs"), "fn children_cached(");
|
||||
assert!(cached.contains("SHARED_WITH_ME_SYNCED_MS"));
|
||||
assert!(cached.contains("Self::shared_folder_freshness_key(&uid)"));
|
||||
assert!(cached.contains("listing_needs_refresh("));
|
||||
assert!(cached.contains("self.shared_listing_stale(&key)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_handler_does_not_wait_out_the_sdk_retry_budget() {
|
||||
// Thumbnails, size upgrades and the share list held mount threads for
|
||||
// minutes on a stalled link (B220).
|
||||
let lib = include_str!("lib.rs");
|
||||
let thumbnail = function_source(lib, "fn thumbnail(");
|
||||
assert_before(thumbnail, "if !self.is_online()", "download_thumbnail");
|
||||
assert!(thumbnail.contains(".block_on_bounded(self.client.download_thumbnail("));
|
||||
let upgrade = function_source(lib, "fn run_size_upgrade(");
|
||||
assert!(upgrade.contains("block_on_within(BATCH_CALL_TIMEOUT"));
|
||||
let shared = function_source(lib, "fn ensure_shared_children(");
|
||||
assert!(shared.contains("block_on_within(BATCH_CALL_TIMEOUT"));
|
||||
assert!(!shared.contains("self.rt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_remote_read_gives_up_at_its_deadline() {
|
||||
// The SDK bounds requests, not reads; a stalled link held one `read`
|
||||
|
||||
@@ -9,10 +9,11 @@
|
||||
//! queued path, and the probe thread here flips it back when Proton answers.
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use parking_lot::{Condvar, Mutex};
|
||||
use pdfs_core::error::session_revoked;
|
||||
use proton_drive_rs::proton_sdk::api::ResponseCode;
|
||||
use proton_drive_rs::proton_sdk::error::{ProtonApiError, ProtonError};
|
||||
use tracing::{debug, info, warn};
|
||||
@@ -28,6 +29,13 @@ use crate::{Core, ONLINE_PROBE_MAX, ONLINE_PROBE_MIN};
|
||||
/// keep waiting for; queueing the change is the better answer.
|
||||
pub(crate) const INTERACTIVE_CALL_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// How long a mount thread waits on a remote read that answers a batch: a
|
||||
/// chunk of file sizes, or the list of shares. Longer than an interactive call,
|
||||
/// since its answer grows with the batch, but well short of the SDK's own retry
|
||||
/// budget, which held a worker for minutes on a stalled link
|
||||
/// (`docs/BUGS.md` B220).
|
||||
pub(crate) const BATCH_CALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// How long one probe may wait for an answer. A probe is a single small request,
|
||||
/// and a probe stuck in the SDK's retry budget would delay noticing the link is
|
||||
/// back by as long as it waits.
|
||||
@@ -54,6 +62,26 @@ pub(crate) struct Link {
|
||||
changed: Condvar,
|
||||
/// Folders listed from the DB alone while offline, to relist once back.
|
||||
pub(crate) stale_listings: Mutex<std::collections::HashSet<u64>>,
|
||||
/// Proton refused the session itself ([`session_revoked`]). The mount is
|
||||
/// offline then, but no probe brings it back: only a new login does, and
|
||||
/// that restarts the daemon. Read by front-ends through `Response::Status`,
|
||||
/// so they can ask the user to sign in again rather than say "offline".
|
||||
pub(crate) session_expired: AtomicBool,
|
||||
/// Proton refused the session, but only because another process rotated
|
||||
/// it (`pdfs unlock`): the keyring holds the live tokens. The mount loop
|
||||
/// ends the mount for the daemon to mount again from the keyring.
|
||||
pub(crate) resume_stored: AtomicBool,
|
||||
}
|
||||
|
||||
impl Link {
|
||||
/// The link of a mount that comes up with Proton refusing its session, or
|
||||
/// not.
|
||||
pub(crate) fn starting(session_expired: bool) -> Self {
|
||||
Self {
|
||||
session_expired: AtomicBool::new(session_expired),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a failed call failed because Proton could not be reached, rather
|
||||
@@ -91,10 +119,25 @@ pub(crate) fn no_answer(after: Duration) -> ProtonError {
|
||||
/// transfer that has stopped moving runs into it.
|
||||
const SLOWEST_UPLOAD: u64 = 32 * 1024;
|
||||
|
||||
/// How many uploads can share the upload cap at once: one per drain worker
|
||||
/// and one per mirror sync task.
|
||||
const UPLOAD_SLOTS: u64 = (crate::DRAIN_WORKERS + crate::sync::SYNC_CONCURRENCY) as u64;
|
||||
|
||||
/// How long an upload of `len` bytes may take before the link is treated as
|
||||
/// down: the budget of a metadata call, plus the bytes at [`SLOWEST_UPLOAD`].
|
||||
pub(crate) fn upload_deadline(len: u64) -> Duration {
|
||||
INTERACTIVE_CALL_TIMEOUT + Duration::from_secs(len / SLOWEST_UPLOAD)
|
||||
///
|
||||
/// An upload cap of `cap` bytes per second (`0` for none) is shared by every
|
||||
/// upload running, so each may get as little as its share of it. Below
|
||||
/// [`SLOWEST_UPLOAD`] a share counts as the slowest rate instead: measured
|
||||
/// against the fixed rate, a large file under a low cap ran out of time on
|
||||
/// every attempt, took the mount offline each time, and started over from
|
||||
/// its first byte.
|
||||
pub(crate) fn upload_deadline(len: u64, cap: u64) -> Duration {
|
||||
let slowest = match cap {
|
||||
0 => SLOWEST_UPLOAD,
|
||||
cap => SLOWEST_UPLOAD.min((cap / UPLOAD_SLOTS).max(1)),
|
||||
};
|
||||
INTERACTIVE_CALL_TIMEOUT + Duration::from_secs(len / slowest)
|
||||
}
|
||||
|
||||
/// Run a remote call, giving up after [`INTERACTIVE_CALL_TIMEOUT`].
|
||||
@@ -161,10 +204,20 @@ impl Core {
|
||||
self.rt.block_on(bounded_by(limit, call))
|
||||
}
|
||||
|
||||
/// Whether Proton refused the session itself, so nothing reaches it until
|
||||
/// the user signs in again.
|
||||
pub(crate) fn session_expired(&self) -> bool {
|
||||
self.link.session_expired.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Report a failed remote call. Returns whether it failed for want of a
|
||||
/// network, in which case the mount is now offline and the caller should
|
||||
/// take its queued path instead of failing.
|
||||
/// network, or of a session, in which case the mount is now offline and the
|
||||
/// caller should take its queued path instead of failing.
|
||||
pub(crate) fn lost_link(&self, e: &ProtonError, what: &str) -> bool {
|
||||
if session_revoked(e) {
|
||||
self.mark_session_expired(what, e);
|
||||
return true;
|
||||
}
|
||||
if !is_network_error(e) {
|
||||
return false;
|
||||
}
|
||||
@@ -172,6 +225,32 @@ impl Core {
|
||||
true
|
||||
}
|
||||
|
||||
/// Flip the mount offline for good: Proton refused the session, and only a
|
||||
/// new login brings it back. Changes stay queued for the daemon that login
|
||||
/// starts. Only the first caller logs it.
|
||||
///
|
||||
/// Unless the keyring holds a newer session than the refused one: then
|
||||
/// the mount ends, for the daemon to mount again on that one.
|
||||
pub(crate) fn mark_session_expired(&self, what: &str, error: &dyn std::fmt::Display) {
|
||||
if let Some(gallery) = &self.gallery {
|
||||
gallery.cancel();
|
||||
}
|
||||
if self.link.resume_stored.load(Ordering::Relaxed) {
|
||||
return self.mark_offline(what, error);
|
||||
}
|
||||
if pdfs_core::auth::stored_session_is_newer() {
|
||||
info!(during = what, %error,
|
||||
"Proton refused a session another process has since refreshed; resuming it");
|
||||
self.link.resume_stored.store(true, Ordering::Relaxed);
|
||||
return self.mark_offline(what, error);
|
||||
}
|
||||
if !self.link.session_expired.swap(true, Ordering::Relaxed) {
|
||||
warn!(during = what, %error,
|
||||
"Proton refused the session; run `pdfs login` to sign in again");
|
||||
}
|
||||
self.mark_offline(what, error);
|
||||
}
|
||||
|
||||
/// Flip the mount offline and wake the probe. Only the first caller of an
|
||||
/// outage logs it; the rest are the same outage.
|
||||
pub(crate) fn mark_offline(&self, what: &str, error: &dyn std::fmt::Display) {
|
||||
@@ -204,11 +283,15 @@ impl Core {
|
||||
self.relist_offline_listings();
|
||||
// Anything written while offline is queued and waiting on exactly this,
|
||||
// including ops that failed on the way down and are sitting out a
|
||||
// backoff sized for a server that keeps refusing.
|
||||
if let Err(e) = self.db.retry_failed_ops_now(crate::now_millis()) {
|
||||
// backoff sized for a server that keeps refusing. Ops Drive refused
|
||||
// keep theirs: the link was not why.
|
||||
if let Err(e) = self.db.retry_unrefused_ops_now(crate::now_millis()) {
|
||||
warn!(error = %e, "moving backed-off ops up after a reconnect failed");
|
||||
}
|
||||
self.wake_drain();
|
||||
// Downloads for pins failed while the link was down, and the events
|
||||
// that would have named the files are behind us (`docs/BUGS.md` B282).
|
||||
self.recheck_all_pins();
|
||||
}
|
||||
|
||||
/// Wait up to `budget` for the link to come back. Returns whether it is up.
|
||||
@@ -276,6 +359,9 @@ impl Core {
|
||||
|
||||
/// Probe until Proton answers, then flip the mount online. Returns `false`
|
||||
/// if the daemon is stopping.
|
||||
///
|
||||
/// A refused session is not probed: every probe would be refused the same
|
||||
/// way, forever (issue #35). The thread waits for the stop instead.
|
||||
fn probe_until_online(&self) -> bool {
|
||||
let started = self.link.lost_at.lock().unwrap_or_else(Instant::now);
|
||||
let mut delay = if started.elapsed() < YOUNG_OUTAGE {
|
||||
@@ -287,6 +373,10 @@ impl Core {
|
||||
if !self.shutdown.sleep(delay) {
|
||||
return false;
|
||||
}
|
||||
if self.session_expired() {
|
||||
delay = ONLINE_PROBE_MAX;
|
||||
continue;
|
||||
}
|
||||
let probe = self.rt.block_on(async {
|
||||
match tokio::time::timeout(PROBE_TIMEOUT, self.drive.get_my_files_folder()).await {
|
||||
Ok(result) => result,
|
||||
@@ -308,6 +398,7 @@ impl Core {
|
||||
self.mark_online();
|
||||
return true;
|
||||
}
|
||||
Err(e) if session_revoked(&e) => self.mark_session_expired("online probe", &e),
|
||||
Err(e) => {
|
||||
debug!(error = %e, ?delay, "online probe failed; still offline");
|
||||
delay = next_probe_delay(delay, started.elapsed());
|
||||
@@ -341,13 +432,31 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn an_upload_gets_longer_the_bigger_it_is() {
|
||||
assert_eq!(upload_deadline(0), INTERACTIVE_CALL_TIMEOUT);
|
||||
assert_eq!(upload_deadline(0, 0), INTERACTIVE_CALL_TIMEOUT);
|
||||
assert_eq!(
|
||||
upload_deadline(1 << 30),
|
||||
upload_deadline(1 << 30, 0),
|
||||
INTERACTIVE_CALL_TIMEOUT + Duration::from_secs(32_768)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_upload_under_a_low_cap_gets_the_time_its_share_of_the_cap_needs() {
|
||||
// A generous cap leaves the deadline as it was.
|
||||
assert_eq!(
|
||||
upload_deadline(1 << 30, 100 << 20),
|
||||
upload_deadline(1 << 30, 0)
|
||||
);
|
||||
// 512 KiB/s shared by every upload slot is below the slowest rate.
|
||||
let share = (512 << 10) / UPLOAD_SLOTS;
|
||||
assert!(share < SLOWEST_UPLOAD);
|
||||
assert_eq!(
|
||||
upload_deadline(1 << 30, 512 << 10),
|
||||
INTERACTIVE_CALL_TIMEOUT + Duration::from_secs((1 << 30) / share)
|
||||
);
|
||||
// A cap below one byte per slot still gives a finite deadline.
|
||||
assert!(upload_deadline(1 << 20, 1) > upload_deadline(1 << 20, 512 << 10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_call_past_its_deadline_reads_as_a_lost_link() {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
|
||||
+186
-33
@@ -12,6 +12,10 @@ pub enum MountOutcome {
|
||||
/// The kernel mount ended on its own (e.g. an external `fusermount -u`).
|
||||
/// The caller may want to remount.
|
||||
Unmounted,
|
||||
/// Proton refused the session, and the keyring holds a newer one: another
|
||||
/// process (`pdfs unlock`) refreshed it, or a new login replaced it. We
|
||||
/// unmounted ourselves; the caller should mount again, which resumes it.
|
||||
ResumeStored,
|
||||
}
|
||||
|
||||
/// Whether `path` is a mountpoint whose FUSE connection is dead — the state a
|
||||
@@ -192,6 +196,43 @@ pub(super) fn clear_stale_mount(mountpoint: &Path) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Make sure `mountpoint` is a directory a mount can go on, clearing a dead
|
||||
/// mount a killed daemon left there first.
|
||||
///
|
||||
/// A dead mount answers `ENOTCONN`, which `create_dir_all` takes for "absent";
|
||||
/// it then tries `mkdir` and fails with `EEXIST`. The daemon retried that error
|
||||
/// forever and never got as far as [`check_mountpoint`], which would have
|
||||
/// cleared the mount (`docs/BUGS.md` B279).
|
||||
pub fn prepare_mountpoint(mountpoint: &Path) -> std::io::Result<()> {
|
||||
if is_stale_mount(mountpoint) {
|
||||
clear_stale_mount(mountpoint);
|
||||
}
|
||||
std::fs::create_dir_all(mountpoint)
|
||||
}
|
||||
|
||||
/// Refuse to mount My files over a folder that already has files in it: FUSE
|
||||
/// would hide them, unsent and out of sight, until the service stops
|
||||
/// (`docs/BUGS.md` B277). Runs before the root fetch and before any worker
|
||||
/// starts, so a refusal costs nothing to retry.
|
||||
///
|
||||
/// A dead mount is cleared first, since it cannot be listed. One of this
|
||||
/// client's own live mounts is left to [`spawn_session`], which clears it as it
|
||||
/// always has; listing it would show Drive's files, not the folder's.
|
||||
pub(super) fn check_mountpoint(mountpoint: &Path) -> std::io::Result<()> {
|
||||
if is_stale_mount(mountpoint) {
|
||||
clear_stale_mount(mountpoint);
|
||||
}
|
||||
if pdfs_core::mountpoint::mounted_fs_type(mountpoint).as_deref()
|
||||
== Some(pdfs_core::mountpoint::OWN_FS_TYPE)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
match pdfs_core::mountpoint::not_empty(mountpoint)? {
|
||||
Some(found) => Err(std::io::Error::other(found)),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `sync_state` key holding the uid of the My Files root, so a later run can
|
||||
/// recover it from `nodes` without the API.
|
||||
const ROOT_UID_KEY: &str = "root_uid";
|
||||
@@ -210,35 +251,66 @@ fn fetch_or_recall_root(
|
||||
drive: &dyn DriveApi,
|
||||
rt: &tokio::runtime::Handle,
|
||||
db: &Db,
|
||||
) -> std::io::Result<(Node, bool)> {
|
||||
) -> std::io::Result<(Node, Start)> {
|
||||
let err = match rt.block_on(drive.get_my_files_folder()) {
|
||||
Ok(root) => {
|
||||
if let Err(e) = db.set_state_str(ROOT_UID_KEY, &root.uid.to_string()) {
|
||||
warn!(error = %e, "persist root uid failed");
|
||||
}
|
||||
return Ok((root, true));
|
||||
return Ok((root, Start::Online));
|
||||
}
|
||||
Err(e) => e,
|
||||
};
|
||||
// Still mounted from the cache, so cached files stay readable and changes
|
||||
// queue for the daemon the next login starts.
|
||||
let start = if pdfs_core::error::session_revoked(&err) {
|
||||
Start::SessionExpired
|
||||
} else {
|
||||
Start::Offline
|
||||
};
|
||||
if start == Start::SessionExpired && pdfs_core::auth::stored_session_is_newer() {
|
||||
return Err(std::io::Error::other(format!(
|
||||
"Proton refused a session another process has since refreshed ({err}); resuming it"
|
||||
)));
|
||||
}
|
||||
let cached = db
|
||||
.state_str(ROOT_UID_KEY)
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|uid| db.node_by_uid(&uid).ok().flatten());
|
||||
match cached {
|
||||
Some(root) if start == Start::SessionExpired => {
|
||||
warn!(error = %err,
|
||||
"Proton refused the session; mounting from cache until `pdfs login`");
|
||||
Ok((root, start))
|
||||
}
|
||||
Some(root) => {
|
||||
warn!(error = %err, "fetch My Files root failed; mounting from cache (offline)");
|
||||
Ok((root, false))
|
||||
Ok((root, start))
|
||||
}
|
||||
None if start == Start::SessionExpired => Err(std::io::Error::other(format!(
|
||||
"Proton refused the session ({err}); run `pdfs login`"
|
||||
))),
|
||||
None => Err(std::io::Error::other(format!("fetch My Files root: {err}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// How a mount comes up, from what fetching its root said.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Start {
|
||||
Online,
|
||||
/// Proton could not be reached: serve the cache and probe for the link.
|
||||
Offline,
|
||||
/// Proton refused the session: serve the cache until a new login.
|
||||
SessionExpired,
|
||||
}
|
||||
|
||||
/// Per-mount settings resolved from [`pdfs_core::config::AppConfig`] by the
|
||||
/// caller, so `mount` itself never reads config or the environment.
|
||||
pub struct MountOptions {
|
||||
/// Proton account the session belongs to, for display and per-user paths.
|
||||
pub username: String,
|
||||
pub gallery_session: Option<proton_drive_rs::proton_sdk::session::ProtonApiSession>,
|
||||
/// What the background conflict sweep may do. See `docs/BUGS.md` B71.
|
||||
pub sweep_mode: SweepMode,
|
||||
/// Upload and download caps in bytes per second (`0` = no cap).
|
||||
@@ -385,6 +457,7 @@ pub(crate) fn mount_with(
|
||||
) -> std::io::Result<MountOutcome> {
|
||||
let MountOptions {
|
||||
username,
|
||||
gallery_session,
|
||||
sweep_mode,
|
||||
upload_limit,
|
||||
download_limit,
|
||||
@@ -393,7 +466,9 @@ pub(crate) fn mount_with(
|
||||
// Start the uptime clock here rather than at the first `pdfs diagnostics`,
|
||||
// so the age it reports is the daemon's own.
|
||||
super::diagnostics::start_clock();
|
||||
let (root, online) = fetch_or_recall_root(drive.as_ref(), &rt, &db)?;
|
||||
check_mountpoint(mountpoint)?;
|
||||
let (root, start) = fetch_or_recall_root(drive.as_ref(), &rt, &db)?;
|
||||
let online = start == Start::Online;
|
||||
let scope = root.tree_event_scope_id();
|
||||
db.mount_upsert_my_files(&mountpoint.to_string_lossy(), &root.uid.to_string(), None)
|
||||
.map_err(|error| std::io::Error::other(format!("project My Files mount: {error}")))?;
|
||||
@@ -407,9 +482,13 @@ pub(crate) fn mount_with(
|
||||
let (sync_tx, sync_rx) = std::sync::mpsc::channel::<sync::SyncMsg>();
|
||||
|
||||
let paused_until = pause::load_paused_until(&db);
|
||||
let gallery_dir = control_socket.with_file_name("gallery-catalog");
|
||||
let gallery =
|
||||
gallery_session.map(|session| Arc::new(super::gallery::Gallery::new(gallery_dir, session)));
|
||||
let core = Core {
|
||||
drive,
|
||||
client,
|
||||
gallery,
|
||||
rt: rt.clone(),
|
||||
maintenance: Arc::new(Mutex::new(Default::default())),
|
||||
primary_root_uid: root.uid.clone(),
|
||||
@@ -429,7 +508,7 @@ pub(crate) fn mount_with(
|
||||
shared_generation: Arc::new(AtomicU64::new(0)),
|
||||
shared_refresh_deadlines: Arc::new(Mutex::new(SharedRefreshDeadlines::default())),
|
||||
online: Arc::new(AtomicBool::new(online)),
|
||||
link: Arc::new(Default::default()),
|
||||
link: Arc::new(link::Link::starting(start == Start::SessionExpired)),
|
||||
sync_paused_until: Arc::new(AtomicI64::new(paused_until)),
|
||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||
hidden: Arc::new(Mutex::new(HashSet::new())),
|
||||
@@ -438,6 +517,7 @@ pub(crate) fn mount_with(
|
||||
landed: Arc::new(Mutex::new(HashMap::new())),
|
||||
unadopted: Arc::new(Mutex::new(HashMap::new())),
|
||||
drain_wake: Arc::new((Mutex::new(false), Condvar::new())),
|
||||
pin_work: Arc::new(Mutex::new(Default::default())),
|
||||
shutdown: Arc::new(crate::shutdown::Shutdown::default()),
|
||||
upload_times: Arc::new(Mutex::new(HashMap::new())),
|
||||
upload_cancel: Arc::new(Mutex::new(HashMap::new())),
|
||||
@@ -472,6 +552,7 @@ pub(crate) fn mount_with(
|
||||
events: Arc::new(Default::default()),
|
||||
indexing: Arc::new(AtomicBool::new(false)),
|
||||
sync_progress: Arc::new(Mutex::new(HashMap::new())),
|
||||
sync_engine: Arc::default(),
|
||||
sync_tx,
|
||||
mounts: Arc::new(Mutex::new(SecondaryMountRegistry::default())),
|
||||
sync_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||
@@ -505,12 +586,11 @@ pub(crate) fn mount_with(
|
||||
// and sweeps all operating on a mount that no longer exists (`docs/BUGS.md` B44).
|
||||
let mut workers: Vec<std::thread::JoinHandle<()>> = Vec::new();
|
||||
for worker in 0..DRAIN_WORKERS {
|
||||
let core = core.clone();
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name(format!("pdfs-drain-{worker}"))
|
||||
.spawn(move || core.run_pending_drain(worker == 0))?,
|
||||
);
|
||||
let drain_core = core.clone();
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name(format!("pdfs-drain-{worker}"))
|
||||
.spawn(move || drain_core.run_pending_drain(worker == 0));
|
||||
keep_or_abandon(&core, control_socket, &mut workers, spawned)?;
|
||||
}
|
||||
|
||||
// Start the folder-sync engine. It watches every mirror folder, polls the
|
||||
@@ -530,39 +610,50 @@ pub(crate) fn mount_with(
|
||||
info!("conflict sweep disabled by configuration");
|
||||
} else {
|
||||
info!(mode = ?sweep_mode, "conflict sweep enabled");
|
||||
let core = core.clone();
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-conflict-sweep".into())
|
||||
.spawn(move || core.run_conflict_sweep_loop())?,
|
||||
);
|
||||
let sweep_core = core.clone();
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("pdfs-conflict-sweep".into())
|
||||
.spawn(move || sweep_core.run_conflict_sweep_loop());
|
||||
keep_or_abandon(&core, control_socket, &mut workers, spawned)?;
|
||||
}
|
||||
|
||||
// Watch the link for the life of the mount: a mount from the cache waits for
|
||||
// the network to come back, and an online one can lose it at any moment and
|
||||
// must notice it coming back just the same.
|
||||
{
|
||||
let core = core.clone();
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-online-probe".into())
|
||||
.spawn(move || core.run_online_probe())?,
|
||||
);
|
||||
let probe_core = core.clone();
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("pdfs-online-probe".into())
|
||||
.spawn(move || probe_core.run_online_probe());
|
||||
keep_or_abandon(&core, control_socket, &mut workers, spawned)?;
|
||||
}
|
||||
|
||||
// Keep pinned files on this computer through changes made elsewhere, and
|
||||
// fetch the ones a failed or missed download left out (`docs/BUGS.md`
|
||||
// B282). Its own thread: a check lists folders and downloads files.
|
||||
{
|
||||
let keeper_core = core.clone();
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("pdfs-pin-keeper".into())
|
||||
.spawn(move || keeper_core.run_pin_keeper());
|
||||
keep_or_abandon(&core, control_socket, &mut workers, spawned)?;
|
||||
}
|
||||
|
||||
// Keep the launcher prompt's "This computer" index fresh. Its own thread:
|
||||
// the walk is I/O-heavy and must never sit in front of a FUSE callback.
|
||||
if matches!(host, Host::Daemon) {
|
||||
if let Some(gallery) = &core.gallery {
|
||||
rt.spawn(gallery.clone().poll(core.clone()));
|
||||
}
|
||||
let db = core.db.clone();
|
||||
let indexing = core.indexing.clone();
|
||||
let transfers = core.transfers.clone();
|
||||
let mountpoint = mountpoint.to_path_buf();
|
||||
let shutdown = core.shutdown.clone();
|
||||
workers.push(
|
||||
std::thread::Builder::new()
|
||||
.name("pdfs-localindex".into())
|
||||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint, shutdown))?,
|
||||
);
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("pdfs-localindex".into())
|
||||
.spawn(move || run_local_index(db, indexing, transfers, mountpoint, shutdown));
|
||||
keep_or_abandon(&core, control_socket, &mut workers, spawned)?;
|
||||
}
|
||||
|
||||
// Bind the control socket before the FUSE session takes over the thread. A
|
||||
@@ -571,12 +662,18 @@ pub(crate) fn mount_with(
|
||||
let old_umask = unsafe { libc::umask(0o77) };
|
||||
let listener_res = UnixListener::bind(control_socket);
|
||||
unsafe { libc::umask(old_umask) };
|
||||
let listener = listener_res?;
|
||||
let listener = match listener_res {
|
||||
Ok(listener) => listener,
|
||||
Err(error) => {
|
||||
abandon_workers(&core, control_socket, workers);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
// Owner-only before anything can connect: a peer on this socket commands the
|
||||
// daemon's authenticated session without a credential of its own (B6).
|
||||
if let Err(e) = pdfs_core::config::restrict_socket(control_socket) {
|
||||
error!(error = %e, "cannot restrict control socket permissions; refusing to serve");
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
abandon_workers(&core, control_socket, workers);
|
||||
return Err(std::io::Error::other(format!(
|
||||
"control socket permissions: {e}"
|
||||
)));
|
||||
@@ -587,7 +684,7 @@ pub(crate) fn mount_with(
|
||||
let bg = match spawn_session(&core, mountpoint, root) {
|
||||
Ok(session) => session,
|
||||
Err(error) => {
|
||||
let _ = std::fs::remove_file(control_socket);
|
||||
abandon_workers(&core, control_socket, workers);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -607,7 +704,7 @@ pub(crate) fn mount_with(
|
||||
Err(error) => {
|
||||
// A failure here abandons the mount, so the background threads
|
||||
// started above have to be told to stop with it.
|
||||
stop_workers(&core, control_socket, workers);
|
||||
abandon_workers(&core, control_socket, workers);
|
||||
let secondaries = core.mounts.lock().close_and_drain();
|
||||
for (id, mount) in secondaries {
|
||||
if let Err(unmount_error) = mount.teardown() {
|
||||
@@ -657,8 +754,12 @@ pub(crate) fn mount_with(
|
||||
}
|
||||
}
|
||||
// The mount is live and the control socket answers: this is the moment the
|
||||
// unit is actually usable, which is what `Type=notify` reports.
|
||||
// unit is actually usable, which is what `Type=notify` reports. Whatever
|
||||
// failure an earlier attempt recorded for the front ends is over.
|
||||
systemd::ready();
|
||||
if let Some(state_dir) = control_socket.parent() {
|
||||
pdfs_core::mountfail::clear(state_dir);
|
||||
}
|
||||
|
||||
// Re-establish on-demand mounts only after the primary session is live.
|
||||
// Keep the worker so shutdown can close publication, wait for any in-flight
|
||||
@@ -719,6 +820,18 @@ pub(crate) fn mount_with(
|
||||
}
|
||||
break MountOutcome::Shutdown;
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||
if core.link.resume_stored.load(Ordering::Relaxed) =>
|
||||
{
|
||||
info!("unmounting to resume the stored session");
|
||||
core.mounts.lock().close();
|
||||
if let Err(e) = teardown_session(&core.session_live, || {
|
||||
umount_session_unblocked(bg, main_conn)
|
||||
}) {
|
||||
warn!(error = %e, "umount_and_join failed");
|
||||
}
|
||||
break MountOutcome::ResumeStored;
|
||||
}
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
|
||||
if bg.guard.is_finished() {
|
||||
info!("mount ended externally");
|
||||
@@ -812,7 +925,47 @@ fn stop_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::Jo
|
||||
// Say so before the joins: a teardown that takes a while must not be read as
|
||||
// a missed watchdog ping and turned into a kill.
|
||||
systemd::stopping();
|
||||
halt_workers(core, control_socket, workers);
|
||||
}
|
||||
|
||||
/// Stop the threads a mount attempt started before it failed.
|
||||
///
|
||||
/// The early returns used to leave them running: every retry started another
|
||||
/// set beside the last one's, and their `Core` kept the database open, so the
|
||||
/// next attempt could fail on its lock instead of the real error
|
||||
/// (`docs/BUGS.md` B264). Unlike [`stop_workers`] this does not tell systemd
|
||||
/// the daemon is stopping, because it is about to try again, and a manager
|
||||
/// told so would wait for the process to exit and then kill it.
|
||||
fn abandon_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
||||
halt_workers(core, control_socket, workers);
|
||||
}
|
||||
|
||||
/// Keep a worker the mount just spawned, or, if the spawn failed, stop the
|
||||
/// ones already running and hand the error back.
|
||||
fn keep_or_abandon(
|
||||
core: &Core,
|
||||
control_socket: &Path,
|
||||
workers: &mut Vec<std::thread::JoinHandle<()>>,
|
||||
spawned: std::io::Result<std::thread::JoinHandle<()>>,
|
||||
) -> std::io::Result<()> {
|
||||
match spawned {
|
||||
Ok(handle) => {
|
||||
workers.push(handle);
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => {
|
||||
abandon_workers(core, control_socket, std::mem::take(workers));
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Signal every worker to stop, then join them up to [`WORKER_JOIN_DEADLINE`].
|
||||
fn halt_workers(core: &Core, control_socket: &Path, workers: Vec<std::thread::JoinHandle<()>>) {
|
||||
core.shutdown.stop();
|
||||
if let Some(gallery) = &core.gallery {
|
||||
gallery.cancel();
|
||||
}
|
||||
core.wake_drain();
|
||||
let _ = core.sync_tx.send(sync::SyncMsg::Stop);
|
||||
// Connect while the socket still exists — that is the whole point of the
|
||||
|
||||
+1395
-250
File diff suppressed because it is too large.
Load diff
+207
-67
@@ -15,6 +15,7 @@ use pdfs_core::control::RestorableFolder;
|
||||
use pdfs_core::mounts::MountMode;
|
||||
use pdfs_core::profile::{
|
||||
PROFILE_DIR_NAME, PROFILE_FILE_NAME, PROFILE_VERSION, Profile, ProfileFolder, ProfilePin,
|
||||
is_profile_file_name, profile_file_name, saved_folder,
|
||||
};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
@@ -66,9 +67,14 @@ impl Core {
|
||||
});
|
||||
}
|
||||
|
||||
/// Serialize this machine's arrangement and upload it to `profile.json`
|
||||
/// inside the device's `.proton-drive-linux` folder, replacing any existing
|
||||
/// one.
|
||||
/// Serialize this machine's arrangement and upload it to its own
|
||||
/// `profile-<machine id>.json` inside the device's `.proton-drive-linux`
|
||||
/// folder, replacing any existing one.
|
||||
///
|
||||
/// Each machine writes its own file because several computers may sync
|
||||
/// into one device. They used to share `profile.json`, and each save
|
||||
/// replaced the other computers' folders with its own (B252). That file is
|
||||
/// left as it is, so a client before 3.5.0 still finds it.
|
||||
///
|
||||
/// Does nothing when no device is registered: a machine that has never
|
||||
/// synced a folder has nothing to describe, and registering a device just to
|
||||
@@ -85,7 +91,11 @@ impl Core {
|
||||
CoreError::internal(format!("bad device root uid: {}", device.root_uid))
|
||||
})?;
|
||||
|
||||
let profile = self.build_profile(&device.uid)?;
|
||||
let machine_id = pdfs_core::config::AppDirs::new()
|
||||
.and_then(|dirs| dirs.machine_id())
|
||||
.map_err(|e| CoreError::internal(format!("machine id: {e}")))?;
|
||||
let file_name = profile_file_name(&machine_id);
|
||||
let profile = self.build_profile(&device.uid, &machine_id)?;
|
||||
let bytes = profile.to_bytes().map_err(CoreError::internal)?;
|
||||
let len = bytes.len() as i64;
|
||||
|
||||
@@ -97,7 +107,7 @@ impl Core {
|
||||
// same name: the profile is a single mutable record, and a folder with
|
||||
// `profile.json` and `profile (1).json` in it teaches the restore
|
||||
// nothing about which to trust.
|
||||
let existing = self.find_device_child_file(&dir_uid, PROFILE_FILE_NAME)?;
|
||||
let existing = self.find_device_child_file(&dir_uid, &file_name)?;
|
||||
match existing {
|
||||
Some(uid) => self
|
||||
.rt
|
||||
@@ -114,7 +124,7 @@ impl Core {
|
||||
self.rt
|
||||
.block_on(self.drive.upload_file_replacing_draft_from(
|
||||
&dir_uid,
|
||||
PROFILE_FILE_NAME,
|
||||
&file_name,
|
||||
"application/json",
|
||||
Cursor::new(bytes),
|
||||
len,
|
||||
@@ -130,20 +140,12 @@ impl Core {
|
||||
}
|
||||
|
||||
/// Snapshot the local arrangement into a [`Profile`].
|
||||
fn build_profile(&self, device_uid: &str) -> CoreResult<Profile> {
|
||||
let folders = self
|
||||
.db
|
||||
.sync_folder_list()
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?
|
||||
.into_iter()
|
||||
.map(|f| ProfileFolder {
|
||||
remote_uid: f.remote_uid,
|
||||
local_path: f.local_path,
|
||||
// A queued switch is where the user asked the folder to be, so
|
||||
// that is what the next machine should set up.
|
||||
mode: MountMode::from(f.pending_mode.unwrap_or(f.mode).as_str()),
|
||||
})
|
||||
.collect();
|
||||
fn build_profile(&self, device_uid: &str, machine_id: &str) -> CoreResult<Profile> {
|
||||
let folders = profile_folders(
|
||||
self.db
|
||||
.sync_folder_list()
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?,
|
||||
);
|
||||
let pins = self
|
||||
.db
|
||||
.pin_list()
|
||||
@@ -162,6 +164,7 @@ impl Core {
|
||||
version: PROFILE_VERSION,
|
||||
device_uid: device_uid.to_string(),
|
||||
hostname: this_hostname(),
|
||||
machine_id: Some(machine_id.to_string()),
|
||||
saved_at: now_secs(),
|
||||
folders,
|
||||
pins,
|
||||
@@ -197,6 +200,31 @@ impl Core {
|
||||
.map(|n| n.uid))
|
||||
}
|
||||
|
||||
/// The untrashed files directly under `parent_uid` whose name passes
|
||||
/// `keep`.
|
||||
fn device_child_files(
|
||||
&self,
|
||||
parent_uid: &NodeUid,
|
||||
keep: impl Fn(&str) -> bool,
|
||||
) -> CoreResult<Vec<NodeUid>> {
|
||||
let uids = self
|
||||
.rt
|
||||
.block_on(self.drive.enumerate_folder_children_node_uids(parent_uid))
|
||||
.map_err(|e| CoreError::from_api(&e, "list device root"))?;
|
||||
if uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let nodes = self
|
||||
.rt
|
||||
.block_on(self.drive.enumerate_nodes(&uids))
|
||||
.map_err(|e| CoreError::from_api(&e, "resolve device root children"))?;
|
||||
Ok(nodes
|
||||
.into_iter()
|
||||
.filter(|n| !n.is_folder() && !n.trashed && keep(&n.name))
|
||||
.map(|n| n.uid)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The device's `.proton-drive-linux` folder, creating it if this is the
|
||||
/// first save on this device.
|
||||
///
|
||||
@@ -218,32 +246,43 @@ impl Core {
|
||||
Ok(uid)
|
||||
}
|
||||
|
||||
/// Download and parse this device's profile, if there is one.
|
||||
/// Download and parse the profiles of every computer that syncs this
|
||||
/// device.
|
||||
///
|
||||
/// Looks in `.proton-drive-linux/` first and falls back to the device root,
|
||||
/// where clients before the B78 fix wrote it. The fallback is read-only: the
|
||||
/// next save writes to the folder, and the legacy document is left alone
|
||||
/// rather than trashed, so an older client on another machine keeps working.
|
||||
fn load_profile(&self, root_uid: &NodeUid) -> CoreResult<Option<Profile>> {
|
||||
let current = match self.find_device_child_folder(root_uid, PROFILE_DIR_NAME)? {
|
||||
Some(dir_uid) => self.find_device_child_file(&dir_uid, PROFILE_FILE_NAME)?,
|
||||
None => None,
|
||||
/// Reads every profile in `.proton-drive-linux/`: one per machine, and the
|
||||
/// shared `profile.json` of clients before 3.5.0. Falls back to the device
|
||||
/// root, where clients before the B78 fix wrote it. The fallback is
|
||||
/// read-only: the next save writes to the folder, and the legacy document
|
||||
/// is left alone rather than trashed, so an older client on another
|
||||
/// machine keeps working. A profile that cannot be read is left out and
|
||||
/// logged; the others still count.
|
||||
fn load_profiles(&self, root_uid: &NodeUid) -> CoreResult<Vec<Profile>> {
|
||||
let mut uids = match self.find_device_child_folder(root_uid, PROFILE_DIR_NAME)? {
|
||||
Some(dir_uid) => self.device_child_files(&dir_uid, is_profile_file_name)?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
let uid = match current {
|
||||
Some(uid) => uid,
|
||||
None => match self.find_device_child_file(root_uid, PROFILE_FILE_NAME)? {
|
||||
Some(uid) => {
|
||||
info!("reading legacy profile from the device root");
|
||||
uid
|
||||
if uids.is_empty() {
|
||||
uids = self.device_child_files(root_uid, |name| name == PROFILE_FILE_NAME)?;
|
||||
if !uids.is_empty() {
|
||||
info!("reading legacy profile from the device root");
|
||||
}
|
||||
}
|
||||
let mut profiles = Vec::new();
|
||||
for uid in uids {
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let parsed = self
|
||||
.rt
|
||||
.block_on(self.drive.download_file_to(&uid, &mut buf))
|
||||
.map_err(|e| CoreError::from_api(&e, "download profile"))
|
||||
.and_then(|()| Profile::parse(&buf).map_err(CoreError::invalid));
|
||||
match parsed {
|
||||
Ok(profile) => profiles.push(profile),
|
||||
Err(e) => {
|
||||
warn!(%uid, error = %e.message, "could not read a profile; leaving it out")
|
||||
}
|
||||
None => return Ok(None),
|
||||
},
|
||||
};
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
self.rt
|
||||
.block_on(self.drive.download_file_to(&uid, &mut buf))
|
||||
.map_err(|e| CoreError::from_api(&e, "download profile"))?;
|
||||
Profile::parse(&buf).map(Some).map_err(CoreError::invalid)
|
||||
}
|
||||
}
|
||||
Ok(profiles)
|
||||
}
|
||||
|
||||
// ---- restore ----------------------------------------
|
||||
@@ -265,15 +304,16 @@ impl Core {
|
||||
CoreError::internal(format!("bad device root uid: {}", device.root_uid))
|
||||
})?;
|
||||
|
||||
let profile = match self.load_profile(&root_uid) {
|
||||
let profiles = match self.load_profiles(&root_uid) {
|
||||
Ok(p) => p,
|
||||
// A missing or unreadable profile is not fatal: the remote tree
|
||||
// alone is enough to offer a restore, just with plainer defaults.
|
||||
Err(e) => {
|
||||
warn!(error = %e.message, "could not read profile; proposing defaults");
|
||||
None
|
||||
warn!(error = %e.message, "could not read profiles; proposing defaults");
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let hostname = this_hostname();
|
||||
|
||||
let uids = self
|
||||
.rt
|
||||
@@ -304,11 +344,12 @@ impl Core {
|
||||
// offering it as a restorable folder would sync the bookkeeping
|
||||
// back onto the machine it describes.
|
||||
.filter(|n| n.is_folder() && !n.trashed && n.name != PROFILE_DIR_NAME)
|
||||
// The name becomes a folder under the home folder: one with a `/`
|
||||
// or a `..` in it would put the restore somewhere else.
|
||||
.filter(|n| crate::remote_name_is_safe(&n.name))
|
||||
.map(|n| {
|
||||
let uid = n.uid.to_string();
|
||||
let saved = profile
|
||||
.as_ref()
|
||||
.and_then(|p| p.folders.iter().find(|f| f.remote_uid == uid));
|
||||
let saved = saved_folder(&profiles, &uid, &hostname);
|
||||
let proposed = saved
|
||||
.map(|f| PathBuf::from(&f.local_path))
|
||||
.filter(|p| path_is_usable_here(p))
|
||||
@@ -339,7 +380,8 @@ impl Core {
|
||||
&self,
|
||||
items: &[pdfs_core::control::RestoreItem],
|
||||
device: Option<&str>,
|
||||
) -> CoreResult<String> {
|
||||
) -> CoreResult<RestoreOutcome> {
|
||||
let _registration = super::devices::SYNC_ROOT_REGISTRATION.lock();
|
||||
if items.is_empty() {
|
||||
return Err(CoreError::invalid("nothing to restore"));
|
||||
}
|
||||
@@ -349,23 +391,46 @@ impl Core {
|
||||
.sync_folder_list()
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
|
||||
// A caller must not smuggle a My files root into a device mapping,
|
||||
// where destructive device-only operations would otherwise be allowed.
|
||||
let device_root = super::parse_uid(&device.root_uid)
|
||||
.ok_or_else(|| CoreError::internal("invalid device root"))?;
|
||||
let children = self.device_root_nodes(&device_root)?;
|
||||
let reserved = crate::devices::sync_root_reserved();
|
||||
let mut roots: Vec<PathBuf> = existing
|
||||
.iter()
|
||||
.map(|f| PathBuf::from(&f.local_path))
|
||||
.collect();
|
||||
let mut restored = 0usize;
|
||||
let mut skipped: Vec<String> = Vec::new();
|
||||
for item in items {
|
||||
let local = PathBuf::from(&item.local_path);
|
||||
if !local.is_absolute() {
|
||||
skipped.push(format!("{}: local path must be absolute", item.local_path));
|
||||
if !children.iter().any(|node| {
|
||||
node.uid.to_string() == item.remote_uid
|
||||
&& node.is_folder()
|
||||
&& !node.trashed
|
||||
&& node.name != PROFILE_DIR_NAME
|
||||
}) {
|
||||
skipped.push(format!(
|
||||
"{}: not a folder under this device",
|
||||
item.local_path
|
||||
));
|
||||
continue;
|
||||
}
|
||||
// The path is proposed from the profile another machine wrote, and
|
||||
// may hold a folder name from Drive: checked like any new root.
|
||||
let local = match pdfs_core::syncroot::resolve(Path::new(&item.local_path)) {
|
||||
Ok(local) => local,
|
||||
Err(e) => {
|
||||
skipped.push(format!("{}: {e}", item.local_path));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if existing.iter().any(|f| f.remote_uid == item.remote_uid) {
|
||||
skipped.push(format!("{}: already synced", item.local_path));
|
||||
continue;
|
||||
}
|
||||
if existing
|
||||
.iter()
|
||||
.any(|f| Path::new(&f.local_path) == local.as_path())
|
||||
{
|
||||
skipped.push(format!("{}: path already in use", item.local_path));
|
||||
if let Some(problem) = pdfs_core::syncroot::problem(&local, &roots, &reserved) {
|
||||
skipped.push(format!("{}: {problem}", item.local_path));
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = std::fs::create_dir_all(&local) {
|
||||
@@ -379,10 +444,19 @@ impl Core {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let id = self
|
||||
.db
|
||||
.sync_folder_add(&local.to_string_lossy(), &item.remote_uid, &device.share_id)
|
||||
.map_err(|e| CoreError::internal(format!("db: {e:?}")))?;
|
||||
let id = match self.db.sync_folder_add(
|
||||
&local.to_string_lossy(),
|
||||
&item.remote_uid,
|
||||
&device.share_id,
|
||||
) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
// The folders restored before this one still need their
|
||||
// watches.
|
||||
let _ = self.sync_tx.send(sync::SyncMsg::Rewatch);
|
||||
return Err(CoreError::internal(format!("db: {e:?}")));
|
||||
}
|
||||
};
|
||||
// `ondemand` is applied through the normal request path, which
|
||||
// mounts the FUSE session and handles the folder being busy — the
|
||||
// restore does not get its own copy of that logic.
|
||||
@@ -390,16 +464,57 @@ impl Core {
|
||||
let _ = self.request_sync_folder_mode(id, "ondemand");
|
||||
}
|
||||
let _ = self.sync_tx.send(sync::SyncMsg::Reconcile(id));
|
||||
roots.push(local);
|
||||
restored += 1;
|
||||
}
|
||||
let _ = self.sync_tx.send(sync::SyncMsg::Rewatch);
|
||||
|
||||
let mut message = format!("restored {restored} folder(s)");
|
||||
if !skipped.is_empty() {
|
||||
message.push_str(&format!("; skipped: {}", skipped.join(", ")));
|
||||
// The reply is gone once the app has shown it; the log keeps why.
|
||||
for reason in &skipped {
|
||||
info!(reason, "restore skipped a folder");
|
||||
}
|
||||
info!(restored, skipped = skipped.len(), "restore complete");
|
||||
Ok(message)
|
||||
Ok(RestoreOutcome { restored, skipped })
|
||||
}
|
||||
}
|
||||
|
||||
/// Only device backups belong in the device's cross-machine profile.
|
||||
fn profile_folders(folders: Vec<pdfs_core::db::StoredSyncFolder>) -> Vec<ProfileFolder> {
|
||||
folders
|
||||
.into_iter()
|
||||
.filter(|f| f.kind == "device")
|
||||
.map(|f| ProfileFolder {
|
||||
remote_uid: f.remote_uid,
|
||||
local_path: f.local_path,
|
||||
mode: MountMode::from(f.pending_mode.unwrap_or(f.mode).as_str()),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// What a restore did: how many folders it attached, and why it left out the
|
||||
/// others.
|
||||
pub(crate) struct RestoreOutcome {
|
||||
pub(crate) restored: usize,
|
||||
pub(crate) skipped: Vec<String>,
|
||||
}
|
||||
|
||||
impl RestoreOutcome {
|
||||
/// The activity entry's headline.
|
||||
pub(crate) fn summary(&self) -> String {
|
||||
match (self.restored, self.skipped.len()) {
|
||||
(n, 0) => format!("restored {n} folder(s)"),
|
||||
(0, k) => format!("restored no folders; skipped {k}"),
|
||||
(n, k) => format!("restored {n} folder(s); skipped {k}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the restore did all it was asked. A skipped folder is a
|
||||
/// failure the user has to see: the reply went out as "restoring folders"
|
||||
/// before any work, and a restore that skipped every folder used to be
|
||||
/// logged as a success, so nothing anywhere said that nothing would
|
||||
/// download (`docs/BUGS.md` B269).
|
||||
pub(crate) fn complete(&self) -> bool {
|
||||
self.skipped.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -438,6 +553,21 @@ fn default_local_path(name: &str) -> PathBuf {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attached_my_files_folders_are_not_in_the_device_profile() {
|
||||
let dir = crate::sim::daemon::scratch("profile-folder-kinds");
|
||||
let db = pdfs_core::db::Db::open(&dir.join("cache.db")).unwrap();
|
||||
db.sync_folder_add("/backup", "vol~backup", "device-share")
|
||||
.unwrap();
|
||||
db.sync_folder_attach("/work", "vol~work", "my-files-share")
|
||||
.unwrap();
|
||||
let folders = profile_folders(db.sync_folder_list().unwrap());
|
||||
assert_eq!(folders.len(), 1);
|
||||
assert_eq!(folders[0].remote_uid, "vol~backup");
|
||||
drop(db);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreign_home_is_not_reused() {
|
||||
assert!(!path_is_usable_here(Path::new(
|
||||
@@ -482,7 +612,7 @@ mod tests {
|
||||
#[test]
|
||||
fn loading_falls_back_to_a_legacy_device_root_profile() {
|
||||
let source = include_str!("profile.rs");
|
||||
let load = function_source(source, "fn load_profile(");
|
||||
let load = function_source(source, "fn load_profiles(");
|
||||
assert!(load.contains("PROFILE_DIR_NAME"), "prefers the folder");
|
||||
assert!(
|
||||
load.find("PROFILE_DIR_NAME").unwrap()
|
||||
@@ -491,6 +621,16 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Several computers may share a device, so each saves to a file named
|
||||
/// after its own machine id and never to the shared `profile.json`
|
||||
/// (B252).
|
||||
#[test]
|
||||
fn each_machine_saves_its_own_profile() {
|
||||
let save = function_source(include_str!("profile.rs"), "fn save_profile(");
|
||||
assert!(save.contains("profile_file_name(&machine_id)"));
|
||||
assert!(!save.contains("PROFILE_FILE_NAME"));
|
||||
}
|
||||
|
||||
/// The profile folder is the client's own bookkeeping: neither the restore
|
||||
/// picker nor `add_sync_folder` may treat it as a user folder.
|
||||
#[test]
|
||||
|
||||
+206
-28
@@ -1070,37 +1070,121 @@ impl Core {
|
||||
offset: u64,
|
||||
size: u64,
|
||||
) -> Result<Vec<u8>, Errno> {
|
||||
if offset >= len || size == 0 {
|
||||
return Ok(Vec::new());
|
||||
let read = assemble_open_read(file, len, uid, base_size, written, offset, size, |s, n| {
|
||||
self.read_range(uid, base_mtime, base_size, s, n, true)
|
||||
.map(Some)
|
||||
})?;
|
||||
Ok(read.unwrap_or_default())
|
||||
}
|
||||
|
||||
/// [`Core::serve_open_read`] from bytes already on this machine only:
|
||||
/// `Ok(None)` the moment a range of the base would have to be downloaded.
|
||||
///
|
||||
/// The dispatch loop answers a read on a write handle with this, and hands
|
||||
/// it to a worker only when it says `None`. Fetching the base there held
|
||||
/// every request of the mount for as long as the download took, up to the
|
||||
/// read deadline (`docs/BUGS.md` B227).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn serve_open_read_local(
|
||||
&self,
|
||||
file: &Arc<File>,
|
||||
len: u64,
|
||||
uid: &NodeUid,
|
||||
base_mtime: i64,
|
||||
base_size: u64,
|
||||
written: &Intervals,
|
||||
offset: u64,
|
||||
size: u64,
|
||||
) -> Result<Option<Vec<u8>>, Errno> {
|
||||
assemble_open_read(file, len, uid, base_size, written, offset, size, |s, n| {
|
||||
Ok(self.cached_range(uid, base_mtime, base_size, s, n))
|
||||
})
|
||||
}
|
||||
|
||||
/// Bytes `[offset, offset + size)` of file `ino` as `read` would serve
|
||||
/// them, but only from this machine: an open write handle, a queued write,
|
||||
/// or the caches. `Ok(None)` the moment any of it would have to be
|
||||
/// downloaded, and for a size still provisional (B100), which only the
|
||||
/// network settles. Short or empty past the end of the file.
|
||||
///
|
||||
/// `copy_file_range` answers with this and leaves anything else to the
|
||||
/// kernel's read/write copy.
|
||||
pub(super) fn read_local(
|
||||
&self,
|
||||
ino: u64,
|
||||
offset: u64,
|
||||
size: u64,
|
||||
) -> Result<Option<Vec<u8>>, Errno> {
|
||||
let handle = self.state().active_writes.get(&ino).map(|h| {
|
||||
(
|
||||
h.file.clone(),
|
||||
h.len,
|
||||
h.uid.clone(),
|
||||
h.base_mtime,
|
||||
h.base_size,
|
||||
h.written.clone(),
|
||||
)
|
||||
});
|
||||
if let Some((file, len, uid, base_mtime, base_size, written)) = handle {
|
||||
return self.serve_open_read_local(
|
||||
&file, len, &uid, base_mtime, base_size, &written, offset, size,
|
||||
);
|
||||
}
|
||||
let end = offset.saturating_add(size).min(len);
|
||||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||||
for (s, e, authored) in written.segments(offset, end) {
|
||||
if authored {
|
||||
let mut buf = vec![0u8; (e - s) as usize];
|
||||
file.read_exact_at(&mut buf, s).map_err(|err| {
|
||||
warn!(%uid, error = %err, "scratch read failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
out.extend_from_slice(&buf);
|
||||
} else {
|
||||
let bend = e.min(base_size);
|
||||
if s < bend {
|
||||
out.extend_from_slice(&self.read_range(
|
||||
uid,
|
||||
base_mtime,
|
||||
base_size,
|
||||
s,
|
||||
bend - s,
|
||||
true,
|
||||
)?);
|
||||
}
|
||||
// Anything past the base is a hole: zero-fill.
|
||||
let zeros = e.saturating_sub(s.max(base_size));
|
||||
out.resize(out.len() + zeros as usize, 0);
|
||||
let (uid, mtime, fsize, provisional) = {
|
||||
let st = self.state();
|
||||
match st.entries.get(&ino) {
|
||||
Some(e) if e.node.is_file() => (
|
||||
e.uid.clone(),
|
||||
e.node.modification_time,
|
||||
node_size(&e.node),
|
||||
matches!(
|
||||
&e.node.kind,
|
||||
NodeKind::File {
|
||||
claimed_size: None,
|
||||
..
|
||||
}
|
||||
),
|
||||
),
|
||||
Some(_) => return Err(Errno::EISDIR),
|
||||
None => return Err(Errno::ENOENT),
|
||||
}
|
||||
};
|
||||
let uid = match is_local_uid(&uid) {
|
||||
true => self.landed_stand_in_uid(&uid).unwrap_or(uid),
|
||||
false => uid,
|
||||
};
|
||||
if let Some(pending) = self.pending.lock().get(&uid).cloned() {
|
||||
let m = &pending.meta;
|
||||
let file = File::open(&pending.path).map_err(|e| {
|
||||
error!(%uid, path = %pending.path.display(), error = %e, "open staged blob failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
let mut written = Intervals::default();
|
||||
for &(s, e) in &m.authored {
|
||||
written.add(s, e);
|
||||
}
|
||||
return self.serve_open_read_local(
|
||||
&Arc::new(file),
|
||||
m.len,
|
||||
&uid,
|
||||
m.base_mtime,
|
||||
m.base_size,
|
||||
&written,
|
||||
offset,
|
||||
size,
|
||||
);
|
||||
}
|
||||
Ok(out)
|
||||
// Created here and never written: an empty file.
|
||||
if is_local_uid(&uid) {
|
||||
return Ok(Some(Vec::new()));
|
||||
}
|
||||
if provisional {
|
||||
return Ok(None);
|
||||
}
|
||||
if offset >= fsize || size == 0 {
|
||||
return Ok(Some(Vec::new()));
|
||||
}
|
||||
Ok(self.cached_range(&uid, mtime, fsize, offset, size.min(fsize - offset)))
|
||||
}
|
||||
|
||||
/// The largest amount [`Core::fill_gaps_cached`] will copy out of the local
|
||||
@@ -1242,6 +1326,49 @@ impl Core {
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes `[offset, offset + size)` of a file open for writing: authored
|
||||
/// ranges from its scratch `file`, the rest from its base through `base`,
|
||||
/// and zeros past the base. `Ok(None)` when `base` says it cannot answer.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn assemble_open_read(
|
||||
file: &File,
|
||||
len: u64,
|
||||
uid: &NodeUid,
|
||||
base_size: u64,
|
||||
written: &Intervals,
|
||||
offset: u64,
|
||||
size: u64,
|
||||
mut base: impl FnMut(u64, u64) -> Result<Option<Vec<u8>>, Errno>,
|
||||
) -> Result<Option<Vec<u8>>, Errno> {
|
||||
if offset >= len || size == 0 {
|
||||
return Ok(Some(Vec::new()));
|
||||
}
|
||||
let end = offset.saturating_add(size).min(len);
|
||||
let mut out = Vec::with_capacity((end - offset) as usize);
|
||||
for (s, e, authored) in written.segments(offset, end) {
|
||||
if authored {
|
||||
let mut buf = vec![0u8; (e - s) as usize];
|
||||
file.read_exact_at(&mut buf, s).map_err(|err| {
|
||||
warn!(%uid, error = %err, "scratch read failed");
|
||||
Errno::EIO
|
||||
})?;
|
||||
out.extend_from_slice(&buf);
|
||||
} else {
|
||||
let bend = e.min(base_size);
|
||||
if s < bend {
|
||||
match base(s, bend - s)? {
|
||||
Some(bytes) => out.extend_from_slice(&bytes),
|
||||
None => return Ok(None),
|
||||
}
|
||||
}
|
||||
// Anything past the base is a hole: zero-fill.
|
||||
let zeros = e.saturating_sub(s.max(base_size));
|
||||
out.resize(out.len() + zeros as usize, 0);
|
||||
}
|
||||
}
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod block_ring_tests {
|
||||
use super::*;
|
||||
@@ -1394,3 +1521,54 @@ mod prefetch_tests {
|
||||
assert!(prefetch.streams.lock().len() <= MAX_PREFETCH_STREAMS + 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod open_read_tests {
|
||||
use super::*;
|
||||
use proton_drive_rs::proton_sdk::ids::{LinkId, VolumeId};
|
||||
|
||||
fn scratch(bytes: &[u8]) -> (File, PathBuf) {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"pdfs-open-read-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let mut file = File::options()
|
||||
.create_new(true)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&path)
|
||||
.unwrap();
|
||||
file.write_all(bytes).unwrap();
|
||||
(file, path)
|
||||
}
|
||||
|
||||
/// The dispatch loop answers from what is on this machine and says so
|
||||
/// when the base would have to be downloaded, for a worker to do it
|
||||
/// (`docs/BUGS.md` B227).
|
||||
#[test]
|
||||
fn a_read_needing_the_base_says_so_instead_of_fetching() {
|
||||
let uid = NodeUid::new(VolumeId::from("v"), LinkId::from("f"));
|
||||
let (file, path) = scratch(b"ABCDEFGH");
|
||||
let mut written = Intervals::default();
|
||||
written.add(0, 4);
|
||||
|
||||
let authored = assemble_open_read(&file, 8, &uid, 8, &written, 0, 4, |_, _| {
|
||||
panic!("an authored range never asks the base")
|
||||
});
|
||||
assert_eq!(authored.unwrap().as_deref(), Some(&b"ABCD"[..]));
|
||||
|
||||
let missing = assemble_open_read(&file, 8, &uid, 8, &written, 2, 6, |_, _| Ok(None));
|
||||
assert_eq!(missing.unwrap(), None);
|
||||
|
||||
let base = assemble_open_read(&file, 10, &uid, 8, &written, 2, 8, |s, n| {
|
||||
assert_eq!((s, n), (4, 4));
|
||||
Ok(Some(b"efgh".to_vec()))
|
||||
});
|
||||
assert_eq!(base.unwrap().as_deref(), Some(&b"CDefgh\0\0"[..]));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
@@ -190,6 +190,7 @@ impl Core {
|
||||
.map_err(|e| CoreError::internal(format!("stat {}: {e}", path.display())))?
|
||||
.len();
|
||||
let media_type = media_type_of(&candidate.name).unwrap_or("application/octet-stream");
|
||||
let thumbnails = self.upload_thumbnails(&path, &candidate.name);
|
||||
|
||||
let guard =
|
||||
self.transfers
|
||||
@@ -199,6 +200,7 @@ impl Core {
|
||||
let reader = CountingReader::new(std::io::BufReader::new(file), &guard);
|
||||
let metadata = PhotoUploadMetadata {
|
||||
capture_time: Some(candidate.wanted),
|
||||
last_modification_time: pdfs_core::timestamp::modification_time(candidate.wanted),
|
||||
..Default::default()
|
||||
};
|
||||
let uploaded = self.rt.block_on(photos.upload_photo_from(
|
||||
@@ -206,7 +208,7 @@ impl Core {
|
||||
media_type,
|
||||
reader,
|
||||
size as i64,
|
||||
Vec::new(),
|
||||
thumbnails,
|
||||
metadata,
|
||||
false,
|
||||
));
|
||||
@@ -291,7 +293,14 @@ impl Core {
|
||||
let mut state = REDATE.lock();
|
||||
state.running = false;
|
||||
state.cancel = None;
|
||||
state.last = outcome.as_ref().ok().cloned();
|
||||
state.last = Some(match &outcome {
|
||||
Ok(summary) => summary.clone(),
|
||||
Err(e) => RedateSummary {
|
||||
error: Some(e.to_string()),
|
||||
dry_run,
|
||||
..Default::default()
|
||||
},
|
||||
});
|
||||
match &outcome {
|
||||
Ok(summary) => info!(
|
||||
candidates = summary.candidates,
|
||||
|
||||
@@ -25,6 +25,7 @@ use proton_drive_rs::NodeKind;
|
||||
use proton_drive_rs::proton_sdk::ids::NodeUid;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::state::State;
|
||||
use super::sync::mirror_subtree_unsynced;
|
||||
use super::{Core, is_local_uid, parse_uid};
|
||||
|
||||
@@ -130,6 +131,24 @@ fn rename_noreplace(from: &Path, to: &Path) -> std::io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether inode `ino` is `root` or lies under it in the tree. Bounded, so a
|
||||
/// parent cycle in corrupt data cannot hang the caller.
|
||||
pub(crate) fn ino_is_under(st: &State, mut ino: u64, root: u64) -> bool {
|
||||
for _ in 0..MAX_TREE_DEPTH {
|
||||
if ino == root {
|
||||
return true;
|
||||
}
|
||||
match st.entries.get(&ino) {
|
||||
Some(entry) if entry.parent != ino => ino = entry.parent,
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// How deep [`ino_is_under`] walks before it gives up.
|
||||
const MAX_TREE_DEPTH: usize = 1024;
|
||||
|
||||
/// Whether `uid` is `root` or lies under it, as far as the nodes table knows.
|
||||
fn under(core: &Core, root: &NodeUid, uid: &str) -> bool {
|
||||
let root = root.to_string();
|
||||
@@ -336,7 +355,7 @@ impl Core {
|
||||
core.require_uid_writable(authority)
|
||||
.map_err(|e| core.errno_error(e, "move access"))?;
|
||||
}
|
||||
if self.has_pending_work_under(&uid)? {
|
||||
if self.has_pending_work_under(ino, &uid)? {
|
||||
return Err(CoreError::conflict(
|
||||
"it has changes that are not uploaded yet; wait for the upload to finish",
|
||||
));
|
||||
@@ -410,10 +429,11 @@ impl Core {
|
||||
Ok(uid)
|
||||
}
|
||||
|
||||
/// Whether the queue owes Drive anything for `uid` or a node under it, or a
|
||||
/// file under it is open for writing. Moving such a node away would strand
|
||||
/// the pending change against a path that no longer leads to it.
|
||||
fn has_pending_work_under(&self, uid: &NodeUid) -> CoreResult<bool> {
|
||||
/// Whether the queue owes Drive anything for `uid` (inode `ino`) or a node
|
||||
/// under it, or a file under it is open for writing. Moving such a node away
|
||||
/// would strand the pending change against a path that no longer leads to
|
||||
/// it.
|
||||
fn has_pending_work_under(&self, ino: u64, uid: &NodeUid) -> CoreResult<bool> {
|
||||
let ops = self
|
||||
.db
|
||||
.pending_ops()
|
||||
@@ -427,12 +447,19 @@ impl Core {
|
||||
}) {
|
||||
return Ok(true);
|
||||
}
|
||||
// Collected first so no State lock is held across the nodes-table reads.
|
||||
let mut writing = Vec::new();
|
||||
// Open handles are placed in the tree by inode, not by uid in the nodes
|
||||
// table. A file opened just after its create landed holds the
|
||||
// placeholder uid until the tree catches up, and the nodes table no
|
||||
// longer has that uid, so the move went ahead under an open write.
|
||||
let mut writing = false;
|
||||
self.for_each_state(|st| {
|
||||
writing.extend(st.active_writes.values().map(|w| w.uid.to_string()));
|
||||
writing = st
|
||||
.active_writes
|
||||
.keys()
|
||||
.chain(&st.releasing)
|
||||
.any(|&open| ino_is_under(st, open, ino));
|
||||
});
|
||||
Ok(writing.iter().any(|w| under(self, uid, w)))
|
||||
Ok(writing)
|
||||
}
|
||||
|
||||
/// Bring the source location level with a move Drive has made.
|
||||
@@ -528,6 +555,7 @@ mod tests {
|
||||
|
||||
fn folder(id: i64, local_path: &str, mode: &str) -> StoredSyncFolder {
|
||||
StoredSyncFolder {
|
||||
kind: "device".into(),
|
||||
id,
|
||||
local_path: local_path.to_string(),
|
||||
remote_uid: format!("vol~root{id}"),
|
||||
|
||||
@@ -111,6 +111,11 @@ impl Core {
|
||||
// stale from here on, and the server may not have applied the swap yet.
|
||||
self.cache.evict(uid);
|
||||
self.evict_reader(uid);
|
||||
// A file kept offline gets the restored content back on disk, and
|
||||
// again when the event of the swap arrives (`docs/BUGS.md` B282).
|
||||
if self.cache.is_pinned(uid) {
|
||||
self.recheck_pinned(uid);
|
||||
}
|
||||
self.log_activity(
|
||||
ActivityKind::Restore,
|
||||
name,
|
||||
|
||||
+113
-20
@@ -9,7 +9,8 @@
|
||||
use std::path::Path;
|
||||
|
||||
use pdfs_core::control::{
|
||||
BookmarkInfo, DirEntry, InvitationInfo, PublicLinkInfo, ShareEntry, ShareEntryKind, SharedItem,
|
||||
BookmarkInfo, DirEntry, InvitationInfo, LinkChange, PublicLinkInfo, ShareEntry, ShareEntryKind,
|
||||
SharedItem,
|
||||
};
|
||||
use pdfs_core::{CoreError, CoreResult};
|
||||
use tracing::debug;
|
||||
@@ -66,6 +67,7 @@ fn shared_entry_of(core: &Core, n: Node) -> DirEntry {
|
||||
size,
|
||||
modified: n.modification_time,
|
||||
pinned: false,
|
||||
pinned_by_folder: false,
|
||||
cached: false,
|
||||
path: core.rel_path_for_uid(&n.uid).unwrap_or_default(),
|
||||
uid: n.uid.to_string(),
|
||||
@@ -103,7 +105,7 @@ impl Core {
|
||||
role: &str,
|
||||
message: Option<&str>,
|
||||
) -> CoreResult<(usize, usize)> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.share_node_for_uid(&uid, emails, role, message)
|
||||
}
|
||||
|
||||
@@ -136,7 +138,7 @@ impl Core {
|
||||
&self,
|
||||
uid: &str,
|
||||
) -> CoreResult<(Vec<ShareEntry>, Option<PublicLinkInfo>)> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.list_share_for_uid(&uid)
|
||||
}
|
||||
|
||||
@@ -213,7 +215,7 @@ impl Core {
|
||||
kind: ShareEntryKind,
|
||||
role: &str,
|
||||
) -> CoreResult<()> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.update_share_role_for_uid(&uid, id, kind, role)
|
||||
}
|
||||
|
||||
@@ -276,7 +278,7 @@ impl Core {
|
||||
id: &str,
|
||||
kind: ShareEntryKind,
|
||||
) -> CoreResult<()> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.remove_share_entry_for_uid(&uid, id, kind)
|
||||
}
|
||||
|
||||
@@ -347,7 +349,7 @@ impl Core {
|
||||
password: Option<&str>,
|
||||
expires: Option<i64>,
|
||||
) -> CoreResult<PublicLinkInfo> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.create_public_link_for_uid(&uid, role, password, expires)
|
||||
}
|
||||
|
||||
@@ -366,6 +368,89 @@ impl Core {
|
||||
Ok(public_link_info(link))
|
||||
}
|
||||
|
||||
/// Change the public link `id` on the node at `rel`, keeping its URL.
|
||||
pub(crate) fn update_public_link(
|
||||
&self,
|
||||
rel: &Path,
|
||||
id: &str,
|
||||
role: Option<&str>,
|
||||
password: LinkChange<String>,
|
||||
expires: LinkChange<i64>,
|
||||
) -> CoreResult<PublicLinkInfo> {
|
||||
let (_ino, uid) = self.resolve(rel)?;
|
||||
self.update_public_link_for_uid(&uid, id, role, password, expires)
|
||||
}
|
||||
|
||||
/// [`Core::update_public_link`] for a node addressed through any daemon
|
||||
/// location.
|
||||
pub(crate) fn update_public_link_by_uid(
|
||||
&self,
|
||||
uid: &str,
|
||||
id: &str,
|
||||
role: Option<&str>,
|
||||
password: LinkChange<String>,
|
||||
expires: LinkChange<i64>,
|
||||
) -> CoreResult<PublicLinkInfo> {
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.update_public_link_for_uid(&uid, id, role, password, expires)
|
||||
}
|
||||
|
||||
/// The SDK sets all three options on every update, so each `Keep` is
|
||||
/// filled in from the link as it stands.
|
||||
fn update_public_link_for_uid(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
id: &str,
|
||||
role: Option<&str>,
|
||||
password: LinkChange<String>,
|
||||
expires: LinkChange<i64>,
|
||||
) -> CoreResult<PublicLinkInfo> {
|
||||
let link = self
|
||||
.rt
|
||||
.block_on(self.client.get_public_link(uid))
|
||||
.map_err(|e| CoreError::from_api(&e, "get public link"))?
|
||||
.filter(|l| l.public_link_id == id)
|
||||
.ok_or_else(|| CoreError::not_found("public link not found"))?;
|
||||
let role = match role {
|
||||
Some(role) => role_from_str(role)?,
|
||||
None => link.role,
|
||||
};
|
||||
let unchanged = role == link.role
|
||||
&& matches!(password, LinkChange::Keep)
|
||||
&& matches!(expires, LinkChange::Keep);
|
||||
if unchanged {
|
||||
return Ok(public_link_info(link));
|
||||
}
|
||||
let password = match password {
|
||||
LinkChange::Keep if link.has_custom_password => {
|
||||
// Sending `None` here would quietly remove the password.
|
||||
Some(link.custom_password.clone().ok_or_else(|| {
|
||||
CoreError::conflict(
|
||||
"the link's password could not be read; set a new one or remove it",
|
||||
)
|
||||
})?)
|
||||
}
|
||||
LinkChange::Keep | LinkChange::Remove => None,
|
||||
LinkChange::Set(password) if password.is_empty() => {
|
||||
return Err(CoreError::invalid("the new password is empty"));
|
||||
}
|
||||
LinkChange::Set(password) => Some(password),
|
||||
};
|
||||
let expires = match expires {
|
||||
LinkChange::Keep => link.expiration_time,
|
||||
LinkChange::Remove => None,
|
||||
LinkChange::Set(at) => Some(at),
|
||||
};
|
||||
let link = self
|
||||
.rt
|
||||
.block_on(
|
||||
self.client
|
||||
.update_public_link(uid, role, password.as_deref(), expires),
|
||||
)
|
||||
.map_err(|e| CoreError::from_api(&e, "update public link"))?;
|
||||
Ok(public_link_info(link))
|
||||
}
|
||||
|
||||
/// Remove the public link `id` from the node at `rel`.
|
||||
pub(crate) fn remove_public_link(&self, rel: &Path, id: &str) -> CoreResult<()> {
|
||||
let (_ino, uid) = self.resolve(rel)?;
|
||||
@@ -375,7 +460,7 @@ impl Core {
|
||||
/// [`Core::remove_public_link`] for a node addressed through any daemon
|
||||
/// location.
|
||||
pub(crate) fn remove_public_link_by_uid(&self, uid: &str, id: &str) -> CoreResult<()> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.remove_public_link_for_uid(&uid, id)
|
||||
}
|
||||
|
||||
@@ -401,7 +486,7 @@ impl Core {
|
||||
|
||||
/// [`Core::stop_sharing`] for a node addressed through any daemon location.
|
||||
pub(crate) fn stop_sharing_by_uid(&self, uid: &str) -> CoreResult<StopSharingOutcome> {
|
||||
let uid = self.resolve_anywhere(uid)?;
|
||||
let uid = self.resolve_shareable(uid)?;
|
||||
self.stop_sharing_for_uid(&uid)
|
||||
}
|
||||
|
||||
@@ -660,6 +745,12 @@ impl Core {
|
||||
/// the shared uids; the per-node detail is then gathered best-effort — a single
|
||||
/// node racing with an unshare drops from the list rather than failing the whole
|
||||
/// request.
|
||||
///
|
||||
/// A share with no member, no invitation and no link is left out: it is
|
||||
/// shared with no one. Stop Sharing leaves exactly that behind, since the SDK
|
||||
/// cannot delete a share, and Drive keeps listing it until its backend gets
|
||||
/// round to removing it, minutes later. A share whose details could not all
|
||||
/// be read stays listed, so a failed lookup never hides a live share.
|
||||
pub(crate) fn list_shared_by_me(&self) -> CoreResult<Vec<SharedItem>> {
|
||||
let uids = self
|
||||
.rt
|
||||
@@ -678,24 +769,26 @@ impl Core {
|
||||
let members = self
|
||||
.rt
|
||||
.block_on(self.client.list_share_members(&uid))
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0);
|
||||
.map(|m| m.len());
|
||||
let proton_invites = self
|
||||
.rt
|
||||
.block_on(self.client.list_share_invitations(&uid))
|
||||
.map(|i| i.len())
|
||||
.unwrap_or(0);
|
||||
.map(|i| i.len());
|
||||
let external_invites = self
|
||||
.rt
|
||||
.block_on(self.client.list_external_invitations(&uid))
|
||||
.map(|i| i.len())
|
||||
.unwrap_or(0);
|
||||
let link = self
|
||||
.rt
|
||||
.block_on(self.client.get_public_link(&uid))
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(public_link_info);
|
||||
.map(|i| i.len());
|
||||
let link = self.rt.block_on(self.client.get_public_link(&uid));
|
||||
if let (Ok(0), Ok(0), Ok(0), Ok(None)) =
|
||||
(&members, &proton_invites, &external_invites, &link)
|
||||
{
|
||||
debug!(%uid, "shared by me: skipping an emptied share");
|
||||
continue;
|
||||
}
|
||||
let members = members.unwrap_or(0);
|
||||
let proton_invites = proton_invites.unwrap_or(0);
|
||||
let external_invites = external_invites.unwrap_or(0);
|
||||
let link = link.ok().flatten().map(public_link_info);
|
||||
items.push(SharedItem {
|
||||
uid: uid.to_string(),
|
||||
is_dir: n.is_folder(),
|
||||
|
||||
+1938
-25
File diff suppressed because it is too large.
Load diff
@@ -31,6 +31,9 @@
|
||||
//! - **A full account** ([`FakeDrive::set_quota`]) refusing uploads with
|
||||
//! `InsufficientQuota`.
|
||||
//!
|
||||
//! It also holds the account's computers, each with a root folder outside My
|
||||
//! files, so a simulated daemon can register its own and sync a local folder.
|
||||
//!
|
||||
//! A test can also hold the next read of one node on its way to Drive
|
||||
//! ([`FakeClient::hold_next_read`]), to make a change while the daemon waits
|
||||
//! for the answer, or on its way back ([`FakeClient::hold_answer_to_next_read`]),
|
||||
@@ -54,9 +57,12 @@ use parking_lot::Mutex;
|
||||
use proton_drive_rs::proton_sdk::account::Quota;
|
||||
use proton_drive_rs::proton_sdk::api::ResponseCode;
|
||||
use proton_drive_rs::proton_sdk::error::{ProtonApiError, ProtonError, Result};
|
||||
use proton_drive_rs::proton_sdk::ids::{DriveEventId, LinkId, NodeUid, VolumeId};
|
||||
use proton_drive_rs::proton_sdk::ids::{
|
||||
DeviceUid, DriveEventId, LinkId, NodeUid, ShareId, VolumeId,
|
||||
};
|
||||
use proton_drive_rs::{
|
||||
DriveEvent, DriveEventScopeId, Node, NodeKind, NodeMoveItem, RevisionState, Thumbnail,
|
||||
AvailableNameTarget, Device as Computer, DeviceType, DriveEvent, DriveEventScopeId, Node,
|
||||
NodeKind, NodeMoveItem, RevisionState, Thumbnail, ThumbnailType,
|
||||
};
|
||||
use sha1::{Digest as _, Sha1};
|
||||
|
||||
@@ -111,6 +117,9 @@ struct Server {
|
||||
block_pattern: Vec<u64>,
|
||||
/// The account's storage limit in bytes.
|
||||
max_space: i64,
|
||||
/// The account's computers, oldest first, as `(name, type, root folder)`.
|
||||
/// [`Computer`] is not `Clone`, so each answer builds them afresh.
|
||||
devices: Vec<(String, DeviceType, NodeUid)>,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
@@ -151,6 +160,17 @@ impl Server {
|
||||
.is_some_and(|remote| remote.file.is_none() && !remote.trashed)
|
||||
}
|
||||
|
||||
fn in_trash(&self, uid: &NodeUid) -> bool {
|
||||
let mut node = self.current(uid);
|
||||
while let Some(remote) = node {
|
||||
if remote.trashed {
|
||||
return true;
|
||||
}
|
||||
node = remote.parent.as_ref().and_then(|uid| self.current(uid));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_below(&self, uid: &NodeUid, ancestor: &NodeUid) -> bool {
|
||||
let mut cursor = Some(uid.clone());
|
||||
while let Some(at) = cursor {
|
||||
@@ -162,6 +182,53 @@ impl Server {
|
||||
false
|
||||
}
|
||||
|
||||
/// The `n`th computer of the account, as Drive lists it.
|
||||
fn computer(&self, n: usize) -> Computer {
|
||||
let (name, device_type, root) = &self.devices[n];
|
||||
Computer {
|
||||
uid: DeviceUid::new(format!("device{}", n + 1)),
|
||||
device_type: *device_type,
|
||||
name: Ok(name.clone()),
|
||||
root_folder_uid: root.clone(),
|
||||
creation_time: self.current(root).map_or(0, |remote| remote.created),
|
||||
last_sync_time: None,
|
||||
share_id: ShareId::new(format!("device-share{}", n + 1)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy `uid` and, for a folder, everything live below it under `parent` as
|
||||
/// `name`, the way Drive's copy endpoint does. Returns the copy's uid.
|
||||
fn copy_subtree(&mut self, uid: &NodeUid, parent: &NodeUid, name: String) -> NodeUid {
|
||||
let source = self
|
||||
.current(uid)
|
||||
.cloned()
|
||||
.expect("the caller checked the source");
|
||||
let copy = self.mint_uid();
|
||||
let now = now_secs();
|
||||
self.put(
|
||||
©,
|
||||
Some(Remote {
|
||||
parent: Some(parent.clone()),
|
||||
name,
|
||||
created: now,
|
||||
..source
|
||||
}),
|
||||
);
|
||||
let children: Vec<(NodeUid, String)> = self
|
||||
.nodes
|
||||
.keys()
|
||||
.filter_map(|child| {
|
||||
let remote = self.current(child)?;
|
||||
(remote.parent.as_ref() == Some(uid) && !remote.trashed)
|
||||
.then(|| (child.clone(), remote.name.clone()))
|
||||
})
|
||||
.collect();
|
||||
for (child, child_name) in children {
|
||||
self.copy_subtree(&child, ©, child_name);
|
||||
}
|
||||
copy
|
||||
}
|
||||
|
||||
fn mint_uid(&mut self) -> NodeUid {
|
||||
self.next_link += 1;
|
||||
uid(&format!("L{}", self.next_link))
|
||||
@@ -305,6 +372,7 @@ impl FakeDrive {
|
||||
events: Vec::new(),
|
||||
block_pattern: vec![pdfs_core::cache::BLOCK_SIZE],
|
||||
max_space: 1 << 40,
|
||||
devices: Vec::new(),
|
||||
};
|
||||
let now = now_secs();
|
||||
server.nodes.insert(
|
||||
@@ -338,6 +406,10 @@ impl FakeDrive {
|
||||
hold: Mutex::new(None),
|
||||
hold_answer: Mutex::new(None),
|
||||
hold_trash: Mutex::new(None),
|
||||
hold_download: Mutex::new(None),
|
||||
thumbnails: Mutex::new(HashMap::new()),
|
||||
content_read: AtomicU64::new(0),
|
||||
hold_any_landing: AtomicBool::new(false),
|
||||
hold_landing: Mutex::new(None),
|
||||
drop_after_create: AtomicBool::new(false),
|
||||
lose_create_reply: AtomicBool::new(false),
|
||||
@@ -616,8 +688,17 @@ struct ClientInner {
|
||||
hold_answer: Mutex<Option<Arc<HoldState>>>,
|
||||
/// The trash [`FakeClient::hold_next_trash`] is waiting for.
|
||||
hold_trash: Mutex<Option<Arc<HoldState>>>,
|
||||
/// The download [`FakeClient::hold_next_download`] is waiting for.
|
||||
hold_download: Mutex<Option<Arc<HoldState>>>,
|
||||
/// Thumbnails set by [`FakeClient::set_thumbnail`].
|
||||
thumbnails: Mutex<HashMap<NodeUid, Vec<u8>>>,
|
||||
/// File content bytes served, by downloads and ranged reads alike.
|
||||
content_read: AtomicU64,
|
||||
/// The landing [`FakeClient::hold_next_landing`] is waiting for.
|
||||
hold_landing: Mutex<Option<Arc<HoldState>>>,
|
||||
/// Set by [`FakeClient::hold_next_landing_of_any`]: hold whichever node
|
||||
/// lands next.
|
||||
hold_any_landing: AtomicBool,
|
||||
/// Set by [`FakeClient::drop_link_after_next_create`].
|
||||
drop_after_create: AtomicBool,
|
||||
/// Set by [`FakeClient::lose_reply_to_next_create`].
|
||||
@@ -755,6 +836,28 @@ impl FakeClient {
|
||||
Held(state)
|
||||
}
|
||||
|
||||
/// Hold the next download of `uid`'s content once Drive has answered it,
|
||||
/// before the bytes reach the daemon, until the answer is dropped.
|
||||
/// Give `uid` a server thumbnail, as an upload that carried one would.
|
||||
pub(crate) fn set_thumbnail(&self, uid: &NodeUid, bytes: Vec<u8>) {
|
||||
self.0.thumbnails.lock().insert(uid.clone(), bytes);
|
||||
}
|
||||
|
||||
/// File content bytes served so far, by whole downloads and ranged reads.
|
||||
pub(crate) fn content_read(&self) -> u64 {
|
||||
self.0.content_read.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn hold_next_download(&self, uid: &NodeUid) -> Held {
|
||||
let state = Arc::new(HoldState {
|
||||
uid: uid.clone(),
|
||||
reached: AtomicBool::new(false),
|
||||
released: AtomicBool::new(false),
|
||||
});
|
||||
*self.0.hold_download.lock() = Some(state.clone());
|
||||
Held(state)
|
||||
}
|
||||
|
||||
/// Hold the drain once the next upload to `uid` has landed and its blob
|
||||
/// is let go of, before the tree has the node (`DriveApi::landing`), until
|
||||
/// the answer is dropped.
|
||||
@@ -768,6 +871,14 @@ impl FakeClient {
|
||||
Held(state)
|
||||
}
|
||||
|
||||
/// [`FakeClient::hold_next_landing`] for whichever node lands next, such
|
||||
/// as a new file, whose uid Drive has not minted yet.
|
||||
pub(crate) fn hold_next_landing_of_any(&self) -> Held {
|
||||
let held = self.hold_next_landing(&uid("*"));
|
||||
self.0.hold_any_landing.store(true, Ordering::SeqCst);
|
||||
held
|
||||
}
|
||||
|
||||
/// Make the next file create on Drive, then hold its answer until the
|
||||
/// answer is dropped, as a slow link does.
|
||||
pub(crate) fn hold_reply_to_next_create(&self) -> Held {
|
||||
@@ -937,6 +1048,12 @@ impl FakeClient {
|
||||
}
|
||||
let parent = new_parent.cloned().or(remote.parent.clone());
|
||||
let name = new_name.unwrap_or(&remote.name).to_owned();
|
||||
if server.in_trash(uid) || parent.as_ref().is_some_and(|uid| server.in_trash(uid)) {
|
||||
return Err(api(
|
||||
ResponseCode::NotEnoughPermissions,
|
||||
"an item involved in the move is in Trash",
|
||||
));
|
||||
}
|
||||
if let Some(parent) = &parent {
|
||||
if !server.is_folder(parent) {
|
||||
return Err(api(
|
||||
@@ -1039,6 +1156,14 @@ fn delete_one(server: &mut Server, uid: &NodeUid) -> Result<()> {
|
||||
|
||||
#[async_trait]
|
||||
impl DriveApi for FakeClient {
|
||||
async fn context_share_id(&self, uid: &NodeUid) -> Result<ShareId> {
|
||||
self.request(false, |server| {
|
||||
assert_eq!(uid, &server.root);
|
||||
Ok(ShareId::new("my-files-share"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_my_files_folder(&self) -> Result<Node> {
|
||||
self.request(false, |server| {
|
||||
let root = server.root.clone();
|
||||
@@ -1383,6 +1508,21 @@ impl DriveApi for FakeClient {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn download_thumbnail(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
_thumbnail_type: ThumbnailType,
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
self.request(false, |server| {
|
||||
server
|
||||
.current(uid)
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| api(ResponseCode::DoesNotExist, "file not found"))
|
||||
})
|
||||
.await?;
|
||||
Ok(self.0.thumbnails.lock().get(uid).cloned())
|
||||
}
|
||||
|
||||
async fn download_file_to_dyn(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
@@ -1396,6 +1536,11 @@ impl DriveApi for FakeClient {
|
||||
.ok_or_else(|| api(ResponseCode::DoesNotExist, "file not found"))
|
||||
})
|
||||
.await?;
|
||||
self.wait_if_held(&self.0.hold_download, std::slice::from_ref(uid))
|
||||
.await;
|
||||
self.0
|
||||
.content_read
|
||||
.fetch_add(content.len() as u64, Ordering::SeqCst);
|
||||
output.write_all(&content).map_err(io_error)
|
||||
}
|
||||
|
||||
@@ -1437,6 +1582,61 @@ impl DriveApi for FakeClient {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn copy_node(
|
||||
&self,
|
||||
uid: &NodeUid,
|
||||
new_parent: &NodeUid,
|
||||
name: Option<&str>,
|
||||
) -> Result<NodeUid> {
|
||||
self.request(true, |server| {
|
||||
if !server.is_folder(new_parent) {
|
||||
return Err(api(ResponseCode::DoesNotExist, "parent folder not found"));
|
||||
}
|
||||
let source = server
|
||||
.current(uid)
|
||||
.filter(|remote| !remote.trashed)
|
||||
.cloned()
|
||||
.ok_or_else(|| api(ResponseCode::DoesNotExist, "node not found"))?;
|
||||
if server.is_below(new_parent, uid) {
|
||||
return Err(api(
|
||||
ResponseCode::InvalidRequirements,
|
||||
"cannot copy a folder into itself",
|
||||
));
|
||||
}
|
||||
let name = name.map_or(source.name.clone(), str::to_owned);
|
||||
if server.live_child_named(new_parent, &name).is_some() {
|
||||
return Err(api(
|
||||
ResponseCode::AlreadyExists,
|
||||
"a file or folder with that name already exists",
|
||||
));
|
||||
}
|
||||
Ok(server.copy_subtree(uid, new_parent, name))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_available_name(
|
||||
&self,
|
||||
parent_uid: &NodeUid,
|
||||
name: &str,
|
||||
target: AvailableNameTarget,
|
||||
) -> Result<String> {
|
||||
self.request(false, |server| {
|
||||
let (stem, extension) = match (target, name.rsplit_once('.')) {
|
||||
(AvailableNameTarget::File, Some((stem, ext))) if !stem.is_empty() => {
|
||||
(stem, format!(".{ext}"))
|
||||
}
|
||||
_ => (name, String::new()),
|
||||
};
|
||||
let free = std::iter::once(name.to_owned())
|
||||
.chain((1..).map(|n| format!("{stem} ({n}){extension}")))
|
||||
.find(|candidate| server.live_child_named(parent_uid, candidate).is_none())
|
||||
.expect("an unbounded sequence has a free name");
|
||||
Ok(free)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn move_nodes_streaming<'a>(
|
||||
&'a self,
|
||||
items: Vec<NodeMoveItem>,
|
||||
@@ -1484,8 +1684,12 @@ impl DriveApi for FakeClient {
|
||||
fn landing(&self, uid: &NodeUid) {
|
||||
let state = {
|
||||
let mut hold = self.0.hold_landing.lock();
|
||||
let any = self.0.hold_any_landing.load(Ordering::SeqCst);
|
||||
match hold.as_ref() {
|
||||
Some(state) if state.uid == *uid => hold.take(),
|
||||
Some(state) if any || state.uid == *uid => {
|
||||
self.0.hold_any_landing.store(false, Ordering::SeqCst);
|
||||
hold.take()
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
@@ -1506,6 +1710,42 @@ impl DriveApi for FakeClient {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn enumerate_devices(&self) -> Result<Vec<Computer>> {
|
||||
self.request(false, |server| {
|
||||
Ok((0..server.devices.len())
|
||||
.map(|n| server.computer(n))
|
||||
.collect())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_device(&self, name: &str, device_type: DeviceType) -> Result<Computer> {
|
||||
self.request(true, |server| {
|
||||
// A device root has no parent: like on Drive, it is not in My files.
|
||||
// It is listed at once, as the My files root is: the listing lag is
|
||||
// about children appearing, and the daemon lists the new root next.
|
||||
let root = server.mint_uid();
|
||||
let now = now_secs();
|
||||
server.nodes.insert(
|
||||
root.clone(),
|
||||
vec![(
|
||||
Instant::now() - HISTORY,
|
||||
Some(Remote {
|
||||
parent: None,
|
||||
name: name.to_owned(),
|
||||
trashed: false,
|
||||
created: now,
|
||||
modified: now,
|
||||
file: None,
|
||||
}),
|
||||
)],
|
||||
);
|
||||
server.devices.push((name.to_owned(), device_type, root));
|
||||
Ok(server.computer(server.devices.len() - 1))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn enumerate_trash_node_uids(&self) -> Result<Vec<NodeUid>> {
|
||||
self.request(false, |server| {
|
||||
Ok(server
|
||||
@@ -1544,13 +1784,19 @@ impl RevisionRead for FakeRevision {
|
||||
|
||||
async fn read_at(&self, offset: u64, length: u64) -> Result<Vec<u8>> {
|
||||
let content = self.content.clone();
|
||||
self.client
|
||||
let read = self
|
||||
.client
|
||||
.request(false, move |_| {
|
||||
let start = (offset as usize).min(content.len());
|
||||
let end = (offset.saturating_add(length) as usize).min(content.len());
|
||||
Ok(content[start..end].to_vec())
|
||||
})
|
||||
.await
|
||||
.await?;
|
||||
self.client
|
||||
.0
|
||||
.content_read
|
||||
.fetch_add(read.len() as u64, Ordering::SeqCst);
|
||||
Ok(read)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +112,8 @@ struct Run {
|
||||
/// Run `profile` on `seed`. It answers a report of what went wrong, if
|
||||
/// anything did.
|
||||
pub(crate) fn run(seed: u64, profile: &'static Profile) -> Result<(), String> {
|
||||
// The simulation suite runs serially. Discard logs left by earlier tests.
|
||||
take_logged();
|
||||
let dir = scratch(&format!("{}-{seed}", profile.name));
|
||||
let mut rng = Rng::new(seed);
|
||||
let drive = FakeDrive::new();
|
||||
@@ -137,6 +139,7 @@ pub(crate) fn run(seed: u64, profile: &'static Profile) -> Result<(), String> {
|
||||
let _ = daemon.stop();
|
||||
}
|
||||
}
|
||||
run.logged.extend(take_logged());
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
@@ -432,6 +435,7 @@ impl Run {
|
||||
|
||||
fn report(&self, failure: &str, dir: &Path) -> String {
|
||||
let _ = std::fs::write(dir.join("steps.log"), self.log.join("\n") + "\n");
|
||||
let _ = std::fs::write(dir.join("daemon.log"), self.logged.join("\n") + "\n");
|
||||
let tail = &self.log[self.log.len().saturating_sub(LOG_TAIL)..];
|
||||
format!(
|
||||
"{} seed {} failed: {failure}\nreplay with PDFS_SIM_SEED={}; state kept in {}\nlast steps:\n{}",
|
||||
|
||||
@@ -212,6 +212,31 @@ pub(crate) fn lid_ino(lid: i64) -> u64 {
|
||||
lid.unsigned_abs() + crate::ROOT_INO
|
||||
}
|
||||
|
||||
/// The inodes of the stored nodes among `nodes`, from their local ids, for
|
||||
/// [`State::intern_batch_with_inos`]. Called before the state lock is taken;
|
||||
/// a node that already has a row keeps its id. Empty when the database fails,
|
||||
/// which leaves the allocation to the lock holder, as before.
|
||||
pub(crate) fn inos_for(db: &Db, nodes: &[Node]) -> HashMap<NodeUid, u64> {
|
||||
let stored: Vec<&Node> = nodes
|
||||
.iter()
|
||||
.filter(|node| !crate::r#virtual::is_virtual_uid(&node.uid))
|
||||
.collect();
|
||||
if stored.is_empty() {
|
||||
return HashMap::new();
|
||||
}
|
||||
match db.lids_for(&stored) {
|
||||
Ok(lids) => stored
|
||||
.iter()
|
||||
.zip(lids)
|
||||
.map(|(node, lid)| (node.uid.clone(), lid_ino(lid)))
|
||||
.collect(),
|
||||
Err(error) => {
|
||||
warn!(%error, "no local ids before the listing; allocating them under the lock");
|
||||
HashMap::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The local id of the node at inode `ino`, the inverse of [`lid_ino`]:
|
||||
/// `None` for the root and a fallback inode.
|
||||
pub(crate) fn ino_lid(ino: u64) -> Option<i64> {
|
||||
@@ -241,6 +266,11 @@ pub(crate) struct State {
|
||||
pub(crate) next_ino: u64,
|
||||
/// Shared write state keyed by inode. Concurrent writers share the scratch file.
|
||||
pub(crate) active_writes: HashMap<u64, WriteHandle>,
|
||||
/// Inodes whose last write handle `release` has taken out of
|
||||
/// [`State::active_writes`] and is still queueing. `close(2)` returns
|
||||
/// before that, and in between the edit is in neither the handles nor the
|
||||
/// queue, so a check for changes not on Drive yet looks here too.
|
||||
pub(crate) releasing: HashSet<u64>,
|
||||
/// Maps file handle id (fh) to inode (ino). Read-only opens use fh 0 and
|
||||
/// have no entry here.
|
||||
pub(crate) handles: HashMap<u64, u64>,
|
||||
@@ -403,6 +433,7 @@ impl State {
|
||||
roots: HashSet::new(),
|
||||
next_ino: FALLBACK_INO,
|
||||
active_writes: HashMap::new(),
|
||||
releasing: HashSet::new(),
|
||||
handles: HashMap::new(),
|
||||
next_fh: 1,
|
||||
dir_snapshots: HashMap::new(),
|
||||
@@ -1074,20 +1105,34 @@ impl State {
|
||||
///
|
||||
/// This is what keeps `ls` on a large folder quick: one commit for the
|
||||
/// listing, rather than one autocommit — and one fsync — per child.
|
||||
pub(crate) fn intern_batch(&mut self, parent: u64, mut nodes: Vec<Node>) -> Vec<u64> {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn intern_batch(&mut self, parent: u64, nodes: Vec<Node>) -> Vec<u64> {
|
||||
self.intern_batch_with_inos(parent, nodes, HashMap::new())
|
||||
}
|
||||
|
||||
/// [`State::intern_batch`], with inodes for new entries that the caller
|
||||
/// allocated before it took the lock, from [`inos_for`]. Allocating them
|
||||
/// is a write transaction, and under the lock every inline handler of
|
||||
/// every mount waited for it (`docs/BUGS.md` B221). A node missing from
|
||||
/// `inos` still gets one here.
|
||||
pub(crate) fn intern_batch_with_inos(
|
||||
&mut self,
|
||||
parent: u64,
|
||||
mut nodes: Vec<Node>,
|
||||
mut inos: HashMap<NodeUid, u64>,
|
||||
) -> Vec<u64> {
|
||||
for node in &mut nodes {
|
||||
self.keep_revision_mtime(node);
|
||||
}
|
||||
self.outbox.push(DbWrite::Upsert(nodes.clone()));
|
||||
let fresh: Vec<&Node> = nodes
|
||||
.iter()
|
||||
.filter(|node| !self.by_uid.contains_key(&node.uid))
|
||||
.filter(|node| !self.by_uid.contains_key(&node.uid) && !inos.contains_key(&node.uid))
|
||||
.collect();
|
||||
let mut inos = if fresh.is_empty() {
|
||||
HashMap::new()
|
||||
} else {
|
||||
self.new_inos(&fresh)
|
||||
};
|
||||
if !fresh.is_empty() {
|
||||
let allocated = self.new_inos(&fresh);
|
||||
inos.extend(allocated);
|
||||
}
|
||||
nodes
|
||||
.into_iter()
|
||||
.map(|node| {
|
||||
@@ -1098,6 +1143,13 @@ impl State {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Record `uid`'s listing as complete once the lock is released, after the
|
||||
/// rows of its children: written the other way round, a crash between the
|
||||
/// two left a folder marked complete with no children to show.
|
||||
pub(crate) fn record_listed(&mut self, uid: NodeUid) {
|
||||
self.outbox.push(DbWrite::SetListed(uid, true));
|
||||
}
|
||||
|
||||
/// Check if a directory inode has any child nodes in memory or in the database.
|
||||
pub(crate) fn has_children(&self, parent: u64) -> bool {
|
||||
// A resident listing is authoritative even when empty. Falling through
|
||||
@@ -1467,6 +1519,44 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A listing's inodes are allocated before the lock is taken, and are the
|
||||
/// ones a node would get under it (`docs/BUGS.md` B221).
|
||||
#[test]
|
||||
fn inodes_allocated_before_the_lock_are_the_ones_the_node_keeps() {
|
||||
let (mut st, _dir) = state();
|
||||
let db = st.db.clone();
|
||||
let listing = vec![
|
||||
node("a", "root", "a.txt", false),
|
||||
node("b", "root", "b.txt", false),
|
||||
];
|
||||
let early = inos_for(&db, &listing);
|
||||
assert_eq!(early.len(), 2);
|
||||
let inos = st.intern_batch_with_inos(0, listing, early.clone());
|
||||
assert_eq!(inos, [early[&uid("a")], early[&uid("b")]]);
|
||||
st.flush_outbox();
|
||||
|
||||
let mut restarted = State::new(db, HashMap::new());
|
||||
assert_eq!(
|
||||
restarted.intern(0, node("b", "root", "b.txt", false)),
|
||||
early[&uid("b")]
|
||||
);
|
||||
}
|
||||
|
||||
/// The listing is recorded complete with its children's rows, when the
|
||||
/// lock goes, never ahead of them.
|
||||
#[test]
|
||||
fn a_listing_is_recorded_complete_after_its_children() {
|
||||
let (mut st, _dir) = state();
|
||||
let root = st.intern(0, node("root", "none", "root", true));
|
||||
st.flush_outbox();
|
||||
st.intern_batch(root, vec![node("a", "root", "a.txt", false)]);
|
||||
st.record_listed(uid("root"));
|
||||
assert!(st.db.children_if_listed(&uid("root")).unwrap().is_none());
|
||||
st.flush_outbox();
|
||||
let kids = st.db.children_if_listed(&uid("root")).unwrap().unwrap();
|
||||
assert_eq!(kids.len(), 1);
|
||||
}
|
||||
|
||||
/// Audit A5. Moving a file out of a folder leaves that folder's DB row
|
||||
/// claiming a complete listing, so once the hot cache drops it, remote
|
||||
/// changes under it stop being seen. `relocate` has to clear both sides.
|
||||
|
||||
@@ -36,6 +36,11 @@ const TICK: Duration = Duration::from_secs(15);
|
||||
/// would train everyone to ignore the real one.
|
||||
const STALL_AFTER: Duration = Duration::from_secs(120);
|
||||
|
||||
/// How long a sync pass may go without reporting progress before it is called
|
||||
/// out. Much longer than [`STALL_AFTER`]: one large file in a pass reports
|
||||
/// nothing until its whole transfer is done.
|
||||
const SYNC_STALL_AFTER: Duration = Duration::from_secs(900);
|
||||
|
||||
/// How often resident size goes into the log.
|
||||
const RSS_EVERY: Duration = Duration::from_secs(300);
|
||||
|
||||
@@ -52,7 +57,10 @@ pub(crate) fn run(core: Core, control_socket: PathBuf) {
|
||||
}
|
||||
|
||||
let mut last_rss = Instant::now();
|
||||
let mut last_ping = Instant::now();
|
||||
// `None` until the first probe, which runs on the first tick: startup has
|
||||
// already used up much of the watchdog's time (`docs/BUGS.md` B279).
|
||||
let mut last_ping: Option<Instant> = None;
|
||||
let mut restored = false;
|
||||
// Per-lane completion counts from the previous tick, to tell a queue that is
|
||||
// being worked through from one that is not moving at all.
|
||||
let mut last_completed = (0_u64, 0_u64);
|
||||
@@ -79,16 +87,22 @@ pub(crate) fn run(core: Core, control_socket: PathBuf) {
|
||||
// throw away in-flight uploads. The probe decides that; a stall only
|
||||
// gets said out loud.
|
||||
if let Some(period) = watchdog
|
||||
&& last_ping.elapsed() >= period
|
||||
&& ping_due(last_ping, period)
|
||||
{
|
||||
if probe(&control_socket) {
|
||||
if !restored {
|
||||
// Startup ran on the longer timeout the daemon asked for.
|
||||
// The socket answers now, so the unit's own one applies.
|
||||
systemd::restore_watchdog();
|
||||
restored = true;
|
||||
}
|
||||
systemd::watchdog_ping();
|
||||
} else {
|
||||
warn!(
|
||||
"daemon did not answer its own control socket; withholding the watchdog ping"
|
||||
);
|
||||
}
|
||||
last_ping = Instant::now();
|
||||
last_ping = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -137,9 +151,34 @@ fn report_stalls(report: &pdfs_core::control::Diagnostics, last_completed: (u64,
|
||||
"the transfer lane has not completed a job since the last check"
|
||||
);
|
||||
}
|
||||
// The engine is one thread for every synced folder, so a dead or stuck one
|
||||
// stops all of them while the worker pool looks idle (B259).
|
||||
if let Some(sync) = &report.sync {
|
||||
if !sync.running {
|
||||
stalled = true;
|
||||
warn!(
|
||||
"the sync engine is not running; synced folders do not sync until the daemon restarts"
|
||||
);
|
||||
} else if let Some(folder) = sync.pass_folder
|
||||
&& sync.pass_idle_secs >= SYNC_STALL_AFTER.as_secs()
|
||||
{
|
||||
stalled = true;
|
||||
warn!(
|
||||
folder,
|
||||
age_secs = sync.pass_age_secs,
|
||||
idle_secs = sync.pass_idle_secs,
|
||||
"a sync pass has not reported progress for a long time"
|
||||
);
|
||||
}
|
||||
}
|
||||
stalled
|
||||
}
|
||||
|
||||
/// Whether the watchdog wants its next ping. The first one is due at once.
|
||||
fn ping_due(last_ping: Option<Instant>, period: Duration) -> bool {
|
||||
last_ping.is_none_or(|at| at.elapsed() >= period)
|
||||
}
|
||||
|
||||
/// One round trip over the control socket: connect, ask for the cheapest
|
||||
/// request there is, read the answer.
|
||||
///
|
||||
@@ -169,7 +208,7 @@ fn probe(control_socket: &Path) -> bool {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pdfs_core::control::{Diagnostics, InflightRequest, WorkerState};
|
||||
use pdfs_core::control::{Diagnostics, InflightRequest, SyncEngineState, WorkerState};
|
||||
|
||||
fn busy_worker(age_secs: u64) -> WorkerState {
|
||||
WorkerState {
|
||||
@@ -180,6 +219,18 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The first ping goes out on the first tick, not a full period after the
|
||||
/// supervisor starts: startup has already spent most of the watchdog's time
|
||||
/// (`docs/BUGS.md` B279).
|
||||
#[test]
|
||||
fn first_watchdog_ping_is_due_at_once() {
|
||||
let period = Duration::from_secs(60);
|
||||
assert!(ping_due(None, period));
|
||||
assert!(!ping_due(Some(Instant::now()), period));
|
||||
let long_ago = Instant::now().checked_sub(period).expect("uptime");
|
||||
assert!(ping_due(Some(long_ago), period));
|
||||
}
|
||||
|
||||
/// A worker doing normal work must not be reported: a WARN every tick is a
|
||||
/// WARN nobody reads.
|
||||
#[test]
|
||||
@@ -226,4 +277,45 @@ mod tests {
|
||||
assert!(!report_stalls(&report, (0, 30)));
|
||||
assert!(report_stalls(&report, (0, 40)));
|
||||
}
|
||||
|
||||
fn with_sync(sync: SyncEngineState) -> Diagnostics {
|
||||
Diagnostics {
|
||||
sync: Some(sync),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A sync engine that has ended stops every synced folder while the
|
||||
/// workers look idle, which is what issue #53 looked like (B259).
|
||||
#[test]
|
||||
fn a_stopped_sync_engine_is_a_stall() {
|
||||
assert!(report_stalls(
|
||||
&with_sync(SyncEngineState::default()),
|
||||
(0, 0)
|
||||
));
|
||||
}
|
||||
|
||||
/// A pass that keeps reporting progress is working, however long it runs;
|
||||
/// one that has gone quiet past the threshold is not.
|
||||
#[test]
|
||||
fn only_a_quiet_sync_pass_is_a_stall() {
|
||||
let busy = SyncEngineState {
|
||||
running: true,
|
||||
pass_folder: Some(3),
|
||||
pass_age_secs: SYNC_STALL_AFTER.as_secs() * 4,
|
||||
pass_idle_secs: 5,
|
||||
panics: 0,
|
||||
};
|
||||
assert!(!report_stalls(&with_sync(busy.clone()), (0, 0)));
|
||||
let quiet = SyncEngineState {
|
||||
pass_idle_secs: SYNC_STALL_AFTER.as_secs(),
|
||||
..busy
|
||||
};
|
||||
assert!(report_stalls(&with_sync(quiet), (0, 0)));
|
||||
let idle = SyncEngineState {
|
||||
running: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!report_stalls(&with_sync(idle), (0, 0)));
|
||||
}
|
||||
}
|
||||
@@ -37,10 +37,12 @@ const CONFLICT_SWEEP_INTERVAL: Duration = Duration::from_secs(300);
|
||||
/// fresh mount does at startup (hydrate, recover, initial enumeration).
|
||||
const CONFLICT_SWEEP_WARMUP: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Strip a ` (sync-conflict <stamp>)` (optionally `-<n>`) marker from `name`,
|
||||
/// returning the original file name it was a copy of. `None` when `name` is not a
|
||||
/// conflict copy. Inverse of the `conflict_name` / `conflict_path_with_suffix`
|
||||
/// formats, so the two halves of the product agree on what a conflict is.
|
||||
/// Strip a ` (sync-conflict <label> <stamp>)` (optionally `-<n>`) marker from
|
||||
/// `name`, returning the original file name it was a copy of. The label names
|
||||
/// the computer that made the copy (B251); copies made before it have the bare
|
||||
/// stamp. `None` when `name` is not a conflict copy. Inverse of the
|
||||
/// `conflict_name` / `conflict_path_with_suffix` formats, so the two halves of
|
||||
/// the product agree on what a conflict is.
|
||||
pub(crate) fn conflict_base_name(name: &str) -> Option<String> {
|
||||
const MARKER: &str = " (sync-conflict ";
|
||||
let start = name.find(MARKER)?;
|
||||
@@ -48,6 +50,20 @@ pub(crate) fn conflict_base_name(name: &str) -> Option<String> {
|
||||
let close_rel = name[after..].find(')')?;
|
||||
let close = after + close_rel;
|
||||
let inner = &name[after..close];
|
||||
// A label is one token of the characters `conflict_label_of` keeps.
|
||||
let inner = match inner.rsplit_once(' ') {
|
||||
Some((label, token)) => {
|
||||
let label_ok = !label.is_empty()
|
||||
&& label
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
|
||||
if !label_ok {
|
||||
return None;
|
||||
}
|
||||
token
|
||||
}
|
||||
None => inner,
|
||||
};
|
||||
// The token is `<digits>` or `<digits>-<digits>`; reject anything else so a
|
||||
// file that merely happens to contain the phrase is not mistaken for a copy.
|
||||
let mut parts = inner.splitn(2, '-');
|
||||
@@ -486,6 +502,24 @@ mod tests {
|
||||
conflict_base_name("notes (sync-conflict 1700-3).txt").as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
// Named after the computer that made it (B251).
|
||||
assert_eq!(
|
||||
conflict_base_name("notes (sync-conflict laptop 1700).txt").as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_base_name("notes (sync-conflict my-pc.local 1700-2).txt").as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_base_name(&crate::conflict_name("notes.txt", 1700)).as_deref(),
|
||||
Some("notes.txt")
|
||||
);
|
||||
assert_eq!(
|
||||
conflict_base_name("x (sync-conflict two words 1).txt"),
|
||||
None
|
||||
);
|
||||
assert_eq!(conflict_base_name("x (sync-conflict 1).txt"), None);
|
||||
// Not a conflict copy.
|
||||
assert_eq!(conflict_base_name("report.xml"), None);
|
||||
// Looks close but the token is not a stamp.
|
||||
|
||||
+870
-155
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,11 @@
|
||||
//! Filesystem watcher, polling, and debounce control plane.
|
||||
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
|
||||
use pdfs_core::control::SyncEngineState;
|
||||
use tracing::error;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// How long the watcher must go quiet before a burst counts as finished.
|
||||
@@ -22,7 +28,10 @@ const POLL_INTERVAL: Duration = Duration::from_secs(120);
|
||||
/// per-item work is a network round-trip, so a folder with thousands of files is
|
||||
/// unusable done one-at-a-time; this bounds concurrency like the bulk-upload
|
||||
/// engine does.
|
||||
pub(super) const SYNC_CONCURRENCY: usize = 8;
|
||||
pub(crate) const SYNC_CONCURRENCY: usize = 8;
|
||||
/// How long after a failed watch the engine tries the watches again. Each try
|
||||
/// walks the folder's whole tree, so not on every burst.
|
||||
const REWATCH_RETRY: Duration = Duration::from_secs(600);
|
||||
|
||||
/// A message to the sync engine's control thread.
|
||||
pub(crate) enum SyncMsg {
|
||||
@@ -53,30 +62,159 @@ pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) -> Option<std::thread::Jo
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the engine is alive and which pass it is on, for
|
||||
/// [`Core::diagnostics`] and the supervisor.
|
||||
///
|
||||
/// A panic in a pass used to end the engine thread. Every later add, sync-now,
|
||||
/// file event and poll then went into a channel nobody read, the folder stayed
|
||||
/// `syncing`, and nothing said so until a restart (`docs/BUGS.md` B259).
|
||||
#[derive(Default)]
|
||||
pub(crate) struct EngineHealth {
|
||||
running: AtomicBool,
|
||||
pass: Mutex<Option<PassClock>>,
|
||||
panics: AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct PassClock {
|
||||
folder: i64,
|
||||
started: Instant,
|
||||
moved: Instant,
|
||||
}
|
||||
|
||||
impl EngineHealth {
|
||||
/// Note that the running pass reported progress.
|
||||
pub(crate) fn moved(&self) {
|
||||
if let Some(pass) = self.pass.lock().as_mut() {
|
||||
pass.moved = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// The engine's state now. `try_lock`, like the rest of the diagnostics:
|
||||
/// a report must not wait on the thing it reports about.
|
||||
pub(crate) fn report(&self) -> SyncEngineState {
|
||||
let pass = self.pass.try_lock().and_then(|pass| *pass);
|
||||
SyncEngineState {
|
||||
running: self.running.load(Ordering::Relaxed),
|
||||
pass_folder: pass.map(|p| p.folder),
|
||||
pass_age_secs: pass.map_or(0, |p| p.started.elapsed().as_secs()),
|
||||
pass_idle_secs: pass.map_or(0, |p| p.moved.elapsed().as_secs()),
|
||||
panics: self.panics.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Run `pass` over `folder` as the current pass. Returns `false` when it
|
||||
/// panicked: the panic ends that pass, not the engine.
|
||||
pub(super) fn contain(&self, folder: i64, pass: impl FnOnce()) -> bool {
|
||||
let now = Instant::now();
|
||||
*self.pass.lock() = Some(PassClock {
|
||||
folder,
|
||||
started: now,
|
||||
moved: now,
|
||||
});
|
||||
let finished = std::panic::catch_unwind(AssertUnwindSafe(pass)).is_ok();
|
||||
*self.pass.lock() = None;
|
||||
if !finished {
|
||||
self.panics.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
finished
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks the engine running for as long as it lives.
|
||||
pub(super) struct Running<'a>(&'a EngineHealth);
|
||||
|
||||
impl<'a> Running<'a> {
|
||||
pub(super) fn start(health: &'a EngineHealth) -> Self {
|
||||
health.running.store(true, Ordering::Relaxed);
|
||||
Self(health)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Running<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.0.running.store(false, Ordering::Relaxed);
|
||||
if std::thread::panicking() {
|
||||
error!("sync engine stopped by a panic; nothing syncs until the daemon restarts");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One folder's pass, contained: a panic in it sets the folder to `error` and
|
||||
/// leaves the engine running for the next one.
|
||||
fn run_pass(core: &Core, folder: &StoredSyncFolder) {
|
||||
if core
|
||||
.sync_engine
|
||||
.contain(folder.id, || core.reconcile_folder(folder))
|
||||
{
|
||||
return;
|
||||
}
|
||||
core.progress_end(folder.id);
|
||||
let _ = core
|
||||
.db
|
||||
.sync_folder_set_state(folder.id, "error", folder.last_sync);
|
||||
error!(
|
||||
id = folder.id,
|
||||
path = %folder.local_path,
|
||||
"sync: pass panicked; the folder is set to error and tried again on the next poll"
|
||||
);
|
||||
core.log_activity(
|
||||
ActivityKind::Sync,
|
||||
format!("sync failed for {}", base_name(&folder.local_path)),
|
||||
"an internal error stopped the pass; it is tried again on the next poll",
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
/// The pass a `Reconcile(id)` asked for. A folder removed since, or one the
|
||||
/// database cannot read, is said so rather than dropped in silence.
|
||||
fn run_pass_by_id(core: &Core, id: i64) {
|
||||
match core.db.sync_folder_get(id) {
|
||||
Ok(Some(folder)) => run_pass(core, &folder),
|
||||
Ok(None) => debug!(id, "sync: folder removed before its pass; skipping"),
|
||||
Err(e) => warn!(id, error = ?e, "sync: cannot read folder; skipping its pass"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The engine control loop. Single-threaded so reconcile passes never overlap.
|
||||
fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
let health = core.sync_engine.clone();
|
||||
let _running = Running::start(&health);
|
||||
// Paths the watcher currently covers, mapped to their folder id, so an event
|
||||
// path can be resolved back to the folder it belongs to.
|
||||
let watched: Mutex<Vec<(PathBuf, i64)>> = Mutex::new(Vec::new());
|
||||
let watched: Arc<Mutex<Vec<(PathBuf, i64)>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
let tx_events = core.sync_tx.clone();
|
||||
let mut watcher =
|
||||
match notify::recommended_watcher(move |res: notify::Result<notify::Event>| {
|
||||
let Ok(event) = res else { return };
|
||||
// Ignore pure metadata/access noise; content changes are what matter.
|
||||
if matches!(event.kind, notify::EventKind::Access(_)) {
|
||||
return;
|
||||
}
|
||||
let _ = tx_events.send(SyncMsg::ReconcileAll);
|
||||
}) {
|
||||
Ok(w) => w,
|
||||
let watched_events = watched.clone();
|
||||
let handler = move |res: notify::Result<notify::Event>| {
|
||||
let event = match res {
|
||||
Ok(event) => event,
|
||||
// Running out of inotify watches for a new subfolder, say: that
|
||||
// subfolder's changes are then found by the poll only (B262).
|
||||
Err(e) => {
|
||||
warn!(error = %e, "filesystem watcher unavailable; sync is poll-only");
|
||||
// Still run the loop so polling and explicit SyncNow work.
|
||||
poll_only_loop(&core, rx);
|
||||
warn!(error = %e, "sync: filesystem watcher error");
|
||||
return;
|
||||
}
|
||||
};
|
||||
// Ignore pure metadata/access noise; content changes are what matter.
|
||||
if matches!(event.kind, notify::EventKind::Access(_)) {
|
||||
return;
|
||||
}
|
||||
let msg = event_target(&watched_events.lock(), &event.paths);
|
||||
let _ = tx_events.send(msg);
|
||||
};
|
||||
// The local walk skips symlinks, so following them here would only watch
|
||||
// trees outside the folder, and run passes over it for changes made there.
|
||||
let config = notify::Config::default().with_follow_symlinks(false);
|
||||
let mut watcher = match notify::RecommendedWatcher::new(handler, config) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "filesystem watcher unavailable; sync is poll-only");
|
||||
// Still run the loop so polling and explicit SyncNow work.
|
||||
poll_only_loop(&core, rx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Periodic remote poll.
|
||||
{
|
||||
@@ -94,8 +232,9 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
.ok();
|
||||
}
|
||||
|
||||
rewatch(&core, &mut watcher, &watched);
|
||||
reconcile_all(&core);
|
||||
let mut unwatched = rewatch(&core, &mut watcher, &watched);
|
||||
let mut last_rewatch = Instant::now();
|
||||
run_burst(&core, &HashSet::new(), true);
|
||||
|
||||
while let Ok(msg) = rx.recv() {
|
||||
let mut ids: HashSet<i64> = HashSet::new();
|
||||
@@ -108,21 +247,16 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||
return;
|
||||
}
|
||||
|
||||
// A folder whose watch failed is tried again now and then, not only
|
||||
// when the folder set changes.
|
||||
if unwatched && last_rewatch.elapsed() >= REWATCH_RETRY {
|
||||
do_rewatch = true;
|
||||
}
|
||||
if do_rewatch {
|
||||
rewatch(&core, &mut watcher, &watched);
|
||||
}
|
||||
if all {
|
||||
reconcile_all(&core);
|
||||
} else {
|
||||
for id in ids {
|
||||
if core.sync_paused() {
|
||||
break;
|
||||
}
|
||||
if let Ok(Some(folder)) = core.db.sync_folder_get(id) {
|
||||
core.reconcile_folder(&folder);
|
||||
}
|
||||
}
|
||||
unwatched = rewatch(&core, &mut watcher, &watched);
|
||||
last_rewatch = Instant::now();
|
||||
}
|
||||
run_burst(&core, &ids, all);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +273,7 @@ fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
||||
}
|
||||
});
|
||||
}
|
||||
reconcile_all(core);
|
||||
run_burst(core, &HashSet::new(), true);
|
||||
while let Ok(msg) = rx.recv() {
|
||||
let mut ids: HashSet<i64> = HashSet::new();
|
||||
let mut all = false;
|
||||
@@ -150,21 +284,59 @@ fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
||||
info!("sync engine stopping");
|
||||
return;
|
||||
}
|
||||
if all {
|
||||
reconcile_all(core);
|
||||
} else {
|
||||
for id in ids {
|
||||
if core.sync_paused() {
|
||||
break;
|
||||
}
|
||||
if let Ok(Some(folder)) = core.db.sync_folder_get(id) {
|
||||
core.reconcile_folder(&folder);
|
||||
}
|
||||
run_burst(core, &ids, all);
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the passes one burst asked for: the folders named by id, then, when the
|
||||
/// burst also holds a `ReconcileAll`, every other folder.
|
||||
///
|
||||
/// The named folders go first. A poll's `ReconcileAll` in the same burst used
|
||||
/// to take their place, and a folder just added, which has the highest id,
|
||||
/// waited behind a full pass over every other folder (`docs/BUGS.md` B260).
|
||||
///
|
||||
/// Skipped while sync is paused, and cut short when a pause lands between
|
||||
/// passes: the resume sends a fresh [`SyncMsg::ReconcileAll`], which walks
|
||||
/// everything the watcher reported meanwhile.
|
||||
fn run_burst(core: &Core, ids: &HashSet<i64>, all: bool) {
|
||||
if core.sync_paused() {
|
||||
debug!("sync: paused, skipping reconcile");
|
||||
return;
|
||||
}
|
||||
let folders = if all {
|
||||
match core.db.sync_folder_list() {
|
||||
Ok(folders) => folders,
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "sync: cannot list folders");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let listed: Vec<i64> = folders.iter().map(|f| f.id).collect();
|
||||
for id in burst_order(ids, &listed) {
|
||||
if core.sync_paused() {
|
||||
break;
|
||||
}
|
||||
// A folder named by id is read again: it may have been added or
|
||||
// changed after the list was read.
|
||||
match folders.iter().find(|f| f.id == id) {
|
||||
Some(folder) if !ids.contains(&id) => run_pass(core, folder),
|
||||
_ => run_pass_by_id(core, id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The order of a burst's passes: the folders named by id, lowest first, then
|
||||
/// the rest of `listed` in its own order.
|
||||
pub(super) fn burst_order(ids: &HashSet<i64>, listed: &[i64]) -> Vec<i64> {
|
||||
let mut order: Vec<i64> = ids.iter().copied().collect();
|
||||
order.sort_unstable();
|
||||
order.extend(listed.iter().copied().filter(|id| !ids.contains(id)));
|
||||
order
|
||||
}
|
||||
|
||||
/// Absorb the rest of an event burst, returning once the watcher has been quiet
|
||||
/// for `quiet` or `cap` has elapsed since the first event — whichever comes
|
||||
/// first. The caller has already classified the event that opened the burst.
|
||||
@@ -236,33 +408,37 @@ pub(super) fn classify(
|
||||
true
|
||||
}
|
||||
|
||||
/// Reconcile every mirror folder in turn. Skipped while sync is paused: the
|
||||
/// resume sends a fresh [`SyncMsg::ReconcileAll`], which walks everything the
|
||||
/// watcher reported meanwhile.
|
||||
fn reconcile_all(core: &Core) {
|
||||
if core.sync_paused() {
|
||||
debug!("sync: paused, skipping reconcile");
|
||||
return;
|
||||
}
|
||||
let folders = match core.db.sync_folder_list() {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "sync: cannot list folders");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for folder in folders {
|
||||
core.reconcile_folder(&folder);
|
||||
/// The reconcile a watcher event asks for: the one mirror folder all its paths
|
||||
/// lie in, so an edit in one folder does not walk every other one too. An event
|
||||
/// that spans folders, or that no watched folder claims (a watch list a moment
|
||||
/// out of date), falls back to every folder.
|
||||
pub(super) fn event_target(watched: &[(PathBuf, i64)], paths: &[PathBuf]) -> SyncMsg {
|
||||
let mut ids = paths.iter().map(|path| {
|
||||
watched
|
||||
.iter()
|
||||
.filter(|(root, _)| path.starts_with(root))
|
||||
// The deepest root, should one mirror folder sit inside another.
|
||||
.max_by_key(|(root, _)| root.components().count())
|
||||
.map(|(_, id)| *id)
|
||||
});
|
||||
match ids.next() {
|
||||
Some(Some(first)) if ids.all(|id| id == Some(first)) => SyncMsg::Reconcile(first),
|
||||
_ => SyncMsg::ReconcileAll,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bring the filesystem watches in line with the current mirror-folder set:
|
||||
/// watch newly-added folders, drop removed ones.
|
||||
/// watch newly-added folders, drop removed ones. Returns whether a folder is
|
||||
/// left unwatched because its watch failed.
|
||||
///
|
||||
/// The watch calls run on a copy, outside the lock: the watcher's event
|
||||
/// callback takes the same lock, and a backend that hands a watch request to
|
||||
/// its event thread and waits for the answer would deadlock against it.
|
||||
fn rewatch(
|
||||
core: &Core,
|
||||
watcher: &mut notify::RecommendedWatcher,
|
||||
watched: &Mutex<Vec<(PathBuf, i64)>>,
|
||||
) {
|
||||
) -> bool {
|
||||
let want: Vec<(PathBuf, i64)> = match core.db.sync_folder_list() {
|
||||
Ok(f) => f
|
||||
.into_iter()
|
||||
@@ -271,29 +447,60 @@ fn rewatch(
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
warn!(error = ?e, "sync: cannot list folders to watch");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let mut have = watched.lock();
|
||||
let mut have = watched.lock().clone();
|
||||
// Drop watches no longer wanted.
|
||||
have.retain(|(path, _)| {
|
||||
if want.iter().any(|(p, _)| p == path) {
|
||||
true
|
||||
} else {
|
||||
let _ = watcher.unwatch(path);
|
||||
false
|
||||
}
|
||||
});
|
||||
for path in keep_wanted(&mut have, &want) {
|
||||
let _ = watcher.unwatch(&path);
|
||||
}
|
||||
// Add newly wanted watches.
|
||||
let mut failed = false;
|
||||
for (path, id) in &want {
|
||||
if have.iter().any(|(p, _)| p == path) {
|
||||
continue;
|
||||
}
|
||||
match watcher.watch(path, RecursiveMode::Recursive) {
|
||||
Ok(()) => have.push((path.clone(), *id)),
|
||||
Err(e) => warn!(path = %path.display(), error = %e, "sync: watch failed"),
|
||||
Err(e) => {
|
||||
// A recursive watch that fails partway keeps the watches it had
|
||||
// added. Their events would match no folder and run a pass over
|
||||
// every one, and they hold watches the next folder needs (B262).
|
||||
let _ = watcher.unwatch(path);
|
||||
failed = true;
|
||||
warn!(
|
||||
path = %path.display(),
|
||||
error = %e,
|
||||
"sync: watch failed; the poll finds this folder's changes until a retry \
|
||||
works. A low fs.inotify.max_user_watches is the usual cause"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
*watched.lock() = have;
|
||||
failed
|
||||
}
|
||||
|
||||
/// Keep the watches in `have` that `want` still asks for, and return the paths
|
||||
/// of the others, to unwatch.
|
||||
///
|
||||
/// A path is matched alone, but takes the id `want` gives it: a folder removed
|
||||
/// and added again in one burst keeps its watch, and its events go to the new
|
||||
/// row rather than to the removed one (`docs/BUGS.md` B272).
|
||||
pub(super) fn keep_wanted(have: &mut Vec<(PathBuf, i64)>, want: &[(PathBuf, i64)]) -> Vec<PathBuf> {
|
||||
let mut dropped = Vec::new();
|
||||
have.retain_mut(|(path, id)| match want.iter().find(|(p, _)| p == path) {
|
||||
Some((_, wanted)) => {
|
||||
*id = *wanted;
|
||||
true
|
||||
}
|
||||
None => {
|
||||
dropped.push(path.clone());
|
||||
false
|
||||
}
|
||||
});
|
||||
dropped
|
||||
}
|
||||
|
||||
// ---- reconcile ------------------------------------------------------------
|
||||
@@ -83,6 +83,46 @@ pub(super) struct RemoteItem {
|
||||
pub(super) is_dir: bool,
|
||||
pub(super) mtime: i64,
|
||||
pub(super) size: i64,
|
||||
/// The active revision's id. The light walk carries it for every file, so
|
||||
/// it is the signal that tells another client's new revision from a link
|
||||
/// merely re-stamped.
|
||||
pub(super) revision: Option<String>,
|
||||
/// The active revision's content SHA-1, read only for files the walk
|
||||
/// enumerated in full (no baseline, or a moved mtime). `None` is never
|
||||
/// proof of anything.
|
||||
pub(super) sha1: Option<String>,
|
||||
}
|
||||
|
||||
/// What a pass saw of a remote file: the signature its baseline records once
|
||||
/// the file is synced, and what an upload checks Drive still holds before it
|
||||
/// supersedes it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) struct RemoteSeen {
|
||||
pub(super) mtime: i64,
|
||||
pub(super) size: i64,
|
||||
pub(super) revision: Option<String>,
|
||||
}
|
||||
|
||||
impl RemoteSeen {
|
||||
/// Whether `node` is still the revision this was taken from. The revision
|
||||
/// id decides it when both sides have one; otherwise the modification time
|
||||
/// and size, which is all a baseline row from before schema 44 holds.
|
||||
pub(super) fn still_current(&self, node: &Node) -> bool {
|
||||
match (&self.revision, crate::node_revision_id(node)) {
|
||||
(Some(seen), Some(now)) => *seen == now,
|
||||
_ => (self.mtime, self.size) == remote_file_sig(node),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&RemoteItem> for RemoteSeen {
|
||||
fn from(item: &RemoteItem) -> Self {
|
||||
Self {
|
||||
mtime: item.mtime,
|
||||
size: item.size,
|
||||
revision: item.revision.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The result of a reconcile pass: what it moved, how many paths were kept as
|
||||
@@ -113,7 +153,7 @@ impl Outcome {
|
||||
Applied::Uploaded => self.uploaded += 1,
|
||||
Applied::Downloaded => self.downloaded += 1,
|
||||
Applied::Conflict => self.conflicts += 1,
|
||||
Applied::Vanished => {}
|
||||
Applied::Vanished | Applied::Adopted => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,10 +197,16 @@ pub(super) enum Pending {
|
||||
CreateDir { rel: String, parent: NodeUid },
|
||||
/// Upload a brand-new local file into `parent`.
|
||||
UploadNew { rel: String, parent: NodeUid },
|
||||
/// Upload a changed local file as a new revision of `uid`.
|
||||
UploadRevision { rel: String, uid: NodeUid },
|
||||
/// Download remote `uid` to the local path, stamping `mtime`. `size` is the
|
||||
/// remote's reported size, used as the transfer's expected total.
|
||||
/// Upload a changed local file as a new revision of `uid`, if Drive still
|
||||
/// holds the revision `seen` describes (B249).
|
||||
UploadRevision {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
seen: RemoteSeen,
|
||||
},
|
||||
/// Download remote `uid` to the local path, stamping `seen.mtime`.
|
||||
/// `seen.size` is the remote's reported size, used as the transfer's
|
||||
/// expected total.
|
||||
///
|
||||
/// `local` is what the planning walk saw at this path, or `None` if nothing
|
||||
/// was there. A download can run for minutes, so the apply step revalidates
|
||||
@@ -170,16 +216,26 @@ pub(super) enum Pending {
|
||||
Download {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
seen: RemoteSeen,
|
||||
local: Option<LocalSig>,
|
||||
},
|
||||
/// First sync of a path present on both sides at the same size, with a
|
||||
/// digest on Drive: hash the local file, and record the baseline if it holds
|
||||
/// the same bytes. A different file of the same size goes the way of
|
||||
/// [`Pending::Conflict`]: two machines syncing one folder must not read as
|
||||
/// in step while they hold different content.
|
||||
Adopt {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
seen: RemoteSeen,
|
||||
sha1: String,
|
||||
local: LocalSig,
|
||||
},
|
||||
/// Both sides changed: set the local copy aside, then download remote `uid`.
|
||||
Conflict {
|
||||
rel: String,
|
||||
uid: NodeUid,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
seen: RemoteSeen,
|
||||
},
|
||||
/// Both sides changed during a [`push pass`](Core::push_pass): upload the local
|
||||
/// copy into `parent` under a conflict name, leaving the remote file as it is.
|
||||
@@ -194,6 +250,7 @@ impl Pending {
|
||||
| Pending::UploadNew { rel, .. }
|
||||
| Pending::UploadRevision { rel, .. }
|
||||
| Pending::Download { rel, .. }
|
||||
| Pending::Adopt { rel, .. }
|
||||
| Pending::Conflict { rel, .. }
|
||||
| Pending::PushConflict { rel, .. } => rel,
|
||||
}
|
||||
@@ -206,7 +263,9 @@ impl Pending {
|
||||
Pending::UploadNew { .. }
|
||||
| Pending::UploadRevision { .. }
|
||||
| Pending::PushConflict { .. } => ActivityKind::Upload,
|
||||
Pending::Download { .. } | Pending::Conflict { .. } => ActivityKind::Download,
|
||||
Pending::Download { .. } | Pending::Adopt { .. } | Pending::Conflict { .. } => {
|
||||
ActivityKind::Download
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,6 +276,7 @@ impl Pending {
|
||||
Pending::UploadNew { .. } => "new file",
|
||||
Pending::UploadRevision { .. } => "new version",
|
||||
Pending::Download { .. } => "from Drive",
|
||||
Pending::Adopt { .. } => "already the same on Drive",
|
||||
Pending::Conflict { .. } => "local changes kept as a conflict copy",
|
||||
Pending::PushConflict { .. } => "local changes uploaded as a conflict copy",
|
||||
}
|
||||
|
||||
@@ -180,33 +180,220 @@ pub(super) fn local_deletions<B, L>(
|
||||
missing
|
||||
}
|
||||
|
||||
/// Refuse to run a pass whose local side has vanished in its entirety.
|
||||
/// How many tracked paths a folder needs before a mass disappearance, short of
|
||||
/// a total one, trips a wipe guard. Below this, a few survivors say little.
|
||||
const MASS_LOSS_MIN_PATHS: usize = 20;
|
||||
|
||||
/// The share of a large folder's tracked paths that may vanish in one pass, in
|
||||
/// tenths, before a wipe guard stops it: 9 of 10 and up is refused.
|
||||
const MASS_LOSS_TENTHS: usize = 9;
|
||||
|
||||
/// Whether `missing` of `total` tracked paths gone at once looks like a lost
|
||||
/// tree rather than deletions. All of them always does. Most of a large
|
||||
/// folder does too: one surviving file used to clear the guard, so a
|
||||
/// half-mounted disk, or a script that emptied the folder and wrote one file
|
||||
/// back, trashed everything else on Drive (B270).
|
||||
fn looks_like_a_wipe(missing: usize, total: usize) -> bool {
|
||||
total > 0
|
||||
&& (missing == total
|
||||
|| (total >= MASS_LOSS_MIN_PATHS && missing * 10 >= total * MASS_LOSS_TENTHS))
|
||||
}
|
||||
|
||||
/// Refuse to run a pass whose local side has vanished in its entirety, or
|
||||
/// nearly so ([`looks_like_a_wipe`]).
|
||||
///
|
||||
/// When every baseline path is absent locally, the likely cause is an unavailable
|
||||
/// mount or unreadable folder rather than a deliberate whole-tree deletion.
|
||||
/// One surviving path is enough to clear the guard.
|
||||
/// When the synced paths are all or nearly all absent locally, the likely
|
||||
/// cause is an unavailable mount or unreadable folder rather than a deliberate
|
||||
/// deletion of the whole tree.
|
||||
pub(super) fn guard_local_wipe<B, L>(
|
||||
baseline: &HashMap<String, B>,
|
||||
local: &HashMap<String, L>,
|
||||
) -> Result<(), String> {
|
||||
if !baseline.is_empty() && baseline.keys().all(|rel| !local.contains_key(rel)) {
|
||||
let missing = baseline
|
||||
.keys()
|
||||
.filter(|rel| !local.contains_key(*rel))
|
||||
.count();
|
||||
if looks_like_a_wipe(missing, baseline.len()) {
|
||||
return Err(format!(
|
||||
"every one of the {} synced paths is missing locally; refusing to trash \
|
||||
them on Drive. Check that the folder is mounted and readable.",
|
||||
"{missing} of the {} synced paths are missing locally; refusing to trash \
|
||||
them on Drive. Check that the folder is mounted and readable; to download \
|
||||
them again, sync the folder from scratch.",
|
||||
baseline.len()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Refuse to run a pass whose remote side has vanished in its entirety, or
|
||||
/// nearly so: the remote twin of [`guard_local_wipe`].
|
||||
///
|
||||
/// An empty or truncated remote walk reads to the planner as "Drive deleted
|
||||
/// everything", which a pass answers by deleting the local copies. When all or
|
||||
/// nearly all synced paths are gone from Drive at once, a failed listing is far
|
||||
/// likelier than a deliberate wipe of the folder, so the pass stops instead.
|
||||
pub(super) fn guard_remote_wipe<B, R>(
|
||||
baseline: &HashMap<String, B>,
|
||||
remote: &HashMap<String, R>,
|
||||
) -> Result<(), String> {
|
||||
let missing = baseline
|
||||
.keys()
|
||||
.filter(|rel| !remote.contains_key(*rel))
|
||||
.count();
|
||||
if looks_like_a_wipe(missing, baseline.len()) {
|
||||
return Err(format!(
|
||||
"{missing} of the {} synced paths are missing on Proton Drive; refusing to \
|
||||
delete them here. If they were deleted on purpose, remove the folder from sync; \
|
||||
to upload them again, sync the folder from scratch.",
|
||||
baseline.len()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What a remote walk saw but could not read: a node the SDK left out because
|
||||
/// it would not decrypt, or a file whose size could not be read.
|
||||
///
|
||||
/// A path missing from the walk is read as deleted on Drive, so a pass must
|
||||
/// not classify one it only failed to read. Since a node that failed to decrypt
|
||||
/// has no name, all a walk can say is that a folder's listing came back short;
|
||||
/// every path under that folder that the walk did not return is then left out,
|
||||
/// as [`Unscanned`] leaves out what the local scan could not read.
|
||||
#[derive(Default)]
|
||||
pub(super) struct RemoteGaps {
|
||||
/// Folders whose listing came back with fewer nodes than it named.
|
||||
short: HashSet<String>,
|
||||
/// Paths seen, but whose entry could not be read.
|
||||
unread: HashSet<String>,
|
||||
/// Folders whose listing failed outright, after the SDK's retries. Nothing
|
||||
/// under one is known, so all of it is left out of the pass (B271).
|
||||
unlisted: HashSet<String>,
|
||||
}
|
||||
|
||||
impl RemoteGaps {
|
||||
/// The listing of `folder` named nodes that did not come back.
|
||||
pub(super) fn short_listing(&mut self, folder: &str) {
|
||||
self.short.insert(folder.to_string());
|
||||
}
|
||||
|
||||
/// `rel` is on Drive, but the walk could not read what it needs of it.
|
||||
pub(super) fn unread(&mut self, rel: String) {
|
||||
self.unread.insert(rel);
|
||||
}
|
||||
|
||||
/// The listing of `folder` failed, so nothing under it was walked.
|
||||
pub(super) fn unlisted(&mut self, folder: &str) {
|
||||
self.unlisted.insert(folder.to_string());
|
||||
}
|
||||
|
||||
pub(super) fn is_empty(&self) -> bool {
|
||||
self.short.is_empty() && self.unread.is_empty() && self.unlisted.is_empty()
|
||||
}
|
||||
|
||||
/// Whether `rel` may be missing from `remote` only because the walk could
|
||||
/// not read it. A path the walk returned is never hidden.
|
||||
pub(super) fn hides<R>(
|
||||
&self,
|
||||
rel: &str,
|
||||
remote: &HashMap<String, R>,
|
||||
remote_dirs: &HashMap<String, NodeUid>,
|
||||
) -> bool {
|
||||
if self.is_empty() || remote.contains_key(rel) {
|
||||
return false;
|
||||
}
|
||||
if self.unread.contains(rel) {
|
||||
return true;
|
||||
}
|
||||
// The deepest folder the walk did list decides: under a short listing,
|
||||
// anything not returned may be one of the nodes it could not read.
|
||||
let mut at = parent_rel(rel);
|
||||
loop {
|
||||
if remote_dirs.contains_key(at) {
|
||||
return self.short.contains(at) || self.unlisted.contains(at);
|
||||
}
|
||||
if at.is_empty() {
|
||||
return false;
|
||||
}
|
||||
at = parent_rel(at);
|
||||
}
|
||||
}
|
||||
|
||||
/// The folders to name to the user, sorted.
|
||||
pub(super) fn report(&self) -> Vec<String> {
|
||||
let mut lines: Vec<String> = self
|
||||
.short
|
||||
.iter()
|
||||
.map(|rel| {
|
||||
let shown = if rel.is_empty() { "the folder" } else { rel };
|
||||
format!("{shown} (items on Proton Drive that cannot be read)")
|
||||
})
|
||||
.chain(
|
||||
self.unread
|
||||
.iter()
|
||||
.map(|rel| format!("{rel} (cannot be read on Proton Drive)")),
|
||||
)
|
||||
.chain(self.unlisted.iter().map(|rel| {
|
||||
format!("{rel} (could not be listed on Proton Drive; tried again next pass)")
|
||||
}))
|
||||
.collect();
|
||||
lines.sort_unstable();
|
||||
lines
|
||||
}
|
||||
}
|
||||
|
||||
/// Why the remote folder at `rel` must not be trashed although it was deleted
|
||||
/// here: something under it is new or changed on Drive since the last sync,
|
||||
/// so it is another device's work that this computer has never had.
|
||||
///
|
||||
/// Trashing the folder would take that work with it, and the next pass would
|
||||
/// then delete the copy this pass just downloaded. `None` when everything under
|
||||
/// it is exactly what the baseline recorded.
|
||||
pub(super) fn remote_subtree_changed(
|
||||
rel: &str,
|
||||
remote: &HashMap<String, RemoteItem>,
|
||||
baseline: &HashMap<String, StoredSyncEntry>,
|
||||
) -> Option<String> {
|
||||
let prefix = format!("{rel}/");
|
||||
let mut changed: Vec<&String> = remote
|
||||
.iter()
|
||||
.filter(|(path, _)| path.starts_with(&prefix))
|
||||
.filter(|(path, item)| match baseline.get(*path) {
|
||||
None => true,
|
||||
Some(_) if item.is_dir => false,
|
||||
Some(base) => {
|
||||
remote_sig(base) != Some((item.mtime, item.size))
|
||||
|| revision_moved(Some(base), item.revision.as_deref())
|
||||
}
|
||||
})
|
||||
.map(|(path, _)| path)
|
||||
.collect();
|
||||
changed.sort_unstable();
|
||||
changed
|
||||
.first()
|
||||
.map(|path| format!("{path} is new or changed on Proton Drive"))
|
||||
}
|
||||
|
||||
/// The size the baseline recorded for `rel`, if its remote signature's mtime
|
||||
/// still matches `mtime`.
|
||||
/// still matches `mtime` and its revision is still `revision`.
|
||||
///
|
||||
/// A row without a revision id, while Drive reports one, cannot vouch for the
|
||||
/// revision, so the size is read for real. Such a row comes from before
|
||||
/// schema 44, or from an upload Drive did not list yet when it was read back.
|
||||
/// Another computer's revision in the same second as that upload kept the
|
||||
/// modification time, and only the size could show it (B254). The pass then
|
||||
/// records the id on the row, so this costs one read per row.
|
||||
pub(super) fn unchanged_remote_size(
|
||||
baseline: &HashMap<String, StoredSyncEntry>,
|
||||
rel: &str,
|
||||
mtime: i64,
|
||||
revision: Option<&str>,
|
||||
) -> Option<i64> {
|
||||
match baseline.get(rel).and_then(remote_sig) {
|
||||
let base = baseline.get(rel)?;
|
||||
if revision_moved(Some(base), revision)
|
||||
|| (base.remote_revision_id.is_none() && revision.is_some())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match remote_sig(base) {
|
||||
Some((recorded, size)) if recorded == mtime => Some(size),
|
||||
_ => None,
|
||||
}
|
||||
@@ -221,6 +408,17 @@ pub(super) fn join_rel(prefix: &str, name: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether Drive's active revision `now` is not the one the baseline row was
|
||||
/// synced at. Only a revision id on both sides can say so; a row from before
|
||||
/// schema 44, or a node read without one, leaves it to the modification time
|
||||
/// and size.
|
||||
pub(super) fn revision_moved(base: Option<&StoredSyncEntry>, now: Option<&str>) -> bool {
|
||||
match (base.and_then(|b| b.remote_revision_id.as_deref()), now) {
|
||||
(Some(recorded), Some(now)) => recorded != now,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stored remote signature of a baseline row, if it has one.
|
||||
pub(super) fn remote_sig(e: &StoredSyncEntry) -> Option<(i64, i64)> {
|
||||
match (&e.remote_rev, &e.remote_hash) {
|
||||
@@ -251,7 +449,8 @@ pub(super) fn rel_to_path(rel: &str) -> PathBuf {
|
||||
rel.split('/').collect()
|
||||
}
|
||||
|
||||
/// The name for a conflict copy of `path`, e.g. `notes (sync-conflict 1700000000).txt`.
|
||||
/// The name for a conflict copy of `path`, e.g.
|
||||
/// `notes (sync-conflict laptop 1700000000).txt`.
|
||||
#[cfg(test)]
|
||||
pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
|
||||
conflict_path_with_suffix(path, stamp, 0)
|
||||
@@ -260,16 +459,17 @@ pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
|
||||
/// As [`conflict_path`], with a deterministic suffix used when that name already
|
||||
/// exists. This keeps conflict preservation from ever replacing an older copy.
|
||||
pub(super) fn conflict_path_with_suffix(path: &Path, stamp: i64, suffix: u32) -> PathBuf {
|
||||
conflict_path_as(path, crate::conflict_label(), stamp, suffix)
|
||||
}
|
||||
|
||||
/// [`conflict_path_with_suffix`] with the machine's label given.
|
||||
pub(super) fn conflict_path_as(path: &Path, label: &str, stamp: i64, suffix: u32) -> PathBuf {
|
||||
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("file");
|
||||
let ext = path.extension().and_then(|s| s.to_str());
|
||||
let suffix = if suffix == 0 {
|
||||
String::new()
|
||||
} else {
|
||||
format!("-{suffix}")
|
||||
};
|
||||
let tag = crate::conflict_tag(label, stamp, suffix);
|
||||
let name = match ext {
|
||||
Some(ext) => format!("{stem} (sync-conflict {stamp}{suffix}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {stamp}{suffix})"),
|
||||
Some(ext) => format!("{stem} (sync-conflict {tag}).{ext}"),
|
||||
None => format!("{stem} (sync-conflict {tag})"),
|
||||
};
|
||||
match path.parent() {
|
||||
Some(dir) => dir.join(name),
|
||||
@@ -290,11 +490,14 @@ pub(super) enum FilePlan {
|
||||
DeleteLocal,
|
||||
DeleteRemote,
|
||||
ForgetBaseline,
|
||||
/// First sync of a file that already exists on both sides with identical
|
||||
/// content: record the baseline without transferring anything. Without this,
|
||||
/// First sync of a file that exists on both sides at the same size, with a
|
||||
/// digest on Drive: record the baseline without transferring anything if
|
||||
/// the local file hashes the same, else keep a conflict copy. Without this,
|
||||
/// a folder synced for the first time cuts a `(sync-conflict)` copy of every
|
||||
/// pre-existing file, because a missing baseline makes both sides read as
|
||||
/// "changed" (see the mass first-sync conflict storm in docs/BUGS.md).
|
||||
/// "changed" (see the mass first-sync conflict storm in docs/BUGS.md). Size
|
||||
/// alone is not enough: a second machine joining the folder would read a
|
||||
/// different file of the same size as in step (B248).
|
||||
AdoptBaseline,
|
||||
}
|
||||
|
||||
@@ -313,8 +516,13 @@ pub(super) fn plan_file(
|
||||
// left the file the same length, so it was never uploaded.
|
||||
baseline.is_none_or(|base| !LocalSig::from(item).same_content(&LocalSig::from(base)))
|
||||
});
|
||||
// Another client's new revision shows in its id even when the link's
|
||||
// modification time and the size come out the same.
|
||||
let remote_changed = remote.is_some_and(|item| {
|
||||
baseline.is_none_or(|base| remote_sig(base) != Some((item.mtime, item.size)))
|
||||
baseline.is_none_or(|base| {
|
||||
remote_sig(base) != Some((item.mtime, item.size))
|
||||
|| revision_moved(Some(base), item.revision.as_deref())
|
||||
})
|
||||
});
|
||||
|
||||
match (local, remote) {
|
||||
@@ -324,10 +532,13 @@ pub(super) fn plan_file(
|
||||
(false, true) => FilePlan::Download,
|
||||
// With a baseline, both-changed is a genuine concurrent edit and must
|
||||
// be preserved as a conflict. Without one, "both changed" only means
|
||||
// this is the first sync and there was nothing to compare to: adopt
|
||||
// the baseline when the sizes already match rather than conflict-copy
|
||||
// an already-identical file. Divergent sizes stay a real conflict.
|
||||
(true, true) if baseline.is_none() && l.size == r.size => FilePlan::AdoptBaseline,
|
||||
// this is the first sync and there was nothing to compare to: when the
|
||||
// sizes match and Drive has a digest to compare against, the apply
|
||||
// step hashes the local file before adopting it. Divergent sizes, or
|
||||
// no digest, stay a real conflict: a missing digest proves nothing.
|
||||
(true, true) if baseline.is_none() && l.size == r.size && r.sha1.is_some() => {
|
||||
FilePlan::AdoptBaseline
|
||||
}
|
||||
(true, true) => FilePlan::Conflict,
|
||||
},
|
||||
(Some(_), None) if baseline.is_none() || local_changed => FilePlan::UploadNew,
|
||||
@@ -359,6 +570,8 @@ mod file_plan_tests {
|
||||
is_dir: false,
|
||||
mtime,
|
||||
size,
|
||||
revision: None,
|
||||
sha1: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -376,6 +589,7 @@ mod file_plan_tests {
|
||||
local_size,
|
||||
remote_rev: Some(remote_mtime.to_string()),
|
||||
remote_hash: Some(remote_size.to_string()),
|
||||
remote_revision_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,12 +711,28 @@ mod file_plan_tests {
|
||||
fn first_sync_of_an_identical_file_adopts_the_baseline_not_a_conflict() {
|
||||
// No baseline yet (first sync). Same size on both sides is a pre-existing
|
||||
// identical file, not a concurrent edit: adopt it rather than cut a
|
||||
// conflict copy of every file in the folder.
|
||||
// conflict copy of every file in the folder. The apply step still
|
||||
// hashes the local file before it records anything.
|
||||
let local = local(10, 40);
|
||||
let remote = RemoteItem {
|
||||
sha1: Some("da39a3ee".into()),
|
||||
..remote(30, 40)
|
||||
};
|
||||
assert_eq!(
|
||||
plan_file(Some(&local), Some(&remote), None),
|
||||
FilePlan::AdoptBaseline
|
||||
);
|
||||
}
|
||||
|
||||
/// A file on Drive without a digest cannot be compared without a download,
|
||||
/// and the same size is not the same content (B248).
|
||||
#[test]
|
||||
fn first_sync_without_a_remote_digest_is_a_conflict() {
|
||||
let local = local(10, 40);
|
||||
let remote = remote(30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&local), Some(&remote), None),
|
||||
FilePlan::AdoptBaseline
|
||||
FilePlan::Conflict
|
||||
);
|
||||
}
|
||||
|
||||
@@ -518,6 +748,73 @@ mod file_plan_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Another computer's new revision can leave the modification time and
|
||||
/// the size as they were. The revision id still tells it apart (B249).
|
||||
#[test]
|
||||
fn a_new_remote_revision_is_a_change_even_at_the_same_mtime_and_size() {
|
||||
let mut base = baseline(10, 20, 30, 40);
|
||||
base.remote_revision_id = Some("r1".into());
|
||||
let unedited = local(10, 20);
|
||||
let same = RemoteItem {
|
||||
revision: Some("r1".into()),
|
||||
..remote(30, 40)
|
||||
};
|
||||
let newer = RemoteItem {
|
||||
revision: Some("r2".into()),
|
||||
..remote(30, 40)
|
||||
};
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&same), Some(&base)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&newer), Some(&base)),
|
||||
FilePlan::Download
|
||||
);
|
||||
let edited = local(11, 20);
|
||||
assert_eq!(
|
||||
plan_file(Some(&edited), Some(&newer), Some(&base)),
|
||||
FilePlan::Conflict
|
||||
);
|
||||
|
||||
// A row from before schema 44 has no id to compare, and a node read
|
||||
// without one says nothing: both fall back to the signature.
|
||||
let old = baseline(10, 20, 30, 40);
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&newer), Some(&old)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
assert_eq!(
|
||||
plan_file(Some(&unedited), Some(&remote(30, 40)), Some(&base)),
|
||||
FilePlan::Unchanged
|
||||
);
|
||||
}
|
||||
|
||||
/// The walk reuses the recorded size only while the revision is the one
|
||||
/// the baseline holds.
|
||||
#[test]
|
||||
fn a_moved_revision_makes_the_walk_read_the_size_again() {
|
||||
let mut base = baseline(10, 20, 30, 40);
|
||||
base.remote_revision_id = Some("r1".into());
|
||||
let rows = HashMap::from([("file".to_string(), base)]);
|
||||
assert_eq!(
|
||||
unchanged_remote_size(&rows, "file", 30, Some("r1")),
|
||||
Some(40)
|
||||
);
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 30, None), Some(40));
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 30, Some("r2")), None);
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 31, Some("r1")), None);
|
||||
}
|
||||
|
||||
/// A row without a revision id reuses its size only while Drive gives no
|
||||
/// id either (B254).
|
||||
#[test]
|
||||
fn a_row_without_a_revision_makes_the_walk_read_the_size_again() {
|
||||
let rows = HashMap::from([("file".to_string(), baseline(10, 20, 30, 40))]);
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 30, Some("r1")), None);
|
||||
assert_eq!(unchanged_remote_size(&rows, "file", 30, None), Some(40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_remote_signature_is_treated_as_changed() {
|
||||
let local = local(10, 20);
|
||||
@@ -530,4 +827,153 @@ mod file_plan_tests {
|
||||
FilePlan::Download
|
||||
);
|
||||
}
|
||||
|
||||
fn uid(id: &str) -> NodeUid {
|
||||
NodeUid::new(VolumeId::from("v"), LinkId::from(id))
|
||||
}
|
||||
|
||||
fn folder() -> RemoteItem {
|
||||
RemoteItem {
|
||||
is_dir: true,
|
||||
mtime: 0,
|
||||
size: 0,
|
||||
..remote(0, 0)
|
||||
}
|
||||
}
|
||||
|
||||
/// An empty or failed remote listing must not read as "Drive deleted
|
||||
/// everything" and delete the local copies.
|
||||
#[test]
|
||||
fn the_remote_wipe_guard_blocks_a_total_disappearance() {
|
||||
let base = HashMap::from([("a".to_string(), ()), ("b".to_string(), ())]);
|
||||
let none: HashMap<String, RemoteItem> = HashMap::new();
|
||||
assert!(guard_remote_wipe(&base, &none).is_err());
|
||||
let one = HashMap::from([("b".to_string(), remote(1, 1))]);
|
||||
assert!(guard_remote_wipe(&base, &one).is_ok());
|
||||
let empty: HashMap<String, ()> = HashMap::new();
|
||||
assert!(guard_remote_wipe(&empty, &none).is_ok());
|
||||
}
|
||||
|
||||
/// One path left of a large folder no longer clears either guard: most
|
||||
/// of it gone at once is a lost tree, not a cleanup (B270).
|
||||
#[test]
|
||||
fn one_survivor_of_a_large_folder_does_not_clear_the_wipe_guards() {
|
||||
let base: HashMap<String, ()> = (0..40).map(|n| (format!("f{n}.txt"), ())).collect();
|
||||
let survivors = |n: usize| -> HashMap<String, ()> {
|
||||
(0..n).map(|n| (format!("f{n}.txt"), ())).collect()
|
||||
};
|
||||
assert!(guard_local_wipe(&base, &survivors(1)).is_err());
|
||||
assert!(guard_remote_wipe(&base, &survivors(1)).is_err());
|
||||
// 36 of 40 gone is nine tenths: still refused.
|
||||
assert!(guard_local_wipe(&base, &survivors(4)).is_err());
|
||||
// 35 of 40 gone is under nine tenths: a pass may delete that much.
|
||||
assert!(guard_local_wipe(&base, &survivors(5)).is_ok());
|
||||
assert!(guard_remote_wipe(&base, &survivors(5)).is_ok());
|
||||
let reason = guard_local_wipe(&base, &survivors(1)).unwrap_err();
|
||||
assert!(reason.starts_with("39 of the 40 synced paths"), "{reason}");
|
||||
}
|
||||
|
||||
/// A small folder still loses most of its files to an ordinary cleanup, so
|
||||
/// only a total loss stops it.
|
||||
#[test]
|
||||
fn a_small_folder_trips_the_wipe_guards_only_when_everything_is_gone() {
|
||||
let base: HashMap<String, ()> = (0..19).map(|n| (format!("f{n}.txt"), ())).collect();
|
||||
let one: HashMap<String, ()> = [("f0.txt".to_string(), ())].into_iter().collect();
|
||||
assert!(guard_local_wipe(&base, &one).is_ok());
|
||||
assert!(guard_remote_wipe(&base, &one).is_ok());
|
||||
let none: HashMap<String, ()> = HashMap::new();
|
||||
assert!(guard_local_wipe(&base, &none).is_err());
|
||||
}
|
||||
|
||||
/// A node that failed to decrypt has no name, so everything a short
|
||||
/// listing did not return is left out, and nothing else.
|
||||
#[test]
|
||||
fn a_short_remote_listing_hides_what_it_did_not_return() {
|
||||
let remote = HashMap::from([
|
||||
("docs".to_string(), folder()),
|
||||
("docs/a.txt".to_string(), remote(1, 1)),
|
||||
("pics".to_string(), folder()),
|
||||
]);
|
||||
let dirs = HashMap::from([
|
||||
(String::new(), uid("root")),
|
||||
("docs".to_string(), uid("docs")),
|
||||
("pics".to_string(), uid("pics")),
|
||||
]);
|
||||
let mut gaps = RemoteGaps::default();
|
||||
assert!(!gaps.hides("docs/b.txt", &remote, &dirs));
|
||||
gaps.short_listing("docs");
|
||||
// Returned paths stand; unreturned ones under the short folder hide,
|
||||
// and so does anything under a subfolder it failed to return.
|
||||
assert!(!gaps.hides("docs/a.txt", &remote, &dirs));
|
||||
assert!(gaps.hides("docs/b.txt", &remote, &dirs));
|
||||
assert!(gaps.hides("docs/sub/deep.txt", &remote, &dirs));
|
||||
// A complete sibling listing still reads deletions as deletions.
|
||||
assert!(!gaps.hides("pics/gone.jpg", &remote, &dirs));
|
||||
assert!(!gaps.hides("top.txt", &remote, &dirs));
|
||||
gaps.short_listing("");
|
||||
assert!(gaps.hides("top.txt", &remote, &dirs));
|
||||
}
|
||||
|
||||
/// A folder whose listing failed was walked no further: nothing under it
|
||||
/// reads as deleted on Drive, and a sibling that listed still does
|
||||
/// (`docs/BUGS.md` B271).
|
||||
#[test]
|
||||
fn a_folder_that_failed_to_list_hides_everything_under_it() {
|
||||
let remote = HashMap::from([
|
||||
("docs".to_string(), folder()),
|
||||
("pics".to_string(), folder()),
|
||||
]);
|
||||
let dirs = HashMap::from([
|
||||
(String::new(), uid("root")),
|
||||
("docs".to_string(), uid("docs")),
|
||||
("pics".to_string(), uid("pics")),
|
||||
]);
|
||||
let mut gaps = RemoteGaps::default();
|
||||
gaps.unlisted("docs");
|
||||
assert!(gaps.hides("docs/a.txt", &remote, &dirs));
|
||||
assert!(gaps.hides("docs/sub/deep.txt", &remote, &dirs));
|
||||
assert!(!gaps.hides("docs", &remote, &dirs));
|
||||
assert!(!gaps.hides("pics/gone.jpg", &remote, &dirs));
|
||||
assert_eq!(
|
||||
gaps.report(),
|
||||
["docs (could not be listed on Proton Drive; tried again next pass)"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_whose_size_could_not_be_read_is_hidden() {
|
||||
let remote: HashMap<String, RemoteItem> = HashMap::new();
|
||||
let dirs = HashMap::from([(String::new(), uid("root"))]);
|
||||
let mut gaps = RemoteGaps::default();
|
||||
gaps.unread("a.txt".to_string());
|
||||
assert!(gaps.hides("a.txt", &remote, &dirs));
|
||||
assert!(!gaps.hides("b.txt", &remote, &dirs));
|
||||
assert_eq!(gaps.report(), ["a.txt (cannot be read on Proton Drive)"]);
|
||||
}
|
||||
|
||||
/// A folder deleted here holds another device's new work on Drive, so
|
||||
/// trashing it would lose that work.
|
||||
#[test]
|
||||
fn a_remote_folder_with_new_or_changed_files_is_not_trashed() {
|
||||
let mut base = HashMap::from([
|
||||
("x".to_string(), baseline(0, 0, 0, 0)),
|
||||
("x/old.txt".to_string(), baseline(1, 1, 5, 6)),
|
||||
]);
|
||||
let mut remote_tree = HashMap::from([
|
||||
("x".to_string(), folder()),
|
||||
("x/old.txt".to_string(), remote(5, 6)),
|
||||
("xy/new.txt".to_string(), remote(1, 1)),
|
||||
]);
|
||||
// Unchanged under x; a name-prefixed sibling does not count.
|
||||
assert_eq!(remote_subtree_changed("x", &remote_tree, &base), None);
|
||||
|
||||
remote_tree.insert("x/old.txt".to_string(), remote(9, 6));
|
||||
assert!(remote_subtree_changed("x", &remote_tree, &base).is_some());
|
||||
|
||||
remote_tree.insert("x/old.txt".to_string(), remote(5, 6));
|
||||
remote_tree.insert("x/sub".to_string(), folder());
|
||||
assert!(remote_subtree_changed("x", &remote_tree, &base).is_some());
|
||||
base.insert("x/sub".to_string(), baseline(0, 0, 0, 0));
|
||||
assert_eq!(remote_subtree_changed("x", &remote_tree, &base), None);
|
||||
}
|
||||
}
|
||||
@@ -64,11 +64,19 @@ fn send_abstract(socket: &UnixDatagram, name: &str, message: &str) -> bool {
|
||||
/// so `systemctl --user start` returns when the mount actually works rather than
|
||||
/// when the process exists.
|
||||
pub(crate) fn ready() {
|
||||
if notify("READY=1\n") {
|
||||
if notify("READY=1\nSTATUS=Mounted\n") {
|
||||
debug!("notified systemd: ready");
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the one-line status `systemctl --user status` shows for the unit, such
|
||||
/// as why the mount keeps failing (`docs/BUGS.md` B263).
|
||||
pub fn status(text: &str) {
|
||||
// A newline would end the `STATUS=` line and start another variable.
|
||||
let line = text.replace('\n', " ");
|
||||
notify(&format!("STATUS={line}\n"));
|
||||
}
|
||||
|
||||
/// Tell the manager the daemon is still healthy.
|
||||
pub(crate) fn watchdog_ping() {
|
||||
let _ = notify("WATCHDOG=1\n");
|
||||
@@ -94,6 +102,37 @@ pub fn idle_sleep(duration: Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a mount may take to start before the watchdog counts it as hung.
|
||||
///
|
||||
/// Startup has no control socket to probe yet, and on a large account it is
|
||||
/// long: hydrating 48 000 nodes took 70 s on 2026-11-12, and the secondary
|
||||
/// mounts followed. Under the 2-minute `WatchdogSec` that run was aborted just
|
||||
/// before its first ping, and left six dead mounts behind (`docs/BUGS.md` B279).
|
||||
const STARTUP_WATCHDOG: Duration = Duration::from_secs(600);
|
||||
|
||||
/// Give the mount about to start [`STARTUP_WATCHDOG`] before the watchdog fires.
|
||||
///
|
||||
/// Sends `WATCHDOG_USEC=`, which replaces the unit's timeout and restarts its
|
||||
/// timer. The supervisor puts the configured timeout back with
|
||||
/// [`restore_watchdog`] once it can prove the daemon answers. A startup that
|
||||
/// wedges for good is still aborted, only later.
|
||||
pub fn stretch_watchdog_for_startup() {
|
||||
if watchdog_period().is_some() {
|
||||
notify(&format!("WATCHDOG_USEC={}\n", STARTUP_WATCHDOG.as_micros()));
|
||||
}
|
||||
}
|
||||
|
||||
/// Put back the timeout the unit configured, undoing
|
||||
/// [`stretch_watchdog_for_startup`].
|
||||
pub(crate) fn restore_watchdog() {
|
||||
if watchdog_period().is_none() {
|
||||
return;
|
||||
}
|
||||
if let Ok(usec) = std::env::var("WATCHDOG_USEC") {
|
||||
notify(&format!("WATCHDOG_USEC={usec}\n"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Tell the manager the daemon is stopping, so a slow teardown is not mistaken
|
||||
/// for a hang by the watchdog it is racing.
|
||||
pub(crate) fn stopping() {
|
||||
|
||||
+919
-178
File diff suppressed because it is too large.
Load diff
@@ -46,6 +46,8 @@ struct JobEntry {
|
||||
detail: Mutex<String>,
|
||||
done: AtomicU64,
|
||||
total: AtomicU64,
|
||||
/// Housekeeping that moves nothing; see [`JobItem::background`].
|
||||
background: bool,
|
||||
}
|
||||
|
||||
/// The set of transfers and jobs currently in flight. Cloned `Arc`-style across
|
||||
@@ -123,6 +125,11 @@ impl TransferRegistry {
|
||||
self.download_limit.set(download);
|
||||
}
|
||||
|
||||
/// The upload cap in bytes per second, `0` when there is none.
|
||||
pub fn upload_cap(&self) -> u64 {
|
||||
self.upload_limit.rate.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn limit(&self, direction: TransferDirection) -> &RateLimit {
|
||||
match direction {
|
||||
TransferDirection::Upload => &self.upload_limit,
|
||||
@@ -160,12 +167,25 @@ impl TransferRegistry {
|
||||
/// that carries its progress and deregisters the job when dropped. The job
|
||||
/// starts indeterminate; give it a `total` once one is known.
|
||||
pub fn begin_job(self: &Arc<Self>, title: impl Into<String>) -> JobGuard {
|
||||
self.register_job(title.into(), false)
|
||||
}
|
||||
|
||||
/// [`TransferRegistry::begin_job`] for housekeeping the user did not ask
|
||||
/// for and that changes nothing in their files, such as an index rebuild or
|
||||
/// a cache check. It is reported, but front ends don't call the drive busy
|
||||
/// over it.
|
||||
pub fn begin_background_job(self: &Arc<Self>, title: impl Into<String>) -> JobGuard {
|
||||
self.register_job(title.into(), true)
|
||||
}
|
||||
|
||||
fn register_job(self: &Arc<Self>, title: String, background: bool) -> JobGuard {
|
||||
let id = self.next.fetch_add(1, Ordering::Relaxed);
|
||||
let entry = Arc::new(JobEntry {
|
||||
title: title.into(),
|
||||
title,
|
||||
detail: Mutex::new(String::new()),
|
||||
done: AtomicU64::new(0),
|
||||
total: AtomicU64::new(0),
|
||||
background,
|
||||
});
|
||||
self.jobs.lock().insert(id, entry.clone());
|
||||
JobGuard {
|
||||
@@ -191,6 +211,7 @@ impl TransferRegistry {
|
||||
detail: e.detail.lock().clone(),
|
||||
done: e.done.load(Ordering::Relaxed),
|
||||
total: e.total.load(Ordering::Relaxed),
|
||||
background: e.background,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -22,6 +22,8 @@ name = "pdfs-prompt"
|
||||
path = "src/prompt.rs"
|
||||
|
||||
[dependencies]
|
||||
proton-gallery-rs.workspace = true
|
||||
base64.workspace = true
|
||||
pdfs-core.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
@@ -40,4 +42,4 @@ pangocairo = "0.22"
|
||||
# Map tiles for the opt-in online map. The same build proton-sdk already pulls in.
|
||||
reqwest = { version = "0.13", default-features = false, features = ["rustls", "webpki-roots"] }
|
||||
# The system libintl (glibc) rather than a bundled gettext build.
|
||||
gettext-rs = { version = "0.7", features = ["gettext-system"] }
|
||||
gettext-rs = { version = "0.8", features = ["gettext-system"] }
|
||||
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
|
||||
<!-- Album: a photo in front of the edge of another, a stack of them. -->
|
||||
<path fill="#222222" d="M 4,1 H 13.5 A 1.5,1.5 0 0 1 15,2.5 V 12 H 13.5 V 2.5 H 4 Z"/>
|
||||
<path fill="#222222" fill-rule="evenodd" d="M 2.5,4 H 10.5 A 1.5,1.5 0 0 1 12,5.5 V 13.5 A 1.5,1.5 0 0 1 10.5,15 H 2.5 A 1.5,1.5 0 0 1 1,13.5 V 5.5 A 1.5,1.5 0 0 1 2.5,4 Z M 2.5,5.5 V 13.5 H 10.5 V 5.5 Z"/>
|
||||
<path fill="#222222" d="M 3.5,12.5 L 5.5,9.2 L 7.2,11.2 L 8.2,10.2 L 9.5,12.5 Z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 550 B |
@@ -1,5 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
|
||||
<!-- Main menu: three bars. Bundled because themes such as Papirus draw
|
||||
open-menu-symbolic as three dots, the same as every row's "more" menu. -->
|
||||
<path fill="#222222" d="M 2,2.5 H 14 A 1,1 0 0 1 14,4.5 H 2 A 1,1 0 0 1 2,2.5 Z M 2,7 H 14 A 1,1 0 0 1 14,9 H 2 A 1,1 0 0 1 2,7 Z M 2,11.5 H 14 A 1,1 0 0 1 14,13.5 H 2 A 1,1 0 0 1 2,11.5 Z"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 441 B |
@@ -0,0 +1,8 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
|
||||
<!-- Photos: a framed landscape with a sun. Stock themes only have
|
||||
image-x-generic, a file type icon, which reads as "an image file"
|
||||
rather than "your photo library". -->
|
||||
<path fill="#222222" fill-rule="evenodd" d="M 3,2 H 13 A 2,2 0 0 1 15,4 V 12 A 2,2 0 0 1 13,14 H 3 A 2,2 0 0 1 1,12 V 4 A 2,2 0 0 1 3,2 Z M 3,3.5 A 0.5,0.5 0 0 0 2.5,4 V 12 A 0.5,0.5 0 0 0 3,12.5 H 13 A 0.5,0.5 0 0 0 13.5,12 V 4 A 0.5,0.5 0 0 0 13,3.5 Z"/>
|
||||
<path fill="#222222" d="M 3.5,11.5 L 6.5,7 L 9,10.2 L 10.5,8.6 L 12.5,11.5 Z"/>
|
||||
<circle fill="#222222" cx="10.5" cy="5.8" r="1.3"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 673 B |
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
|
||||
<!-- Slideshow: a play mark inside a photo frame, so it is not mistaken for
|
||||
playing the video on screen. -->
|
||||
<path fill="#222222" fill-rule="evenodd" d="M 3,2 H 13 A 2,2 0 0 1 15,4 V 12 A 2,2 0 0 1 13,14 H 3 A 2,2 0 0 1 1,12 V 4 A 2,2 0 0 1 3,2 Z M 3,3.5 A 0.5,0.5 0 0 0 2.5,4 V 12 A 0.5,0.5 0 0 0 3,12.5 H 13 A 0.5,0.5 0 0 0 13.5,12 V 4 A 0.5,0.5 0 0 0 13,3.5 Z"/>
|
||||
<path fill="#222222" d="M 6.5,5.2 L 11,8 L 6.5,10.8 Z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 527 B |
@@ -14,8 +14,10 @@
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-details-pane-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-link-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-map-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-menu-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-people-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-photos-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-album-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-slideshow-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-share-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-sync-symbolic.svg</file>
|
||||
<file preprocess="xml-stripblanks">scalable/actions/pdfs-synced-symbolic.svg</file>
|
||||
|
||||
@@ -18,7 +18,9 @@
|
||||
.sidebar-status image.warning { color: var(--warning-color); }
|
||||
.sidebar-status image.error { color: var(--error-color); }
|
||||
.sidebar-quota progressbar trough, .sidebar-quota progressbar progress { min-height: 4px; }
|
||||
.sidebar-account { padding: 4px 4px 4px 8px; }
|
||||
/* The account row is the main menu's button: a row, not a pushbutton, so
|
||||
* the name keeps the text weight. */
|
||||
button.sidebar-account { padding: 4px 8px; font-weight: normal; }
|
||||
/* A count of what waits on a page, at the end of its sidebar row */
|
||||
.sidebar-badge {
|
||||
min-width: 10px;
|
||||
@@ -48,7 +50,10 @@
|
||||
.badge-cloud { color: color-mix(in srgb, currentColor 55%, transparent); }
|
||||
.badge-issue { color: var(--error-color); }
|
||||
.browser-statusbar { background-color: alpha(currentColor, 0.025); }
|
||||
scale.browser-status-meter trough { min-width: 104px; min-height: 6px; }
|
||||
/* The folder on screen, last in the breadcrumb trail: a flat button's side
|
||||
* padding, so the "›" before it is spaced like the ones between buttons. */
|
||||
.crumb-current { padding: 0 10px; }
|
||||
scale.browser-status-meter trough { min-width: 140px; min-height: 6px; }
|
||||
.bulk-bar { padding: 6px 8px; border-radius: 12px; background: color-mix(in srgb, var(--accent-bg-color) 12%, transparent); }
|
||||
.dropzone { border: 2px dashed alpha(currentColor, 0.25); border-radius: 16px; padding: 28px 18px; transition: background 160ms ease, border-color 160ms ease; }
|
||||
.dropzone-active { border-color: var(--accent-bg-color); background: color-mix(in srgb, var(--accent-bg-color) 10%, transparent); }
|
||||
@@ -56,14 +61,28 @@ scale.browser-status-meter trough { min-width: 104px; min-height: 6px; }
|
||||
|
||||
/* Photo viewer: always light-on-dark, whatever the system style */
|
||||
.photo-viewer-window { background-color: #111014; }
|
||||
.viewer-top-bar { background: linear-gradient(to bottom, rgba(0, 0, 0, 0.75), rgba(0, 0, 0, 0)); padding: 10px 16px 28px 20px; color: white; }
|
||||
.media-player-window video { background-color: black; }
|
||||
.media-player-note { color: rgba(255, 255, 255, 0.5); }
|
||||
.media-player-note .file-thumbnail { border-radius: 12px; }
|
||||
.viewer-top-bar { background: linear-gradient(to bottom, rgba(0, 0, 0, 0.85), rgba(0, 0, 0, 0.45) 60%, rgba(0, 0, 0, 0)); padding: 10px 16px 40px 20px; color: white; }
|
||||
.viewer-title { font-weight: 700; font-size: 1.05rem; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); color: white; }
|
||||
.viewer-counter { font-size: 0.8rem; color: rgba(255, 255, 255, 0.72); text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); }
|
||||
.viewer-action-btn { color: white; background-color: rgba(255, 255, 255, 0.12); border-radius: 50%; min-width: 34px; min-height: 34px; margin-left: 4px; }
|
||||
.viewer-action-btn:hover { background-color: rgba(255, 255, 255, 0.28); color: white; }
|
||||
/* Dark discs with a white glyph read on a white photo as well as on a black one. */
|
||||
.viewer-action-btn { color: white; background-color: rgba(0, 0, 0, 0.5); border-radius: 50%; min-width: 36px; min-height: 36px; margin-left: 4px; -gtk-icon-shadow: 0 1px 2px rgba(0, 0, 0, 0.6); }
|
||||
.viewer-action-btn:hover { background-color: rgba(0, 0, 0, 0.75); color: white; }
|
||||
.viewer-action-btn:checked { background-color: var(--accent-bg-color); color: var(--accent-fg-color); }
|
||||
.viewer-close-btn { background-color: rgba(255, 255, 255, 0.2); margin-left: 10px; }
|
||||
.viewer-close-btn { margin-left: 12px; }
|
||||
.viewer-close-btn:hover { background-color: var(--destructive-bg-color); color: var(--destructive-fg-color); }
|
||||
.viewer-group-pill { color: white; background-color: rgba(0, 0, 0, 0.5); border-radius: 999px; min-height: 36px; padding: 0 14px; font-weight: 700; font-size: 0.85rem; }
|
||||
.viewer-group-pill:hover { background-color: rgba(0, 0, 0, 0.75); color: white; }
|
||||
.viewer-group-pill.lit { background-color: var(--accent-bg-color); color: var(--accent-fg-color); }
|
||||
/* The thumbnail strip along the lightbox's bottom edge, the photo on screen ringed in the accent. */
|
||||
.viewer-strip-bar { background: linear-gradient(to top, rgba(0, 0, 0, 0.8), rgba(0, 0, 0, 0)); padding: 28px 12px 10px 12px; }
|
||||
.viewer-strip { background: none; }
|
||||
.viewer-strip > row { padding: 2px; margin: 0 2px; border-radius: 8px; background: none; }
|
||||
.viewer-strip > row:hover { background-color: rgba(255, 255, 255, 0.15); }
|
||||
.viewer-strip > row:selected { background-color: var(--accent-bg-color); }
|
||||
.viewer-strip-thumb { border-radius: 6px; background-color: rgba(255, 255, 255, 0.08); }
|
||||
.viewer-nav-btn { background-color: rgba(0, 0, 0, 0.45); color: white; margin: 20px; min-width: 44px; min-height: 44px; border-radius: 50%; opacity: 0.7; transition: opacity 150ms ease, background-color 150ms ease; }
|
||||
.viewer-nav-btn:hover { background-color: rgba(0, 0, 0, 0.85); color: white; opacity: 1; }
|
||||
.viewer-nav-btn:disabled { opacity: 0; }
|
||||
@@ -75,7 +94,12 @@ scale.browser-status-meter trough { min-width: 104px; min-height: 6px; }
|
||||
.viewer-info-panel { background-color: var(--window-bg-color); border-left: 1px solid alpha(currentColor, 0.12); }
|
||||
|
||||
/* Photos grid */
|
||||
.gallery-day { font-size: 0.82rem; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; color: alpha(currentColor, 0.65); padding: 2px 2px 6px 2px; }
|
||||
.gallery-day { font-size: 1.1rem; font-weight: 700; padding: 4px 2px 8px 2px; }
|
||||
/* The check that picks a whole section: on hover, and always while selecting. */
|
||||
.gallery-day-check { opacity: 0; transition: opacity 160ms ease; min-width: 24px; min-height: 24px; margin-bottom: 4px; }
|
||||
.gallery-heading:hover .gallery-day-check, .gallery-heading.selecting .gallery-day-check, .gallery-day-check:focus-visible { opacity: 1; }
|
||||
/* The section at the top of the grid, floating once its heading scrolled off. */
|
||||
.gallery-float-heading { font-weight: 700; padding: 4px 12px; border-radius: 999px; background-color: color-mix(in srgb, var(--popover-bg-color) 92%, transparent); color: var(--popover-fg-color); box-shadow: 0 1px 4px rgba(0, 0, 0, 0.25); }
|
||||
.photo-tile { padding: 0; margin: 0; min-height: 0; min-width: 0; border: none; border-radius: 4px; background: none; box-shadow: none; }
|
||||
.photo-tile:focus { outline: 2px solid var(--accent-color); outline-offset: -2px; }
|
||||
.photo-thumb { transition: transform 220ms ease; }
|
||||
@@ -89,16 +113,21 @@ scale.browser-status-meter trough { min-width: 104px; min-height: 6px; }
|
||||
.photo-video-badge { color: white; background: rgba(0, 0, 0, 0.45); border-radius: 999px; padding: 8px; min-width: 20px; min-height: 20px; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.5); transition: background 160ms ease; }
|
||||
.photo-video-badge-small { padding: 3px; min-width: 0; min-height: 0; }
|
||||
.photo-tile:hover .photo-video-badge { background: color-mix(in srgb, var(--accent-bg-color) 85%, transparent); }
|
||||
.photo-favorite-badge { color: white; -gtk-icon-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); }
|
||||
.photo-group-badge { font-size: 0.68rem; font-weight: 700; letter-spacing: 0.04em; color: white; background: rgba(0, 0, 0, 0.55); border-radius: 6px; padding: 1px 5px; margin: 6px; text-shadow: 0 1px 2px rgba(0, 0, 0, 0.9); }
|
||||
.memory-card { padding: 0; border-radius: 12px; }
|
||||
.memory-cover { border-radius: 12px; background: alpha(currentColor, 0.06); }
|
||||
.memory-label { font-weight: 700; color: white; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); padding: 22px 10px 6px 10px; background: linear-gradient(to top, rgba(0, 0, 0, 0.6), rgba(0, 0, 0, 0)); }
|
||||
/* The button handles hover and clicks. The FlowBox cell can be wider than it. */
|
||||
flowbox.photo-album-grid > flowboxchild:hover,
|
||||
flowbox.photo-album-grid > flowboxchild:active { background: transparent; }
|
||||
.album-card { padding: 0; border-radius: 16px; transition: background 180ms ease; }
|
||||
.album-card:hover { background: alpha(currentColor, 0.07); }
|
||||
.album-cover { border-radius: 16px; background: alpha(currentColor, 0.06); box-shadow: 0 4px 16px rgba(0, 0, 0, 0.32); transition: box-shadow 180ms ease; }
|
||||
.album-card:hover .album-cover { box-shadow: 0 10px 26px rgba(0, 0, 0, 0.5); }
|
||||
.album-card:hover .photo-thumb { transform: scale(1.05); }
|
||||
.view-switch button, .view-switch toggle { padding-left: 18px; padding-right: 18px; }
|
||||
listview.drive-gallery-rows > row { padding: 0; }
|
||||
.view-switch button, .view-switch toggle { padding-left: 12px; padding-right: 12px; }
|
||||
|
||||
/* Shared with Me: what a share lets me do, as a tinted pill beside the name. */
|
||||
.role-pill { padding: 2px 10px; border-radius: 999px; font-weight: 600; color: var(--accent-color); background-color: color-mix(in srgb, var(--accent-bg-color) 16%, transparent); }
|
||||
|
||||
@@ -10,9 +10,14 @@
|
||||
//! `dead_code` allowances below are for — not code nobody uses.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use pdfs_core::control::{PhotoKind, SearchHit};
|
||||
|
||||
/// How long a `stat` of the mount may take before the file counts as not
|
||||
/// there. A frozen mount never answers, and every caller has a fallback.
|
||||
const STAT_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum DriveActivation {
|
||||
Folder,
|
||||
@@ -55,18 +60,32 @@ pub(crate) fn mounted_target(mountpoint: &Path, hit: &SearchHit) -> Option<PathB
|
||||
present(mounted_or_relative(mountpoint, hit))
|
||||
}
|
||||
|
||||
/// [`mounted_target`] for callers holding a mountpoint-relative path rather
|
||||
/// than a search hit — the browser's listing entries, which are by definition
|
||||
/// paths in the primary mount's tree.
|
||||
#[allow(dead_code)] // browser only
|
||||
pub(crate) fn mounted_target_rel(mountpoint: &Path, relative_path: &str) -> Option<PathBuf> {
|
||||
present(mounted_path(mountpoint, relative_path))
|
||||
}
|
||||
|
||||
fn present(path: PathBuf) -> Option<PathBuf> {
|
||||
path.symlink_metadata().is_ok().then_some(path)
|
||||
}
|
||||
|
||||
/// Whether anything is at `path`, asked on a thread of its own and given up
|
||||
/// on after [`STAT_TIMEOUT`]. `None` means "not there" or "no answer".
|
||||
///
|
||||
/// For the GTK binaries, whose main thread must not `stat` the mount: a
|
||||
/// mount that hangs hung the whole window with it. The thread of a `stat`
|
||||
/// that never returns is left behind; there is no way to cancel one.
|
||||
#[allow(dead_code)] // GTK binaries only
|
||||
pub(crate) async fn present_within(path: PathBuf) -> Option<PathBuf> {
|
||||
let (tx, rx) = async_channel::bounded(1);
|
||||
std::thread::spawn(move || {
|
||||
let _ = tx.send_blocking(present(path));
|
||||
});
|
||||
match gtk4::glib::future_with_timeout(STAT_TIMEOUT, rx.recv()).await {
|
||||
Ok(Ok(target)) => target,
|
||||
Ok(Err(_)) => None,
|
||||
Err(_) => {
|
||||
tracing::warn!("the mount did not answer in time");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The mount path for a hit, whether or not anything is there — for callers
|
||||
/// that have no fallback (a folder cannot be materialised) or already know the
|
||||
/// entry came from a mount listing.
|
||||
|
||||
@@ -100,6 +100,9 @@ fn on_feed(ui: &Rc<Ui>, state: &Rc<FeedState>, feed: Feed, hidden: bool) {
|
||||
follow_takeout(ui, state, &jobs);
|
||||
}
|
||||
Feed::Event(Event::Changed { topics }) if hidden => {
|
||||
if topics.contains(&Topic::Photos) {
|
||||
check_photo_viewer(ui);
|
||||
}
|
||||
state.deferred.borrow_mut().extend(topics);
|
||||
}
|
||||
Feed::Event(Event::Changed { topics }) => follow_changes(ui, state, &topics),
|
||||
@@ -128,10 +131,57 @@ fn follow_changes(ui: &Rc<Ui>, state: &FeedState, topics: &[Topic]) {
|
||||
if changed(Topic::Shares) {
|
||||
refresh_invitations_badge(ui, Duration::ZERO);
|
||||
}
|
||||
match ui.stack.visible_child_name().as_deref() {
|
||||
Some("browser") if changed(Topic::Files) => load_browser(ui),
|
||||
// The timeline is left alone: reloading it would throw away the place
|
||||
// the user scrolled to. F5 reloads it on request.
|
||||
// On any page: a grid, a list or a player may show a tile still blank.
|
||||
if changed(Topic::Thumbnails) {
|
||||
refresh_file_thumbnails(ui);
|
||||
}
|
||||
let page = ui.stack.visible_child_name();
|
||||
if changed(Topic::GalleryVisibility) || changed(Topic::GalleryVisibilitySynced) {
|
||||
if let Some(browser) = ui.drive_galleries.borrow().upgrade() {
|
||||
browser.refresh_visibility(changed(Topic::GalleryVisibility));
|
||||
}
|
||||
for window in gtk4::Window::list_toplevels() {
|
||||
if let Ok(window) = window.downcast::<gtk4::Window>()
|
||||
&& window.has_css_class("photo-viewer-window")
|
||||
{
|
||||
window.close();
|
||||
}
|
||||
}
|
||||
ui.gallery
|
||||
.thumb_generation
|
||||
.set(ui.gallery.thumb_generation.get().wrapping_add(1));
|
||||
ui.gallery.thumb_queue.borrow_mut().clear();
|
||||
ui.gallery.decode_queue.borrow_mut().clear();
|
||||
ui.gallery.thumb_wanted.borrow_mut().clear();
|
||||
ui.gallery.photo_tex.borrow_mut().clear();
|
||||
ui.gallery.photo_tex_order.borrow_mut().clear();
|
||||
ui.gallery.model.remove_all();
|
||||
ui.gallery.groups.remove_all();
|
||||
ui.gallery.row_by_photo.borrow_mut().clear();
|
||||
ui.gallery.timeline_stale.set(true);
|
||||
if page.as_deref() == Some("gallery") {
|
||||
match ui.gallery.content.visible_child_name().as_deref() {
|
||||
Some("albums") => load_albums(ui),
|
||||
Some("drive-galleries") => {}
|
||||
Some("places") => load_places(ui),
|
||||
_ => load_gallery(ui, false),
|
||||
}
|
||||
}
|
||||
}
|
||||
if changed(Topic::Photos) {
|
||||
check_photo_viewer(ui);
|
||||
if page.as_deref() != Some("gallery") || !photo_grid_shown(ui) {
|
||||
ui.gallery.timeline_stale.set(true);
|
||||
} else {
|
||||
refresh_visible_photos(ui);
|
||||
}
|
||||
}
|
||||
match page.as_deref() {
|
||||
// A search on screen is left alone: reloading would swap its hits for
|
||||
// the folder's listing while the query stays in the box.
|
||||
Some("browser") if changed(Topic::Files) && ui.browser.search.text().trim().is_empty() => {
|
||||
load_browser(ui)
|
||||
}
|
||||
Some("gallery")
|
||||
if changed(Topic::Photos)
|
||||
&& ui.gallery.content.visible_child_name().as_deref() == Some("albums") =>
|
||||
|
||||
Loaded 100 of 229 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user