mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-03 16:41:26 +01:00
feat(core,fuse,cli): reuse cached content and take several sources in moves
- A mirror download first copies the file out of the content cache when the same revision is cached whole or block by block, so moving a file out of an on-demand folder into a mirror costs no traffic for what was already read. - `pdfs move a b c DEST` moves every source, reports each failure and exits non-zero if any failed. - Mirror-to-mirror moves rename with RENAME_NOREPLACE, so a name that appeared at the destination after the check is never overwritten. - The local copy a move out of a mirror drops is checked again just before it is deleted; anything written into it meanwhile keeps the copy and is uploaded as new by the next pass.
This commit is contained in:
1 parent
5f9d07aff1
commit
261d5dbf84
8 files changed
+236
-20
No files matched your search
+34
-13
@@ -290,18 +290,17 @@ enum Command {
|
||||
/// New name (a single path component).
|
||||
new_name: String,
|
||||
},
|
||||
/// Move a file or folder into another folder via the running daemon.
|
||||
/// Move files or folders into another folder via the running daemon.
|
||||
///
|
||||
/// The two paths may be in different locations: My files and any synced
|
||||
/// The paths may be in different locations: My files and any synced
|
||||
/// folder, on-demand or mirrored. The move happens on Proton Drive, so
|
||||
/// nothing is downloaded or uploaded again, unlike `mv` between two
|
||||
/// locations.
|
||||
/// locations. The last path is the destination folder.
|
||||
Move {
|
||||
/// File/folder path: inside a location, or relative to the mountpoint.
|
||||
path: PathBuf,
|
||||
/// Destination folder path: inside a location, or relative to the
|
||||
/// mountpoint.
|
||||
new_parent: PathBuf,
|
||||
/// Files/folders to move, then the destination folder: each inside a
|
||||
/// location, or relative to the mountpoint.
|
||||
#[arg(required = true, num_args = 2.., value_name = "PATH")]
|
||||
paths: Vec<PathBuf>,
|
||||
},
|
||||
/// Trash a file or folder via the running daemon.
|
||||
Rm {
|
||||
@@ -806,7 +805,10 @@ fn main() -> Result<()> {
|
||||
scope,
|
||||
} => cmd_search(query, limit, scope),
|
||||
Command::Rename { path, new_name } => cmd_rename(path, new_name),
|
||||
Command::Move { path, new_parent } => cmd_move(path, new_parent),
|
||||
Command::Move { mut paths } => {
|
||||
let new_parent = paths.pop().expect("clap requires at least two paths");
|
||||
cmd_move(paths, new_parent)
|
||||
}
|
||||
Command::Rm { path } => cmd_rm(path),
|
||||
Command::Mkdir { parent, name } => cmd_mkdir(parent, name),
|
||||
Command::Upload { sources, parent } => cmd_upload(sources, parent),
|
||||
@@ -2445,15 +2447,34 @@ fn cmd_rename(path: PathBuf, new_name: String) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_move(path: PathBuf, new_parent: PathBuf) -> Result<()> {
|
||||
/// Move each of `sources` into `new_parent`, one request each. A refusal names
|
||||
/// its source and the rest still move; the command fails if any did not.
|
||||
fn cmd_move(sources: Vec<PathBuf>, new_parent: PathBuf) -> Result<()> {
|
||||
let new_parent = path_arg(&new_parent)?;
|
||||
let mut failed = 0;
|
||||
for source in &sources {
|
||||
let outcome = path_arg(source).and_then(|path| {
|
||||
match control_request(CtlRequest::Move {
|
||||
path: path_arg(&path)?,
|
||||
new_parent: path_arg(&new_parent)?,
|
||||
path,
|
||||
new_parent: new_parent.clone(),
|
||||
})? {
|
||||
CtlResponse::Ok { message } => println!("{message}"),
|
||||
CtlResponse::Ok { message } => Ok(message),
|
||||
CtlResponse::Error { message, kind } => bail!("{}", cli_error(kind, &message)),
|
||||
other => bail!("unexpected response: {other:?}"),
|
||||
}
|
||||
});
|
||||
match outcome {
|
||||
Ok(message) => println!("{message}"),
|
||||
Err(e) if sources.len() == 1 => return Err(e),
|
||||
Err(e) => {
|
||||
eprintln!("{}: {e:#}", source.display());
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
bail!("{failed} of {} could not be moved", sources.len());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -730,6 +730,60 @@ impl ContentCache {
|
||||
Some(buf)
|
||||
}
|
||||
|
||||
/// Write the whole content of `uid` at `(mtime, size)` to `dst` from the
|
||||
/// cache alone, and report whether that worked.
|
||||
///
|
||||
/// A whole-file blob is cloned. Otherwise every block of the revision has to
|
||||
/// be cached, under its recorded geometry or, failing that, the uniform one
|
||||
/// the read path would use. Anything short of the full content leaves
|
||||
/// nothing at `dst`, so a caller falls back to a download with a clean slate.
|
||||
///
|
||||
/// This is what lets a mirror folder take a file an on-demand mount has
|
||||
/// already read, after a move between them, without fetching it again. The
|
||||
/// trust is the read path's: the same `(mtime, size)` key, and blocks that
|
||||
/// were validated when they were fetched.
|
||||
pub fn copy_cached_to(&self, uid: &NodeUid, mtime: i64, size: u64, dst: &Path) -> bool {
|
||||
if size == 0 {
|
||||
return false;
|
||||
}
|
||||
if let Some(blob) = self.valid_blob(uid, mtime, size) {
|
||||
if Self::clone_or_copy(&blob, dst).is_ok_and(|len| len == size) {
|
||||
self.touch(Self::key(uid));
|
||||
return true;
|
||||
}
|
||||
let _ = std::fs::remove_file(dst);
|
||||
return false;
|
||||
}
|
||||
let geometry = self
|
||||
.block_geometry(uid, mtime, size)
|
||||
.unwrap_or_else(|| BlockGeometry::uniform(size));
|
||||
if geometry.size() != size {
|
||||
return false;
|
||||
}
|
||||
let spans = geometry.spans(0, size);
|
||||
// Checked up front so a file with one block missing costs stats, not a
|
||||
// copy of every block before it.
|
||||
if !spans
|
||||
.iter()
|
||||
.all(|&span| self.has_block(uid, mtime, size, span))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let written = (|| -> Option<()> {
|
||||
use std::io::Write;
|
||||
let mut out = std::fs::File::create(dst).ok()?;
|
||||
for span in spans {
|
||||
out.write_all(&self.cached_block(uid, mtime, size, span)?)
|
||||
.ok()?;
|
||||
}
|
||||
(out.metadata().ok()?.len() == size).then_some(())
|
||||
})();
|
||||
if written.is_none() {
|
||||
let _ = std::fs::remove_file(dst);
|
||||
}
|
||||
written.is_some()
|
||||
}
|
||||
|
||||
/// Store `bytes` as the cached content for `uid`, tagged with `(mtime,
|
||||
/// size)`. The blob is written to a temp file then renamed so a concurrent
|
||||
/// reader never sees a partial blob; the meta tag is written last so a
|
||||
@@ -2561,6 +2615,46 @@ mod tests {
|
||||
assert!(c.cached_block(&u, 100, 5000, blk(5000, 0)).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cached_blob_is_copied_out_whole() {
|
||||
let (c, d) = cache();
|
||||
let u = uid("blob");
|
||||
c.store(&u, 7, 5, b"hello").unwrap();
|
||||
let dst = d.path().join("out");
|
||||
assert!(c.copy_cached_to(&u, 7, 5, &dst));
|
||||
assert_eq!(std::fs::read(&dst).unwrap(), b"hello");
|
||||
// Another revision is not this one.
|
||||
assert!(!c.copy_cached_to(&u, 8, 5, &d.path().join("stale")));
|
||||
assert!(!d.path().join("stale").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_blocks_are_copied_out_in_order() {
|
||||
let (c, d) = cache();
|
||||
let u = uid("blocks");
|
||||
let size = BLOCK_SIZE + 3;
|
||||
let first = vec![1_u8; BLOCK_SIZE as usize];
|
||||
c.store_block(&u, 7, size, blk(size, 0), &first).unwrap();
|
||||
c.store_block(&u, 7, size, blk(size, 1), b"end").unwrap();
|
||||
let dst = d.path().join("out");
|
||||
assert!(c.copy_cached_to(&u, 7, size, &dst));
|
||||
let bytes = std::fs::read(&dst).unwrap();
|
||||
assert_eq!(bytes.len() as u64, size);
|
||||
assert_eq!(&bytes[..BLOCK_SIZE as usize], first.as_slice());
|
||||
assert_eq!(&bytes[BLOCK_SIZE as usize..], b"end");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_with_a_block_missing_is_not_copied_out() {
|
||||
let (c, d) = cache();
|
||||
let u = uid("partial");
|
||||
let size = BLOCK_SIZE + 3;
|
||||
c.store_block(&u, 7, size, blk(size, 1), b"end").unwrap();
|
||||
let dst = d.path().join("out");
|
||||
assert!(!c.copy_cached_to(&u, 7, size, &dst));
|
||||
assert!(!dst.exists(), "a partial copy must not be left behind");
|
||||
}
|
||||
|
||||
/// The [`BlockSpan`] the uniform geometry gives block `idx` of a file of
|
||||
/// `size` bytes — the shape these tests were written against, and the one
|
||||
/// this client's own uploads still have.
|
||||
|
||||
@@ -93,6 +93,44 @@ fn join_rel(parent: &str, name: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Rename `from` to `to`, failing with `EEXIST` rather than replacing
|
||||
/// whatever appeared at `to` since it was checked. A filesystem that cannot
|
||||
/// promise that gets a plain rename after one more look.
|
||||
fn rename_noreplace(from: &Path, to: &Path) -> std::io::Result<()> {
|
||||
use std::ffi::CString;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
|
||||
let cstr = |p: &Path| {
|
||||
CString::new(p.as_os_str().as_bytes())
|
||||
.map_err(|_| std::io::Error::from_raw_os_error(libc::EINVAL))
|
||||
};
|
||||
let (c_from, c_to) = (cstr(from)?, cstr(to)?);
|
||||
// SAFETY: both are valid NUL-terminated paths that outlive the call.
|
||||
let rc = unsafe {
|
||||
libc::syscall(
|
||||
libc::SYS_renameat2,
|
||||
libc::AT_FDCWD,
|
||||
c_from.as_ptr(),
|
||||
libc::AT_FDCWD,
|
||||
c_to.as_ptr(),
|
||||
libc::RENAME_NOREPLACE,
|
||||
)
|
||||
};
|
||||
if rc == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let err = std::io::Error::last_os_error();
|
||||
match err.raw_os_error() {
|
||||
Some(libc::EINVAL | libc::ENOSYS) => {
|
||||
if std::fs::symlink_metadata(to).is_ok() {
|
||||
return Err(std::io::Error::from_raw_os_error(libc::EEXIST));
|
||||
}
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
_ => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `uid` is `root` or lies under it, as far as the nodes table knows.
|
||||
fn under(core: &Core, root: &NodeUid, uid: &str) -> bool {
|
||||
let root = root.to_string();
|
||||
@@ -186,12 +224,18 @@ impl Core {
|
||||
{
|
||||
let from = src.to_path_buf();
|
||||
let to = dest_parent.join(&name);
|
||||
match std::fs::rename(&from, &to) {
|
||||
match rename_noreplace(&from, &to) {
|
||||
Ok(()) => renamed = Some((from, to)),
|
||||
Err(e) if e.raw_os_error() == Some(libc::EXDEV) => {
|
||||
info!(from = %folder.local_path, to = %dest_folder.local_path,
|
||||
"move: mirror folders are on different filesystems");
|
||||
}
|
||||
Err(e) if e.raw_os_error() == Some(libc::EEXIST) => {
|
||||
return Err(CoreError::conflict(format!(
|
||||
"{name} already exists in {}",
|
||||
dest_parent.display()
|
||||
)));
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(CoreError::internal(format!(
|
||||
"could not move {} to {}: {e}",
|
||||
@@ -425,11 +469,24 @@ impl Core {
|
||||
}
|
||||
}
|
||||
Place::Mirror { folder, rel } => {
|
||||
// Drive has all of it (checked before the move), so the local
|
||||
// copy is only a stale duplicate now.
|
||||
// Drive had all of it when the move was checked, so the local
|
||||
// copy is only a stale duplicate now. Something written into it
|
||||
// since would go with it, so look again first; if anything is
|
||||
// new, the copy stays and the next pass uploads it as new.
|
||||
let changed = match self.db.sync_entries(folder.id) {
|
||||
Ok(baseline) => {
|
||||
mirror_subtree_unsynced(Path::new(&folder.local_path), rel, &baseline)
|
||||
}
|
||||
Err(e) => Some(format!("db: {e}")),
|
||||
};
|
||||
if let Err(e) = self.db.sync_entries_remove_subtree(folder.id, rel) {
|
||||
warn!(error = ?e, "move: could not drop the sync baseline");
|
||||
}
|
||||
if let Some(reason) = changed {
|
||||
warn!(path = %src.display(), reason,
|
||||
"move: the local copy changed during the move; keeping it");
|
||||
return;
|
||||
}
|
||||
let removed = if src.is_dir() {
|
||||
std::fs::remove_dir_all(src)
|
||||
} else {
|
||||
@@ -523,4 +580,24 @@ mod tests {
|
||||
assert_eq!(join_rel("", "a"), "a");
|
||||
assert_eq!(join_rel("x/y", "a"), "x/y/a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_local_rename_never_replaces_what_is_already_there() {
|
||||
let dir = std::env::temp_dir().join(format!("pdfs-relocate-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let (a, b, c) = (dir.join("a"), dir.join("b"), dir.join("c"));
|
||||
std::fs::write(&a, b"moved").unwrap();
|
||||
std::fs::write(&b, b"kept").unwrap();
|
||||
|
||||
let err = rename_noreplace(&a, &b).unwrap_err();
|
||||
assert_eq!(err.raw_os_error(), Some(libc::EEXIST));
|
||||
assert_eq!(std::fs::read(&b).unwrap(), b"kept");
|
||||
assert!(a.exists());
|
||||
|
||||
rename_noreplace(&a, &c).unwrap();
|
||||
assert!(!a.exists());
|
||||
assert_eq!(std::fs::read(&c).unwrap(), b"moved");
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -1328,7 +1328,21 @@ impl Core {
|
||||
.map(|d| d.as_nanos())
|
||||
.unwrap_or(0)
|
||||
));
|
||||
{
|
||||
// A file an on-demand mount already read, say one just moved here from
|
||||
// it, is in the content cache under the same revision key.
|
||||
let seeded = size > 0 && self.cache.copy_cached_to(uid, mtime, size as u64, &tmp);
|
||||
if seeded {
|
||||
std::fs::File::open(&tmp)
|
||||
.and_then(|f| f.sync_all())
|
||||
.map_err(|e| {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
format!("sync tmp {rel}: {e}")
|
||||
})?;
|
||||
debug!(
|
||||
rel,
|
||||
"sync: placed from the content cache instead of downloading"
|
||||
);
|
||||
} else {
|
||||
let out = std::fs::File::create(&tmp).map_err(|e| format!("create tmp {rel}: {e}"))?;
|
||||
// Count the bytes as they land, so the download reports progress like
|
||||
// an on-demand hydration does.
|
||||
|
||||
@@ -547,9 +547,10 @@ The primary mount and every on-demand folder are separate FUSE sessions, and a m
|
||||
`Request::Move` with two absolute paths is therefore resolved by `Core::move_between` against every location: a live mount through `rooted_at`, otherwise the mirror folder with the longest `local_path` prefix, whose uids come from the `sync_entry` baseline. Two paths in one inode space still go to `move_to`. Relative paths keep their old meaning.
|
||||
|
||||
- **Nothing that exists only on this disk may be lost.** A mounted source is refused while it or anything under it has a queued op or a file open for writing. A mirror source is refused unless every file under it matches its baseline and every baseline row still has its file (`mirror_subtree_unsynced`); ignored names and symlinks count as unsynced, because the local copy is removed after the move.
|
||||
- **Mirror to mirror renames along.** The local copy is renamed first and its baseline rows are moved with `sync_entries_move`, so neither folder's next pass has anything to do. If `move_node` fails, the rename is undone. `EXDEV` between two filesystems falls back to the check above, then drops the source copy.
|
||||
- **Mirror to mirror renames along.** The local copy is renamed first, with `RENAME_NOREPLACE` so nothing that appeared at the destination meanwhile is overwritten, and its baseline rows are moved with `sync_entries_move`, so neither folder's next pass has anything to do. If `move_node` fails, the rename is undone. `EXDEV` between two filesystems falls back to the check above, then drops the source copy.
|
||||
- **A dropped mirror copy is checked twice.** The check runs again just before the source copy is deleted. A file written into it during the move keeps the whole copy; its baseline is gone, so the next pass uploads it as new.
|
||||
- **Sync passes are held off.** Every mirror folder involved is locked through `sync_lock`, by ascending id, with a five-second bound. A folder that is still busy is reported rather than waited on.
|
||||
- **The local side is levelled afterwards.** A mounted source forgets the node in every inode space and notifies the kernel. A mounted destination has its listing cleared. A mirror destination that received nothing locally gets a reconcile, which downloads the node.
|
||||
- **The local side is levelled afterwards.** A mounted source forgets the node in every inode space and notifies the kernel. A mounted destination has its listing cleared. A mirror destination that received nothing locally gets a reconcile, which downloads the node. That download first asks `ContentCache::copy_cached_to` for the file: a whole cached blob, or every block of it, is copied into place instead of fetched, so a move out of an on-demand folder costs no traffic for what was already opened there.
|
||||
|
||||
### Open-for-Write Deferral for Mirror Sync
|
||||
**File**: [`sync.rs`](../crates/pdfs-fuse/src/sync.rs)
|
||||
|
||||
@@ -20,6 +20,10 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
|
||||
level. A move is refused, with the reason, when the source still has changes that are not on
|
||||
Proton Drive, when the two locations are on different volumes, or when a synced folder stays
|
||||
busy with a sync pass.
|
||||
- **`pdfs move` takes several sources.** `pdfs move a b c DEST` moves all three and
|
||||
reports each one that could not move.
|
||||
- **Moving into a mirrored folder reuses cached content.** A file already cached by
|
||||
an online-only folder is copied into place instead of downloaded again.
|
||||
|
||||
## [2.6.2] — 2026-09-28
|
||||
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ talks to the running service (`proton-drive.service`); `login`, `logout`, `mount
|
||||
| `pdfs search QUERY [--in FOLDER] [--limit N]` | Search names in the local index. |
|
||||
| `pdfs mkdir PARENT NAME` | Create a folder. |
|
||||
| `pdfs rename PATH NEW_NAME` | Rename a file or folder. |
|
||||
| `pdfs move PATH NEW_PARENT` | Move into another folder. The two may be in different locations (My files, an on-demand or a mirrored synced folder); the move happens on Proton Drive, without downloading or uploading anything. |
|
||||
| `pdfs move PATH... NEW_PARENT` | Move one or more files or folders into another folder. The two may be in different locations (My files, an on-demand or a mirrored synced folder); the move happens on Proton Drive, without downloading or uploading anything. |
|
||||
| `pdfs rm PATH` | Move to the Trash. |
|
||||
| `pdfs upload SOURCES... [-t FOLDER]` | Upload local files and folders (recursively) in the background. Follow with `pdfs transfers`. |
|
||||
| `pdfs refresh [TARGET] [--full]` | Forget a cached listing so the next read fetches it again. `TARGET` is a folder, `trash` or `photos`; `--full` with `photos` re-reads every photo's metadata. |
|
||||
|
||||
@@ -106,8 +106,13 @@ Drive, so it takes seconds whatever the size. For example:
|
||||
|
||||
```sh
|
||||
pdfs move ~/Documents/Scans ~/ProtonDrive/Archive
|
||||
pdfs move ~/Documents/a.pdf ~/Documents/b.pdf ~/ProtonDrive/Archive
|
||||
```
|
||||
|
||||
The last path is the destination folder; everything before it moves there.
|
||||
Files already opened in an online-only folder are not downloaded again when
|
||||
they move into a mirrored one.
|
||||
|
||||
A move out of a synced folder waits until its files are on Proton Drive. A file that is still
|
||||
uploading, has changes that are not uploaded, or is excluded by `.pdfsignore` stops the move,
|
||||
and the message names it.
|
||||
|
||||
Reference in new issue
Block a user