This commit is contained in:
nvrl committed 2026-07-22 02:17:19 +02:00
1 parent 242a28bb8f
commit 0a1fa95eb4
10 files changed
+1772 -1750

No files matched your search

+251
View File
@@ -0,0 +1,251 @@
//! Filesystem watcher, polling, and debounce control plane.
use super::*;
/// How long the watcher must go quiet before a burst counts as finished.
///
/// Measured from the *last* event, not the first: applications do not write a
/// file once. An editor saves by writing a temp file and renaming it over the
/// target; an export or a database dump writes continuously for as long as it
/// takes. Settling a fixed interval after the first event walks the tree while
/// the file is still growing, and uploads a torn snapshot as a real revision —
/// corrected on the next pass, but only after spending the encrypt-and-upload
/// on a file that was never in that state.
const DEBOUNCE: Duration = Duration::from_secs(2);
/// Ceiling on the settle wait, so a directory under sustained change still
/// syncs. Without it, copying a large tree in — events arriving forever —
/// would postpone the reconcile for as long as the copy ran.
const MAX_SETTLE: Duration = Duration::from_secs(30);
/// How often to re-walk remotes so changes made on other clients are pulled in.
const POLL_INTERVAL: Duration = Duration::from_secs(120);
/// How many uploads/downloads/folder-creations a reconcile runs at once. The
/// per-item work is a network round-trip, so a folder with thousands of files is
/// unusable done one-at-a-time; this bounds concurrency like the bulk-upload
/// engine does.
pub(super) const SYNC_CONCURRENCY: usize = 8;
/// A message to the sync engine's control thread.
pub(crate) enum SyncMsg {
/// Reconcile one folder by id (a local change, or a targeted `SyncNow`).
Reconcile(i64),
/// Reconcile every mirror folder (periodic poll, or startup).
ReconcileAll,
/// Re-read the folder set and adjust the filesystem watches (after add/remove).
Rewatch,
}
/// Start the sync engine: a control thread that owns the filesystem watcher and
/// serialises reconcile passes, plus a periodic poll thread. The engine receives
/// on `rx`; senders live in [`Core::sync_tx`].
pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) {
if let Err(e) = std::thread::Builder::new()
.name("pdfs-sync".into())
.spawn(move || engine_loop(core, rx))
{
warn!(error = %e, "failed to start sync engine");
}
}
/// The engine control loop. Single-threaded so reconcile passes never overlap.
fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
// Paths the watcher currently covers, mapped to their folder id, so an event
// path can be resolved back to the folder it belongs to.
let watched: Mutex<Vec<(PathBuf, i64)>> = Mutex::new(Vec::new());
let tx_events = core.sync_tx.clone();
let mut watcher =
match notify::recommended_watcher(move |res: notify::Result<notify::Event>| {
let Ok(event) = res else { return };
// Ignore pure metadata/access noise; content changes are what matter.
if matches!(event.kind, notify::EventKind::Access(_)) {
return;
}
let _ = tx_events.send(SyncMsg::ReconcileAll);
}) {
Ok(w) => w,
Err(e) => {
warn!(error = %e, "filesystem watcher unavailable; sync is poll-only");
// Still run the loop so polling and explicit SyncNow work.
poll_only_loop(&core, rx);
return;
}
};
// Periodic remote poll.
{
let tx = core.sync_tx.clone();
std::thread::Builder::new()
.name("pdfs-sync-poll".into())
.spawn(move || {
loop {
std::thread::sleep(POLL_INTERVAL);
if tx.send(SyncMsg::ReconcileAll).is_err() {
break;
}
}
})
.ok();
}
rewatch(&core, &mut watcher, &watched);
reconcile_all(&core);
while let Ok(msg) = rx.recv() {
let mut ids: HashSet<i64> = HashSet::new();
let mut all = false;
let mut do_rewatch = false;
classify(msg, &mut ids, &mut all, &mut do_rewatch);
settle(&rx, &mut ids, &mut all, &mut do_rewatch);
if do_rewatch {
rewatch(&core, &mut watcher, &watched);
}
if all {
reconcile_all(&core);
} else {
for id in ids {
if let Ok(Some(folder)) = core.db.sync_folder_get(id) {
core.reconcile_folder(&folder);
}
}
}
}
}
/// Fallback loop when no filesystem watcher could be created: poll + SyncNow only.
fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
{
let tx = core.sync_tx.clone();
std::thread::spawn(move || {
loop {
std::thread::sleep(POLL_INTERVAL);
if tx.send(SyncMsg::ReconcileAll).is_err() {
break;
}
}
});
}
reconcile_all(core);
while let Ok(msg) = rx.recv() {
let mut ids: HashSet<i64> = HashSet::new();
let mut all = false;
let mut do_rewatch = false;
classify(msg, &mut ids, &mut all, &mut do_rewatch);
settle(&rx, &mut ids, &mut all, &mut do_rewatch);
if all {
reconcile_all(core);
} else {
for id in ids {
if let Ok(Some(folder)) = core.db.sync_folder_get(id) {
core.reconcile_folder(&folder);
}
}
}
}
}
/// Absorb the rest of an event burst, returning once the watcher has been quiet
/// for `quiet` or `cap` has elapsed since the first event — whichever comes
/// first. The caller has already classified the event that opened the burst.
///
/// Split out with its timings as parameters so the settling behaviour can be
/// tested in milliseconds rather than in the tens of seconds the real constants
/// describe.
pub(super) fn settle_with(
rx: &Receiver<SyncMsg>,
ids: &mut HashSet<i64>,
all: &mut bool,
do_rewatch: &mut bool,
quiet: Duration,
cap: Duration,
) {
let deadline = Instant::now() + cap;
loop {
let now = Instant::now();
if now >= deadline {
break;
}
// Never wait past the cap, however quiet the watcher goes.
let wait = quiet.min(deadline - now);
match rx.recv_timeout(wait) {
// Another event: the burst is still going, so the quiet window
// starts again from here.
Ok(m) => classify(m, ids, all, do_rewatch),
// Quiet for a full window: whatever was being written has stopped.
Err(RecvTimeoutError::Timeout) => break,
// Every sender is gone; the caller's loop is about to end anyway.
Err(RecvTimeoutError::Disconnected) => break,
}
}
}
/// [`settle_with`] at the real timings.
fn settle(rx: &Receiver<SyncMsg>, ids: &mut HashSet<i64>, all: &mut bool, do_rewatch: &mut bool) {
settle_with(rx, ids, all, do_rewatch, DEBOUNCE, MAX_SETTLE);
}
pub(super) fn classify(msg: SyncMsg, ids: &mut HashSet<i64>, all: &mut bool, rewatch: &mut bool) {
match msg {
SyncMsg::Reconcile(id) => {
ids.insert(id);
}
SyncMsg::ReconcileAll => *all = true,
SyncMsg::Rewatch => *rewatch = true,
}
}
/// Reconcile every mirror folder in turn.
fn reconcile_all(core: &Core) {
let folders = match core.db.sync_folder_list() {
Ok(f) => f,
Err(e) => {
warn!(error = ?e, "sync: cannot list folders");
return;
}
};
for folder in folders {
core.reconcile_folder(&folder);
}
}
/// Bring the filesystem watches in line with the current mirror-folder set:
/// watch newly-added folders, drop removed ones.
fn rewatch(
core: &Core,
watcher: &mut notify::RecommendedWatcher,
watched: &Mutex<Vec<(PathBuf, i64)>>,
) {
let want: Vec<(PathBuf, i64)> = match core.db.sync_folder_list() {
Ok(f) => f
.into_iter()
.filter(|f| f.mode == "mirror")
.map(|f| (PathBuf::from(f.local_path), f.id))
.collect(),
Err(e) => {
warn!(error = ?e, "sync: cannot list folders to watch");
return;
}
};
let mut have = watched.lock();
// Drop watches no longer wanted.
have.retain(|(path, _)| {
if want.iter().any(|(p, _)| p == path) {
true
} else {
let _ = watcher.unwatch(path);
false
}
});
// Add newly wanted watches.
for (path, id) in &want {
if have.iter().any(|(p, _)| p == path) {
continue;
}
match watcher.watch(path, RecursiveMode::Recursive) {
Ok(()) => have.push((path.clone(), *id)),
Err(e) => warn!(path = %path.display(), error = %e, "sync: watch failed"),
}
}
}
// ---- reconcile ------------------------------------------------------------
+81
View File
@@ -0,0 +1,81 @@
//! Local open-for-write detection used to defer torn uploads.
use super::*;
// ---- open-for-write detection ---------------------------------------------
/// Scan `/proc/*/fd` once and return the set of **canonical** paths under
/// `root` that any process currently holds open for writing (`O_WRONLY` or
/// `O_RDWR`). The scan is best-effort: unreadable `/proc/*/fd` directories
/// (other users' processes, or kernel threads) are silently skipped.
///
/// Cost: one `readdir` of `/proc`, then for every process one `readdir` of its
/// `/proc/<pid>/fd/`, and one `readlink` + one `open` of `/proc/<pid>/fdinfo/<n>`
/// per open fd that resolves under `root`. For a typical desktop with a few
/// hundred processes and a handful of files inside the sync root, this
/// completes in low single-digit milliseconds.
pub(super) fn open_for_write_set(root: &Path) -> HashSet<PathBuf> {
let mut result = HashSet::new();
let Ok(root_canonical) = std::fs::canonicalize(root) else {
return result;
};
let Ok(proc_entries) = std::fs::read_dir("/proc") else {
return result;
};
for proc_entry in proc_entries.filter_map(|e| e.ok()) {
let pid_name = proc_entry.file_name();
// Skip non-numeric entries (kernel threads, /proc/self, etc.)
if !pid_name
.to_str()
.is_some_and(|s| s.bytes().all(|b| b.is_ascii_digit()))
{
continue;
}
let fd_dir = PathBuf::from("/proc").join(&pid_name).join("fd");
let Ok(fds) = std::fs::read_dir(&fd_dir) else {
continue;
};
for fd_entry in fds.filter_map(|e| e.ok()) {
// readlink on /proc/<pid>/fd/<n> gives the target path.
let link = fd_dir.join(fd_entry.file_name());
let Ok(target) = std::fs::read_link(&link) else {
continue;
};
// Quick prefix check before the more expensive fdinfo read.
if !target.starts_with(&root_canonical) {
continue;
}
// Read the fd's flags from /proc/<pid>/fdinfo/<n> to determine
// the access mode. The first line is "pos:\t<offset>", the second
// is "flags:\t<octal>". We only need the low two bits of flags:
// 0 = O_RDONLY, 1 = O_WRONLY, 2 = O_RDWR
let fdinfo_path = PathBuf::from("/proc")
.join(&pid_name)
.join("fdinfo")
.join(fd_entry.file_name());
if let Ok(contents) = std::fs::read_to_string(&fdinfo_path)
&& is_write_mode(&contents)
{
result.insert(target);
}
}
}
result
}
/// Parse the `flags:` line out of a `/proc/<pid>/fdinfo/<n>` file and return
/// true if the low two bits indicate write access (`O_WRONLY = 1` or
/// `O_RDWR = 2`).
pub(super) fn is_write_mode(fdinfo: &str) -> bool {
for line in fdinfo.lines() {
if let Some(rest) = line.strip_prefix("flags:\t") {
if let Ok(flags) = u32::from_str_radix(rest.trim_start_matches('0'), 8) {
let access = flags & 0o3; // O_ACCMODE
return access == 1 || access == 2; // O_WRONLY or O_RDWR
}
// An empty octal string ("0" stripped to "") means flags == 0 == O_RDONLY.
return false;
}
}
false
}
+164
View File
@@ -0,0 +1,164 @@
//! Reconciliation snapshots, queued operations, and pass outcomes.
use super::*;
pub(super) struct LocalItem {
pub(super) is_dir: bool,
pub(super) mtime: i64,
pub(super) size: i64,
/// True when another process holds this file open for writing (detected via
/// `/proc/*/fd`). An open-for-write file is kept in the map — so it is not
/// misread as a deletion — but treated as unchanged for upload purposes,
/// deferring the upload until the writer closes the file.
pub(super) open_for_write: bool,
}
/// One item found while walking a remote tree.
pub(super) struct RemoteItem {
pub(super) uid: NodeUid,
pub(super) is_dir: bool,
pub(super) mtime: i64,
pub(super) size: i64,
}
/// The result of a reconcile pass: what it moved, how many paths were kept as
/// conflict copies, and how many failed to apply (and so still need another
/// pass). The counts drive both the folder's state and its activity summary.
#[derive(Default)]
pub(super) struct Outcome {
pub(super) uploaded: usize,
pub(super) downloaded: usize,
pub(super) created: usize,
pub(super) deleted: usize,
pub(super) conflicts: usize,
pub(super) errors: usize,
/// Files skipped because another process held them open for writing.
pub(super) deferred: usize,
}
impl Outcome {
/// Fold in one applied op.
pub(super) fn record(&mut self, applied: &Applied) {
match applied {
Applied::Dir(..) => self.created += 1,
Applied::Uploaded => self.uploaded += 1,
Applied::Downloaded => self.downloaded += 1,
Applied::Conflict => self.conflicts += 1,
}
}
/// Whether the pass moved nothing at all — the common case on a poll of an
/// unchanged folder, which should not add a line to the activity feed.
pub(super) fn is_empty(&self) -> bool {
self.uploaded == 0
&& self.downloaded == 0
&& self.created == 0
&& self.deleted == 0
&& self.conflicts == 0
&& self.errors == 0
&& self.deferred == 0
}
/// A human summary of the pass: "3 uploaded, 1 downloaded, 2 failed".
pub(super) fn summary(&self) -> String {
let mut parts = Vec::new();
for (n, label) in [
(self.uploaded, "uploaded"),
(self.downloaded, "downloaded"),
(self.created, "folder(s) created"),
(self.deleted, "deleted"),
(self.conflicts, "conflicted"),
(self.deferred, "deferred (open for write)"),
(self.errors, "failed"),
] {
if n > 0 {
parts.push(format!("{n} {label}"));
}
}
parts.join(", ")
}
}
/// A network operation queued during classification and run concurrently in a
/// per-depth batch. Parent uids are resolved up front (the parent folder is one
/// depth shallower and already created), so tasks share nothing mutable.
pub(super) enum Pending {
/// Create a new remote folder under `parent`.
CreateDir { rel: String, parent: NodeUid },
/// Upload a brand-new local file into `parent`.
UploadNew { rel: String, parent: NodeUid },
/// Upload a changed local file as a new revision of `uid`.
UploadRevision { rel: String, uid: NodeUid },
/// Download remote `uid` to the local path, stamping `mtime`. `size` is the
/// remote's reported size, used as the transfer's expected total.
Download {
rel: String,
uid: NodeUid,
mtime: i64,
size: i64,
},
/// Both sides changed: set the local copy aside, then download remote `uid`.
Conflict {
rel: String,
uid: NodeUid,
mtime: i64,
size: i64,
},
/// Both sides changed during a [`push pass`](Core::push_pass): upload the local
/// copy into `parent` under a conflict name, leaving the remote file as it is.
PushConflict { rel: String, parent: NodeUid },
}
impl Pending {
/// The path this op acts on, relative to the folder root.
pub(super) fn rel(&self) -> &str {
match self {
Pending::CreateDir { rel, .. }
| Pending::UploadNew { rel, .. }
| Pending::UploadRevision { rel, .. }
| Pending::Download { rel, .. }
| Pending::Conflict { rel, .. }
| Pending::PushConflict { rel, .. } => rel,
}
}
/// How this op reads in the activity feed.
pub(super) fn kind(&self) -> ActivityKind {
match self {
Pending::CreateDir { .. } => ActivityKind::CreateFolder,
Pending::UploadNew { .. }
| Pending::UploadRevision { .. }
| Pending::PushConflict { .. } => ActivityKind::Upload,
Pending::Download { .. } | Pending::Conflict { .. } => ActivityKind::Download,
}
}
/// The activity line's detail, which distinguishes ops that share a kind.
pub(super) fn detail(&self) -> &'static str {
match self {
Pending::CreateDir { .. } => "on Drive",
Pending::UploadNew { .. } => "new file",
Pending::UploadRevision { .. } => "new version",
Pending::Download { .. } => "from Drive",
Pending::Conflict { .. } => "local changes kept as a conflict copy",
Pending::PushConflict { .. } => "local changes uploaded as a conflict copy",
}
}
}
/// Why a full reconcile pass stopped without a result.
pub(super) enum PassAbort {
/// An on-demand switch was queued while the pass was running. Everything left
/// to do is work on a local copy about to be evicted, so the pass gives up its
/// remaining work to a [`push pass`](Core::push_pass) instead. Not a failure —
/// nothing is left half-applied, because each step is applied whole.
Interrupted,
/// The pass could not establish its diff (a walk or the baseline load failed).
Failed(String),
}
impl From<String> for PassAbort {
fn from(e: String) -> Self {
PassAbort::Failed(e)
}
}
+176
View File
@@ -0,0 +1,176 @@
//! Pure reconciliation ordering, safety guards, signatures, and path rules.
use super::*;
/// The paths a reconcile pass will classify: the union of the local, remote,
/// and baseline states, minus anything the ignore rules exclude, shallowest
/// first so a parent folder is created before its children are placed in it.
///
/// The ignore filter belongs *here*, on the union, rather than on the local walk
/// alone — filtering the walk alone would be actively destructive. A file synced
/// before it became ignored is absent from `local` but present in `remote` and
/// `baseline`, which is precisely the "local deleted, remote untouched" shape
/// `do_reconcile` responds to by trashing the remote copy. Adding a line to
/// `.pdfsignore` must never delete anything from Drive.
///
/// Ignored baseline rows are deliberately left in the database rather than
/// removed: with the row intact, un-ignoring the path later re-adopts the
/// existing remote file instead of reading as a brand-new local one.
pub(super) fn classification_order<L, R, B>(
local: &HashMap<String, L>,
remote: &HashMap<String, R>,
baseline: &HashMap<String, B>,
rules: &IgnoreRules,
) -> Vec<String>
where
L: HasKind,
R: HasKind,
{
let mut paths: HashSet<String> = HashSet::new();
paths.extend(local.keys().cloned());
paths.extend(remote.keys().cloned());
paths.extend(baseline.keys().cloned());
let mut order: Vec<String> = paths.into_iter().collect();
if !rules.is_empty() {
order.retain(|rel| {
// A baseline-only path has no kind recorded, so it is tested as
// both; see `filter_baseline` for why erring towards "ignored" is
// the safe direction here.
let kind = local
.get(rel)
.map(HasKind::is_dir)
.or_else(|| remote.get(rel).map(HasKind::is_dir));
match kind {
Some(is_dir) => !rules.is_ignored(rel, is_dir),
None => !rules.is_ignored(rel, false) && !rules.is_ignored(rel, true),
}
});
}
order.sort_by_key(|p| p.matches('/').count());
order
}
/// Lets [`classification_order`] ask either walk's item whether it is a
/// directory without knowing which walk it came from.
pub(super) trait HasKind {
fn is_dir(&self) -> bool;
}
impl HasKind for LocalItem {
fn is_dir(&self) -> bool {
self.is_dir
}
}
impl HasKind for RemoteItem {
fn is_dir(&self) -> bool {
self.is_dir
}
}
/// The baseline minus its ignored paths, for [`guard_local_wipe`].
///
/// The guard asks "did every synced path vanish locally?", and an ignored path
/// is absent from the local walk by rule rather than by loss. Left in, a rule
/// covering the whole tree would trip the guard on every pass and wedge that
/// folder's sync for good.
///
/// A baseline row does not record whether it was a file or a folder, so a path
/// counts as ignored if it matches as either. Erring towards "ignored" only ever
/// shrinks the set the guard checks, which weakens a safety net rather than
/// causing a deletion — the wrong direction to be wrong in is the other one.
pub(super) fn filter_baseline<'a, B>(
baseline: &'a HashMap<String, B>,
rules: &IgnoreRules,
) -> HashMap<String, &'a B> {
baseline
.iter()
.filter(|(rel, _)| !rules.is_ignored(rel, false) && !rules.is_ignored(rel, true))
.map(|(rel, entry)| (rel.clone(), entry))
.collect()
}
/// Refuse to run a pass whose local side has vanished in its entirety.
///
/// When every baseline path is absent locally, the likely cause is an unavailable
/// mount or unreadable folder rather than a deliberate whole-tree deletion.
/// One surviving path is enough to clear the guard.
pub(super) fn guard_local_wipe<B, L>(
baseline: &HashMap<String, B>,
local: &HashMap<String, L>,
) -> Result<(), String> {
if baseline.len() >= 2 && baseline.keys().all(|rel| !local.contains_key(rel)) {
return Err(format!(
"every one of the {} synced paths is missing locally; refusing to trash \
them on Drive. Check that the folder is mounted and readable.",
baseline.len()
));
}
Ok(())
}
/// The size the baseline recorded for `rel`, if its remote signature's mtime
/// still matches `mtime`.
pub(super) fn unchanged_remote_size(
baseline: &HashMap<String, StoredSyncEntry>,
rel: &str,
mtime: i64,
) -> Option<i64> {
match baseline.get(rel).and_then(remote_sig) {
Some((recorded, size)) if recorded == mtime => Some(size),
_ => None,
}
}
/// Join a child `name` onto a walk's `prefix`, giving a rel path.
pub(super) fn join_rel(prefix: &str, name: &str) -> String {
if prefix.is_empty() {
name.to_string()
} else {
format!("{prefix}/{name}")
}
}
/// The stored remote signature of a baseline row, if it has one.
pub(super) fn remote_sig(e: &StoredSyncEntry) -> Option<(i64, i64)> {
match (&e.remote_rev, &e.remote_hash) {
(Some(m), Some(s)) => Some((m.parse().ok()?, s.parse().ok()?)),
_ => None,
}
}
/// The parent of a `/`-joined relative path (`""` for a top-level entry).
pub(super) fn parent_rel(rel: &str) -> &str {
match rel.rfind('/') {
Some(i) => &rel[..i],
None => "",
}
}
/// The final component of a `/`-joined relative path.
pub(crate) fn base_name(rel: &str) -> &str {
match rel.rfind('/') {
Some(i) => &rel[i + 1..],
None => rel,
}
}
/// Turn a `/`-joined relative path into an OS path (`/` is already the separator
/// on Linux, this keeps the intent explicit).
pub(super) fn rel_to_path(rel: &str) -> PathBuf {
rel.split('/').collect()
}
/// The name for a conflict copy of `path`, e.g. `notes (sync-conflict 1700000000).txt`.
pub(super) fn conflict_path(path: &Path, stamp: i64) -> PathBuf {
let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or("file");
let ext = path.extension().and_then(|s| s.to_str());
let name = match ext {
Some(ext) => format!("{stem} (sync-conflict {stamp}).{ext}"),
None => format!("{stem} (sync-conflict {stamp})"),
};
match path.parent() {
Some(dir) => dir.join(name),
None => PathBuf::from(name),
}
}