mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-04 16:56:59 +01:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73af39ee65 | ||
|
|
48aaad23ce | ||
|
|
c64bb23e3c | ||
|
|
8d4a005aa2 | ||
|
|
9460c93b50 | ||
|
|
bd50ba5959 | ||
|
|
27201914ee | ||
|
|
2775f0681d | ||
|
|
97277d41dd | ||
|
|
d49738bf0b | ||
|
|
48945b9c97 | ||
|
|
b860ac3b49 | ||
|
|
42f3a18177 | ||
|
|
6f504d2e83 | ||
|
|
8c1c37a928 | ||
|
|
e4d2340bbf | ||
|
|
c1b72e455e | ||
|
|
2e3b7748e6 |
No files matched your search
@@ -0,0 +1,122 @@
|
|||||||
|
//! Log priorities for journald.
|
||||||
|
//!
|
||||||
|
//! journald files each line a service writes at the unit's default priority,
|
||||||
|
//! info, unless the line opens with a syslog `<N>` prefix. Without one every
|
||||||
|
//! WARN and ERROR the daemon logs is stored as info, and the
|
||||||
|
//! `journalctl -p warning` that the troubleshooting guide asks for shows
|
||||||
|
//! nothing (`docs/BUGS.md` B258).
|
||||||
|
|
||||||
|
use std::fmt;
|
||||||
|
use std::os::unix::fs::MetadataExt;
|
||||||
|
|
||||||
|
use tracing::{Event, Level, Subscriber};
|
||||||
|
use tracing_subscriber::fmt::format::Writer;
|
||||||
|
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
|
||||||
|
use tracing_subscriber::registry::LookupSpan;
|
||||||
|
|
||||||
|
/// Whether standard output is the journal stream systemd connected this
|
||||||
|
/// process to.
|
||||||
|
///
|
||||||
|
/// `JOURNAL_STREAM` alone does not say so: a terminal started from a user unit
|
||||||
|
/// inherits it, and every `pdfs` run in that terminal would print the
|
||||||
|
/// prefixes. systemd sets it to the stream's `device:inode`, so it is compared
|
||||||
|
/// against what fd 1 actually is.
|
||||||
|
pub(crate) fn stdout_is_journal() -> bool {
|
||||||
|
let Some(stream) = std::env::var_os("JOURNAL_STREAM") else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(stdout) = std::fs::metadata("/proc/self/fd/1") else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
stream
|
||||||
|
.to_str()
|
||||||
|
.is_some_and(|value| names_stream(value, stdout.dev(), stdout.ino()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a `JOURNAL_STREAM` value names the file `dev`:`ino`.
|
||||||
|
fn names_stream(value: &str, dev: u64, ino: u64) -> bool {
|
||||||
|
value
|
||||||
|
.split_once(':')
|
||||||
|
.is_some_and(|(d, i)| d.parse() == Ok(dev) && i.parse() == Ok(ino))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The syslog priority journald should file a `level` line at.
|
||||||
|
fn priority(level: &Level) -> u8 {
|
||||||
|
match *level {
|
||||||
|
Level::ERROR => 3,
|
||||||
|
Level::WARN => 4,
|
||||||
|
Level::INFO => 6,
|
||||||
|
Level::DEBUG | Level::TRACE => 7,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An event format that opens each line with its `<N>` priority, then writes
|
||||||
|
/// the line as `F` would.
|
||||||
|
pub(crate) struct Prioritized<F>(pub(crate) F);
|
||||||
|
|
||||||
|
impl<S, N, F> FormatEvent<S, N> for Prioritized<F>
|
||||||
|
where
|
||||||
|
S: Subscriber + for<'a> LookupSpan<'a>,
|
||||||
|
N: for<'a> FormatFields<'a> + 'static,
|
||||||
|
F: FormatEvent<S, N>,
|
||||||
|
{
|
||||||
|
fn format_event(
|
||||||
|
&self,
|
||||||
|
ctx: &FmtContext<'_, S, N>,
|
||||||
|
mut writer: Writer<'_>,
|
||||||
|
event: &Event<'_>,
|
||||||
|
) -> fmt::Result {
|
||||||
|
write!(writer, "<{}>", priority(event.metadata().level()))?;
|
||||||
|
self.0.format_event(ctx, writer, event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::io;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct Buf(Arc<Mutex<Vec<u8>>>);
|
||||||
|
|
||||||
|
impl io::Write for Buf {
|
||||||
|
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||||
|
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||||
|
Ok(bytes.len())
|
||||||
|
}
|
||||||
|
fn flush(&mut self) -> io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn each_line_opens_with_the_priority_of_its_level() {
|
||||||
|
let buf = Buf(Arc::new(Mutex::new(Vec::new())));
|
||||||
|
let writer = buf.clone();
|
||||||
|
let subscriber = tracing_subscriber::fmt()
|
||||||
|
.with_max_level(Level::DEBUG)
|
||||||
|
.with_ansi(false)
|
||||||
|
.with_writer(move || writer.clone())
|
||||||
|
.event_format(Prioritized(tracing_subscriber::fmt::format()))
|
||||||
|
.finish();
|
||||||
|
tracing::subscriber::with_default(subscriber, || {
|
||||||
|
tracing::error!("e");
|
||||||
|
tracing::warn!("w");
|
||||||
|
tracing::info!("i");
|
||||||
|
tracing::debug!("d");
|
||||||
|
});
|
||||||
|
let out = String::from_utf8(buf.0.lock().unwrap().clone()).unwrap();
|
||||||
|
let prefixes: Vec<&str> = out.lines().map(|line| &line[..3]).collect();
|
||||||
|
assert_eq!(prefixes, ["<3>", "<4>", "<6>", "<7>"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn journal_stream_must_name_the_same_file() {
|
||||||
|
assert!(names_stream("8:12345", 8, 12345));
|
||||||
|
assert!(!names_stream("8:12345", 8, 54321));
|
||||||
|
assert!(!names_stream("9:12345", 8, 12345));
|
||||||
|
assert!(!names_stream("8", 8, 12345));
|
||||||
|
assert!(!names_stream("", 8, 12345));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -20,6 +20,8 @@ use pdfs_core::control::{
|
|||||||
use pdfs_core::db::Db;
|
use pdfs_core::db::Db;
|
||||||
use pdfs_core::service;
|
use pdfs_core::service;
|
||||||
|
|
||||||
|
mod journal;
|
||||||
|
|
||||||
/// `println!` for this binary, shadowing std's, which panics when stdout is
|
/// `println!` for this binary, shadowing std's, which panics when stdout is
|
||||||
/// closed: `pdfs ls | head` ended in a "Broken pipe" panic once `head` had its
|
/// closed: `pdfs ls | head` ended in a "Broken pipe" panic once `head` had its
|
||||||
/// lines. This one hands the failure to [`stdout_written`] instead.
|
/// lines. This one hands the failure to [`stdout_written`] instead.
|
||||||
@@ -879,15 +881,22 @@ fn cli_error(kind: ErrorKind, message: &str) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> Result<()> {
|
fn main() -> Result<()> {
|
||||||
tracing_subscriber::fmt()
|
let filter = tracing_subscriber::EnvFilter::try_from_default_env()
|
||||||
.with_env_filter(
|
|
||||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
|
||||||
// `pgp` logs a benign WARN when it re-serializes a Proton secret-key
|
// `pgp` logs a benign WARN when it re-serializes a Proton secret-key
|
||||||
// packet with a longer (but equivalent) length encoding than the
|
// packet with a longer (but equivalent) length encoding than the
|
||||||
// original; drop it to error so normal runs stay quiet.
|
// original; drop it to error so normal runs stay quiet.
|
||||||
.unwrap_or_else(|_| "info,pgp=error".into()),
|
.unwrap_or_else(|_| "info,pgp=error".into());
|
||||||
)
|
if journal::stdout_is_journal() {
|
||||||
|
// journalctl colours lines by the priority the prefix gives them, so
|
||||||
|
// the escape codes would only be noise in the stored text.
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_env_filter(filter)
|
||||||
|
.with_ansi(false)
|
||||||
|
.event_format(journal::Prioritized(tracing_subscriber::fmt::format()))
|
||||||
.init();
|
.init();
|
||||||
|
} else {
|
||||||
|
tracing_subscriber::fmt().with_env_filter(filter).init();
|
||||||
|
}
|
||||||
|
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
JSON_OUTPUT.store(cli.json, std::sync::atomic::Ordering::Relaxed);
|
JSON_OUTPUT.store(cli.json, std::sync::atomic::Ordering::Relaxed);
|
||||||
@@ -1556,9 +1565,13 @@ fn cmd_versions(action: VersionCmd) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_members(path: PathBuf) -> Result<()> {
|
fn cmd_members(path: PathBuf) -> Result<()> {
|
||||||
match control_request(CtlRequest::ListShare {
|
let response = control_request(CtlRequest::ListShare {
|
||||||
path: path_arg(&path)?,
|
path: path_arg(&path)?,
|
||||||
})? {
|
})?;
|
||||||
|
if emit_json(&response)? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
match response {
|
||||||
CtlResponse::Share { entries, link } => {
|
CtlResponse::Share { entries, link } => {
|
||||||
if entries.is_empty() {
|
if entries.is_empty() {
|
||||||
println!("No members or pending invitations.");
|
println!("No members or pending invitations.");
|
||||||
@@ -1625,6 +1638,7 @@ fn cmd_public_link(action: PublicLinkCmd) -> Result<()> {
|
|||||||
password,
|
password,
|
||||||
expires,
|
expires,
|
||||||
})? {
|
})? {
|
||||||
|
response if emit_json(&response)? => {}
|
||||||
CtlResponse::PublicLink { link } => {
|
CtlResponse::PublicLink { link } => {
|
||||||
println!("{}", link.url.as_deref().unwrap_or("(no url returned)"));
|
println!("{}", link.url.as_deref().unwrap_or("(no url returned)"));
|
||||||
}
|
}
|
||||||
@@ -1646,6 +1660,7 @@ fn cmd_public_link(action: PublicLinkCmd) -> Result<()> {
|
|||||||
password: link_change(password, no_password),
|
password: link_change(password, no_password),
|
||||||
expires: link_change(expires, no_expiry),
|
expires: link_change(expires, no_expiry),
|
||||||
})? {
|
})? {
|
||||||
|
response if emit_json(&response)? => {}
|
||||||
CtlResponse::PublicLink { link } => {
|
CtlResponse::PublicLink { link } => {
|
||||||
println!("{}", link.url.as_deref().unwrap_or("(no url returned)"));
|
println!("{}", link.url.as_deref().unwrap_or("(no url returned)"));
|
||||||
}
|
}
|
||||||
@@ -3048,9 +3063,13 @@ fn cmd_copy(sources: Vec<PathBuf>, new_parent: PathBuf) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn cmd_du(path: PathBuf) -> Result<()> {
|
fn cmd_du(path: PathBuf) -> Result<()> {
|
||||||
match control_request(CtlRequest::FolderSize {
|
let response = control_request(CtlRequest::FolderSize {
|
||||||
path: path_arg(&path)?,
|
path: path_arg(&path)?,
|
||||||
})? {
|
})?;
|
||||||
|
if emit_json(&response)? {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
match response {
|
||||||
CtlResponse::FolderSize { size, items } => {
|
CtlResponse::FolderSize { size, items } => {
|
||||||
let noun = if items == 1 { "item" } else { "items" };
|
let noun = if items == 1 { "item" } else { "items" };
|
||||||
println!(
|
println!(
|
||||||
@@ -3086,6 +3105,24 @@ fn human_secs(secs: u64) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The sync engine's line in `pdfs diagnostics`: whether it runs, and which
|
||||||
|
/// folder's pass it is on.
|
||||||
|
fn sync_engine_line(sync: &pdfs_core::control::SyncEngineState) -> String {
|
||||||
|
let mut line = match (sync.running, sync.pass_folder) {
|
||||||
|
(false, _) => "STOPPED; synced folders do not sync until the daemon restarts".to_string(),
|
||||||
|
(true, None) => "idle".to_string(),
|
||||||
|
(true, Some(folder)) => format!(
|
||||||
|
"pass on folder {folder} for {}, last progress {} ago",
|
||||||
|
human_secs(sync.pass_age_secs),
|
||||||
|
human_secs(sync.pass_idle_secs),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
if sync.panics > 0 {
|
||||||
|
line.push_str(&format!("; {} pass(es) panicked", sync.panics));
|
||||||
|
}
|
||||||
|
line
|
||||||
|
}
|
||||||
|
|
||||||
fn cmd_diagnostics() -> Result<()> {
|
fn cmd_diagnostics() -> Result<()> {
|
||||||
let response = control_request(CtlRequest::Diagnostics)?;
|
let response = control_request(CtlRequest::Diagnostics)?;
|
||||||
if emit_json(&response)? {
|
if emit_json(&response)? {
|
||||||
@@ -3121,6 +3158,9 @@ fn cmd_diagnostics() -> Result<()> {
|
|||||||
human_secs(worker.age_secs)
|
human_secs(worker.age_secs)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if let Some(sync) = &d.sync {
|
||||||
|
println!("sync engine: {}", sync_engine_line(sync));
|
||||||
|
}
|
||||||
if d.inflight.is_empty() {
|
if d.inflight.is_empty() {
|
||||||
println!("in-flight requests: none");
|
println!("in-flight requests: none");
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -1980,6 +1980,29 @@ pub struct Diagnostics {
|
|||||||
pub control_handler_limit: usize,
|
pub control_handler_limit: usize,
|
||||||
/// Queued mutations not yet drained.
|
/// Queued mutations not yet drained.
|
||||||
pub pending_ops: u64,
|
pub pending_ops: u64,
|
||||||
|
/// The folder-sync engine. `None` from a daemon that predates the field.
|
||||||
|
#[serde(default)]
|
||||||
|
pub sync: Option<SyncEngineState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the folder-sync engine is doing, for [`Diagnostics`].
|
||||||
|
///
|
||||||
|
/// The engine is one thread that runs every synced folder's passes in turn.
|
||||||
|
/// Dead or stuck, it looked exactly like an idle one: workers idle, nothing
|
||||||
|
/// queued, nothing in the journal (`docs/BUGS.md` B259).
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
|
||||||
|
pub struct SyncEngineState {
|
||||||
|
/// Whether the engine thread is running. Once it has ended, nothing syncs
|
||||||
|
/// until the daemon restarts.
|
||||||
|
pub running: bool,
|
||||||
|
/// The synced folder whose pass is running, if one is.
|
||||||
|
pub pass_folder: Option<i64>,
|
||||||
|
/// How long that pass has been running.
|
||||||
|
pub pass_age_secs: u64,
|
||||||
|
/// How long since that pass last reported progress.
|
||||||
|
pub pass_idle_secs: u64,
|
||||||
|
/// Passes that ended in a panic since the daemon started.
|
||||||
|
pub panics: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The daemon's reply to a [`Request`].
|
/// The daemon's reply to a [`Request`].
|
||||||
|
|||||||
@@ -989,9 +989,15 @@ impl Core {
|
|||||||
.db
|
.db
|
||||||
.sync_entries(id)
|
.sync_entries(id)
|
||||||
.map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?;
|
.map_err(|e| SwitchBlocked::Failed(format!("db: {e:?}")))?;
|
||||||
if let Some(reason) =
|
let rules = self.ignore_rules(&local);
|
||||||
sync::mirror_tree_unsynced(&local, &baseline, &self.ignore_rules(&local))
|
// Emptied before any pass saw it, the folder is still `idle`, and
|
||||||
{
|
// the check below would refuse the switch for good. The push pass
|
||||||
|
// of a queued switch forgets those paths instead (`docs/BUGS.md`
|
||||||
|
// B255).
|
||||||
|
if sync::mirror_tree_gone(&local, &baseline, &rules) {
|
||||||
|
return Err(SwitchBlocked::NotNow);
|
||||||
|
}
|
||||||
|
if let Some(reason) = sync::mirror_tree_unsynced(&local, &baseline, &rules) {
|
||||||
return Err(SwitchBlocked::Failed(format!(
|
return Err(SwitchBlocked::Failed(format!(
|
||||||
"going online-only would delete what is only on this computer: {reason}"
|
"going online-only would delete what is only on this computer: {reason}"
|
||||||
)));
|
)));
|
||||||
|
|||||||
@@ -163,6 +163,7 @@ impl Core {
|
|||||||
control_handlers: super::control::active_handlers(),
|
control_handlers: super::control::active_handlers(),
|
||||||
control_handler_limit: super::control::handler_limit(),
|
control_handler_limit: super::control::handler_limit(),
|
||||||
pending_ops: self.pending_ops_quickly(),
|
pending_ops: self.pending_ops_quickly(),
|
||||||
|
sync: Some(self.sync_engine.report()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -728,6 +728,8 @@ struct Core {
|
|||||||
/// can say what a pass is doing rather than just "syncing". An entry exists
|
/// can say what a pass is doing rather than just "syncing". An entry exists
|
||||||
/// only while that folder's reconcile pass is running.
|
/// only while that folder's reconcile pass is running.
|
||||||
sync_progress: Arc<Mutex<HashMap<i64, SyncProgress>>>,
|
sync_progress: Arc<Mutex<HashMap<i64, SyncProgress>>>,
|
||||||
|
/// Whether the folder-sync engine is alive, and which pass it is on.
|
||||||
|
sync_engine: Arc<sync::EngineHealth>,
|
||||||
/// Channel to the folder-sync engine: nudges it to
|
/// Channel to the folder-sync engine: nudges it to
|
||||||
/// reconcile a folder, reconcile everything, or re-scan its watch set.
|
/// reconcile a folder, reconcile everything, or re-scan its watch set.
|
||||||
sync_tx: std::sync::mpsc::Sender<sync::SyncMsg>,
|
sync_tx: std::sync::mpsc::Sender<sync::SyncMsg>,
|
||||||
@@ -5989,6 +5991,7 @@ impl Core {
|
|||||||
if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) {
|
if let Some(p) = self.sync_progress.lock().get_mut(&folder_id) {
|
||||||
f(p);
|
f(p);
|
||||||
}
|
}
|
||||||
|
self.sync_engine.moved();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set the number of items the scan expects to check, from the size of the
|
/// Set the number of items the scan expects to check, from the size of the
|
||||||
@@ -7312,13 +7315,15 @@ mod size_upgrade_tests {
|
|||||||
fn a_listing_waits_for_the_whole_size_batch_but_not_forever() {
|
fn a_listing_waits_for_the_whole_size_batch_but_not_forever() {
|
||||||
let slot = Arc::new(SizeUpgrade::default());
|
let slot = Arc::new(SizeUpgrade::default());
|
||||||
let worker = slot.clone();
|
let worker = slot.clone();
|
||||||
|
// Read before the spawn: the worker's sleeps can start before this
|
||||||
|
// thread runs again, which made the bound below fail on a busy runner.
|
||||||
|
let started = Instant::now();
|
||||||
let t = std::thread::spawn(move || {
|
let t = std::thread::spawn(move || {
|
||||||
std::thread::sleep(Duration::from_millis(30));
|
std::thread::sleep(Duration::from_millis(30));
|
||||||
worker.chunk_done();
|
worker.chunk_done();
|
||||||
std::thread::sleep(Duration::from_millis(60));
|
std::thread::sleep(Duration::from_millis(60));
|
||||||
worker.finish();
|
worker.finish();
|
||||||
});
|
});
|
||||||
let started = Instant::now();
|
|
||||||
slot.wait_finished(started + Duration::from_secs(5));
|
slot.wait_finished(started + Duration::from_secs(5));
|
||||||
// Past the first chunk, up to the end of the batch.
|
// Past the first chunk, up to the end of the batch.
|
||||||
assert!(started.elapsed() >= Duration::from_millis(90));
|
assert!(started.elapsed() >= Duration::from_millis(90));
|
||||||
|
|||||||
@@ -507,6 +507,7 @@ pub(crate) fn mount_with(
|
|||||||
events: Arc::new(Default::default()),
|
events: Arc::new(Default::default()),
|
||||||
indexing: Arc::new(AtomicBool::new(false)),
|
indexing: Arc::new(AtomicBool::new(false)),
|
||||||
sync_progress: Arc::new(Mutex::new(HashMap::new())),
|
sync_progress: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
sync_engine: Arc::default(),
|
||||||
sync_tx,
|
sync_tx,
|
||||||
mounts: Arc::new(Mutex::new(SecondaryMountRegistry::default())),
|
mounts: Arc::new(Mutex::new(SecondaryMountRegistry::default())),
|
||||||
sync_locks: Arc::new(Mutex::new(HashMap::new())),
|
sync_locks: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
|||||||
@@ -130,6 +130,15 @@ impl Daemon {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the mount believes Proton is reachable, as `pdfs status` shows
|
||||||
|
/// it.
|
||||||
|
pub(crate) fn online(&self) -> io::Result<bool> {
|
||||||
|
match self.request(&Request::Status)? {
|
||||||
|
Response::Status { online, .. } => Ok(online),
|
||||||
|
other => Err(io::Error::other(format!("unexpected answer {other:?}"))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Send one control request, as the `pdfs` CLI does, and fail on a
|
/// Send one control request, as the `pdfs` CLI does, and fail on a
|
||||||
/// refusal.
|
/// refusal.
|
||||||
pub(crate) fn request(&self, request: &Request) -> io::Result<Response> {
|
pub(crate) fn request(&self, request: &Request) -> io::Result<Response> {
|
||||||
@@ -648,7 +657,19 @@ mod tests {
|
|||||||
.pending()
|
.pending()
|
||||||
.is_ok_and(|items| items.iter().any(|item| item.attempts > 0))
|
.is_ok_and(|items| items.iter().any(|item| item.attempts > 0))
|
||||||
}));
|
}));
|
||||||
|
// The drain records the failure just before it takes the mount
|
||||||
|
// offline. Left online, only the event poll would, much later.
|
||||||
|
assert!(wait_until(Duration::from_secs(2), || daemon
|
||||||
|
.online()
|
||||||
|
.is_ok_and(|online| !online)));
|
||||||
daemon.client.set_online(true);
|
daemon.client.set_online(true);
|
||||||
|
// The probe that sees the link return runs 2 s after the drop, then
|
||||||
|
// 6 s, and a loaded run can reach this line after the first one. So
|
||||||
|
// the 3 s below count from the mount being back, not from the link:
|
||||||
|
// an op left on its backoff waits 4 s or more after its failure.
|
||||||
|
assert!(wait_until(Duration::from_secs(30), || daemon
|
||||||
|
.online()
|
||||||
|
.unwrap_or(false)));
|
||||||
assert!(
|
assert!(
|
||||||
wait_until(Duration::from_secs(3), || {
|
wait_until(Duration::from_secs(3), || {
|
||||||
daemon.pending().is_ok_and(|items| items.is_empty())
|
daemon.pending().is_ok_and(|items| items.is_empty())
|
||||||
@@ -2981,4 +3002,69 @@ mod tests {
|
|||||||
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||||
let _ = std::fs::remove_dir_all(&dir);
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Emptied before any pass saw it, the folder is still `idle`, and the
|
||||||
|
/// switch was refused for good: every path read as "deleted here but not on
|
||||||
|
/// Proton Drive yet" (B255, found by the account run).
|
||||||
|
#[test]
|
||||||
|
#[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"]
|
||||||
|
fn a_folder_emptied_before_any_pass_still_goes_on_demand() {
|
||||||
|
let drive = FakeDrive::new();
|
||||||
|
let dir = scratch("ondemand-emptied-idle");
|
||||||
|
let daemon = Daemon::start(&dir, drive.client(1, Faults::lan())).unwrap();
|
||||||
|
let local = dir.join("docs");
|
||||||
|
let id = sync_folder(&daemon, &local, "mirror");
|
||||||
|
std::fs::create_dir(local.join("sub")).unwrap();
|
||||||
|
std::fs::write(local.join("sub/a.txt"), b"alpha").unwrap();
|
||||||
|
std::fs::write(local.join("sub/b.txt"), b"beta").unwrap();
|
||||||
|
daemon.request(&Request::SyncNow { id: Some(id) }).unwrap();
|
||||||
|
let on_drive = |name: &str, content: &[u8]| {
|
||||||
|
drive.tree().iter().any(|(path, entry)| {
|
||||||
|
path.ends_with(name) && *entry == Entry::File(Arc::new(content.to_vec()))
|
||||||
|
})
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
wait_until(Duration::from_secs(30), || on_drive("sub/a.txt", b"alpha")
|
||||||
|
&& on_drive("sub/b.txt", b"beta")
|
||||||
|
&& listed_folder(&daemon, id).is_some_and(|f| f.state == "idle")),
|
||||||
|
"the folder synced: {:?}",
|
||||||
|
listed_folder(&daemon, id)
|
||||||
|
);
|
||||||
|
|
||||||
|
// In one rename, so no pass sees half of it, and asked at once, well
|
||||||
|
// inside the watcher's debounce, so no pass sees it empty either.
|
||||||
|
std::fs::rename(local.join("sub"), dir.join("moved-away")).unwrap();
|
||||||
|
let asked = daemon.request(&Request::SetSyncFolderMode {
|
||||||
|
id,
|
||||||
|
mode: "ondemand".into(),
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
matches!(asked, Ok(Response::Ok { .. })),
|
||||||
|
"the switch was refused: {asked:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
listed_folder(&daemon, id)
|
||||||
|
.is_some_and(|f| f.pending_mode.as_deref() == Some("ondemand")),
|
||||||
|
"the switch waits for a pass: {:?}",
|
||||||
|
listed_folder(&daemon, id)
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
wait_until(Duration::from_secs(30), || listed_folder(&daemon, id)
|
||||||
|
.is_some_and(|f| f.mode == "ondemand" && f.pending_mode.is_none())
|
||||||
|
&& is_mounted(&local)),
|
||||||
|
"the queued switch landed: {:?}",
|
||||||
|
listed_folder(&daemon, id)
|
||||||
|
);
|
||||||
|
assert_eq!(std::fs::read(local.join("sub/a.txt")).unwrap(), b"alpha");
|
||||||
|
assert_eq!(std::fs::read(local.join("sub/b.txt")).unwrap(), b"beta");
|
||||||
|
assert!(
|
||||||
|
on_drive("sub/a.txt", b"alpha") && on_drive("sub/b.txt", b"beta"),
|
||||||
|
"nothing was trashed on Drive: {:?}",
|
||||||
|
drive.tree().keys().collect::<Vec<_>>()
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown)));
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -36,6 +36,11 @@ const TICK: Duration = Duration::from_secs(15);
|
|||||||
/// would train everyone to ignore the real one.
|
/// would train everyone to ignore the real one.
|
||||||
const STALL_AFTER: Duration = Duration::from_secs(120);
|
const STALL_AFTER: Duration = Duration::from_secs(120);
|
||||||
|
|
||||||
|
/// How long a sync pass may go without reporting progress before it is called
|
||||||
|
/// out. Much longer than [`STALL_AFTER`]: one large file in a pass reports
|
||||||
|
/// nothing until its whole transfer is done.
|
||||||
|
const SYNC_STALL_AFTER: Duration = Duration::from_secs(900);
|
||||||
|
|
||||||
/// How often resident size goes into the log.
|
/// How often resident size goes into the log.
|
||||||
const RSS_EVERY: Duration = Duration::from_secs(300);
|
const RSS_EVERY: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
@@ -137,6 +142,26 @@ fn report_stalls(report: &pdfs_core::control::Diagnostics, last_completed: (u64,
|
|||||||
"the transfer lane has not completed a job since the last check"
|
"the transfer lane has not completed a job since the last check"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
// The engine is one thread for every synced folder, so a dead or stuck one
|
||||||
|
// stops all of them while the worker pool looks idle (B259).
|
||||||
|
if let Some(sync) = &report.sync {
|
||||||
|
if !sync.running {
|
||||||
|
stalled = true;
|
||||||
|
warn!(
|
||||||
|
"the sync engine is not running; synced folders do not sync until the daemon restarts"
|
||||||
|
);
|
||||||
|
} else if let Some(folder) = sync.pass_folder
|
||||||
|
&& sync.pass_idle_secs >= SYNC_STALL_AFTER.as_secs()
|
||||||
|
{
|
||||||
|
stalled = true;
|
||||||
|
warn!(
|
||||||
|
folder,
|
||||||
|
age_secs = sync.pass_age_secs,
|
||||||
|
idle_secs = sync.pass_idle_secs,
|
||||||
|
"a sync pass has not reported progress for a long time"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
stalled
|
stalled
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,7 +194,7 @@ fn probe(control_socket: &Path) -> bool {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use pdfs_core::control::{Diagnostics, InflightRequest, WorkerState};
|
use pdfs_core::control::{Diagnostics, InflightRequest, SyncEngineState, WorkerState};
|
||||||
|
|
||||||
fn busy_worker(age_secs: u64) -> WorkerState {
|
fn busy_worker(age_secs: u64) -> WorkerState {
|
||||||
WorkerState {
|
WorkerState {
|
||||||
@@ -226,4 +251,45 @@ mod tests {
|
|||||||
assert!(!report_stalls(&report, (0, 30)));
|
assert!(!report_stalls(&report, (0, 30)));
|
||||||
assert!(report_stalls(&report, (0, 40)));
|
assert!(report_stalls(&report, (0, 40)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn with_sync(sync: SyncEngineState) -> Diagnostics {
|
||||||
|
Diagnostics {
|
||||||
|
sync: Some(sync),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A sync engine that has ended stops every synced folder while the
|
||||||
|
/// workers look idle, which is what issue #53 looked like (B259).
|
||||||
|
#[test]
|
||||||
|
fn a_stopped_sync_engine_is_a_stall() {
|
||||||
|
assert!(report_stalls(
|
||||||
|
&with_sync(SyncEngineState::default()),
|
||||||
|
(0, 0)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A pass that keeps reporting progress is working, however long it runs;
|
||||||
|
/// one that has gone quiet past the threshold is not.
|
||||||
|
#[test]
|
||||||
|
fn only_a_quiet_sync_pass_is_a_stall() {
|
||||||
|
let busy = SyncEngineState {
|
||||||
|
running: true,
|
||||||
|
pass_folder: Some(3),
|
||||||
|
pass_age_secs: SYNC_STALL_AFTER.as_secs() * 4,
|
||||||
|
pass_idle_secs: 5,
|
||||||
|
panics: 0,
|
||||||
|
};
|
||||||
|
assert!(!report_stalls(&with_sync(busy.clone()), (0, 0)));
|
||||||
|
let quiet = SyncEngineState {
|
||||||
|
pass_idle_secs: SYNC_STALL_AFTER.as_secs(),
|
||||||
|
..busy
|
||||||
|
};
|
||||||
|
assert!(report_stalls(&with_sync(quiet), (0, 0)));
|
||||||
|
let idle = SyncEngineState {
|
||||||
|
running: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(!report_stalls(&with_sync(idle), (0, 0)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -48,8 +48,10 @@ mod engine;
|
|||||||
mod local_scan;
|
mod local_scan;
|
||||||
mod model;
|
mod model;
|
||||||
mod planner;
|
mod planner;
|
||||||
|
#[cfg(test)]
|
||||||
|
use engine::Running;
|
||||||
pub(crate) use engine::SYNC_CONCURRENCY;
|
pub(crate) use engine::SYNC_CONCURRENCY;
|
||||||
pub(crate) use engine::{SyncMsg, spawn};
|
pub(crate) use engine::{EngineHealth, SyncMsg, spawn};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use engine::{classify, event_target, settle_with};
|
use engine::{classify, event_target, settle_with};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -67,6 +69,11 @@ use planner::{
|
|||||||
plan_file, rel_to_path, remote_sig, remote_subtree_changed, revision_moved, tracked_baseline,
|
plan_file, rel_to_path, remote_sig, remote_subtree_changed, revision_moved, tracked_baseline,
|
||||||
unchanged_remote_size,
|
unchanged_remote_size,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// A pass that took at least this long is logged at info even when it changed
|
||||||
|
/// nothing, so the journal says where a slow engine spends its time.
|
||||||
|
const SLOW_PASS: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
/// What a [`Pending`] op did, so the reconcile can update shared state on the
|
/// What a [`Pending`] op did, so the reconcile can update shared state on the
|
||||||
/// engine thread (never inside a task).
|
/// engine thread (never inside a task).
|
||||||
enum Applied {
|
enum Applied {
|
||||||
@@ -154,6 +161,14 @@ impl Core {
|
|||||||
// The cost is one small file read against a pass that is about to walk the
|
// The cost is one small file read against a pass that is about to walk the
|
||||||
// whole tree.
|
// whole tree.
|
||||||
let rules = self.ignore_rules(&local_root);
|
let rules = self.ignore_rules(&local_root);
|
||||||
|
// A first pass is the one a user waits on after adding or restoring a
|
||||||
|
// folder, so it is said at info; the poll's passes stay at debug.
|
||||||
|
if folder.last_sync == 0 {
|
||||||
|
info!(id = folder.id, path = %folder.local_path, "sync: first pass starting");
|
||||||
|
} else {
|
||||||
|
debug!(id = folder.id, "sync: pass starting");
|
||||||
|
}
|
||||||
|
let started = Instant::now();
|
||||||
self.progress_begin(folder.id);
|
self.progress_begin(folder.id);
|
||||||
let result = if push_only {
|
let result = if push_only {
|
||||||
self.push_pass(folder.id, &local_root, &remote_root, &rules)
|
self.push_pass(folder.id, &local_root, &remote_root, &rules)
|
||||||
@@ -195,6 +210,18 @@ impl Core {
|
|||||||
"idle"
|
"idle"
|
||||||
};
|
};
|
||||||
let _ = self.db.sync_folder_set_state(folder.id, state, now_secs());
|
let _ = self.db.sync_folder_set_state(folder.id, state, now_secs());
|
||||||
|
let elapsed = started.elapsed();
|
||||||
|
if folder.last_sync == 0 || !outcome.is_empty() || elapsed >= SLOW_PASS {
|
||||||
|
info!(
|
||||||
|
id = folder.id,
|
||||||
|
state,
|
||||||
|
elapsed_ms = elapsed.as_millis() as u64,
|
||||||
|
summary = %outcome.summary(),
|
||||||
|
"sync: pass finished"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
debug!(id = folder.id, state, "sync: pass finished");
|
||||||
|
}
|
||||||
self.report_unscanned(folder.id, name, &outcome.unscanned);
|
self.report_unscanned(folder.id, name, &outcome.unscanned);
|
||||||
// Summarise a pass that actually moved something. A poll over an
|
// Summarise a pass that actually moved something. A poll over an
|
||||||
// unchanged folder does nothing and says nothing — otherwise the
|
// unchanged folder does nothing and says nothing — otherwise the
|
||||||
@@ -2253,6 +2280,27 @@ pub(crate) fn mirror_tree_unsynced(
|
|||||||
.map(|r| format!("{r} was deleted here but not on Proton Drive yet"))
|
.map(|r| format!("{r} was deleted here but not on Proton Drive yet"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a mirror folder's local copy is gone in full: the folder is still
|
||||||
|
/// there, but not one tracked path of its baseline is.
|
||||||
|
///
|
||||||
|
/// [`mirror_tree_unsynced`] reads that as deletions not carried up yet, which
|
||||||
|
/// would refuse the switch to on-demand for good. It is the wipe guard's case
|
||||||
|
/// instead, which the push pass a queued switch runs settles without trashing
|
||||||
|
/// anything on Drive (#46), so the switch waits for that pass.
|
||||||
|
pub(crate) fn mirror_tree_gone(
|
||||||
|
root: &Path,
|
||||||
|
baseline: &HashMap<String, StoredSyncEntry>,
|
||||||
|
rules: &IgnoreRules,
|
||||||
|
) -> bool {
|
||||||
|
let tracked = filter_baseline(baseline, rules);
|
||||||
|
root.is_dir()
|
||||||
|
&& !tracked.is_empty()
|
||||||
|
&& tracked.keys().all(|rel| {
|
||||||
|
std::fs::symlink_metadata(root.join(rel_to_path(rel)))
|
||||||
|
.is_err_and(|e| e.kind() == std::io::ErrorKind::NotFound)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// The walk behind both checks. `rules`, when given, only names an ignored
|
/// The walk behind both checks. `rules`, when given, only names an ignored
|
||||||
/// path as such in the reason; it is still something only this disk has.
|
/// path as such in the reason; it is still something only this disk has.
|
||||||
fn unsynced_local(
|
fn unsynced_local(
|
||||||
@@ -2417,6 +2465,50 @@ fn system_mtime_ns(meta: &std::fs::Metadata) -> Option<i64> {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// A panic in a pass ends that pass and is counted; the engine is not
|
||||||
|
/// left believing the pass still runs (`docs/BUGS.md` B259).
|
||||||
|
#[test]
|
||||||
|
fn a_panicking_pass_is_contained_and_counted() {
|
||||||
|
let health = EngineHealth::default();
|
||||||
|
assert!(!health.contain(7, || panic!("a pass that panics")));
|
||||||
|
let report = health.report();
|
||||||
|
assert_eq!(report.panics, 1);
|
||||||
|
assert_eq!(report.pass_folder, None);
|
||||||
|
assert!(health.contain(7, || {}));
|
||||||
|
assert_eq!(health.report().panics, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// While a pass runs, the report names its folder, which is what tells a
|
||||||
|
/// long pass apart from an idle engine.
|
||||||
|
#[test]
|
||||||
|
fn the_running_pass_is_reported() {
|
||||||
|
let health = EngineHealth::default();
|
||||||
|
let mut seen = None;
|
||||||
|
health.contain(7, || seen = health.report().pass_folder);
|
||||||
|
assert_eq!(seen, Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The engine reads as running only while its thread lives, including
|
||||||
|
/// when the thread ends in a panic.
|
||||||
|
#[test]
|
||||||
|
fn the_engine_reads_as_stopped_once_its_thread_ends() {
|
||||||
|
let health = Arc::new(EngineHealth::default());
|
||||||
|
let thread_health = health.clone();
|
||||||
|
let (started_tx, started_rx) = std::sync::mpsc::channel();
|
||||||
|
let (end_tx, end_rx) = std::sync::mpsc::channel::<()>();
|
||||||
|
let thread = std::thread::spawn(move || {
|
||||||
|
let _running = Running::start(&thread_health);
|
||||||
|
started_tx.send(()).unwrap();
|
||||||
|
let _ = end_rx.recv();
|
||||||
|
panic!("the engine thread panics");
|
||||||
|
});
|
||||||
|
started_rx.recv().unwrap();
|
||||||
|
assert!(health.report().running);
|
||||||
|
end_tx.send(()).unwrap();
|
||||||
|
assert!(thread.join().is_err());
|
||||||
|
assert!(!health.report().running);
|
||||||
|
}
|
||||||
|
|
||||||
/// A removal of something already gone has produced the state the sync
|
/// A removal of something already gone has produced the state the sync
|
||||||
/// engine wanted, so its baseline is dropped rather than retried forever.
|
/// engine wanted, so its baseline is dropped rather than retried forever.
|
||||||
/// Anything else stays an error and keeps the baseline (`docs/BUGS.md` B42).
|
/// Anything else stays an error and keeps the baseline (`docs/BUGS.md` B42).
|
||||||
@@ -3538,6 +3630,26 @@ mod tests {
|
|||||||
assert!(reason.unwrap().contains("link"));
|
assert!(reason.unwrap().contains("link"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Emptied before any pass saw it, the folder waits for the push pass that
|
||||||
|
/// forgets those paths, instead of being refused for good (B255).
|
||||||
|
#[test]
|
||||||
|
fn only_a_local_copy_gone_in_full_counts_as_gone() {
|
||||||
|
let (root, baseline) = synced_subtree("evict-gone");
|
||||||
|
let rules = rules_for(&[]);
|
||||||
|
let whole = mirror_tree_gone(&root, &baseline, &rules);
|
||||||
|
std::fs::remove_dir_all(root.join("dir/sub")).unwrap();
|
||||||
|
let partly = mirror_tree_gone(&root, &baseline, &rules);
|
||||||
|
std::fs::remove_dir_all(root.join("dir")).unwrap();
|
||||||
|
let emptied = mirror_tree_gone(&root, &baseline, &rules);
|
||||||
|
let none_synced = mirror_tree_gone(&root, &HashMap::new(), &rules);
|
||||||
|
std::fs::remove_dir_all(&root).unwrap();
|
||||||
|
let no_root = mirror_tree_gone(&root, &baseline, &rules);
|
||||||
|
assert!(!whole && !partly, "a deletion with survivors is a real one");
|
||||||
|
assert!(emptied);
|
||||||
|
assert!(!none_synced, "an empty baseline has nothing to lose");
|
||||||
|
assert!(!no_root, "a missing folder is the pass's error to report");
|
||||||
|
}
|
||||||
|
|
||||||
/// The same rule one level up: if the sync root itself cannot be read, the
|
/// The same rule one level up: if the sync root itself cannot be read, the
|
||||||
/// pass sees nothing, and "nothing" must never mean "delete everything".
|
/// pass sees nothing, and "nothing" must never mean "delete everything".
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
//! Filesystem watcher, polling, and debounce control plane.
|
//! Filesystem watcher, polling, and debounce control plane.
|
||||||
|
|
||||||
|
use std::panic::AssertUnwindSafe;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
|
||||||
|
use pdfs_core::control::SyncEngineState;
|
||||||
|
use tracing::error;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
/// How long the watcher must go quiet before a burst counts as finished.
|
/// How long the watcher must go quiet before a burst counts as finished.
|
||||||
@@ -53,8 +59,124 @@ pub(crate) fn spawn(core: Core, rx: Receiver<SyncMsg>) -> Option<std::thread::Jo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether the engine is alive and which pass it is on, for
|
||||||
|
/// [`Core::diagnostics`] and the supervisor.
|
||||||
|
///
|
||||||
|
/// A panic in a pass used to end the engine thread. Every later add, sync-now,
|
||||||
|
/// file event and poll then went into a channel nobody read, the folder stayed
|
||||||
|
/// `syncing`, and nothing said so until a restart (`docs/BUGS.md` B259).
|
||||||
|
#[derive(Default)]
|
||||||
|
pub(crate) struct EngineHealth {
|
||||||
|
running: AtomicBool,
|
||||||
|
pass: Mutex<Option<PassClock>>,
|
||||||
|
panics: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct PassClock {
|
||||||
|
folder: i64,
|
||||||
|
started: Instant,
|
||||||
|
moved: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EngineHealth {
|
||||||
|
/// Note that the running pass reported progress.
|
||||||
|
pub(crate) fn moved(&self) {
|
||||||
|
if let Some(pass) = self.pass.lock().as_mut() {
|
||||||
|
pass.moved = Instant::now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The engine's state now. `try_lock`, like the rest of the diagnostics:
|
||||||
|
/// a report must not wait on the thing it reports about.
|
||||||
|
pub(crate) fn report(&self) -> SyncEngineState {
|
||||||
|
let pass = self.pass.try_lock().and_then(|pass| *pass);
|
||||||
|
SyncEngineState {
|
||||||
|
running: self.running.load(Ordering::Relaxed),
|
||||||
|
pass_folder: pass.map(|p| p.folder),
|
||||||
|
pass_age_secs: pass.map_or(0, |p| p.started.elapsed().as_secs()),
|
||||||
|
pass_idle_secs: pass.map_or(0, |p| p.moved.elapsed().as_secs()),
|
||||||
|
panics: self.panics.load(Ordering::Relaxed),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run `pass` over `folder` as the current pass. Returns `false` when it
|
||||||
|
/// panicked: the panic ends that pass, not the engine.
|
||||||
|
pub(super) fn contain(&self, folder: i64, pass: impl FnOnce()) -> bool {
|
||||||
|
let now = Instant::now();
|
||||||
|
*self.pass.lock() = Some(PassClock {
|
||||||
|
folder,
|
||||||
|
started: now,
|
||||||
|
moved: now,
|
||||||
|
});
|
||||||
|
let finished = std::panic::catch_unwind(AssertUnwindSafe(pass)).is_ok();
|
||||||
|
*self.pass.lock() = None;
|
||||||
|
if !finished {
|
||||||
|
self.panics.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
finished
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Marks the engine running for as long as it lives.
|
||||||
|
pub(super) struct Running<'a>(&'a EngineHealth);
|
||||||
|
|
||||||
|
impl<'a> Running<'a> {
|
||||||
|
pub(super) fn start(health: &'a EngineHealth) -> Self {
|
||||||
|
health.running.store(true, Ordering::Relaxed);
|
||||||
|
Self(health)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Running<'_> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.running.store(false, Ordering::Relaxed);
|
||||||
|
if std::thread::panicking() {
|
||||||
|
error!("sync engine stopped by a panic; nothing syncs until the daemon restarts");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One folder's pass, contained: a panic in it sets the folder to `error` and
|
||||||
|
/// leaves the engine running for the next one.
|
||||||
|
fn run_pass(core: &Core, folder: &StoredSyncFolder) {
|
||||||
|
if core
|
||||||
|
.sync_engine
|
||||||
|
.contain(folder.id, || core.reconcile_folder(folder))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
core.progress_end(folder.id);
|
||||||
|
let _ = core
|
||||||
|
.db
|
||||||
|
.sync_folder_set_state(folder.id, "error", folder.last_sync);
|
||||||
|
error!(
|
||||||
|
id = folder.id,
|
||||||
|
path = %folder.local_path,
|
||||||
|
"sync: pass panicked; the folder is set to error and tried again on the next poll"
|
||||||
|
);
|
||||||
|
core.log_activity(
|
||||||
|
ActivityKind::Sync,
|
||||||
|
format!("sync failed for {}", base_name(&folder.local_path)),
|
||||||
|
"an internal error stopped the pass; it is tried again on the next poll",
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The pass a `Reconcile(id)` asked for. A folder removed since, or one the
|
||||||
|
/// database cannot read, is said so rather than dropped in silence.
|
||||||
|
fn run_pass_by_id(core: &Core, id: i64) {
|
||||||
|
match core.db.sync_folder_get(id) {
|
||||||
|
Ok(Some(folder)) => run_pass(core, &folder),
|
||||||
|
Ok(None) => debug!(id, "sync: folder removed before its pass; skipping"),
|
||||||
|
Err(e) => warn!(id, error = ?e, "sync: cannot read folder; skipping its pass"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The engine control loop. Single-threaded so reconcile passes never overlap.
|
/// The engine control loop. Single-threaded so reconcile passes never overlap.
|
||||||
fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
||||||
|
let health = core.sync_engine.clone();
|
||||||
|
let _running = Running::start(&health);
|
||||||
// Paths the watcher currently covers, mapped to their folder id, so an event
|
// Paths the watcher currently covers, mapped to their folder id, so an event
|
||||||
// path can be resolved back to the folder it belongs to.
|
// path can be resolved back to the folder it belongs to.
|
||||||
let watched: Arc<Mutex<Vec<(PathBuf, i64)>>> = Arc::new(Mutex::new(Vec::new()));
|
let watched: Arc<Mutex<Vec<(PathBuf, i64)>>> = Arc::new(Mutex::new(Vec::new()));
|
||||||
@@ -120,9 +242,7 @@ fn engine_loop(core: Core, rx: Receiver<SyncMsg>) {
|
|||||||
if core.sync_paused() {
|
if core.sync_paused() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if let Ok(Some(folder)) = core.db.sync_folder_get(id) {
|
run_pass_by_id(&core, id);
|
||||||
core.reconcile_folder(&folder);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -159,9 +279,7 @@ fn poll_only_loop(core: &Core, rx: Receiver<SyncMsg>) {
|
|||||||
if core.sync_paused() {
|
if core.sync_paused() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if let Ok(Some(folder)) = core.db.sync_folder_get(id) {
|
run_pass_by_id(core, id);
|
||||||
core.reconcile_folder(&folder);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -254,7 +372,7 @@ fn reconcile_all(core: &Core) {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
for folder in folders {
|
for folder in folders {
|
||||||
core.reconcile_folder(&folder);
|
run_pass(core, &folder);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+409
-3
@@ -68,6 +68,20 @@ original wording.
|
|||||||
|
|
||||||
| Entry | Title | What is left |
|
| Entry | Title | What is left |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
| [B273](#b273--a-restore-proposes-another-computers-paths-when-the-two-share-a-hostname) | A restore proposes another computer's paths when the two share a hostname | Not fixed |
|
||||||
|
| [B272](#b272--after-a-folder-is-removed-and-added-again-its-file-events-go-to-the-old-id) | After a folder is removed and added again, its file events go to the old id | Not fixed |
|
||||||
|
| [B271](#b271--one-failed-listing-makes-a-large-first-sync-start-over-from-nothing) | One failed listing makes a large first sync start over from nothing | Not fixed |
|
||||||
|
| [B270](#b270--one-file-left-in-a-mirror-folder-clears-the-wipe-guard-for-all-the-others) | One file left in a mirror folder clears the wipe guard for all the others | Not fixed |
|
||||||
|
| [B269](#b269--a-restore-that-skipped-every-folder-reports-success) | A restore that skipped every folder reports success | Not fixed |
|
||||||
|
| [B268](#b268--pdfs-diagnose-says-no-problems-found-while-sync-is-stuck) | `pdfs diagnose` says "No problems found" while sync is stuck | Not fixed |
|
||||||
|
| [B267](#b267--a-key-failure-at-a-start-from-a-warm-cache-shows-as-offline-for-good) | A key failure at a start from a warm cache shows as offline for good | Not fixed |
|
||||||
|
| [B266](#b266--pin-unpin-search-and-node-loads-report-a-database-error-as-offline) | Pin, unpin, search and node loads report a database error as "offline" | Not fixed |
|
||||||
|
| [B265](#b265--address-keys-whose-token-is-for-an-inactive-account-key-never-open) | Address keys whose token is for an inactive account key never open | Needs an SDK change; not reproduced |
|
||||||
|
| [B264](#b264--a-mount-that-fails-after-starting-its-workers-leaves-them-running) | A mount that fails after starting its workers leaves them running | Not fixed |
|
||||||
|
| [B263](#b263--a-mount-that-can-never-succeed-retries-every-5-s-and-tells-only-the-journal) | A mount that can never succeed retries every 5 s and tells only the journal | Not fixed |
|
||||||
|
| [B262](#b262--watching-a-large-synced-folder-holds-up-the-sync-engine-and-a-failed-watch-is-never-retried) | Watching a large synced folder holds up the sync engine, and a failed watch is never retried | Not fixed |
|
||||||
|
| [B261](#b261--a-folder-that-has-never-synced-reads-up-to-date) | A folder that has never synced reads "up to date" | Not fixed |
|
||||||
|
| [B260](#b260--the-poll-pushes-a-new-folders-first-sync-behind-every-other-folder) | The poll pushes a new folder's first sync behind every other folder | Not fixed |
|
||||||
| [B134](#b134--a-file-deleted-online-while-the-drain-checks-its-write-is-kept-as-a-conflict-copy) | A file deleted online while the drain checks its write is kept as a conflict copy | Only with `local_first` off |
|
| [B134](#b134--a-file-deleted-online-while-the-drain-checks-its-write-is-kept-as-a-conflict-copy) | A file deleted online while the drain checks its write is kept as a conflict copy | Only with `local_first` off |
|
||||||
| [B91](#b91--the-daemon-hung-overnight-and-nothing-could-say-what-it-was-waiting-on) | The daemon hung overnight, and nothing could say what it was waiting on | The cause is not diagnosed; the daemon now reports where it is stuck and is restarted |
|
| [B91](#b91--the-daemon-hung-overnight-and-nothing-could-say-what-it-was-waiting-on) | The daemon hung overnight, and nothing could say what it was waiting on | The cause is not diagnosed; the daemon now reports where it is stuck and is restarted |
|
||||||
| [B67](#b67--fresh-state-restore-can-silently-select-a-new-device-high-22) | Fresh-state restore can silently select a new device (HIGH-22) | An ambiguous device after a loss of local state is not asked about |
|
| [B67](#b67--fresh-state-restore-can-silently-select-a-new-device-high-22) | Fresh-state restore can silently select a new device (HIGH-22) | An ambiguous device after a loss of local state is not asked about |
|
||||||
@@ -80,6 +94,7 @@ original wording.
|
|||||||
|
|
||||||
| Entry | Title | What is left |
|
| Entry | Title | What is left |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
| [B259](#b259--the-sync-engine-can-die-or-hang-and-nothing-shows-it-until-a-restart) | The sync engine can die or hang, and nothing shows it until a restart | No pass deadline; an ended engine is reported, not restarted; the trigger in #53 is not known |
|
||||||
| [B199](#b199--a-stopped-share-stays-on-the-shared-by-me-page-for-minutes) | A stopped share stays on the Shared by me page for minutes | The share itself is not deleted; the SDK has no call for it |
|
| [B199](#b199--a-stopped-share-stays-on-the-shared-by-me-page-for-minutes) | A stopped share stays on the Shared by me page for minutes | The share itself is not deleted; the SDK has no call for it |
|
||||||
| [B88](#b88--pdfs-pin-rename-move-delete-and-sharing-still-cannot-name-a-path-under-a-secondary-mount) | `pdfs pin`, `rename`, `move`, `delete` and sharing still cannot name a path under a secondary mount | `pin`, `unpin`, opening a file, thumbnails, uploads, sharing and versions |
|
| [B88](#b88--pdfs-pin-rename-move-delete-and-sharing-still-cannot-name-a-path-under-a-secondary-mount) | `pdfs pin`, `rename`, `move`, `delete` and sharing still cannot name a path under a secondary mount | `pin`, `unpin`, opening a file, thumbnails, uploads, sharing and versions |
|
||||||
| [B72](#b72--pdfs-trash-never-returns) | `pdfs trash` never returns | The slowness of the first refresh is unmeasured |
|
| [B72](#b72--pdfs-trash-never-returns) | `pdfs trash` never returns | The slowness of the first refresh is unmeasured |
|
||||||
@@ -100,6 +115,7 @@ a live account since.
|
|||||||
|
|
||||||
| Entry | Title |
|
| Entry | Title |
|
||||||
|---|---|
|
|---|---|
|
||||||
|
| [B258](#b258--journalctl--p-warning-shows-nothing-whatever-the-daemon-logged) | `journalctl -p warning` shows nothing, whatever the daemon logged |
|
||||||
| [B256](#b256--pdfs-copy-right-after-a-save-copies-the-old-version) | `pdfs copy` right after a save copies the old version |
|
| [B256](#b256--pdfs-copy-right-after-a-save-copies-the-old-version) | `pdfs copy` right after a save copies the old version |
|
||||||
| [B255](#b255--a-synced-folder-whose-files-are-all-gone-here-is-stuck-for-good) | A synced folder whose files are all gone here is stuck for good |
|
| [B255](#b255--a-synced-folder-whose-files-are-all-gone-here-is-stuck-for-good) | A synced folder whose files are all gone here is stuck for good |
|
||||||
| [B254](#b254--an-edit-in-the-same-second-as-a-new-upload-never-arrives) | An edit in the same second as a new upload never arrives |
|
| [B254](#b254--an-edit-in-the-same-second-as-a-new-upload-never-arrives) | An edit in the same second as a new upload never arrives |
|
||||||
@@ -359,6 +375,384 @@ a live account since.
|
|||||||
|
|
||||||
Newest first.
|
Newest first.
|
||||||
|
|
||||||
|
### B273 — A restore proposes another computer's paths when the two share a hostname
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #46.
|
||||||
|
**Where:** `saved_folder` in `crates/pdfs-core/src/profile.rs`
|
||||||
|
|
||||||
|
**Repro:** Two computers with the same hostname, for example both left at a distribution's
|
||||||
|
default `archlinux`, sync the same device. Restore a folder on one of them. The proposed local
|
||||||
|
path and mode come from whichever of the two profiles is found first, not from this computer's.
|
||||||
|
|
||||||
|
**Cause.** `saved_folder` prefers the profile whose `hostname` matches. The machine id that
|
||||||
|
B252 added tells computers apart, but this lookup does not use it.
|
||||||
|
|
||||||
|
**Fix.** Not done. Match on the machine id first, then fall back to the hostname for profiles
|
||||||
|
written before it existed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B272 — After a folder is removed and added again, its file events go to the old id
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #53.
|
||||||
|
**Where:** `rewatch` in `crates/pdfs-fuse/src/sync/engine.rs`
|
||||||
|
|
||||||
|
**Repro:** Remove a synced folder and add the same path again before the engine handles the
|
||||||
|
first `Rewatch`, so both land in one burst. Edits in the folder are not synced until the
|
||||||
|
two-minute poll, and the same goes for every later edit until the daemon restarts.
|
||||||
|
|
||||||
|
**Cause.** `rewatch` matches the watches it has against the ones it wants by path alone. The
|
||||||
|
path is still wanted, so the old entry stays with the removed folder's id. Its events become
|
||||||
|
`Reconcile(old id)`, `sync_folder_get` finds no row, and the request is dropped.
|
||||||
|
|
||||||
|
**Fix.** Not done. Compare `(path, id)` pairs, and update the id of a path that is still
|
||||||
|
watched.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B271 — One failed listing makes a large first sync start over from nothing
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #46: a folder of about 19,900 items restored on a
|
||||||
|
second computer sat at `scanning…` with nothing downloaded, and the journal showed
|
||||||
|
`RequestTimeout (http 408)`.
|
||||||
|
**Where:** `walk_remote` in `crates/pdfs-fuse/src/sync.rs`
|
||||||
|
|
||||||
|
**Repro:** Restore or add a mirror folder with tens of thousands of items on Drive, on a link
|
||||||
|
that drops a request now and then. The pass walks the whole remote tree before it downloads
|
||||||
|
anything. When one folder's listing fails after the SDK's retries, the pass fails, and the
|
||||||
|
next one walks the whole tree again. On a slow enough link it never gets past the walk.
|
||||||
|
|
||||||
|
**Cause.** Each `enumerate_folder_children_node_uids`, `enumerate_nodes_light` and
|
||||||
|
`enumerate_nodes` call in the walk ends the whole walk through `?`. Nothing is applied until
|
||||||
|
the walk is done. Files that did download keep their baseline rows, so only the walk is lost,
|
||||||
|
but on a first sync that is all of it. Progress is also counted only after a folder's full
|
||||||
|
listing, so a large folder shows `scanning…` with 0 checked for a long time.
|
||||||
|
|
||||||
|
**Fix.** Not done. Record a failed subtree as a gap, as undecryptable nodes already are: no
|
||||||
|
deletions under it, and the rest of the pass still applies.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B270 — One file left in a mirror folder clears the wipe guard for all the others
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #46, whose reporter emptied a synced folder
|
||||||
|
several times.
|
||||||
|
**Where:** `guard_local_wipe` and `guard_remote_wipe` in `crates/pdfs-fuse/src/sync/planner.rs`
|
||||||
|
|
||||||
|
**Repro:** Sync a mirror folder with many files. Make all of them but one unavailable here: a
|
||||||
|
half-mounted disk, a restored backup that brought back one file, or a script that emptied the
|
||||||
|
folder and wrote one file back. The next pass trashes every other file on Drive. On a device
|
||||||
|
that two computers sync, the other computer then deletes its copies too.
|
||||||
|
|
||||||
|
**Cause.** The guard refuses a pass only when every baseline path is missing. One surviving
|
||||||
|
path is enough to clear it, so it guards against an empty folder and nothing short of that.
|
||||||
|
The remote guard has the same shape.
|
||||||
|
|
||||||
|
**Fix.** Not done. Also refuse when most of a large baseline is missing at once, and leave the
|
||||||
|
choice to the user: delete on the other side, or sync from scratch.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B269 — A restore that skipped every folder reports success
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #46: the journal showed `restore complete
|
||||||
|
restored=0 skipped=1` several times, and the user saw nothing.
|
||||||
|
**Where:** the `RestoreSyncFolders` and `RestoreDeviceFolders` handler in
|
||||||
|
`crates/pdfs-fuse/src/control.rs`; `restore_sync_folders` in `crates/pdfs-fuse/src/profile.rs`
|
||||||
|
|
||||||
|
**Repro:** Restore a folder from another computer to a path that is already a sync root here,
|
||||||
|
for example a folder left over from an earlier attempt. `pdfs devices join` prints `restoring
|
||||||
|
folders`, and the app shows the same. Nothing downloads, and no error appears anywhere.
|
||||||
|
|
||||||
|
**Cause.** The daemon acks before it works, because a restore outlasts the socket timeout. The
|
||||||
|
result, `restored 0 folder(s); skipped: …`, is then logged as a successful activity, so the
|
||||||
|
reason is only in the activity text and the journal. When the database insert fails partway,
|
||||||
|
the `?` also skips the `Rewatch` for the folders already restored.
|
||||||
|
|
||||||
|
**Fix.** Not done. Log a restore that skipped folders as a failure, name the reason in the app
|
||||||
|
and the CLI, and offer to replace or reset the folder in the way. Send the `Rewatch` on every
|
||||||
|
exit path.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B268 — `pdfs diagnose` says "No problems found" while sync is stuck
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, issue #53: a folder that never synced, and `diagnose` found nothing.
|
||||||
|
**Where:** `diagnose` in `crates/pdfs-cli/src/main.rs`
|
||||||
|
|
||||||
|
**Repro:** Leave a synced folder in `error`, or have queued writes that keep failing, or pause
|
||||||
|
sync. `pdfs diagnose` reports every check `[ok]` and exits 0.
|
||||||
|
|
||||||
|
**Cause.** The `Status` reply is matched with `..`, which drops `failing_ops`, `failing_error`,
|
||||||
|
`parked_uploads`, `staged_oldest_secs` and `paused`. `diagnose` never asks for the synced
|
||||||
|
folders or for `Diagnostics`, so a folder in error and a stalled worker are not seen. A
|
||||||
|
control socket file with no daemon behind it is also reported `[ok]`; only the next line says
|
||||||
|
the daemon does not answer.
|
||||||
|
|
||||||
|
**Fix.** Not done. Warn on failing or parked writes, a pause, old staged data and folders in
|
||||||
|
error, and report the sync engine's state once B259 exposes it.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B267 — A key failure at a start from a warm cache shows as offline for good
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #40. B202 fixed the same path for a refused
|
||||||
|
session only.
|
||||||
|
**Where:** `fetch_or_recall_root` in `crates/pdfs-fuse/src/mount.rs`; `run_online_probe` in
|
||||||
|
`crates/pdfs-fuse/src/link.rs`
|
||||||
|
|
||||||
|
**Repro:** Sign in once so the cache holds the My Files root. Then start the daemon in a state
|
||||||
|
where the root cannot be decrypted, for example the address-key failure of B265. The mount
|
||||||
|
comes up "offline", the app says you are offline, and it stays so while the network is fine.
|
||||||
|
|
||||||
|
**Cause.** Any error from `get_my_files_folder` other than a revoked session becomes
|
||||||
|
`Start::Offline` when a cached root exists. The online probe then retries the same call and
|
||||||
|
logs its failures at debug level, so the journal at the default level says nothing.
|
||||||
|
|
||||||
|
**Fix.** Not done. Use `Start::Offline` only for an error `is_network_error` accepts, and
|
||||||
|
report the others as their own state through `Status`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B266 — Pin, unpin, search and node loads report a database error as "offline"
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #38. B205 fixed the photo upload path only.
|
||||||
|
**Where:** `CoreError::from_api` in `crates/pdfs-core/src/error.rs`; its callers in
|
||||||
|
`crates/pdfs-fuse/src/lib.rs` (`pin`, `unpin`, the searches, `load node`)
|
||||||
|
|
||||||
|
**Repro:** Pin a folder while the cache database is locked or full. The app says "You're
|
||||||
|
offline" and drops the reason.
|
||||||
|
|
||||||
|
**Cause.** `from_api` classifies any error that is not a `ProtonError` as `Offline`, because
|
||||||
|
it was written for SDK calls, where that means a transport error. These callers pass it
|
||||||
|
`pdfs_core::Error` values from the cache and the database. A `ProtonError` wrapped in
|
||||||
|
`pdfs_core::Error::Proton` also misses the downcast, since the source chain is not walked.
|
||||||
|
|
||||||
|
**Fix.** Not done. Use `CoreError::internal` at the database callers, and walk the source
|
||||||
|
chain in `from_api` before falling back.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B265 — Address keys whose token is for an inactive account key never open
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #40: a Duo account whose mount failed on every
|
||||||
|
start with `fetch My Files root: cryptography error: failed to decrypt: missing key`, after three
|
||||||
|
`failed to decrypt address key token` warnings. The web app worked. B210's fix does not apply,
|
||||||
|
because the account has an active volume.
|
||||||
|
**Where:** `user_keys` and `decrypt_address` in `proton-sdk` (`crates/proton-sdk/src/account.rs`
|
||||||
|
in `proton-sdk-rs`)
|
||||||
|
|
||||||
|
**Repro:** Not reproduced. An account whose address-key tokens are encrypted to an account key
|
||||||
|
that is no longer active cannot open those address keys, and so cannot open the My Files share.
|
||||||
|
|
||||||
|
**Cause.** `user_keys` skips every inactive account key, even one the password opens, and the
|
||||||
|
address-key tokens are decrypted only with what it returns. `keycheck.rs` in this repository
|
||||||
|
already reports "token is for inactive account key", so the case is known. Separately,
|
||||||
|
`addresses()` fails as a whole through `?` when one address has no usable primary key, which
|
||||||
|
takes the working addresses down with it.
|
||||||
|
|
||||||
|
**Fix.** Not done, and it belongs in the SDK. Use every account key that opens for decrypting
|
||||||
|
tokens, and only the active ones for signing. Skip and log an address that cannot be opened.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B264 — A mount that fails after starting its workers leaves them running
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #40.
|
||||||
|
**Where:** `run_mount` in `crates/pdfs-fuse/src/mount.rs`
|
||||||
|
|
||||||
|
**Repro:** Make the mount fail after the workers start: a busy mountpoint, or a control socket
|
||||||
|
whose permissions cannot be set. The daemon retries every 5 s, and the drain, sync, sweep,
|
||||||
|
probe and index threads of the failed attempt keep running beside the next one.
|
||||||
|
|
||||||
|
**Cause.** The early returns after the workers are spawned (the `?` on the socket bind, the
|
||||||
|
permission check, a `spawn_session` error, and a `?` on a later thread spawn) skip
|
||||||
|
`stop_workers`. The threads hold a `Core`, and the `Core` holds the database. If that keeps
|
||||||
|
the `flock` held, every later attempt fails with "another daemon is using cache.db" instead
|
||||||
|
of the real error. That part is inferred from the code and not observed.
|
||||||
|
|
||||||
|
**Fix.** Not done. Bind the socket and check the mountpoint before any worker starts, and stop
|
||||||
|
the workers on every error path.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B263 — A mount that can never succeed retries every 5 s and tells only the journal
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, issue #40: the app showed "Connecting… waiting for the Proton Drive
|
||||||
|
service" for days, `pdfs diagnose` only reported the control socket missing, and the reporter
|
||||||
|
uninstalled.
|
||||||
|
**Where:** the mount loop in `crates/pdfs-cli/src/main.rs`; `run_mount` in
|
||||||
|
`crates/pdfs-fuse/src/mount.rs`
|
||||||
|
|
||||||
|
**Repro:** Start the daemon with a session whose keys cannot open My Files (B265). The journal
|
||||||
|
shows `mount failed; retrying in 5s` forever. The app and the tray wait for the service, and
|
||||||
|
nothing says why.
|
||||||
|
|
||||||
|
**Cause.** Every mount error gets the same 5 s retry, whether it is a network error or one that
|
||||||
|
no retry fixes, such as a key or a 4xx answer. Each attempt builds a new client and sends about
|
||||||
|
four requests, so a broken account sends tens of thousands a day, which may also feed
|
||||||
|
Proton's sign-in blocks (#39). The control socket is bound only after the mount succeeds, and
|
||||||
|
the error is written nowhere a front end can read it.
|
||||||
|
|
||||||
|
**Fix.** Not done. Back off exponentially, treat key, `KeysLocked` and 4xx errors as permanent
|
||||||
|
with a long back-off, and write the last error where the app, the tray and `diagnose` can
|
||||||
|
show it (a state file, and `STATUS=` to systemd).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B262 — Watching a large synced folder holds up the sync engine, and a failed watch is never retried
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #53.
|
||||||
|
**Where:** `rewatch` and `engine_loop` in `crates/pdfs-fuse/src/sync/engine.rs`
|
||||||
|
|
||||||
|
**Repro:** Add a synced folder with many subfolders, or one that holds a symlink to a large
|
||||||
|
tree, on a system near `fs.inotify.max_user_watches`. The new folder's first pass waits until
|
||||||
|
the recursive watch is set up. When the watch fails, the journal has one `sync: watch failed`
|
||||||
|
line, the folder is never watched, and later changes are found only by the two-minute poll.
|
||||||
|
|
||||||
|
**Cause.**
|
||||||
|
- `rewatch` runs on the engine thread before the reconcile, and notify's recursive watch walks
|
||||||
|
the whole tree, at startup for every folder.
|
||||||
|
- notify follows symlinks by default, and the local walk does not.
|
||||||
|
- When `watch` fails partway, the watches added before the failure stay registered, but the
|
||||||
|
folder is not recorded as watched. Their events match no folder and become `ReconcileAll`,
|
||||||
|
and they use up watches the next folder needs.
|
||||||
|
- Errors the watcher reports later are dropped in its callback.
|
||||||
|
|
||||||
|
**Fix.** Not done. Reconcile first, unwatch after a partial failure and retry with a back-off,
|
||||||
|
turn off symlink following, and show a watch failure in the folder's state.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B261 — A folder that has never synced reads "up to date"
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, issue #53: "In Computers I add the folder. It appears as synced."
|
||||||
|
**Where:** `sync_folder` in `crates/pdfs-core/src/db/migrations.rs`; `sync_state_label` in
|
||||||
|
`crates/pdfs-gui/src/app/pages/devices.rs`; the location rows in
|
||||||
|
`crates/pdfs-gui/src/app/pages/locations.rs`
|
||||||
|
|
||||||
|
**Repro:** Add a synced folder. Until its first pass starts, the app shows it as up to date.
|
||||||
|
The same happens after a switch from on-demand to mirror, after **Sync From Scratch…**, and for
|
||||||
|
every folder while sync is paused for the whole account.
|
||||||
|
|
||||||
|
**Cause.** The `state` column defaults to `idle`, and every state the label does not know reads
|
||||||
|
"up to date". `last_sync` is 0 for a folder that has never synced, but the rows do not look at
|
||||||
|
it, and a test pins the label for that case. The rows only show a folder's own pause, not the
|
||||||
|
account-wide one.
|
||||||
|
|
||||||
|
**Fix.** Not done. Show "waiting for first sync" when `last_sync` is 0, and show the
|
||||||
|
account-wide pause on the folder rows.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B260 — The poll pushes a new folder's first sync behind every other folder
|
||||||
|
|
||||||
|
**Status:** Open
|
||||||
|
**Found:** 2026-10-31, code audit after issue #53.
|
||||||
|
**Where:** `engine_loop` in `crates/pdfs-fuse/src/sync/engine.rs`
|
||||||
|
|
||||||
|
**Repro:** Sync a few large mirror folders whose passes take longer than two minutes, then add
|
||||||
|
another folder. Its first pass waits until every other folder has been walked again, each
|
||||||
|
time.
|
||||||
|
|
||||||
|
**Cause.** The poll sends `ReconcileAll` every two minutes whether or not a pass is still
|
||||||
|
running. When a burst holds both a `ReconcileAll` and a folder's `Reconcile(id)`, the loop runs
|
||||||
|
only `reconcile_all`, which goes in id order. A new folder has the highest id, so it always
|
||||||
|
goes last.
|
||||||
|
|
||||||
|
**Fix.** Not done. Run the folders asked for by id before the rest, and don't queue a poll
|
||||||
|
while a pass runs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B259 — The sync engine can die or hang, and nothing shows it until a restart
|
||||||
|
|
||||||
|
**Status:** Partly fixed. A hung pass is reported but not ended, and a panic outside a pass
|
||||||
|
still stops the engine; it is reported, not restarted.
|
||||||
|
**Found:** 2026-10-31, issue #53: a newly added folder did not sync until the reporter restarted
|
||||||
|
the app. All FUSE workers were idle, nothing was queued, and the warning-level journal was
|
||||||
|
empty (B258).
|
||||||
|
**Where:** `spawn` and `engine_loop` in `crates/pdfs-fuse/src/sync/engine.rs`;
|
||||||
|
`reconcile_pass` in `crates/pdfs-fuse/src/sync.rs`; `crates/pdfs-fuse/src/diagnostics.rs`
|
||||||
|
|
||||||
|
**Repro:** Not reproduced. Make one pass panic, or block in a call that does not return. Every
|
||||||
|
later add, `pdfs sync now`, file event and poll is dropped without a word. The folder that was
|
||||||
|
syncing stays `syncing` with its last progress. A restart runs a pass over every folder, which
|
||||||
|
is why it helps.
|
||||||
|
|
||||||
|
**Cause.**
|
||||||
|
- The engine is one thread, and nothing catches a panic in a pass. When it unwinds, the
|
||||||
|
receiver is gone and every `let _ = sync_tx.send(..)` fails silently. The poll thread stops
|
||||||
|
at its first failed send.
|
||||||
|
- The thread's `JoinHandle` is checked only at shutdown.
|
||||||
|
- A pass has no overall deadline. Each request is bounded, but a pass is not; B165 dealt with
|
||||||
|
the stop only.
|
||||||
|
- Nothing logs a pass starting, or ending without changes. `Diagnostics` covers the FUSE
|
||||||
|
workers and the request queue, not this thread, so "workers idle, nothing queued" says
|
||||||
|
nothing about sync.
|
||||||
|
- A database error when the engine reads a folder drops that folder's request without a log.
|
||||||
|
|
||||||
|
**Fix.**
|
||||||
|
- Each pass runs under `catch_unwind` (`EngineHealth::contain` in
|
||||||
|
`crates/pdfs-fuse/src/sync/engine.rs`). A panic ends that pass only: its progress is cleared,
|
||||||
|
the folder is set to `error`, the activity feed says so, and the next poll tries again.
|
||||||
|
- `Diagnostics` carries the engine's state: running or not, the folder of the running pass, how
|
||||||
|
long it has run and how long since it last reported progress, and how many passes panicked.
|
||||||
|
`pdfs diagnostics` prints it as `sync engine: …`.
|
||||||
|
- The supervisor warns when the engine thread has ended, and when a pass has reported no
|
||||||
|
progress for 15 minutes.
|
||||||
|
- A first pass logs its start and end at info, as does any pass that changed something or
|
||||||
|
took 30 s or more. A request for a folder the database cannot read is logged.
|
||||||
|
- Not done: a deadline for a pass, and restarting an engine thread that has ended. The
|
||||||
|
channel's receiver ends with the thread, so a restart needs a new channel in every `Core`
|
||||||
|
clone.
|
||||||
|
|
||||||
|
**Test:** `a_panicking_pass_is_contained_and_counted`, `the_running_pass_is_reported` and
|
||||||
|
`the_engine_reads_as_stopped_once_its_thread_ends` in `pdfs-fuse`;
|
||||||
|
`a_stopped_sync_engine_is_a_stall` and `only_a_quiet_sync_pass_is_a_stall` in the supervisor.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### B258 — `journalctl -p warning` shows nothing, whatever the daemon logged
|
||||||
|
|
||||||
|
**Status:** Fixed, not verified live
|
||||||
|
**Found:** 2026-10-31, issue #53: the reporter ran `journalctl --user -u proton-drive.service -b
|
||||||
|
-p warning`, as `docs/TROUBLESHOOTING.md` tells them to, and got `-- No entries --`.
|
||||||
|
**Where:** `main` in `crates/pdfs-cli/src/main.rs`; `packaging/proton-drive.service`
|
||||||
|
|
||||||
|
**Repro:** Make the daemon log a warning, then run the command above. It prints no entries.
|
||||||
|
Without `-p` the warning is there.
|
||||||
|
|
||||||
|
**Cause.** The daemon logs with plain `tracing_subscriber::fmt()` to standard output. journald
|
||||||
|
stores every line of a service's output at the unit's default priority, info, unless the line
|
||||||
|
starts with a `<N>` prefix. So every `WARN` and `ERROR` line is stored as info, and the
|
||||||
|
troubleshooting guide's filter hides all of them.
|
||||||
|
|
||||||
|
**Fix.** When standard output is the journal stream (`JOURNAL_STREAM` names the same
|
||||||
|
device and inode as fd 1), each line starts with the `<N>` prefix for its level, and ANSI colours
|
||||||
|
are off. A terminal that merely inherited `JOURNAL_STREAM` keeps the plain format
|
||||||
|
(`crates/pdfs-cli/src/journal.rs`).
|
||||||
|
|
||||||
|
**Test:** `each_line_opens_with_the_priority_of_its_level` and
|
||||||
|
`journal_stream_must_name_the_same_file` in `pdfs-cli`. On 2026-10-31, `pdfs diagnose` run with
|
||||||
|
`RUST_LOG=debug` under `systemd-run --user -p StandardOutput=journal` stored its debug lines at
|
||||||
|
`PRIORITY=7` and its report at 6.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### B256 — `pdfs copy` right after a save copies the old version
|
### B256 — `pdfs copy` right after a save copies the old version
|
||||||
|
|
||||||
**Status:** Fixed, not verified live
|
**Status:** Fixed, not verified live
|
||||||
@@ -402,14 +796,20 @@ folder lost its files is not known yet.
|
|||||||
|
|
||||||
**Fix.** The push pass for a switch to on-demand forgets the baseline rows of a local copy that
|
**Fix.** The push pass for a switch to on-demand forgets the baseline rows of a local copy that
|
||||||
is gone in full, instead of trashing them on Drive, so the switch goes ahead and the mount shows
|
is gone in full, instead of trashing them on Drive, so the switch goes ahead and the mount shows
|
||||||
what Drive has. **Sync From Scratch…** in the app and `pdfs sync reset ID` clear a mirror
|
what Drive has. A switch asked for before any pass saw the folder empty found it still `idle`,
|
||||||
|
and the check for what only this computer has refused it at once: every path read as "deleted
|
||||||
|
here but not on Proton Drive yet". The account run found that. Such a switch now waits for the
|
||||||
|
same push pass (`mirror_tree_gone` in `crates/pdfs-fuse/src/sync.rs`, checked in
|
||||||
|
`apply_sync_folder_mode`). **Sync From Scratch…** in the app and `pdfs sync reset ID` clear a mirror
|
||||||
folder's baseline, so the next pass compares both sides as on the first sync: one-sided files
|
folder's baseline, so the next pass compares both sides as on the first sync: one-sided files
|
||||||
are copied across, files that differ keep both versions, and nothing is deleted. Restore now
|
are copied across, files that differ keep both versions, and nothing is deleted. Restore now
|
||||||
logs why it skipped a folder.
|
logs why it skipped a folder.
|
||||||
|
|
||||||
**Test:** The simulations `a_folder_whose_local_copy_is_gone_still_goes_on_demand` (which
|
**Test:** The simulations `a_folder_whose_local_copy_is_gone_still_goes_on_demand` (which
|
||||||
failed in exactly the reported state before the fix) and
|
failed in exactly the reported state before the fix),
|
||||||
`a_folder_whose_local_copy_is_gone_downloads_again_after_a_reset`.
|
`a_folder_emptied_before_any_pass_still_goes_on_demand` (which failed with the account run's
|
||||||
|
error) and `a_folder_whose_local_copy_is_gone_downloads_again_after_a_reset`; the acceptance case
|
||||||
|
"an emptied mirror folder switches to on-demand".
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -1517,6 +1917,8 @@ what it holds. What was synced in it is still deleted.
|
|||||||
### B210 — After a password reset, the mount retries "missing key" forever
|
### B210 — After a password reset, the mount retries "missing key" forever
|
||||||
|
|
||||||
**Status:** Fixed, not verified live
|
**Status:** Fixed, not verified live
|
||||||
|
**See also:** B265. The account in issue #40 has an active volume, so this fix does not run for
|
||||||
|
it; its "missing key" comes from address keys that do not open.
|
||||||
**Found:** 2026-10-15, issue #40: on a fresh 3.4.0 install the sign-in worked, but the daemon
|
**Found:** 2026-10-15, issue #40: on a fresh 3.4.0 install the sign-in worked, but the daemon
|
||||||
logged `mount failed: fetch My Files root: cryptography error: failed to decrypt: missing key`
|
logged `mount failed: fetch My Files root: cryptography error: failed to decrypt: missing key`
|
||||||
every 5 seconds. The app stayed on "Connecting…" because the daemon never got far enough to
|
every 5 seconds. The app stayed on "Connecting…" because the daemon never got far enough to
|
||||||
@@ -1662,6 +2064,7 @@ front-end is the stale one, and restarting the service would not help.
|
|||||||
### B205 — A photo upload the SDK refuses says "You're offline"
|
### B205 — A photo upload the SDK refuses says "You're offline"
|
||||||
|
|
||||||
**Status:** Fixed, not verified live
|
**Status:** Fixed, not verified live
|
||||||
|
**See also:** B266, the same misclassification on the pin, unpin, search and node-load paths.
|
||||||
**Found:** 2026-10-11, issue #38: uploading an image from the Photos page showed "You're offline"
|
**Found:** 2026-10-11, issue #38: uploading an image from the Photos page showed "You're offline"
|
||||||
while other uploads went through. Each `upload_photo` failed about 5 µs after it started, with no
|
while other uploads went through. Each `upload_photo` failed about 5 µs after it started, with no
|
||||||
request sent, and the journal gave no reason.
|
request sent, and the journal gave no reason.
|
||||||
@@ -1746,6 +2149,7 @@ taken as this session's, so the daemon keeps writing its later rotations.
|
|||||||
### B202 — A refused session shows as offline forever, and nothing says to sign in again
|
### B202 — A refused session shows as offline forever, and nothing says to sign in again
|
||||||
|
|
||||||
**Status:** Fixed, not verified live
|
**Status:** Fixed, not verified live
|
||||||
|
**See also:** B267. Other errors at a start from a warm cache still mount as offline.
|
||||||
**Found:** 2026-10-09, issue #35: the tray and `pdfs diagnose` said "offline, 3 queued", and
|
**Found:** 2026-10-09, issue #35: the tray and `pdfs diagnose` said "offline, 3 queued", and
|
||||||
diagnose also said "keyring session: ok", while every call in the log failed with
|
diagnose also said "keyring session: ok", while every call in the log failed with
|
||||||
`InvalidRefreshToken (http 400)`. `pdfs sync now` and restarting the service changed nothing.
|
`InvalidRefreshToken (http 400)`. `pdfs sync now` and restarting the service changed nothing.
|
||||||
@@ -2687,6 +3091,8 @@ fix).
|
|||||||
### B165 — A stop waits on a stuck thread until systemd kills the daemon
|
### B165 — A stop waits on a stuck thread until systemd kills the daemon
|
||||||
|
|
||||||
**Status:** Fixed (unverified).
|
**Status:** Fixed (unverified).
|
||||||
|
**See also:** B259. A pass still has no deadline while the daemon runs, and a stuck or dead
|
||||||
|
sync engine is not reported.
|
||||||
**Found:** 2026-10-04, by the account run on 2.10.0 (cleanup after on-demand pairing 2). The
|
**Found:** 2026-10-04, by the account run on 2.10.0 (cleanup after on-demand pairing 2). The
|
||||||
daemon was stopped at 15:53:04 and logged "control socket closing", but never "sync engine
|
daemon was stopped at 15:53:04 and logged "control socket closing", but never "sync engine
|
||||||
stopping" or "daemon stopping". systemd killed it 15 s later (`TimeoutStopSec`). The run's
|
stopping" or "daemon stopping". systemd killed it 15 s later (`TimeoutStopSec`). The run's
|
||||||
|
|||||||
+12
-4
@@ -11,7 +11,7 @@ scratch (user data in `staging/` and `recovery/` is never touched by this).
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
## [3.6.0] — 2026-10-26
|
## [3.6.0] — 2026-10-27
|
||||||
|
|
||||||
Schema: **44**, unchanged.
|
Schema: **44**, unchanged.
|
||||||
|
|
||||||
@@ -46,6 +46,9 @@ Schema: **44**, unchanged.
|
|||||||
file on one side only is copied to the other, and a file that differs keeps both versions.
|
file on one side only is copied to the other, and a file that differs keeps both versions.
|
||||||
Nothing is deleted. From the terminal: `pdfs sync reset ID`.
|
Nothing is deleted. From the terminal: `pdfs sync reset ID`.
|
||||||
|
|
||||||
|
- **`--json` for `pdfs members`, `pdfs du` and `pdfs public-link create` / `update`**, so scripts
|
||||||
|
can read a link's id, role, password flag and expiry, and a folder's size and item count.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
- **Copying a file inside the mount is faster** when the computer already has the file, for
|
- **Copying a file inside the mount is faster** when the computer already has the file, for
|
||||||
example with `cp` or a file manager: the copy is made on this computer, about four times
|
example with `cp` or a file manager: the copy is made on this computer, about four times
|
||||||
@@ -58,14 +61,19 @@ Schema: **44**, unchanged.
|
|||||||
pages for files, synced folders, Photos, sharing, computers, and the launcher and tray.
|
pages for files, synced folders, Photos, sharing, computers, and the launcher and tray.
|
||||||
Installation puts the COPR repository first for Fedora and explains updating and uninstalling
|
Installation puts the COPR repository first for Fedora and explains updating and uninstalling
|
||||||
for each kind of package. Each release publishes `docs/` to the wiki.
|
for each kind of package. Each release publishes `docs/` to the wiki.
|
||||||
|
- **Acceptance cases for the 3.6.0 features:** `copy_file_range` over several calls and at
|
||||||
|
offsets, `pdfs copy` (including B256), `pdfs du`, `pdfs public-link update`, Sync From Scratch,
|
||||||
|
and the switch of an emptied mirror folder to on-demand (B255). `docs/TESTING.md` lists what to
|
||||||
|
check by hand in the app.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- **A synced folder whose files were all gone from this computer no longer gets stuck** (#46).
|
- **A synced folder whose files were all gone from this computer no longer gets stuck** (#46).
|
||||||
The sync still refuses to delete everything on Proton Drive, but a switch to online only now
|
The sync still refuses to delete everything on Proton Drive, but a switch to online only now
|
||||||
goes ahead and shows what Proton Drive has, instead of waiting for good. To download the
|
goes ahead and shows what Proton Drive has, instead of waiting for good. This includes a
|
||||||
files again instead, sync the folder from scratch. Restoring folders also logs why it
|
switch asked for right after the files went, which was refused with "… was deleted here but
|
||||||
skipped a folder.
|
not on Proton Drive yet". To download the files again instead, sync the folder from scratch.
|
||||||
|
Restoring folders also logs why it skipped a folder.
|
||||||
|
|
||||||
- **The first sync of a folder shows how far its scan got** (#46). Before anything downloads,
|
- **The first sync of a folder shows how far its scan got** (#46). Before anything downloads,
|
||||||
the first pass reads every item on Proton Drive, which takes many minutes for a large folder.
|
the first pass reads every item on Proton Drive, which takes many minutes for a large folder.
|
||||||
|
|||||||
+3
-2
@@ -190,8 +190,9 @@ commands are for scripts and for cases where the mount is not available.
|
|||||||
## Scripting with `--json`
|
## Scripting with `--json`
|
||||||
|
|
||||||
The global `--json` flag makes query commands print JSON: `status`, `ls`, `pins`, `sync list`,
|
The global `--json` flag makes query commands print JSON: `status`, `ls`, `pins`, `sync list`,
|
||||||
`devices list`, `locations`, `transfers`, `activity`, `trash` and `cache inspect`. Commands that change
|
`devices list`, `locations`, `transfers`, `activity`, `trash`, `cache inspect`, `members` and `du`.
|
||||||
something keep their human-readable output; check their exit code.
|
`public-link create` and `public-link update` print the link they made or changed. Other commands
|
||||||
|
that change something keep their human-readable output; check their exit code.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pdfs --json status | jq -r '.mount.mountpoint'
|
pdfs --json status | jq -r '.mount.mountpoint'
|
||||||
|
|||||||
+31
-1
@@ -116,7 +116,10 @@ for something a network filesystem cannot do.
|
|||||||
the move cases in every pairing of on-demand and mirror. Before a folder switches to on-demand,
|
the move cases in every pairing of on-demand and mirror. Before a folder switches to on-demand,
|
||||||
a new file is written into it; once mounted, the file must be listed with its bytes. In every
|
a new file is written into it; once mounted, the file must be listed with its bytes. In every
|
||||||
pairing each folder must still hold the file written into it at setup.
|
pairing each folder must still hold the file written into it at setup.
|
||||||
4. Removes everything it made, locally and on Drive, and reports `cleanup`. A run that leaves
|
4. Registers a third sync folder for the recovery cases (`account` in `--list`): Sync From Scratch
|
||||||
|
must change nothing that already matched, and a mirror folder whose local files are all gone
|
||||||
|
must still switch to on-demand, with every file intact on Drive (B255).
|
||||||
|
5. Removes everything it made, locally and on Drive, and reports `cleanup`. A run that leaves
|
||||||
anything behind fails.
|
anything behind fails.
|
||||||
|
|
||||||
Cleanup finds remote folders by uid, never by name, so nothing of yours is touched. It removes
|
Cleanup finds remote folders by uid, never by name, so nothing of yours is touched. It removes
|
||||||
@@ -199,6 +202,15 @@ Besides the syscall contract:
|
|||||||
both directions, several sources in one `pdfs move`, and a tree through My files and out again.
|
both directions, several sources in one `pdfs move`, and a tree through My files and out again.
|
||||||
The refusals must move nothing and lose nothing: a name already taken, a folder into itself, a
|
The refusals must move nothing and lose nothing: a name already taken, a folder into itself, a
|
||||||
mirrored source holding something Drive lacks, a file still open for writing.
|
mirrored source holding something Drive lacks, a file still open for writing.
|
||||||
|
- **`copy_file_range`.** A 17 MiB copy that takes several calls, copies into the middle of a file
|
||||||
|
and past its end, a short copy at the source's end, and a copy from a file whose writes are not
|
||||||
|
released yet. On a live mount the results are also read back from Drive.
|
||||||
|
- **Copies, sizes and links on Drive** (My files only). `pdfs copy` of a tree and of a file beside
|
||||||
|
itself, read back as listed and after `pdfs refresh`, with nothing queued; the refusal while a
|
||||||
|
file has writes Drive lacks, and never the version before a save (B256). `pdfs du` counts every
|
||||||
|
item below a folder and keeps counting one moved to the trash. `pdfs public-link update` keeps
|
||||||
|
the URL and every option it was not asked to change, and refuses a link id that is not the
|
||||||
|
folder's.
|
||||||
- **The CLI.** `pdfs` into a closed pipe exits 0 with nothing on stderr; `pdfs mkdir`, `rename`
|
- **The CLI.** `pdfs` into a closed pipe exits 0 with nothing on stderr; `pdfs mkdir`, `rename`
|
||||||
and `rm` by local path, and their refusals.
|
and `rm` by local path, and their refusals.
|
||||||
|
|
||||||
@@ -263,3 +275,21 @@ typo it is. A release run leaves it unset.
|
|||||||
|
|
||||||
What no automated run covers yet is listed under
|
What no automated run covers yet is listed under
|
||||||
[Verification still owed](ROADMAP.md#verification-still-owed).
|
[Verification still owed](ROADMAP.md#verification-still-owed).
|
||||||
|
|
||||||
|
### By hand in the app
|
||||||
|
|
||||||
|
The suite drives the CLI, not the app. Before a release that changes one of these screens, check
|
||||||
|
it by hand on a test account:
|
||||||
|
|
||||||
|
1. **Make a Copy** on a file in My files: the copy appears beside it with ` (1)`, and opens with
|
||||||
|
the same content. **Copy To…** a folder into another folder: the whole tree arrives.
|
||||||
|
2. Make a Copy on a file that is still open in an editor with unsaved changes: the app refuses,
|
||||||
|
and no copy appears.
|
||||||
|
3. **Details** of a folder: **Size on Proton Drive** shows a size and an item count, and its
|
||||||
|
tooltip says items in the trash are counted. Details of a file still shows **Size**.
|
||||||
|
4. **Share → Link settings** on a link with a password: change **Link role**. The link in the
|
||||||
|
dialog stays the same, and **Remove the password** is still offered. Then remove the password
|
||||||
|
and pick an expiry under **Expires**: the link row shows when it expires.
|
||||||
|
5. **Sync From Scratch…** on a mirror folder: after the pass, the folder holds the same files and
|
||||||
|
no conflict copies.
|
||||||
|
6. Each of the above in a second language (`LANGUAGE=de pdfs-app`): no English left.
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
pkgbase = proton-drive-for-linux-bin
|
pkgbase = proton-drive-for-linux-bin
|
||||||
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)
|
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)
|
||||||
pkgver = 3.5.0
|
pkgver = 3.6.0
|
||||||
pkgrel = 1
|
pkgrel = 1
|
||||||
url = https://proton-drive.narl.io
|
url = https://proton-drive.narl.io
|
||||||
install = proton-drive.install
|
install = proton-drive.install
|
||||||
@@ -22,21 +22,21 @@ pkgbase = proton-drive-for-linux-bin
|
|||||||
optdepends = xdg-utils: "Open folder" action in the tray menu
|
optdepends = xdg-utils: "Open folder" action in the tray menu
|
||||||
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
||||||
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
||||||
provides = proton-drive-for-linux=3.5.0
|
provides = proton-drive-for-linux=3.6.0
|
||||||
provides = proton-drive-linux=3.5.0
|
provides = proton-drive-linux=3.6.0
|
||||||
provides = pdfs
|
provides = pdfs
|
||||||
conflicts = proton-drive-linux
|
conflicts = proton-drive-linux
|
||||||
conflicts = proton-drive-for-linux
|
conflicts = proton-drive-for-linux
|
||||||
conflicts = proton-drive-for-linux-git
|
conflicts = proton-drive-for-linux-git
|
||||||
options = !strip
|
options = !strip
|
||||||
options = !debug
|
options = !debug
|
||||||
source = proton-drive-linux-3.5.0-x86_64.tar.gz::https://github.com/narrrl/proton-drive-linux/releases/download/v3.5.0/proton-drive-linux-3.5.0-x86_64.tar.gz
|
source = proton-drive-linux-3.6.0-x86_64.tar.gz::https://github.com/narrrl/proton-drive-linux/releases/download/v3.6.0/proton-drive-linux-3.6.0-x86_64.tar.gz
|
||||||
source = io.narl.proton-drive-linux-3.5.0.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.5.0/packaging/io.narl.proton-drive-linux.desktop
|
source = io.narl.proton-drive-linux-3.6.0.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.6.0/packaging/io.narl.proton-drive-linux.desktop
|
||||||
source = io.narl.proton-drive-linux-tray-3.5.0.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.5.0/packaging/io.narl.proton-drive-linux-tray.desktop
|
source = io.narl.proton-drive-linux-tray-3.6.0.desktop::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.6.0/packaging/io.narl.proton-drive-linux-tray.desktop
|
||||||
source = io.narl.proton-drive-linux-3.5.0.svg::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.5.0/packaging/io.narl.proton-drive-linux.svg
|
source = io.narl.proton-drive-linux-3.6.0.svg::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.6.0/packaging/io.narl.proton-drive-linux.svg
|
||||||
source = proton-drive-3.5.0.service::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.5.0/packaging/proton-drive.service
|
source = proton-drive-3.6.0.service::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.6.0/packaging/proton-drive.service
|
||||||
source = LICENSE-3.5.0::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.5.0/LICENSE
|
source = LICENSE-3.6.0::https://raw.githubusercontent.com/narrrl/proton-drive-linux/v3.6.0/LICENSE
|
||||||
sha256sums = 902f60de5f471cb15c84d0ecb29cb4644a0ac52ce36d2be54c16b4befeba8de0
|
sha256sums = 8e9b76b0df238af1c8978c33b4c27c0c548e1faa81b368d0d271ed564d0a76c7
|
||||||
sha256sums = d190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123
|
sha256sums = d190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123
|
||||||
sha256sums = 1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0
|
sha256sums = 1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0
|
||||||
sha256sums = e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828
|
sha256sums = e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Maintainer: Nils Pukropp <contact@narl.io>
|
# Maintainer: Nils Pukropp <contact@narl.io>
|
||||||
|
|
||||||
pkgname=proton-drive-for-linux-bin
|
pkgname=proton-drive-for-linux-bin
|
||||||
pkgver=3.5.0
|
pkgver=3.6.0
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)"
|
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (binaries)"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
@@ -36,7 +36,7 @@ source=("proton-drive-linux-$pkgver-x86_64.tar.gz::$_repo/releases/download/$_ta
|
|||||||
"io.narl.proton-drive-linux-$pkgver.svg::$_raw/packaging/io.narl.proton-drive-linux.svg"
|
"io.narl.proton-drive-linux-$pkgver.svg::$_raw/packaging/io.narl.proton-drive-linux.svg"
|
||||||
"proton-drive-$pkgver.service::$_raw/packaging/proton-drive.service"
|
"proton-drive-$pkgver.service::$_raw/packaging/proton-drive.service"
|
||||||
"LICENSE-$pkgver::$_raw/LICENSE")
|
"LICENSE-$pkgver::$_raw/LICENSE")
|
||||||
sha256sums=('902f60de5f471cb15c84d0ecb29cb4644a0ac52ce36d2be54c16b4befeba8de0'
|
sha256sums=('8e9b76b0df238af1c8978c33b4c27c0c548e1faa81b368d0d271ed564d0a76c7'
|
||||||
'd190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123'
|
'd190d6771fff0b975271fb62fd3fb0fa8bbf17d7278c10853c18557855b0c123'
|
||||||
'1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0'
|
'1d95e250370220e3bb6aead6ce34d1d6cc759f42c4803b75f1667e78ff47c6c0'
|
||||||
'e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828'
|
'e598a93c2715c3a9fdc7fb937902083d4a1135e8fbdfe6fe7e37f0383ebbd828'
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
pkgbase = proton-drive-for-linux-git
|
pkgbase = proton-drive-for-linux-git
|
||||||
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)
|
pkgdesc = Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)
|
||||||
pkgver = 3.5.0.r0.geb25c16
|
pkgver = 3.6.0.r0.g2720191
|
||||||
pkgrel = 1
|
pkgrel = 1
|
||||||
url = https://proton-drive.narl.io
|
url = https://proton-drive.narl.io
|
||||||
install = proton-drive.install
|
install = proton-drive.install
|
||||||
@@ -25,8 +25,8 @@ pkgbase = proton-drive-for-linux-git
|
|||||||
optdepends = xdg-utils: "Open folder" action in the tray menu
|
optdepends = xdg-utils: "Open folder" action in the tray menu
|
||||||
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
||||||
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
||||||
provides = proton-drive-for-linux=3.5.0.r0.geb25c16
|
provides = proton-drive-for-linux=3.6.0.r0.g2720191
|
||||||
provides = proton-drive-linux=3.5.0.r0.geb25c16
|
provides = proton-drive-linux=3.6.0.r0.g2720191
|
||||||
provides = pdfs
|
provides = pdfs
|
||||||
conflicts = proton-drive-linux
|
conflicts = proton-drive-linux
|
||||||
conflicts = proton-drive-for-linux
|
conflicts = proton-drive-for-linux
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# Maintainer: Nils Pukropp <contact@narl.io>
|
# Maintainer: Nils Pukropp <contact@narl.io>
|
||||||
|
|
||||||
pkgname=proton-drive-for-linux-git
|
pkgname=proton-drive-for-linux-git
|
||||||
pkgver=3.5.0.r0.geb25c16
|
pkgver=3.6.0.r0.g2720191
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)"
|
pkgdesc="Unofficial Proton Drive client: FUSE files-on-demand mount, CLI, GTK4 app, tray (git)"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
pkgbase = proton-drive-for-linux
|
pkgbase = proton-drive-for-linux
|
||||||
pkgdesc = Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray
|
pkgdesc = Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray
|
||||||
pkgver = 3.5.0
|
pkgver = 3.6.0
|
||||||
pkgrel = 1
|
pkgrel = 1
|
||||||
url = https://proton-drive.narl.io
|
url = https://proton-drive.narl.io
|
||||||
install = proton-drive.install
|
install = proton-drive.install
|
||||||
@@ -24,13 +24,13 @@ pkgbase = proton-drive-for-linux
|
|||||||
optdepends = xdg-utils: "Open folder" action in the tray menu
|
optdepends = xdg-utils: "Open folder" action in the tray menu
|
||||||
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
optdepends = fzf: search-as-you-type launcher (pdfs-prompt --fzf)
|
||||||
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
optdepends = foot: terminal that hosts pdfs-prompt --fzf when started from a keybinding
|
||||||
provides = proton-drive-linux=3.5.0
|
provides = proton-drive-linux=3.6.0
|
||||||
provides = pdfs
|
provides = pdfs
|
||||||
conflicts = proton-drive-linux
|
conflicts = proton-drive-linux
|
||||||
conflicts = proton-drive-for-linux-bin
|
conflicts = proton-drive-for-linux-bin
|
||||||
conflicts = proton-drive-for-linux-git
|
conflicts = proton-drive-for-linux-git
|
||||||
options = !lto
|
options = !lto
|
||||||
source = proton-drive-for-linux-3.5.0.tar.gz::https://github.com/narrrl/proton-drive-linux/archive/refs/tags/v3.5.0.tar.gz
|
source = proton-drive-for-linux-3.6.0.tar.gz::https://github.com/narrrl/proton-drive-linux/archive/refs/tags/v3.6.0.tar.gz
|
||||||
sha256sums = 5eb0e001ec2301ffb8b2b8429e907fab32810d6eba541bf978a0ea97867d8bc9
|
sha256sums = d9b16b1027816a379d6a11c6396d779f751a6f94becaa16b17439420f561d13d
|
||||||
|
|
||||||
pkgname = proton-drive-for-linux
|
pkgname = proton-drive-for-linux
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# Maintainer: Nils Pukropp <contact@narl.io>
|
# Maintainer: Nils Pukropp <contact@narl.io>
|
||||||
|
|
||||||
pkgname=proton-drive-for-linux
|
pkgname=proton-drive-for-linux
|
||||||
pkgver=3.5.0
|
pkgver=3.6.0
|
||||||
pkgrel=1
|
pkgrel=1
|
||||||
pkgdesc="Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray"
|
pkgdesc="Unofficial Proton Drive client: files-on-demand FUSE mount, CLI, GTK4 app and tray"
|
||||||
arch=('x86_64')
|
arch=('x86_64')
|
||||||
@@ -26,7 +26,7 @@ options=('!lto')
|
|||||||
# Stops the running daemon before removal, so the mount stops mirroring deletes.
|
# Stops the running daemon before removal, so the mount stops mirroring deletes.
|
||||||
install=proton-drive.install
|
install=proton-drive.install
|
||||||
source=("$pkgname-$pkgver.tar.gz::$_repo/archive/refs/tags/v$pkgver.tar.gz")
|
source=("$pkgname-$pkgver.tar.gz::$_repo/archive/refs/tags/v$pkgver.tar.gz")
|
||||||
sha256sums=('5eb0e001ec2301ffb8b2b8429e907fab32810d6eba541bf978a0ea97867d8bc9')
|
sha256sums=('d9b16b1027816a379d6a11c6396d779f751a6f94becaa16b17439420f561d13d')
|
||||||
|
|
||||||
_srcname="proton-drive-linux-$pkgver"
|
_srcname="proton-drive-linux-$pkgver"
|
||||||
|
|
||||||
|
|||||||
+506
-9
@@ -566,6 +566,357 @@ def test_mmap_and_copy_paths(ctx: Context) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_range(ctx: Context, key: str, src: int, dst: int, count: int, offset_src: int, offset_dst: int) -> int | None:
|
||||||
|
"""`copy_file_range` until `count` bytes moved or the source ran out.
|
||||||
|
|
||||||
|
Returns how many bytes moved, or None when the first call was refused
|
||||||
|
cleanly. The mount copies at most 16 MiB a call, and `cp` loops the same
|
||||||
|
way, so a large copy reaches it as several calls with growing offsets.
|
||||||
|
"""
|
||||||
|
moved = 0
|
||||||
|
first = ctx.probe(
|
||||||
|
key,
|
||||||
|
lambda: os.copy_file_range(src, dst, count, offset_src, offset_dst),
|
||||||
|
allow=CLEAN_UNSUPPORTED | {errno.EXDEV},
|
||||||
|
)
|
||||||
|
if not first.ok:
|
||||||
|
return None
|
||||||
|
step = first.value
|
||||||
|
calls = 1
|
||||||
|
while step and moved + step < count:
|
||||||
|
moved += step
|
||||||
|
step = os.copy_file_range(src, dst, count - moved, offset_src + moved, offset_dst + moved)
|
||||||
|
calls += 1
|
||||||
|
ctx.note(f"{key}.calls", calls)
|
||||||
|
return moved + step
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_file_range_ranges(ctx: Context) -> None:
|
||||||
|
"""`copy_file_range` beyond the first call, at offsets, and from an open file.
|
||||||
|
|
||||||
|
The mount answers the call from bytes this machine already has: the write
|
||||||
|
handle's scratch, a pending upload, or the cache. Each answer is a write
|
||||||
|
into the destination, so it has to land exactly where the offsets say,
|
||||||
|
fill a gap past the end with zeros, and survive the upload.
|
||||||
|
"""
|
||||||
|
root = ctx.root
|
||||||
|
big = pattern(17 * 1024 * 1024 + 4099, "cfr-big")
|
||||||
|
source = root / "cfr-source.bin"
|
||||||
|
write_durable(source, big)
|
||||||
|
|
||||||
|
def copy(key: str, src_path: Path, dst_path: Path, count: int, offset_src: int = 0, offset_dst: int = 0) -> int | None:
|
||||||
|
src = os.open(src_path, os.O_RDONLY)
|
||||||
|
dst = os.open(dst_path, os.O_CREAT | os.O_WRONLY, 0o600)
|
||||||
|
try:
|
||||||
|
moved = _copy_range(ctx, key, src, dst, count, offset_src, offset_dst)
|
||||||
|
os.fsync(dst)
|
||||||
|
finally:
|
||||||
|
os.close(src)
|
||||||
|
os.close(dst)
|
||||||
|
return moved
|
||||||
|
|
||||||
|
expected: dict[Path, bytes] = {}
|
||||||
|
|
||||||
|
whole = root / "cfr-whole.bin"
|
||||||
|
moved = copy("whole", source, whole, len(big))
|
||||||
|
if moved is not None:
|
||||||
|
check(moved == len(big), f"whole: copy_file_range moved {moved} bytes, expected {len(big)}")
|
||||||
|
expected[whole] = big
|
||||||
|
|
||||||
|
# Into the middle of a file that exists, and past the end of another.
|
||||||
|
middle = root / "cfr-middle.bin"
|
||||||
|
base = pattern(300_000, "cfr-base")
|
||||||
|
write_durable(middle, base)
|
||||||
|
moved = copy("middle", source, middle, 100_000, offset_src=5_000_000, offset_dst=123_457)
|
||||||
|
if moved is not None:
|
||||||
|
check(moved == 100_000, f"middle: copy_file_range moved {moved} bytes, expected {100_000}")
|
||||||
|
expected[middle] = base[:123_457] + big[5_000_000:5_100_000] + base[223_457:]
|
||||||
|
gap = root / "cfr-gap.bin"
|
||||||
|
write_durable(gap, b"head")
|
||||||
|
moved = copy("gap", source, gap, 70_001, offset_src=1, offset_dst=200_000)
|
||||||
|
if moved is not None:
|
||||||
|
check(moved == 70_001, f"gap: copy_file_range moved {moved} bytes, expected {70_001}")
|
||||||
|
expected[gap] = b"head" + bytes(200_000 - 4) + big[1:70_002]
|
||||||
|
|
||||||
|
# Past the source's end the copy is short, and at it nothing moves.
|
||||||
|
tail = root / "cfr-tail.bin"
|
||||||
|
moved = copy("tail", source, tail, 10_000, offset_src=len(big) - 99)
|
||||||
|
if moved is not None:
|
||||||
|
check(moved == 99, f"tail: copy_file_range moved {moved} bytes, expected {99}")
|
||||||
|
expected[tail] = big[-99:]
|
||||||
|
|
||||||
|
# From a file whose last write has not been released yet.
|
||||||
|
unsaved = root / "cfr-unsaved.bin"
|
||||||
|
fresh = pattern(600_001, "cfr-unsaved")
|
||||||
|
writer = os.open(unsaved, os.O_CREAT | os.O_RDWR | os.O_TRUNC, 0o600)
|
||||||
|
try:
|
||||||
|
write_all(writer, fresh)
|
||||||
|
moved = copy("unsaved", unsaved, root / "cfr-from-unsaved.bin", len(fresh))
|
||||||
|
finally:
|
||||||
|
os.close(writer)
|
||||||
|
if moved is not None:
|
||||||
|
check(moved == len(fresh), f"unsaved: copy_file_range moved {moved} bytes, expected {len(fresh)}")
|
||||||
|
expected[root / "cfr-from-unsaved.bin"] = fresh
|
||||||
|
|
||||||
|
def verify(when: str) -> None:
|
||||||
|
for path, data in expected.items():
|
||||||
|
size = os.lstat(path).st_size
|
||||||
|
check(size == len(data), f"{path.name} {when}: {size} bytes, expected {len(data)}")
|
||||||
|
check_bytes(read(path), data, f"{path.name} {when}")
|
||||||
|
|
||||||
|
verify("right after the copy")
|
||||||
|
if not ctx.is_live or ctx.daemon is None:
|
||||||
|
return
|
||||||
|
ctx.daemon.wait_for_queue()
|
||||||
|
verify("after the upload")
|
||||||
|
# A server-side copy has a new uid, so nothing of it is cached: reading
|
||||||
|
# it reads what Drive holds.
|
||||||
|
if _my_files_root(ctx) is None:
|
||||||
|
return
|
||||||
|
landed = root / "cfr-landed"
|
||||||
|
landed.mkdir()
|
||||||
|
ctx.daemon.wait_for_queue()
|
||||||
|
for path in expected:
|
||||||
|
_pdfs_copy(ctx.daemon, path, landed)
|
||||||
|
for path, data in expected.items():
|
||||||
|
check_bytes(read(landed / path.name), data, f"{path.name} as Drive holds it")
|
||||||
|
|
||||||
|
|
||||||
|
def _my_files_root(ctx: Context) -> Path | None:
|
||||||
|
"""The My files mount the case runs in, or None anywhere else.
|
||||||
|
|
||||||
|
`pdfs copy`, `pdfs du` and public links address a path inside My files;
|
||||||
|
synced folders have no path of their own there.
|
||||||
|
"""
|
||||||
|
if ctx.daemon is None:
|
||||||
|
return None
|
||||||
|
mountpoint = (ctx.daemon.status().get("mount") or {}).get("mountpoint")
|
||||||
|
if not mountpoint:
|
||||||
|
return None
|
||||||
|
mountpoint = Path(mountpoint).resolve()
|
||||||
|
return mountpoint if mountpoint in ctx.root.resolve().parents else None
|
||||||
|
|
||||||
|
|
||||||
|
def _require_my_files(ctx: Context) -> Daemon:
|
||||||
|
daemon = ctx.require_daemon()
|
||||||
|
if _my_files_root(ctx) is None:
|
||||||
|
raise Skip("needs a root inside My files")
|
||||||
|
return daemon
|
||||||
|
|
||||||
|
|
||||||
|
COPIED_TO = re.compile(r"^copied to (.+)$")
|
||||||
|
|
||||||
|
|
||||||
|
def _pdfs_copy(daemon: Daemon, source: Path, dest: Path) -> str:
|
||||||
|
"""`pdfs copy`, returning the name the copy got."""
|
||||||
|
message = daemon.command("copy", str(source), str(dest)).strip()
|
||||||
|
match = COPIED_TO.match(message)
|
||||||
|
check(match is not None, f"pdfs copy {source.name}: unexpected reply {message!r}")
|
||||||
|
return match[1]
|
||||||
|
|
||||||
|
|
||||||
|
def _pdfs_copy_refused(daemon: Daemon, source: Path, dest: Path) -> str | None:
|
||||||
|
"""`pdfs copy` expected to be refused for changes Drive lacks: the refusal,
|
||||||
|
or None if the copy went ahead (and returned its name in the log)."""
|
||||||
|
try:
|
||||||
|
daemon.command("copy", str(source), str(dest))
|
||||||
|
except RuntimeError as error:
|
||||||
|
detail = str(error)
|
||||||
|
check("have not reached Proton Drive" in detail, f"pdfs copy {source.name}: unexpected refusal: {detail}")
|
||||||
|
return detail
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_copy(ctx: Context) -> None:
|
||||||
|
"""`pdfs copy` makes an exact copy on Drive and uploads nothing.
|
||||||
|
|
||||||
|
A tree and a lone file are copied; the file into its own folder, where
|
||||||
|
its name is taken and the copy gets the next free one. The copy is read
|
||||||
|
back as listed and again after its listing is dropped, and nothing of it
|
||||||
|
may wait in the queue: Drive made it, so there is nothing to send.
|
||||||
|
"""
|
||||||
|
daemon = _require_my_files(ctx)
|
||||||
|
root = ctx.root
|
||||||
|
name = f"tree-{uuid.uuid4().hex[:8]}"
|
||||||
|
files = _move_tree(root, name)
|
||||||
|
dest = root / "copies"
|
||||||
|
dest.mkdir()
|
||||||
|
daemon.wait_for_queue()
|
||||||
|
|
||||||
|
check(_pdfs_copy(daemon, root / name, dest) == name, f"the copy of {name} was renamed")
|
||||||
|
note = root / name / "note.txt"
|
||||||
|
twin = _pdfs_copy(daemon, note, note.parent)
|
||||||
|
check(twin != note.name, f"the copy of {note.name} beside it kept the name {twin!r}")
|
||||||
|
check((note.parent / twin).is_file(), f"{twin} is not listed beside {note.name}")
|
||||||
|
queued = [
|
||||||
|
item
|
||||||
|
for item in daemon.queue()
|
||||||
|
if root.name in (parts := Path(item.get("path", "")).parts) and ("copies" in parts or twin in parts)
|
||||||
|
]
|
||||||
|
check(not queued, f"a server copy queued work: {_describe_uploads(queued)}")
|
||||||
|
|
||||||
|
def verify(when: str) -> None:
|
||||||
|
for relative, data in files.items():
|
||||||
|
size = os.lstat(dest / relative).st_size
|
||||||
|
check(size == len(data), f"{relative} {when}: {size} bytes, expected {len(data)}")
|
||||||
|
check_bytes(read(dest / relative), data, f"{relative} {when}")
|
||||||
|
check((dest / name / "empty-dir").is_dir(), f"the empty folder is missing {when}")
|
||||||
|
check_bytes(read(note.parent / twin), files[Path(name) / "note.txt"], f"{twin} {when}")
|
||||||
|
|
||||||
|
verify("right after the copy")
|
||||||
|
folders = sorted({dest / relative.parent for relative in files} | {dest / name, note.parent})
|
||||||
|
for folder in folders:
|
||||||
|
daemon.command("refresh", str(folder))
|
||||||
|
verify("after dropping the cached listing")
|
||||||
|
_unchanged_at(root, files, "after copying it")
|
||||||
|
|
||||||
|
|
||||||
|
def _unchanged_at(root: Path, files: dict[Path, bytes], what: str) -> None:
|
||||||
|
for relative, data in files.items():
|
||||||
|
check_bytes(read(root / relative), data, f"{relative} {what}")
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_copy_never_copies_an_old_version(ctx: Context) -> None:
|
||||||
|
"""`pdfs copy` of a file being written is refused, and never copies the
|
||||||
|
version before the last save (B256).
|
||||||
|
|
||||||
|
Drive copies what it holds. A file open with unsaved writes, or saved but
|
||||||
|
not yet uploaded, holds more here than there: the copy has to wait.
|
||||||
|
"""
|
||||||
|
daemon = _require_my_files(ctx)
|
||||||
|
root = ctx.root
|
||||||
|
path = root / "being-written.txt"
|
||||||
|
write_durable(path, b"first version")
|
||||||
|
# Its own folder: every case of a run shares the root, and "pdfs copy"
|
||||||
|
# already made "copies" there.
|
||||||
|
dest = root / "b256-copies"
|
||||||
|
dest.mkdir()
|
||||||
|
daemon.wait_for_queue()
|
||||||
|
|
||||||
|
fd = os.open(path, os.O_WRONLY | os.O_TRUNC)
|
||||||
|
try:
|
||||||
|
write_all(fd, b"second version, still open")
|
||||||
|
check(
|
||||||
|
_pdfs_copy_refused(daemon, path, dest) is not None,
|
||||||
|
"pdfs copy went ahead while the file had unsaved writes",
|
||||||
|
)
|
||||||
|
check(not (dest / path.name).exists(), "a refused copy left a file behind")
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
# Right after the close: refused while the upload is owed, then the new
|
||||||
|
# bytes. Never the first version.
|
||||||
|
saved = b"third version, saved"
|
||||||
|
write_durable(path, saved)
|
||||||
|
deadline = time.monotonic() + daemon.timeout
|
||||||
|
refusals = 0
|
||||||
|
while _pdfs_copy_refused(daemon, path, dest) is not None:
|
||||||
|
refusals += 1
|
||||||
|
check(time.monotonic() < deadline, f"pdfs copy was still refused after {daemon.timeout}s")
|
||||||
|
time.sleep(1)
|
||||||
|
ctx.note("refusals before the upload landed", refusals)
|
||||||
|
check_bytes(read(dest / path.name), saved, "the copy made right after a save")
|
||||||
|
|
||||||
|
|
||||||
|
def test_folder_size(ctx: Context) -> None:
|
||||||
|
"""`pdfs du` counts exactly what is below a folder, trash included.
|
||||||
|
|
||||||
|
Drive adds the size up from encrypted bytes, so it is at least the sum of
|
||||||
|
the files. A file moved to the trash still counts, as `pdfs du` says.
|
||||||
|
"""
|
||||||
|
daemon = _require_my_files(ctx)
|
||||||
|
root = ctx.root
|
||||||
|
name = f"sized-{uuid.uuid4().hex[:8]}"
|
||||||
|
files = _move_tree(root, name)
|
||||||
|
folder = root / name
|
||||||
|
items = sum(1 for _ in folder.rglob("*"))
|
||||||
|
daemon.wait_for_queue()
|
||||||
|
|
||||||
|
def size() -> dict:
|
||||||
|
return json.loads(daemon.command("du", str(folder), json_output=True))
|
||||||
|
|
||||||
|
# Drive may add a new tree up a moment after it lands.
|
||||||
|
deadline = time.monotonic() + daemon.timeout
|
||||||
|
while (answer := size())["items"] != items and time.monotonic() < deadline:
|
||||||
|
time.sleep(2)
|
||||||
|
check(answer["items"] == items, f"pdfs du counts {answer['items']} items, expected {items}")
|
||||||
|
total = sum(len(data) for data in files.values())
|
||||||
|
check(answer["size"] >= total, f"pdfs du reports {answer['size']} bytes for {total} bytes of files")
|
||||||
|
ctx.info(f"{answer['size']} bytes on Drive for {total} bytes in {items} items")
|
||||||
|
|
||||||
|
os.unlink(folder / "note.txt")
|
||||||
|
daemon.wait_for_queue()
|
||||||
|
check(size()["items"] == items, "a file moved to the trash stopped counting in pdfs du")
|
||||||
|
|
||||||
|
|
||||||
|
def _public_link(daemon: Daemon, path: Path) -> dict | None:
|
||||||
|
return json.loads(daemon.command("members", str(path), json_output=True)).get("link")
|
||||||
|
|
||||||
|
|
||||||
|
def _link_token(url: str | None) -> str:
|
||||||
|
"""The part of a public link's URL that names the link, without the
|
||||||
|
password fragment."""
|
||||||
|
check(bool(url), "the link has no URL")
|
||||||
|
return url.split("#", 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_link_update(ctx: Context) -> None:
|
||||||
|
"""`pdfs public-link update` changes a link and keeps its URL.
|
||||||
|
|
||||||
|
Each option changed alone leaves the others as they were: a role change
|
||||||
|
keeps the password, removing the password keeps the expiry. A link id
|
||||||
|
that is not the folder's link is refused.
|
||||||
|
"""
|
||||||
|
daemon = _require_my_files(ctx)
|
||||||
|
folder = ctx.root / "linked"
|
||||||
|
folder.mkdir()
|
||||||
|
write_durable(folder / "inside.txt", b"behind a public link")
|
||||||
|
daemon.wait_for_queue()
|
||||||
|
|
||||||
|
password = f"pw-{uuid.uuid4().hex[:12]}"
|
||||||
|
created = json.loads(
|
||||||
|
daemon.command("public-link", "create", str(folder), "--password", password, json_output=True)
|
||||||
|
)["link"]
|
||||||
|
link_id, token = created["id"], _link_token(created["url"])
|
||||||
|
try:
|
||||||
|
check(created["has_password"], "the new link has no password")
|
||||||
|
|
||||||
|
def update(*args: str) -> dict:
|
||||||
|
link = json.loads(
|
||||||
|
daemon.command("public-link", "update", str(folder), link_id, *args, json_output=True)
|
||||||
|
)["link"]
|
||||||
|
check(link["id"] == link_id, f"the update replaced the link: {link['id']} for {link_id}")
|
||||||
|
if link.get("url"):
|
||||||
|
check(_link_token(link["url"]) == token, "the update changed the link's URL")
|
||||||
|
listed = _public_link(daemon, folder)
|
||||||
|
check(listed is not None and listed["id"] == link_id, f"the link is gone after the update: {listed}")
|
||||||
|
return listed
|
||||||
|
|
||||||
|
listed = update("--role", "editor")
|
||||||
|
check(listed["role"] == "editor", f"the role is {listed['role']} after changing it to editor")
|
||||||
|
check(listed["has_password"], "changing the role removed the password")
|
||||||
|
|
||||||
|
expires = int(time.time()) + 7 * 24 * 3600
|
||||||
|
listed = update("--no-password", "--expires", str(expires))
|
||||||
|
check(not listed["has_password"], "--no-password kept the password")
|
||||||
|
check(listed["expires"] == expires, f"the expiry is {listed['expires']}, expected {expires}")
|
||||||
|
check(listed["role"] == "editor", "removing the password changed the role")
|
||||||
|
|
||||||
|
listed = update("--no-expiry")
|
||||||
|
check(listed["expires"] is None, f"--no-expiry kept the expiry {listed['expires']}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
daemon.command("public-link", "update", str(folder), f"not-{link_id}", "--role", "viewer")
|
||||||
|
except RuntimeError as error:
|
||||||
|
check("not found" in str(error), f"a wrong link id was refused for another reason: {error}")
|
||||||
|
else:
|
||||||
|
check(False, "an update with a wrong link id went ahead")
|
||||||
|
check(_public_link(daemon, folder)["role"] == "editor", "a refused update changed the link")
|
||||||
|
finally:
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
daemon.command("public-link", "remove", str(folder), link_id)
|
||||||
|
|
||||||
|
|
||||||
def test_namespace_and_errors(ctx: Context) -> None:
|
def test_namespace_and_errors(ctx: Context) -> None:
|
||||||
root = ctx.root
|
root = ctx.root
|
||||||
a, b = root / "dir-a", root / "dir-b"
|
a, b = root / "dir-a", root / "dir-b"
|
||||||
@@ -2625,6 +2976,7 @@ TESTS = [
|
|||||||
Case("positioned and vectored I/O", test_positioned_and_vectored_io),
|
Case("positioned and vectored I/O", test_positioned_and_vectored_io),
|
||||||
Case("truncate, growth, sparse ranges, and EOF", test_resize_and_sparse_io),
|
Case("truncate, growth, sparse ranges, and EOF", test_resize_and_sparse_io),
|
||||||
Case("mmap and zero-copy copy paths", test_mmap_and_copy_paths),
|
Case("mmap and zero-copy copy paths", test_mmap_and_copy_paths),
|
||||||
|
Case("copy_file_range across calls, at offsets, and from an open file", test_copy_file_range_ranges),
|
||||||
Case("namespace operations and POSIX errors", test_namespace_and_errors),
|
Case("namespace operations and POSIX errors", test_namespace_and_errors),
|
||||||
Case("open-handle lifetime across unlink and rename", test_open_lifetime),
|
Case("open-handle lifetime across unlink and rename", test_open_lifetime),
|
||||||
Case("names, lookup, stat, and enumeration", test_names_and_enumeration),
|
Case("names, lookup, stat, and enumeration", test_names_and_enumeration),
|
||||||
@@ -2670,6 +3022,10 @@ TESTS = [
|
|||||||
Case("regression B116: trashing a folder drops the creates queued in its subfolders", test_regression_b116_trashed_folder_drops_deeper_creates, (LIVE,)),
|
Case("regression B116: trashing a folder drops the creates queued in its subfolders", test_regression_b116_trashed_folder_drops_deeper_creates, (LIVE,)),
|
||||||
Case("regression B117: a moved file's queued upload keeps its path", test_regression_b117_moved_upload_keeps_its_path, (LIVE,)),
|
Case("regression B117: a moved file's queued upload keeps its path", test_regression_b117_moved_upload_keeps_its_path, (LIVE,)),
|
||||||
Case("regression B119: an uploaded file stays cached", test_regression_b119_uploaded_file_stays_cached, (LIVE,)),
|
Case("regression B119: an uploaded file stays cached", test_regression_b119_uploaded_file_stays_cached, (LIVE,)),
|
||||||
|
Case("pdfs copy: a copy on Drive reads back exactly and uploads nothing", test_server_copy, (LIVE,)),
|
||||||
|
Case("regression B256: pdfs copy never copies the version before a save", test_server_copy_never_copies_an_old_version, (LIVE,)),
|
||||||
|
Case("pdfs du counts what is below a folder, trash included", test_folder_size, (LIVE,)),
|
||||||
|
Case("pdfs public-link update keeps the URL and the options not changed", test_public_link_update, (LIVE,)),
|
||||||
Case("durability across a daemon restart", test_durability_across_restart, (LIVE,)),
|
Case("durability across a daemon restart", test_durability_across_restart, (LIVE,)),
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2942,7 +3298,7 @@ def select_cases(kind: str, pool: list[Case] | None = None) -> list[Case]:
|
|||||||
# typo worth failing on.
|
# typo worth failing on.
|
||||||
for needle in needles:
|
for needle in needles:
|
||||||
check(
|
check(
|
||||||
any(needle in case.name.lower() for case in TESTS + MOVE_CASES),
|
any(needle in case.name.lower() for case in TESTS + MOVE_CASES + RECOVERY_CASES),
|
||||||
f"PDFS_ACCEPTANCE_ONLY: {needle!r} matched no tests",
|
f"PDFS_ACCEPTANCE_ONLY: {needle!r} matched no tests",
|
||||||
)
|
)
|
||||||
return [case for case in cases if any(needle in case.name.lower() for needle in needles)]
|
return [case for case in cases if any(needle in case.name.lower() for needle in needles)]
|
||||||
@@ -3658,30 +4014,36 @@ class Janitor:
|
|||||||
return [
|
return [
|
||||||
f"{name} ({entry['uid']}) is still in the trash"
|
f"{name} ({entry['uid']}) is still in the trash"
|
||||||
for entry in matches
|
for entry in matches
|
||||||
if not self.purge(entry["uid"], name)
|
if not self.purge(entry["uid"], name, listed=True)
|
||||||
]
|
]
|
||||||
|
|
||||||
def purge(self, uid: str, name: str) -> bool:
|
def purge(self, uid: str, name: str, listed: bool = False) -> bool:
|
||||||
"""Permanently delete one trashed node the run owns, by uid.
|
"""Permanently delete one trashed node the run owns, by uid.
|
||||||
|
|
||||||
By uid and never by name: a name can match something of the user's, a
|
By uid and never by name: a name can match something of the user's, a
|
||||||
uid cannot. The node has to show up in the trash first, which is also
|
uid cannot. The node has to show up in the trash first, which is also
|
||||||
what proves the trash step before it worked, and it has to be gone from
|
what proves the trash step before it worked, and it has to be gone from
|
||||||
a fresh listing afterwards.
|
a fresh listing afterwards. `listed` says the caller just found it in
|
||||||
|
the trash.
|
||||||
|
|
||||||
The server's own answer comes first, though: it only deletes a node
|
The server's own answer comes first, though: it only deletes a node
|
||||||
that is in the trash, and it answers per uid. A listing is a whole-trash
|
that is in the trash, and it answers per uid. A listing is a whole-trash
|
||||||
enumeration, which on a large trash can time out on every attempt and
|
enumeration, which on a large trash can time out on every attempt and
|
||||||
would otherwise keep a folder the server would delete on request.
|
would otherwise keep a folder the server would delete on request.
|
||||||
"""
|
"""
|
||||||
with contextlib.suppress(Exception):
|
last_error = ""
|
||||||
|
try:
|
||||||
reply = self.daemon.command("delete-forever", uid, stdin="yes\n")
|
reply = self.daemon.command("delete-forever", uid, stdin="yes\n")
|
||||||
if re.search(r"\bpermanently deleted 1 item", reply):
|
if re.search(r"\bpermanently deleted 1 item", reply):
|
||||||
print(f"[cleanup] permanently deleted {name}")
|
print(f"[cleanup] permanently deleted {name}")
|
||||||
return True
|
return True
|
||||||
|
except Exception as error: # noqa: BLE001 - retried, then reported
|
||||||
|
last_error = f": {error}"
|
||||||
deadline = time.monotonic() + self.daemon.timeout
|
deadline = time.monotonic() + self.daemon.timeout
|
||||||
attempted = False
|
# A node seen in the trash before that delete and gone after it was
|
||||||
last_error = ""
|
# deleted by it, whatever it answered. Counting it as never sent kept
|
||||||
|
# waiting for a node that was already gone, and failed the run.
|
||||||
|
attempted = listed
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
if self.daemon.trashed(uid):
|
if self.daemon.trashed(uid):
|
||||||
@@ -3793,7 +4155,7 @@ class Janitor:
|
|||||||
for entry in trashed:
|
for entry in trashed:
|
||||||
if STALE_ROOT.match(entry["name"]) and entry["name"] not in live_names:
|
if STALE_ROOT.match(entry["name"]) and entry["name"] not in live_names:
|
||||||
print(f"[reap] permanently deleting an abandoned root from the trash: {entry['name']}")
|
print(f"[reap] permanently deleting an abandoned root from the trash: {entry['name']}")
|
||||||
if not orphan.purge(entry["uid"], entry["name"]):
|
if not orphan.purge(entry["uid"], entry["name"], listed=True):
|
||||||
leftovers.append(f"{entry['name']} ({entry['uid']}) is still in the trash")
|
leftovers.append(f"{entry['name']} ({entry['uid']}) is still in the trash")
|
||||||
return leftovers
|
return leftovers
|
||||||
|
|
||||||
@@ -3894,7 +4256,7 @@ class ManagedSyncPair:
|
|||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"cannot find {self.pdfs!r}; set PDFS_ACCEPTANCE_PDFS to the CLI binary"
|
f"cannot find {self.pdfs!r}; set PDFS_ACCEPTANCE_PDFS to the CLI binary"
|
||||||
)
|
)
|
||||||
if self.paths[0] == self.paths[1]:
|
if len(set(self.paths)) != len(self.paths):
|
||||||
raise ValueError("managed live paths must be different directories")
|
raise ValueError("managed live paths must be different directories")
|
||||||
deadline = time.monotonic() + self.timeout
|
deadline = time.monotonic() + self.timeout
|
||||||
last_error = None
|
last_error = None
|
||||||
@@ -4537,6 +4899,138 @@ def run_move_contract(
|
|||||||
return run
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Getting a synced folder unstuck
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class RecoveryContext:
|
||||||
|
"""One mirror folder of the run's own, for cases that reset or empty it.
|
||||||
|
|
||||||
|
The mode matrix needs its folders' sentinels intact, so these cases get a
|
||||||
|
folder nobody else checks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, pair: ManagedSyncPair, daemon: Daemon) -> None:
|
||||||
|
self.pair = pair
|
||||||
|
self.daemon = daemon
|
||||||
|
self.root = pair.paths[0]
|
||||||
|
self.id = pair.ids[self.root]
|
||||||
|
self.infos: list[str] = []
|
||||||
|
|
||||||
|
def folder(self) -> dict:
|
||||||
|
return next(item for item in self.pair.folders() if Path(item["local_path"]).resolve() == self.root)
|
||||||
|
|
||||||
|
def tree(self) -> tuple[str, dict[Path, bytes]]:
|
||||||
|
"""A fresh tree in the folder, with one file named after it, so that a
|
||||||
|
trash entry of it cannot be the user's."""
|
||||||
|
name = f"kept-{uuid.uuid4().hex[:8]}"
|
||||||
|
files = _move_tree(self.root, name)
|
||||||
|
files[Path(name) / f"{name}.txt"] = b"named after its tree " + name.encode()
|
||||||
|
write_durable(self.root / name / f"{name}.txt", files[Path(name) / f"{name}.txt"])
|
||||||
|
return name, files
|
||||||
|
|
||||||
|
def trashed(self, names: set[str]) -> list[str]:
|
||||||
|
self.daemon.command("refresh", "trash")
|
||||||
|
return sorted(entry["name"] for entry in self.daemon.trash() if entry["name"] in names)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_reset_changes_nothing(rc: RecoveryContext) -> None:
|
||||||
|
"""Sync From Scratch compares the folder with Drive again and changes
|
||||||
|
nothing that already matched: no deletes, no conflict copies.
|
||||||
|
|
||||||
|
`pdfs sync reset` forgets the folder's record of what it synced. The next
|
||||||
|
pass then sees every file on both sides with no history, which is the
|
||||||
|
case that must copy what is missing and leave matching files alone.
|
||||||
|
"""
|
||||||
|
name, files = rc.tree()
|
||||||
|
rc.pair.force_sync(rc.root)
|
||||||
|
rc.daemon.command("sync", "reset", str(rc.id))
|
||||||
|
# Added after the reset, so the pass after it has to upload it.
|
||||||
|
later = Path(name) / "added-after-the-reset.txt"
|
||||||
|
files[later] = b"only here when the pass starts " + name.encode()
|
||||||
|
write_durable(rc.root / later, files[later])
|
||||||
|
rc.pair.force_sync(rc.root)
|
||||||
|
rc.pair.force_sync(rc.root)
|
||||||
|
|
||||||
|
_unchanged_at(rc.root, files, "after Sync From Scratch")
|
||||||
|
copies = [str(path) for path in rc.root.rglob("*") if "(sync-conflict" in path.name]
|
||||||
|
check(not copies, f"Sync From Scratch made conflict copies: {copies}")
|
||||||
|
trashed = rc.trashed({name, f"{name}.txt"})
|
||||||
|
check(not trashed, f"Sync From Scratch trashed {trashed}")
|
||||||
|
state = rc.folder()
|
||||||
|
check(state["state"] == "idle", f"the folder is {state['state']} after Sync From Scratch")
|
||||||
|
|
||||||
|
|
||||||
|
def test_emptied_folder_switches_to_ondemand(rc: RecoveryContext) -> None:
|
||||||
|
"""A mirror folder whose local files are all gone switches to online only,
|
||||||
|
and Drive keeps every file (B255, issue #46).
|
||||||
|
|
||||||
|
The wipe guard rightly refuses to pass the local loss on to Drive, but it
|
||||||
|
also blocked the push pass a queued switch runs first, so the switch never
|
||||||
|
landed and the folder stayed in error. Through the mount afterwards, every
|
||||||
|
file has to read back as it was.
|
||||||
|
"""
|
||||||
|
names: set[str] = set()
|
||||||
|
files: dict[Path, bytes] = {}
|
||||||
|
for _ in range(2):
|
||||||
|
name, tree = rc.tree()
|
||||||
|
names |= {name, f"{name}.txt"}
|
||||||
|
files.update(tree)
|
||||||
|
rc.pair.force_sync(rc.root)
|
||||||
|
for child in list(rc.root.iterdir()):
|
||||||
|
if child.is_dir() and not child.is_symlink():
|
||||||
|
shutil.rmtree(child)
|
||||||
|
else:
|
||||||
|
child.unlink()
|
||||||
|
rc.daemon.command("sync", "mode", str(rc.id), "ondemand")
|
||||||
|
|
||||||
|
# The folder may pass through error while the guard holds the deletions
|
||||||
|
# back; the switch has to land anyway.
|
||||||
|
deadline = time.monotonic() + rc.pair.timeout
|
||||||
|
while True:
|
||||||
|
state = rc.folder()
|
||||||
|
if state["mode"] == "ondemand" and state.get("pending_mode") is None and is_mountpoint(rc.root):
|
||||||
|
break
|
||||||
|
check(time.monotonic() < deadline, f"the switch to on-demand did not land: {state}")
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
for relative, data in files.items():
|
||||||
|
data_read = rc.pair.through_outage(lambda relative=relative: read(rc.root / relative), f"reading {relative}", rc.infos)
|
||||||
|
check_bytes(data_read, data, f"{relative} on Drive after the local copy was gone")
|
||||||
|
sentinel = rc.root / "pdfs-mode-preservation.bin"
|
||||||
|
check_bytes(read(sentinel), rc.pair.sentinels[rc.root], f"{sentinel.name} on Drive")
|
||||||
|
trashed = rc.trashed(names)
|
||||||
|
check(not trashed, f"the switch trashed {trashed} on Drive")
|
||||||
|
|
||||||
|
|
||||||
|
RECOVERY_CASES = [
|
||||||
|
Case("Sync From Scratch changes nothing that already matched", test_sync_reset_changes_nothing, (LIVE,)),
|
||||||
|
Case("regression B255: an emptied mirror folder switches to on-demand", test_emptied_folder_switches_to_ondemand, (LIVE,)),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def run_sync_recovery(janitor: Janitor, args) -> None:
|
||||||
|
"""The recovery cases, in a sync folder of their own."""
|
||||||
|
cases = select_cases(LIVE, RECOVERY_CASES)
|
||||||
|
if not cases:
|
||||||
|
return
|
||||||
|
timeout = int(os.environ.get("PDFS_ACCEPTANCE_SYNC_TIMEOUT", "180"))
|
||||||
|
path = janitor.new_sync_folder()
|
||||||
|
pair = ManagedSyncPair([path], timeout)
|
||||||
|
daemon = Daemon.discover(timeout)
|
||||||
|
check(daemon is not None, "the daemon stopped answering")
|
||||||
|
run = Run("sync folder recovery", LIVE, path)
|
||||||
|
REPORT.append(run)
|
||||||
|
announce(run, {"folder": f"mirror, {path}", "cases": describe_cases(LIVE, cases, RECOVERY_CASES)})
|
||||||
|
started = time.monotonic()
|
||||||
|
pair.create()
|
||||||
|
context = RecoveryContext(pair, daemon)
|
||||||
|
run_cases(run, cases, lambda _case: context, max(args.timeout, 8 * timeout), args.fail_fast)
|
||||||
|
run.wall = time.monotonic() - started
|
||||||
|
CONSOLE.fact("result", tally(run.counts(), run.wall))
|
||||||
|
|
||||||
|
|
||||||
def is_mountpoint(path: Path) -> bool:
|
def is_mountpoint(path: Path) -> bool:
|
||||||
"""True for a distinct mount, including FUSE mounts over an existing dir."""
|
"""True for a distinct mount, including FUSE mounts over an existing dir."""
|
||||||
return os.path.ismount(path)
|
return os.path.ismount(path)
|
||||||
@@ -4694,6 +5188,8 @@ def main() -> int:
|
|||||||
print(f"{','.join(case.kinds):<18} {case.name}")
|
print(f"{','.join(case.kinds):<18} {case.name}")
|
||||||
for case in MOVE_CASES:
|
for case in MOVE_CASES:
|
||||||
print(f"{'managed':<18} {case.name}")
|
print(f"{'managed':<18} {case.name}")
|
||||||
|
for case in RECOVERY_CASES:
|
||||||
|
print(f"{'account':<18} {case.name}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
install_interrupt_handlers()
|
install_interrupt_handlers()
|
||||||
@@ -4814,6 +5310,7 @@ def run_account(reference: Run, args) -> None:
|
|||||||
)
|
)
|
||||||
paths = [janitor.new_sync_folder(), janitor.new_sync_folder()]
|
paths = [janitor.new_sync_folder(), janitor.new_sync_folder()]
|
||||||
run_managed_matrix(paths, reference, args, janitor)
|
run_managed_matrix(paths, reference, args, janitor)
|
||||||
|
run_sync_recovery(janitor, args)
|
||||||
finally:
|
finally:
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
leftovers = janitor.cleanup()
|
leftovers = janitor.cleanup()
|
||||||
|
|||||||
Reference in new issue
Block a user