revisions: a save keeps what it replaces

Every write of a document's text keeps the text it replaces, in the same
transaction. Autosave writes every few seconds, so an ordinary save keeps
one at most every ten minutes; `:w!`, a save over the SFTP mount and a
restore always keep one, because each may undo something. A rename alone,
or a document that was empty, keeps nothing.

`gh` or `:history` lists the open document's earlier versions, the newest
first, and shows the highlighted one as a line diff against the text as it
is now. Enter brings it back, and what it replaces becomes a version of
its own, so a restore is undone the same way.

The daily sweep forgets versions older than WORKSPACE_REVISION_DAYS (90)
but never the newest 20 of a document, and keeps an upload as long as a
version still embeds it. Versions stay out of the export, the mount and
the readable mirror; a document in the trash shows none.
This commit is contained in:
nvrl committed 2026-10-20 18:40:50 +02:00
1 parent 8ffad6350f
commit fc2be66b0c
17 files changed
+887 -31

No files matched your search

+7
View File
@@ -24,6 +24,9 @@ pub struct Config {
pub vault_idle: std::time::Duration,
/// How many days a deleted node waits in the trash before it is purged.
pub trash_days: i64,
/// How many days a document's earlier versions are kept, past the newest
/// few that always are.
pub revision_days: i64,
pub backup: BackupConfig,
pub mail: MailConfig,
pub sftp: SftpConfig,
@@ -44,6 +47,7 @@ impl std::fmt::Debug for Config {
.field("secure_cookie", &self.secure_cookie)
.field("vault_idle", &self.vault_idle)
.field("trash_days", &self.trash_days)
.field("revision_days", &self.revision_days)
.field("backup", &self.backup)
.field("mail", &self.mail)
.field("sftp", &self.sftp)
@@ -293,6 +297,7 @@ impl Config {
// A day at least, so an undo is never raced by the sweep; ten years
// at most, as for the sessions.
let trash_days = number("TRASH_DAYS", 30, 1..=3650)? as i64;
let revision_days = number("REVISION_DAYS", 90, 1..=3650)? as i64;
let secure_cookie = flag("SECURE_COOKIE")?;
Ok(Self {
@@ -306,6 +311,7 @@ impl Config {
secure_cookie,
vault_idle,
trash_days,
revision_days,
backup: BackupConfig::from_env()?,
mail: MailConfig::from_env()?,
sftp: SftpConfig::from_env()?,
@@ -344,6 +350,7 @@ impl Config {
secure_cookie: false,
vault_idle: std::time::Duration::from_secs(60),
trash_days: 30,
revision_days: 90,
backup: BackupConfig {
device_name: "narl-diary".into(),
interval: None,
+5 -1
View File
@@ -120,7 +120,11 @@ async fn main() -> Result<()> {
let notify = notify::Notifier::new(db.clone(), Arc::clone(&config))?;
notify::spawn(Arc::clone(&notify));
sweep::spawn(db.clone(), config.uploads_dir(), config.trash_days);
sweep::spawn(
db.clone(),
config.uploads_dir(),
sweep::Keep::from(&*config),
);
let epoch = auth::Epoch::load(&db, &config).await?;
let sessions = Arc::new(auth::Sessions::load(&db).await?);
+3
View File
@@ -6,6 +6,7 @@ pub mod inspect;
pub mod manifest;
pub mod media;
pub mod nodes;
pub mod revisions;
pub mod seal;
pub mod session;
pub mod share;
@@ -69,6 +70,8 @@ pub fn api_router(max_upload: usize) -> Router<AppState> {
.route("/nodes/{id}", get(nodes::get))
.route("/nodes/{id}", put(nodes::update))
.route("/nodes/{id}", delete(nodes::remove))
.route("/nodes/{id}/revisions", get(revisions::list))
.route("/nodes/{id}/revisions/{rev}", get(revisions::get))
.route("/nodes/move", post(nodes::move_many))
.route("/nodes/{id}/move", post(nodes::move_node))
.route("/import", post(import::import_root).layer(uploads))
+13 -1
View File
@@ -10,7 +10,7 @@ use sqlx::{Row, SqliteConnection, SqlitePool};
use crate::{
auth::Session,
error::{AppError, AppResult},
routes::{media, trash},
routes::{media, revisions, trash},
state::AppState,
};
@@ -162,6 +162,10 @@ pub struct UpdateInput {
/// write goes through whatever happened in between.
#[serde(default)]
pub base_updated_at: Option<i64>,
/// The text sent is a revision brought back: what it replaces is kept
/// as a revision of its own, however recently the last one was.
#[serde(default)]
pub restore: bool,
}
#[derive(Deserialize)]
@@ -702,6 +706,14 @@ pub async fn update(
} else {
input.body.as_str()
};
let reason = if input.restore {
revisions::Reason::Restore
} else if input.base_updated_at.is_none() {
revisions::Reason::Force
} else {
revisions::Reason::Edit
};
revisions::keep(&mut tx, id, body, reason).await?;
if kind == FILE {
sqlx::query(
"UPDATE media SET filename = ?1 WHERE id = (SELECT media_id FROM nodes WHERE id = ?2)",
+342
View File
@@ -0,0 +1,342 @@
//! Revisions: what a document said before it was written over, so a `:w!`
//! that should not have been, a save over the mount, or a paragraph deleted
//! last week can be taken back.
//!
//! A revision is kept by the write that replaces the text, in the same
//! transaction, so there is never a save without the version it replaced.
//! The sweep forgets them after `WORKSPACE_REVISION_DAYS`, but always keeps
//! the newest few of every document.
use axum::{
extract::{Path, State},
Json,
};
use serde::Serialize;
use sqlx::{Row, SqliteConnection, SqlitePool};
use crate::{
auth::Session,
error::{AppError, AppResult},
routes::trash,
state::AppState,
};
/// Why a revision was kept.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Reason {
/// An ordinary save, autosave included.
Edit,
/// `:w!`: a write that did not check what it wrote over.
Force,
/// A save over the SFTP mount.
Sftp,
/// A revision brought back, replacing what was there.
Restore,
}
impl Reason {
fn as_str(self) -> &'static str {
match self {
Reason::Edit => "edit",
Reason::Force => "force",
Reason::Sftp => "sftp",
Reason::Restore => "restore",
}
}
}
/// Autosave writes every few seconds; an ordinary save keeps a revision only
/// when the newest one is at least this old.
const EDIT_GAP: i64 = 10 * 60;
/// How many of a document's revisions the sweep never forgets, however old.
pub const ALWAYS_KEPT: i64 = 20;
/// Keep what a document says now, before `new_body` replaces it. Called by
/// every write of a document's text, in its transaction. Returns whether a
/// revision was kept.
pub async fn keep(
db: &mut SqliteConnection,
id: i64,
new_body: &str,
reason: Reason,
) -> AppResult<bool> {
let Some(row) = sqlx::query("SELECT kind, name, body FROM nodes WHERE id = ?1")
.bind(id)
.fetch_optional(&mut *db)
.await?
else {
return Ok(false);
};
let body: String = row.get("body");
// Only text is worth keeping: a rename is not a loss, and an empty
// document has nothing to lose.
if row.get::<String, _>("kind") != crate::routes::nodes::DOCUMENT
|| body == new_body
|| body.trim().is_empty()
{
return Ok(false);
}
let now = crate::now();
if reason == Reason::Edit {
let newest: Option<i64> =
sqlx::query_scalar("SELECT MAX(saved_at) FROM revisions WHERE node_id = ?1")
.bind(id)
.fetch_one(&mut *db)
.await?;
if newest.is_some_and(|at| now - at < EDIT_GAP) {
return Ok(false);
}
}
sqlx::query(
"INSERT INTO revisions (node_id, name, body, saved_at, reason)
VALUES (?1, ?2, ?3, ?4, ?5)",
)
.bind(id)
.bind(row.get::<String, _>("name"))
.bind(&body)
.bind(now)
.bind(reason.as_str())
.execute(&mut *db)
.await?;
Ok(true)
}
/// Forget the revisions saved before `cutoff`, except the newest
/// `ALWAYS_KEPT` of each document. Returns how many went.
pub async fn forget_older(db: &SqlitePool, cutoff: i64) -> sqlx::Result<u64> {
let done = sqlx::query(
"DELETE FROM revisions
WHERE saved_at < ?1
AND id NOT IN (
SELECT r.id FROM revisions r
WHERE r.node_id = revisions.node_id
ORDER BY r.saved_at DESC, r.id DESC
LIMIT ?2
)",
)
.bind(cutoff)
.bind(ALWAYS_KEPT)
.execute(db)
.await?;
Ok(done.rows_affected())
}
#[derive(Serialize)]
pub struct RevisionSummary {
pub id: i64,
pub name: String,
pub saved_at: i64,
pub reason: String,
/// Characters of text.
pub size: i64,
}
#[derive(Serialize)]
pub struct Revision {
pub id: i64,
pub name: String,
pub body: String,
pub saved_at: i64,
pub reason: String,
}
/// A document in the trash has no history to read, as it has no text: both
/// come back with it.
async fn readable(state: &AppState, id: i64) -> AppResult<()> {
if trash::in_trash(&mut *state.db.acquire().await?, id).await? {
return Err(AppError::NotFound);
}
Ok(())
}
/// `GET /api/nodes/{id}/revisions` — the newest first.
pub async fn list(
_: Session,
State(state): State<AppState>,
Path(id): Path<i64>,
) -> AppResult<Json<Vec<RevisionSummary>>> {
readable(&state, id).await?;
let rows = sqlx::query(
"SELECT id, name, saved_at, reason, length(body) AS size FROM revisions
WHERE node_id = ?1 ORDER BY saved_at DESC, id DESC",
)
.bind(id)
.fetch_all(&state.db)
.await?;
Ok(Json(
rows.iter()
.map(|row| RevisionSummary {
id: row.get("id"),
name: row.get("name"),
saved_at: row.get("saved_at"),
reason: row.get("reason"),
size: row.get("size"),
})
.collect(),
))
}
/// `GET /api/nodes/{id}/revisions/{rev}` — one, with its text.
pub async fn get(
_: Session,
State(state): State<AppState>,
Path((id, rev)): Path<(i64, i64)>,
) -> AppResult<Json<Revision>> {
readable(&state, id).await?;
let row = sqlx::query(
"SELECT id, name, body, saved_at, reason FROM revisions WHERE id = ?1 AND node_id = ?2",
)
.bind(rev)
.bind(id)
.fetch_optional(&state.db)
.await?
.ok_or(AppError::NotFound)?;
Ok(Json(Revision {
id: row.get("id"),
name: row.get("name"),
body: row.get("body"),
saved_at: row.get("saved_at"),
reason: row.get("reason"),
}))
}
#[cfg(test)]
mod tests {
use super::{forget_older, keep, Reason, ALWAYS_KEPT};
use sqlx::SqlitePool;
async fn db() -> SqlitePool {
let db = sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.unwrap();
sqlx::migrate!("./migrations").run(&db).await.unwrap();
db
}
async fn document(db: &SqlitePool, slug: &str, body: &str) -> i64 {
sqlx::query_scalar(
"INSERT INTO nodes (parent_id, kind, name, slug, body, created_at, updated_at)
VALUES ((SELECT id FROM nodes WHERE slug = 'work'), 'document', ?1, ?1, ?2, 0, 0)
RETURNING id",
)
.bind(slug)
.bind(body)
.fetch_one(db)
.await
.unwrap()
}
async fn write(db: &SqlitePool, id: i64, body: &str, reason: Reason) -> bool {
let mut conn = db.acquire().await.unwrap();
let kept = keep(&mut conn, id, body, reason).await.unwrap();
sqlx::query("UPDATE nodes SET body = ?1 WHERE id = ?2")
.bind(body)
.bind(id)
.execute(&mut *conn)
.await
.unwrap();
kept
}
async fn bodies(db: &SqlitePool, id: i64) -> Vec<String> {
sqlx::query_scalar("SELECT body FROM revisions WHERE node_id = ?1 ORDER BY id")
.bind(id)
.fetch_all(db)
.await
.unwrap()
}
/// Autosave keeps one revision per ten minutes; `:w!` and a save over
/// the mount always keep one; nothing is kept of an unchanged text or
/// of an empty one.
#[tokio::test]
async fn what_a_write_keeps() {
let db = db().await;
let id = document(&db, "plan", "").await;
assert!(!write(&db, id, "one", Reason::Edit).await);
assert!(write(&db, id, "two", Reason::Edit).await);
assert!(!write(&db, id, "three", Reason::Edit).await);
assert!(!write(&db, id, "three", Reason::Force).await);
assert!(write(&db, id, "four", Reason::Force).await);
assert!(write(&db, id, "five", Reason::Sftp).await);
assert_eq!(bodies(&db, id).await, ["one", "three", "four"]);
sqlx::query("UPDATE revisions SET saved_at = saved_at - 3600")
.execute(&db)
.await
.unwrap();
assert!(write(&db, id, "six", Reason::Edit).await);
}
/// Through the API: a save that checked what it wrote over keeps one
/// revision per ten minutes, `:w!` and a restore always keep one.
#[tokio::test]
async fn saves_keep_what_they_replace() {
use axum::extract::{Path, State};
let state = crate::routes::tests::state_in(".".into()).await;
let id = document(&state.db, "plan", "first").await;
let save = |body: &'static str, base: bool, restore: bool| {
let state = state.clone();
async move {
let input = serde_json::from_value(serde_json::json!({
"name": "plan",
"body": body,
"base_updated_at": base.then_some(i64::MAX),
"restore": restore,
}))
.unwrap();
let _saved = crate::routes::nodes::update(
crate::auth::Session { id: "test".into() },
State(state),
Path(id),
axum::Json(input),
)
.await
.unwrap();
}
};
save("second", true, false).await;
save("third", true, false).await;
save("fourth", false, false).await;
save("first", true, true).await;
let kept: Vec<(String, String)> =
sqlx::query_as("SELECT body, reason FROM revisions WHERE node_id = ?1 ORDER BY id")
.bind(id)
.fetch_all(&state.db)
.await
.unwrap();
let kept: Vec<(&str, &str)> = kept.iter().map(|(b, r)| (b.as_str(), r.as_str())).collect();
assert_eq!(
kept,
[("first", "edit"), ("third", "force"), ("fourth", "restore")]
);
}
/// Old revisions go, except the newest few of every document.
#[tokio::test]
async fn the_newest_stay_however_old() {
let db = db().await;
let busy = document(&db, "busy", "start").await;
let quiet = document(&db, "quiet", "start").await;
for n in 0..ALWAYS_KEPT + 5 {
write(&db, busy, &format!("busy {n}"), Reason::Force).await;
}
write(&db, quiet, "only", Reason::Force).await;
sqlx::query("UPDATE revisions SET saved_at = id")
.execute(&db)
.await
.unwrap();
assert_eq!(forget_older(&db, i64::MAX).await.unwrap(), 5);
assert_eq!(bodies(&db, busy).await.len() as i64, ALWAYS_KEPT);
assert_eq!(bodies(&db, busy).await[0], "busy 4");
assert_eq!(bodies(&db, quiet).await, ["start"]);
}
}
+21 -3
View File
@@ -35,7 +35,7 @@ use crate::{
Backup,
},
error::{AppError, AppResult},
routes::{media, nodes, seal, trash},
routes::{media, nodes, revisions, seal, trash},
state::AppState,
vault,
};
@@ -757,6 +757,14 @@ impl SpaceFs {
let (title, body) = split_heading(&text);
let now = crate::now();
self.own_writes.insert(id, now);
// One transaction, so the text this replaces is kept as a revision
// exactly when it is replaced.
let mut tx = crate::db::begin_write(&self.db)
.await
.map_err(|_| StatusCode::Failure)?;
revisions::keep(&mut tx, id, &body, revisions::Reason::Sftp)
.await
.map_err(|_| StatusCode::Failure)?;
sqlx::query(
"UPDATE nodes SET name = COALESCE(?1, name), body = ?2, updated_at = ?3 WHERE id = ?4",
)
@@ -764,12 +772,13 @@ impl SpaceFs {
.bind(&body)
.bind(now)
.bind(id)
.execute(&self.db)
.execute(&mut *tx)
.await
.map_err(|_| StatusCode::Failure)?;
media::link_to_node(&mut *self.conn().await?, id, &body)
media::link_to_node(&mut tx, id, &body)
.await
.map_err(|_| StatusCode::Failure)?;
tx.commit().await.map_err(|_| StatusCode::Failure)?;
self.changed();
Ok(())
}
@@ -2483,6 +2492,15 @@ mod tests {
(id, "Renamed", "v2\n")
);
assert!(fs.scratch.is_empty());
// The save over the mount kept what it replaced.
let kept: Vec<(String, String)> =
sqlx::query_as("SELECT body, reason FROM revisions WHERE node_id = ?1 ORDER BY id")
.bind(id)
.fetch_all(&fs.db)
.await
.expect("revisions");
assert_eq!(kept, [("v1\n".to_string(), "sftp".to_string())]);
}
#[tokio::test]
+75 -20
View File
@@ -1,7 +1,8 @@
//! Housekeeping for `uploads/`: what a crash, a killed upload or a forgotten
//! embed leaves behind. Nothing else ever deletes these, so without this they
//! would only grow — on the disk and in every backup of it. The trash is
//! emptied here too, of what has waited in it long enough.
//! emptied here too, of what has waited in it long enough, and the revisions
//! of documents are thinned out.
use std::{collections::HashSet, path::Path, time::Duration};
@@ -19,22 +20,39 @@ const UNREFERENCED_DAYS: i64 = 7;
const EVERY: Duration = Duration::from_secs(24 * 60 * 60);
pub fn spawn(db: SqlitePool, uploads: std::path::PathBuf, trash_days: i64) {
/// How long what the sweep thins out is kept, in days.
#[derive(Clone, Copy)]
pub struct Keep {
pub trash_days: i64,
pub revision_days: i64,
}
impl From<&crate::config::Config> for Keep {
fn from(config: &crate::config::Config) -> Self {
Self {
trash_days: config.trash_days,
revision_days: config.revision_days,
}
}
}
pub fn spawn(db: SqlitePool, uploads: std::path::PathBuf, keep: Keep) {
crate::tasks::supervise("the uploads sweep", move || {
let (db, uploads) = (db.clone(), uploads.clone());
sweep_forever(db, uploads, trash_days)
sweep_forever(db, uploads, keep)
});
}
async fn sweep_forever(db: SqlitePool, uploads: std::path::PathBuf, trash_days: i64) {
async fn sweep_forever(db: SqlitePool, uploads: std::path::PathBuf, keep: Keep) {
{
let mut ticker = tokio::time::interval(EVERY);
loop {
ticker.tick().await;
match run(&db, &uploads, trash_days).await {
match run(&db, &uploads, keep).await {
Ok(swept) if swept.any() => tracing::info!(
"swept: {} out of the trash; uploads: {} partial, {} without a row, {} that nothing embeds",
"swept: {} out of the trash, {} old revisions; uploads: {} partial, {} without a row, {} that nothing embeds",
swept.trashed,
swept.revisions,
swept.partial,
swept.stray,
swept.unreferenced
@@ -50,6 +68,8 @@ async fn sweep_forever(db: SqlitePool, uploads: std::path::PathBuf, trash_days:
pub struct Swept {
/// Things deleted long enough ago to be purged from the trash.
pub trashed: usize,
/// Revisions older than they are kept for.
pub revisions: u64,
pub partial: usize,
pub stray: usize,
pub unreferenced: usize,
@@ -57,19 +77,23 @@ pub struct Swept {
impl Swept {
fn any(&self) -> bool {
self.trashed + self.partial + self.stray + self.unreferenced > 0
self.trashed + self.revisions as usize + self.partial + self.stray + self.unreferenced > 0
}
}
pub async fn run(db: &SqlitePool, uploads: &Path, trash_days: i64) -> anyhow::Result<Swept> {
// First, so the files only the purged nodes held are released and go in
// this same pass.
pub async fn run(db: &SqlitePool, uploads: &Path, keep: Keep) -> anyhow::Result<Swept> {
// First, so the files only the purged nodes and the forgotten revisions
// held are released and go in this same pass.
let trashed =
crate::routes::trash::purge_older(db, uploads, crate::now() - trash_days * 86_400)
crate::routes::trash::purge_older(db, uploads, crate::now() - keep.trash_days * 86_400)
.await
.map_err(|e| anyhow::anyhow!("emptying the trash: {e:?}"))?;
let revisions =
crate::routes::revisions::forget_older(db, crate::now() - keep.revision_days * 86_400)
.await?;
let mut swept = Swept {
trashed,
revisions,
..Swept::default()
};
@@ -89,7 +113,9 @@ pub async fn run(db: &SqlitePool, uploads: &Path, trash_days: i64) -> anyhow::Re
-- Belt and braces: a body or a card that names the file keeps it,
-- whatever the link table says.
AND NOT EXISTS (SELECT 1 FROM nodes n WHERE instr(n.body, m.id) > 0)
AND NOT EXISTS (SELECT 1 FROM cards c WHERE instr(c.body, m.id) > 0)",
AND NOT EXISTS (SELECT 1 FROM cards c WHERE instr(c.body, m.id) > 0)
-- An earlier version of a document brings its pictures back with it.
AND NOT EXISTS (SELECT 1 FROM revisions r WHERE instr(r.body, m.id) > 0)",
)
.bind(cutoff)
.fetch_all(db)
@@ -141,7 +167,7 @@ pub async fn run(db: &SqlitePool, uploads: &Path, trash_days: i64) -> anyhow::Re
#[cfg(test)]
mod tests {
use super::{run, Swept};
use super::{run, Keep, Swept};
#[tokio::test]
async fn only_what_nothing_knows_about_goes() {
@@ -158,7 +184,8 @@ mod tests {
let embedded = "22222222-2222-4222-8222-222222222222";
let forgotten = "33333333-3333-4333-8333-333333333333";
let stray = "44444444-4444-4444-8444-444444444444";
for id in [kept, embedded, forgotten] {
let remembered = "66666666-6666-4666-8666-666666666666";
for id in [kept, embedded, forgotten, remembered] {
sqlx::query("INSERT INTO media (id, filename, mime, size, created_at) VALUES (?1, 'x', 'image/png', 1, ?2)")
.bind(id)
.bind(old)
@@ -187,9 +214,26 @@ mod tests {
.execute(&db)
.await
.unwrap();
// `remembered` only by an earlier version of that document.
sqlx::query(
"INSERT INTO revisions (node_id, name, body, saved_at, reason)
SELECT id, 'a', ?1, 0, 'edit' FROM nodes WHERE slug = 'a'",
)
.bind(format!("![](/api/media/{remembered})"))
.execute(&db)
.await
.unwrap();
let hour_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 60 * 60);
for name in [kept, embedded, forgotten, stray, "x.part", "notes.txt"] {
for name in [
kept,
embedded,
remembered,
forgotten,
stray,
"x.part",
"notes.txt",
] {
let path = dir.path().join(name);
std::fs::write(&path, b"x").unwrap();
std::fs::File::options()
@@ -202,11 +246,21 @@ mod tests {
let fresh = "55555555-5555-4555-8555-555555555555.part";
std::fs::write(dir.path().join(fresh), b"x").unwrap();
let swept = run(&db, dir.path(), 30).await.unwrap();
let swept = run(
&db,
dir.path(),
Keep {
trash_days: 30,
revision_days: 90,
},
)
.await
.unwrap();
assert_eq!(
swept,
Swept {
trashed: 0,
revisions: 0,
partial: 1,
stray: 2,
unreferenced: 1
@@ -217,10 +271,11 @@ mod tests {
.unwrap()
.map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
.collect();
let expected: std::collections::BTreeSet<String> = [kept, embedded, "notes.txt", fresh]
.iter()
.map(|s| s.to_string())
.collect();
let expected: std::collections::BTreeSet<String> =
[kept, embedded, remembered, "notes.txt", fresh]
.iter()
.map(|s| s.to_string())
.collect();
assert_eq!(left, expected);
}
}