From f0135d34ac08f549495e092e83b6a2e3b2831d5f Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Sat, 5 Sep 2026 09:45:44 +0200 Subject: [PATCH] feat: :export! writes the whole diary out as a zip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was no way to get everything out — only one entry at a time, by hand. A diary you cannot leave is not a diary you can trust. The archive is entries/ as one markdown file per entry named by its day, media/ with every embedded file under its real name, and a README. Media links are rewritten to relative paths, so the entries render in any markdown reader with no help from this application. Names are restricted to characters that need no escaping inside a markdown link. It is built on the blocking pool into a temporary file and streamed, so a diary with photographs in it never has to fit in memory. --- Cargo.lock | 89 +++++++++++++++++ Cargo.toml | 2 + src/routes/export.rs | 211 ++++++++++++++++++++++++++++++++++++++++ src/routes/mod.rs | 2 + web/src/lib/commands.ts | 10 ++ 5 files changed, 314 insertions(+) create mode 100644 src/routes/export.rs diff --git a/Cargo.lock b/Cargo.lock index dcb282d..d8b99b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "async-compression" version = "0.4.44" @@ -316,6 +325,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -432,6 +452,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.12" @@ -907,6 +933,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.3" @@ -1031,6 +1063,7 @@ dependencies = [ "sha2 0.10.9", "sqlx", "subtle", + "tempfile", "tokio", "tokio-util", "tower", @@ -1038,6 +1071,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "zip", ] [[package]] @@ -1375,6 +1409,19 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -1842,6 +1889,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.20" @@ -2420,6 +2480,23 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.7" @@ -2431,3 +2508,15 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 8132b03..8cb9249 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ mime_guess = "2" rust-embed = { version = "8", features = ["interpolate-folder-path"] } regex = "1" anyhow = "1" +zip = { version = "2", default-features = false, features = ["deflate"] } +tempfile = "3.27.0" [profile.release] opt-level = 3 diff --git a/src/routes/export.rs b/src/routes/export.rs new file mode 100644 index 0000000..6b94dc8 --- /dev/null +++ b/src/routes/export.rs @@ -0,0 +1,211 @@ +use std::io::{Seek, SeekFrom, Write}; + +use axum::{ + body::Body, + extract::State, + http::header, + response::{IntoResponse, Response}, +}; +use sqlx::Row; +use tokio_util::io::ReaderStream; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; + +use crate::{auth::Session, error::AppResult, state::AppState}; + +/// One entry, flattened out of the database ahead of the blocking zip work. +struct ExportEntry { + id: i64, + title: String, + body: String, + created_at: i64, +} + +/// `2026-09-05-first-light-12.md`. The id is always on the end, so two entries +/// written on the same day under the same title cannot collide. +fn entry_filename(entry: &ExportEntry) -> String { + let slug: String = entry + .title + .to_lowercase() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + let slug = slug.trim_matches('-').split('-').filter(|s| !s.is_empty()).collect::>().join("-"); + let slug: String = slug.chars().take(60).collect(); + let day = day_string(entry.created_at); + if slug.is_empty() { + format!("{day}-entry-{}.md", entry.id) + } else { + format!("{day}-{slug}-{}.md", entry.id) + } +} + +/// `YYYY-MM-DD` in UTC, from a Unix timestamp, without pulling in a date crate. +fn day_string(seconds: i64) -> String { + let days = seconds.div_euclid(86_400); + // Civil-from-days, Howard Hinnant's algorithm. + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!("{y:04}-{m:02}-{d:02}") +} + +/// Uploads are stored under a uuid; the archive gives them their name back. +/// Only characters that are safe in a path *and* need no escaping inside a +/// markdown link survive, because the entries link to these names directly. +fn media_filename(id: &str, original: &str) -> String { + let safe: String = original + .chars() + .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_') { c } else { '_' }) + .collect(); + let safe = safe.trim_matches(['.', '_']).to_string(); + if safe.is_empty() { + id.to_string() + } else { + format!("{id}-{safe}") + } +} + +const README: &str = "\ +This is a complete copy of a ~/diary. + + entries/ one markdown file per entry, named by the day it is about + media/ every file the entries embed + +Embedded media is linked relatively, so the entries render correctly in any +markdown reader as long as the two folders stay next to each other. Nothing +here needs the diary application to read it. +"; + +/// Everything, as one zip. A diary you cannot get out of is not a diary you +/// can trust, so this is deliberately a plain archive of plain files. +pub async fn export(_: Session, State(state): State) -> AppResult { + let entries: Vec = sqlx::query( + "SELECT id, title, body, created_at FROM entries ORDER BY created_at, id", + ) + .fetch_all(&state.db) + .await? + .into_iter() + .map(|row| ExportEntry { + id: row.get("id"), + title: row.get("title"), + body: row.get("body"), + created_at: row.get("created_at"), + }) + .collect(); + + let media: Vec<(String, String)> = sqlx::query("SELECT id, filename FROM media") + .fetch_all(&state.db) + .await? + .into_iter() + .map(|row| (row.get("id"), row.get("filename"))) + .collect(); + + let uploads = state.config.uploads_dir(); + + // Zip writing is synchronous and reads every upload off disk, so it happens + // on the blocking pool, into a temporary file rather than into memory: a + // diary with photographs in it does not fit comfortably in a Vec. + let file = tokio::task::spawn_blocking(move || -> anyhow::Result { + let mut zip = ZipWriter::new(tempfile::tempfile()?); + let text = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + // Media is overwhelmingly already-compressed; deflating it again costs + // a lot of CPU to save nothing. + let blob = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + zip.start_file("README.txt", text)?; + zip.write_all(README.as_bytes())?; + + let names: Vec<(String, String)> = media + .iter() + .map(|(id, filename)| (id.clone(), media_filename(id, filename))) + .collect(); + + for entry in &entries { + let mut body = entry.body.clone(); + for (id, name) in &names { + body = body.replace(&format!("/api/media/{id}"), &format!("../media/{name}")); + } + let document = if entry.title.trim().is_empty() { + body + } else { + format!("# {}\n\n{body}", entry.title.trim()) + }; + + zip.start_file(format!("entries/{}", entry_filename(entry)), text)?; + zip.write_all(document.as_bytes())?; + } + + for (id, name) in &names { + // A row whose file has gone missing must not fail the whole backup. + let Ok(bytes) = std::fs::read(uploads.join(id)) else { + continue; + }; + zip.start_file(format!("media/{name}"), blob)?; + zip.write_all(&bytes)?; + } + + let mut file = zip.finish()?; + file.seek(SeekFrom::Start(0))?; + Ok(file) + }) + .await??; + + let size = file.metadata()?.len(); + let name = format!("diary-{}.zip", day_string(crate::now())); + + Ok(( + [ + (header::CONTENT_TYPE, "application/zip".to_string()), + (header::CONTENT_LENGTH, size.to_string()), + ( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{name}\""), + ), + (header::CACHE_CONTROL, "no-store".to_string()), + ], + Body::from_stream(ReaderStream::new(tokio::fs::File::from_std(file))), + ) + .into_response()) +} + +#[cfg(test)] +mod tests { + use super::{day_string, entry_filename, media_filename, ExportEntry}; + + #[test] + fn renders_days_without_a_date_crate() { + assert_eq!(day_string(0), "1970-01-01"); + assert_eq!(day_string(1_757_030_400), "2025-09-05"); + assert_eq!(day_string(951_782_400), "2000-02-29"); // a leap day the 100-year rule skips + assert_eq!(day_string(-1), "1969-12-31"); + } + + fn entry(title: &str, id: i64) -> ExportEntry { + ExportEntry { id, title: title.into(), body: String::new(), created_at: 0 } + } + + #[test] + fn entry_names_are_readable_and_unique() { + assert_eq!(entry_filename(&entry("First Light", 12)), "1970-01-01-first-light-12.md"); + assert_eq!(entry_filename(&entry("", 3)), "1970-01-01-entry-3.md"); + assert_eq!(entry_filename(&entry("!!! ???", 4)), "1970-01-01-entry-4.md"); + // Same day, same title, different entries. + assert_ne!(entry_filename(&entry("a", 1)), entry_filename(&entry("a", 2))); + } + + #[test] + fn media_names_cannot_escape_their_folder() { + assert_eq!(media_filename("abc", "../../etc/passwd"), "abc-etc_passwd"); + // Nothing that would need escaping in the markdown link that points at it. + assert_eq!(media_filename("abc", "my photo (1).png"), "abc-my_photo__1_.png"); + assert_eq!(media_filename("abc", "photo.jpg"), "abc-photo.jpg"); + assert_eq!(media_filename("abc", "..."), "abc"); + } +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index eeea88b..f0b8306 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -1,4 +1,5 @@ pub mod entries; +pub mod export; pub mod media; pub mod session; pub mod share; @@ -16,6 +17,7 @@ pub fn api_router() -> Router { .route("/login", post(session::login)) .route("/logout", post(session::logout)) .route("/entries", get(entries::list).post(entries::create)) + .route("/export", get(export::export)) .route("/entries/{id}", get(entries::get)) .route("/entries/{id}", put(entries::update)) .route("/entries/{id}", delete(entries::remove)) diff --git a/web/src/lib/commands.ts b/web/src/lib/commands.ts index ea2e011..a25c56d 100644 --- a/web/src/lib/commands.ts +++ b/web/src/lib/commands.ts @@ -312,6 +312,16 @@ export const commands: CommandSpec[] = [ URL.revokeObjectURL(url) diary.say(`wrote ${anchor.download}`) }, + bang: { + help: 'download the whole diary — every entry and every file — as a zip', + run: async () => { + // A plain navigation, so the browser streams the archive straight to + // disk instead of the tab holding all of it in memory first. + await diary.flush() + diary.say('building the archive — the download starts on its own') + location.href = '/api/export' + }, + }, }, { name: 'upload',