From f42c0c6e4b00be6227b8dec74693481910d33a29 Mon Sep 17 00:00:00 2001 From: Nils Pukropp Date: Sat, 5 Sep 2026 09:07:28 +0200 Subject: [PATCH] perf: build entry excerpts in SQL, not in Rust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list query selected every entry's full body so that excerpt() could keep the first 180 characters of it. Listing a few years of diary moved megabytes over the wire to render a sidebar, on every keystroke of an incremental search. It now reads the leading 1024 characters, which is more than an excerpt can use, and infers truncation from the length of what came back. Also pairs WAL with synchronous=NORMAL and an explicit busy_timeout — the usual settings for it, and neither was being set. --- src/main.rs | 9 +++++- src/routes/entries.rs | 74 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/src/main.rs b/src/main.rs index cd178e3..1a10300 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,9 @@ use std::sync::Arc; use anyhow::{Context, Result}; use axum::Router; -use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; +use sqlx::sqlite::{ + SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions, SqliteSynchronous, +}; use axum::http::{header, HeaderValue}; use tower_http::{ compression::CompressionLayer, set_header::SetResponseHeaderLayer, trace::TraceLayer, @@ -47,6 +49,11 @@ async fn main() -> Result<()> { .filename(config.data_dir.join("diary.db")) .create_if_missing(true) .journal_mode(SqliteJournalMode::Wal) + // The standard pairing for WAL: fsync at checkpoints rather + // than on every commit. A crash can cost the last transaction, + // never the database. + .synchronous(SqliteSynchronous::Normal) + .busy_timeout(std::time::Duration::from_secs(10)) .foreign_keys(true), ) .await diff --git a/src/routes/entries.rs b/src/routes/entries.rs index 7aff5e4..23d6e7c 100644 --- a/src/routes/entries.rs +++ b/src/routes/entries.rs @@ -55,17 +55,24 @@ pub struct EntryInput { pub created_at: Option, } -/// A short, plain-ish preview of the body for the entry list. -fn excerpt(body: &str) -> String { - let flat: String = body +/// How much of a body the list query reads. Comfortably more than an excerpt +/// needs, and a rounding error next to entries that run to thousands of words. +const HEAD_CHARS: usize = 1024; +const EXCERPT_CHARS: usize = 180; + +/// A short, plain-ish preview of the body for the entry list. `head` is only +/// the leading `HEAD_CHARS` of the entry, so truncation is inferred from its +/// length rather than measured against the whole body. +fn excerpt(head: &str) -> String { + let flat: String = head .lines() .map(str::trim) .filter(|line| !line.is_empty() && !line.starts_with("![") && !line.starts_with("```")) .map(|line| line.trim_start_matches(['#', '>', '-', '*', ' '])) .collect::>() .join(" "); - let mut out: String = flat.chars().take(180).collect(); - if flat.chars().count() > 180 { + let mut out: String = flat.chars().take(EXCERPT_CHARS).collect(); + if flat.chars().count() > EXCERPT_CHARS || head.chars().count() >= HEAD_CHARS { out.push('…'); } out @@ -99,14 +106,15 @@ pub async fn list( let rows = match search.and_then(fts_query) { Some(query) => { - sqlx::query( - "SELECT e.id, e.title, e.body, e.created_at, e.updated_at, e.share_token + sqlx::query(&format!( + "SELECT e.id, e.title, substr(e.body, 1, {HEAD_CHARS}) AS body, + e.created_at, e.updated_at, e.share_token FROM entries_fts JOIN entries e ON e.id = entries_fts.rowid WHERE entries_fts MATCH ?1 ORDER BY rank - LIMIT ?2 OFFSET ?3", - ) + LIMIT ?2 OFFSET ?3" + )) .bind(query) .bind(limit) .bind(offset) @@ -114,12 +122,13 @@ pub async fn list( .await? } None => { - sqlx::query( - "SELECT id, title, body, created_at, updated_at, share_token + sqlx::query(&format!( + "SELECT id, title, substr(body, 1, {HEAD_CHARS}) AS body, + created_at, updated_at, share_token FROM entries ORDER BY created_at DESC, id DESC - LIMIT ?1 OFFSET ?2", - ) + LIMIT ?1 OFFSET ?2" + )) .bind(limit) .bind(offset) .fetch_all(&state.db) @@ -130,12 +139,12 @@ pub async fn list( let entries = rows .into_iter() .map(|row| { - let body: String = row.get("body"); + let head: String = row.get("body"); let share_token: Option = row.get("share_token"); EntrySummary { id: row.get("id"), title: row.get("title"), - excerpt: excerpt(&body), + excerpt: excerpt(&head), created_at: row.get("created_at"), updated_at: row.get("updated_at"), shared: share_token.is_some(), @@ -253,3 +262,38 @@ async fn load(state: &AppState, id: i64) -> AppResult { share_token, }) } + +#[cfg(test)] +mod tests { + use super::{excerpt, fts_query, EXCERPT_CHARS, HEAD_CHARS}; + + #[test] + fn strips_markdown_furniture() { + let body = "# Title\n\n![pic](/api/media/x)\n\n> a quote\n- a point"; + assert_eq!(excerpt(body), "Title a quote a point"); + } + + #[test] + fn marks_a_body_that_was_cut_short() { + assert!(!excerpt("short").ends_with('…')); + assert!(excerpt(&"word ".repeat(60)).ends_with('…')); + // A head at the read limit is truncated even when it flattens small. + assert!(excerpt(&"![x](/api/media/y)\n".repeat(HEAD_CHARS / 18)).ends_with('…')); + } + + #[test] + fn excerpt_never_exceeds_its_budget() { + let out = excerpt(&"a".repeat(HEAD_CHARS)); + assert_eq!(out.chars().count(), EXCERPT_CHARS + 1); + } + + #[test] + fn fts_query_discards_operators() { + // Anything that could be read as FTS syntax must not survive as syntax. + assert_eq!(fts_query("hello"), Some("\"hello\"*".into())); + assert_eq!(fts_query("a OR b"), Some("\"a\"* AND \"OR\"* AND \"b\"*".into())); + assert_eq!(fts_query("foo\"NEAR/2\"bar"), Some("\"foo\"* AND \"NEAR\"* AND \"2\"* AND \"bar\"*".into())); + assert_eq!(fts_query(" "), None); + assert_eq!(fts_query("***"), None); + } +}