diff --git a/README.md b/README.md index 4dd9eba..3c97c23 100644 --- a/README.md +++ b/README.md @@ -507,7 +507,7 @@ limit at least that large too. of every file that is text — by its type, or by its first 8 KiB reading as UTF-8 without a NUL — is indexed, up to its first 256 KiB, the same part the viewer shows. A file found that way lists the words around the match under its -name. The text is read when a file arrives, by upload, import or the SFTP +name, the words that matched lit. The text is read when a file arrives, by upload, import or the SFTP mount, again when its bytes change, and at start for any file still without it. PDFs, office documents and images are not read. @@ -848,7 +848,7 @@ sealed or not. | `GET` | `/api/sessions` | every signed-in browser, the one seen last first | | `DELETE` | `/api/sessions/{id}` | sign another browser out, and lock the vault | | `GET`/`POST` | `/api/spaces` | the spaces / create one | -| `GET`/`POST` | `/api/nodes` | children (`?parent=`) or search (`?q=`, `?space=`; a file matched by its text has `in_text` and the words around the match as `excerpt`) / create, from a template with `template_id` | +| `GET`/`POST` | `/api/nodes` | children (`?parent=`) or search (`?q=`, `?space=`; a file matched by its text has `in_text` and the words around the match as `excerpt`, each matched word between U+0002 and U+0003) / create, from a template with `template_id` | | `GET`/`PUT`/`DELETE` | `/api/nodes/{id}` | read / update only the fields sent — `name`, `body`, `created_at`, `has_board`; with `base_updated_at`, refused with `409` once it was saved since / delete into the trash, subtree and all — or for good, for something already in it | | `GET`/`DELETE` | `/api/trash` | what was deleted, the last first / empty it | | `DELETE` | `/api/trash/{id}` | delete one thing in the trash for good | diff --git a/docs/images/agenda.png b/docs/images/agenda.png index 99680de..b3cf591 100644 Binary files a/docs/images/agenda.png and b/docs/images/agenda.png differ diff --git a/docs/images/board.png b/docs/images/board.png index 6437be1..64bd84a 100644 Binary files a/docs/images/board.png and b/docs/images/board.png differ diff --git a/docs/images/calendar.png b/docs/images/calendar.png index 8bd439d..2222752 100644 Binary files a/docs/images/calendar.png and b/docs/images/calendar.png differ diff --git a/docs/images/document.png b/docs/images/document.png index 72d1ce2..ebdc38d 100644 Binary files a/docs/images/document.png and b/docs/images/document.png differ diff --git a/docs/images/editor.png b/docs/images/editor.png index 2d4fc07..9e4c6fa 100644 Binary files a/docs/images/editor.png and b/docs/images/editor.png differ diff --git a/docs/images/file-certificate.png b/docs/images/file-certificate.png index cd8b8a3..712c231 100644 Binary files a/docs/images/file-certificate.png and b/docs/images/file-certificate.png differ diff --git a/docs/images/file-table.png b/docs/images/file-table.png index c3841ec..cce4063 100644 Binary files a/docs/images/file-table.png and b/docs/images/file-table.png differ diff --git a/docs/images/help.png b/docs/images/help.png index 24d0ac2..b079701 100644 Binary files a/docs/images/help.png and b/docs/images/help.png differ diff --git a/docs/images/history.png b/docs/images/history.png index d1e4816..6060872 100644 Binary files a/docs/images/history.png and b/docs/images/history.png differ diff --git a/docs/images/login.png b/docs/images/login.png index 37a42d3..f2f1743 100644 Binary files a/docs/images/login.png and b/docs/images/login.png differ diff --git a/docs/images/math.png b/docs/images/math.png index f6b84bd..ec3ea04 100644 Binary files a/docs/images/math.png and b/docs/images/math.png differ diff --git a/docs/images/mobile-document.png b/docs/images/mobile-document.png index 6739841..c39ebe4 100644 Binary files a/docs/images/mobile-document.png and b/docs/images/mobile-document.png differ diff --git a/docs/images/mobile-tree.png b/docs/images/mobile-tree.png index 1244cdb..4d886ab 100644 Binary files a/docs/images/mobile-tree.png and b/docs/images/mobile-tree.png differ diff --git a/docs/images/overview.png b/docs/images/overview.png index 6330f8c..e48e6b5 100644 Binary files a/docs/images/overview.png and b/docs/images/overview.png differ diff --git a/docs/images/search.png b/docs/images/search.png index aa851d9..39d997f 100644 Binary files a/docs/images/search.png and b/docs/images/search.png differ diff --git a/docs/images/tiles.png b/docs/images/tiles.png index 9a0c5f4..f37b61f 100644 Binary files a/docs/images/tiles.png and b/docs/images/tiles.png differ diff --git a/docs/wiki/API.md b/docs/wiki/API.md index b1baa07..91b4d78 100644 --- a/docs/wiki/API.md +++ b/docs/wiki/API.md @@ -14,7 +14,7 @@ sealed or not. | `GET` | `/api/sessions` | every signed-in browser, the one seen last first | | `DELETE` | `/api/sessions/{id}` | sign another browser out, and lock the vault | | `GET`/`POST` | `/api/spaces` | the spaces / create one | -| `GET`/`POST` | `/api/nodes` | children (`?parent=`) or search (`?q=`, `?space=`; a file matched by its text has `in_text` and the words around the match as `excerpt`) / create, from a template with `template_id` | +| `GET`/`POST` | `/api/nodes` | children (`?parent=`) or search (`?q=`, `?space=`; a file matched by its text has `in_text` and the words around the match as `excerpt`, each matched word between U+0002 and U+0003) / create, from a template with `template_id` | | `GET`/`PUT`/`DELETE` | `/api/nodes/{id}` | read / update only the fields sent — `name`, `body`, `created_at`, `has_board`; with `base_updated_at`, refused with `409` once it was saved since / delete into the trash, subtree and all — or for good, for something already in it | | `GET`/`DELETE` | `/api/trash` | what was deleted, the last first / empty it | | `DELETE` | `/api/trash/{id}` | delete one thing in the trash for good | diff --git a/docs/wiki/Files-and-Search.md b/docs/wiki/Files-and-Search.md index c6fc222..25d6421 100644 --- a/docs/wiki/Files-and-Search.md +++ b/docs/wiki/Files-and-Search.md @@ -68,7 +68,7 @@ A search reads the text files in the tree as well as the documents. A file counts as text by its type, or when its first 8 KiB read as UTF-8 with no NUL byte, which covers `.log` files and Makefiles. The first 256 KiB of each is indexed, the same part the viewer shows. A file found by its text shows the -words around the match under its name. +words around the match under its name, with the words that matched lit. The text is read when a file arrives, by upload, import or the SFTP mount, again when its bytes change, and at start for any file still missing it. PDFs, diff --git a/src/main.rs b/src/main.rs index de7f14e..402b3ec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -155,7 +155,7 @@ async fn main() -> Result<()> { // first start, so the server does not wait for it. let (db, uploads) = (state.db.clone(), config.uploads_dir()); tokio::spawn(async move { - match routes::file_text::catch_up(&db, &uploads).await { + match routes::file_text::catch_up(&db, &uploads, None).await { Ok(0) => {} Ok(read) => tracing::info!("read {read} files for search"), Err(e) => tracing::warn!("could not index files for search: {e:?}"), diff --git a/src/routes/file_text.rs b/src/routes/file_text.rs index d11546b..85ed684 100644 --- a/src/routes/file_text.rs +++ b/src/routes/file_text.rs @@ -3,9 +3,11 @@ //! A file's bytes live in `uploads/`, out of reach of SQLite's full-text //! index, so the text of the ones that are text is copied into `file_text` //! beside them. There is one way in, [`catch_up`]: it reads every file that -//! has no row yet. Whatever puts a file in the tree — an upload, an import, an -//! SFTP save, a decrypt — calls it once its own transaction is done, and the -//! server calls it at start for anything that slipped past. The triggers of +//! has no row yet, in the part of the tree it is pointed at. Whatever puts a +//! file in the tree — an upload, an import, an SFTP save, a decrypt — calls it +//! for the folder it wrote into once its own transaction is done, and the +//! server calls it for the whole tree at start, for anything that slipped +//! past. The triggers of //! 0024 take care of the other direction: new bytes or sealing drop the row, //! and a sealed file never gets one. //! @@ -72,17 +74,28 @@ fn utf8_but_for_a_cut(bytes: &[u8]) -> bool { } } -/// Read every unsealed file in the tree that has no row yet. Returns how many -/// were read. A file whose bytes are missing is skipped and asked about again -/// next time; one that is not text gets an empty row and is not. -pub async fn catch_up(db: &SqlitePool, uploads: &Path) -> AppResult { +/// Read every unsealed file that has no row yet: at or under `under`, or in +/// the whole tree when it is `None`. Returns how many were read. A file whose +/// bytes are missing is skipped and asked about again next time; one that is +/// not text gets an empty row and is not. A write asks only about where it +/// wrote, so an upload into one folder does not walk every file there is. +pub async fn catch_up(db: &SqlitePool, uploads: &Path, under: Option) -> AppResult { let missing = sqlx::query( "SELECT n.id, n.media_id, m.mime FROM nodes n JOIN media m ON m.id = n.media_id LEFT JOIN file_text t ON t.node_id = n.id - WHERE n.kind = 'file' AND m.encrypted = 0 AND t.node_id IS NULL", + WHERE n.kind = 'file' AND m.encrypted = 0 AND t.node_id IS NULL + AND (?1 IS NULL OR n.id IN ( + WITH RECURSIVE sub (id) AS ( + SELECT ?1 + UNION ALL + SELECT c.id FROM nodes c JOIN sub ON c.parent_id = sub.id + ) + SELECT id FROM sub + ))", ) + .bind(under) .fetch_all(db) .await?; @@ -122,8 +135,8 @@ pub async fn catch_up(db: &SqlitePool, uploads: &Path) -> AppResult { /// [`catch_up`] after a write, where a failure is worth a line in the log /// but not the write's answer: the file is there either way, and the next /// start reads it. -pub async fn refresh(db: &SqlitePool, uploads: &Path) { - if let Err(e) = catch_up(db, uploads).await { +pub async fn refresh(db: &SqlitePool, uploads: &Path, under: Option) { + if let Err(e) = catch_up(db, uploads, under).await { tracing::warn!("could not index files for search: {e:?}"); } } @@ -235,13 +248,13 @@ mod tests { .await; assert_eq!( - catch_up(&state.db, &state.config.uploads_dir()) + catch_up(&state.db, &state.config.uploads_dir(), None) .await .unwrap(), 2 ); assert_eq!( - catch_up(&state.db, &state.config.uploads_dir()) + catch_up(&state.db, &state.config.uploads_dir(), None) .await .unwrap(), 0 @@ -250,7 +263,7 @@ mod tests { assert_eq!(hits.len(), 1); let (name, in_text, excerpt) = &hits[0]; assert_eq!((name.as_str(), *in_text), ("notes.txt", true)); - assert!(excerpt.contains("otter sleeps"), "{excerpt}"); + assert!(excerpt.contains("\u{2}otter\u{3} sleeps"), "{excerpt}"); // Found by its name, it is not marked. assert!(!found(&state, "notes").await[0].1); @@ -263,6 +276,39 @@ mod tests { assert!(found(&state, "otter").await.is_empty()); } + /// After a write only the folder written into is read; the rest waits + /// for the pass at start. + #[tokio::test] + async fn a_write_reads_only_where_it_wrote() { + let dir = tempfile::tempdir().unwrap(); + let state = crate::routes::tests::state_in(dir.path().into()).await; + std::fs::create_dir_all(state.config.uploads_dir()).unwrap(); + let notes = file(&state, "notes.txt", b"the otter sleeps", false).await; + let space: i64 = sqlx::query_scalar("SELECT parent_id FROM nodes WHERE id = ?1") + .bind(notes) + .fetch_one(&state.db) + .await + .unwrap(); + let elsewhere: i64 = sqlx::query_scalar( + "SELECT id FROM nodes WHERE parent_id IS NULL AND id <> ?1 AND role IS NULL LIMIT 1", + ) + .bind(space) + .fetch_one(&state.db) + .await + .unwrap(); + let uploads = state.config.uploads_dir(); + + assert_eq!( + catch_up(&state.db, &uploads, Some(elsewhere)) + .await + .unwrap(), + 0 + ); + assert!(found(&state, "otter").await.is_empty()); + assert_eq!(catch_up(&state.db, &uploads, Some(space)).await.unwrap(), 1); + assert_eq!(found(&state, "otter").await.len(), 1); + } + /// A sealed file is never read, a file that gets sealed drops out at /// once, and the trigger refuses its text whatever tries to store it. #[tokio::test] @@ -272,7 +318,7 @@ mod tests { std::fs::create_dir_all(state.config.uploads_dir()).unwrap(); file(&state, "secret.txt", b"badger plans", true).await; let open = file(&state, "open.txt", b"heron plans", false).await; - catch_up(&state.db, &state.config.uploads_dir()) + catch_up(&state.db, &state.config.uploads_dir(), None) .await .unwrap(); assert!(found(&state, "badger").await.is_empty()); @@ -299,7 +345,7 @@ mod tests { .unwrap(); assert!(found(&state, "heron").await.is_empty()); assert_eq!( - catch_up(&state.db, &state.config.uploads_dir()) + catch_up(&state.db, &state.config.uploads_dir(), None) .await .unwrap(), 0 @@ -323,7 +369,7 @@ mod tests { let uploads = state.config.uploads_dir(); std::fs::create_dir_all(&uploads).unwrap(); let id = file(&state, "todo.md", b"buy lentils", false).await; - catch_up(&state.db, &uploads).await.unwrap(); + catch_up(&state.db, &uploads, None).await.unwrap(); std::fs::write(uploads.join("v2"), b"buy chickpeas").unwrap(); sqlx::query( @@ -338,7 +384,7 @@ mod tests { .await .unwrap(); assert!(found(&state, "lentils").await.is_empty()); - catch_up(&state.db, &uploads).await.unwrap(); + catch_up(&state.db, &uploads, None).await.unwrap(); assert_eq!(found(&state, "chickpeas").await.len(), 1); } } diff --git a/src/routes/import.rs b/src/routes/import.rs index e0bef9a..79b2831 100644 --- a/src/routes/import.rs +++ b/src/routes/import.rs @@ -559,7 +559,11 @@ async fn run(state: &AppState, into: Into, multipart: Multipart) -> AppResult Some(node.id), + Into::Root => None, + }; + file_text::refresh(&state.db, &state.config.uploads_dir(), under).await; state.backup.signal(); Ok(report) } diff --git a/src/routes/media.rs b/src/routes/media.rs index 996c122..8c59a17 100644 --- a/src/routes/media.rs +++ b/src/routes/media.rs @@ -331,7 +331,7 @@ pub async fn upload_into( if created.is_empty() { return Err(AppError::BadRequest("no files in upload".into())); } - file_text::refresh(&state.db, &state.config.uploads_dir()).await; + file_text::refresh(&state.db, &state.config.uploads_dir(), Some(parent)).await; state.backup.signal(); let mut out = Vec::with_capacity(created.len()); diff --git a/src/routes/nodes.rs b/src/routes/nodes.rs index 7c04b92..3f2fcc6 100644 --- a/src/routes/nodes.rs +++ b/src/routes/nodes.rs @@ -79,7 +79,9 @@ pub struct NodeSummary { #[serde(skip_serializing_if = "Option::is_none")] pub location: Option, /// Search results only: a file found by what it says rather than by its - /// name, with the words around the match as its excerpt. + /// name, with the words around the match as its excerpt — each word that + /// matched between U+0002 and U+0003, which no text file has a use for, + /// for the client to light. #[serde(skip_serializing_if = "std::ops::Not::not")] pub in_text: bool, } @@ -580,7 +582,7 @@ pub async fn list( hits (id, rank, snippet) AS ( SELECT rowid, rank, NULL FROM nodes_fts WHERE nodes_fts MATCH ?1 UNION ALL - SELECT rowid, rank, snippet(files_fts, 0, '', '', '…', 16) + SELECT rowid, rank, snippet(files_fts, 0, char(2), char(3), '…', 16) FROM files_fts WHERE files_fts MATCH ?1 ), best AS ( diff --git a/src/routes/seal.rs b/src/routes/seal.rs index e556d2a..d6cb01b 100644 --- a/src/routes/seal.rs +++ b/src/routes/seal.rs @@ -250,7 +250,7 @@ async fn reseal(state: &AppState, id: i64, seal: bool) -> AppResult { if files > 0 { // Sealing dropped the text with the old bytes; opened, it is read again. if !seal { - file_text::refresh(&state.db, &state.config.uploads_dir()).await; + file_text::refresh(&state.db, &state.config.uploads_dir(), Some(id)).await; } state.backup.signal(); } diff --git a/src/sftp/fs.rs b/src/sftp/fs.rs index b83244c..ed9a7c0 100644 --- a/src/sftp/fs.rs +++ b/src/sftp/fs.rs @@ -985,7 +985,7 @@ impl SpaceFs { let _ = tokio::fs::remove_file(self.uploads.join(&media_id)).await; return Err(StatusCode::Failure); } - file_text::refresh(&self.db, &self.uploads).await; + file_text::refresh(&self.db, &self.uploads, Some(parent)).await; self.changed(); Ok(()) } diff --git a/web/src/lib/Sidebar.svelte b/web/src/lib/Sidebar.svelte index fb9127a..0ccd810 100644 --- a/web/src/lib/Sidebar.svelte +++ b/web/src/lib/Sidebar.svelte @@ -3,7 +3,7 @@ import { carry, into, release } from './carry' import Icon from './Icon.svelte' import { workspace } from './store.svelte' - import { formatBytes, formatDay, formatTime, formatWeekday, isTouchDevice } from './util' + import { formatBytes, formatDay, formatTime, formatWeekday, hitParts, isTouchDevice } from './util' let list = $state() /** Files are being dragged over the list, which takes them into this folder. */ @@ -130,7 +130,7 @@ {label(node)}{#if isContainer(node)}/{/if} {#if workspace.query && node.location}{node.location}{/if} - {#if workspace.query && node.in_text}{node.excerpt}{/if} + {#if workspace.query && node.in_text}{#each hitParts(node.excerpt) as part, at (at)}{#if part.lit}{part.text}{:else}{part.text}{/if}{/each}{/if} {#if workspace.isMarked(node.id)}m{/if} {#if node.shared}{/if} @@ -270,6 +270,8 @@ color: var(--muted); } .row.active .hit { color: var(--ink); } + .hit mark { background: none; color: var(--light); } + .row.active .hit mark { color: var(--ink); font-weight: 700; } .aside { flex: none; font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; } .badge { flex: none; font-size: 11px; font-weight: 700; color: var(--light-dim); } .shared { diff --git a/web/src/lib/util.ts b/web/src/lib/util.ts index eb0813f..25ecd0f 100644 --- a/web/src/lib/util.ts +++ b/web/src/lib/util.ts @@ -346,3 +346,19 @@ export function hardWrapped(paragraph: string): boolean { const sentences = body.filter((line) => /[.!?:;]$/.test(line)).length return fits && sentences <= Math.floor((body.length - 1) / 3) } + +/** + * A search excerpt cut into what to draw plainly and what to light: the + * server puts each matched word between U+0002 and U+0003. A cut through the + * middle of a match leaves an opening without its close, lit to the end. + */ +export function hitParts(excerpt: string): { text: string; lit: boolean }[] { + const parts: { text: string; lit: boolean }[] = [] + let lit = false + for (const piece of excerpt.split(/([\u0002\u0003])/)) { + if (piece === '\u0002') lit = true + else if (piece === '\u0003') lit = false + else if (piece) parts.push({ text: piece, lit }) + } + return parts +}