mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-04 16:56:59 +01:00
prompt refactor
This commit is contained in:
1 parent
6cba77d3cb
commit
39ab4aae48
14 files changed
+1409
-111
No files matched your search
@@ -101,6 +101,19 @@ pub enum Request {
|
||||
/// Independent of [`Request::Search`] so a front-end can fire both at once and
|
||||
/// render whichever lands first. Replies with [`Response::LocalResults`].
|
||||
SearchLocal { query: String, limit: usize },
|
||||
/// Search both the Drive metadata index and the local-file index in one
|
||||
/// round-trip. This is the preferred prompt API; the two legacy search
|
||||
/// requests remain available for older clients. `limit` applies to each
|
||||
/// source independently. `filters` is applied before limiting, so selecting
|
||||
/// (for example) images cannot hide valid matches below an unfiltered top
|
||||
/// `limit`. Older clients omit it and therefore search both sources and all
|
||||
/// kinds. Replies with [`Response::SearchResultsV2`].
|
||||
SearchV2 {
|
||||
query: String,
|
||||
limit: usize,
|
||||
#[serde(default)]
|
||||
filters: SearchFilters,
|
||||
},
|
||||
/// Rename a file or folder. `path` is mountpoint-relative; `new_name` is a
|
||||
/// single path component (no separators). Replies with [`Response::Ok`].
|
||||
Rename { path: String, new_name: String },
|
||||
@@ -665,6 +678,10 @@ pub struct SearchHit {
|
||||
pub pinned: bool,
|
||||
/// Node uid in `volume~link` form.
|
||||
pub uid: String,
|
||||
/// Relevance assigned by the daemon. Higher scores sort first. Legacy
|
||||
/// daemons omit it, in which case clients retain the daemon's result order.
|
||||
#[serde(default)]
|
||||
pub score: i64,
|
||||
}
|
||||
|
||||
/// One hit in a [`Request::SearchLocal`] result: a file on this machine, outside
|
||||
@@ -679,6 +696,107 @@ pub struct LocalHit {
|
||||
pub size: u64,
|
||||
/// Modification time, epoch seconds.
|
||||
pub modified: i64,
|
||||
/// Relevance assigned by the daemon. Higher scores sort first. Legacy
|
||||
/// daemons omit it, in which case clients retain the daemon's result order.
|
||||
#[serde(default)]
|
||||
pub score: i64,
|
||||
}
|
||||
|
||||
/// A searchable corpus in [`Request::SearchV2`]. Kept explicit on the wire so
|
||||
/// future sources can be added without multiplying request variants.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum SearchSource {
|
||||
Drive,
|
||||
Local,
|
||||
}
|
||||
|
||||
/// Coarse result kind used by the prompt's facets. Classification belongs in
|
||||
/// core/the daemon and must happen before the per-source limit is applied.
|
||||
#[derive(Serialize, Deserialize, Debug, Default, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum SearchKind {
|
||||
#[default]
|
||||
All,
|
||||
Folders,
|
||||
Documents,
|
||||
Images,
|
||||
Media,
|
||||
}
|
||||
|
||||
impl SearchKind {
|
||||
/// Classify a result before the caller applies its final limit. This stays
|
||||
/// beside the wire enum so every front end and both search corpora agree.
|
||||
pub fn accepts(self, name: &str, is_dir: bool) -> bool {
|
||||
if self == Self::All {
|
||||
return true;
|
||||
}
|
||||
if self == Self::Folders {
|
||||
return is_dir;
|
||||
}
|
||||
if is_dir {
|
||||
return false;
|
||||
}
|
||||
let extension = std::path::Path::new(name)
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
match self {
|
||||
Self::Documents => matches!(
|
||||
extension.as_str(),
|
||||
"pdf"
|
||||
| "doc"
|
||||
| "docx"
|
||||
| "odt"
|
||||
| "rtf"
|
||||
| "txt"
|
||||
| "md"
|
||||
| "xls"
|
||||
| "xlsx"
|
||||
| "ods"
|
||||
| "csv"
|
||||
| "ppt"
|
||||
| "pptx"
|
||||
| "odp"
|
||||
| "epub"
|
||||
),
|
||||
Self::Images => matches!(
|
||||
extension.as_str(),
|
||||
"png" | "jpg" | "jpeg" | "webp" | "gif" | "bmp" | "svg" | "avif" | "heic" | "tiff"
|
||||
),
|
||||
Self::Media => {
|
||||
PhotoKind::classify(Some(name), None) == PhotoKind::Video
|
||||
|| matches!(
|
||||
extension.as_str(),
|
||||
"mp3" | "flac" | "wav" | "ogg" | "opus" | "m4a" | "aac"
|
||||
)
|
||||
}
|
||||
Self::All | Self::Folders => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Optional constraints for a unified search. The default deliberately means
|
||||
/// the original SearchV2 behaviour, preserving old JSON requests.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SearchFilters {
|
||||
/// Sources to query. An explicit empty list searches neither source.
|
||||
#[serde(default = "default_search_sources")]
|
||||
pub sources: Vec<SearchSource>,
|
||||
#[serde(default)]
|
||||
pub kind: SearchKind,
|
||||
}
|
||||
|
||||
impl Default for SearchFilters {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
sources: default_search_sources(),
|
||||
kind: SearchKind::All,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_search_sources() -> Vec<SearchSource> {
|
||||
vec![SearchSource::Drive, SearchSource::Local]
|
||||
}
|
||||
|
||||
/// What kind of media a timeline entry is, so the Photos page can split into
|
||||
@@ -947,6 +1065,14 @@ pub enum Response {
|
||||
/// is true while a scan of the machine is still running, so a front-end can
|
||||
/// say "still indexing" instead of "no matches" on a cold first launch.
|
||||
LocalResults { hits: Vec<LocalHit>, indexing: bool },
|
||||
/// Drive and local-file search results returned together (reply to
|
||||
/// [`Request::SearchV2`]). `local_indexing` distinguishes a genuinely empty
|
||||
/// local result set from one observed while the initial scan is running.
|
||||
SearchResultsV2 {
|
||||
drive_hits: Vec<SearchHit>,
|
||||
local_hits: Vec<LocalHit>,
|
||||
local_indexing: bool,
|
||||
},
|
||||
/// A snapshot of what the daemon is working on (reply to
|
||||
/// [`Request::GetQueueStatus`]): `items` are byte-moving transfers, `jobs`
|
||||
/// the longer non-transfer work around them (scans, folder skeletons, the
|
||||
@@ -1104,6 +1230,58 @@ mod tests {
|
||||
assert_eq!(PhotoKind::classify(Some("x.jpg"), Some("video/mp4")), Photo);
|
||||
}
|
||||
|
||||
/// Keep every extension accepted by the shared classifier covered. Drive
|
||||
/// activation uses this classification before it knows whether opening a
|
||||
/// file should stream through FUSE or materialise a complete cache entry;
|
||||
/// silently dropping one of these formats would therefore regress large
|
||||
/// videos back to the expensive whole-file path.
|
||||
#[test]
|
||||
fn photo_kind_recognises_every_supported_video_extension() {
|
||||
for ext in VIDEO_EXTS {
|
||||
let lower = format!("movie.{ext}");
|
||||
let upper = format!("MOVIE.{}", ext.to_ascii_uppercase());
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some(&lower), None),
|
||||
PhotoKind::Video,
|
||||
"lower-case .{ext}"
|
||||
);
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some(&upper), None),
|
||||
PhotoKind::Video,
|
||||
"upper-case .{ext}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn photo_kind_uses_video_mime_only_when_extension_is_not_authoritative() {
|
||||
use PhotoKind::*;
|
||||
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some("extensionless"), Some("video/mp4; codecs=avc1")),
|
||||
Video
|
||||
);
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some("clip.unknown"), Some("video/x-matroska")),
|
||||
Video
|
||||
);
|
||||
// Audio is not a video and must not accidentally take the video
|
||||
// streaming policy merely because both are time-based media.
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some("recording"), Some("audio/flac")),
|
||||
Photo
|
||||
);
|
||||
// A known still/raw suffix remains authoritative over server metadata.
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some("poster.AVIF"), Some("video/mp4")),
|
||||
Photo
|
||||
);
|
||||
assert_eq!(
|
||||
PhotoKind::classify(Some("negative.NEF"), Some("video/mp4")),
|
||||
Raw
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn photo_kind_i64_round_trips() {
|
||||
for k in [PhotoKind::Photo, PhotoKind::Video, PhotoKind::Raw] {
|
||||
@@ -1167,6 +1345,149 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unified_search_request_and_response_roundtrip() {
|
||||
let request = Request::SearchV2 {
|
||||
query: "quarterly report".into(),
|
||||
limit: 37,
|
||||
filters: SearchFilters::default(),
|
||||
};
|
||||
let line = serde_json::to_string(&request).unwrap();
|
||||
assert_eq!(
|
||||
line,
|
||||
r#"{"SearchV2":{"query":"quarterly report","limit":37,"filters":{"sources":["Drive","Local"],"kind":"All"}}}"#
|
||||
);
|
||||
let decoded: Request = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(line, serde_json::to_string(&decoded).unwrap());
|
||||
|
||||
let response = Response::SearchResultsV2 {
|
||||
drive_hits: vec![SearchHit {
|
||||
name: "report.pdf".into(),
|
||||
path: "Work/report.pdf".into(),
|
||||
is_dir: false,
|
||||
size: 12,
|
||||
modified: 123,
|
||||
pinned: true,
|
||||
uid: "vol~link".into(),
|
||||
score: 985,
|
||||
}],
|
||||
local_hits: vec![LocalHit {
|
||||
name: "notes.txt".into(),
|
||||
path: "/home/me/notes.txt".into(),
|
||||
is_dir: false,
|
||||
size: 7,
|
||||
modified: 456,
|
||||
score: 720,
|
||||
}],
|
||||
local_indexing: true,
|
||||
};
|
||||
let line = serde_json::to_string(&response).unwrap();
|
||||
assert!(!line.contains('\n'));
|
||||
let decoded: Response = serde_json::from_str(&line).unwrap();
|
||||
match decoded {
|
||||
Response::SearchResultsV2 {
|
||||
drive_hits,
|
||||
local_hits,
|
||||
local_indexing,
|
||||
} => {
|
||||
assert_eq!(drive_hits.len(), 1);
|
||||
assert_eq!(drive_hits[0].uid, "vol~link");
|
||||
assert_eq!(drive_hits[0].score, 985);
|
||||
assert_eq!(local_hits.len(), 1);
|
||||
assert_eq!(local_hits[0].path, "/home/me/notes.txt");
|
||||
assert_eq!(local_hits[0].score, 720);
|
||||
assert!(local_indexing);
|
||||
}
|
||||
other => panic!("expected unified search results, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_v2_legacy_wire_form_gets_compatible_defaults() {
|
||||
let decoded: Request =
|
||||
serde_json::from_str(r#"{"SearchV2":{"query":"old client","limit":20}}"#).unwrap();
|
||||
match decoded {
|
||||
Request::SearchV2 {
|
||||
query,
|
||||
limit,
|
||||
filters,
|
||||
} => {
|
||||
assert_eq!(query, "old client");
|
||||
assert_eq!(limit, 20);
|
||||
assert_eq!(filters, SearchFilters::default());
|
||||
}
|
||||
other => panic!("expected SearchV2, got {other:?}"),
|
||||
}
|
||||
|
||||
// New daemons add scores, but old-daemon replies remain readable and
|
||||
// signal that their existing order should be retained with score 0.
|
||||
let decoded: Response = serde_json::from_str(
|
||||
r#"{"SearchResultsV2":{"drive_hits":[{"name":"a","path":"a","is_dir":false,"size":1,"modified":2,"pinned":false,"uid":"v~a"}],"local_hits":[{"name":"b","path":"/b","is_dir":false,"size":3,"modified":4}],"local_indexing":false}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
match decoded {
|
||||
Response::SearchResultsV2 {
|
||||
drive_hits,
|
||||
local_hits,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(drive_hits[0].score, 0);
|
||||
assert_eq!(local_hits[0].score, 0);
|
||||
}
|
||||
other => panic!("expected SearchResultsV2, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_v2_filters_roundtrip() {
|
||||
let request = Request::SearchV2 {
|
||||
query: "holiday".into(),
|
||||
limit: 50,
|
||||
filters: SearchFilters {
|
||||
sources: vec![SearchSource::Drive],
|
||||
kind: SearchKind::Images,
|
||||
},
|
||||
};
|
||||
let line = serde_json::to_string(&request).unwrap();
|
||||
let decoded: Request = serde_json::from_str(&line).unwrap();
|
||||
assert!(matches!(
|
||||
decoded,
|
||||
Request::SearchV2 {
|
||||
filters: SearchFilters { sources, kind: SearchKind::Images },
|
||||
..
|
||||
} if sources == vec![SearchSource::Drive]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_kind_classifies_before_results_are_limited() {
|
||||
assert!(SearchKind::Folders.accepts("movie.mp4", true));
|
||||
assert!(!SearchKind::Folders.accepts("movie.mp4", false));
|
||||
assert!(SearchKind::Documents.accepts("REPORT.PDF", false));
|
||||
assert!(SearchKind::Images.accepts("scan.HEIC", false));
|
||||
assert!(SearchKind::Media.accepts("film.mkv", false));
|
||||
assert!(SearchKind::Media.accepts("recording.flac", false));
|
||||
assert!(!SearchKind::Media.accepts("poster.jpg", false));
|
||||
assert!(SearchKind::All.accepts("anything", true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_search_wire_forms_still_decode() {
|
||||
let drive: Request =
|
||||
serde_json::from_str(r#"{"Search":{"query":"old","limit":20}}"#).unwrap();
|
||||
assert!(matches!(
|
||||
drive,
|
||||
Request::Search { query, limit } if query == "old" && limit == 20
|
||||
));
|
||||
|
||||
let local: Request =
|
||||
serde_json::from_str(r#"{"SearchLocal":{"query":"old","limit":10}}"#).unwrap();
|
||||
assert!(matches!(
|
||||
local,
|
||||
Request::SearchLocal { query, limit } if query == "old" && limit == 10
|
||||
));
|
||||
}
|
||||
|
||||
/// The trash requests carry uids rather than paths; they must survive the same
|
||||
/// line-delimited round-trip, since a mangled uid would restore or destroy the
|
||||
/// wrong node.
|
||||
|
||||
@@ -7,6 +7,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
use crate::localindex::LocalEntry;
|
||||
|
||||
use super::nodes::candidate_trigrams;
|
||||
use super::utils::{TRIGRAM_MIN, like_escape};
|
||||
|
||||
/// One hit from [`Db::search_local`]: an indexed file on the machine itself, not
|
||||
@@ -147,6 +148,58 @@ impl Db {
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Bounded candidates for the fuzzy scorer, including matches in the full
|
||||
/// local path. ORed trigrams admit misspellings while the FTS index prevents
|
||||
/// this from becoming an unbounded filesystem-table scan.
|
||||
pub fn search_local_candidates(&self, query: &str, limit: usize) -> Result<Vec<LocalFileHit>> {
|
||||
let query = query.trim();
|
||||
if query.is_empty() || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let terms = candidate_trigrams(query);
|
||||
let lane_limit = limit.div_ceil(2);
|
||||
let mut rows: Vec<LocalFileHit> = if terms.is_empty() {
|
||||
let pat = format!("%{}%", like_escape(query));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT path, name, is_dir, size, mtime FROM local_files
|
||||
WHERE name LIKE ?1 ESCAPE '\\' OR path LIKE ?1 ESCAPE '\\'
|
||||
ORDER BY mtime DESC LIMIT ?2",
|
||||
)?;
|
||||
stmt.query_map(params![pat, limit as i64], local_hit)?
|
||||
.collect::<std::result::Result<_, _>>()?
|
||||
} else {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT f.path, f.name, f.is_dir, f.size, f.mtime
|
||||
FROM local_fts x JOIN local_files f ON f.id = x.rowid
|
||||
WHERE local_fts MATCH ?1
|
||||
ORDER BY x.rank LIMIT ?2",
|
||||
)?;
|
||||
stmt.query_map(params![terms.join(" OR "), lane_limit as i64], local_hit)?
|
||||
.collect::<std::result::Result<_, _>>()?
|
||||
};
|
||||
if !terms.is_empty()
|
||||
&& let Some(first) = query.chars().next()
|
||||
{
|
||||
let prefix = format!("{}%", like_escape(&first.to_string()));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT path, name, is_dir, size, mtime FROM local_files
|
||||
WHERE name COLLATE NOCASE LIKE ?1 ESCAPE '\\'
|
||||
ORDER BY name COLLATE NOCASE LIMIT ?2",
|
||||
)?;
|
||||
let fallback = stmt
|
||||
.query_map(params![prefix, lane_limit as i64], local_hit)?
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
for hit in fallback {
|
||||
if !rows.iter().any(|existing| existing.path == hit.path) {
|
||||
rows.push(hit);
|
||||
}
|
||||
}
|
||||
rows.truncate(limit);
|
||||
}
|
||||
Ok(rows)
|
||||
}
|
||||
}
|
||||
|
||||
/// Row → [`LocalFileHit`]. Every local-search query selects the same
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::Db;
|
||||
use crate::Result;
|
||||
|
||||
/// Current schema version. Bump on every forward migration added below.
|
||||
pub(super) const SCHEMA_VERSION: i64 = 15;
|
||||
pub(super) const SCHEMA_VERSION: i64 = 16;
|
||||
|
||||
impl Db {
|
||||
pub(super) fn migrate(&self) -> Result<()> {
|
||||
@@ -81,6 +81,9 @@ impl Db {
|
||||
if current < 15 {
|
||||
tx.execute_batch(MIGRATION_V15)?;
|
||||
}
|
||||
if current < 16 {
|
||||
tx.execute_batch(MIGRATION_V16)?;
|
||||
}
|
||||
tx.execute(
|
||||
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
@@ -369,3 +372,37 @@ CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
INSERT INTO nodes_fts (rowid, name)
|
||||
SELECT rowid, name FROM nodes WHERE trashed = 0;
|
||||
";
|
||||
|
||||
/// Schema v16: fuzzy search needs a bounded candidate set that includes parent
|
||||
/// paths. Keeping the path in the trigram indexes lets SQLite narrow both name
|
||||
/// and path candidates without walking either base table for every keystroke.
|
||||
/// Node paths are initially backfilled below and subsequently maintained by the
|
||||
/// node write path (including descendants after a folder move/rename).
|
||||
const MIGRATION_V16: &str = "
|
||||
DROP TABLE IF EXISTS nodes_fts;
|
||||
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
||||
name, path, tokenize='trigram'
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_nodes_name_nocase ON nodes(name COLLATE NOCASE);
|
||||
INSERT INTO nodes_fts (rowid, name, path)
|
||||
WITH RECURSIVE paths(rowid, uid, path) AS (
|
||||
SELECT n.rowid, n.uid, '' FROM nodes n
|
||||
WHERE n.parent_uid IS NULL
|
||||
OR NOT EXISTS (SELECT 1 FROM nodes p WHERE p.uid = n.parent_uid)
|
||||
UNION ALL
|
||||
SELECT n.rowid, n.uid,
|
||||
CASE WHEN paths.path = '' THEN n.name ELSE paths.path || '/' || n.name END
|
||||
FROM nodes n JOIN paths ON n.parent_uid = paths.uid
|
||||
)
|
||||
SELECT n.rowid, n.name, paths.path
|
||||
FROM nodes n JOIN paths ON paths.rowid = n.rowid
|
||||
WHERE n.trashed = 0;
|
||||
|
||||
DROP TABLE IF EXISTS local_fts;
|
||||
CREATE VIRTUAL TABLE local_fts USING fts5(
|
||||
name, path, content='local_files', content_rowid='id', tokenize='trigram'
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_local_files_name_nocase
|
||||
ON local_files(name COLLATE NOCASE);
|
||||
INSERT INTO local_fts(local_fts) VALUES('rebuild');
|
||||
";
|
||||
@@ -81,12 +81,42 @@ impl Db {
|
||||
params![uid],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
tx.execute("DELETE FROM nodes_fts WHERE rowid = ?1", params![rowid])?;
|
||||
if !node.trashed {
|
||||
tx.execute(
|
||||
"INSERT INTO nodes_fts (rowid, name) VALUES (?1, ?2)",
|
||||
params![rowid, node.name],
|
||||
// A folder rename/move changes every descendant's searchable path.
|
||||
// Refresh the affected subtree, while retaining efficient rowid
|
||||
// deletes for the overwhelmingly common one-file case.
|
||||
let mut affected = vec![rowid];
|
||||
if node.is_folder() {
|
||||
let mut stmt = tx.prepare(
|
||||
"WITH RECURSIVE descendants(rowid, uid) AS (
|
||||
SELECT rowid, uid FROM nodes WHERE parent_uid = ?1
|
||||
UNION ALL
|
||||
SELECT n.rowid, n.uid FROM nodes n
|
||||
JOIN descendants d ON n.parent_uid = d.uid
|
||||
) SELECT rowid FROM descendants",
|
||||
)?;
|
||||
for descendant in stmt.query_map(params![uid], |row| row.get(0))? {
|
||||
affected.push(descendant?);
|
||||
}
|
||||
}
|
||||
for affected_rowid in affected {
|
||||
tx.execute(
|
||||
"DELETE FROM nodes_fts WHERE rowid = ?1",
|
||||
params![affected_rowid],
|
||||
)?;
|
||||
let indexed: Option<(String, String)> = tx
|
||||
.query_row(
|
||||
"SELECT name, uid FROM nodes WHERE rowid = ?1 AND trashed = 0",
|
||||
params![affected_rowid],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some((name, indexed_uid)) = indexed {
|
||||
let path = path_of(&tx, &indexed_uid)?;
|
||||
tx.execute(
|
||||
"INSERT INTO nodes_fts (rowid, name, path) VALUES (?1, ?2, ?3)",
|
||||
params![affected_rowid, name, path],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
@@ -175,6 +205,68 @@ impl Db {
|
||||
Ok(hits)
|
||||
}
|
||||
|
||||
/// Return a bounded, deliberately broad candidate pool for fuzzy ranking.
|
||||
/// Unlike [`search`](Self::search), trigram terms are ORed across both the
|
||||
/// basename and parent path, so a typo can still share enough trigrams to
|
||||
/// enter the pool. Final relevance ordering belongs to the caller.
|
||||
pub fn search_candidates(&self, query: &str, limit: usize) -> Result<Vec<SearchHit>> {
|
||||
let query = query.trim();
|
||||
if query.is_empty() || limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let conn = self.conn.lock();
|
||||
let terms = candidate_trigrams(query);
|
||||
let lane_limit = limit.div_ceil(2);
|
||||
let mut rows: Vec<(String, String)> = if terms.is_empty() {
|
||||
let pat = format!("%{}%", like_escape(query));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_json, uid FROM nodes
|
||||
WHERE name LIKE ?1 ESCAPE '\\' AND trashed = 0 AND node_json IS NOT NULL
|
||||
ORDER BY name LIMIT ?2",
|
||||
)?;
|
||||
collect_pairs(stmt.query_map(params![pat, limit as i64], pair)?)?
|
||||
} else {
|
||||
let expression = terms.join(" OR ");
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT n.node_json, n.uid
|
||||
FROM nodes_fts f JOIN nodes n ON n.rowid = f.rowid
|
||||
WHERE nodes_fts MATCH ?1 AND n.trashed = 0 AND n.node_json IS NOT NULL
|
||||
ORDER BY f.rank LIMIT ?2",
|
||||
)?;
|
||||
collect_pairs(stmt.query_map(params![expression, lane_limit as i64], pair)?)?
|
||||
};
|
||||
// A short substitution can destroy every trigram (`vedio` vs `video`).
|
||||
// Feed the scorer an additional indexed same-initial lane so it can
|
||||
// recover those candidates without scanning the full node table.
|
||||
if !terms.is_empty()
|
||||
&& let Some(first) = query.chars().next()
|
||||
{
|
||||
let prefix = format!("{}%", like_escape(&first.to_string()));
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT node_json, uid FROM nodes
|
||||
WHERE name COLLATE NOCASE LIKE ?1 ESCAPE '\\'
|
||||
AND trashed = 0 AND node_json IS NOT NULL
|
||||
ORDER BY name COLLATE NOCASE LIMIT ?2",
|
||||
)?;
|
||||
let fallback =
|
||||
collect_pairs(stmt.query_map(params![prefix, lane_limit as i64], pair)?)?;
|
||||
for row in fallback {
|
||||
if !rows.iter().any(|(_, uid)| uid == &row.1) {
|
||||
rows.push(row);
|
||||
}
|
||||
}
|
||||
rows.truncate(limit);
|
||||
}
|
||||
rows.into_iter()
|
||||
.map(|(json, uid)| {
|
||||
Ok(SearchHit {
|
||||
node: serde_json::from_str(&json)?,
|
||||
path: path_of(&conn, &uid)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Mark (or unmark) a folder's child listing as complete. A listed folder
|
||||
/// rehydrates its `children` map on mount even when empty; an unlisted one
|
||||
/// re-enumerates from the remote on next access.
|
||||
@@ -261,6 +353,23 @@ impl Db {
|
||||
// in sync on every store/read/evict, so it is authoritative for eviction.
|
||||
}
|
||||
|
||||
/// Unique quoted character trigrams suitable for an FTS5 OR expression.
|
||||
pub(super) fn candidate_trigrams(query: &str) -> Vec<String> {
|
||||
let chars: Vec<char> = query.to_lowercase().chars().collect();
|
||||
let mut terms = Vec::new();
|
||||
for window in chars.windows(TRIGRAM_MIN) {
|
||||
if window.iter().all(|c| c.is_whitespace()) {
|
||||
continue;
|
||||
}
|
||||
let raw: String = window.iter().collect();
|
||||
let quoted = format!("\"{}\"", raw.replace('"', "\"\""));
|
||||
if !terms.contains("ed) {
|
||||
terms.push(quoted);
|
||||
}
|
||||
}
|
||||
terms
|
||||
}
|
||||
|
||||
/// Effective plaintext size of a node for the indexed `size` column: the
|
||||
/// claimed size when known, else the on-storage size; folders are 0.
|
||||
pub(super) fn node_size(node: &Node) -> i64 {
|
||||
|
||||
@@ -542,6 +542,95 @@ fn search_short_query_like_fallback() {
|
||||
assert_eq!(hits[0].node.name, "ab.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fuzzy_candidates_include_typo_and_parent_path_matches() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("projects", Some("root"), "Projects"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("report", "projects", "quarterly-report.pdf", 1))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("video", "projects", "video.mp4", 1))
|
||||
.unwrap();
|
||||
|
||||
let typo = db.search_candidates("quartrly", 20).unwrap();
|
||||
assert!(
|
||||
typo.iter()
|
||||
.any(|hit| hit.node.name == "quarterly-report.pdf")
|
||||
);
|
||||
assert!(
|
||||
db.search_candidates("vedio", 20)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|hit| hit.node.name == "video.mp4")
|
||||
);
|
||||
|
||||
let path = db.search_candidates("projects", 20).unwrap();
|
||||
assert!(
|
||||
path.iter()
|
||||
.any(|hit| hit.node.name == "quarterly-report.pdf")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renaming_parent_refreshes_descendant_search_paths() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
db.upsert_node(&folder("root", None, "My Files")).unwrap();
|
||||
db.upsert_node(&folder("folder", Some("root"), "zzbeforezz"))
|
||||
.unwrap();
|
||||
db.upsert_node(&file("child", "folder", "plain.txt", 1))
|
||||
.unwrap();
|
||||
assert!(
|
||||
db.search_candidates("zzbeforezz", 20)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|h| h.node.name == "plain.txt")
|
||||
);
|
||||
|
||||
db.upsert_node(&folder("folder", Some("root"), "yyafteryy"))
|
||||
.unwrap();
|
||||
assert!(
|
||||
!db.search_candidates("zzbeforezz", 20)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|h| h.node.name == "plain.txt")
|
||||
);
|
||||
assert!(
|
||||
db.search_candidates("yyafteryy", 20)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|h| h.node.name == "plain.txt")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_fuzzy_candidates_include_typo_and_parent_path() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
let generation = db.local_begin_scan().unwrap();
|
||||
db.local_upsert_batch(
|
||||
generation,
|
||||
&[
|
||||
local(
|
||||
"/home/u/Projects/quarterly-report.pdf",
|
||||
"quarterly-report.pdf",
|
||||
false,
|
||||
),
|
||||
local("/home/u/Videos/video.mp4", "video.mp4", false),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
db.local_finish_scan(generation, 1).unwrap();
|
||||
|
||||
assert_eq!(db.search_local_candidates("quartrly", 20).unwrap().len(), 1);
|
||||
assert_eq!(db.search_local_candidates("projects", 20).unwrap().len(), 1);
|
||||
assert!(
|
||||
db.search_local_candidates("vedio", 20)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|hit| hit.name == "video.mp4")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_drops_fts_row_on_delete_and_trash() {
|
||||
let db = Db::open_in_memory().unwrap();
|
||||
|
||||
@@ -10,6 +10,7 @@ pub mod db;
|
||||
pub mod error;
|
||||
pub mod localindex;
|
||||
pub mod profile;
|
||||
pub mod search;
|
||||
pub mod service;
|
||||
pub mod shell;
|
||||
pub mod syncignore;
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
//! Shared, deterministic search-result relevance scoring.
|
||||
//!
|
||||
//! The scorer deliberately operates on a small candidate set rather than
|
||||
//! replacing an index. Callers should use SQLite (or another index) to find
|
||||
//! candidates, then use this module to put local and remote results on the
|
||||
//! same relevance scale.
|
||||
|
||||
/// Scores a candidate against a user query.
|
||||
///
|
||||
/// Every query term must match either `basename` or `parent_path`. Basename
|
||||
/// matches are weighted twice as strongly as path matches. The returned score
|
||||
/// is stable and only meaningful when compared with scores produced by this
|
||||
/// function.
|
||||
#[must_use]
|
||||
pub fn relevance_score(query: &str, basename: &str, parent_path: &str) -> Option<i64> {
|
||||
let query = normalize(query);
|
||||
if query.is_empty() {
|
||||
return Some(0);
|
||||
}
|
||||
|
||||
let name = normalize(basename);
|
||||
let path = normalize(parent_path);
|
||||
let query_terms: Vec<_> = query.split_whitespace().collect();
|
||||
let name_words: Vec<_> = name.split_whitespace().collect();
|
||||
let path_words: Vec<_> = path.split_whitespace().collect();
|
||||
|
||||
let mut total = 0_i64;
|
||||
for term in query_terms {
|
||||
let name_score = field_term_score(term, &name, &name_words).map(|score| score * 2);
|
||||
let path_score = field_term_score(term, &path, &path_words);
|
||||
total += name_score.into_iter().chain(path_score).max()?;
|
||||
}
|
||||
|
||||
// Whole-field bonuses make ordering intuitive without changing whether a
|
||||
// candidate matches. They dominate small differences between term scores.
|
||||
if name == query {
|
||||
total += 20_000;
|
||||
} else if name.starts_with(&query) {
|
||||
total += 10_000;
|
||||
} else if name.contains(&query) {
|
||||
total += 4_000;
|
||||
}
|
||||
|
||||
Some(total)
|
||||
}
|
||||
|
||||
fn normalize(value: &str) -> String {
|
||||
let mut normalized = String::with_capacity(value.len());
|
||||
let mut last_was_space = true;
|
||||
|
||||
for character in value.chars().flat_map(char::to_lowercase) {
|
||||
if character.is_alphanumeric() {
|
||||
normalized.push(character);
|
||||
last_was_space = false;
|
||||
} else if !last_was_space {
|
||||
normalized.push(' ');
|
||||
last_was_space = true;
|
||||
}
|
||||
}
|
||||
|
||||
if last_was_space {
|
||||
normalized.pop();
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
fn field_term_score(term: &str, field: &str, words: &[&str]) -> Option<i64> {
|
||||
if field == term {
|
||||
return Some(10_000);
|
||||
}
|
||||
if field.starts_with(term) {
|
||||
return Some(8_500);
|
||||
}
|
||||
if words.contains(&term) {
|
||||
return Some(8_000);
|
||||
}
|
||||
if words.iter().any(|word| word.starts_with(term)) {
|
||||
return Some(7_000);
|
||||
}
|
||||
if field.contains(term) {
|
||||
return Some(6_000);
|
||||
}
|
||||
|
||||
// Subsequence matching supports useful abbreviations such as "qtrrpt".
|
||||
let subsequence = std::iter::once(field)
|
||||
.chain(words.iter().copied())
|
||||
.filter_map(|word| subsequence_gap(term, word))
|
||||
.min();
|
||||
if let Some(gap) = subsequence {
|
||||
return Some(4_500 - i64::try_from(gap.min(500)).unwrap_or(500));
|
||||
}
|
||||
|
||||
// Typo matching is deliberately restricted to substantial terms to avoid
|
||||
// noisy matches for one- and two-letter searches.
|
||||
if term.chars().count() >= 4
|
||||
&& words
|
||||
.iter()
|
||||
.any(|word| bounded_damerau_levenshtein(term, word, typo_limit(term)))
|
||||
{
|
||||
return Some(3_500);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn subsequence_gap(needle: &str, haystack: &str) -> Option<usize> {
|
||||
let needle_length = needle.chars().count();
|
||||
let mut needle = needle.chars();
|
||||
let mut wanted = needle.next()?;
|
||||
let mut first = None;
|
||||
|
||||
for (index, character) in haystack.chars().enumerate() {
|
||||
if character == wanted {
|
||||
first.get_or_insert(index);
|
||||
match needle.next() {
|
||||
Some(next) => wanted = next,
|
||||
None => return Some(index + 1 - first.unwrap_or(index) - needle_length),
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn typo_limit(term: &str) -> usize {
|
||||
usize::from(term.chars().count() >= 5) + 1
|
||||
}
|
||||
|
||||
/// A bounded optimal-string-alignment distance check. This treats a single
|
||||
/// adjacent transposition (for example, `vidoe`) as one edit.
|
||||
fn bounded_damerau_levenshtein(left: &str, right: &str, limit: usize) -> bool {
|
||||
let left: Vec<_> = left.chars().collect();
|
||||
let right: Vec<_> = right.chars().collect();
|
||||
if left.len().abs_diff(right.len()) > limit || left.len().max(right.len()) > 64 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut distances = vec![vec![0_usize; right.len() + 1]; left.len() + 1];
|
||||
for (index, row) in distances.iter_mut().enumerate() {
|
||||
row[0] = index;
|
||||
}
|
||||
for (index, distance) in distances[0].iter_mut().enumerate() {
|
||||
*distance = index;
|
||||
}
|
||||
|
||||
for i in 1..=left.len() {
|
||||
for j in 1..=right.len() {
|
||||
let cost = usize::from(left[i - 1] != right[j - 1]);
|
||||
distances[i][j] = (distances[i - 1][j] + 1)
|
||||
.min(distances[i][j - 1] + 1)
|
||||
.min(distances[i - 1][j - 1] + cost);
|
||||
if i > 1 && j > 1 && left[i - 1] == right[j - 2] && left[i - 2] == right[j - 1] {
|
||||
distances[i][j] = distances[i][j].min(distances[i - 2][j - 2] + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
distances[left.len()][right.len()] <= limit
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::relevance_score;
|
||||
|
||||
#[test]
|
||||
fn exact_name_beats_prefix_and_substring() {
|
||||
let exact = relevance_score("report", "report", "work").unwrap();
|
||||
let prefix = relevance_score("report", "report final", "work").unwrap();
|
||||
let substring = relevance_score("report", "annual report", "work").unwrap();
|
||||
assert!(exact > prefix);
|
||||
assert!(prefix > substring);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn basename_is_weighted_above_parent_path() {
|
||||
let name = relevance_score("budget", "budget notes", "archive").unwrap();
|
||||
let path = relevance_score("budget", "notes", "work/budget").unwrap();
|
||||
assert!(name > path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_typos_and_adjacent_transpositions() {
|
||||
assert!(relevance_score("vedio", "video.mp4", "media").is_some());
|
||||
assert!(relevance_score("vidoe", "video.mp4", "media").is_some());
|
||||
assert!(relevance_score("presentaton", "presentation.pdf", "work").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn separators_form_searchable_word_boundaries() {
|
||||
assert!(relevance_score("final draft", "Final_Draft-v2.pdf", "work").is_some());
|
||||
assert!(relevance_score("quarter report", "quarterly-report.pdf", "work").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_case_folding_preserves_accented_matches() {
|
||||
assert!(relevance_score("CAFÉ", "Café Photos", "Trips").is_some());
|
||||
assert!(relevance_score("ångström", "ÅNGSTRÖM notes", "Research").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_terms_across_name_and_path() {
|
||||
assert!(relevance_score("tax invoice", "invoice-1042.pdf", "archive/tax/2025").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordered_subsequence_supports_abbreviations() {
|
||||
assert!(relevance_score("qtrrpt", "quarter-report.pdf", "finance").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unrelated_candidates_and_short_typos() {
|
||||
assert_eq!(relevance_score("video", "holiday.jpg", "photos"), None);
|
||||
assert_eq!(relevance_score("xy", "xz.txt", "misc"), None);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user