nvrl/proton-stream
1
0 Fork 0
mirror of https://github.com/narrrl/proton-stream.git synced 2026-11-03 16:41:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(core,app,android,ios): look at shares again on their own, and say when they were last looked at

Browse Source 
No client crawled a share unless someone pressed Refresh, so a new
episode stayed out of the library until then, and nothing said how stale
the library was.

Catalog schema v11 adds share_crawls: when each share's last crawl
finished and was attempted, how many files it stored and how many were
new, and why the last attempt failed. A failure keeps the last good
crawl's numbers.

Every client now crawls, quietly, any share not looked at for six hours:
the desktop at launch and every half hour while open, the phones at
launch and on their existing catch-up timer. A quiet crawl says only
news ("anime: 3 new files") and leaves failures on the share's row
rather than in a toast.

Share rows on all three show "120 files - crawled 2 h ago", or the last
failure until a crawl succeeds.
This commit is contained in:
nvrl committed 2026-10-12 22:22:25 +02:00
1 parent a306d02cf1
commit 5807493a4a
13 files changed
+552 -75
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/pstr-android/src/lib.rs
+72 -25
View File
@@ -140,6 +140,18 @@ pub struct ShareRecord {
/// A folder of the signed-in account rather than a public link.
#[uniffi(default = false)]
pub from_account: bool,
/// When the last crawl here finished, Unix seconds. `None` before one has.
#[uniffi(default = None)]
pub crawled_at: Option<i64>,
/// Playable files that crawl stored.
#[uniffi(default = None)]
pub crawl_files: Option<u64>,
/// Of those, how many no earlier crawl had seen.
#[uniffi(default = None)]
pub crawl_added: Option<u64>,
/// Why the last attempt failed, until one succeeds.
#[uniffi(default = None)]
pub crawl_error: Option<String>,
}
fn share_record(share: Share) -> ShareRecord {
@@ -148,6 +160,10 @@ fn share_record(share: Share) -> ShareRecord {
id: share.id,
name: share.name,
has_custom_password: share.has_custom_password,
crawled_at: None,
crawl_files: None,
crawl_added: None,
crawl_error: None,
}
}
@@ -1259,31 +1275,52 @@ impl AndroidEngine {
Ok(share_record(share))
}
/// Configured shares with nothing in the catalog: brought in by sync,
/// possibly while the app was in the background, or added here and never
/// crawled to the end. Crawl each. An empty share is crawled every time,
/// which costs one listing.
/// Configured shares due a crawl: never crawled here — brought in by sync,
/// possibly while the app was in the background, or added and never
/// crawled to the end — or not looked at for
/// [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`], so a share grows
/// without anyone pressing Refresh. Crawl each.
pub fn uncrawled_shares(&self) -> Result<Vec<String>, BridgeError> {
let shares = self.store.list().map_err(BridgeError::from_display)?;
let catalog = self.catalog.lock();
let mut uncrawled = Vec::new();
for share in shares {
if catalog
.files(&share.id)
.map_err(BridgeError::from_display)?
.is_empty()
{
uncrawled.push(share.id);
}
}
Ok(uncrawled)
let crawls = self
.catalog
.lock()
.crawls()
.map_err(BridgeError::from_display)?;
let now = now();
Ok(shares
.into_iter()
.filter(|share| {
crawls.get(&share.id).is_none_or(|record| {
record.is_stale(now, pstr_core::catalog::CRAWL_STALE_AFTER_SECS)
})
})
.map(|share| share.id)
.collect())
}
/// Every configured share, with what its last crawl here came to.
pub fn shares(&self) -> Result<Vec<ShareRecord>, BridgeError> {
self.store
.list()
.map_err(BridgeError::from_display)
.map(|shares| shares.into_iter().map(share_record).collect())
let shares = self.store.list().map_err(BridgeError::from_display)?;
let crawls = self
.catalog
.lock()
.crawls()
.map_err(BridgeError::from_display)?;
Ok(shares
.into_iter()
.map(|share| {
let crawl = crawls.get(&share.id).cloned();
let mut record = share_record(share);
if let Some(crawl) = crawl {
record.crawled_at = (crawl.crawled_at > 0).then_some(crawl.crawled_at);
record.crawl_files = (crawl.crawled_at > 0).then_some(crawl.files);
record.crawl_added = (crawl.crawled_at > 0).then_some(crawl.added);
record.crawl_error = crawl.error;
}
record
})
.collect())
}
/// Add a public link, once it has opened with the password given. A link
@@ -2498,11 +2535,21 @@ impl AndroidEngine {
.lock()
.all_offline_files()
.map_err(BridgeError::from_display)?;
let nodes = connection
.library
.crawl(&id)
.await
.map_err(BridgeError::from_display)?;
let nodes = match connection.library.crawl(&id).await {
Ok(nodes) => nodes,
Err(error) => {
// On the share's row until a crawl succeeds: a link revoked
// upstream otherwise just stops growing.
if let Err(error) = self
.catalog
.lock()
.record_crawl_failure(&id, &error.to_string())
{
log::warn!("record a failed crawl: {error}");
}
return Err(BridgeError::from_display(error));
}
};
let rows = build_rows(&id, &nodes);
self.catalog
.lock()
crates/pstr-app/src/app.rs
+54 -11
View File
@@ -95,6 +95,10 @@ pub enum Page {
Player,
}
/// How often, in seconds, an open window looks for shares due a crawl. What
/// counts as due is [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`].
const STALE_CHECK_EVERY: f64 = 30.0 * 60.0;
/// The pages in the top bar, in order. Ctrl+1 to Ctrl+5 go by position here.
const NAV: [Page; 5] = [
Page::Library,
@@ -530,6 +534,10 @@ pub struct App {
pub page: Page,
pub library: Library,
pub shares: Vec<Share>,
/// What the last crawl of each share came to, for its row.
pub crawls: HashMap<String, pstr_core::catalog::CrawlRecord>,
/// When, on egui's clock, to next look for shares due a crawl.
next_stale_check: f64,
/// Proton's own per-file thumbnails.
pub thumbs: ImageCache,
/// Artwork from a metadata provider, keyed by title key.
@@ -631,7 +639,9 @@ impl App {
engine.load_library();
engine.forget_orphaned_shares();
engine.load_metadata();
engine.load_crawls();
engine.connect();
engine.crawl_stale();
let settings = engine.metadata_config();
Ok(Self {
@@ -641,6 +651,8 @@ impl App {
page: Page::Library,
library: Library::default(),
shares: Vec::new(),
crawls: HashMap::new(),
next_stale_check: STALE_CHECK_EVERY,
thumbs: ImageCache::default(),
posters: ImageCache::default(),
metadata: HashMap::new(),
@@ -761,20 +773,35 @@ impl App {
share_id,
nodes,
files,
added,
seconds,
quiet,
} => {
let name = self.share_name(&share_id);
self.crawl_progress.remove(&name);
self.note(
ctx,
format!("{name}: {files} playable of {nodes} nodes in {seconds:.0}s"),
false,
);
// Only a crawl long enough to have walked away from.
if seconds >= 30.0 {
self.notify(ctx, &format!("Crawled {name}"), &format!("{files} files"));
if quiet {
// Looked at by the app, not the viewer: only news.
if added > 0 {
let text = format!(
"{name}: {}",
ui::library::plural(added as usize, "new file")
);
self.note(ctx, &text, false);
self.notify(ctx, &format!("New in {name}"), &text);
}
} else {
self.note(
ctx,
format!("{name}: {files} playable of {nodes} nodes in {seconds:.0}s"),
false,
);
// Only a crawl long enough to have walked away from.
if seconds >= 30.0 {
self.notify(ctx, &format!("Crawled {name}"), &format!("{files} files"));
}
}
}
Event::Crawls(crawls) => self.crawls = crawls,
Event::CrawlFinished => {
self.crawling = false;
self.crawl_progress.clear();
@@ -1807,6 +1834,15 @@ impl eframe::App for App {
let ctx = ui.ctx().clone();
let ctx = &ctx;
self.pump(ctx, frame);
// A window left open for days still picks up what was added: the
// launch check alone would only ever run once.
let now = ctx.input(|input| input.time);
if now >= self.next_stale_check {
self.next_stale_check = now + STALE_CHECK_EVERY;
if now > 0.0 && !self.crawling {
self.engine.crawl_stale();
}
}
self.warm_next();
self.auto_skip(ctx);
self.window.observe(ctx);
@@ -2000,6 +2036,7 @@ impl eframe::App for App {
page,
library,
shares,
crawls,
thumbs,
posters,
metadata,
@@ -2107,9 +2144,15 @@ impl eframe::App for App {
},
&mut actions,
),
Page::Shares => {
ui::shares::show(ui, shares, library, form, account, &mut actions)
}
Page::Shares => ui::shares::show(
ui,
shares,
crawls,
library,
form,
account,
&mut actions,
),
Page::Settings => {
ui::settings::show(ui, settings, prefs, api_key, &mut actions)
}
crates/pstr-app/src/engine.rs
+94 -9
View File
@@ -193,8 +193,15 @@ pub enum Event {
share_id: String,
nodes: usize,
files: usize,
/// Files no earlier crawl had seen.
added: u64,
seconds: f64,
/// Started by the app rather than the viewer, so only news is worth
/// saying.
quiet: bool,
},
/// What the last crawl of each share came to, by share id.
Crawls(HashMap<String, pstr_core::catalog::CrawlRecord>),
/// A crawl is under way: which share, and how many nodes so far.
CrawlProgress { share: String, found: usize },
/// Every requested crawl is done.
@@ -402,6 +409,14 @@ const DOWNLOAD_CONCURRENCY: usize = 3;
/// launch and whenever a player stops.
const SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5 * 60);
/// Unix seconds, now.
pub fn unix_now() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|since| since.as_secs() as i64)
.unwrap_or(0)
}
/// Enrichment as currently configured.
struct Enrichment {
config: MetadataConfig,
@@ -663,7 +678,7 @@ impl Engine {
// The open connection predates the share, and crawling it through
// that would find it "not open".
engine.connection.lock().take();
engine.connect_and_crawl(Some(share.id)).await;
engine.connect_and_crawl(Some(vec![share.id]), false).await;
});
}
@@ -682,7 +697,7 @@ impl Engine {
engine.emit(Event::ShareAdded(share.name.clone()));
engine.load_shares();
engine.connection.lock().take();
engine.connect_and_crawl(Some(share.id)).await;
engine.connect_and_crawl(Some(vec![share.id]), false).await;
});
}
@@ -944,7 +959,7 @@ impl Engine {
self.load_shares();
self.load_library();
for id in profile.shares_added.iter().chain(&profile.shares_changed) {
self.connect_and_crawl(Some(id.clone())).await;
self.connect_and_crawl(Some(vec![id.clone()]), false).await;
}
}
@@ -1277,18 +1292,65 @@ impl Engine {
pub fn crawl(&self, share: Option<String>) {
let engine = self.clone();
self.runtime.spawn(async move {
engine.connect_and_crawl(share).await;
engine
.connect_and_crawl(share.map(|id| vec![id]), false)
.await;
});
}
async fn connect_and_crawl(&self, share: Option<String>) {
/// Crawl, quietly, every share no crawl has looked at for
/// [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`] — or ever.
///
/// Without this a share only grew when someone thought to press Refresh,
/// so the episode that came out yesterday was not in the library until
/// then. Quiet: a failure is logged and recorded on the share's row rather
/// than toasted, and a crawl that found nothing new says nothing.
pub fn crawl_stale(&self) {
let engine = self.clone();
self.runtime.spawn(async move {
let stale = {
let crawls = match engine.catalog.lock().crawls() {
Ok(crawls) => crawls,
Err(error) => return tracing::warn!(%error, "read crawl records"),
};
let Ok(shares) = engine.store.list() else {
return;
};
let now = unix_now();
shares
.into_iter()
.filter(|share| {
crawls.get(&share.id).is_none_or(|record| {
record.is_stale(now, pstr_core::catalog::CRAWL_STALE_AFTER_SECS)
})
})
.map(|share| share.id)
.collect::<Vec<_>>()
};
if !stale.is_empty() {
engine.connect_and_crawl(Some(stale), true).await;
}
});
}
/// Read what the last crawl of each share came to, for the shares page.
pub fn load_crawls(&self) {
let engine = self.clone();
self.runtime
.spawn_blocking(move || match engine.catalog.lock().crawls() {
Ok(crawls) => engine.emit(Event::Crawls(crawls)),
Err(error) => tracing::warn!(%error, "read crawl records"),
});
}
async fn connect_and_crawl(&self, shares: Option<Vec<String>>, quiet: bool) {
let Some((library, _)) = self.ensure_connected().await else {
self.emit(Event::CrawlFinished);
return;
};
let targets: Vec<String> = match share {
Some(id) => vec![id],
let targets: Vec<String> = match shares {
Some(ids) => ids,
None => library.share_ids().map(str::to_string).collect(),
};
@@ -1298,7 +1360,7 @@ impl Engine {
futures::future::join_all(
targets
.into_iter()
.map(|share_id| self.crawl_share(&library, share_id, cancel.clone())),
.map(|share_id| self.crawl_share(&library, share_id, cancel.clone(), quiet)),
)
.await;
self.emit(Event::CrawlFinished);
@@ -1342,6 +1404,7 @@ impl Engine {
library: &SharedLibrary,
share_id: String,
mut cancel: tokio::sync::watch::Receiver<u64>,
quiet: bool,
) {
let started = std::time::Instant::now();
let name = self
@@ -1376,7 +1439,19 @@ impl Engine {
}) => match result {
Ok(nodes) => nodes,
Err(error) => {
self.fail(&format!("crawl {share_id}"), error);
if let Err(error) = self
.catalog
.lock()
.record_crawl_failure(&share_id, &error.to_string())
{
tracing::warn!(%error, "record a failed crawl");
}
self.load_crawls();
if quiet {
tracing::warn!(%error, "crawl {name}");
} else {
self.fail(&format!("crawl {name}"), error);
}
return;
}
},
@@ -1455,12 +1530,22 @@ impl Engine {
return;
}
let added = self
.catalog
.lock()
.crawls()
.ok()
.and_then(|crawls| crawls.get(&share_id).map(|record| record.added))
.unwrap_or(0);
self.emit(Event::Crawled {
share_id,
nodes: nodes.len(),
files,
added,
seconds: started.elapsed().as_secs_f64(),
quiet,
});
self.load_crawls();
self.load_library();
}
crates/pstr-app/src/ui/account.rs
+1 -7
View File
@@ -315,13 +315,7 @@ fn signed_in(
/// "Synced 3 min ago · 2 positions from other devices".
fn sync_line(at: i64, report: &SyncReport, now: i64) -> String {
let ago = (now - at).max(0);
let when = match ago {
0..60 => "just now".to_owned(),
60..3600 => format!("{} min ago", ago / 60),
_ => format!("{} h ago", ago / 3600),
};
let mut line = format!("Synced {when}");
let mut line = format!("Synced {}", ui::ago(at, now));
if report.applied > 0 {
line.push_str(&format!(
" · {} from other devices",
crates/pstr-app/src/ui/mod.rs
+13
View File
@@ -172,6 +172,19 @@ pub fn wide_key(title_key: &str) -> String {
format!("wide:{title_key}")
}
/// "just now", "5 min ago", "3 h ago", "2 days ago": how long before `now`
/// the Unix time `at` was.
pub fn ago(at: i64, now: i64) -> String {
let ago = (now - at).max(0);
match ago {
0..60 => "just now".to_owned(),
60..3600 => format!("{} min ago", ago / 60),
3600..86_400 => format!("{} h ago", ago / 3600),
86_400..172_800 => "1 day ago".to_owned(),
_ => format!("{} days ago", ago / 86_400),
}
}
/// `1:03:47`, or `4:12` for anything under an hour.
pub fn format_time(seconds: f64) -> String {
if !seconds.is_finite() || seconds < 0.0 {
crates/pstr-app/src/ui/shares.rs
+28 -2
View File
@@ -6,7 +6,10 @@
//! below is the only place a link is ever visible, and only while it is being
//! typed.
use std::collections::HashMap;
use pstr_core::Share;
use pstr_core::catalog::CrawlRecord;
use pstr_core::library::Library;
use crate::app::{Action, ShareForm};
@@ -16,6 +19,7 @@ use crate::ui;
pub fn show(
ui: &mut egui::Ui,
shares: &[Share],
crawls: &HashMap<String, CrawlRecord>,
library: &Library,
form: &mut ShareForm,
account: &mut ui::account::AccountPanel,
@@ -36,7 +40,7 @@ pub fn show(
.iter()
.filter(|title| title.share_ids.contains(&share.id))
.count();
share_row(ui, share, titles, actions);
share_row(ui, share, titles, crawls.get(&share.id), actions);
}
ui.add_space(theme::space::XXL);
@@ -48,7 +52,13 @@ pub fn show(
});
}
fn share_row(ui: &mut egui::Ui, share: &Share, titles: usize, actions: &mut Vec<Action>) {
fn share_row(
ui: &mut egui::Ui,
share: &Share,
titles: usize,
crawl: Option<&CrawlRecord>,
actions: &mut Vec<Action>,
) {
egui::Frame::new()
.fill(theme::card())
.corner_radius(egui::CornerRadius::same(theme::radius::MD))
@@ -67,7 +77,23 @@ fn share_row(ui: &mut egui::Ui, share: &Share, titles: usize, actions: &mut Vec<
if share.has_custom_password {
detail.push_str(" · password protected");
}
match crawl {
Some(crawl) if crawl.crawled_at > 0 => {
detail.push_str(" · crawled ");
detail.push_str(&ui::ago(crawl.crawled_at, crate::engine::unix_now()));
}
_ => detail.push_str(" · not crawled yet"),
}
ui.label(ui::muted(detail)).on_hover_text(&share.id);
// Why the last look failed, until one succeeds: a link
// revoked upstream otherwise just stops growing.
if let Some(error) = crawl.and_then(|crawl| crawl.error.as_deref()) {
ui.label(
theme::Role::Caption
.rich(format!("Last crawl failed: {error}"))
.color(theme::danger()),
);
}
});
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
crates/pstr-core/src/catalog.rs
+191 -3
View File
@@ -24,7 +24,7 @@ use crate::metadata::{
use crate::naming::{self, ParsedName};
/// Bump when adding a migration.
pub const SCHEMA_VERSION: i64 = 10;
pub const SCHEMA_VERSION: i64 = 11;
const MIGRATION_V1: &str = r#"
CREATE TABLE nodes (
@@ -207,6 +207,51 @@ INSERT OR IGNORE INTO node_first_seen (share_id, link_id, seen_at)
UPDATE title_metadata SET enriched_at = NULL;
"#;
/// What the last crawl of each share came to.
///
/// Per installation, like the rest of the catalog: when this device last
/// looked at a share is nothing another device needs. `crawled_at` is the last
/// crawl that finished, `attempted_at` the last that was tried, so a share that
/// keeps failing is not retried on every launch. `added` is how many files the
/// last finished crawl found that had never been seen before.
const MIGRATION_V11: &str = r#"
CREATE TABLE share_crawls (
share_id TEXT PRIMARY KEY,
crawled_at INTEGER NOT NULL,
attempted_at INTEGER NOT NULL,
files INTEGER NOT NULL,
added INTEGER NOT NULL,
error TEXT
);
"#;
/// How old a share's last crawl may be before an app looks again on its own.
/// Long enough that a launch rarely costs a crawl; short enough that a new
/// episode turns up the same day.
pub const CRAWL_STALE_AFTER_SECS: i64 = 6 * 60 * 60;
/// One share's row of `share_crawls`. See [`MIGRATION_V11`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CrawlRecord {
/// Unix seconds; `0` for a share no crawl of has finished.
pub crawled_at: i64,
pub attempted_at: i64,
/// Playable files the last finished crawl stored.
pub files: u64,
/// Of those, how many no earlier crawl had seen.
pub added: u64,
/// Why the last attempt failed, when it did.
pub error: Option<String>,
}
impl CrawlRecord {
/// Whether the share is due another look: nothing has been tried within
/// `age_secs` of `now`.
pub fn is_stale(&self, now: i64, age_secs: i64) -> bool {
now - self.crawled_at.max(self.attempted_at) >= age_secs
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OfflineFile {
pub revision_id: String,
@@ -408,6 +453,12 @@ impl Catalog {
tx.pragma_update(None, "user_version", 10)?;
tx.commit()?;
}
if version < 11 {
let tx = self.conn.transaction()?;
tx.execute_batch(MIGRATION_V11)?;
tx.pragma_update(None, "user_version", 11)?;
tx.commit()?;
}
Ok(())
}
@@ -469,10 +520,23 @@ impl Catalog {
}
// Only files: a folder appearing is not something anyone watches.
tx.execute(
let now = unix_now();
let added = tx.execute(
"INSERT OR IGNORE INTO node_first_seen (share_id, link_id, seen_at)
SELECT share_id, link_id, ?2 FROM nodes WHERE share_id = ?1 AND is_folder = 0",
params![share_id, unix_now()],
params![share_id, now],
)?;
let files = nodes.iter().filter(|node| !node.is_folder).count();
tx.execute(
"INSERT INTO share_crawls (share_id, crawled_at, attempted_at, files, added, error)
VALUES (?1, ?2, ?2, ?3, ?4, NULL)
ON CONFLICT (share_id) DO UPDATE SET
crawled_at = excluded.crawled_at,
attempted_at = excluded.attempted_at,
files = excluded.files,
added = excluded.added,
error = NULL",
params![share_id, now, files as i64, added as i64],
)?;
if !retain_offline {
@@ -483,6 +547,41 @@ impl Catalog {
Ok(())
}
/// Record that a crawl of `share_id` failed, keeping what the last
/// finished one stored.
pub fn record_crawl_failure(&self, share_id: &str, error: &str) -> Result<()> {
self.conn.execute(
"INSERT INTO share_crawls (share_id, crawled_at, attempted_at, files, added, error)
VALUES (?1, 0, ?2, 0, 0, ?3)
ON CONFLICT (share_id) DO UPDATE SET
attempted_at = excluded.attempted_at,
error = excluded.error",
params![share_id, unix_now(), error],
)?;
Ok(())
}
/// What the last crawl of each share came to, by share id. A share never
/// crawled has no entry.
pub fn crawls(&self) -> Result<HashMap<String, CrawlRecord>> {
let mut statement = self.conn.prepare(
"SELECT share_id, crawled_at, attempted_at, files, added, error FROM share_crawls",
)?;
let rows = statement.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
CrawlRecord {
crawled_at: row.get(1)?,
attempted_at: row.get(2)?,
files: row.get::<_, i64>(3)?.max(0) as u64,
added: row.get::<_, i64>(4)?.max(0) as u64,
error: row.get(5)?,
},
))
})?;
Ok(rows.collect::<std::result::Result<_, _>>()?)
}
/// Every file in a share, folders excluded, ordered for display.
pub fn files(&self, share_id: &str) -> Result<Vec<CatalogNode>> {
let mut statement = self.conn.prepare(
@@ -579,6 +678,10 @@ impl Catalog {
"DELETE FROM node_first_seen WHERE share_id = ?1",
params![share_id],
)?;
tx.execute(
"DELETE FROM share_crawls WHERE share_id = ?1",
params![share_id],
)?;
tx.commit()?;
Ok(())
}
@@ -2133,6 +2236,91 @@ mod tests {
assert!(catalog.enrichment_ages().expect("ages").is_empty());
}
#[test]
fn a_crawl_records_when_it_finished_and_what_it_added() {
let mut catalog = Catalog::in_memory().expect("open");
catalog
.replace_share("s", &[file("s", "a", "Show S01E01.mkv")])
.expect("crawl");
catalog
.replace_share(
"s",
&[
file("s", "a", "Show S01E01.mkv"),
file("s", "b", "Show S01E02.mkv"),
],
)
.expect("recrawl");
let record = &catalog.crawls().expect("read")["s"];
assert!(record.crawled_at > 0);
assert_eq!(record.attempted_at, record.crawled_at);
assert_eq!((record.files, record.added), (2, 1));
assert_eq!(record.error, None);
}
/// A failure says why without forgetting what the last good crawl found,
/// and the next good crawl clears it.
#[test]
fn a_failed_crawl_keeps_the_last_good_one_until_the_next() {
let mut catalog = Catalog::in_memory().expect("open");
catalog
.replace_share("s", &[file("s", "a", "Show S01E01.mkv")])
.expect("crawl");
catalog
.conn
.execute(
"UPDATE share_crawls SET crawled_at = 5, attempted_at = 5",
[],
)
.expect("age it");
catalog
.record_crawl_failure("s", "link expired")
.expect("fail");
let record = catalog.crawls().expect("read")["s"].clone();
assert_eq!((record.crawled_at, record.files), (5, 1));
assert!(record.attempted_at > 5);
assert_eq!(record.error.as_deref(), Some("link expired"));
assert!(!record.is_stale(record.attempted_at + 10, 60));
catalog
.replace_share("s", &[file("s", "a", "Show S01E01.mkv")])
.expect("recrawl");
assert_eq!(catalog.crawls().expect("read")["s"].error, None);
}
#[test]
fn a_removed_share_forgets_its_crawls() {
let mut catalog = Catalog::in_memory().expect("open");
catalog
.replace_share("s", &[file("s", "a", "Show S01E01.mkv")])
.expect("crawl");
catalog.remove_share("s").expect("remove");
assert!(catalog.crawls().expect("read").is_empty());
}
#[test]
fn migration_v11_opens_a_v10_catalog() {
let conn = Connection::open_in_memory().expect("open");
for migration in [
MIGRATION_V1,
MIGRATION_V2,
MIGRATION_V3,
MIGRATION_V4,
MIGRATION_V5,
MIGRATION_V6,
MIGRATION_V7,
MIGRATION_V8,
MIGRATION_V9,
MIGRATION_V10,
] {
conn.execute_batch(migration).expect("migrate");
}
conn.pragma_update(None, "user_version", 10)
.expect("set version");
let catalog = Catalog::from_connection(conn).expect("upgrade");
assert!(catalog.crawls().expect("read").is_empty());
}
/// A file keeps the moment it first appeared across recrawls, and one
/// that was there before this was recorded reads as 0.
#[test]