feat(core,app,android,ios): look at shares again on their own, and say when they were last looked at

No client crawled a share unless someone pressed Refresh, so a new
episode stayed out of the library until then, and nothing said how stale
the library was.

Catalog schema v11 adds share_crawls: when each share's last crawl
finished and was attempted, how many files it stored and how many were
new, and why the last attempt failed. A failure keeps the last good
crawl's numbers.

Every client now crawls, quietly, any share not looked at for six hours:
the desktop at launch and every half hour while open, the phones at
launch and on their existing catch-up timer. A quiet crawl says only
news ("anime: 3 new files") and leaves failures on the share's row
rather than in a toast.

Share rows on all three show "120 files - crawled 2 h ago", or the last
failure until a crawl succeeds.

crates/pstr-app/src/app.rs

@@ -95,6 +95,10 @@ pub enum Page {
     Player,
 }
 
+/// How often, in seconds, an open window looks for shares due a crawl. What
+/// counts as due is [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`].
+const STALE_CHECK_EVERY: f64 = 30.0 * 60.0;
+
 /// The pages in the top bar, in order. Ctrl+1 to Ctrl+5 go by position here.
 const NAV: [Page; 5] = [
     Page::Library,
@@ -530,6 +534,10 @@ pub struct App {
     pub page: Page,
     pub library: Library,
     pub shares: Vec<Share>,
+    /// What the last crawl of each share came to, for its row.
+    pub crawls: HashMap<String, pstr_core::catalog::CrawlRecord>,
+    /// When, on egui's clock, to next look for shares due a crawl.
+    next_stale_check: f64,
     /// Proton's own per-file thumbnails.
     pub thumbs: ImageCache,
     /// Artwork from a metadata provider, keyed by title key.
@@ -631,7 +639,9 @@ impl App {
         engine.load_library();
         engine.forget_orphaned_shares();
         engine.load_metadata();
+        engine.load_crawls();
         engine.connect();
+        engine.crawl_stale();
         let settings = engine.metadata_config();
 
         Ok(Self {
@@ -641,6 +651,8 @@ impl App {
             page: Page::Library,
             library: Library::default(),
             shares: Vec::new(),
+            crawls: HashMap::new(),
+            next_stale_check: STALE_CHECK_EVERY,
             thumbs: ImageCache::default(),
             posters: ImageCache::default(),
             metadata: HashMap::new(),
@@ -761,20 +773,35 @@ impl App {
                     share_id,
                     nodes,
                     files,
+                    added,
                     seconds,
+                    quiet,
                 } => {
                     let name = self.share_name(&share_id);
                     self.crawl_progress.remove(&name);
-                    self.note(
-                        ctx,
-                        format!("{name}: {files} playable of {nodes} nodes in {seconds:.0}s"),
-                        false,
-                    );
-                    // Only a crawl long enough to have walked away from.
-                    if seconds >= 30.0 {
-                        self.notify(ctx, &format!("Crawled {name}"), &format!("{files} files"));
+                    if quiet {
+                        // Looked at by the app, not the viewer: only news.
+                        if added > 0 {
+                            let text = format!(
+                                "{name}: {}",
+                                ui::library::plural(added as usize, "new file")
+                            );
+                            self.note(ctx, &text, false);
+                            self.notify(ctx, &format!("New in {name}"), &text);
+                        }
+                    } else {
+                        self.note(
+                            ctx,
+                            format!("{name}: {files} playable of {nodes} nodes in {seconds:.0}s"),
+                            false,
+                        );
+                        // Only a crawl long enough to have walked away from.
+                        if seconds >= 30.0 {
+                            self.notify(ctx, &format!("Crawled {name}"), &format!("{files} files"));
+                        }
                     }
                 }
+                Event::Crawls(crawls) => self.crawls = crawls,
                 Event::CrawlFinished => {
                     self.crawling = false;
                     self.crawl_progress.clear();
@@ -1807,6 +1834,15 @@ impl eframe::App for App {
         let ctx = ui.ctx().clone();
         let ctx = &ctx;
         self.pump(ctx, frame);
+        // A window left open for days still picks up what was added: the
+        // launch check alone would only ever run once.
+        let now = ctx.input(|input| input.time);
+        if now >= self.next_stale_check {
+            self.next_stale_check = now + STALE_CHECK_EVERY;
+            if now > 0.0 && !self.crawling {
+                self.engine.crawl_stale();
+            }
+        }
         self.warm_next();
         self.auto_skip(ctx);
         self.window.observe(ctx);
@@ -2000,6 +2036,7 @@ impl eframe::App for App {
                 page,
                 library,
                 shares,
+                crawls,
                 thumbs,
                 posters,
                 metadata,
@@ -2107,9 +2144,15 @@ impl eframe::App for App {
                             },
                             &mut actions,
                         ),
-                        Page::Shares => {
-                            ui::shares::show(ui, shares, library, form, account, &mut actions)
-                        }
+                        Page::Shares => ui::shares::show(
+                            ui,
+                            shares,
+                            crawls,
+                            library,
+                            form,
+                            account,
+                            &mut actions,
+                        ),
                         Page::Settings => {
                             ui::settings::show(ui, settings, prefs, api_key, &mut actions)
                         }

crates/pstr-app/src/engine.rs

@@ -193,8 +193,15 @@ pub enum Event {
         share_id: String,
         nodes: usize,
         files: usize,
+        /// Files no earlier crawl had seen.
+        added: u64,
         seconds: f64,
+        /// Started by the app rather than the viewer, so only news is worth
+        /// saying.
+        quiet: bool,
     },
+    /// What the last crawl of each share came to, by share id.
+    Crawls(HashMap<String, pstr_core::catalog::CrawlRecord>),
     /// A crawl is under way: which share, and how many nodes so far.
     CrawlProgress { share: String, found: usize },
     /// Every requested crawl is done.
@@ -402,6 +409,14 @@ const DOWNLOAD_CONCURRENCY: usize = 3;
 /// launch and whenever a player stops.
 const SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5 * 60);
 
+/// Unix seconds, now.
+pub fn unix_now() -> i64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|since| since.as_secs() as i64)
+        .unwrap_or(0)
+}
+
 /// Enrichment as currently configured.
 struct Enrichment {
     config: MetadataConfig,
@@ -663,7 +678,7 @@ impl Engine {
             // The open connection predates the share, and crawling it through
             // that would find it "not open".
             engine.connection.lock().take();
-            engine.connect_and_crawl(Some(share.id)).await;
+            engine.connect_and_crawl(Some(vec![share.id]), false).await;
         });
     }
 
@@ -682,7 +697,7 @@ impl Engine {
             engine.emit(Event::ShareAdded(share.name.clone()));
             engine.load_shares();
             engine.connection.lock().take();
-            engine.connect_and_crawl(Some(share.id)).await;
+            engine.connect_and_crawl(Some(vec![share.id]), false).await;
         });
     }
 
@@ -944,7 +959,7 @@ impl Engine {
         self.load_shares();
         self.load_library();
         for id in profile.shares_added.iter().chain(&profile.shares_changed) {
-            self.connect_and_crawl(Some(id.clone())).await;
+            self.connect_and_crawl(Some(vec![id.clone()]), false).await;
         }
     }
 
@@ -1277,18 +1292,65 @@ impl Engine {
     pub fn crawl(&self, share: Option<String>) {
         let engine = self.clone();
         self.runtime.spawn(async move {
-            engine.connect_and_crawl(share).await;
+            engine
+                .connect_and_crawl(share.map(|id| vec![id]), false)
+                .await;
         });
     }
 
-    async fn connect_and_crawl(&self, share: Option<String>) {
+    /// Crawl, quietly, every share no crawl has looked at for
+    /// [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`] — or ever.
+    ///
+    /// Without this a share only grew when someone thought to press Refresh,
+    /// so the episode that came out yesterday was not in the library until
+    /// then. Quiet: a failure is logged and recorded on the share's row rather
+    /// than toasted, and a crawl that found nothing new says nothing.
+    pub fn crawl_stale(&self) {
+        let engine = self.clone();
+        self.runtime.spawn(async move {
+            let stale = {
+                let crawls = match engine.catalog.lock().crawls() {
+                    Ok(crawls) => crawls,
+                    Err(error) => return tracing::warn!(%error, "read crawl records"),
+                };
+                let Ok(shares) = engine.store.list() else {
+                    return;
+                };
+                let now = unix_now();
+                shares
+                    .into_iter()
+                    .filter(|share| {
+                        crawls.get(&share.id).is_none_or(|record| {
+                            record.is_stale(now, pstr_core::catalog::CRAWL_STALE_AFTER_SECS)
+                        })
+                    })
+                    .map(|share| share.id)
+                    .collect::<Vec<_>>()
+            };
+            if !stale.is_empty() {
+                engine.connect_and_crawl(Some(stale), true).await;
+            }
+        });
+    }
+
+    /// Read what the last crawl of each share came to, for the shares page.
+    pub fn load_crawls(&self) {
+        let engine = self.clone();
+        self.runtime
+            .spawn_blocking(move || match engine.catalog.lock().crawls() {
+                Ok(crawls) => engine.emit(Event::Crawls(crawls)),
+                Err(error) => tracing::warn!(%error, "read crawl records"),
+            });
+    }
+
+    async fn connect_and_crawl(&self, shares: Option<Vec<String>>, quiet: bool) {
         let Some((library, _)) = self.ensure_connected().await else {
             self.emit(Event::CrawlFinished);
             return;
         };
 
-        let targets: Vec<String> = match share {
-            Some(id) => vec![id],
+        let targets: Vec<String> = match shares {
+            Some(ids) => ids,
             None => library.share_ids().map(str::to_string).collect(),
         };
 
@@ -1298,7 +1360,7 @@ impl Engine {
         futures::future::join_all(
             targets
                 .into_iter()
-                .map(|share_id| self.crawl_share(&library, share_id, cancel.clone())),
+                .map(|share_id| self.crawl_share(&library, share_id, cancel.clone(), quiet)),
         )
         .await;
         self.emit(Event::CrawlFinished);
@@ -1342,6 +1404,7 @@ impl Engine {
         library: &SharedLibrary,
         share_id: String,
         mut cancel: tokio::sync::watch::Receiver<u64>,
+        quiet: bool,
     ) {
         let started = std::time::Instant::now();
         let name = self
@@ -1376,7 +1439,19 @@ impl Engine {
             }) => match result {
                 Ok(nodes) => nodes,
                 Err(error) => {
-                    self.fail(&format!("crawl {share_id}"), error);
+                    if let Err(error) = self
+                        .catalog
+                        .lock()
+                        .record_crawl_failure(&share_id, &error.to_string())
+                    {
+                        tracing::warn!(%error, "record a failed crawl");
+                    }
+                    self.load_crawls();
+                    if quiet {
+                        tracing::warn!(%error, "crawl {name}");
+                    } else {
+                        self.fail(&format!("crawl {name}"), error);
+                    }
                     return;
                 }
             },
@@ -1455,12 +1530,22 @@ impl Engine {
             return;
         }
 
+        let added = self
+            .catalog
+            .lock()
+            .crawls()
+            .ok()
+            .and_then(|crawls| crawls.get(&share_id).map(|record| record.added))
+            .unwrap_or(0);
         self.emit(Event::Crawled {
             share_id,
             nodes: nodes.len(),
             files,
+            added,
             seconds: started.elapsed().as_secs_f64(),
+            quiet,
         });
+        self.load_crawls();
         self.load_library();
     }
 

crates/pstr-app/src/ui/account.rs

@@ -315,13 +315,7 @@ fn signed_in(
 
 /// "Synced 3 min ago · 2 positions from other devices".
 fn sync_line(at: i64, report: &SyncReport, now: i64) -> String {
-    let ago = (now - at).max(0);
-    let when = match ago {
-        0..60 => "just now".to_owned(),
-        60..3600 => format!("{} min ago", ago / 60),
-        _ => format!("{} h ago", ago / 3600),
-    };
-    let mut line = format!("Synced {when}");
+    let mut line = format!("Synced {}", ui::ago(at, now));
     if report.applied > 0 {
         line.push_str(&format!(
             "  ·  {} from other devices",

crates/pstr-app/src/ui/mod.rs

@@ -172,6 +172,19 @@ pub fn wide_key(title_key: &str) -> String {
     format!("wide:{title_key}")
 }
 
+/// "just now", "5 min ago", "3 h ago", "2 days ago": how long before `now`
+/// the Unix time `at` was.
+pub fn ago(at: i64, now: i64) -> String {
+    let ago = (now - at).max(0);
+    match ago {
+        0..60 => "just now".to_owned(),
+        60..3600 => format!("{} min ago", ago / 60),
+        3600..86_400 => format!("{} h ago", ago / 3600),
+        86_400..172_800 => "1 day ago".to_owned(),
+        _ => format!("{} days ago", ago / 86_400),
+    }
+}
+
 /// `1:03:47`, or `4:12` for anything under an hour.
 pub fn format_time(seconds: f64) -> String {
     if !seconds.is_finite() || seconds < 0.0 {

crates/pstr-app/src/ui/shares.rs

@@ -6,7 +6,10 @@
 //! below is the only place a link is ever visible, and only while it is being
 //! typed.
 
+use std::collections::HashMap;
+
 use pstr_core::Share;
+use pstr_core::catalog::CrawlRecord;
 use pstr_core::library::Library;
 
 use crate::app::{Action, ShareForm};
@@ -16,6 +19,7 @@ use crate::ui;
 pub fn show(
     ui: &mut egui::Ui,
     shares: &[Share],
+    crawls: &HashMap<String, CrawlRecord>,
     library: &Library,
     form: &mut ShareForm,
     account: &mut ui::account::AccountPanel,
@@ -36,7 +40,7 @@ pub fn show(
                     .iter()
                     .filter(|title| title.share_ids.contains(&share.id))
                     .count();
-                share_row(ui, share, titles, actions);
+                share_row(ui, share, titles, crawls.get(&share.id), actions);
             }
 
             ui.add_space(theme::space::XXL);
@@ -48,7 +52,13 @@ pub fn show(
         });
 }
 
-fn share_row(ui: &mut egui::Ui, share: &Share, titles: usize, actions: &mut Vec<Action>) {
+fn share_row(
+    ui: &mut egui::Ui,
+    share: &Share,
+    titles: usize,
+    crawl: Option<&CrawlRecord>,
+    actions: &mut Vec<Action>,
+) {
     egui::Frame::new()
         .fill(theme::card())
         .corner_radius(egui::CornerRadius::same(theme::radius::MD))
@@ -67,7 +77,23 @@ fn share_row(ui: &mut egui::Ui, share: &Share, titles: usize, actions: &mut Vec<
                     if share.has_custom_password {
                         detail.push_str("  ·  password protected");
                     }
+                    match crawl {
+                        Some(crawl) if crawl.crawled_at > 0 => {
+                            detail.push_str("  ·  crawled ");
+                            detail.push_str(&ui::ago(crawl.crawled_at, crate::engine::unix_now()));
+                        }
+                        _ => detail.push_str("  ·  not crawled yet"),
+                    }
                     ui.label(ui::muted(detail)).on_hover_text(&share.id);
+                    // Why the last look failed, until one succeeds: a link
+                    // revoked upstream otherwise just stops growing.
+                    if let Some(error) = crawl.and_then(|crawl| crawl.error.as_deref()) {
+                        ui.label(
+                            theme::Role::Caption
+                                .rich(format!("Last crawl failed: {error}"))
+                                .color(theme::danger()),
+                        );
+                    }
                 });
 
                 ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {