diff --git a/android/app/src/main/java/io/narl/protonstream/ui/AppViewModel.kt b/android/app/src/main/java/io/narl/protonstream/ui/AppViewModel.kt index 26f7376..433b6ca 100644 --- a/android/app/src/main/java/io/narl/protonstream/ui/AppViewModel.kt +++ b/android/app/src/main/java/io/narl/protonstream/ui/AppViewModel.kt @@ -297,7 +297,9 @@ class AppViewModel(context: Context, private val workManager: WorkManager) : Vie } } reloadNow() - if (failed.isNotEmpty()) { + // A catch-up is the app's doing, not the viewer's: a failure goes on + // the share's row rather than in their face, and only news is said. + if (failed.isNotEmpty() && announce) { mutableState.update { it.copy(message = failed.joinToString("\n")) } return } @@ -305,13 +307,19 @@ class AppViewModel(context: Context, private val workManager: WorkManager) : Vie if (announce) { val added = (mutableState.value.titles.map { it.key }.toSet() - before).size mutableState.update { it.copy(message = if (added == 1) "1 title added" else "$added titles added") } + } else { + val files = mutableState.value.shares.filter { it.id in ids }.sumOf { it.crawlAdded ?: 0uL } + if (files > 0uL) { + mutableState.update { it.copy(message = if (files == 1uL) "1 new file" else "$files new files") } + } } } /** - * Crawl shares the catalog has nothing of: brought by a sync that ran in - * [io.narl.protonstream.sync.WatchSyncWorker], or an add whose crawl never - * finished. + * Crawl shares due a look: ones the catalog has nothing of — brought by a + * sync that ran in [io.narl.protonstream.sync.WatchSyncWorker], or an add + * whose crawl never finished — and ones not crawled for six hours, so new + * episodes turn up without a pull to refresh. */ fun catchUp() { if (catchingUp) return diff --git a/android/app/src/main/java/io/narl/protonstream/ui/SharesScreen.kt b/android/app/src/main/java/io/narl/protonstream/ui/SharesScreen.kt index aa152e9..1abad13 100644 --- a/android/app/src/main/java/io/narl/protonstream/ui/SharesScreen.kt +++ b/android/app/src/main/java/io/narl/protonstream/ui/SharesScreen.kt @@ -30,6 +30,7 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.input.PasswordVisualTransformation +import androidx.compose.ui.text.style.TextOverflow import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.ui.text.input.KeyboardType import androidx.compose.ui.window.DialogProperties @@ -167,6 +168,25 @@ internal fun SharesScreen( * Three differently styled controls beside the name squeezed it into a column * a word wide; one icon leaves the row its width. */ +/** "120 files · crawled 2 h ago", or that it has not been yet. */ +internal fun crawlLine(share: ShareRecord, now: Long = System.currentTimeMillis() / 1_000): String { + val at = share.crawledAt ?: return "Not crawled yet" + val files = share.crawlFiles?.toLong()?.let { if (it == 1L) "1 file" else "$it files" } + return listOfNotNull(files, "crawled ${ago(at, now)}").joinToString(" · ") +} + +/** "just now", "5 min ago", "3 h ago", "2 days ago" — the desktop's `ui::ago`. */ +internal fun ago(at: Long, now: Long): String { + val seconds = (now - at).coerceAtLeast(0) + return when { + seconds < 60 -> "just now" + seconds < 3_600 -> "${seconds / 60} min ago" + seconds < 86_400 -> "${seconds / 3_600} h ago" + seconds < 172_800 -> "1 day ago" + else -> "${seconds / 86_400} days ago" + } +} + @Composable private fun ShareRow( share: ShareRecord, @@ -178,13 +198,27 @@ private fun ShareRow( ListItem( headlineContent = { Text(share.name, fontWeight = FontWeight.SemiBold) }, supportingContent = { - Text( - when { - share.fromAccount -> "Folder of your Drive" - share.hasCustomPassword -> "Link and custom password, stored securely" - else -> "Public link" - }, - ) + Column { + Text( + when { + share.fromAccount -> "Folder of your Drive" + share.hasCustomPassword -> "Link and custom password" + else -> "Public link" + }, + ) + Text(crawlLine(share), style = MaterialTheme.typography.bodySmall) + // Until a crawl succeeds: a link revoked upstream otherwise + // just stops growing. + share.crawlError?.let { + Text( + "Last crawl failed: $it", + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.bodySmall, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } }, leadingContent = { Icon( diff --git a/android/app/src/test/java/io/narl/protonstream/ui/ScreenshotTest.kt b/android/app/src/test/java/io/narl/protonstream/ui/ScreenshotTest.kt index eb9b83d..d075425 100644 --- a/android/app/src/test/java/io/narl/protonstream/ui/ScreenshotTest.kt +++ b/android/app/src/test/java/io/narl/protonstream/ui/ScreenshotTest.kt @@ -309,9 +309,17 @@ class ScreenshotTest { }, shelves = listOf(ShelfRecord("From MAPPA", listOf("oshi-no-ko", "bleach", "akira"))), ), + // Crawled relative to the clock, so the row reads "2 h ago" on + // whatever day the test runs. shares = listOf( - ShareRecord("anime", "anime", true), - ShareRecord("films", "films", false), + ShareRecord( + "anime", + "anime", + true, + crawledAt = System.currentTimeMillis() / 1_000 - 2 * 3_600, + crawlFiles = 120uL, + ), + ShareRecord("films", "films", false, crawlError = "the link was revoked or has expired"), ), storage = StorageUsageRecord(6_400_000_000uL, 3uL, 0uL, 1_100_000_000uL), ) diff --git a/android/app/src/test/screenshots/shares-phone.png b/android/app/src/test/screenshots/shares-phone.png index be4d525..edc2ffe 100644 Binary files a/android/app/src/test/screenshots/shares-phone.png and b/android/app/src/test/screenshots/shares-phone.png differ diff --git a/crates/pstr-android/src/lib.rs b/crates/pstr-android/src/lib.rs index a39fc2b..b84e53d 100644 --- a/crates/pstr-android/src/lib.rs +++ b/crates/pstr-android/src/lib.rs @@ -140,6 +140,18 @@ pub struct ShareRecord { /// A folder of the signed-in account rather than a public link. #[uniffi(default = false)] pub from_account: bool, + /// When the last crawl here finished, Unix seconds. `None` before one has. + #[uniffi(default = None)] + pub crawled_at: Option, + /// Playable files that crawl stored. + #[uniffi(default = None)] + pub crawl_files: Option, + /// Of those, how many no earlier crawl had seen. + #[uniffi(default = None)] + pub crawl_added: Option, + /// Why the last attempt failed, until one succeeds. + #[uniffi(default = None)] + pub crawl_error: Option, } fn share_record(share: Share) -> ShareRecord { @@ -148,6 +160,10 @@ fn share_record(share: Share) -> ShareRecord { id: share.id, name: share.name, has_custom_password: share.has_custom_password, + crawled_at: None, + crawl_files: None, + crawl_added: None, + crawl_error: None, } } @@ -1259,31 +1275,52 @@ impl AndroidEngine { Ok(share_record(share)) } - /// Configured shares with nothing in the catalog: brought in by sync, - /// possibly while the app was in the background, or added here and never - /// crawled to the end. Crawl each. An empty share is crawled every time, - /// which costs one listing. + /// Configured shares due a crawl: never crawled here — brought in by sync, + /// possibly while the app was in the background, or added and never + /// crawled to the end — or not looked at for + /// [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`], so a share grows + /// without anyone pressing Refresh. Crawl each. pub fn uncrawled_shares(&self) -> Result, BridgeError> { let shares = self.store.list().map_err(BridgeError::from_display)?; - let catalog = self.catalog.lock(); - let mut uncrawled = Vec::new(); - for share in shares { - if catalog - .files(&share.id) - .map_err(BridgeError::from_display)? - .is_empty() - { - uncrawled.push(share.id); - } - } - Ok(uncrawled) + let crawls = self + .catalog + .lock() + .crawls() + .map_err(BridgeError::from_display)?; + let now = now(); + Ok(shares + .into_iter() + .filter(|share| { + crawls.get(&share.id).is_none_or(|record| { + record.is_stale(now, pstr_core::catalog::CRAWL_STALE_AFTER_SECS) + }) + }) + .map(|share| share.id) + .collect()) } + /// Every configured share, with what its last crawl here came to. pub fn shares(&self) -> Result, BridgeError> { - self.store - .list() - .map_err(BridgeError::from_display) - .map(|shares| shares.into_iter().map(share_record).collect()) + let shares = self.store.list().map_err(BridgeError::from_display)?; + let crawls = self + .catalog + .lock() + .crawls() + .map_err(BridgeError::from_display)?; + Ok(shares + .into_iter() + .map(|share| { + let crawl = crawls.get(&share.id).cloned(); + let mut record = share_record(share); + if let Some(crawl) = crawl { + record.crawled_at = (crawl.crawled_at > 0).then_some(crawl.crawled_at); + record.crawl_files = (crawl.crawled_at > 0).then_some(crawl.files); + record.crawl_added = (crawl.crawled_at > 0).then_some(crawl.added); + record.crawl_error = crawl.error; + } + record + }) + .collect()) } /// Add a public link, once it has opened with the password given. A link @@ -2498,11 +2535,21 @@ impl AndroidEngine { .lock() .all_offline_files() .map_err(BridgeError::from_display)?; - let nodes = connection - .library - .crawl(&id) - .await - .map_err(BridgeError::from_display)?; + let nodes = match connection.library.crawl(&id).await { + Ok(nodes) => nodes, + Err(error) => { + // On the share's row until a crawl succeeds: a link revoked + // upstream otherwise just stops growing. + if let Err(error) = self + .catalog + .lock() + .record_crawl_failure(&id, &error.to_string()) + { + log::warn!("record a failed crawl: {error}"); + } + return Err(BridgeError::from_display(error)); + } + }; let rows = build_rows(&id, &nodes); self.catalog .lock() diff --git a/crates/pstr-app/src/app.rs b/crates/pstr-app/src/app.rs index 5dd34b7..cc0f624 100644 --- a/crates/pstr-app/src/app.rs +++ b/crates/pstr-app/src/app.rs @@ -95,6 +95,10 @@ pub enum Page { Player, } +/// How often, in seconds, an open window looks for shares due a crawl. What +/// counts as due is [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`]. +const STALE_CHECK_EVERY: f64 = 30.0 * 60.0; + /// The pages in the top bar, in order. Ctrl+1 to Ctrl+5 go by position here. const NAV: [Page; 5] = [ Page::Library, @@ -530,6 +534,10 @@ pub struct App { pub page: Page, pub library: Library, pub shares: Vec, + /// What the last crawl of each share came to, for its row. + pub crawls: HashMap, + /// When, on egui's clock, to next look for shares due a crawl. + next_stale_check: f64, /// Proton's own per-file thumbnails. pub thumbs: ImageCache, /// Artwork from a metadata provider, keyed by title key. @@ -631,7 +639,9 @@ impl App { engine.load_library(); engine.forget_orphaned_shares(); engine.load_metadata(); + engine.load_crawls(); engine.connect(); + engine.crawl_stale(); let settings = engine.metadata_config(); Ok(Self { @@ -641,6 +651,8 @@ impl App { page: Page::Library, library: Library::default(), shares: Vec::new(), + crawls: HashMap::new(), + next_stale_check: STALE_CHECK_EVERY, thumbs: ImageCache::default(), posters: ImageCache::default(), metadata: HashMap::new(), @@ -761,20 +773,35 @@ impl App { share_id, nodes, files, + added, seconds, + quiet, } => { let name = self.share_name(&share_id); self.crawl_progress.remove(&name); - self.note( - ctx, - format!("{name}: {files} playable of {nodes} nodes in {seconds:.0}s"), - false, - ); - // Only a crawl long enough to have walked away from. - if seconds >= 30.0 { - self.notify(ctx, &format!("Crawled {name}"), &format!("{files} files")); + if quiet { + // Looked at by the app, not the viewer: only news. + if added > 0 { + let text = format!( + "{name}: {}", + ui::library::plural(added as usize, "new file") + ); + self.note(ctx, &text, false); + self.notify(ctx, &format!("New in {name}"), &text); + } + } else { + self.note( + ctx, + format!("{name}: {files} playable of {nodes} nodes in {seconds:.0}s"), + false, + ); + // Only a crawl long enough to have walked away from. + if seconds >= 30.0 { + self.notify(ctx, &format!("Crawled {name}"), &format!("{files} files")); + } } } + Event::Crawls(crawls) => self.crawls = crawls, Event::CrawlFinished => { self.crawling = false; self.crawl_progress.clear(); @@ -1807,6 +1834,15 @@ impl eframe::App for App { let ctx = ui.ctx().clone(); let ctx = &ctx; self.pump(ctx, frame); + // A window left open for days still picks up what was added: the + // launch check alone would only ever run once. + let now = ctx.input(|input| input.time); + if now >= self.next_stale_check { + self.next_stale_check = now + STALE_CHECK_EVERY; + if now > 0.0 && !self.crawling { + self.engine.crawl_stale(); + } + } self.warm_next(); self.auto_skip(ctx); self.window.observe(ctx); @@ -2000,6 +2036,7 @@ impl eframe::App for App { page, library, shares, + crawls, thumbs, posters, metadata, @@ -2107,9 +2144,15 @@ impl eframe::App for App { }, &mut actions, ), - Page::Shares => { - ui::shares::show(ui, shares, library, form, account, &mut actions) - } + Page::Shares => ui::shares::show( + ui, + shares, + crawls, + library, + form, + account, + &mut actions, + ), Page::Settings => { ui::settings::show(ui, settings, prefs, api_key, &mut actions) } diff --git a/crates/pstr-app/src/engine.rs b/crates/pstr-app/src/engine.rs index 9573077..474513d 100644 --- a/crates/pstr-app/src/engine.rs +++ b/crates/pstr-app/src/engine.rs @@ -193,8 +193,15 @@ pub enum Event { share_id: String, nodes: usize, files: usize, + /// Files no earlier crawl had seen. + added: u64, seconds: f64, + /// Started by the app rather than the viewer, so only news is worth + /// saying. + quiet: bool, }, + /// What the last crawl of each share came to, by share id. + Crawls(HashMap), /// A crawl is under way: which share, and how many nodes so far. CrawlProgress { share: String, found: usize }, /// Every requested crawl is done. @@ -402,6 +409,14 @@ const DOWNLOAD_CONCURRENCY: usize = 3; /// launch and whenever a player stops. const SYNC_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5 * 60); +/// Unix seconds, now. +pub fn unix_now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since| since.as_secs() as i64) + .unwrap_or(0) +} + /// Enrichment as currently configured. struct Enrichment { config: MetadataConfig, @@ -663,7 +678,7 @@ impl Engine { // The open connection predates the share, and crawling it through // that would find it "not open". engine.connection.lock().take(); - engine.connect_and_crawl(Some(share.id)).await; + engine.connect_and_crawl(Some(vec![share.id]), false).await; }); } @@ -682,7 +697,7 @@ impl Engine { engine.emit(Event::ShareAdded(share.name.clone())); engine.load_shares(); engine.connection.lock().take(); - engine.connect_and_crawl(Some(share.id)).await; + engine.connect_and_crawl(Some(vec![share.id]), false).await; }); } @@ -944,7 +959,7 @@ impl Engine { self.load_shares(); self.load_library(); for id in profile.shares_added.iter().chain(&profile.shares_changed) { - self.connect_and_crawl(Some(id.clone())).await; + self.connect_and_crawl(Some(vec![id.clone()]), false).await; } } @@ -1277,18 +1292,65 @@ impl Engine { pub fn crawl(&self, share: Option) { let engine = self.clone(); self.runtime.spawn(async move { - engine.connect_and_crawl(share).await; + engine + .connect_and_crawl(share.map(|id| vec![id]), false) + .await; }); } - async fn connect_and_crawl(&self, share: Option) { + /// Crawl, quietly, every share no crawl has looked at for + /// [`pstr_core::catalog::CRAWL_STALE_AFTER_SECS`] — or ever. + /// + /// Without this a share only grew when someone thought to press Refresh, + /// so the episode that came out yesterday was not in the library until + /// then. Quiet: a failure is logged and recorded on the share's row rather + /// than toasted, and a crawl that found nothing new says nothing. + pub fn crawl_stale(&self) { + let engine = self.clone(); + self.runtime.spawn(async move { + let stale = { + let crawls = match engine.catalog.lock().crawls() { + Ok(crawls) => crawls, + Err(error) => return tracing::warn!(%error, "read crawl records"), + }; + let Ok(shares) = engine.store.list() else { + return; + }; + let now = unix_now(); + shares + .into_iter() + .filter(|share| { + crawls.get(&share.id).is_none_or(|record| { + record.is_stale(now, pstr_core::catalog::CRAWL_STALE_AFTER_SECS) + }) + }) + .map(|share| share.id) + .collect::>() + }; + if !stale.is_empty() { + engine.connect_and_crawl(Some(stale), true).await; + } + }); + } + + /// Read what the last crawl of each share came to, for the shares page. + pub fn load_crawls(&self) { + let engine = self.clone(); + self.runtime + .spawn_blocking(move || match engine.catalog.lock().crawls() { + Ok(crawls) => engine.emit(Event::Crawls(crawls)), + Err(error) => tracing::warn!(%error, "read crawl records"), + }); + } + + async fn connect_and_crawl(&self, shares: Option>, quiet: bool) { let Some((library, _)) = self.ensure_connected().await else { self.emit(Event::CrawlFinished); return; }; - let targets: Vec = match share { - Some(id) => vec![id], + let targets: Vec = match shares { + Some(ids) => ids, None => library.share_ids().map(str::to_string).collect(), }; @@ -1298,7 +1360,7 @@ impl Engine { futures::future::join_all( targets .into_iter() - .map(|share_id| self.crawl_share(&library, share_id, cancel.clone())), + .map(|share_id| self.crawl_share(&library, share_id, cancel.clone(), quiet)), ) .await; self.emit(Event::CrawlFinished); @@ -1342,6 +1404,7 @@ impl Engine { library: &SharedLibrary, share_id: String, mut cancel: tokio::sync::watch::Receiver, + quiet: bool, ) { let started = std::time::Instant::now(); let name = self @@ -1376,7 +1439,19 @@ impl Engine { }) => match result { Ok(nodes) => nodes, Err(error) => { - self.fail(&format!("crawl {share_id}"), error); + if let Err(error) = self + .catalog + .lock() + .record_crawl_failure(&share_id, &error.to_string()) + { + tracing::warn!(%error, "record a failed crawl"); + } + self.load_crawls(); + if quiet { + tracing::warn!(%error, "crawl {name}"); + } else { + self.fail(&format!("crawl {name}"), error); + } return; } }, @@ -1455,12 +1530,22 @@ impl Engine { return; } + let added = self + .catalog + .lock() + .crawls() + .ok() + .and_then(|crawls| crawls.get(&share_id).map(|record| record.added)) + .unwrap_or(0); self.emit(Event::Crawled { share_id, nodes: nodes.len(), files, + added, seconds: started.elapsed().as_secs_f64(), + quiet, }); + self.load_crawls(); self.load_library(); } diff --git a/crates/pstr-app/src/ui/account.rs b/crates/pstr-app/src/ui/account.rs index fd56fc9..7e4d81e 100644 --- a/crates/pstr-app/src/ui/account.rs +++ b/crates/pstr-app/src/ui/account.rs @@ -315,13 +315,7 @@ fn signed_in( /// "Synced 3 min ago · 2 positions from other devices". fn sync_line(at: i64, report: &SyncReport, now: i64) -> String { - let ago = (now - at).max(0); - let when = match ago { - 0..60 => "just now".to_owned(), - 60..3600 => format!("{} min ago", ago / 60), - _ => format!("{} h ago", ago / 3600), - }; - let mut line = format!("Synced {when}"); + let mut line = format!("Synced {}", ui::ago(at, now)); if report.applied > 0 { line.push_str(&format!( " · {} from other devices", diff --git a/crates/pstr-app/src/ui/mod.rs b/crates/pstr-app/src/ui/mod.rs index fb41be1..4588dae 100644 --- a/crates/pstr-app/src/ui/mod.rs +++ b/crates/pstr-app/src/ui/mod.rs @@ -172,6 +172,19 @@ pub fn wide_key(title_key: &str) -> String { format!("wide:{title_key}") } +/// "just now", "5 min ago", "3 h ago", "2 days ago": how long before `now` +/// the Unix time `at` was. +pub fn ago(at: i64, now: i64) -> String { + let ago = (now - at).max(0); + match ago { + 0..60 => "just now".to_owned(), + 60..3600 => format!("{} min ago", ago / 60), + 3600..86_400 => format!("{} h ago", ago / 3600), + 86_400..172_800 => "1 day ago".to_owned(), + _ => format!("{} days ago", ago / 86_400), + } +} + /// `1:03:47`, or `4:12` for anything under an hour. pub fn format_time(seconds: f64) -> String { if !seconds.is_finite() || seconds < 0.0 { diff --git a/crates/pstr-app/src/ui/shares.rs b/crates/pstr-app/src/ui/shares.rs index aadb466..8f1419a 100644 --- a/crates/pstr-app/src/ui/shares.rs +++ b/crates/pstr-app/src/ui/shares.rs @@ -6,7 +6,10 @@ //! below is the only place a link is ever visible, and only while it is being //! typed. +use std::collections::HashMap; + use pstr_core::Share; +use pstr_core::catalog::CrawlRecord; use pstr_core::library::Library; use crate::app::{Action, ShareForm}; @@ -16,6 +19,7 @@ use crate::ui; pub fn show( ui: &mut egui::Ui, shares: &[Share], + crawls: &HashMap, library: &Library, form: &mut ShareForm, account: &mut ui::account::AccountPanel, @@ -36,7 +40,7 @@ pub fn show( .iter() .filter(|title| title.share_ids.contains(&share.id)) .count(); - share_row(ui, share, titles, actions); + share_row(ui, share, titles, crawls.get(&share.id), actions); } ui.add_space(theme::space::XXL); @@ -48,7 +52,13 @@ pub fn show( }); } -fn share_row(ui: &mut egui::Ui, share: &Share, titles: usize, actions: &mut Vec) { +fn share_row( + ui: &mut egui::Ui, + share: &Share, + titles: usize, + crawl: Option<&CrawlRecord>, + actions: &mut Vec, +) { egui::Frame::new() .fill(theme::card()) .corner_radius(egui::CornerRadius::same(theme::radius::MD)) @@ -67,7 +77,23 @@ fn share_row(ui: &mut egui::Ui, share: &Share, titles: usize, actions: &mut Vec< if share.has_custom_password { detail.push_str(" · password protected"); } + match crawl { + Some(crawl) if crawl.crawled_at > 0 => { + detail.push_str(" · crawled "); + detail.push_str(&ui::ago(crawl.crawled_at, crate::engine::unix_now())); + } + _ => detail.push_str(" · not crawled yet"), + } ui.label(ui::muted(detail)).on_hover_text(&share.id); + // Why the last look failed, until one succeeds: a link + // revoked upstream otherwise just stops growing. + if let Some(error) = crawl.and_then(|crawl| crawl.error.as_deref()) { + ui.label( + theme::Role::Caption + .rich(format!("Last crawl failed: {error}")) + .color(theme::danger()), + ); + } }); ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { diff --git a/crates/pstr-core/src/catalog.rs b/crates/pstr-core/src/catalog.rs index 31b8ea9..097521a 100644 --- a/crates/pstr-core/src/catalog.rs +++ b/crates/pstr-core/src/catalog.rs @@ -24,7 +24,7 @@ use crate::metadata::{ use crate::naming::{self, ParsedName}; /// Bump when adding a migration. -pub const SCHEMA_VERSION: i64 = 10; +pub const SCHEMA_VERSION: i64 = 11; const MIGRATION_V1: &str = r#" CREATE TABLE nodes ( @@ -207,6 +207,51 @@ INSERT OR IGNORE INTO node_first_seen (share_id, link_id, seen_at) UPDATE title_metadata SET enriched_at = NULL; "#; +/// What the last crawl of each share came to. +/// +/// Per installation, like the rest of the catalog: when this device last +/// looked at a share is nothing another device needs. `crawled_at` is the last +/// crawl that finished, `attempted_at` the last that was tried, so a share that +/// keeps failing is not retried on every launch. `added` is how many files the +/// last finished crawl found that had never been seen before. +const MIGRATION_V11: &str = r#" +CREATE TABLE share_crawls ( + share_id TEXT PRIMARY KEY, + crawled_at INTEGER NOT NULL, + attempted_at INTEGER NOT NULL, + files INTEGER NOT NULL, + added INTEGER NOT NULL, + error TEXT +); +"#; + +/// How old a share's last crawl may be before an app looks again on its own. +/// Long enough that a launch rarely costs a crawl; short enough that a new +/// episode turns up the same day. +pub const CRAWL_STALE_AFTER_SECS: i64 = 6 * 60 * 60; + +/// One share's row of `share_crawls`. See [`MIGRATION_V11`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CrawlRecord { + /// Unix seconds; `0` for a share no crawl of has finished. + pub crawled_at: i64, + pub attempted_at: i64, + /// Playable files the last finished crawl stored. + pub files: u64, + /// Of those, how many no earlier crawl had seen. + pub added: u64, + /// Why the last attempt failed, when it did. + pub error: Option, +} + +impl CrawlRecord { + /// Whether the share is due another look: nothing has been tried within + /// `age_secs` of `now`. + pub fn is_stale(&self, now: i64, age_secs: i64) -> bool { + now - self.crawled_at.max(self.attempted_at) >= age_secs + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct OfflineFile { pub revision_id: String, @@ -408,6 +453,12 @@ impl Catalog { tx.pragma_update(None, "user_version", 10)?; tx.commit()?; } + if version < 11 { + let tx = self.conn.transaction()?; + tx.execute_batch(MIGRATION_V11)?; + tx.pragma_update(None, "user_version", 11)?; + tx.commit()?; + } Ok(()) } @@ -469,10 +520,23 @@ impl Catalog { } // Only files: a folder appearing is not something anyone watches. - tx.execute( + let now = unix_now(); + let added = tx.execute( "INSERT OR IGNORE INTO node_first_seen (share_id, link_id, seen_at) SELECT share_id, link_id, ?2 FROM nodes WHERE share_id = ?1 AND is_folder = 0", - params![share_id, unix_now()], + params![share_id, now], + )?; + let files = nodes.iter().filter(|node| !node.is_folder).count(); + tx.execute( + "INSERT INTO share_crawls (share_id, crawled_at, attempted_at, files, added, error) + VALUES (?1, ?2, ?2, ?3, ?4, NULL) + ON CONFLICT (share_id) DO UPDATE SET + crawled_at = excluded.crawled_at, + attempted_at = excluded.attempted_at, + files = excluded.files, + added = excluded.added, + error = NULL", + params![share_id, now, files as i64, added as i64], )?; if !retain_offline { @@ -483,6 +547,41 @@ impl Catalog { Ok(()) } + /// Record that a crawl of `share_id` failed, keeping what the last + /// finished one stored. + pub fn record_crawl_failure(&self, share_id: &str, error: &str) -> Result<()> { + self.conn.execute( + "INSERT INTO share_crawls (share_id, crawled_at, attempted_at, files, added, error) + VALUES (?1, 0, ?2, 0, 0, ?3) + ON CONFLICT (share_id) DO UPDATE SET + attempted_at = excluded.attempted_at, + error = excluded.error", + params![share_id, unix_now(), error], + )?; + Ok(()) + } + + /// What the last crawl of each share came to, by share id. A share never + /// crawled has no entry. + pub fn crawls(&self) -> Result> { + let mut statement = self.conn.prepare( + "SELECT share_id, crawled_at, attempted_at, files, added, error FROM share_crawls", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + CrawlRecord { + crawled_at: row.get(1)?, + attempted_at: row.get(2)?, + files: row.get::<_, i64>(3)?.max(0) as u64, + added: row.get::<_, i64>(4)?.max(0) as u64, + error: row.get(5)?, + }, + )) + })?; + Ok(rows.collect::>()?) + } + /// Every file in a share, folders excluded, ordered for display. pub fn files(&self, share_id: &str) -> Result> { let mut statement = self.conn.prepare( @@ -579,6 +678,10 @@ impl Catalog { "DELETE FROM node_first_seen WHERE share_id = ?1", params![share_id], )?; + tx.execute( + "DELETE FROM share_crawls WHERE share_id = ?1", + params![share_id], + )?; tx.commit()?; Ok(()) } @@ -2133,6 +2236,91 @@ mod tests { assert!(catalog.enrichment_ages().expect("ages").is_empty()); } + #[test] + fn a_crawl_records_when_it_finished_and_what_it_added() { + let mut catalog = Catalog::in_memory().expect("open"); + catalog + .replace_share("s", &[file("s", "a", "Show S01E01.mkv")]) + .expect("crawl"); + catalog + .replace_share( + "s", + &[ + file("s", "a", "Show S01E01.mkv"), + file("s", "b", "Show S01E02.mkv"), + ], + ) + .expect("recrawl"); + let record = &catalog.crawls().expect("read")["s"]; + assert!(record.crawled_at > 0); + assert_eq!(record.attempted_at, record.crawled_at); + assert_eq!((record.files, record.added), (2, 1)); + assert_eq!(record.error, None); + } + + /// A failure says why without forgetting what the last good crawl found, + /// and the next good crawl clears it. + #[test] + fn a_failed_crawl_keeps_the_last_good_one_until_the_next() { + let mut catalog = Catalog::in_memory().expect("open"); + catalog + .replace_share("s", &[file("s", "a", "Show S01E01.mkv")]) + .expect("crawl"); + catalog + .conn + .execute( + "UPDATE share_crawls SET crawled_at = 5, attempted_at = 5", + [], + ) + .expect("age it"); + catalog + .record_crawl_failure("s", "link expired") + .expect("fail"); + let record = catalog.crawls().expect("read")["s"].clone(); + assert_eq!((record.crawled_at, record.files), (5, 1)); + assert!(record.attempted_at > 5); + assert_eq!(record.error.as_deref(), Some("link expired")); + assert!(!record.is_stale(record.attempted_at + 10, 60)); + + catalog + .replace_share("s", &[file("s", "a", "Show S01E01.mkv")]) + .expect("recrawl"); + assert_eq!(catalog.crawls().expect("read")["s"].error, None); + } + + #[test] + fn a_removed_share_forgets_its_crawls() { + let mut catalog = Catalog::in_memory().expect("open"); + catalog + .replace_share("s", &[file("s", "a", "Show S01E01.mkv")]) + .expect("crawl"); + catalog.remove_share("s").expect("remove"); + assert!(catalog.crawls().expect("read").is_empty()); + } + + #[test] + fn migration_v11_opens_a_v10_catalog() { + let conn = Connection::open_in_memory().expect("open"); + for migration in [ + MIGRATION_V1, + MIGRATION_V2, + MIGRATION_V3, + MIGRATION_V4, + MIGRATION_V5, + MIGRATION_V6, + MIGRATION_V7, + MIGRATION_V8, + MIGRATION_V9, + MIGRATION_V10, + ] { + conn.execute_batch(migration).expect("migrate"); + } + conn.pragma_update(None, "user_version", 10) + .expect("set version"); + let catalog = Catalog::from_connection(conn).expect("upgrade"); + assert!(catalog.crawls().expect("read").is_empty()); + } + /// A file keeps the moment it first appeared across recrawls, and one /// that was there before this was recorded reads as 0. #[test] diff --git a/ios/ProtonStream/App/AppModel.swift b/ios/ProtonStream/App/AppModel.swift index b01b9d3..a48fbb8 100644 --- a/ios/ProtonStream/App/AppModel.swift +++ b/ios/ProtonStream/App/AppModel.swift @@ -242,7 +242,9 @@ final class AppModel { } } await reloadNow() - if !failed.isEmpty { + // A catch-up is the app's doing, not the viewer's: a failure goes on + // the share's row rather than in their face, and only news is said. + if !failed.isEmpty, announce { message = failed.joined(separator: "\n") return } @@ -250,6 +252,9 @@ final class AppModel { if announce { let added = Set(titles.map(\.key)).subtracting(before).count message = added == 1 ? "1 title added" : "\(added) titles added" + } else { + let files = shares.filter { ids.contains($0.id) }.reduce(UInt64(0)) { $0 + ($1.crawlAdded ?? 0) } + if files > 0 { message = files == 1 ? "1 new file" : "\(files) new files" } } } @@ -430,9 +435,10 @@ final class AppModel { } } - /// On launch and back on screen: crawl shares the catalog has nothing of — - /// brought by a sync that ran in the background, or an add whose crawl - /// never finished. + /// On launch and back on screen: crawl shares due a look — ones the + /// catalog has nothing of, brought by a sync that ran in the background or + /// an add whose crawl never finished, and ones not crawled for six hours, + /// so new episodes turn up without a pull to refresh. func enteredForeground() { guard !catchingUp else { return } catchingUp = true diff --git a/ios/ProtonStream/UI/SharesView.swift b/ios/ProtonStream/UI/SharesView.swift index 2453c6e..064909b 100644 --- a/ios/ProtonStream/UI/SharesView.swift +++ b/ios/ProtonStream/UI/SharesView.swift @@ -97,6 +97,25 @@ struct SharesView: View { extension ShareRecord: Identifiable {} /// One share: its name, how it is unlocked, and its actions behind a menu. +/// "120 files · crawled 2 h ago", or that it has not been yet. +func crawlLine(_ share: ShareRecord, now: Int64 = Int64(Date.now.timeIntervalSince1970)) -> String { + guard let at = share.crawledAt else { return "Not crawled yet" } + let files = share.crawlFiles.map { $0 == 1 ? "1 file" : "\($0) files" } + return [files, "crawled \(ago(at, now: now))"].compactMap { $0 }.joined(separator: " · ") +} + +/// "just now", "5 min ago", "3 h ago", "2 days ago" — the desktop's `ui::ago`. +func ago(_ at: Int64, now: Int64) -> String { + let seconds = max(now - at, 0) + switch seconds { + case ..<60: return "just now" + case ..<3_600: return "\(seconds / 60) min ago" + case ..<86_400: return "\(seconds / 3_600) h ago" + case ..<172_800: return "1 day ago" + default: return "\(seconds / 86_400) days ago" + } +} + private struct ShareRow: View { let share: ShareRecord let onRefresh: () -> Void @@ -110,8 +129,14 @@ private struct ShareRow: View { VStack(alignment: .leading, spacing: 2) { Text(share.name).textStyle(.bodyLarge, semibold: true).foregroundStyle(scheme.onSurface) Text(share.fromAccount ? "Folder of your Drive" - : share.hasCustomPassword ? "Link and custom password, stored securely" : "Public link") + : share.hasCustomPassword ? "Link and custom password" : "Public link") .textStyle(.bodyMedium).foregroundStyle(scheme.onSurfaceVariant) + Text(crawlLine(share)).textStyle(.bodySmall).foregroundStyle(scheme.onSurfaceVariant) + // Until a crawl succeeds: a link revoked upstream otherwise + // just stops growing. + if let error = share.crawlError { + Text("Last crawl failed: \(error)").textStyle(.bodySmall).foregroundStyle(scheme.error).lineLimit(2) + } } Spacer(minLength: 0) Menu {