mirror of
https://github.com/narrrl/proton-stream.git
synced 2026-11-04 00:47:00 +01:00
feature(gui) themes + chore(deps) version bump
This commit is contained in:
1 parent
4e6778e1ea
commit
ad24557588
18 files changed
+782
-92
No files matched your search
@@ -152,6 +152,7 @@ pub enum FlavorChoice {
|
||||
Frappe,
|
||||
Macchiato,
|
||||
Mocha,
|
||||
Persona5,
|
||||
}
|
||||
|
||||
/// The one strong colour. The shared `pstr_core::appearance::Accent`.
|
||||
@@ -165,6 +166,7 @@ pub enum AccentChoice {
|
||||
Blue,
|
||||
Teal,
|
||||
Peach,
|
||||
Red,
|
||||
}
|
||||
|
||||
/// What the viewer chose, stored where the desktop client reads it too.
|
||||
@@ -2348,6 +2350,7 @@ fn appearance_record(appearance: Appearance) -> AppearanceRecord {
|
||||
Flavor::Frappe => FlavorChoice::Frappe,
|
||||
Flavor::Macchiato => FlavorChoice::Macchiato,
|
||||
Flavor::Mocha => FlavorChoice::Mocha,
|
||||
Flavor::Persona5 => FlavorChoice::Persona5,
|
||||
},
|
||||
accent: match appearance.accent {
|
||||
Accent::Mauve => AccentChoice::Mauve,
|
||||
@@ -2358,6 +2361,7 @@ fn appearance_record(appearance: Appearance) -> AppearanceRecord {
|
||||
Accent::Blue => AccentChoice::Blue,
|
||||
Accent::Teal => AccentChoice::Teal,
|
||||
Accent::Peach => AccentChoice::Peach,
|
||||
Accent::Red => AccentChoice::Red,
|
||||
},
|
||||
gradients: appearance.gradients,
|
||||
}
|
||||
@@ -2371,6 +2375,7 @@ fn appearance_choice(record: AppearanceRecord) -> Appearance {
|
||||
FlavorChoice::Frappe => Flavor::Frappe,
|
||||
FlavorChoice::Macchiato => Flavor::Macchiato,
|
||||
FlavorChoice::Mocha => Flavor::Mocha,
|
||||
FlavorChoice::Persona5 => Flavor::Persona5,
|
||||
},
|
||||
accent: match record.accent {
|
||||
AccentChoice::Mauve => Accent::Mauve,
|
||||
@@ -2381,6 +2386,7 @@ fn appearance_choice(record: AppearanceRecord) -> Appearance {
|
||||
AccentChoice::Blue => Accent::Blue,
|
||||
AccentChoice::Teal => Accent::Teal,
|
||||
AccentChoice::Peach => Accent::Peach,
|
||||
AccentChoice::Red => Accent::Red,
|
||||
},
|
||||
gradients: record.gradients,
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ use pstr_core::metadata::{EpisodeGuide, MetadataConfig, MetadataRecord};
|
||||
use crate::engine::{
|
||||
DownloadItem, DownloadKey, Engine, Event, ImageCache, describe_failures, watch_state,
|
||||
};
|
||||
use crate::pacing::Pacer;
|
||||
use crate::playback::{Playback, PlaybackTarget};
|
||||
use crate::ui::player::UpNextCard;
|
||||
use crate::{theme, ui};
|
||||
@@ -297,6 +298,9 @@ pub struct App {
|
||||
pub offline_files: std::collections::HashSet<DownloadKey>,
|
||||
pub confirm_partial_delete: Option<DownloadKey>,
|
||||
pub status: Option<Status>,
|
||||
/// The frame ceiling, where this session is pacing itself rather than
|
||||
/// letting vsync do it — see [`crate::pacing`].
|
||||
pacer: Pacer,
|
||||
}
|
||||
|
||||
impl App {
|
||||
@@ -347,6 +351,7 @@ impl App {
|
||||
offline_files: std::collections::HashSet::new(),
|
||||
confirm_partial_delete: None,
|
||||
status: None,
|
||||
pacer: Pacer::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -973,6 +978,10 @@ impl App {
|
||||
|
||||
impl eframe::App for App {
|
||||
fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) {
|
||||
// Before the timer: this is the frame budget being spent waiting, not
|
||||
// the frame taking longer, and `FrameTimer` reports slow frames.
|
||||
self.pacer.wait();
|
||||
|
||||
let _timer = FrameTimer::new(&self.page);
|
||||
let ctx = ui.ctx().clone();
|
||||
let ctx = &ctx;
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
|
||||
pub mod app;
|
||||
pub mod engine;
|
||||
pub mod pacing;
|
||||
pub mod playback;
|
||||
pub mod theme;
|
||||
pub mod ui;
|
||||
|
||||
@@ -31,7 +31,7 @@ fn main() -> anyhow::Result<()> {
|
||||
);
|
||||
let dirs = AppDirs::ensure().context("resolve app directories")?;
|
||||
|
||||
let options = eframe::NativeOptions {
|
||||
let mut options = eframe::NativeOptions {
|
||||
viewport: egui::ViewportBuilder::default()
|
||||
.with_title("proton-stream")
|
||||
.with_inner_size([1180.0, 780.0])
|
||||
@@ -39,6 +39,10 @@ fn main() -> anyhow::Result<()> {
|
||||
.with_app_id("io.narl.proton-stream"),
|
||||
..Default::default()
|
||||
};
|
||||
// Off on Wayland, where a swap on a window nobody is looking at never
|
||||
// returns and the compositor decides the app has hung. `pstr_app::pacing`
|
||||
// has the whole of it, and caps the frame rate in vsync's place.
|
||||
options.glow_options.vsync = pstr_app::pacing::vsync();
|
||||
|
||||
eframe::run_native(
|
||||
"proton-stream",
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! How often the window is allowed to draw — and why it does not simply let
|
||||
//! vsync decide on Wayland.
|
||||
//!
|
||||
//! ## The bug this exists for
|
||||
//!
|
||||
//! A Wayland compositor throttles a client by *withholding* the `wl_surface`
|
||||
//! frame callback, and it withholds it for as long as the surface is not being
|
||||
//! shown — a window on another workspace, or behind a fullscreen one. Mesa's
|
||||
//! EGL implements a swap interval of 1 by waiting on exactly that callback, so
|
||||
//! `eglSwapBuffers` on a hidden surface does not return late, it does not
|
||||
//! return at all:
|
||||
//!
|
||||
//! ```text
|
||||
//! eframe run_ui_and_paint ─▶ glutin swap_buffers ─▶ eglSwapBuffers
|
||||
//! │
|
||||
//! wl_display_dispatch_queue ◀─┘ ppoll, forever
|
||||
//! ```
|
||||
//!
|
||||
//! That is the whole of the UI thread, and the same thread is the one that
|
||||
//! answers `xdg_wm_base.ping`. Hyprland pings a client it has not heard from
|
||||
//! and puts up an "application is not responding" dialog when the pong does not
|
||||
//! come, which is what a viewer sees a few seconds after switching away from a
|
||||
//! playing window. Nothing is actually wrong — coming back releases the frame
|
||||
//! callback and the app carries on — but a dialog offering to kill the player
|
||||
//! is not an acceptable way to say "this window is not visible".
|
||||
//!
|
||||
//! winit cannot rescue this for us: `WindowEvent::Occluded` is X11, macOS and
|
||||
//! web only — its Wayland implementation was deliberately removed in 0.29 — so
|
||||
//! eframe never learns the window is hidden and keeps painting it.
|
||||
//!
|
||||
//! ## What is done instead
|
||||
//!
|
||||
//! On Wayland the swap interval is set to 0 ([`vsync`]), which makes
|
||||
//! `eglSwapBuffers` return immediately whether the surface is being shown or
|
||||
//! not, and the frame rate is capped here instead ([`Pacer`]). Nothing is lost
|
||||
//! by dropping vsync there: a Wayland compositor composites from the last
|
||||
//! buffer a client committed, at its own refresh, so a client that commits too
|
||||
//! often wastes work but cannot tear. X11 and Windows keep vsync, where a
|
||||
//! client really does drive the scanout and hidden windows do not block.
|
||||
//!
|
||||
//! The cap is a ceiling and not a rate: frames are drawn when something asks
|
||||
//! for one — a new mpv frame, an animation, a keystroke — so an idle window
|
||||
//! still draws nothing, and a 24 fps film still draws 24 times a second.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// The default ceiling, in frames a second.
|
||||
///
|
||||
/// Above the rate of any video this plays and of every animation in the app.
|
||||
/// It is deliberately not the display's refresh rate, which neither eframe nor
|
||||
/// egui gives us — a window on a 144 Hz panel therefore animates at 60 while
|
||||
/// vsync is off. [`CAP_VAR`] is the way out for someone who minds.
|
||||
const DEFAULT_CAP: f32 = 60.0;
|
||||
|
||||
/// Overrides [`DEFAULT_CAP`]. `0` removes the ceiling altogether.
|
||||
const CAP_VAR: &str = "PSTR_FRAME_CAP";
|
||||
|
||||
/// Whether this process is talking to a Wayland compositor, decided the way
|
||||
/// winit decides it: either variable, non-empty, Wayland first.
|
||||
pub fn on_wayland() -> bool {
|
||||
["WAYLAND_DISPLAY", "WAYLAND_SOCKET"]
|
||||
.iter()
|
||||
.any(|name| std::env::var_os(name).is_some_and(|value| !value.is_empty()))
|
||||
}
|
||||
|
||||
/// Whether to ask for a swap interval of 1. False on Wayland — see the module
|
||||
/// documentation.
|
||||
pub fn vsync() -> bool {
|
||||
!on_wayland()
|
||||
}
|
||||
|
||||
/// The ceiling to pace to, or `None` when vsync is doing the pacing.
|
||||
fn cap() -> Option<f32> {
|
||||
if vsync() {
|
||||
return None;
|
||||
}
|
||||
let Ok(value) = std::env::var(CAP_VAR) else {
|
||||
return Some(DEFAULT_CAP);
|
||||
};
|
||||
// Not a number, or a negative one, is a typo rather than a request to
|
||||
// free-run — free-running is what `0` says, explicitly.
|
||||
let parsed = value
|
||||
.trim()
|
||||
.parse::<f32>()
|
||||
.ok()
|
||||
.filter(|fps| fps.is_finite() && *fps >= 0.0);
|
||||
match parsed {
|
||||
Some(fps) if fps > 0.0 => Some(fps),
|
||||
Some(_) => None,
|
||||
None => {
|
||||
tracing::warn!("{CAP_VAR}={value:?} is not a frame rate; using {DEFAULT_CAP}");
|
||||
Some(DEFAULT_CAP)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds the draw loop to a maximum rate.
|
||||
///
|
||||
/// Sleeping is the only thing that can do this: a frame is under way by the
|
||||
/// time the app sees it, and every other source of repaints — mpv, egui's own
|
||||
/// animations — asks for the *next* one immediately. The sleep is at most one
|
||||
/// frame's worth, which is far inside the several seconds a compositor waits
|
||||
/// for a pong.
|
||||
pub struct Pacer {
|
||||
interval: Option<Duration>,
|
||||
last: Instant,
|
||||
}
|
||||
|
||||
impl Default for Pacer {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Pacer {
|
||||
/// The pacer this session wants: capped when vsync is off, a no-op when it
|
||||
/// is on.
|
||||
pub fn new() -> Self {
|
||||
Self::with_cap(cap())
|
||||
}
|
||||
|
||||
fn with_cap(fps: Option<f32>) -> Self {
|
||||
Self {
|
||||
interval: fps.map(|fps| Duration::from_secs_f32(1.0 / fps)),
|
||||
last: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this pacer does anything at all.
|
||||
pub fn is_capped(&self) -> bool {
|
||||
self.interval.is_some()
|
||||
}
|
||||
|
||||
/// Called once at the top of a frame: sleeps out whatever is left of the
|
||||
/// last one's budget.
|
||||
pub fn wait(&mut self) {
|
||||
let Some(interval) = self.interval else {
|
||||
return;
|
||||
};
|
||||
let elapsed = self.last.elapsed();
|
||||
if let Some(left) = interval.checked_sub(elapsed) {
|
||||
std::thread::sleep(left);
|
||||
}
|
||||
self.last = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_uncapped_pacer_never_sleeps() {
|
||||
let mut pacer = Pacer::with_cap(None);
|
||||
assert!(!pacer.is_capped());
|
||||
let before = Instant::now();
|
||||
pacer.wait();
|
||||
pacer.wait();
|
||||
assert!(before.elapsed() < Duration::from_millis(50));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_capped_pacer_holds_the_second_frame_back() {
|
||||
let mut pacer = Pacer::with_cap(Some(50.0));
|
||||
assert!(pacer.is_capped());
|
||||
pacer.wait();
|
||||
let before = Instant::now();
|
||||
pacer.wait();
|
||||
// 20 ms apart, less whatever the first call spent. Sleeping is only
|
||||
// ever long, so the lower bound is the one worth asserting on.
|
||||
assert!(
|
||||
before.elapsed() >= Duration::from_millis(10),
|
||||
"second frame was not held back: {:?}",
|
||||
before.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_frame_that_took_longer_than_the_budget_is_not_delayed_further() {
|
||||
let mut pacer = Pacer::with_cap(Some(1000.0));
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
let before = Instant::now();
|
||||
pacer.wait();
|
||||
assert!(before.elapsed() < Duration::from_millis(2));
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,8 @@
|
||||
use crate::config::{AppDirs, read_json, write_json};
|
||||
use crate::error::Result;
|
||||
|
||||
/// A palette family: Catppuccin's four, plus the one this app shipped with.
|
||||
/// A palette family: Catppuccin's four, the one this app shipped with, and one
|
||||
/// borrowed from a menu screen.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum Flavor {
|
||||
@@ -22,15 +23,18 @@ pub enum Flavor {
|
||||
Frappe,
|
||||
Macchiato,
|
||||
Mocha,
|
||||
/// Black, white and one loud red — the Persona 5 menus.
|
||||
Persona5,
|
||||
}
|
||||
|
||||
impl Flavor {
|
||||
pub const ALL: [Self; 5] = [
|
||||
pub const ALL: [Self; 6] = [
|
||||
Self::Proton,
|
||||
Self::Mocha,
|
||||
Self::Macchiato,
|
||||
Self::Frappe,
|
||||
Self::Latte,
|
||||
Self::Persona5,
|
||||
];
|
||||
|
||||
pub fn label(self) -> &'static str {
|
||||
@@ -40,6 +44,7 @@ impl Flavor {
|
||||
Self::Frappe => "Catppuccin Frappé",
|
||||
Self::Macchiato => "Catppuccin Macchiato",
|
||||
Self::Mocha => "Catppuccin Mocha",
|
||||
Self::Persona5 => "Persona 5",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +55,7 @@ impl Flavor {
|
||||
Self::Frappe => "Dark, warm, the lowest contrast of the three",
|
||||
Self::Macchiato => "Dark, between Frappé and Mocha",
|
||||
Self::Mocha => "Dark, the deepest of Catppuccin's three",
|
||||
Self::Persona5 => "Black and white with one loud red — pick the Red accent for it",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,10 +88,15 @@ pub enum Accent {
|
||||
Blue,
|
||||
Teal,
|
||||
Peach,
|
||||
/// The flavour's warning colour, used on purpose. Every flavour has one
|
||||
/// because everything here needs a colour for a failure, and in
|
||||
/// [`Flavor::Persona5`] it is the point of the palette rather than an
|
||||
/// afterthought.
|
||||
Red,
|
||||
}
|
||||
|
||||
impl Accent {
|
||||
pub const ALL: [Self; 8] = [
|
||||
pub const ALL: [Self; 9] = [
|
||||
Self::Mauve,
|
||||
Self::Pink,
|
||||
Self::Sky,
|
||||
@@ -94,6 +105,7 @@ impl Accent {
|
||||
Self::Blue,
|
||||
Self::Teal,
|
||||
Self::Peach,
|
||||
Self::Red,
|
||||
];
|
||||
|
||||
pub fn label(self) -> &'static str {
|
||||
@@ -106,6 +118,7 @@ impl Accent {
|
||||
Self::Blue => "Blue",
|
||||
Self::Teal => "Teal",
|
||||
Self::Peach => "Peach",
|
||||
Self::Red => "Red",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -423,6 +436,36 @@ const MOCHA: Ramp = Ramp {
|
||||
light: false,
|
||||
};
|
||||
|
||||
/// Persona 5's menus: black, white, and a red that is not asking permission.
|
||||
///
|
||||
/// The three colours that matter are `crust`/`base` (black), `text` (white) and
|
||||
/// `red` — everything between them is that same red bled into the black, which
|
||||
/// is what keeps a card from reading as grey furniture in a palette that has no
|
||||
/// grey in it. The other hues exist because an accent can be any of them; they
|
||||
/// are pushed to full saturation to sit on a black page, which is the one thing
|
||||
/// this palette has in common with [`PROTON`].
|
||||
const PERSONA5: Ramp = Ramp {
|
||||
base: rgb(0x0a0708),
|
||||
mantle: rgb(0x140d0f),
|
||||
crust: rgb(0x000000),
|
||||
surface0: rgb(0x1d1215),
|
||||
surface1: rgb(0x33191d),
|
||||
subtext0: rgb(0xb59a9d),
|
||||
text: rgb(0xf5f2f2),
|
||||
pink: rgb(0xff2e63),
|
||||
mauve: rgb(0xb13bd6),
|
||||
sky: rgb(0x2ec7ff),
|
||||
sapphire: rgb(0x1f9bd0),
|
||||
blue: rgb(0x2f5ee0),
|
||||
lavender: rgb(0xa07bff),
|
||||
teal: rgb(0x12c2a0),
|
||||
green: rgb(0x2fbf4f),
|
||||
peach: rgb(0xff7a1a),
|
||||
yellow: rgb(0xffc400),
|
||||
red: rgb(0xe60012),
|
||||
light: false,
|
||||
};
|
||||
|
||||
impl Ramp {
|
||||
const fn of(flavor: Flavor) -> Self {
|
||||
match flavor {
|
||||
@@ -431,6 +474,7 @@ impl Ramp {
|
||||
Flavor::Frappe => FRAPPE,
|
||||
Flavor::Macchiato => MACCHIATO,
|
||||
Flavor::Mocha => MOCHA,
|
||||
Flavor::Persona5 => PERSONA5,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,6 +494,7 @@ impl Ramp {
|
||||
Accent::Blue => (self.blue, self.sapphire),
|
||||
Accent::Teal => (self.teal, self.green),
|
||||
Accent::Peach => (self.peach, self.yellow),
|
||||
Accent::Red => (self.red, self.pink),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -630,6 +675,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persona_5_is_black_with_that_red_in_it() {
|
||||
let palette = Palette::resolve(Appearance {
|
||||
flavor: Flavor::Persona5,
|
||||
accent: Accent::Red,
|
||||
gradients: false,
|
||||
});
|
||||
assert_eq!(palette.accent, PERSONA5.red);
|
||||
assert!(!palette.light);
|
||||
// Black enough that the artwork is the brightest thing on the page:
|
||||
// darker than the palette this app shipped with, which is the other
|
||||
// near-black one.
|
||||
assert!(luminance(palette.background) < luminance(PROTON.base));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_flavour_can_wear_its_own_warning_colour() {
|
||||
for flavor in Flavor::ALL {
|
||||
let palette = Palette::resolve(Appearance {
|
||||
flavor,
|
||||
accent: Accent::Red,
|
||||
gradients: true,
|
||||
});
|
||||
// The accent is the flavour's red — deepened on the light flavour,
|
||||
// where it would otherwise be too bright to put a label on.
|
||||
assert!(palette.accent.r > palette.accent.b, "{flavor:?}");
|
||||
assert_ne!(palette.accent, palette.background, "{flavor:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turning_gradients_off_leaves_one_colour_to_draw() {
|
||||
let flat = Palette::resolve(Appearance {
|
||||
|
||||
@@ -337,43 +337,77 @@ impl SharedLibrary {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Walk one share's whole subtree, depth first, yielding every node.
|
||||
/// Walk one share's whole subtree, level by level, yielding every node.
|
||||
///
|
||||
/// There is no recursion helper on the visitor path — the authenticated
|
||||
/// client's `get_node_hierarchy` / `get_node_by_path` have no counterpart
|
||||
/// there — so the walk is here.
|
||||
///
|
||||
/// **Breadth first and fanned out, because a media share is wide.** The
|
||||
/// shape of one is a root holding a folder per series, each holding a folder
|
||||
/// per season: a couple of hundred folders that are each one listing request
|
||||
/// plus a detail fetch, and almost all of them siblings. Walking that one
|
||||
/// folder at a time spends the whole crawl waiting on a round trip it could
|
||||
/// have overlapped — the crawl was minutes of latency and seconds of work.
|
||||
/// Taking a whole level at once is what collapses it.
|
||||
///
|
||||
/// The client is cheap to clone and its clones share one node-key cache and
|
||||
/// one single-flight map, so siblings that need the same ancestor key wait
|
||||
/// on one derivation rather than each starting their own. `buffered` rather
|
||||
/// than `buffer_unordered`: the crawl's output order is otherwise a race,
|
||||
/// and a catalog that reorders itself between two identical crawls is
|
||||
/// needlessly hard to diff.
|
||||
pub async fn crawl(&self, share_id: &str) -> Result<Vec<Node>> {
|
||||
use futures::stream::{StreamExt as _, TryStreamExt as _};
|
||||
|
||||
let client = self
|
||||
.client(share_id)
|
||||
.ok_or_else(|| Error::NotFound(format!("share {share_id} is not open")))?;
|
||||
|
||||
let root = client.get_root_node().await?;
|
||||
let mut found = Vec::new();
|
||||
let mut folders = vec![root.uid.clone()];
|
||||
found.push(root);
|
||||
let mut level = vec![root.uid.clone()];
|
||||
let mut found = vec![root];
|
||||
|
||||
while let Some(folder) = folders.pop() {
|
||||
let child_uids: Vec<NodeUid> =
|
||||
client.enumerate_folder_children_node_uids(&folder).await?;
|
||||
if child_uids.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Chunked and fanned out inside the SDK, and the node keys it
|
||||
// unlocks along the way are cached there, so a deep tree does not
|
||||
// re-derive the same ancestors per child.
|
||||
let children = client.enumerate_nodes(&child_uids).await?;
|
||||
for child in children {
|
||||
if child.is_folder() {
|
||||
folders.push(child.uid.clone());
|
||||
while !level.is_empty() {
|
||||
let listings = level.into_iter().map(|folder| {
|
||||
let client = client.clone();
|
||||
async move {
|
||||
let child_uids: Vec<NodeUid> =
|
||||
client.enumerate_folder_children_node_uids(&folder).await?;
|
||||
if child_uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
// Chunked and fanned out inside the SDK too, so this is a
|
||||
// multiplier on that concurrency rather than the only one.
|
||||
client.enumerate_nodes(&child_uids).await
|
||||
}
|
||||
});
|
||||
|
||||
let mut listings = futures::stream::iter(listings).buffered(CRAWL_CONCURRENCY);
|
||||
let mut next = Vec::new();
|
||||
while let Some(children) = listings.try_next().await? {
|
||||
for child in children {
|
||||
if child.is_folder() {
|
||||
next.push(child.uid.clone());
|
||||
}
|
||||
found.push(child);
|
||||
}
|
||||
found.push(child);
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
|
||||
Ok(found)
|
||||
}
|
||||
}
|
||||
|
||||
/// How many folders of one level to list at once.
|
||||
///
|
||||
/// Each of these fans out again inside the SDK — up to four detail batches per
|
||||
/// folder — so this is not the number of requests in flight but a multiplier on
|
||||
/// it. Six keeps the ceiling in the low tens: enough to hide the round trip on
|
||||
/// a wide level, well short of a burst a share server would read as abuse.
|
||||
const CRAWL_CONCURRENCY: usize = 6;
|
||||
|
||||
/// The app-version string Proton identifies this client by.
|
||||
///
|
||||
/// The `external-drive-` prefix is **not** decoration: the API parses the part
|
||||
|
||||
@@ -10,6 +10,8 @@ authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
pstr-core.workspace = true
|
||||
# `time` for the rate limiter's sleeps, `sync` for the mutex it paces behind.
|
||||
tokio = { workspace = true, features = ["time", "sync"] }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
@@ -19,4 +21,4 @@ tracing.workspace = true
|
||||
reqwest.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] }
|
||||
@@ -14,11 +14,30 @@ use pstr_core::metadata::{EpisodeMetadata, ProviderId, TitleMetadata};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::limiter::RateLimiter;
|
||||
use crate::matching::{Candidate, Query};
|
||||
use crate::provider::Provider;
|
||||
|
||||
const ENDPOINT: &str = "https://graphql.anilist.co";
|
||||
|
||||
/// What AniList documents as its ceiling: ninety requests a minute.
|
||||
///
|
||||
/// A starting point, not a belief — the API has been running in a degraded mode
|
||||
/// that allows thirty for a long time now, and says so in `X-RateLimit-Limit` on
|
||||
/// every answer. The limiter retunes itself off that header after the first
|
||||
/// request, so the only cost of starting optimistic is one `429` on a run that
|
||||
/// begins during a degraded window.
|
||||
const REQUESTS_PER_MINUTE: u32 = 90;
|
||||
|
||||
/// How many times to sit out a `429` before giving up on one request.
|
||||
///
|
||||
/// Two, because the pause it asks for is the rest of its rate-limit window and
|
||||
/// the limiter holds every other lookup for the same stretch — a third round is
|
||||
/// minutes of a scan spent waiting rather than matching. Giving up returns
|
||||
/// [`Error::RateLimited`], which is a failure and therefore not cached: the
|
||||
/// title stays askable and the next run picks it up.
|
||||
const RETRIES: usize = 2;
|
||||
|
||||
/// How many answers to score. AniList's own relevance ordering is good, and the
|
||||
/// right title is essentially always in the first handful; asking for more costs
|
||||
/// them bandwidth and buys nothing.
|
||||
@@ -66,11 +85,16 @@ query ($id: Int) {
|
||||
|
||||
pub struct AniList {
|
||||
http: reqwest::Client,
|
||||
/// Shared by every lookup in flight — see [`crate::limiter`].
|
||||
limiter: RateLimiter,
|
||||
}
|
||||
|
||||
impl AniList {
|
||||
pub fn new(http: reqwest::Client) -> Self {
|
||||
Self { http }
|
||||
Self {
|
||||
http,
|
||||
limiter: RateLimiter::per_minute(REQUESTS_PER_MINUTE),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,24 +156,40 @@ impl Provider for AniList {
|
||||
}
|
||||
|
||||
impl AniList {
|
||||
/// One GraphQL request, with AniList's two ways of failing — an HTTP status
|
||||
/// and an `errors` array under a 200 — folded into one.
|
||||
/// One GraphQL request, paced, with AniList's two ways of failing — an HTTP
|
||||
/// status and an `errors` array under a 200 — folded into one.
|
||||
///
|
||||
/// A `429` is waited out rather than returned: a scan sends hundreds of
|
||||
/// these, and the request that trips the limit is not the one that was
|
||||
/// wrong. See [`crate::limiter`].
|
||||
async fn query<T: serde::de::DeserializeOwned>(
|
||||
&self,
|
||||
body: serde_json::Value,
|
||||
) -> Result<Option<T>> {
|
||||
let response = self.http.post(ENDPOINT).json(&body).send().await?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
// 429 is the one worth naming: AniList rate-limits by the minute,
|
||||
// and a caller that knows it was throttled can back off rather than
|
||||
// cache a miss for a title that would have matched.
|
||||
return Err(if status.as_u16() == 429 {
|
||||
Error::RateLimited
|
||||
} else {
|
||||
Error::Http(format!("AniList answered {status}"))
|
||||
});
|
||||
}
|
||||
let mut attempts = 0usize;
|
||||
let response = loop {
|
||||
self.limiter.acquire().await;
|
||||
let response = self.http.post(ENDPOINT).json(&body).send().await?;
|
||||
let status = response.status();
|
||||
|
||||
if status.as_u16() == 429 {
|
||||
// The pause holds every other lookup too, so the rest of the
|
||||
// run does not spend the window earning more 429s.
|
||||
let waited = self.limiter.throttled(response.headers()).await;
|
||||
attempts += 1;
|
||||
if attempts > RETRIES {
|
||||
return Err(Error::RateLimited);
|
||||
}
|
||||
tracing::debug!("AniList is throttling; waiting {waited:?}");
|
||||
continue;
|
||||
}
|
||||
|
||||
self.limiter.observe(response.headers()).await;
|
||||
if !status.is_success() {
|
||||
return Err(Error::Http(format!("AniList answered {status}")));
|
||||
}
|
||||
break response;
|
||||
};
|
||||
|
||||
let payload: Response<T> = response.json().await?;
|
||||
if let Some(errors) = payload.errors
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
|
||||
pub mod anilist;
|
||||
pub mod error;
|
||||
pub mod limiter;
|
||||
pub mod matching;
|
||||
pub mod provider;
|
||||
pub mod service;
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
//! Spacing requests out, so a provider does not have to.
|
||||
//!
|
||||
//! Scanning a new library is the one moment this app talks to a metadata
|
||||
//! provider in bulk: every title is one search, a matched series is one more
|
||||
//! request per season, and a hundred-folder share is several hundred requests
|
||||
//! in a burst. AniList allows ninety a minute — thirty while its API is in the
|
||||
//! degraded mode it has been in for a while — and answers everything past that
|
||||
//! with a `429`. Without pacing, the first thirty titles match and the rest of
|
||||
//! the run fails; because a failure is deliberately not cached (see
|
||||
//! [`crate::service`]), the next scan starts over and fails in the same place.
|
||||
//!
|
||||
//! So requests take a slot here first. The rate is not a constant but the
|
||||
//! provider's own: `X-RateLimit-Limit` on any answer retunes it, and a `429`
|
||||
//! parks every caller for as long as `Retry-After` asks. One limiter is shared
|
||||
//! by every lookup in flight — it lives in the provider, the provider lives in
|
||||
//! an `Arc` inside [`crate::MetadataService`], and cloning the service to hand
|
||||
//! it to a worker shares the limiter rather than copying it.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// Never faster than this, whatever a header claims.
|
||||
const MIN_INTERVAL: Duration = Duration::from_millis(20);
|
||||
|
||||
/// Never slower than this. A provider that asks for a longer pause gets it via
|
||||
/// [`RateLimiter::throttled`]; this is only the floor the steady rate decays to,
|
||||
/// so a misread header cannot wedge a scan forever.
|
||||
const MAX_INTERVAL: Duration = Duration::from_secs(5);
|
||||
|
||||
/// How long to honour a `429` that came with no `Retry-After`.
|
||||
const DEFAULT_BACKOFF: Duration = Duration::from_secs(60);
|
||||
|
||||
/// The longest a single throttle may park requests for.
|
||||
///
|
||||
/// A scan is something a person is watching happen. Two minutes of silence is
|
||||
/// worse than a partial answer they can ask to complete later, and the titles
|
||||
/// that did not get through stay unmatched and askable.
|
||||
const MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
|
||||
/// A shared, self-tuning pacer for one provider's API.
|
||||
pub struct RateLimiter {
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
struct State {
|
||||
/// The earliest a request may be sent.
|
||||
next: Instant,
|
||||
/// The spacing between requests.
|
||||
interval: Duration,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
/// A limiter starting at `per_minute` requests a minute.
|
||||
///
|
||||
/// A starting point only: the first answer that carries a limit header
|
||||
/// replaces it with what the provider actually allows right now.
|
||||
pub fn per_minute(per_minute: u32) -> Self {
|
||||
Self::new(interval_for(per_minute.max(1)))
|
||||
}
|
||||
|
||||
fn new(interval: Duration) -> Self {
|
||||
Self {
|
||||
state: Mutex::new(State {
|
||||
next: Instant::now(),
|
||||
interval,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait until a request may be sent, then claim that slot.
|
||||
///
|
||||
/// Re-checks after every sleep rather than reserving a slot up front, so a
|
||||
/// [`throttled`](Self::throttled) that lands while callers are already
|
||||
/// waiting holds them too — which is the whole point of a shared limiter
|
||||
/// during a burst.
|
||||
pub async fn acquire(&self) {
|
||||
loop {
|
||||
let wait = {
|
||||
let mut state = self.state.lock().await;
|
||||
let now = Instant::now();
|
||||
if state.next <= now {
|
||||
state.next = now + state.interval;
|
||||
return;
|
||||
}
|
||||
state.next - now
|
||||
};
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Retune from a provider's own rate-limit headers.
|
||||
pub async fn observe(&self, headers: &reqwest::header::HeaderMap) {
|
||||
let Some(limit) = header_number(headers, "x-ratelimit-limit") else {
|
||||
return;
|
||||
};
|
||||
let interval = interval_for(limit.max(1) as u32);
|
||||
let mut state = self.state.lock().await;
|
||||
if state.interval != interval {
|
||||
tracing::debug!("provider allows {limit}/min; pacing at {interval:?}");
|
||||
state.interval = interval;
|
||||
}
|
||||
}
|
||||
|
||||
/// Park every caller after a `429`.
|
||||
///
|
||||
/// `Retry-After` is seconds, and is the provider telling us exactly how
|
||||
/// long its window has left to run — a shorter guess would just earn
|
||||
/// another `429`.
|
||||
pub async fn throttled(&self, headers: &reqwest::header::HeaderMap) -> Duration {
|
||||
let wait = header_number(headers, "retry-after")
|
||||
.map(|seconds| Duration::from_secs(seconds.max(1)))
|
||||
.unwrap_or(DEFAULT_BACKOFF)
|
||||
.min(MAX_BACKOFF);
|
||||
|
||||
let mut state = self.state.lock().await;
|
||||
let until = Instant::now() + wait;
|
||||
if until > state.next {
|
||||
state.next = until;
|
||||
}
|
||||
wait
|
||||
}
|
||||
}
|
||||
|
||||
/// `90` a minute → one every 667ms.
|
||||
fn interval_for(per_minute: u32) -> Duration {
|
||||
Duration::from_secs_f64(60.0 / f64::from(per_minute)).clamp(MIN_INTERVAL, MAX_INTERVAL)
|
||||
}
|
||||
|
||||
fn header_number(headers: &reqwest::header::HeaderMap, name: &str) -> Option<u64> {
|
||||
headers.get(name)?.to_str().ok()?.trim().parse::<u64>().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn headers(pairs: &[(&str, &str)]) -> reqwest::header::HeaderMap {
|
||||
let mut map = reqwest::header::HeaderMap::new();
|
||||
for (name, value) in pairs {
|
||||
map.insert(
|
||||
reqwest::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
|
||||
value.parse().expect("header value"),
|
||||
);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rate_becomes_the_spacing_it_implies() {
|
||||
assert_eq!(interval_for(60), Duration::from_secs(1));
|
||||
assert_eq!(interval_for(30), Duration::from_secs(2));
|
||||
// Clamped at both ends: a provider claiming one request a minute must
|
||||
// not stall a scan for a minute per title, and one claiming a million
|
||||
// still gets paced.
|
||||
assert_eq!(interval_for(1), MAX_INTERVAL);
|
||||
assert_eq!(interval_for(1_000_000), MIN_INTERVAL);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_first_request_of_a_run_does_not_wait() {
|
||||
let limiter = RateLimiter::per_minute(30);
|
||||
let started = Instant::now();
|
||||
limiter.acquire().await;
|
||||
assert!(started.elapsed() < Duration::from_millis(50));
|
||||
}
|
||||
|
||||
/// The one that matters: two requests in a row are spaced, not sent
|
||||
/// together.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_second_request_waits_out_the_interval() {
|
||||
let limiter = RateLimiter::per_minute(60);
|
||||
limiter.acquire().await;
|
||||
let started = Instant::now();
|
||||
limiter.acquire().await;
|
||||
assert!(started.elapsed() >= Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_429_parks_the_next_request_for_as_long_as_it_asked() {
|
||||
let limiter = RateLimiter::per_minute(600);
|
||||
let waited = limiter.throttled(&headers(&[("retry-after", "12")])).await;
|
||||
assert_eq!(waited, Duration::from_secs(12));
|
||||
|
||||
let started = Instant::now();
|
||||
limiter.acquire().await;
|
||||
assert!(started.elapsed() >= Duration::from_secs(12));
|
||||
}
|
||||
|
||||
/// A throttle with no header still has to park for something, and a
|
||||
/// ridiculous one is capped — a scan is something a person is watching.
|
||||
#[tokio::test]
|
||||
async fn a_429_without_a_header_backs_off_by_a_capped_default() {
|
||||
let limiter = RateLimiter::per_minute(90);
|
||||
assert_eq!(
|
||||
limiter.throttled(&headers(&[])).await,
|
||||
DEFAULT_BACKOFF.min(MAX_BACKOFF)
|
||||
);
|
||||
assert_eq!(
|
||||
limiter
|
||||
.throttled(&headers(&[("retry-after", "86400")]))
|
||||
.await,
|
||||
MAX_BACKOFF
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_provider_s_own_limit_retunes_the_pacing() {
|
||||
let limiter = RateLimiter::per_minute(90);
|
||||
limiter
|
||||
.observe(&headers(&[("x-ratelimit-limit", "30")]))
|
||||
.await;
|
||||
assert_eq!(
|
||||
limiter.state.lock().await.interval,
|
||||
Duration::from_secs(2),
|
||||
"AniList's degraded mode allows 30/min and says so on every answer"
|
||||
);
|
||||
|
||||
// Nonsense, and an absent header, leave the pacing alone.
|
||||
limiter
|
||||
.observe(&headers(&[("x-ratelimit-limit", "soon")]))
|
||||
.await;
|
||||
limiter.observe(&headers(&[])).await;
|
||||
assert_eq!(limiter.state.lock().await.interval, Duration::from_secs(2));
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ use pstr_core::metadata::{EpisodeMetadata, ProviderId, TitleMetadata};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::limiter::RateLimiter;
|
||||
use crate::matching::{Candidate, Query};
|
||||
use crate::provider::Provider;
|
||||
|
||||
@@ -37,13 +38,23 @@ const STILL_SIZE: &str = "w300";
|
||||
/// at here, give or take; `original` is several megabytes of print-resolution
|
||||
/// artwork per title, which is bandwidth spent on pixels no tile can show.
|
||||
const IMAGE_BASE: &str = "https://image.tmdb.org/t/p";
|
||||
|
||||
const POSTER_SIZE: &str = "w500";
|
||||
const BACKDROP_SIZE: &str = "w780";
|
||||
|
||||
/// TMDB's published ceiling is around fifty requests a second and it no longer
|
||||
/// enforces a daily one. This is well under it: the point of pacing here is a
|
||||
/// burst that would look like abuse, not a rate anyone is close to.
|
||||
const REQUESTS_PER_MINUTE: u32 = 600;
|
||||
|
||||
/// As [`crate::anilist`]'s.
|
||||
const RETRIES: usize = 2;
|
||||
|
||||
pub struct Tmdb {
|
||||
http: reqwest::Client,
|
||||
api_key: String,
|
||||
language: String,
|
||||
limiter: RateLimiter,
|
||||
}
|
||||
|
||||
impl Tmdb {
|
||||
@@ -52,6 +63,7 @@ impl Tmdb {
|
||||
http,
|
||||
api_key,
|
||||
language,
|
||||
limiter: RateLimiter::per_minute(REQUESTS_PER_MINUTE),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,28 +84,12 @@ impl Provider for Tmdb {
|
||||
return Err(Error::MissingApiKey(ProviderId::Tmdb));
|
||||
}
|
||||
|
||||
let response = self
|
||||
.http
|
||||
.get(ENDPOINT)
|
||||
.query(&[
|
||||
("api_key", self.api_key.as_str()),
|
||||
("query", query.name.as_str()),
|
||||
("language", self.language.as_str()),
|
||||
("include_adult", "false"),
|
||||
])
|
||||
.send()
|
||||
let payload: SearchResponse = self
|
||||
.get(
|
||||
ENDPOINT,
|
||||
&[("query", query.name.as_str()), ("include_adult", "false")],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(match status.as_u16() {
|
||||
401 => Error::MissingApiKey(ProviderId::Tmdb),
|
||||
429 => Error::RateLimited,
|
||||
_ => Error::Http(format!("TMDB answered {status}")),
|
||||
});
|
||||
}
|
||||
|
||||
let payload: SearchResponse = response.json().await?;
|
||||
Ok(payload
|
||||
.results
|
||||
.into_iter()
|
||||
@@ -172,7 +168,11 @@ impl Provider for Tmdb {
|
||||
}
|
||||
|
||||
impl Tmdb {
|
||||
/// One authenticated GET, with TMDB's failures named.
|
||||
/// One authenticated GET, paced, with TMDB's failures named.
|
||||
///
|
||||
/// TMDB is far more generous than AniList and a library-sized run rarely
|
||||
/// troubles it, but the pacing costs nothing here and the `429` handling is
|
||||
/// the same code — see [`crate::limiter`].
|
||||
async fn get<T: serde::de::DeserializeOwned>(
|
||||
&self,
|
||||
url: &str,
|
||||
@@ -181,23 +181,40 @@ impl Tmdb {
|
||||
if self.api_key.trim().is_empty() {
|
||||
return Err(Error::MissingApiKey(ProviderId::Tmdb));
|
||||
}
|
||||
let mut request = self.http.get(url).query(&[
|
||||
("api_key", self.api_key.as_str()),
|
||||
("language", self.language.as_str()),
|
||||
]);
|
||||
for pair in extra {
|
||||
request = request.query(&[pair]);
|
||||
}
|
||||
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(match status.as_u16() {
|
||||
401 => Error::MissingApiKey(ProviderId::Tmdb),
|
||||
429 => Error::RateLimited,
|
||||
_ => Error::Http(format!("TMDB answered {status}")),
|
||||
});
|
||||
}
|
||||
let mut attempts = 0usize;
|
||||
let response = loop {
|
||||
let mut request = self.http.get(url).query(&[
|
||||
("api_key", self.api_key.as_str()),
|
||||
("language", self.language.as_str()),
|
||||
]);
|
||||
for pair in extra {
|
||||
request = request.query(&[pair]);
|
||||
}
|
||||
|
||||
self.limiter.acquire().await;
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
|
||||
if status.as_u16() == 429 {
|
||||
let waited = self.limiter.throttled(response.headers()).await;
|
||||
attempts += 1;
|
||||
if attempts > RETRIES {
|
||||
return Err(Error::RateLimited);
|
||||
}
|
||||
tracing::debug!("TMDB is throttling; waiting {waited:?}");
|
||||
continue;
|
||||
}
|
||||
|
||||
if !status.is_success() {
|
||||
return Err(match status.as_u16() {
|
||||
401 => Error::MissingApiKey(ProviderId::Tmdb),
|
||||
_ => Error::Http(format!("TMDB answered {status}")),
|
||||
});
|
||||
}
|
||||
break response;
|
||||
};
|
||||
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user