Turn artist radio into a station sheet with reasons on each track

The radio button (r) opens a sheet with the artist, their tags and their closest
neighbours instead of switching the feed at once; tapping a neighbour walks the
artist graph. Starting jumps to the radio's first track, the top-bar chip shows the
station, each radio card says why it is there, and an end card offers the way back
when the radio runs dry.

GET /local/artist feeds the sheet and starts crawling a new artist's neighbourhood
(at most every 10 minutes per artist). Artist-radio tracks carry why: seed, related,
or via the neighbour that leads to them.

backend/src/crawler.rs

@@ -14,7 +14,7 @@
 
 use std::{
     cmp::Reverse,
-    collections::{BinaryHeap, HashSet},
+    collections::{BinaryHeap, HashMap, HashSet},
     sync::{Arc, Mutex},
     time::Duration,
 };
@@ -50,6 +50,8 @@ const IDLE_DELAY: Duration = Duration::from_secs(20); // extra wait between requ
 const MIN_POOL: i64 = 500; // crawl at full pace until the catalogue has this many tracks
 const REFRESH_AFTER: f64 = 86400.0; // re-check an artist's newest page for uploads this often
 const RESEED_EVERY: Duration = Duration::from_secs(600);
+/// An artist radio crawls its artist's neighbourhood at most this often.
+const EXPLORE_EVERY: Duration = Duration::from_secs(600);
 
 #[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
 enum Job {
@@ -73,6 +75,7 @@ pub struct Crawler {
     db: Db,
     queue: Mutex<Queue>,
     wake: Notify,
+    explored: Mutex<HashMap<String, Instant>>,
 }
 
 fn s(v: &Value, k: &str) -> Option<String> {
@@ -86,6 +89,7 @@ impl Crawler {
             db,
             queue: Mutex::new(Queue { heap: BinaryHeap::new(), queued: HashSet::new(), seq: 0 }),
             wake: Notify::new(),
+            explored: Mutex::new(HashMap::new()),
         })
     }
 
@@ -150,6 +154,14 @@ impl Crawler {
     /// An artist radio was opened: crawl the artist, and the related pages of their
     /// best tracks that were not read yet, to fill in the artist graph around them.
     pub async fn explore_artist(&self, handle: String) {
+        {
+            let mut explored = self.explored.lock().unwrap_or_else(|e| e.into_inner());
+            let now = Instant::now();
+            explored.retain(|_, at| now.duration_since(*at) < EXPLORE_EVERY);
+            if explored.insert(handle.clone(), now).is_some() {
+                return;
+            }
+        }
         self.push(P_FEEDBACK, Job::Artist(handle.clone()));
         let h = handle.clone();
         let tracks = self

backend/src/db.rs

@@ -278,6 +278,36 @@ pub fn artist_neighbours(conn: &Connection, handle: &str) -> rusqlite::Result<Ve
     .collect()
 }
 
+/// An artist as seen on their tracks.
+#[derive(Debug, Serialize)]
+pub struct Artist {
+    pub handle: String,
+    pub name: String,
+    pub img: Option<String>,
+    pub verified: bool,
+    pub tracks: i64,
+}
+
+pub fn artist(conn: &Connection, handle: &str) -> rusqlite::Result<Option<Artist>> {
+    conn.prepare_cached(
+        "SELECT handle, artist_name, artist_img, artist_verified, COUNT(*) FROM tracks WHERE handle=? GROUP BY handle",
+    )?
+    .query_row([handle], |r| {
+        Ok(Artist { handle: r.get(0)?, name: r.get(1)?, img: r.get(2)?, verified: r.get(3)?, tracks: r.get(4)? })
+    })
+    .optional()
+}
+
+/// An artist's most common style tags; tempo tags left out.
+pub fn artist_tags(conn: &Connection, handle: &str, n: usize) -> rusqlite::Result<Vec<String>> {
+    conn.prepare_cached(
+        "SELECT g.tag FROM track_tags g JOIN tracks t ON t.id = g.track_id \
+         WHERE t.handle=? AND g.tag NOT GLOB '[0-9]*' GROUP BY g.tag ORDER BY COUNT(*) DESC, g.tag LIMIT ?",
+    )?
+    .query_map(params![handle, n as i64], |r| r.get(0))?
+    .collect()
+}
+
 /// How many tracks each artist has with `tag`.
 pub fn tag_counts(conn: &Connection, tag: &str) -> rusqlite::Result<Vec<(String, i64)>> {
     conn.prepare_cached(

backend/src/http/app.rs

@@ -144,6 +144,47 @@ pub async fn event(State(app): St, Extension(current): Extension<CurrentUser>, J
     Ok((no_store(), Json(json!({ "ok": true }))).into_response())
 }
 
+/// How many neighbours the radio sheet shows, and how few count as still exploring.
+const SHEET_NEIGHBOURS: usize = 12;
+const FEW_NEIGHBOURS: usize = 3;
+
+/// An artist and the ones next to them in the artist graph, for the radio sheet.
+/// Starts crawling the neighbourhood when it is new, so the sheet can poll while
+/// `exploring` is true.
+pub async fn artist(State(app): St, Extension(current): Extension<CurrentUser>, Query(q): Query<HashMap<String, String>>) -> Res {
+    let uid = match current.require() {
+        Ok(u) => u.id,
+        Err(r) => return Ok(r),
+    };
+    let Some(handle) = q.get("handle").filter(|h| valid_handle(h)).cloned() else {
+        return Ok(StatusCode::BAD_REQUEST.into_response());
+    };
+    app.crawler.explore_artist(handle.clone()).await;
+    let body = app
+        .db
+        .call(move |c| {
+            let Some(artist) = db::artist(c, &handle)? else { return Ok(None) };
+            let mut neighbours = vec![];
+            for (h, weight) in ranker::scaled_neighbours(c, &handle)?.into_iter().take(SHEET_NEIGHBOURS) {
+                if let Some(a) = db::artist(c, &h)? {
+                    neighbours.push(json!({ "handle": a.handle, "name": a.name, "img": a.img, "weight": weight }));
+                }
+            }
+            Ok(Some(json!({
+                "followed": db::follows(c, uid)?.contains(&handle),
+                "tags": db::artist_tags(c, &handle, 4)?,
+                "exploring": neighbours.len() < FEW_NEIGHBOURS,
+                "artist": artist,
+                "neighbours": neighbours,
+            })))
+        })
+        .await?;
+    Ok(match body {
+        Some(b) => (no_store(), Json(b)).into_response(),
+        None => json_error(StatusCode::NOT_FOUND, "unknown artist"),
+    })
+}
+
 /// Valid pl8list handles, as seen in every handle crawled so far.
 fn valid_handle(h: &str) -> bool {
     (1..=64).contains(&h.len()) && h.bytes().all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) && h != "." && h != ".."

backend/src/http/mod.rs

@@ -316,6 +316,7 @@ pub fn router(app: Arc<App>) -> Router {
         .route("/local/me", get(app::me))
         .route("/local/feed", get(app::feed))
         .route("/local/likes", get(app::likes))
+        .route("/local/artist", get(app::artist))
         .route("/local/event", post(app::event))
         .route("/local/follow", post(app::follow))
         .route("/account/export", get(account::export))

backend/src/http/tests.rs

@@ -1,6 +1,6 @@
 //! End-to-end tests of the HTTP API against a fake pl8list.
 
-use std::{net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
+use std::{collections::HashMap, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
 
 use axum::{
     Router,
@@ -417,3 +417,49 @@ async fn feed_modes_filter_and_tag_tracks() {
     let res = get("/local/feed?mode=artist&artist=h").await.unwrap();
     assert_eq!(json_body(res).await["tracks"].as_array().unwrap().len(), 1);
 }
+
+#[tokio::test]
+async fn artist_sheet_and_radio_reasons() {
+    let (r, app, _) = setup("radio", "http://127.0.0.1:9").await;
+    let token = app
+        .db
+        .call(|c| {
+            let t = |id, handle: &str, title: &str| crate::db::Track {
+                id,
+                handle: handle.into(),
+                artist_name: handle.to_uppercase(),
+                slug: format!("s{id}"),
+                title: title.into(),
+                bpm: Some(140.0),
+                full_access: true,
+                ..Default::default()
+            };
+            crate::db::upsert_tracks(c, &[t(1, "a", "Riddim one"), t(2, "b", "Two"), t(3, "c", "Three")], None)?;
+            crate::db::add_related(c, 1, &["b".into()])?;
+            crate::db::add_related(c, 2, &["c".into()])?;
+            crate::db::upsert_user(c, 5, "tester", None, None)?;
+            crate::db::create_session(c, 5)
+        })
+        .await
+        .unwrap();
+    let cookie = format!("__Host-pl8inf={token}");
+    let get = |uri: &str| r.clone().oneshot(req("GET", uri, None, Some(&cookie), "203.0.113.1"));
+
+    let res = get("/local/artist?handle=a").await.unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+    let body = json_body(res).await;
+    assert_eq!(body["artist"]["name"], "A");
+    assert_eq!(body["tags"], serde_json::json!(["riddim"]));
+    assert_eq!(body["neighbours"][0]["handle"], "b");
+    assert_eq!(body["exploring"], true);
+    assert_eq!(get("/local/artist?handle=..").await.unwrap().status(), StatusCode::BAD_REQUEST);
+    assert_eq!(get("/local/artist?handle=nobody").await.unwrap().status(), StatusCode::NOT_FOUND);
+
+    let res = get("/local/feed?mode=artist&artist=a").await.unwrap();
+    let body = json_body(res).await;
+    let why: HashMap<i64, serde_json::Value> =
+        body["tracks"].as_array().unwrap().iter().map(|t| (t["id"].as_i64().unwrap(), t["why"].clone())).collect();
+    assert_eq!(why[&1], serde_json::json!({ "kind": "seed" }));
+    assert_eq!(why[&2], serde_json::json!({ "kind": "related" }));
+    assert_eq!(why[&3], serde_json::json!({ "kind": "via", "handle": "b", "name": "B" }));
+}

backend/src/ranker.rs

@@ -48,6 +48,10 @@ pub trait Ranker: Send + Sync {
     fn bonus(&self, _track: &Track) -> f64 {
         0.0
     }
+    /// Why a picked track is in this feed, sent along as `why`.
+    fn reason(&self, _track: &Track) -> Option<Value> {
+        None
+    }
 }
 
 /// The original mix.
@@ -164,8 +168,7 @@ impl Ranker for Fresh {
 
 struct ArtistRadio {
     seed: String,
-    /// Neighbours, 0..1 by how closely they are linked.
-    near: HashMap<String, f64>,
+    near: HashMap<String, Near>,
 }
 
 const RADIO: Weights = Weights { artist: 0.5, related: 0.5, ..CLASSIC };
@@ -179,32 +182,57 @@ impl Ranker for ArtistRadio {
     }
     // The artist themselves now and then; mostly the ones around them.
     fn bonus(&self, t: &Track) -> f64 {
-        if t.handle == self.seed { 1.0 } else { 2.0 * self.near.get(&t.handle).copied().unwrap_or(0.0) }
+        if t.handle == self.seed { 1.0 } else { 2.0 * self.near.get(&t.handle).map_or(0.0, |n| n.score) }
+    }
+    fn reason(&self, t: &Track) -> Option<Value> {
+        if t.handle == self.seed {
+            return Some(json!({ "kind": "seed" }));
+        }
+        Some(match &self.near.get(&t.handle)?.via {
+            None => json!({ "kind": "related" }),
+            Some(via) => json!({ "kind": "via", "handle": via }),
+        })
     }
 }
 
-/// Neighbours of an artist up to two hops away, scored 0..1. A second-hop artist
-/// scores at most 0.4 of the link that leads to it.
-fn neighbourhood(conn: &Connection, seed: &str) -> rusqlite::Result<HashMap<String, f64>> {
-    let scaled = |links: Vec<(String, i64)>| {
-        let max = links.iter().map(|l| l.1).max().unwrap_or(1).max(1) as f64;
-        links.into_iter().map(|(h, n)| (h, (n as f64 / max).sqrt())).collect::<Vec<_>>()
-    };
-    let first = scaled(crate::db::artist_neighbours(conn, seed)?);
-    let mut near: HashMap<String, f64> = first.iter().cloned().collect();
-    let mut by_score = first;
-    by_score.sort_by(|a, b| b.1.total_cmp(&a.1));
-    for (h, w) in by_score.into_iter().take(10) {
-        for (h2, w2) in scaled(crate::db::artist_neighbours(conn, &h)?) {
-            if h2 != seed {
-                let s = near.entry(h2).or_default();
-                *s = s.max(0.4 * w * w2);
+/// An artist near a radio's seed.
+#[derive(Debug, PartialEq)]
+pub struct Near {
+    /// 0..1 by how closely linked.
+    pub score: f64,
+    /// For a second hop: the seed's neighbour that leads here.
+    pub via: Option<String>,
+}
+
+/// Neighbours of an artist up to two hops away. A second-hop artist scores at most
+/// 0.4 of the link that leads to it.
+fn neighbourhood(conn: &Connection, seed: &str) -> rusqlite::Result<HashMap<String, Near>> {
+    let first = scaled_neighbours(conn, seed)?;
+    let mut near: HashMap<String, Near> = first.iter().map(|(h, w)| (h.clone(), Near { score: *w, via: None })).collect();
+    for (h, w) in first.into_iter().take(10) {
+        for (h2, w2) in scaled_neighbours(conn, &h)? {
+            if h2 == seed {
+                continue;
+            }
+            let score = 0.4 * w * w2;
+            let n = near.entry(h2).or_insert(Near { score: 0.0, via: Some(h.clone()) });
+            if n.via.is_some() && score > n.score {
+                *n = Near { score, via: Some(h.clone()) };
             }
         }
     }
     Ok(near)
 }
 
+/// Direct neighbours of an artist, closest first, scored 0..1 against the closest.
+pub fn scaled_neighbours(conn: &Connection, handle: &str) -> rusqlite::Result<Vec<(String, f64)>> {
+    let links = crate::db::artist_neighbours(conn, handle)?;
+    let max = links.iter().map(|l| l.1).max().unwrap_or(1).max(1) as f64;
+    let mut out: Vec<(String, f64)> = links.into_iter().map(|(h, n)| (h, (n as f64 / max).sqrt())).collect();
+    out.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
+    Ok(out)
+}
+
 /// How strongly each artist is linked to `tag`, 0..1: their own tagged tracks (three
 /// or more count fully), or half of what the artists next to them in the graph have.
 pub fn tag_affinity(conn: &Connection, tag: &str) -> rusqlite::Result<HashMap<String, f64>> {
@@ -391,12 +419,27 @@ pub fn pick(conn: &Connection, user_id: i64, n: usize, f: Filters, ranker: &dyn
         let best = (0..scored.len()).max_by(|&a, &b| penalised(&scored[a]).total_cmp(&penalised(&scored[b]))).unwrap_or(0);
         let (s, r) = scored.remove(best);
         seen.push(r.handle.clone());
-        out.push(serialize(&r, Some(s)));
+        let mut v = serialize(&r, Some(s));
+        if let Some(why) = ranker.reason(&r) {
+            v["why"] = why;
+        }
+        out.push(v);
     }
     add_tags(conn, &mut out)?;
+    name_reasons(conn, &mut out)?;
     Ok(out)
 }
 
+/// Artist names for reasons that point at another artist.
+fn name_reasons(conn: &Connection, tracks: &mut [Value]) -> rusqlite::Result<()> {
+    for t in tracks {
+        if let Some(h) = t["why"]["handle"].as_str().map(String::from) {
+            t["why"]["name"] = json!(crate::db::artist(conn, &h)?.map_or(h, |a| a.name));
+        }
+    }
+    Ok(())
+}
+
 fn add_tags(conn: &Connection, tracks: &mut [Value]) -> rusqlite::Result<()> {
     for t in tracks {
         if let Some(id) = t["id"].as_i64() {
@@ -509,8 +552,9 @@ mod tests {
         db::add_related(&mut c, 1, &["b".into()]).unwrap();
         db::add_related(&mut c, 2, &["c".into()]).unwrap();
         let near = neighbourhood(&c, "a").unwrap();
-        assert_eq!(near["b"], 1.0);
-        assert!((near["c"] - 0.4).abs() < 1e-9);
+        assert_eq!(near["b"], Near { score: 1.0, via: None });
+        assert!((near["c"].score - 0.4).abs() < 1e-9);
+        assert_eq!(near["c"].via.as_deref(), Some("b"));
         assert_eq!(ids(&c, Mode::Artist("a".into()), 10), [1, 2, 3]);
     }