mirror of
https://github.com/narrrl/pl8list-rewind.git
synced 2026-11-03 16:41:26 +01:00
Rewrite backend in Rust, add Impressum, privacy policy and account tools
New axum server in backend/ with the same HTTP API and database as the Python one. The pl8list session is only held in memory during the feed crawl and logged out afterwards. Local sessions use a __Host- cookie, 30-day idle expiry and weekly token rotation; the old cookie is migrated. The audio and image proxy only accepts known parameters, fixes the content type, follows redirects only to pl8list's file host and caches segments and images on disk. Strict security headers, origin checks on POST, body and concurrency limits. /impressum and /datenschutz are rendered from env variables. The account menu offers data export, sign out on all devices and account deletion. hls.js is self-hosted and artist images are proxied, so the browser only talks to this server. The Docker image builds a static binary on scratch.
This commit is contained in:
nvrl committed
1 parent
61af1a6dd2
commit
54a436ae48
35 files changed
+6389
-42
No files matched your search
Generated
+2339
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "pl8inf"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.104"
|
||||
axum = "0.8.9"
|
||||
base64 = "0.23.1"
|
||||
bytes = "1.12.1"
|
||||
futures-util = "0.3.34"
|
||||
hex = "0.4.3"
|
||||
percent-encoding = "2.3.2"
|
||||
rand = "0.10.3"
|
||||
reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "http2", "stream", "form", "query", "json"] }
|
||||
rusqlite = { version = "0.41.1", features = ["bundled"] }
|
||||
rust-embed = "8.13.0"
|
||||
secrecy = "0.10.3"
|
||||
serde = { version = "1.0.229", features = ["derive"] }
|
||||
serde_json = "1.0.154"
|
||||
sha2 = "0.11.1"
|
||||
time = { version = "0.3.55", features = ["parsing"] }
|
||||
tokio = { version = "1.54.1", features = ["full"] }
|
||||
tower = { version = "0.5.3", features = ["limit", "timeout", "util"] }
|
||||
tower-http = { version = "0.7.1", features = ["timeout", "limit", "trace", "set-header"] }
|
||||
tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
|
||||
url = "2.5.8"
|
||||
|
||||
[dev-dependencies]
|
||||
wiremock = "0.6.5"
|
||||
@@ -0,0 +1,198 @@
|
||||
//! Disk cache for immutable upstream content: audio segments and artist images.
|
||||
//!
|
||||
//! Files are named after a hash of their key, with an extension that records the
|
||||
//! content type. The index lives in memory and is rebuilt from the directory at
|
||||
//! startup. When the cache grows past its limit the least recently used files go.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
sync::Mutex,
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
use bytes::Bytes;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Content types we pass on, with the file extension they are stored under.
|
||||
pub const SEGMENT_TYPES: &[(&str, &str)] = &[
|
||||
("video/mp2t", "ts"),
|
||||
("audio/aac", "aac"),
|
||||
("audio/mpeg", "mp3"),
|
||||
("audio/mp4", "m4a"),
|
||||
("video/mp4", "mp4"),
|
||||
];
|
||||
pub const IMAGE_TYPES: &[(&str, &str)] = &[
|
||||
("image/jpeg", "jpg"),
|
||||
("image/png", "png"),
|
||||
("image/webp", "webp"),
|
||||
("image/gif", "gif"),
|
||||
("image/avif", "avif"),
|
||||
];
|
||||
|
||||
fn type_for_ext(ext: &str) -> Option<&'static str> {
|
||||
SEGMENT_TYPES.iter().chain(IMAGE_TYPES).find(|(_, e)| *e == ext).map(|(t, _)| *t)
|
||||
}
|
||||
|
||||
/// The allowlisted type matching an upstream `Content-Type`, ignoring parameters.
|
||||
pub fn allowed_type(upstream: Option<&str>, allow: &[(&'static str, &'static str)]) -> Option<(&'static str, &'static str)> {
|
||||
let t = upstream?.split(';').next()?.trim().to_ascii_lowercase();
|
||||
allow.iter().find(|(a, _)| *a == t).copied()
|
||||
}
|
||||
|
||||
struct Entry {
|
||||
ext: &'static str,
|
||||
size: u64,
|
||||
used: u64,
|
||||
}
|
||||
|
||||
struct Index {
|
||||
entries: HashMap<String, Entry>,
|
||||
total: u64,
|
||||
clock: u64,
|
||||
}
|
||||
|
||||
pub struct Cache {
|
||||
dir: Option<PathBuf>,
|
||||
max: u64,
|
||||
index: Mutex<Index>,
|
||||
}
|
||||
|
||||
fn hash(key: &str) -> String {
|
||||
hex::encode(Sha256::digest(key.as_bytes()))
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
/// Opens (and creates) the directory. A cache that cannot be used is disabled, not fatal.
|
||||
pub fn open(dir: PathBuf, max: u64) -> Self {
|
||||
let mut index = Index { entries: HashMap::new(), total: 0, clock: 0 };
|
||||
let dir = match std::fs::create_dir_all(&dir).and_then(|_| std::fs::read_dir(&dir)) {
|
||||
Ok(read) => {
|
||||
let mut found: Vec<(SystemTime, String, Entry)> = read
|
||||
.flatten()
|
||||
.filter_map(|f| {
|
||||
let name = f.file_name().into_string().ok()?;
|
||||
let (h, ext) = name.split_once('.')?;
|
||||
let ext = SEGMENT_TYPES.iter().chain(IMAGE_TYPES).find(|(_, e)| *e == ext)?.1;
|
||||
let meta = f.metadata().ok()?;
|
||||
Some((meta.modified().ok()?, h.to_string(), Entry { ext, size: meta.len(), used: 0 }))
|
||||
})
|
||||
.collect();
|
||||
found.sort_by_key(|(t, _, _)| *t);
|
||||
for (_, h, mut e) in found {
|
||||
index.clock += 1;
|
||||
e.used = index.clock;
|
||||
index.total += e.size;
|
||||
index.entries.insert(h, e);
|
||||
}
|
||||
tracing::info!("cache: {} files, {} MB in {}", index.entries.len(), index.total >> 20, dir.display());
|
||||
Some(dir)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("cache disabled, {}: {e}", dir.display());
|
||||
None
|
||||
}
|
||||
};
|
||||
Cache { dir, max, index: Mutex::new(index) }
|
||||
}
|
||||
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, Index> {
|
||||
self.index.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
pub async fn get(&self, key: &str) -> Option<(Bytes, &'static str)> {
|
||||
let dir = self.dir.as_ref()?;
|
||||
let h = hash(key);
|
||||
let ext = {
|
||||
let mut idx = self.lock();
|
||||
idx.clock += 1;
|
||||
let clock = idx.clock;
|
||||
let e = idx.entries.get_mut(&h)?;
|
||||
e.used = clock;
|
||||
e.ext
|
||||
};
|
||||
match tokio::fs::read(dir.join(format!("{h}.{ext}"))).await {
|
||||
Ok(b) => Some((Bytes::from(b), type_for_ext(ext)?)),
|
||||
Err(_) => {
|
||||
let mut idx = self.lock();
|
||||
if let Some(e) = idx.entries.remove(&h) {
|
||||
idx.total -= e.size;
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn put(&self, key: &str, ext: &'static str, body: &Bytes) {
|
||||
let Some(dir) = &self.dir else { return };
|
||||
let h = hash(key);
|
||||
let tmp = dir.join(format!(".{h}.tmp"));
|
||||
let path = dir.join(format!("{h}.{ext}"));
|
||||
if let Err(e) = async {
|
||||
tokio::fs::write(&tmp, body).await?;
|
||||
tokio::fs::rename(&tmp, &path).await
|
||||
}
|
||||
.await
|
||||
{
|
||||
tracing::warn!("cache write failed: {e}");
|
||||
let _ = tokio::fs::remove_file(&tmp).await;
|
||||
return;
|
||||
}
|
||||
let evict = {
|
||||
let mut idx = self.lock();
|
||||
idx.clock += 1;
|
||||
let entry = Entry { ext, size: body.len() as u64, used: idx.clock };
|
||||
idx.total += entry.size;
|
||||
if let Some(old) = idx.entries.insert(h, entry) {
|
||||
idx.total -= old.size;
|
||||
}
|
||||
let mut evict = vec![];
|
||||
if idx.total > self.max {
|
||||
let mut by_age: Vec<(u64, String)> = idx.entries.iter().map(|(k, e)| (e.used, k.clone())).collect();
|
||||
by_age.sort();
|
||||
let target = self.max / 10 * 9;
|
||||
for (_, k) in by_age {
|
||||
if idx.total <= target {
|
||||
break;
|
||||
}
|
||||
if let Some(e) = idx.entries.remove(&k) {
|
||||
idx.total -= e.size;
|
||||
evict.push(format!("{k}.{}", e.ext));
|
||||
}
|
||||
}
|
||||
}
|
||||
evict
|
||||
};
|
||||
for name in evict {
|
||||
let _ = tokio::fs::remove_file(dir.join(name)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn evicts_least_recently_used() {
|
||||
let dir = std::env::temp_dir().join(format!("pl8inf-cache-test-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let c = Cache::open(dir.clone(), 25);
|
||||
let body = Bytes::from_static(&[0u8; 10]);
|
||||
c.put("a", "ts", &body).await;
|
||||
c.put("b", "ts", &body).await;
|
||||
assert!(c.get("a").await.is_some()); // a is now newer than b
|
||||
c.put("c", "ts", &body).await; // 30 > 25: evict down to 22, b goes first
|
||||
assert!(c.get("b").await.is_none());
|
||||
assert!(c.get("a").await.is_some() || c.get("c").await.is_some());
|
||||
assert_eq!(c.get("c").await.unwrap().1, "video/mp2t");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_type_allowlist() {
|
||||
assert_eq!(allowed_type(Some("video/MP2T; charset=x"), SEGMENT_TYPES), Some(("video/mp2t", "ts")));
|
||||
assert_eq!(allowed_type(Some("text/html"), SEGMENT_TYPES), None);
|
||||
assert_eq!(allowed_type(Some("image/svg+xml"), IMAGE_TYPES), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//! Configuration from the environment (or `.env` next to the working directory).
|
||||
|
||||
use std::{env, path::PathBuf};
|
||||
|
||||
pub struct Config {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub db_path: PathBuf,
|
||||
/// Reverse proxies in front of us that append to `X-Forwarded-For`. 0 = none.
|
||||
pub proxy_hops: usize,
|
||||
/// `https://host` the site is served from. Used for the Origin check; when unset
|
||||
/// the Origin must match the Host header.
|
||||
pub public_origin: Option<String>,
|
||||
/// Off only for plain-http development on another machine: drops `Secure`,
|
||||
/// the `__Host-` cookie prefix and HSTS.
|
||||
pub secure: bool,
|
||||
pub cache_dir: PathBuf,
|
||||
pub cache_max_bytes: u64,
|
||||
/// pl8list itself; only changed by tests.
|
||||
pub pl8_base: String,
|
||||
pub legal: Legal,
|
||||
}
|
||||
|
||||
/// Operator details for the Impressum and the privacy policy. Kept out of the repository.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Legal {
|
||||
pub name: String,
|
||||
pub address: String,
|
||||
pub email: String,
|
||||
pub phone: String,
|
||||
pub hosting: String,
|
||||
}
|
||||
|
||||
impl Legal {
|
||||
pub fn is_complete(&self) -> bool {
|
||||
!self.name.is_empty() && !self.address.is_empty() && !self.email.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
fn var(name: &str) -> Option<String> {
|
||||
env::var(name).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
fn flag(name: &str) -> Option<bool> {
|
||||
var(name).map(|v| matches!(v.as_str(), "1" | "true" | "yes"))
|
||||
}
|
||||
|
||||
pub fn load_dotenv(path: &str) {
|
||||
let Ok(text) = std::fs::read_to_string(path) else { return };
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if let Some((k, v)) = line.split_once('=') {
|
||||
let k = k.trim();
|
||||
if env::var_os(k).is_none() {
|
||||
// SAFETY: called once at startup before any other thread exists.
|
||||
unsafe { env::set_var(k, v.trim().trim_matches(|c| c == '"' || c == '\'')) };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn from_env() -> Self {
|
||||
let db_path = PathBuf::from(var("PL8_DB").unwrap_or_else(|| "pl8list.db".into()));
|
||||
let cache_dir = var("CACHE_DIR").map(PathBuf::from).unwrap_or_else(|| {
|
||||
db_path.parent().map(|p| p.join("cache")).unwrap_or_else(|| PathBuf::from("cache"))
|
||||
});
|
||||
// TRUST_PROXY=1 from the Python version means one proxy hop.
|
||||
let proxy_hops = var("TRUSTED_PROXY_HOPS")
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(if flag("TRUST_PROXY") == Some(true) { 1 } else { 0 });
|
||||
Config {
|
||||
host: var("HOST").unwrap_or_else(|| "127.0.0.1".into()),
|
||||
port: var("PORT").and_then(|v| v.parse().ok()).unwrap_or(8790),
|
||||
db_path,
|
||||
proxy_hops,
|
||||
public_origin: var("PUBLIC_ORIGIN").map(|o| o.trim_end_matches('/').to_string()),
|
||||
secure: flag("INSECURE_COOKIES") != Some(true),
|
||||
cache_dir,
|
||||
cache_max_bytes: var("CACHE_MAX_MB").and_then(|v| v.parse().ok()).unwrap_or(2048u64) << 20,
|
||||
pl8_base: var("PL8_BASE").unwrap_or_else(|| "https://pl8list.com".into()),
|
||||
legal: Legal {
|
||||
name: var("IMPRESSUM_NAME").unwrap_or_default(),
|
||||
// One line per address line; "\n" or "|" separate lines in .env.
|
||||
address: var("IMPRESSUM_ADDRESS").unwrap_or_default().replace("\\n", "\n").replace('|', "\n"),
|
||||
email: var("IMPRESSUM_EMAIL").unwrap_or_default(),
|
||||
phone: var("IMPRESSUM_PHONE").unwrap_or_default(),
|
||||
hosting: var("HOSTING_PROVIDER").unwrap_or_default().replace("\\n", "\n").replace('|', "\n"),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
//! Background discovery crawler.
|
||||
//!
|
||||
//! There is no "all tracks" endpoint, so the pool grows by snowballing:
|
||||
//! home page sections, the activity ticker, best-selling artists and user search
|
||||
//! give us artists; artist pages give us their tracks; "related" pages of tracks
|
||||
//! users liked or finished pull in neighbours. Everything goes through one priority
|
||||
//! queue so feedback is crawled first.
|
||||
//!
|
||||
//! The catalogue is shared by all users and crawled as a guest. The crawler runs
|
||||
//! at full pace only while someone has used the app recently.
|
||||
//!
|
||||
//! A user's pl8list session never enters the queue: `crawl_feed` owns it in its own
|
||||
//! task, reads the personal feed, logs the session out at pl8list and drops it.
|
||||
|
||||
use std::{
|
||||
cmp::Reverse,
|
||||
collections::{BinaryHeap, HashSet},
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use secrecy::SecretString;
|
||||
use serde_json::Value;
|
||||
use tokio::{sync::Notify, time::Instant};
|
||||
|
||||
use crate::{
|
||||
db::{self, Db},
|
||||
pl8::{Pl8, normalize},
|
||||
};
|
||||
|
||||
/// Search needs at least 3 characters. Genre-ish words find lots of artist handles.
|
||||
const SEARCH_TERMS: &[&str] = &[
|
||||
"dub", "bass", "dnb", "jungle", "garage", "grime", "riddim", "dubstep", "trap", "techno",
|
||||
"house", "beats", "music", "sound", "prod", "audio", "records", "drill", "breaks", "wave",
|
||||
"deep", "dark", "the", "official", "dj", "mc", "lofi", "halftime", "neuro", "tek",
|
||||
];
|
||||
|
||||
// Lower number = sooner.
|
||||
const P_FEEDBACK: i64 = 0;
|
||||
const P_SEED: i64 = 10;
|
||||
const P_ARTIST_FIRST: i64 = 20;
|
||||
const P_RELATED: i64 = 30;
|
||||
const P_ARTIST_MORE: i64 = 40;
|
||||
const P_SEARCH: i64 = 60;
|
||||
|
||||
const ACTIVE_WINDOW: f64 = 1800.0; // a user seen this recently counts as active
|
||||
const IDLE_DELAY: Duration = Duration::from_secs(20); // extra wait between requests when nobody is active
|
||||
const MIN_POOL: i64 = 500; // crawl at full pace until the catalogue has this many tracks
|
||||
const REFRESH_AFTER: f64 = 86400.0; // re-check an artist's newest page for uploads this often
|
||||
const RESEED_EVERY: Duration = Duration::from_secs(600);
|
||||
const FEED_MAX_PAGES: i64 = 20;
|
||||
const FEED_DEADLINE: Duration = Duration::from_secs(120);
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
|
||||
enum Job {
|
||||
Home,
|
||||
Ticker,
|
||||
BestArtists,
|
||||
Search(String),
|
||||
Artist(String),
|
||||
ArtistRefresh(String),
|
||||
Related(i64, String, String),
|
||||
}
|
||||
|
||||
struct Queue {
|
||||
heap: BinaryHeap<Reverse<(i64, u64, Job)>>,
|
||||
queued: HashSet<Job>,
|
||||
seq: u64,
|
||||
}
|
||||
|
||||
pub struct Crawler {
|
||||
pl8: Arc<Pl8>,
|
||||
db: Db,
|
||||
queue: Mutex<Queue>,
|
||||
wake: Notify,
|
||||
}
|
||||
|
||||
fn s(v: &Value, k: &str) -> Option<String> {
|
||||
v.get(k).and_then(Value::as_str).filter(|s| !s.is_empty()).map(String::from)
|
||||
}
|
||||
|
||||
impl Crawler {
|
||||
pub fn new(pl8: Arc<Pl8>, db: Db) -> Arc<Self> {
|
||||
Arc::new(Crawler {
|
||||
pl8,
|
||||
db,
|
||||
queue: Mutex::new(Queue { heap: BinaryHeap::new(), queued: HashSet::new(), seq: 0 }),
|
||||
wake: Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn push(&self, priority: i64, job: Job) {
|
||||
let mut q = self.queue.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if !q.queued.insert(job.clone()) {
|
||||
return;
|
||||
}
|
||||
q.seq += 1;
|
||||
let seq = q.seq;
|
||||
q.heap.push(Reverse((priority, seq, job)));
|
||||
drop(q);
|
||||
self.wake.notify_one();
|
||||
}
|
||||
|
||||
fn try_pop(&self) -> Option<Job> {
|
||||
let mut q = self.queue.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let Reverse((_, _, job)) = q.heap.pop()?;
|
||||
q.queued.remove(&job);
|
||||
Some(job)
|
||||
}
|
||||
|
||||
async fn pop(&self) -> Job {
|
||||
loop {
|
||||
if let Some(job) = self.try_pop() {
|
||||
return job;
|
||||
}
|
||||
if tokio::time::timeout(Duration::from_secs(30), self.wake.notified()).await.is_err()
|
||||
&& let Err(e) = self.refill().await {
|
||||
tracing::warn!("crawler refill: {e:#}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Feedback hooks, called from the web server ----------------------------
|
||||
|
||||
/// Liked or listened through: crawl its related tracks and its artist's catalogue.
|
||||
pub async fn on_positive(&self, track_id: i64) {
|
||||
let row = self
|
||||
.db
|
||||
.call(move |c| {
|
||||
rusqlite::OptionalExtension::optional(c.query_row(
|
||||
"SELECT handle, slug FROM tracks WHERE id=?",
|
||||
[track_id],
|
||||
|r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)),
|
||||
))
|
||||
})
|
||||
.await;
|
||||
if let Ok(Some((handle, slug))) = row {
|
||||
self.push(P_FEEDBACK, Job::Related(track_id, handle.clone(), slug));
|
||||
self.push(P_FEEDBACK, Job::Artist(handle));
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the user's personal feed (artists in their pl8list library) with their
|
||||
/// freshly logged-in session, then end that session at pl8list. Takes ownership
|
||||
/// of the session; it is dropped (and zeroed) when this task ends.
|
||||
pub fn crawl_feed(self: &Arc<Self>, session: SecretString) {
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move {
|
||||
let res = tokio::time::timeout(FEED_DEADLINE, me.read_feed(&session)).await;
|
||||
match res {
|
||||
Ok(Ok(pages)) => tracing::info!("personal feed: {pages} pages"),
|
||||
Ok(Err(e)) => tracing::warn!("personal feed failed: {e:#}"),
|
||||
Err(_) => tracing::warn!("personal feed timed out"),
|
||||
}
|
||||
me.pl8.logout(&session).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn read_feed(&self, session: &SecretString) -> anyhow::Result<i64> {
|
||||
let mut page = 1;
|
||||
loop {
|
||||
let d = self.pl8.feed(session, page).await?.unwrap_or(Value::Null);
|
||||
let list = d.get("tracks");
|
||||
let rows = normalize::items(list);
|
||||
for r in &rows {
|
||||
self.push(P_ARTIST_FIRST, Job::Artist(r.handle.clone()));
|
||||
}
|
||||
self.db.call(move |c| db::upsert_tracks(c, &rows, None)).await?;
|
||||
let total = d.get("totalPages").and_then(Value::as_i64).unwrap_or(0);
|
||||
if page >= total.min(FEED_MAX_PAGES) {
|
||||
return Ok(page);
|
||||
}
|
||||
page += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Main loop -------------------------------------------------------------
|
||||
|
||||
pub fn start(self: &Arc<Self>) {
|
||||
let me = self.clone();
|
||||
tokio::spawn(async move { me.run().await });
|
||||
}
|
||||
|
||||
async fn run(&self) {
|
||||
self.seed();
|
||||
let mut last_seed = Instant::now();
|
||||
loop {
|
||||
let job = self.pop().await;
|
||||
if let Err(e) = self.run_job(job.clone()).await {
|
||||
tracing::warn!("crawl {job:?} failed: {e:#}");
|
||||
}
|
||||
if !self.busy().await {
|
||||
tokio::time::sleep(IDLE_DELAY).await;
|
||||
}
|
||||
if last_seed.elapsed() > RESEED_EVERY {
|
||||
self.seed();
|
||||
last_seed = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Crawl at full pace while someone is listening or the catalogue is still small.
|
||||
async fn busy(&self) -> bool {
|
||||
self.db
|
||||
.call(|c| Ok(db::active_since(c, db::now() - ACTIVE_WINDOW)? > 0 || db::track_count(c)? < MIN_POOL))
|
||||
.await
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
fn seed(&self) {
|
||||
self.push(P_SEED, Job::Home);
|
||||
self.push(P_SEED, Job::Ticker);
|
||||
self.push(P_SEED, Job::BestArtists);
|
||||
for term in SEARCH_TERMS {
|
||||
self.push(P_SEARCH, Job::Search(term.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue ran dry: queue more artist pages, related lookups and stale artists.
|
||||
async fn refill(&self) -> anyhow::Result<()> {
|
||||
let (more, related, stale) = self
|
||||
.db
|
||||
.call(|c| {
|
||||
let col = |sql: &str, arg: Option<f64>| -> rusqlite::Result<Vec<String>> {
|
||||
let mut st = c.prepare(sql)?;
|
||||
|
||||
match arg {
|
||||
Some(a) => st.query_map([a], |r| r.get(0))?.collect(),
|
||||
None => st.query_map([], |r| r.get(0))?.collect(),
|
||||
}
|
||||
};
|
||||
let more = col(
|
||||
"SELECT handle FROM artists WHERE total_pages IS NULL OR pages_fetched < total_pages \
|
||||
ORDER BY verified DESC, pages_fetched ASC LIMIT 20",
|
||||
None,
|
||||
)?;
|
||||
let related: Vec<(i64, String, String)> = c
|
||||
.prepare(
|
||||
"SELECT id, handle, slug FROM tracks WHERE related_fetched=0 \
|
||||
ORDER BY saves + downloads DESC LIMIT 10",
|
||||
)?
|
||||
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?
|
||||
.collect::<Result<_, _>>()?;
|
||||
let stale = col(
|
||||
"SELECT handle FROM artists WHERE pages_fetched > 0 AND last_crawled < ? \
|
||||
ORDER BY last_crawled LIMIT 20",
|
||||
Some(db::now() - REFRESH_AFTER),
|
||||
)?;
|
||||
Ok((more, related, stale))
|
||||
})
|
||||
.await?;
|
||||
for h in more {
|
||||
self.push(P_ARTIST_MORE, Job::Artist(h));
|
||||
}
|
||||
for (id, h, s) in related {
|
||||
self.push(P_RELATED, Job::Related(id, h, s));
|
||||
}
|
||||
for h in stale {
|
||||
self.push(P_ARTIST_MORE, Job::ArtistRefresh(h));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Jobs ------------------------------------------------------------------
|
||||
|
||||
async fn add_artist(&self, handle: String, source: &'static str, name: Option<String>, img: Option<String>, verified: bool, priority: i64) -> anyhow::Result<()> {
|
||||
let h = handle.clone();
|
||||
let known: Option<i64> = self
|
||||
.db
|
||||
.call(move |c| {
|
||||
let known = rusqlite::OptionalExtension::optional(c.query_row(
|
||||
"SELECT pages_fetched FROM artists WHERE handle=?",
|
||||
[&h],
|
||||
|r| r.get::<_, Option<i64>>(0),
|
||||
))?
|
||||
.flatten();
|
||||
db::upsert_artist(c, &h, name.as_deref(), img.as_deref(), verified, source)?;
|
||||
Ok(known)
|
||||
})
|
||||
.await?;
|
||||
if known.unwrap_or(0) == 0 {
|
||||
self.push(priority, Job::Artist(handle));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upsert(&self, rows: Vec<db::Track>, related_of: Option<i64>) -> anyhow::Result<usize> {
|
||||
self.db.call(move |c| db::upsert_tracks(c, &rows, related_of)).await
|
||||
}
|
||||
|
||||
async fn ticker_events(&self, events: Option<&Value>) -> anyhow::Result<()> {
|
||||
for e in events.and_then(Value::as_array).into_iter().flatten() {
|
||||
if let Some(h) = s(e, "artistHandle") {
|
||||
self.add_artist(h, "ticker", s(e, "artistName"), None, false, P_ARTIST_FIRST).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_job(&self, job: Job) -> anyhow::Result<()> {
|
||||
match job {
|
||||
Job::Home => {
|
||||
let d = self.pl8.home().await?.unwrap_or(Value::Null);
|
||||
for section in ["bestSellers", "recommended"] {
|
||||
self.upsert(normalize::items(d.get(section).and_then(|s| s.get("items"))), None).await?;
|
||||
}
|
||||
self.ticker_events(d.get("tickerEvents")).await
|
||||
}
|
||||
Job::Ticker => {
|
||||
let d = self.pl8.activity_ticker().await?.unwrap_or(Value::Null);
|
||||
self.ticker_events(d.get("events")).await
|
||||
}
|
||||
Job::BestArtists => {
|
||||
let d = self.pl8.best_selling_artists().await?.unwrap_or(Value::Null);
|
||||
for a in d.get("artists").and_then(Value::as_array).into_iter().flatten() {
|
||||
if let Some(h) = s(a, "artist_handle") {
|
||||
let verified = a.get("artist_isVerified").and_then(Value::as_bool).unwrap_or(false);
|
||||
self.add_artist(h, "best_sellers", s(a, "artist_name"), s(a, "artist_img"), verified, P_SEED).await?;
|
||||
}
|
||||
}
|
||||
self.upsert(normalize::items(d.get("recommended").and_then(|r| r.get("items"))), None).await?;
|
||||
Ok(())
|
||||
}
|
||||
Job::ArtistRefresh(handle) => {
|
||||
// Newest uploads are on page 1, so re-reading it catches new tracks.
|
||||
if let Some(d) = self.pl8.artist(&handle, 1).await? {
|
||||
let new = self.upsert(normalize::items(d.get("tracks")), None).await?;
|
||||
if new > 0 {
|
||||
tracing::info!("artist {handle}: {new} new uploads");
|
||||
}
|
||||
}
|
||||
self.db
|
||||
.call(move |c| c.execute("UPDATE artists SET last_crawled=? WHERE handle=?", rusqlite::params![db::now(), handle]))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
Job::Artist(handle) => self.artist_page(handle).await,
|
||||
Job::Related(track_id, handle, slug) => {
|
||||
let d = self.pl8.related(&handle, &slug).await?;
|
||||
self.db.call(move |c| c.execute("UPDATE tracks SET related_fetched=1 WHERE id=?", [track_id])).await?;
|
||||
let Some(d) = d else { return Ok(()) };
|
||||
let rows = normalize::related(&d);
|
||||
let artists: Vec<_> = rows.iter().map(|r| (r.handle.clone(), r.artist_name.clone(), r.artist_img.clone(), r.artist_verified)).collect();
|
||||
self.upsert(rows, Some(track_id)).await?;
|
||||
for (h, name, img, verified) in artists {
|
||||
self.add_artist(h, "related", Some(name), img, verified, P_RELATED).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Job::Search(term) => self.search_page(term).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn artist_page(&self, handle: String) -> anyhow::Result<()> {
|
||||
let h = handle.clone();
|
||||
let (fetched, total) = self
|
||||
.db
|
||||
.call(move |c| {
|
||||
Ok(rusqlite::OptionalExtension::optional(c.query_row(
|
||||
"SELECT pages_fetched, total_pages FROM artists WHERE handle=?",
|
||||
[&h],
|
||||
|r| Ok((r.get::<_, Option<i64>>(0)?.unwrap_or(0), r.get::<_, Option<i64>>(1)?)),
|
||||
))?
|
||||
.unwrap_or((0, None)))
|
||||
})
|
||||
.await?;
|
||||
if total.is_some_and(|t| fetched >= t) {
|
||||
return Ok(());
|
||||
}
|
||||
let page = fetched + 1;
|
||||
let h = handle.clone();
|
||||
let Some(d) = self.pl8.artist(&handle, page).await? else {
|
||||
self.db
|
||||
.call(move |c| c.execute("UPDATE artists SET total_pages=0, last_crawled=? WHERE handle=?", rusqlite::params![db::now(), h]))
|
||||
.await?;
|
||||
return Ok(());
|
||||
};
|
||||
let new = self.upsert(normalize::items(d.get("tracks")), None).await?;
|
||||
let total = d.get("totalPages").and_then(Value::as_i64).unwrap_or(0);
|
||||
self.db
|
||||
.call(move |c| {
|
||||
c.execute(
|
||||
"UPDATE artists SET pages_fetched=?, total_pages=?, last_crawled=? WHERE handle=?",
|
||||
rusqlite::params![page, total, db::now(), h],
|
||||
)
|
||||
})
|
||||
.await?;
|
||||
tracing::info!("artist {handle} page {page}/{total}: {new} new tracks");
|
||||
if page < total {
|
||||
self.push(P_ARTIST_MORE + page, Job::Artist(handle));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn search_page(&self, term: String) -> anyhow::Result<()> {
|
||||
let t = term.clone();
|
||||
let (fetched, total) = self
|
||||
.db
|
||||
.call(move |c| {
|
||||
Ok(rusqlite::OptionalExtension::optional(c.query_row(
|
||||
"SELECT pages_fetched, total_pages FROM search_terms WHERE term=?",
|
||||
[&t],
|
||||
|r| Ok((r.get::<_, Option<i64>>(0)?.unwrap_or(0), r.get::<_, Option<i64>>(1)?)),
|
||||
))?
|
||||
.unwrap_or((0, None)))
|
||||
})
|
||||
.await?;
|
||||
if total.is_some_and(|t| fetched >= t) {
|
||||
return Ok(());
|
||||
}
|
||||
let page = fetched + 1;
|
||||
let d = self.pl8.search_users(&term, page).await?.unwrap_or(Value::Null);
|
||||
for item in d.get("items").and_then(Value::as_array).into_iter().flatten() {
|
||||
let Some(a) = item.get("artist") else { continue };
|
||||
if let Some(h) = s(a, "handle") {
|
||||
let verified = a.get("isVerified").and_then(Value::as_bool).unwrap_or(false);
|
||||
let prio = if verified { P_ARTIST_FIRST } else { P_SEARCH + 5 };
|
||||
self.add_artist(h, "search", s(a, "name"), s(a, "img"), verified, prio).await?;
|
||||
}
|
||||
}
|
||||
let total = d.get("totalPages").and_then(Value::as_i64).unwrap_or(0);
|
||||
let t = term.clone();
|
||||
self.db
|
||||
.call(move |c| {
|
||||
c.execute(
|
||||
"INSERT INTO search_terms (term, pages_fetched, total_pages) VALUES (?,?,?) \
|
||||
ON CONFLICT(term) DO UPDATE SET pages_fetched=excluded.pages_fetched, total_pages=excluded.total_pages",
|
||||
rusqlite::params![t, page, total],
|
||||
)
|
||||
})
|
||||
.await?;
|
||||
if page < total {
|
||||
self.push(P_SEARCH + page, Job::Search(term));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
//! SQLite store: the shared track catalogue, plus per-user accounts and listening events.
|
||||
//!
|
||||
//! The schema is the one the Python version created, so both can open the same file.
|
||||
//! Changes are additive and tracked with `PRAGMA user_version`.
|
||||
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{Arc, Mutex},
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use base64::Engine;
|
||||
use rand::Rng;
|
||||
use rusqlite::{Connection, OptionalExtension, Row, params, params_from_iter};
|
||||
use serde::Serialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Sessions end after this long without use.
|
||||
pub const SESSION_IDLE_DAYS: f64 = 30.0;
|
||||
/// "served" events only steer which tracks are new to a user; older ones are dropped.
|
||||
pub const SERVED_RETENTION_DAYS: f64 = 180.0;
|
||||
|
||||
const SCHEMA_V1: &str = r#"
|
||||
CREATE TABLE IF NOT EXISTS artists (
|
||||
handle TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
img TEXT,
|
||||
verified INTEGER DEFAULT 0,
|
||||
source TEXT,
|
||||
pages_fetched INTEGER DEFAULT 0,
|
||||
total_pages INTEGER,
|
||||
last_crawled REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tracks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
handle TEXT NOT NULL,
|
||||
artist_name TEXT,
|
||||
artist_img TEXT,
|
||||
artist_verified INTEGER DEFAULT 0,
|
||||
slug TEXT NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
bpm REAL,
|
||||
key TEXT,
|
||||
duration REAL,
|
||||
preview_duration REAL,
|
||||
price INTEGER DEFAULT 0,
|
||||
created_at TEXT,
|
||||
upload_id TEXT,
|
||||
preview_upload_id TEXT,
|
||||
waveform TEXT,
|
||||
preview_waveform TEXT,
|
||||
full_access INTEGER DEFAULT 0,
|
||||
saves INTEGER DEFAULT 0,
|
||||
downloads INTEGER DEFAULT 0,
|
||||
purchases INTEGER DEFAULT 0,
|
||||
comments INTEGER DEFAULT 0,
|
||||
related_of INTEGER,
|
||||
related_fetched INTEGER DEFAULT 0,
|
||||
updated_at REAL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS tracks_handle ON tracks(handle);
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
track_id INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
seconds REAL,
|
||||
ts REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY,
|
||||
handle TEXT NOT NULL,
|
||||
name TEXT,
|
||||
img TEXT,
|
||||
created_at REAL NOT NULL,
|
||||
last_seen REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at REAL NOT NULL,
|
||||
expires_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS search_terms (
|
||||
term TEXT PRIMARY KEY,
|
||||
pages_fetched INTEGER DEFAULT 0,
|
||||
total_pages INTEGER
|
||||
);
|
||||
"#;
|
||||
|
||||
const SCHEMA_V2: &str = r#"
|
||||
CREATE INDEX IF NOT EXISTS sessions_user ON sessions(user_id);
|
||||
CREATE INDEX IF NOT EXISTS events_kind_ts ON events(kind, ts);
|
||||
"#;
|
||||
|
||||
pub fn now() -> f64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs_f64()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Db(Arc<Mutex<Connection>>);
|
||||
|
||||
impl Db {
|
||||
pub fn open(path: &Path) -> anyhow::Result<Self> {
|
||||
let conn = Connection::open(path)?;
|
||||
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||
conn.pragma_update(None, "foreign_keys", "ON")?;
|
||||
conn.busy_timeout(std::time::Duration::from_secs(5))?;
|
||||
migrate(&conn)?;
|
||||
Ok(Db(Arc::new(Mutex::new(conn))))
|
||||
}
|
||||
|
||||
/// Run `f` on the connection off the async runtime.
|
||||
pub async fn call<T, F>(&self, f: F) -> anyhow::Result<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce(&mut Connection) -> rusqlite::Result<T> + Send + 'static,
|
||||
{
|
||||
let db = self.0.clone();
|
||||
let out = tokio::task::spawn_blocking(move || {
|
||||
let mut conn = db.lock().unwrap_or_else(|e| e.into_inner());
|
||||
f(&mut conn)
|
||||
})
|
||||
.await??;
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
fn migrate(conn: &Connection) -> rusqlite::Result<()> {
|
||||
let version: i64 = conn.pragma_query_value(None, "user_version", |r| r.get(0))?;
|
||||
if version < 1 {
|
||||
conn.execute_batch(SCHEMA_V1)?;
|
||||
// Databases from the single-user version have events without an owner.
|
||||
let has_user: bool = conn
|
||||
.prepare("SELECT 1 FROM pragma_table_info('events') WHERE name='user_id'")?
|
||||
.exists([])?;
|
||||
if !has_user {
|
||||
conn.execute("DELETE FROM events", [])?;
|
||||
conn.execute("ALTER TABLE events ADD COLUMN user_id INTEGER", [])?;
|
||||
}
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS events_user ON events(user_id, kind, track_id)", [])?;
|
||||
conn.pragma_update(None, "user_version", 1)?;
|
||||
}
|
||||
if version < 2 {
|
||||
conn.execute_batch(SCHEMA_V2)?;
|
||||
conn.pragma_update(None, "user_version", 2)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Tracks ------------------------------------------------------------------
|
||||
|
||||
/// One row of `tracks`, as normalised from pl8list or read back.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Track {
|
||||
pub id: i64,
|
||||
pub handle: String,
|
||||
pub artist_name: String,
|
||||
pub artist_img: Option<String>,
|
||||
pub artist_verified: bool,
|
||||
pub slug: String,
|
||||
pub title: String,
|
||||
pub description: Option<String>,
|
||||
pub bpm: Option<f64>,
|
||||
pub key: Option<String>,
|
||||
pub duration: Option<f64>,
|
||||
pub preview_duration: Option<f64>,
|
||||
pub price: f64,
|
||||
pub created_at: Option<String>,
|
||||
pub upload_id: Option<String>,
|
||||
pub preview_upload_id: Option<String>,
|
||||
pub waveform: Option<String>,
|
||||
pub preview_waveform: Option<String>,
|
||||
pub full_access: bool,
|
||||
pub saves: i64,
|
||||
pub downloads: i64,
|
||||
pub purchases: i64,
|
||||
pub comments: i64,
|
||||
pub related_of: Option<i64>,
|
||||
}
|
||||
|
||||
const TRACK_COLS: [&str; 23] = [
|
||||
"id", "handle", "artist_name", "artist_img", "artist_verified", "slug", "title", "description",
|
||||
"bpm", "key", "duration", "preview_duration", "price", "created_at", "upload_id",
|
||||
"preview_upload_id", "waveform", "preview_waveform", "full_access", "saves", "downloads",
|
||||
"purchases", "comments",
|
||||
];
|
||||
|
||||
impl Track {
|
||||
pub fn from_row(r: &Row) -> rusqlite::Result<Self> {
|
||||
Ok(Track {
|
||||
id: r.get("id")?,
|
||||
handle: r.get("handle")?,
|
||||
artist_name: r.get::<_, Option<String>>("artist_name")?.unwrap_or_default(),
|
||||
artist_img: r.get("artist_img")?,
|
||||
artist_verified: r.get::<_, Option<i64>>("artist_verified")?.unwrap_or(0) != 0,
|
||||
slug: r.get("slug")?,
|
||||
title: r.get::<_, Option<String>>("title")?.unwrap_or_default(),
|
||||
description: r.get("description")?,
|
||||
bpm: r.get("bpm")?,
|
||||
key: r.get("key")?,
|
||||
duration: r.get("duration")?,
|
||||
preview_duration: r.get("preview_duration")?,
|
||||
price: r.get::<_, Option<f64>>("price")?.unwrap_or(0.0),
|
||||
created_at: r.get("created_at")?,
|
||||
upload_id: r.get("upload_id")?,
|
||||
preview_upload_id: r.get("preview_upload_id")?,
|
||||
waveform: r.get("waveform")?,
|
||||
preview_waveform: r.get("preview_waveform")?,
|
||||
full_access: r.get::<_, Option<i64>>("full_access")?.unwrap_or(0) != 0,
|
||||
saves: r.get::<_, Option<i64>>("saves")?.unwrap_or(0),
|
||||
downloads: r.get::<_, Option<i64>>("downloads")?.unwrap_or(0),
|
||||
purchases: r.get::<_, Option<i64>>("purchases")?.unwrap_or(0),
|
||||
comments: r.get::<_, Option<i64>>("comments")?.unwrap_or(0),
|
||||
related_of: r.get("related_of")?,
|
||||
})
|
||||
}
|
||||
|
||||
fn values(&self) -> Vec<rusqlite::types::Value> {
|
||||
use rusqlite::types::Value;
|
||||
let opt = |s: &Option<String>| s.clone().map(Value::Text).unwrap_or(Value::Null);
|
||||
let optf = |f: Option<f64>| f.map(Value::Real).unwrap_or(Value::Null);
|
||||
// Whole prices stay integers, as the Python version stored them.
|
||||
let price = if self.price.fract() == 0.0 { Value::Integer(self.price as i64) } else { Value::Real(self.price) };
|
||||
vec![
|
||||
Value::Integer(self.id),
|
||||
Value::Text(self.handle.clone()),
|
||||
Value::Text(self.artist_name.clone()),
|
||||
opt(&self.artist_img),
|
||||
Value::Integer(self.artist_verified as i64),
|
||||
Value::Text(self.slug.clone()),
|
||||
Value::Text(self.title.clone()),
|
||||
opt(&self.description),
|
||||
optf(self.bpm),
|
||||
opt(&self.key),
|
||||
optf(self.duration),
|
||||
optf(self.preview_duration),
|
||||
price,
|
||||
opt(&self.created_at),
|
||||
opt(&self.upload_id),
|
||||
opt(&self.preview_upload_id),
|
||||
opt(&self.waveform),
|
||||
opt(&self.preview_waveform),
|
||||
Value::Integer(self.full_access as i64),
|
||||
Value::Integer(self.saves),
|
||||
Value::Integer(self.downloads),
|
||||
Value::Integer(self.purchases),
|
||||
Value::Integer(self.comments),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert or refresh tracks and their artists. Returns how many were new.
|
||||
pub fn upsert_tracks(conn: &mut Connection, rows: &[Track], related_of: Option<i64>) -> rusqlite::Result<usize> {
|
||||
let tx = conn.transaction()?;
|
||||
let mut new = 0;
|
||||
let now = now();
|
||||
{
|
||||
let sets = TRACK_COLS[1..].iter().map(|c| format!("{c}=?")).collect::<Vec<_>>().join(",");
|
||||
let mut update = tx.prepare_cached(&format!("UPDATE tracks SET {sets}, updated_at=? WHERE id=?"))?;
|
||||
let mut insert = tx.prepare_cached(&format!(
|
||||
"INSERT INTO tracks ({}, related_of, updated_at) VALUES ({})",
|
||||
TRACK_COLS.join(","),
|
||||
vec!["?"; TRACK_COLS.len() + 2].join(",")
|
||||
))?;
|
||||
let mut artist = tx.prepare_cached(
|
||||
"INSERT INTO artists (handle, name, img, verified, source) VALUES (?,?,?,?,'track') \
|
||||
ON CONFLICT(handle) DO UPDATE SET name=excluded.name, img=excluded.img, verified=excluded.verified",
|
||||
)?;
|
||||
for r in rows {
|
||||
let mut vals = r.values();
|
||||
let id = vals[0].clone();
|
||||
let changed = {
|
||||
let mut v = vals[1..].to_vec();
|
||||
v.push(rusqlite::types::Value::Real(now));
|
||||
v.push(id);
|
||||
update.execute(params_from_iter(v))?
|
||||
};
|
||||
if changed == 0 {
|
||||
new += 1;
|
||||
vals.push(related_of.map(rusqlite::types::Value::Integer).unwrap_or(rusqlite::types::Value::Null));
|
||||
vals.push(rusqlite::types::Value::Real(now));
|
||||
insert.execute(params_from_iter(vals))?;
|
||||
}
|
||||
artist.execute(params![r.handle, r.artist_name, r.artist_img, r.artist_verified as i64])?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(new)
|
||||
}
|
||||
|
||||
pub fn upsert_artist(
|
||||
conn: &Connection,
|
||||
handle: &str,
|
||||
name: Option<&str>,
|
||||
img: Option<&str>,
|
||||
verified: bool,
|
||||
source: &str,
|
||||
) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO artists (handle, name, img, verified, source) VALUES (?,?,?,?,?) \
|
||||
ON CONFLICT(handle) DO UPDATE SET name=COALESCE(excluded.name, name), \
|
||||
img=COALESCE(excluded.img, img), verified=MAX(verified, excluded.verified)",
|
||||
params![handle, name, img, verified as i64, source],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_event(conn: &Connection, user_id: i64, track_id: i64, kind: &str, seconds: Option<f64>) -> rusqlite::Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO events (user_id, track_id, kind, seconds, ts) VALUES (?,?,?,?,?)",
|
||||
params![user_id, track_id, kind, seconds, now()],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn track_count(conn: &Connection) -> rusqlite::Result<i64> {
|
||||
conn.query_row("SELECT COUNT(*) FROM tracks", [], |r| r.get(0))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct Stats {
|
||||
pub tracks: i64,
|
||||
pub artists: i64,
|
||||
pub artists_crawled: i64,
|
||||
pub served: i64,
|
||||
pub liked: usize,
|
||||
}
|
||||
|
||||
pub fn stats(conn: &Connection, user_id: i64) -> rusqlite::Result<Stats> {
|
||||
conn.query_row(
|
||||
"SELECT (SELECT COUNT(*) FROM tracks), (SELECT COUNT(*) FROM artists), \
|
||||
(SELECT COUNT(*) FROM artists WHERE pages_fetched > 0), \
|
||||
(SELECT COUNT(DISTINCT track_id) FROM events WHERE user_id=? AND kind='served')",
|
||||
[user_id],
|
||||
|r| Ok(Stats { tracks: r.get(0)?, artists: r.get(1)?, artists_crawled: r.get(2)?, served: r.get(3)?, liked: 0 }),
|
||||
)
|
||||
}
|
||||
|
||||
// Accounts ----------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct User {
|
||||
pub id: i64,
|
||||
pub handle: String,
|
||||
pub name: Option<String>,
|
||||
pub img: Option<String>,
|
||||
pub created_at: f64,
|
||||
pub last_seen: Option<f64>,
|
||||
}
|
||||
|
||||
impl User {
|
||||
fn from_row(r: &Row) -> rusqlite::Result<Self> {
|
||||
Ok(User {
|
||||
id: r.get("id")?,
|
||||
handle: r.get("handle")?,
|
||||
name: r.get("name")?,
|
||||
img: r.get("img")?,
|
||||
created_at: r.get("created_at")?,
|
||||
last_seen: r.get("last_seen")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn upsert_user(conn: &Connection, id: i64, handle: &str, name: Option<&str>, img: Option<&str>) -> rusqlite::Result<()> {
|
||||
let now = now();
|
||||
conn.execute(
|
||||
"INSERT INTO users (id, handle, name, img, created_at, last_seen) VALUES (?,?,?,?,?,?) \
|
||||
ON CONFLICT(id) DO UPDATE SET handle=excluded.handle, name=excluded.name, img=excluded.img, \
|
||||
last_seen=excluded.last_seen",
|
||||
params![id, handle, name, img, now, now],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn hash_token(token: &str) -> String {
|
||||
hex::encode(Sha256::digest(token.as_bytes()))
|
||||
}
|
||||
|
||||
/// 32 random bytes, URL-safe base64, like Python's `secrets.token_urlsafe(32)`.
|
||||
pub fn new_token() -> String {
|
||||
let mut bytes = [0u8; 32];
|
||||
rand::rng().fill_bytes(&mut bytes);
|
||||
base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes)
|
||||
}
|
||||
|
||||
/// Returns the raw token for the cookie; only its hash is stored.
|
||||
pub fn create_session(conn: &Connection, user_id: i64) -> rusqlite::Result<String> {
|
||||
let token = new_token();
|
||||
let now = now();
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at) VALUES (?,?,?,?)",
|
||||
params![hash_token(&token), user_id, now, now + SESSION_IDLE_DAYS * 86400.0],
|
||||
)?;
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
pub struct SessionHit {
|
||||
pub user: User,
|
||||
pub created_at: f64,
|
||||
pub expires_at: f64,
|
||||
}
|
||||
|
||||
pub fn session_user(conn: &Connection, token: &str) -> rusqlite::Result<Option<SessionHit>> {
|
||||
let now = now();
|
||||
let hit = conn
|
||||
.query_row(
|
||||
"SELECT u.*, s.created_at AS s_created, s.expires_at AS s_expires \
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token_hash=? AND s.expires_at > ?",
|
||||
params![hash_token(token), now],
|
||||
|r| Ok(SessionHit { user: User::from_row(r)?, created_at: r.get("s_created")?, expires_at: r.get("s_expires")? }),
|
||||
)
|
||||
.optional()?;
|
||||
if let Some(h) = &hit {
|
||||
if h.user.last_seen.unwrap_or(0.0) < now - 60.0 {
|
||||
conn.execute("UPDATE users SET last_seen=? WHERE id=?", params![now, h.user.id])?;
|
||||
}
|
||||
// Sliding expiry, written at most once a day.
|
||||
if h.expires_at < now + (SESSION_IDLE_DAYS - 1.0) * 86400.0 {
|
||||
conn.execute(
|
||||
"UPDATE sessions SET expires_at=? WHERE token_hash=? AND expires_at > ?",
|
||||
params![now + SESSION_IDLE_DAYS * 86400.0, hash_token(token), now + 120.0],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(hit)
|
||||
}
|
||||
|
||||
/// Replace a session with a new token. The old one keeps working for a minute so
|
||||
/// requests already in flight do not fail.
|
||||
pub fn rotate_session(conn: &Connection, old_token: &str, user_id: i64) -> rusqlite::Result<String> {
|
||||
let token = create_session(conn, user_id)?;
|
||||
conn.execute(
|
||||
"UPDATE sessions SET expires_at=MIN(expires_at, ?) WHERE token_hash=?",
|
||||
params![now() + 60.0, hash_token(old_token)],
|
||||
)?;
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
pub fn delete_session(conn: &Connection, token: &str) -> rusqlite::Result<()> {
|
||||
conn.execute("DELETE FROM sessions WHERE token_hash=?", [hash_token(token)])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_user_sessions(conn: &Connection, user_id: i64) -> rusqlite::Result<()> {
|
||||
conn.execute("DELETE FROM sessions WHERE user_id=?", [user_id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn active_since(conn: &Connection, ts: f64) -> rusqlite::Result<i64> {
|
||||
conn.query_row("SELECT COUNT(*) FROM users WHERE last_seen > ?", [ts], |r| r.get(0))
|
||||
}
|
||||
|
||||
/// Everything stored about one user (Art. 15 / 20 GDPR).
|
||||
pub fn export_user(conn: &Connection, user_id: i64) -> rusqlite::Result<serde_json::Value> {
|
||||
let user = conn.query_row("SELECT * FROM users WHERE id=?", [user_id], User::from_row)?;
|
||||
let sessions: Vec<serde_json::Value> = conn
|
||||
.prepare("SELECT created_at, expires_at FROM sessions WHERE user_id=? ORDER BY created_at")?
|
||||
.query_map([user_id], |r| {
|
||||
Ok(serde_json::json!({ "created_at": r.get::<_, f64>(0)?, "expires_at": r.get::<_, f64>(1)? }))
|
||||
})?
|
||||
.collect::<Result<_, _>>()?;
|
||||
let events: Vec<serde_json::Value> = conn
|
||||
.prepare(
|
||||
"SELECT e.ts, e.kind, e.seconds, e.track_id, t.handle, t.slug, t.title FROM events e \
|
||||
LEFT JOIN tracks t ON t.id = e.track_id WHERE e.user_id=? ORDER BY e.ts",
|
||||
)?
|
||||
.query_map([user_id], |r| {
|
||||
Ok(serde_json::json!({
|
||||
"ts": r.get::<_, f64>(0)?,
|
||||
"kind": r.get::<_, String>(1)?,
|
||||
"seconds": r.get::<_, Option<f64>>(2)?,
|
||||
"track_id": r.get::<_, i64>(3)?,
|
||||
"track": r.get::<_, Option<String>>(4)?.zip(r.get::<_, Option<String>>(5)?)
|
||||
.map(|(h, s)| format!("https://pl8list.com/{h}/{s}")),
|
||||
"title": r.get::<_, Option<String>>(6)?,
|
||||
}))
|
||||
})?
|
||||
.collect::<Result<_, _>>()?;
|
||||
Ok(serde_json::json!({
|
||||
"exported_at": now(),
|
||||
"user": user,
|
||||
"sessions": sessions,
|
||||
"events": events,
|
||||
"note": "Times are Unix timestamps in seconds. Session tokens are stored only as hashes and are not included.",
|
||||
}))
|
||||
}
|
||||
|
||||
/// Remove a user and everything linked to them (Art. 17 GDPR).
|
||||
pub fn delete_user(conn: &mut Connection, user_id: i64) -> rusqlite::Result<()> {
|
||||
let tx = conn.transaction()?;
|
||||
tx.execute("DELETE FROM events WHERE user_id=?", [user_id])?;
|
||||
tx.execute("DELETE FROM sessions WHERE user_id=?", [user_id])?;
|
||||
tx.execute("DELETE FROM users WHERE id=?", [user_id])?;
|
||||
tx.commit()
|
||||
}
|
||||
|
||||
/// Drop expired sessions and old "served" events. Returns (sessions, events) removed.
|
||||
pub fn purge(conn: &Connection) -> rusqlite::Result<(usize, usize)> {
|
||||
let now = now();
|
||||
let s = conn.execute("DELETE FROM sessions WHERE expires_at <= ?", [now])?;
|
||||
let e = conn.execute(
|
||||
"DELETE FROM events WHERE kind='served' AND ts < ?",
|
||||
[now - SERVED_RETENTION_DAYS * 86400.0],
|
||||
)?;
|
||||
Ok((s, e))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn mem() -> Connection {
|
||||
let c = Connection::open_in_memory().unwrap();
|
||||
migrate(&c).unwrap();
|
||||
c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sessions_store_only_hashes_and_rotate() {
|
||||
let c = mem();
|
||||
upsert_user(&c, 7, "someone", None, None).unwrap();
|
||||
let token = create_session(&c, 7).unwrap();
|
||||
let stored: String = c.query_row("SELECT token_hash FROM sessions", [], |r| r.get(0)).unwrap();
|
||||
assert_ne!(stored, token);
|
||||
assert_eq!(stored, hash_token(&token));
|
||||
assert_eq!(session_user(&c, &token).unwrap().unwrap().user.id, 7);
|
||||
|
||||
let new = rotate_session(&c, &token, 7).unwrap();
|
||||
assert!(session_user(&c, &new).unwrap().is_some());
|
||||
let old_expiry: f64 = c
|
||||
.query_row("SELECT expires_at FROM sessions WHERE token_hash=?", [hash_token(&token)], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert!(old_expiry <= now() + 61.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_user_removes_everything() {
|
||||
let mut c = mem();
|
||||
upsert_user(&c, 1, "a", None, None).unwrap();
|
||||
upsert_user(&c, 2, "b", None, None).unwrap();
|
||||
create_session(&c, 1).unwrap();
|
||||
create_session(&c, 2).unwrap();
|
||||
add_event(&c, 1, 10, "like", None).unwrap();
|
||||
add_event(&c, 2, 10, "like", None).unwrap();
|
||||
delete_user(&mut c, 1).unwrap();
|
||||
let count = |sql: &str| c.query_row(sql, [], |r| r.get::<_, i64>(0)).unwrap();
|
||||
assert_eq!(count("SELECT COUNT(*) FROM users WHERE id=1"), 0);
|
||||
assert_eq!(count("SELECT COUNT(*) FROM sessions WHERE user_id=1"), 0);
|
||||
assert_eq!(count("SELECT COUNT(*) FROM events WHERE user_id=1"), 0);
|
||||
assert_eq!(count("SELECT COUNT(*) FROM events WHERE user_id=2"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upsert_counts_new_tracks() {
|
||||
let mut c = mem();
|
||||
let t = Track { id: 1, handle: "h".into(), artist_name: "H".into(), slug: "s".into(), title: "T".into(), price: 4.5, ..Default::default() };
|
||||
assert_eq!(upsert_tracks(&mut c, std::slice::from_ref(&t), None).unwrap(), 1);
|
||||
assert_eq!(upsert_tracks(&mut c, &[t], None).unwrap(), 0);
|
||||
let price: f64 = c.query_row("SELECT price FROM tracks", [], |r| r.get(0)).unwrap();
|
||||
assert_eq!(price, 4.5);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
//! Data export, account deletion and signing out everywhere (Art. 15, 17, 20 GDPR).
|
||||
|
||||
use axum::{
|
||||
Extension, Json,
|
||||
extract::State,
|
||||
http::{StatusCode, header},
|
||||
response::IntoResponse,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{CurrentUser, Res, St, auth::signed_out, json_error};
|
||||
use crate::db;
|
||||
|
||||
pub async fn export(State(app): St, Extension(current): Extension<CurrentUser>) -> Res {
|
||||
let user = match current.require() {
|
||||
Ok(u) => u.clone(),
|
||||
Err(r) => return Ok(r),
|
||||
};
|
||||
let uid = user.id;
|
||||
let data = app.db.call(move |c| db::export_user(c, uid)).await?;
|
||||
let body = serde_json::to_vec_pretty(&data)?;
|
||||
let safe: String = user.handle.chars().filter(|c| c.is_ascii_alphanumeric() || "_.-".contains(*c)).collect();
|
||||
let disposition = format!("attachment; filename=\"pl8inf-{safe}.json\"");
|
||||
Ok((
|
||||
[
|
||||
(header::CONTENT_TYPE, "application/json".to_string()),
|
||||
(header::CACHE_CONTROL, "no-store".to_string()),
|
||||
(header::CONTENT_DISPOSITION, disposition),
|
||||
],
|
||||
body,
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
pub async fn delete(State(app): St, Extension(current): Extension<CurrentUser>, Json(body): Json<Value>) -> Res {
|
||||
let user = match current.require() {
|
||||
Ok(u) => u.clone(),
|
||||
Err(r) => return Ok(r),
|
||||
};
|
||||
if body.get("confirm").and_then(Value::as_str) != Some(&user.handle) {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "type your handle to confirm"));
|
||||
}
|
||||
let uid = user.id;
|
||||
app.db.call(move |c| db::delete_user(c, uid)).await?;
|
||||
tracing::info!("account deleted");
|
||||
Ok(signed_out(&app, json!({ "ok": true })))
|
||||
}
|
||||
|
||||
pub async fn logout_all(State(app): St, Extension(current): Extension<CurrentUser>) -> Res {
|
||||
let uid = match current.require() {
|
||||
Ok(u) => u.id,
|
||||
Err(r) => return Ok(r),
|
||||
};
|
||||
app.db.call(move |c| db::delete_user_sessions(c, uid)).await?;
|
||||
Ok(signed_out(&app, json!({ "ok": true })))
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
//! The feed, likes and listening events.
|
||||
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
|
||||
use axum::{
|
||||
Extension, Json,
|
||||
extract::{Query, State},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::time::Instant;
|
||||
|
||||
use super::{CurrentUser, Res, St, json_error, no_store};
|
||||
use crate::{db, ranker};
|
||||
|
||||
const EVENT_KINDS: &[&str] = &["play", "like", "unlike", "skip", "complete", "dislike"];
|
||||
/// On a cold start the crawler needs a few seconds before there is anything to serve.
|
||||
const FEED_WAIT: Duration = Duration::from_secs(10);
|
||||
|
||||
pub async fn me(Extension(current): Extension<CurrentUser>) -> Response {
|
||||
match current.user {
|
||||
Some(u) => (no_store(), Json(json!({ "user": { "handle": u.handle, "name": u.name, "img": u.img } }))).into_response(),
|
||||
None => json_error(StatusCode::UNAUTHORIZED, "not signed in"),
|
||||
}
|
||||
}
|
||||
|
||||
fn num(q: &HashMap<String, String>, name: &str) -> Option<f64> {
|
||||
q.get(name).and_then(|v| v.parse::<f64>().ok()).filter(|v| v.is_finite())
|
||||
}
|
||||
|
||||
pub async fn feed(State(app): St, Extension(current): Extension<CurrentUser>, Query(q): Query<HashMap<String, String>>) -> Res {
|
||||
let user = match current.require() {
|
||||
Ok(u) => u.clone(),
|
||||
Err(r) => return Ok(r),
|
||||
};
|
||||
let Some(ranker) = ranker::by_name(q.get("mode").map(String::as_str)) else {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "unknown mode"));
|
||||
};
|
||||
let n = num(&q, "n").map(|n| n as usize).unwrap_or(10).clamp(1, 30);
|
||||
let filters = ranker::Filters {
|
||||
free_only: q.get("free").map(String::as_str) == Some("1"),
|
||||
bpm_min: num(&q, "bpm_min"),
|
||||
bpm_max: num(&q, "bpm_max"),
|
||||
};
|
||||
let deadline = Instant::now() + FEED_WAIT;
|
||||
let uid = user.id;
|
||||
let tracks = loop {
|
||||
let tracks = app.db.call(move |c| ranker::pick(c, uid, n, filters, ranker)).await?;
|
||||
if !tracks.is_empty() || Instant::now() >= deadline {
|
||||
break tracks;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(1500)).await;
|
||||
};
|
||||
let ids: Vec<i64> = tracks.iter().filter_map(|t| t["id"].as_i64()).collect();
|
||||
let stats = app
|
||||
.db
|
||||
.call(move |c| {
|
||||
for id in ids {
|
||||
db::add_event(c, uid, id, "served", None)?;
|
||||
}
|
||||
let mut s = db::stats(c, uid)?;
|
||||
s.liked = ranker::taste(c, uid)?.liked_ids.len();
|
||||
Ok(s)
|
||||
})
|
||||
.await?;
|
||||
Ok((no_store(), Json(json!({ "tracks": tracks, "stats": stats }))).into_response())
|
||||
}
|
||||
|
||||
pub async fn likes(State(app): St, Extension(current): Extension<CurrentUser>) -> Res {
|
||||
let uid = match current.require() {
|
||||
Ok(u) => u.id,
|
||||
Err(r) => return Ok(r),
|
||||
};
|
||||
let tracks = app.db.call(move |c| ranker::likes(c, uid)).await?;
|
||||
Ok((no_store(), Json(json!({ "tracks": tracks }))).into_response())
|
||||
}
|
||||
|
||||
pub async fn event(State(app): St, Extension(current): Extension<CurrentUser>, Json(body): Json<Value>) -> Res {
|
||||
let uid = match current.require() {
|
||||
Ok(u) => u.id,
|
||||
Err(r) => return Ok(r),
|
||||
};
|
||||
let track_id = body.get("track_id").and_then(|v| v.as_i64().or_else(|| v.as_str()?.parse().ok()));
|
||||
let kind = body.get("kind").and_then(Value::as_str).filter(|k| EVENT_KINDS.contains(k));
|
||||
let seconds = match body.get("seconds") {
|
||||
None | Some(Value::Null) => Some(None),
|
||||
Some(v) => v.as_f64().filter(|s| s.is_finite() && *s >= 0.0).map(Some),
|
||||
};
|
||||
let (Some(track_id), Some(kind), Some(seconds)) = (track_id, kind, seconds) else {
|
||||
return Ok(StatusCode::BAD_REQUEST.into_response());
|
||||
};
|
||||
if !app.limits.events.allow(&uid.to_string()) {
|
||||
return Ok(StatusCode::TOO_MANY_REQUESTS.into_response());
|
||||
}
|
||||
let kind = kind.to_string();
|
||||
let k = kind.clone();
|
||||
app.db.call(move |c| db::add_event(c, uid, track_id, &k, seconds)).await?;
|
||||
if kind == "like" || kind == "complete" {
|
||||
app.crawler.on_positive(track_id).await;
|
||||
}
|
||||
Ok((no_store(), Json(json!({ "ok": true }))).into_response())
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
//! Sign-in through pl8list: email, emailed 6-digit code, then our own session cookie.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU32, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Extension, Json,
|
||||
extract::State,
|
||||
http::{HeaderMap, StatusCode, header},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{App, CurrentUser, LOGIN_COOKIE, Peer, Res, SESSION_COOKIE, St, client_ip, json_error, no_store, security};
|
||||
use crate::{db, pl8::login::LoginFlow};
|
||||
|
||||
const LOGIN_TTL: Duration = Duration::from_secs(900);
|
||||
const MAX_PENDING: usize = 500;
|
||||
const MAX_RESENDS: u32 = 2;
|
||||
const MAX_CODE_ATTEMPTS: u32 = 5;
|
||||
const TOO_MANY: &str = "too many attempts, wait a few minutes";
|
||||
|
||||
/// A login waiting for its emailed code, keyed by a random id in a short-lived cookie.
|
||||
pub struct Pending {
|
||||
created: Instant,
|
||||
attempts: AtomicU32,
|
||||
resends: AtomicU32,
|
||||
/// Also serialises resend/verify calls of one login.
|
||||
flow: tokio::sync::Mutex<LoginFlow>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct PendingLogins(Mutex<HashMap<String, Arc<Pending>>>);
|
||||
|
||||
impl PendingLogins {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, Arc<Pending>>> {
|
||||
self.0.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// Returns the new flow id, or `None` when too many logins are pending.
|
||||
fn add(&self, flow: LoginFlow) -> Option<(String, Arc<Pending>)> {
|
||||
let mut map = self.lock();
|
||||
map.retain(|_, p| p.created.elapsed() < LOGIN_TTL);
|
||||
if map.len() >= MAX_PENDING {
|
||||
return None;
|
||||
}
|
||||
let id = db::new_token();
|
||||
let p = Arc::new(Pending {
|
||||
created: Instant::now(),
|
||||
attempts: AtomicU32::new(0),
|
||||
resends: AtomicU32::new(0),
|
||||
flow: tokio::sync::Mutex::new(flow),
|
||||
});
|
||||
map.insert(id.clone(), p.clone());
|
||||
Some((id, p))
|
||||
}
|
||||
|
||||
fn get(&self, id: Option<&str>) -> Option<Arc<Pending>> {
|
||||
let p = self.lock().get(id?).cloned()?;
|
||||
(p.created.elapsed() < LOGIN_TTL).then_some(p)
|
||||
}
|
||||
|
||||
fn drop(&self, id: &str) {
|
||||
self.lock().remove(id);
|
||||
}
|
||||
|
||||
pub fn prune(&self) {
|
||||
self.lock().retain(|_, p| p.created.elapsed() < LOGIN_TTL);
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_email(e: &str) -> bool {
|
||||
let Some((local, domain)) = e.split_once('@') else { return false };
|
||||
e.len() <= 254
|
||||
&& (1..=64).contains(&local.len())
|
||||
&& !domain.contains('@')
|
||||
&& domain.split('.').count() >= 2
|
||||
&& domain.split('.').all(|p| !p.is_empty())
|
||||
&& !e.chars().any(|c| c.is_whitespace() || c.is_control())
|
||||
}
|
||||
|
||||
fn str_field(body: &Value, key: &str) -> String {
|
||||
body.get(key).and_then(Value::as_str).unwrap_or_default().trim().to_string()
|
||||
}
|
||||
|
||||
fn allow_login_mail(app: &App, ip: &str, email: &str) -> bool {
|
||||
app.limits.login_ip.allow(ip) && app.limits.login_email.allow(email) && app.limits.login_global.allow("")
|
||||
}
|
||||
|
||||
pub async fn start(State(app): St, peer: Peer, headers: HeaderMap, Json(body): Json<Value>) -> Res {
|
||||
let email = str_field(&body, "email").to_lowercase();
|
||||
if !valid_email(&email) {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "enter a valid email"));
|
||||
}
|
||||
if !allow_login_mail(&app, &client_ip(&app, &headers, peer.0), &email) {
|
||||
return Ok(json_error(StatusCode::TOO_MANY_REQUESTS, TOO_MANY));
|
||||
}
|
||||
let Some((id, pending)) = app.logins.add(LoginFlow::new(email.clone())) else {
|
||||
return Ok(json_error(StatusCode::TOO_MANY_REQUESTS, "too many sign-ins right now, try again in a few minutes"));
|
||||
};
|
||||
if let Err(e) = pending.flow.lock().await.start(&app.pl8).await {
|
||||
app.logins.drop(&id);
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, &e.0));
|
||||
}
|
||||
let cookie = app.cookies.set(LOGIN_COOKIE, &id, LOGIN_TTL.as_secs() as i64);
|
||||
Ok(([(header::SET_COOKIE, cookie)], no_store(), Json(json!({ "ok": true, "email": email }))).into_response())
|
||||
}
|
||||
|
||||
fn login_cookie<'a>(app: &App, headers: &'a HeaderMap) -> Option<&'a str> {
|
||||
security::cookie(headers, &app.cookies.name(LOGIN_COOKIE))
|
||||
}
|
||||
|
||||
fn expired() -> Response {
|
||||
json_error(StatusCode::BAD_REQUEST, "login expired, start again")
|
||||
}
|
||||
|
||||
pub async fn resend(State(app): St, peer: Peer, headers: HeaderMap) -> Res {
|
||||
let Some(pending) = app.logins.get(login_cookie(&app, &headers)) else { return Ok(expired()) };
|
||||
if pending.resends.fetch_add(1, Ordering::SeqCst) >= MAX_RESENDS {
|
||||
return Ok(json_error(StatusCode::TOO_MANY_REQUESTS, "no more resends, start again"));
|
||||
}
|
||||
let mut flow = pending.flow.lock().await;
|
||||
if !allow_login_mail(&app, &client_ip(&app, &headers, peer.0), &flow.email) {
|
||||
return Ok(json_error(StatusCode::TOO_MANY_REQUESTS, TOO_MANY));
|
||||
}
|
||||
if let Err(e) = flow.resend(&app.pl8).await {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, &e.0));
|
||||
}
|
||||
Ok((no_store(), Json(json!({ "ok": true }))).into_response())
|
||||
}
|
||||
|
||||
pub async fn verify(State(app): St, headers: HeaderMap, Json(body): Json<Value>) -> Res {
|
||||
let id = login_cookie(&app, &headers).map(String::from);
|
||||
let Some(pending) = app.logins.get(id.as_deref()) else { return Ok(expired()) };
|
||||
let id = id.unwrap_or_default();
|
||||
let code = str_field(&body, "code");
|
||||
if code.len() != 6 || !code.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Ok(json_error(StatusCode::BAD_REQUEST, "the code has 6 digits"));
|
||||
}
|
||||
if pending.attempts.fetch_add(1, Ordering::SeqCst) >= MAX_CODE_ATTEMPTS {
|
||||
app.logins.drop(&id);
|
||||
return Ok(json_error(StatusCode::TOO_MANY_REQUESTS, "too many wrong codes, start again"));
|
||||
}
|
||||
let result = pending.flow.lock().await.verify(&app.pl8, &code).await;
|
||||
let (user, pl8_session) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(e) => return Ok(json_error(StatusCode::BAD_REQUEST, &e.0)),
|
||||
};
|
||||
app.logins.drop(&id);
|
||||
// The pl8list session goes straight to the feed crawl, which logs it out afterwards.
|
||||
app.crawler.crawl_feed(pl8_session);
|
||||
let u = user.clone();
|
||||
let token = app
|
||||
.db
|
||||
.call(move |c| {
|
||||
db::upsert_user(c, u.id, &u.handle, u.name.as_deref(), u.img.as_deref())?;
|
||||
db::create_session(c, u.id)
|
||||
})
|
||||
.await?;
|
||||
tracing::info!("signed in: {}", user.handle);
|
||||
let mut res = (no_store(), Json(json!({ "user": { "handle": user.handle, "name": user.name } }))).into_response();
|
||||
let h = res.headers_mut();
|
||||
h.append(header::SET_COOKIE, app.cookies.set(SESSION_COOKIE, &token, (db::SESSION_IDLE_DAYS * 86400.0) as i64));
|
||||
h.append(header::SET_COOKIE, app.cookies.clear(LOGIN_COOKIE));
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub async fn logout(State(app): St, Extension(current): Extension<CurrentUser>) -> Res {
|
||||
if let Some(token) = current.token {
|
||||
app.db.call(move |c| db::delete_session(c, &token)).await?;
|
||||
}
|
||||
Ok(signed_out(&app, json!({ "ok": true })))
|
||||
}
|
||||
|
||||
/// Response that removes the session cookie (and the Python version's one).
|
||||
pub fn signed_out(app: &App, body: Value) -> Response {
|
||||
let mut res = (no_store(), Json(body)).into_response();
|
||||
res.headers_mut().append(header::SET_COOKIE, app.cookies.clear(SESSION_COOKIE));
|
||||
if app.cookies.secure {
|
||||
res.headers_mut().append(header::SET_COOKIE, app.cookies.clear_legacy(SESSION_COOKIE));
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn email_validation() {
|
||||
assert!(valid_email("a@b.co"));
|
||||
assert!(!valid_email("a@b"));
|
||||
assert!(!valid_email("a b@c.de"));
|
||||
assert!(!valid_email("@c.de"));
|
||||
assert!(!valid_email("a@@c.de"));
|
||||
assert!(!valid_email("a@c..de"));
|
||||
assert!(!valid_email(&format!("{}@c.de", "x".repeat(65))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
//! Impressum (§ 5 DDG, § 18 MStV) and Datenschutzerklärung (Art. 13 GDPR).
|
||||
//!
|
||||
//! The texts live in `templates/`; operator details come from the environment so
|
||||
//! they are not committed. Pages are rendered once at startup.
|
||||
|
||||
use axum::{
|
||||
http::header,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
|
||||
use crate::config::Legal;
|
||||
|
||||
const IMPRESSUM: &str = include_str!("../../templates/impressum.html");
|
||||
const DATENSCHUTZ: &str = include_str!("../../templates/datenschutz.html");
|
||||
const UPDATED: &str = "1. November 2026";
|
||||
|
||||
pub struct Pages {
|
||||
impressum: String,
|
||||
datenschutz: String,
|
||||
}
|
||||
|
||||
fn escape(s: &str) -> String {
|
||||
s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """).replace('\'', "'")
|
||||
}
|
||||
|
||||
fn lines(s: &str) -> String {
|
||||
s.lines().map(|l| escape(l.trim())).filter(|l| !l.is_empty()).collect::<Vec<_>>().join("<br>\n")
|
||||
}
|
||||
|
||||
fn fill(template: &str, l: &Legal) -> String {
|
||||
let missing = "<strong>[nicht konfiguriert]</strong>";
|
||||
let or_missing = |v: String| if v.is_empty() { missing.to_string() } else { v };
|
||||
let email = if l.email.is_empty() {
|
||||
missing.to_string()
|
||||
} else {
|
||||
format!("<a href=\"mailto:{0}\">{0}</a>", escape(&l.email))
|
||||
};
|
||||
let phone = if l.phone.is_empty() { String::new() } else { format!("<br>\nTelefon: {}", escape(&l.phone)) };
|
||||
template
|
||||
.replace("{{name}}", &or_missing(escape(&l.name)))
|
||||
.replace("{{address}}", &or_missing(lines(&l.address)))
|
||||
.replace("{{email}}", &email)
|
||||
.replace("{{phone}}", &phone)
|
||||
.replace("{{hosting}}", &or_missing(lines(&l.hosting)))
|
||||
.replace("{{updated}}", UPDATED)
|
||||
}
|
||||
|
||||
impl Pages {
|
||||
pub fn render(l: &Legal) -> Self {
|
||||
if !l.is_complete() {
|
||||
tracing::warn!("IMPRESSUM_NAME, IMPRESSUM_ADDRESS and IMPRESSUM_EMAIL are not all set; the Impressum is incomplete");
|
||||
}
|
||||
if l.hosting.is_empty() {
|
||||
tracing::warn!("HOSTING_PROVIDER is not set; the privacy policy names no hosting provider");
|
||||
}
|
||||
Pages { impressum: fill(IMPRESSUM, l), datenschutz: fill(DATENSCHUTZ, l) }
|
||||
}
|
||||
}
|
||||
|
||||
fn html(body: String) -> Response {
|
||||
([(header::CONTENT_TYPE, "text/html; charset=utf-8"), (header::CACHE_CONTROL, "no-cache")], body).into_response()
|
||||
}
|
||||
|
||||
pub async fn impressum(axum::extract::State(app): super::St) -> Response {
|
||||
html(app.legal.impressum.clone())
|
||||
}
|
||||
|
||||
pub async fn datenschutz(axum::extract::State(app): super::St) -> Response {
|
||||
html(app.legal.datenschutz.clone())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn operator_details_are_escaped() {
|
||||
let l = Legal { name: "A <b>".into(), address: "Str. 1\n12345 Ort".into(), email: "a@b.de".into(), ..Default::default() };
|
||||
let p = Pages::render(&l);
|
||||
assert!(p.impressum.contains("A <b>"));
|
||||
assert!(p.impressum.contains("Str. 1<br>\n12345 Ort"));
|
||||
assert!(!p.impressum.contains("{{"));
|
||||
assert!(!p.datenschutz.contains("{{"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
//! Routes, shared state and the middleware every request goes through.
|
||||
|
||||
pub mod account;
|
||||
pub mod app;
|
||||
pub mod auth;
|
||||
pub mod legal;
|
||||
pub mod proxy;
|
||||
pub mod security;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::{net::SocketAddr, sync::Arc, time::Duration};
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{ConnectInfo, DefaultBodyLimit, Path, Request, State},
|
||||
http::{HeaderMap, HeaderValue, Method, StatusCode, header},
|
||||
middleware::{self, Next},
|
||||
response::{IntoResponse, Response},
|
||||
routing::{get, post},
|
||||
};
|
||||
use rust_embed::Embed;
|
||||
use serde_json::json;
|
||||
use tower::limit::GlobalConcurrencyLimitLayer;
|
||||
use tower_http::timeout::TimeoutLayer;
|
||||
|
||||
use crate::{
|
||||
cache::Cache,
|
||||
config::Config,
|
||||
crawler::Crawler,
|
||||
db::{self, Db, User},
|
||||
pl8::Pl8,
|
||||
ratelimit::RateLimiter,
|
||||
};
|
||||
use security::Cookies;
|
||||
|
||||
pub const SESSION_COOKIE: &str = "pl8inf";
|
||||
pub const LOGIN_COOKIE: &str = "pl8inf_login";
|
||||
/// Sessions get a new token once they are this old.
|
||||
const ROTATE_AFTER: f64 = 7.0 * 86400.0;
|
||||
|
||||
pub struct Limits {
|
||||
pub login_ip: RateLimiter,
|
||||
pub login_email: RateLimiter,
|
||||
/// Every start and resend makes pl8list send a mail; cap them for the whole server.
|
||||
pub login_global: RateLimiter,
|
||||
pub events: RateLimiter,
|
||||
pub proxy: RateLimiter,
|
||||
}
|
||||
|
||||
pub struct App {
|
||||
pub cfg: Config,
|
||||
pub db: Db,
|
||||
pub pl8: Arc<Pl8>,
|
||||
pub crawler: Arc<Crawler>,
|
||||
pub cache: Cache,
|
||||
pub logins: auth::PendingLogins,
|
||||
pub limits: Limits,
|
||||
pub cookies: Cookies,
|
||||
pub segments: proxy::SegmentKeys,
|
||||
pub legal: legal::Pages,
|
||||
}
|
||||
|
||||
pub type St = State<Arc<App>>;
|
||||
|
||||
impl App {
|
||||
pub fn new(cfg: Config, db: Db, pl8: Arc<Pl8>, crawler: Arc<Crawler>) -> Arc<Self> {
|
||||
let cache = Cache::open(cfg.cache_dir.clone(), cfg.cache_max_bytes);
|
||||
let legal = legal::Pages::render(&cfg.legal);
|
||||
Arc::new(App {
|
||||
cookies: Cookies { secure: cfg.secure },
|
||||
limits: Limits {
|
||||
login_ip: RateLimiter::new(5, 900),
|
||||
login_email: RateLimiter::new(3, 900),
|
||||
login_global: RateLimiter::new(30, 600),
|
||||
events: RateLimiter::new(600, 600),
|
||||
proxy: RateLimiter::new(1200, 60),
|
||||
},
|
||||
logins: auth::PendingLogins::default(),
|
||||
segments: proxy::SegmentKeys::default(),
|
||||
cache,
|
||||
legal,
|
||||
cfg,
|
||||
db,
|
||||
pl8,
|
||||
crawler,
|
||||
})
|
||||
}
|
||||
|
||||
/// Hourly housekeeping.
|
||||
pub async fn purge(&self) {
|
||||
match self.db.call(|c| db::purge(c)).await {
|
||||
Ok((s, e)) if s + e > 0 => tracing::info!("purged {s} expired sessions, {e} old served events"),
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::warn!("purge failed: {e:#}"),
|
||||
}
|
||||
self.logins.prune();
|
||||
self.segments.prune();
|
||||
}
|
||||
}
|
||||
|
||||
// Errors ------------------------------------------------------------------
|
||||
|
||||
pub struct AppError(anyhow::Error);
|
||||
|
||||
impl<E: Into<anyhow::Error>> From<E> for AppError {
|
||||
fn from(e: E) -> Self {
|
||||
AppError(e.into())
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for AppError {
|
||||
fn into_response(self) -> Response {
|
||||
tracing::error!("{:#}", self.0);
|
||||
StatusCode::INTERNAL_SERVER_ERROR.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
pub type Res = Result<Response, AppError>;
|
||||
|
||||
pub fn json_error(status: StatusCode, msg: &str) -> Response {
|
||||
(status, no_store(), Json(json!({ "error": msg }))).into_response()
|
||||
}
|
||||
|
||||
pub fn no_store() -> [(header::HeaderName, &'static str); 1] {
|
||||
[(header::CACHE_CONTROL, "no-store")]
|
||||
}
|
||||
|
||||
// Session -----------------------------------------------------------------
|
||||
|
||||
/// The signed-in user, put into request extensions by `session`.
|
||||
#[derive(Clone)]
|
||||
pub struct CurrentUser {
|
||||
pub user: Option<User>,
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
impl CurrentUser {
|
||||
pub fn require(&self) -> Result<&User, Response> {
|
||||
self.user.as_ref().ok_or_else(|| StatusCode::UNAUTHORIZED.into_response())
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up the session cookie, rotate old tokens and move the Python version's
|
||||
/// unprefixed cookie to the `__Host-` one.
|
||||
async fn session(State(app): St, mut req: Request, next: Next) -> Response {
|
||||
let name = app.cookies.name(SESSION_COOKIE);
|
||||
let headers = req.headers();
|
||||
let (token, legacy) = match security::cookie(headers, &name) {
|
||||
Some(t) => (Some(t.to_string()), false),
|
||||
None if app.cookies.secure => (security::cookie(headers, SESSION_COOKIE).map(String::from), true),
|
||||
None => (None, false),
|
||||
};
|
||||
let mut set_cookie: Vec<HeaderValue> = vec![];
|
||||
let mut current = CurrentUser { user: None, token: token.clone() };
|
||||
if let Some(t) = token {
|
||||
let lookup = t.clone();
|
||||
if let Ok(Some(hit)) = app.db.call(move |c| db::session_user(c, &lookup)).await {
|
||||
let mut new_token = None;
|
||||
if hit.created_at < db::now() - ROTATE_AFTER {
|
||||
let (old, uid) = (t.clone(), hit.user.id);
|
||||
new_token = app.db.call(move |c| db::rotate_session(c, &old, uid)).await.ok();
|
||||
} else if legacy {
|
||||
new_token = Some(t.clone());
|
||||
}
|
||||
if let Some(nt) = new_token {
|
||||
set_cookie.push(app.cookies.set(SESSION_COOKIE, &nt, (db::SESSION_IDLE_DAYS * 86400.0) as i64));
|
||||
current.token = Some(nt);
|
||||
}
|
||||
current.user = Some(hit.user);
|
||||
}
|
||||
if legacy {
|
||||
set_cookie.push(app.cookies.clear_legacy(SESSION_COOKIE));
|
||||
}
|
||||
}
|
||||
req.extensions_mut().insert(current);
|
||||
let mut res = next.run(req).await;
|
||||
// Handlers that set the session cookie themselves (sign-in, sign-out) win.
|
||||
let handled = res.headers().get_all(header::SET_COOKIE).iter().any(|v| v.as_bytes().starts_with(format!("{name}=").as_bytes()));
|
||||
if !handled {
|
||||
for c in set_cookie {
|
||||
res.headers_mut().append(header::SET_COOKIE, c);
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
/// POSTs must come from our own pages.
|
||||
async fn origin_check(State(app): St, req: Request, next: Next) -> Response {
|
||||
if req.method() == Method::POST && !security::same_origin(req.headers(), app.cfg.public_origin.as_deref()) {
|
||||
return StatusCode::FORBIDDEN.into_response();
|
||||
}
|
||||
next.run(req).await
|
||||
}
|
||||
|
||||
pub fn client_ip(app: &App, headers: &HeaderMap, peer: SocketAddr) -> String {
|
||||
security::ip_key(security::client_ip(headers, peer, app.cfg.proxy_hops))
|
||||
}
|
||||
|
||||
pub type Peer = ConnectInfo<SocketAddr>;
|
||||
|
||||
// Static files ------------------------------------------------------------
|
||||
|
||||
#[derive(Embed)]
|
||||
#[folder = "../static/"]
|
||||
struct Static;
|
||||
|
||||
fn content_type(path: &str) -> &'static str {
|
||||
match path.rsplit('.').next().unwrap_or_default() {
|
||||
"html" => "text/html; charset=utf-8",
|
||||
"js" => "text/javascript; charset=utf-8",
|
||||
"css" => "text/css; charset=utf-8",
|
||||
"png" => "image/png",
|
||||
"svg" => "image/svg+xml",
|
||||
"webmanifest" => "application/manifest+json",
|
||||
"txt" => "text/plain; charset=utf-8",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
fn static_file(path: &str) -> Response {
|
||||
match Static::get(path) {
|
||||
Some(f) => ([(header::CONTENT_TYPE, content_type(path)), (header::CACHE_CONTROL, "no-cache")], f.data).into_response(),
|
||||
None => StatusCode::NOT_FOUND.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn index() -> Response {
|
||||
static_file("index.html")
|
||||
}
|
||||
|
||||
async fn static_route(Path(path): Path<String>) -> Response {
|
||||
static_file(&path)
|
||||
}
|
||||
|
||||
async fn healthz() -> Response {
|
||||
(no_store(), Json(json!({ "ok": true }))).into_response()
|
||||
}
|
||||
|
||||
// Router ------------------------------------------------------------------
|
||||
|
||||
pub fn router(app: Arc<App>) -> Router {
|
||||
let secure = app.cfg.secure;
|
||||
Router::new()
|
||||
.route("/", get(index))
|
||||
.route("/static/{*path}", get(static_route))
|
||||
.route("/healthz", get(healthz))
|
||||
.route("/impressum", get(legal::impressum))
|
||||
.route("/datenschutz", get(legal::datenschutz))
|
||||
.route("/privacy", get(legal::datenschutz))
|
||||
.route("/auth/start", post(auth::start))
|
||||
.route("/auth/resend", post(auth::resend))
|
||||
.route("/auth/verify", post(auth::verify))
|
||||
.route("/auth/logout", post(auth::logout))
|
||||
.route("/local/me", get(app::me))
|
||||
.route("/local/feed", get(app::feed))
|
||||
.route("/local/likes", get(app::likes))
|
||||
.route("/local/event", post(app::event))
|
||||
.route("/account/export", get(account::export))
|
||||
.route("/account/delete", post(account::delete))
|
||||
.route("/account/logout-all", post(account::logout_all))
|
||||
.route("/api/{handle}/{slug}/hls/playlist.m3u8", get(proxy::playlist))
|
||||
.route("/api/{handle}/{slug}/hls/segment/{n}", get(proxy::segment))
|
||||
.route("/img/{id}", get(proxy::image))
|
||||
.layer(middleware::from_fn_with_state(app.clone(), session))
|
||||
.layer(middleware::from_fn_with_state(app.clone(), origin_check))
|
||||
.layer(DefaultBodyLimit::max(4096))
|
||||
.layer(TimeoutLayer::with_status_code(StatusCode::REQUEST_TIMEOUT, Duration::from_secs(30)))
|
||||
.layer(GlobalConcurrencyLimitLayer::new(512))
|
||||
.layer(middleware::from_fn(move |req, next| security::headers(secure, req, next)))
|
||||
.with_state(app)
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
//! Audio and image proxy.
|
||||
//!
|
||||
//! The browser only talks to us, so visitors' addresses never reach pl8list. Only
|
||||
//! HLS playlists, their segments and artist images are proxied, each with a fixed
|
||||
//! or allowlisted content type, so nothing pl8list returns can run as script on
|
||||
//! our origin. Previews and free tracks stream fine as a guest.
|
||||
//!
|
||||
//! Segments and images are cached on disk. A segment URL carries a signed token
|
||||
//! (`st`) that changes with every playlist, so the cache key is the upload id inside
|
||||
//! that token. We cannot check pl8list's signature, so we only trust tokens that
|
||||
//! came out of a playlist we fetched ourselves.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Extension,
|
||||
extract::{Path, Query, State},
|
||||
http::{StatusCode, header},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::StreamExt;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{App, CurrentUser, Res, St};
|
||||
use crate::cache::{self, IMAGE_TYPES, SEGMENT_TYPES};
|
||||
|
||||
const MAX_PLAYLIST: usize = 256 << 10;
|
||||
const MAX_SEGMENT: usize = 16 << 20;
|
||||
const MAX_IMAGE: usize = 5 << 20;
|
||||
const TOKEN_TTL: Duration = Duration::from_secs(6 * 3600);
|
||||
const MAX_TOKENS: usize = 50_000;
|
||||
|
||||
/// Segment tokens seen in playlists from pl8list, with the upload they belong to.
|
||||
#[derive(Default)]
|
||||
pub struct SegmentKeys(Mutex<HashMap<String, (String, Instant)>>);
|
||||
|
||||
impl SegmentKeys {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, (String, Instant)>> {
|
||||
self.0.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
fn insert(&self, token: String, upload: String) {
|
||||
let mut m = self.lock();
|
||||
if m.len() >= MAX_TOKENS {
|
||||
m.retain(|_, (_, t)| t.elapsed() < TOKEN_TTL);
|
||||
if m.len() >= MAX_TOKENS {
|
||||
m.clear();
|
||||
}
|
||||
}
|
||||
m.insert(token, (upload, Instant::now()));
|
||||
}
|
||||
|
||||
fn upload(&self, token: &str) -> Option<String> {
|
||||
self.lock().get(token).filter(|(_, t)| t.elapsed() < TOKEN_TTL).map(|(u, _)| u.clone())
|
||||
}
|
||||
|
||||
pub fn prune(&self) {
|
||||
self.lock().retain(|_, (_, t)| t.elapsed() < TOKEN_TTL);
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles, slugs and upload ids on pl8list only use these characters.
|
||||
fn safe_segment(s: &str, max: usize) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.len() <= max
|
||||
&& s != "."
|
||||
&& s != ".."
|
||||
&& s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'.' || b == b'-')
|
||||
}
|
||||
|
||||
/// The upload id pl8list put into a segment token (`base64url(json).signature`).
|
||||
fn token_upload(token: &str) -> Option<String> {
|
||||
let payload = URL_SAFE_NO_PAD.decode(token.split('.').next()?).ok()?;
|
||||
let v: Value = serde_json::from_slice(&payload).ok()?;
|
||||
v.get("u")?.as_str().filter(|u| safe_segment(u, 128)).map(String::from)
|
||||
}
|
||||
|
||||
fn require_user(app: &App, current: &CurrentUser) -> Result<(), Response> {
|
||||
let user = current.require()?;
|
||||
if !app.limits.proxy.allow(&user.id.to_string()) {
|
||||
return Err(StatusCode::TOO_MANY_REQUESTS.into_response());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read an upstream body, giving up past `max` bytes.
|
||||
async fn read_capped(res: reqwest::Response, max: usize) -> Result<Bytes, StatusCode> {
|
||||
if res.content_length().is_some_and(|l| l as usize > max) {
|
||||
return Err(StatusCode::BAD_GATEWAY);
|
||||
}
|
||||
let mut body = BytesMut::new();
|
||||
let mut stream = res.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|_| StatusCode::BAD_GATEWAY)?;
|
||||
if body.len() + chunk.len() > max {
|
||||
return Err(StatusCode::BAD_GATEWAY);
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(body.freeze())
|
||||
}
|
||||
|
||||
async fn fetch(app: &App, url: url::Url) -> Result<reqwest::Response, StatusCode> {
|
||||
let res = app.pl8.request(url, None).send().await.map_err(|e| {
|
||||
tracing::debug!("proxy: {}", e.without_url());
|
||||
StatusCode::BAD_GATEWAY
|
||||
})?;
|
||||
match res.status().as_u16() {
|
||||
200 => Ok(res),
|
||||
403 => Err(StatusCode::FORBIDDEN),
|
||||
404 => Err(StatusCode::NOT_FOUND),
|
||||
_ => Err(StatusCode::BAD_GATEWAY),
|
||||
}
|
||||
}
|
||||
|
||||
fn media(ctype: &'static str, body: Bytes, cache_control: &'static str) -> Response {
|
||||
([(header::CONTENT_TYPE, ctype), (header::CACHE_CONTROL, cache_control)], body).into_response()
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct PlaylistQuery {
|
||||
v: Option<String>,
|
||||
preview: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn playlist(
|
||||
State(app): St,
|
||||
Extension(current): Extension<CurrentUser>,
|
||||
Path((handle, slug)): Path<(String, String)>,
|
||||
Query(q): Query<PlaylistQuery>,
|
||||
) -> Res {
|
||||
if let Err(r) = require_user(&app, ¤t) {
|
||||
return Ok(r);
|
||||
}
|
||||
if !safe_segment(&handle, 64) || !safe_segment(&slug, 160) || q.v.as_deref().is_some_and(|v| !safe_segment(v, 128)) {
|
||||
return Ok(StatusCode::NOT_FOUND.into_response());
|
||||
}
|
||||
let mut url = app.pl8.url(&["api", &handle, &slug, "hls", "playlist.m3u8"]);
|
||||
{
|
||||
let mut qp = url.query_pairs_mut();
|
||||
if let Some(v) = &q.v {
|
||||
qp.append_pair("v", v);
|
||||
}
|
||||
if q.preview.as_deref() == Some("true") {
|
||||
qp.append_pair("preview", "true");
|
||||
}
|
||||
}
|
||||
let body = match fetch(&app, url).await {
|
||||
Ok(res) => match read_capped(res, MAX_PLAYLIST).await {
|
||||
Ok(b) => b,
|
||||
Err(s) => return Ok(s.into_response()),
|
||||
},
|
||||
Err(s) => return Ok(s.into_response()),
|
||||
};
|
||||
let Ok(text) = std::str::from_utf8(&body) else { return Ok(StatusCode::BAD_GATEWAY.into_response()) };
|
||||
if !text.starts_with("#EXTM3U") {
|
||||
return Ok(StatusCode::BAD_GATEWAY.into_response());
|
||||
}
|
||||
for line in text.lines().filter(|l| l.starts_with("segment/")) {
|
||||
if let Some(token) = line.split_once("st=").map(|(_, t)| t.split('&').next().unwrap_or_default())
|
||||
&& let Some(upload) = token_upload(token) {
|
||||
app.segments.insert(token.to_string(), upload);
|
||||
}
|
||||
}
|
||||
Ok(media("application/vnd.apple.mpegurl", body, "no-store"))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SegmentQuery {
|
||||
st: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn segment(
|
||||
State(app): St,
|
||||
Extension(current): Extension<CurrentUser>,
|
||||
Path((handle, slug, n)): Path<(String, String, String)>,
|
||||
Query(q): Query<SegmentQuery>,
|
||||
) -> Res {
|
||||
if let Err(r) = require_user(&app, ¤t) {
|
||||
return Ok(r);
|
||||
}
|
||||
let token = q.st.unwrap_or_default();
|
||||
let valid_token = token.len() <= 2048 && token.bytes().all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b));
|
||||
let Ok(n) = n.parse::<u32>() else { return Ok(StatusCode::NOT_FOUND.into_response()) };
|
||||
if !safe_segment(&handle, 64) || !safe_segment(&slug, 160) || !valid_token {
|
||||
return Ok(StatusCode::NOT_FOUND.into_response());
|
||||
}
|
||||
let key = app.segments.upload(&token).map(|u| format!("seg:{u}:{n}"));
|
||||
if let Some(k) = &key
|
||||
&& let Some((body, ctype)) = app.cache.get(k).await {
|
||||
return Ok(media(ctype, body, "private, max-age=86400"));
|
||||
}
|
||||
let mut url = app.pl8.url(&["api", &handle, &slug, "hls", "segment", &n.to_string()]);
|
||||
if !token.is_empty() {
|
||||
url.query_pairs_mut().append_pair("st", &token);
|
||||
}
|
||||
let res = match fetch(&app, url).await {
|
||||
Ok(r) => r,
|
||||
Err(s) => return Ok(s.into_response()),
|
||||
};
|
||||
let upstream_type = res.headers().get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok());
|
||||
let Some((ctype, ext)) = cache::allowed_type(upstream_type, SEGMENT_TYPES) else {
|
||||
return Ok(StatusCode::BAD_GATEWAY.into_response());
|
||||
};
|
||||
let body = match read_capped(res, MAX_SEGMENT).await {
|
||||
Ok(b) => b,
|
||||
Err(s) => return Ok(s.into_response()),
|
||||
};
|
||||
if let Some(k) = &key {
|
||||
app.cache.put(k, ext, &body).await;
|
||||
}
|
||||
Ok(media(ctype, body, "private, max-age=86400"))
|
||||
}
|
||||
|
||||
pub async fn image(State(app): St, Extension(current): Extension<CurrentUser>, Path(id): Path<String>) -> Res {
|
||||
if let Err(r) = require_user(&app, ¤t) {
|
||||
return Ok(r);
|
||||
}
|
||||
if id.len() > 64 || id.is_empty() || !id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') {
|
||||
return Ok(StatusCode::NOT_FOUND.into_response());
|
||||
}
|
||||
let key = format!("img:{id}");
|
||||
if let Some((body, ctype)) = app.cache.get(&key).await {
|
||||
return Ok(media(ctype, body, "private, max-age=86400"));
|
||||
}
|
||||
let res = match fetch(&app, app.pl8.url(&["img", &id])).await {
|
||||
Ok(r) => r,
|
||||
Err(s) => return Ok(s.into_response()),
|
||||
};
|
||||
let upstream_type = res.headers().get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok());
|
||||
let Some((ctype, ext)) = cache::allowed_type(upstream_type, IMAGE_TYPES) else {
|
||||
return Ok(StatusCode::BAD_GATEWAY.into_response());
|
||||
};
|
||||
let body = match read_capped(res, MAX_IMAGE).await {
|
||||
Ok(b) => b,
|
||||
Err(s) => return Ok(s.into_response()),
|
||||
};
|
||||
app.cache.put(&key, ext, &body).await;
|
||||
Ok(media(ctype, body, "private, max-age=86400"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn path_segments() {
|
||||
assert!(safe_segment("garp-chorus-flute-vip-final-v5", 160));
|
||||
assert!(!safe_segment("..", 160));
|
||||
assert!(!safe_segment(".", 160));
|
||||
assert!(!safe_segment("a/b", 160));
|
||||
assert!(!safe_segment("a%2Fb", 160));
|
||||
assert!(!safe_segment("", 160));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_upload_from_token() {
|
||||
let t = "eyJ1IjoiMjU1YzhmZGUtYTUyYy00Njc2LTk2MzgtMGExOGZkNTRmN2UyLndhdiIsImgiOiJnYXJwYmUiLCJzIjoiZ2FycC1zY290dGlzY2gtc3F1YXJlLWZpbmFsLXYzLTIiLCJlIjoxNzkzNTMwMTgwfQ.nPGK9LdPDvmHDwhqlCbHbaEtJhLLqCwUIS26MlCvYR8";
|
||||
assert_eq!(token_upload(t).as_deref(), Some("255c8fde-a52c-4676-9638-0a18fd54f7e2.wav"));
|
||||
assert_eq!(token_upload("garbage"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_tokens_get_no_cache_key() {
|
||||
let keys = SegmentKeys::default();
|
||||
keys.insert("known".into(), "u1".into());
|
||||
assert_eq!(keys.upload("known").as_deref(), Some("u1"));
|
||||
assert_eq!(keys.upload("forged"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
//! Client address, Origin check, cookies and response headers.
|
||||
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
use axum::{
|
||||
extract::Request,
|
||||
http::{HeaderMap, HeaderValue, header},
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
|
||||
/// The address rate limits use.
|
||||
///
|
||||
/// Each trusted proxy appends the address it saw to `X-Forwarded-For`, so with
|
||||
/// `hops` proxies the client is the `hops`-th entry from the right. Everything to
|
||||
/// the left of it was written by the client and is ignored.
|
||||
pub fn client_ip(headers: &HeaderMap, peer: SocketAddr, hops: usize) -> IpAddr {
|
||||
if hops == 0 {
|
||||
return peer.ip();
|
||||
}
|
||||
let entries: Vec<&str> = headers
|
||||
.get_all("x-forwarded-for")
|
||||
.iter()
|
||||
.filter_map(|v| v.to_str().ok())
|
||||
.flat_map(|v| v.split(','))
|
||||
.map(str::trim)
|
||||
.collect();
|
||||
entries
|
||||
.len()
|
||||
.checked_sub(hops)
|
||||
.and_then(|i| entries[i].parse().ok())
|
||||
.unwrap_or(peer.ip())
|
||||
}
|
||||
|
||||
/// Rate-limit key for an address. IPv6 users usually get a whole /64, so limit by that.
|
||||
pub fn ip_key(ip: IpAddr) -> String {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => v4.to_string(),
|
||||
IpAddr::V6(v6) => match v6.to_ipv4_mapped() {
|
||||
Some(v4) => v4.to_string(),
|
||||
None => {
|
||||
let s = v6.segments();
|
||||
format!("{:x}:{:x}:{:x}:{:x}::/64", s[0], s[1], s[2], s[3])
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Reject cross-site POSTs. Browsers send Origin on every POST and Sec-Fetch-Site
|
||||
/// on every request; either one pointing elsewhere is enough to refuse.
|
||||
pub fn same_origin(headers: &HeaderMap, public_origin: Option<&str>) -> bool {
|
||||
if let Some(site) = headers.get("sec-fetch-site").and_then(|v| v.to_str().ok())
|
||||
&& site != "same-origin" && site != "none" {
|
||||
return false;
|
||||
}
|
||||
let Some(origin) = headers.get(header::ORIGIN).and_then(|v| v.to_str().ok()) else { return true };
|
||||
match public_origin {
|
||||
Some(expected) => origin == expected,
|
||||
None => {
|
||||
let host = headers.get(header::HOST).and_then(|v| v.to_str().ok()).unwrap_or_default();
|
||||
origin.split_once("://").map(|(_, h)| h) == Some(host)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cookie<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
|
||||
headers
|
||||
.get_all(header::COOKIE)
|
||||
.iter()
|
||||
.filter_map(|v| v.to_str().ok())
|
||||
.flat_map(|v| v.split(';'))
|
||||
.filter_map(|c| c.trim().split_once('='))
|
||||
.find(|(k, _)| *k == name)
|
||||
.map(|(_, v)| v)
|
||||
.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
pub struct Cookies {
|
||||
pub secure: bool,
|
||||
}
|
||||
|
||||
impl Cookies {
|
||||
/// `__Host-` makes browsers refuse the cookie unless it is Secure, host-only and Path=/.
|
||||
pub fn name(&self, base: &str) -> String {
|
||||
if self.secure { format!("__Host-{base}") } else { base.to_string() }
|
||||
}
|
||||
|
||||
pub fn set(&self, base: &str, value: &str, max_age: i64) -> HeaderValue {
|
||||
let secure = if self.secure { "; Secure" } else { "" };
|
||||
let v = format!("{}={value}; Path=/; HttpOnly; SameSite=Lax; Max-Age={max_age}{secure}", self.name(base));
|
||||
HeaderValue::from_str(&v).expect("cookie values are URL-safe base64")
|
||||
}
|
||||
|
||||
pub fn clear(&self, base: &str) -> HeaderValue {
|
||||
self.set(base, "", 0)
|
||||
}
|
||||
|
||||
/// Remove a cookie that was set without the prefix by the Python version.
|
||||
pub fn clear_legacy(&self, name: &str) -> HeaderValue {
|
||||
HeaderValue::from_str(&format!("{name}=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0")).expect("static")
|
||||
}
|
||||
}
|
||||
|
||||
const CSP: &str = "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; \
|
||||
media-src 'self' blob:; worker-src 'self' blob:; connect-src 'self'; object-src 'none'; \
|
||||
base-uri 'none'; form-action 'self'; frame-ancestors 'none'";
|
||||
|
||||
const PERMISSIONS: &str = "camera=(), microphone=(), geolocation=(), payment=(), usb=(), \
|
||||
browsing-topics=(), interest-cohort=()";
|
||||
|
||||
/// Headers on every response.
|
||||
pub async fn headers(secure: bool, req: Request, next: Next) -> Response {
|
||||
let mut res = next.run(req).await;
|
||||
let h = res.headers_mut();
|
||||
let set = |h: &mut HeaderMap, k: &'static str, v: &'static str| {
|
||||
h.entry(k).or_insert(HeaderValue::from_static(v));
|
||||
};
|
||||
set(h, "content-security-policy", CSP);
|
||||
set(h, "x-content-type-options", "nosniff");
|
||||
set(h, "referrer-policy", "strict-origin-when-cross-origin");
|
||||
set(h, "x-frame-options", "DENY");
|
||||
set(h, "permissions-policy", PERMISSIONS);
|
||||
set(h, "cross-origin-opener-policy", "same-origin");
|
||||
set(h, "cross-origin-resource-policy", "same-origin");
|
||||
if secure {
|
||||
set(h, "strict-transport-security", "max-age=31536000; includeSubDomains");
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn h(pairs: &[(&'static str, &'static str)]) -> HeaderMap {
|
||||
let mut m = HeaderMap::new();
|
||||
for (k, v) in pairs {
|
||||
m.append(*k, HeaderValue::from_static(v));
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xff_uses_entry_added_by_proxy() {
|
||||
let peer: SocketAddr = "172.18.0.2:5000".parse().unwrap();
|
||||
let spoofed = h(&[("x-forwarded-for", "6.6.6.6, 203.0.113.9")]);
|
||||
assert_eq!(client_ip(&spoofed, peer, 1).to_string(), "203.0.113.9");
|
||||
assert_eq!(client_ip(&spoofed, peer, 2).to_string(), "6.6.6.6");
|
||||
assert_eq!(client_ip(&spoofed, peer, 0), peer.ip());
|
||||
// Fewer entries than proxies, or garbage: fall back to the peer.
|
||||
assert_eq!(client_ip(&spoofed, peer, 3), peer.ip());
|
||||
assert_eq!(client_ip(&h(&[("x-forwarded-for", "nonsense")]), peer, 1), peer.ip());
|
||||
assert_eq!(client_ip(&HeaderMap::new(), peer, 1), peer.ip());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipv6_is_grouped_by_64() {
|
||||
let a: IpAddr = "2001:db8:1:2:aaaa::1".parse().unwrap();
|
||||
let b: IpAddr = "2001:db8:1:2:bbbb::9".parse().unwrap();
|
||||
assert_eq!(ip_key(a), ip_key(b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn origin_check() {
|
||||
assert!(same_origin(&h(&[("host", "pl8.narl.io"), ("origin", "https://pl8.narl.io")]), None));
|
||||
assert!(!same_origin(&h(&[("host", "pl8.narl.io"), ("origin", "https://evil.example")]), None));
|
||||
assert!(!same_origin(&h(&[("sec-fetch-site", "cross-site")]), None));
|
||||
assert!(same_origin(&h(&[("origin", "https://pl8.narl.io")]), Some("https://pl8.narl.io")));
|
||||
assert!(!same_origin(&h(&[("origin", "http://pl8.narl.io")]), Some("https://pl8.narl.io")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cookie_attributes() {
|
||||
let c = Cookies { secure: true };
|
||||
let v = c.set("pl8inf", "tok", 60);
|
||||
assert_eq!(v.to_str().unwrap(), "__Host-pl8inf=tok; Path=/; HttpOnly; SameSite=Lax; Max-Age=60; Secure");
|
||||
let jar = h(&[("cookie", "a=1; __Host-pl8inf=tok; b=")]);
|
||||
assert_eq!(cookie(&jar, "__Host-pl8inf"), Some("tok"));
|
||||
assert_eq!(cookie(&jar, "b"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
//! End-to-end tests of the HTTP API against a fake pl8list.
|
||||
|
||||
use std::{net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::Body,
|
||||
extract::ConnectInfo,
|
||||
http::{Request, StatusCode, header},
|
||||
};
|
||||
use base64::Engine;
|
||||
use tower::ServiceExt;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{body_string_contains, header as has_header, method, path, query_param},
|
||||
};
|
||||
|
||||
use super::{App, router};
|
||||
use crate::{config::{Config, Legal}, crawler::Crawler, db::Db, pl8::Pl8};
|
||||
|
||||
const LOGGED_IN: &str = "LOGGED-IN-PL8-SESSION-SECRET";
|
||||
|
||||
fn db_path(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("pl8inf-test-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir.join("test.db")
|
||||
}
|
||||
|
||||
async fn setup(name: &str, pl8_base: &str) -> (Router, Arc<App>, PathBuf) {
|
||||
let db_path = db_path(name);
|
||||
let cfg = Config {
|
||||
host: "127.0.0.1".into(),
|
||||
port: 0,
|
||||
db_path: db_path.clone(),
|
||||
proxy_hops: 1,
|
||||
public_origin: Some("https://pl8.example".into()),
|
||||
secure: true,
|
||||
cache_dir: db_path.parent().unwrap().join("cache"),
|
||||
cache_max_bytes: 1 << 20,
|
||||
pl8_base: pl8_base.into(),
|
||||
legal: Legal::default(),
|
||||
};
|
||||
let db = Db::open(&db_path).unwrap();
|
||||
let pl8 = Arc::new(Pl8::new(pl8_base, Duration::ZERO).unwrap());
|
||||
let crawler = Crawler::new(pl8.clone(), db.clone());
|
||||
let app = App::new(cfg, db, pl8, crawler);
|
||||
(router(app.clone()), app, db_path)
|
||||
}
|
||||
|
||||
fn req(method: &str, uri: &str, body: Option<&str>, cookie: Option<&str>, xff: &str) -> Request<Body> {
|
||||
let mut b = Request::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header(header::HOST, "pl8.example")
|
||||
.header("x-forwarded-for", xff);
|
||||
if body.is_some() {
|
||||
b = b.header(header::CONTENT_TYPE, "application/json").header(header::ORIGIN, "https://pl8.example");
|
||||
}
|
||||
if let Some(c) = cookie {
|
||||
b = b.header(header::COOKIE, c);
|
||||
}
|
||||
let mut r = b.body(Body::from(body.unwrap_or_default().to_string())).unwrap();
|
||||
r.extensions_mut().insert(ConnectInfo(SocketAddr::from(([172, 18, 0, 2], 4000))));
|
||||
r
|
||||
}
|
||||
|
||||
fn set_cookies(res: &axum::response::Response) -> Vec<String> {
|
||||
res.headers().get_all(header::SET_COOKIE).iter().map(|v| v.to_str().unwrap().to_string()).collect()
|
||||
}
|
||||
|
||||
fn cookie_pair(cookies: &[String], name: &str) -> String {
|
||||
cookies.iter().find(|c| c.starts_with(&format!("{name}="))).unwrap().split(';').next().unwrap().to_string()
|
||||
}
|
||||
|
||||
async fn fake_pl8list(logouts: u64) -> MockServer {
|
||||
let server = MockServer::start().await;
|
||||
let guest = base64::engine::general_purpose::STANDARD.encode(r#"{"pl8list_guest_id":"g-1"}"#);
|
||||
Mock::given(method("GET")).and(path("/login"))
|
||||
.respond_with(ResponseTemplate::new(200).insert_header("set-cookie", format!("_session={guest}.sig; Path=/; HttpOnly").as_str()))
|
||||
.mount(&server).await;
|
||||
Mock::given(method("POST")).and(path("/login")).and(body_string_contains("pl8list_guest_id=g-1"))
|
||||
.respond_with(ResponseTemplate::new(204).insert_header("x-remix-redirect", "/verify?email=a%40b.de&mode=login"))
|
||||
.mount(&server).await;
|
||||
Mock::given(method("POST")).and(path("/verify")).and(body_string_contains("code=000000"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_string(r#"{"error":"wrong code"}"#))
|
||||
.mount(&server).await;
|
||||
Mock::given(method("POST")).and(path("/verify")).and(body_string_contains("code=123456"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_string("{}")
|
||||
.insert_header("set-cookie", format!("_session={LOGGED_IN}; Path=/; HttpOnly").as_str()))
|
||||
.mount(&server).await;
|
||||
Mock::given(method("GET")).and(path("/")).and(query_param("_data", "root"))
|
||||
.and(has_header("cookie", format!("_session={LOGGED_IN}").as_str()))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_string(r#"{"currentUser":{"id":42,"handle":"tester","name":"T"}}"#))
|
||||
.mount(&server).await;
|
||||
Mock::given(method("GET")).and(path("/api/feed"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_string(r#"{"tracks":[],"totalPages":1}"#))
|
||||
.mount(&server).await;
|
||||
Mock::given(method("GET")).and(path("/logout"))
|
||||
.and(has_header("cookie", format!("_session={LOGGED_IN}").as_str()))
|
||||
.respond_with(ResponseTemplate::new(204))
|
||||
.expect(logouts)
|
||||
.named("logout of the user's pl8list session")
|
||||
.mount(&server).await;
|
||||
server
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn login_logs_pl8list_session_out_and_never_stores_it() {
|
||||
let pl8 = fake_pl8list(1).await;
|
||||
let (r, _app, db_path) = setup("login", &pl8.uri()).await;
|
||||
|
||||
let res = r.clone().oneshot(req("POST", "/auth/start", Some(r#"{"email":"A@b.de"}"#), None, "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let login = cookie_pair(&set_cookies(&res), "__Host-pl8inf_login");
|
||||
|
||||
let res = r.clone().oneshot(req("POST", "/auth/verify", Some(r#"{"code":"000000"}"#), Some(&login), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::BAD_REQUEST);
|
||||
|
||||
let res = r.clone().oneshot(req("POST", "/auth/verify", Some(r#"{"code":"123456"}"#), Some(&login), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let cookies = set_cookies(&res);
|
||||
assert!(cookies.iter().any(|c| c.starts_with("__Host-pl8inf=") && c.contains("Secure") && c.contains("HttpOnly")));
|
||||
let session = cookie_pair(&cookies, "__Host-pl8inf");
|
||||
|
||||
let res = r.clone().oneshot(req("GET", "/local/me", None, Some(&session), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
|
||||
// The feed crawl logs the pl8list session out in the background.
|
||||
for _ in 0..50 {
|
||||
let logged_out = pl8.received_requests().await.unwrap().iter().any(|q| q.url.path() == "/logout");
|
||||
if logged_out {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
pl8.verify().await;
|
||||
|
||||
let raw = std::fs::read(&db_path).unwrap();
|
||||
let wal = std::fs::read(db_path.with_extension("db-wal")).unwrap_or_default();
|
||||
let needle = LOGGED_IN.as_bytes();
|
||||
assert!(!raw.windows(needle.len()).any(|w| w == needle), "pl8list session found in database file");
|
||||
assert!(!wal.windows(needle.len()).any(|w| w == needle), "pl8list session found in WAL");
|
||||
assert!(!raw.windows(5).any(|w| w == b"a@b.d") && !wal.windows(5).any(|w| w == b"a@b.d"), "email stored");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spoofed_forwarded_for_does_not_reset_ip_limit() {
|
||||
let pl8 = fake_pl8list(0).await;
|
||||
let (r, _app, _) = setup("xff", &pl8.uri()).await;
|
||||
let mut statuses = vec![];
|
||||
for i in 0..7 {
|
||||
let body = format!(r#"{{"email":"u{i}@b.de"}}"#);
|
||||
let xff = format!("6.6.6.{i}, 198.51.100.7");
|
||||
statuses.push(r.clone().oneshot(req("POST", "/auth/start", Some(&body), None, &xff)).await.unwrap().status());
|
||||
}
|
||||
assert_eq!(&statuses[..5], &[StatusCode::OK; 5]);
|
||||
assert_eq!(&statuses[5..], &[StatusCode::TOO_MANY_REQUESTS; 2]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cross_site_posts_and_anonymous_reads_are_refused() {
|
||||
let (r, _app, _) = setup("csrf", "http://127.0.0.1:9").await;
|
||||
let mut cross = req("POST", "/auth/logout", Some("{}"), None, "203.0.113.1");
|
||||
cross.headers_mut().insert(header::ORIGIN, "https://evil.example".parse().unwrap());
|
||||
assert_eq!(r.clone().oneshot(cross).await.unwrap().status(), StatusCode::FORBIDDEN);
|
||||
|
||||
for uri in ["/local/feed", "/local/likes", "/account/export", "/img/abc", "/api/a/b/hls/playlist.m3u8"] {
|
||||
let res = r.clone().oneshot(req("GET", uri, None, None, "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::UNAUTHORIZED, "{uri}");
|
||||
}
|
||||
let res = r.clone().oneshot(req("GET", "/", None, None, "203.0.113.1")).await.unwrap();
|
||||
let h = res.headers();
|
||||
assert!(h["content-security-policy"].to_str().unwrap().contains("script-src 'self';"));
|
||||
assert!(h.contains_key("strict-transport-security"));
|
||||
assert_eq!(h["cross-origin-opener-policy"], "same-origin");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_account_needs_handle_and_removes_data() {
|
||||
let (r, app, _) = setup("delete", "http://127.0.0.1:9").await;
|
||||
let token = app
|
||||
.db
|
||||
.call(|c| {
|
||||
crate::db::upsert_user(c, 5, "tester", None, None)?;
|
||||
crate::db::add_event(c, 5, 1, "like", None)?;
|
||||
crate::db::create_session(c, 5)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let cookie = format!("__Host-pl8inf={token}");
|
||||
let res = r.clone().oneshot(req("POST", "/account/delete", Some(r#"{"confirm":"nope"}"#), Some(&cookie), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::BAD_REQUEST);
|
||||
let res = r.clone().oneshot(req("GET", "/account/export", None, Some(&cookie), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let res = r.clone().oneshot(req("POST", "/account/delete", Some(r#"{"confirm":"tester"}"#), Some(&cookie), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let left = app
|
||||
.db
|
||||
.call(|c| c.query_row("SELECT (SELECT COUNT(*) FROM users) + (SELECT COUNT(*) FROM sessions) + (SELECT COUNT(*) FROM events)", [], |r| r.get::<_, i64>(0)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(left, 0);
|
||||
let res = r.clone().oneshot(req("GET", "/local/me", None, Some(&cookie), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_cookie_moves_to_host_prefix() {
|
||||
let (r, app, _) = setup("legacy", "http://127.0.0.1:9").await;
|
||||
let token = app
|
||||
.db
|
||||
.call(|c| {
|
||||
crate::db::upsert_user(c, 5, "tester", None, None)?;
|
||||
crate::db::create_session(c, 5)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let res = r.oneshot(req("GET", "/local/me", None, Some(&format!("pl8inf={token}")), "203.0.113.1")).await.unwrap();
|
||||
assert_eq!(res.status(), StatusCode::OK);
|
||||
let cookies = set_cookies(&res);
|
||||
assert!(cookies.iter().any(|c| c.starts_with(&format!("__Host-pl8inf={token};"))));
|
||||
assert!(cookies.iter().any(|c| c.starts_with("pl8inf=;") && c.contains("Max-Age=0")));
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//! Infinite-scroll front end for pl8list, with sign-in through pl8list accounts.
|
||||
//!
|
||||
//! pl8inf # then open http://127.0.0.1:8790
|
||||
//! pl8inf --healthcheck # exit 0 if the server on $PORT answers /healthz
|
||||
//!
|
||||
//! Users sign in with their pl8list email: pl8list mails them a code, they type it
|
||||
//! here, and this server checks it with pl8list. Afterwards they get a session
|
||||
//! cookie for this app only. Their pl8list session is used once to read their
|
||||
//! personal feed, then logged out at pl8list; it is never written to disk.
|
||||
//! See `config.rs` for the environment variables.
|
||||
|
||||
#![allow(clippy::result_large_err)] // handlers return axum `Response` as their error
|
||||
|
||||
mod cache;
|
||||
mod config;
|
||||
mod crawler;
|
||||
mod db;
|
||||
mod http;
|
||||
mod pl8;
|
||||
mod ranker;
|
||||
mod ratelimit;
|
||||
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
net::{SocketAddr, TcpStream},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use config::Config;
|
||||
|
||||
fn healthcheck() -> std::process::ExitCode {
|
||||
let port = std::env::var("PORT").ok().and_then(|p| p.parse().ok()).unwrap_or(8790u16);
|
||||
let ok = (|| -> std::io::Result<bool> {
|
||||
let mut s = TcpStream::connect_timeout(&SocketAddr::from(([127, 0, 0, 1], port)), Duration::from_secs(2))?;
|
||||
s.set_read_timeout(Some(Duration::from_secs(2)))?;
|
||||
s.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")?;
|
||||
let mut head = [0u8; 12];
|
||||
s.read_exact(&mut head)?;
|
||||
Ok(head.ends_with(b"200"))
|
||||
})();
|
||||
if matches!(ok, Ok(true)) { std::process::ExitCode::SUCCESS } else { std::process::ExitCode::FAILURE }
|
||||
}
|
||||
|
||||
async fn shutdown() {
|
||||
let ctrl_c = tokio::signal::ctrl_c();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("signal handler");
|
||||
tokio::select! { _ = ctrl_c => {}, _ = term.recv() => {} }
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
let _ = ctrl_c.await;
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<std::process::ExitCode> {
|
||||
if std::env::args().any(|a| a == "--healthcheck") {
|
||||
return Ok(healthcheck());
|
||||
}
|
||||
config::load_dotenv(".env");
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "pl8inf=info".into()),
|
||||
)
|
||||
.with_target(false)
|
||||
.init();
|
||||
|
||||
let cfg = Config::from_env();
|
||||
if !cfg.secure {
|
||||
tracing::warn!("INSECURE_COOKIES=1: cookies without Secure, no HSTS. Use only for local development.");
|
||||
}
|
||||
let db = db::Db::open(&cfg.db_path)?;
|
||||
let pl8 = Arc::new(pl8::Pl8::new(&cfg.pl8_base, Duration::from_millis(700))?);
|
||||
let crawler = crawler::Crawler::new(pl8.clone(), db.clone());
|
||||
crawler.start();
|
||||
|
||||
let addr: SocketAddr = format!("{}:{}", cfg.host, cfg.port).parse()?;
|
||||
let app = http::App::new(cfg, db, pl8, crawler);
|
||||
{
|
||||
let app = app.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(3600));
|
||||
loop {
|
||||
tick.tick().await;
|
||||
app.purge().await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(addr).await?;
|
||||
tracing::info!("listening on http://{addr}");
|
||||
axum::serve(listener, http::router(app).into_make_service_with_connect_info::<SocketAddr>())
|
||||
.with_graceful_shutdown(shutdown())
|
||||
.await?;
|
||||
Ok(std::process::ExitCode::SUCCESS)
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! pl8list's passwordless login: POST email, pl8list mails a 6-digit code, POST code.
|
||||
//!
|
||||
//! The site keeps the pending login in its `_session` cookie, so one flow holds that
|
||||
//! cookie between the two steps. After `verify` succeeds the cookie is a fully
|
||||
//! logged-in pl8list session. It is handed to the caller exactly once and is never
|
||||
//! logged, stored or kept in the flow.
|
||||
|
||||
use base64::{
|
||||
Engine,
|
||||
alphabet,
|
||||
engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},
|
||||
};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::Pl8;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct LoginError(pub String);
|
||||
|
||||
impl std::fmt::Display for LoginError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn err(msg: impl Into<String>) -> LoginError {
|
||||
LoginError(msg.into())
|
||||
}
|
||||
|
||||
/// The pl8list account that just signed in.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Pl8User {
|
||||
pub id: i64,
|
||||
pub handle: String,
|
||||
pub name: Option<String>,
|
||||
pub img: Option<String>,
|
||||
}
|
||||
|
||||
pub struct LoginFlow {
|
||||
pub email: String,
|
||||
session: Option<SecretString>,
|
||||
}
|
||||
|
||||
struct Reply {
|
||||
status: u16,
|
||||
remix_redirect: Option<String>,
|
||||
payload: Option<Value>,
|
||||
}
|
||||
|
||||
impl LoginFlow {
|
||||
pub fn new(email: String) -> Self {
|
||||
LoginFlow { email, session: None }
|
||||
}
|
||||
|
||||
async fn call(&mut self, pl8: &Pl8, path: &str, query: &[(&str, &str)], route: &str, form: Option<&[(&str, &str)]>) -> Result<Reply, LoginError> {
|
||||
let mut url = pl8.url(&[path]);
|
||||
{
|
||||
let mut q = url.query_pairs_mut();
|
||||
for (k, v) in query {
|
||||
q.append_pair(k, v);
|
||||
}
|
||||
q.append_pair("_data", route);
|
||||
}
|
||||
let mut req = match form {
|
||||
Some(fields) => pl8.http().post(url).form(fields),
|
||||
None => pl8.http().get(url),
|
||||
};
|
||||
if let Some(s) = &self.session {
|
||||
req = req.header("Cookie", format!("_session={}", s.expose_secret()));
|
||||
}
|
||||
let res = req.send().await.map_err(|e| err(format!("pl8list unreachable: {}", e.without_url())))?;
|
||||
for c in res.headers().get_all("set-cookie") {
|
||||
if let Some(v) = c.to_str().ok().and_then(|c| c.strip_prefix("_session=")) {
|
||||
let v = v.split(';').next().unwrap_or_default();
|
||||
self.session = (!v.is_empty()).then(|| SecretString::from(v.to_string()));
|
||||
}
|
||||
}
|
||||
let status = res.status().as_u16();
|
||||
let remix_redirect = res.headers().get("x-remix-redirect").and_then(|v| v.to_str().ok()).map(String::from);
|
||||
let payload = res.bytes().await.ok().and_then(|b| serde_json::from_slice(&b).ok());
|
||||
Ok(Reply { status, remix_redirect, payload })
|
||||
}
|
||||
|
||||
/// The guest id pl8list stores in its cookie; the login forms echo it back.
|
||||
fn guest_id(&self) -> String {
|
||||
let Some(s) = &self.session else { return String::new() };
|
||||
let raw = percent_decode_str(s.expose_secret()).decode_utf8_lossy();
|
||||
let first = raw.split('.').next().unwrap_or_default();
|
||||
let engine = GeneralPurpose::new(
|
||||
&alphabet::STANDARD,
|
||||
GeneralPurposeConfig::new().with_decode_padding_mode(DecodePaddingMode::Indifferent),
|
||||
);
|
||||
engine
|
||||
.decode(first)
|
||||
.ok()
|
||||
.and_then(|b| serde_json::from_slice::<Value>(&b).ok())
|
||||
.and_then(|v| v.get("pl8list_guest_id").and_then(Value::as_str).map(String::from))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn payload_str(reply: &Reply, key: &str) -> Option<String> {
|
||||
reply.payload.as_ref()?.get(key)?.as_str().map(String::from)
|
||||
}
|
||||
|
||||
/// Ask pl8list to mail a login code.
|
||||
pub async fn start(&mut self, pl8: &Pl8) -> Result<(), LoginError> {
|
||||
self.call(pl8, "login", &[], "routes/login", None).await?;
|
||||
let guest = self.guest_id();
|
||||
let email = self.email.clone();
|
||||
let reply = self
|
||||
.call(pl8, "login", &[], "routes/login", Some(&[("email", &email), ("pl8list_guest_id", &guest)]))
|
||||
.await?;
|
||||
if !reply.remix_redirect.as_deref().unwrap_or_default().contains("/verify") {
|
||||
return Err(err(Self::payload_str(&reply, "error")
|
||||
.unwrap_or_else(|| format!("pl8list refused the login (HTTP {})", reply.status))));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn resend(&mut self, pl8: &Pl8) -> Result<(), LoginError> {
|
||||
let guest = self.guest_id();
|
||||
let email = self.email.clone();
|
||||
let query = [("email", email.as_str()), ("mode", "login")];
|
||||
let reply = self
|
||||
.call(pl8, "verify", &query, "routes/verify", Some(&[("action", "resend"), ("pl8list_guest_id", &guest)]))
|
||||
.await?;
|
||||
match Self::payload_str(&reply, "resendError") {
|
||||
Some(e) => Err(err(e)),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit the code. On success returns the account and its logged-in pl8list session;
|
||||
/// the flow no longer holds the session afterwards.
|
||||
pub async fn verify(&mut self, pl8: &Pl8, code: &str) -> Result<(Pl8User, SecretString), LoginError> {
|
||||
let guest = self.guest_id();
|
||||
let email = self.email.clone();
|
||||
let query = [("email", email.as_str()), ("mode", "login")];
|
||||
let reply = self
|
||||
.call(pl8, "verify", &query, "routes/verify", Some(&[("action", "verify"), ("code", code), ("pl8list_guest_id", &guest)]))
|
||||
.await?;
|
||||
if let Some(e) = Self::payload_str(&reply, "error") {
|
||||
return Err(err(e));
|
||||
}
|
||||
let session = self.session.take().ok_or_else(|| err("login failed, try again"))?;
|
||||
let root = pl8.root(Some(&session)).await.ok().flatten();
|
||||
let user = root.as_ref().and_then(|r| r.get("currentUser")).filter(|u| u.is_object());
|
||||
let Some(user) = user else {
|
||||
// Not logged in after all; nothing to end at pl8list.
|
||||
return Err(err("login failed, try again"));
|
||||
};
|
||||
let handle = user.get("handle").and_then(Value::as_str).filter(|h| !h.is_empty());
|
||||
let (Some(id), Some(handle)) = (user.get("id").and_then(Value::as_i64), handle) else {
|
||||
pl8.logout(&session).await;
|
||||
return Err(err("finish setting up your account on pl8list.com first (choose a handle)"));
|
||||
};
|
||||
let text = |k: &str| user.get(k).and_then(Value::as_str).map(String::from);
|
||||
Ok((Pl8User { id, handle: handle.to_string(), name: text("name"), img: text("img") }, session))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! Client for pl8list.com.
|
||||
//!
|
||||
//! pl8list is a Remix app, so every page loader can be read as JSON by adding
|
||||
//! `?_data=<route id>` to the page URL. Resource routes under /api/ return JSON
|
||||
//! directly. All JSON requests share one throttle so we stay polite.
|
||||
//!
|
||||
//! Public data (artist pages, related tracks, previews, free tracks) works without
|
||||
//! an account, so the crawler runs as a guest. A user's own session is only needed
|
||||
//! for login and for reading their personal feed.
|
||||
|
||||
pub mod login;
|
||||
pub mod normalize;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::{StatusCode, redirect};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use serde_json::Value;
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
use url::Url;
|
||||
|
||||
pub const USER_AGENT: &str = "pl8list-wrapper/0.3 (infinite-scroll client, https://pl8.narl.io)";
|
||||
/// Signed storage URLs for audio and images point here.
|
||||
const FILES_HOST: &str = "files.pl8list.com";
|
||||
|
||||
pub struct Pl8 {
|
||||
http: reqwest::Client,
|
||||
base: Url,
|
||||
min_interval: Duration,
|
||||
last: Mutex<Instant>,
|
||||
}
|
||||
|
||||
impl Pl8 {
|
||||
pub fn new(base: &str, min_interval: Duration) -> anyhow::Result<Self> {
|
||||
let base = Url::parse(base)?;
|
||||
let own_host = base.host_str().unwrap_or_default().to_string();
|
||||
let own_scheme = base.scheme().to_string();
|
||||
// Follow redirects only to pl8list and its storage host, never anywhere else.
|
||||
let policy = redirect::Policy::custom(move |attempt| {
|
||||
let url = attempt.url();
|
||||
let host = url.host_str().unwrap_or_default();
|
||||
let ok_host = host == own_host || host == FILES_HOST;
|
||||
let ok_scheme = url.scheme() == "https" || url.scheme() == own_scheme;
|
||||
if attempt.previous().len() >= 3 || !ok_host || !ok_scheme {
|
||||
attempt.stop()
|
||||
} else {
|
||||
attempt.follow()
|
||||
}
|
||||
});
|
||||
let http = reqwest::Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.redirect(policy)
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
Ok(Pl8 { http, base, min_interval, last: Mutex::new(Instant::now() - min_interval) })
|
||||
}
|
||||
|
||||
async fn throttle(&self) {
|
||||
let mut last = self.last.lock().await;
|
||||
let next = *last + self.min_interval;
|
||||
tokio::time::sleep_until(next).await;
|
||||
*last = Instant::now();
|
||||
}
|
||||
|
||||
/// URL on pl8list from already-decoded path segments.
|
||||
pub fn url(&self, segments: &[&str]) -> Url {
|
||||
let mut url = self.base.clone();
|
||||
url.path_segments_mut().expect("base is a URL with a path").clear().extend(segments);
|
||||
url
|
||||
}
|
||||
|
||||
pub fn request(&self, url: Url, session: Option<&SecretString>) -> reqwest::RequestBuilder {
|
||||
let req = self.http.get(url);
|
||||
match session {
|
||||
Some(s) => req.header("Cookie", format!("_session={}", s.expose_secret())),
|
||||
None => req,
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON GET with retries on overload. 403/404 give `None`.
|
||||
pub async fn get_json(
|
||||
&self,
|
||||
segments: &[&str],
|
||||
params: &[(&str, String)],
|
||||
route: Option<&str>,
|
||||
session: Option<&SecretString>,
|
||||
) -> anyhow::Result<Option<Value>> {
|
||||
let mut url = self.url(segments);
|
||||
{
|
||||
let mut q = url.query_pairs_mut();
|
||||
for (k, v) in params {
|
||||
q.append_pair(k, v);
|
||||
}
|
||||
if let Some(r) = route {
|
||||
q.append_pair("_data", r);
|
||||
}
|
||||
}
|
||||
for attempt in 0..4u32 {
|
||||
self.throttle().await;
|
||||
let res = self.request(url.clone(), session).send().await?;
|
||||
match res.status() {
|
||||
s if s.is_success() => {
|
||||
let body = res.bytes().await?;
|
||||
return Ok(if body.is_empty() { None } else { Some(serde_json::from_slice(&body)?) });
|
||||
}
|
||||
StatusCode::TOO_MANY_REQUESTS | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE
|
||||
| StatusCode::GATEWAY_TIMEOUT
|
||||
if attempt < 3 =>
|
||||
{
|
||||
tokio::time::sleep(Duration::from_secs(3 << attempt)).await;
|
||||
}
|
||||
StatusCode::FORBIDDEN | StatusCode::NOT_FOUND => return Ok(None),
|
||||
s => anyhow::bail!("pl8list {} for {}", s, url.path()),
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
// Loaders -------------------------------------------------------------
|
||||
//
|
||||
// Paginated loaders ignore `page` unless `pagination=true` is also set,
|
||||
// which is what the site's own useInfiniteLoading hook sends.
|
||||
|
||||
pub async fn root(&self, session: Option<&SecretString>) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&[], &[], Some("root"), session).await
|
||||
}
|
||||
|
||||
pub async fn home(&self) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&[], &[], Some("routes/_index"), None).await
|
||||
}
|
||||
|
||||
pub async fn feed(&self, session: &SecretString, page: i64) -> anyhow::Result<Option<Value>> {
|
||||
let p = [("page", page.to_string()), ("pagination", "true".into())];
|
||||
self.get_json(&["api", "feed"], &p, None, Some(session)).await
|
||||
}
|
||||
|
||||
pub async fn artist(&self, handle: &str, page: i64) -> anyhow::Result<Option<Value>> {
|
||||
let p = [("page", page.to_string()), ("pagination", "true".into())];
|
||||
self.get_json(&[handle], &p, Some("routes/$handle"), None).await
|
||||
}
|
||||
|
||||
pub async fn related(&self, handle: &str, slug: &str) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&[handle, slug, "related"], &[], Some("routes/$handle_.$slug.related"), None).await
|
||||
}
|
||||
|
||||
pub async fn best_selling_artists(&self) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&["best-selling-artists"], &[], Some("routes/best-selling-artists"), None).await
|
||||
}
|
||||
|
||||
pub async fn activity_ticker(&self) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&["api", "activity-ticker"], &[], None, None).await
|
||||
}
|
||||
|
||||
pub async fn search_users(&self, q: &str, page: i64) -> anyhow::Result<Option<Value>> {
|
||||
let p = [("q", q.to_string()), ("page", page.to_string()), ("pagination", "true".into())];
|
||||
self.get_json(&["search", "users"], &p, Some("routes/search.users"), None).await
|
||||
}
|
||||
|
||||
/// End a session at pl8list. Errors are ignored: we drop the session either way.
|
||||
pub async fn logout(&self, session: &SecretString) {
|
||||
let mut url = self.url(&["logout"]);
|
||||
url.query_pairs_mut().append_pair("_data", "routes/logout");
|
||||
if let Err(e) = self.request(url, Some(session)).send().await {
|
||||
tracing::warn!("pl8list logout failed: {}", e.without_url());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn http(&self) -> &reqwest::Client {
|
||||
&self.http
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//! Flatten pl8list loader JSON into `tracks` rows.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::db::Track;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Extra {
|
||||
full_access: bool,
|
||||
saves: i64,
|
||||
downloads: i64,
|
||||
purchases: i64,
|
||||
comments: i64,
|
||||
}
|
||||
|
||||
fn int(v: Option<&Value>) -> i64 {
|
||||
v.and_then(|v| v.as_i64().or_else(|| v.as_f64().map(|f| f as i64))).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn text(t: &Value, key: &str) -> Option<String> {
|
||||
t.get(key).and_then(Value::as_str).filter(|s| !s.is_empty()).map(String::from)
|
||||
}
|
||||
|
||||
fn num(t: &Value, key: &str) -> Option<f64> {
|
||||
t.get(key).and_then(Value::as_f64)
|
||||
}
|
||||
|
||||
fn json_text(t: &Value, key: &str) -> Option<String> {
|
||||
t.get(key).filter(|v| !v.is_null() && v.as_array().is_none_or(|a| !a.is_empty())).map(Value::to_string)
|
||||
}
|
||||
|
||||
/// A loader track item (`{track, hasFullAccess, saveCount, ...}`).
|
||||
pub fn item(item: &Value) -> Option<Track> {
|
||||
track(item.get("track")?, Extra {
|
||||
full_access: item.get("hasFullAccess").and_then(Value::as_bool).unwrap_or(false),
|
||||
saves: int(item.get("saveCount")),
|
||||
downloads: int(item.get("downloadCount")),
|
||||
purchases: int(item.get("purchaseCount")),
|
||||
comments: int(item.get("commentCount")),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn items(list: Option<&Value>) -> Vec<Track> {
|
||||
list.and_then(Value::as_array).map(|a| a.iter().filter_map(item).collect()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The related loader returns bare tracks plus per-id maps for access and counts.
|
||||
pub fn related(data: &Value) -> Vec<Track> {
|
||||
let counts = data.get("counts");
|
||||
let count = |name: &str, k: &str| int(counts.and_then(|c| c.get(name)).and_then(|m| m.get(k)));
|
||||
let Some(tracks) = data.get("tracks").and_then(Value::as_array) else { return vec![] };
|
||||
tracks
|
||||
.iter()
|
||||
.filter_map(|t| {
|
||||
let k = t.get("id")?.as_i64()?.to_string();
|
||||
track(t, Extra {
|
||||
full_access: data.get("access").and_then(|a| a.get(&k)).and_then(Value::as_bool).unwrap_or(false),
|
||||
saves: count("saveCount", &k),
|
||||
downloads: count("downloadCount", &k),
|
||||
purchases: count("purchaseCount", &k),
|
||||
comments: count("commentCount", &k),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn track(t: &Value, extra: Extra) -> Option<Track> {
|
||||
let user = t.get("user")?;
|
||||
let handle = text(user, "handle")?;
|
||||
if t.get("isDeleted").and_then(Value::as_bool) == Some(true) || t.get("isPublic").and_then(Value::as_bool) == Some(false) {
|
||||
return None;
|
||||
}
|
||||
let slug = text(t, "slug")?;
|
||||
Some(Track {
|
||||
id: t.get("id")?.as_i64()?,
|
||||
artist_name: text(user, "name").unwrap_or_else(|| handle.clone()),
|
||||
artist_img: text(user, "img"),
|
||||
artist_verified: user.get("isVerified").and_then(Value::as_bool).unwrap_or(false),
|
||||
title: text(t, "title").or_else(|| text(t, "filename")).unwrap_or_else(|| slug.clone()),
|
||||
description: text(t, "description"),
|
||||
bpm: num(t, "bpm"),
|
||||
key: text(t, "key"),
|
||||
duration: num(t, "duration"),
|
||||
preview_duration: num(t, "previewDuration"),
|
||||
price: num(t, "price").unwrap_or(0.0),
|
||||
created_at: text(t, "createdAt"),
|
||||
upload_id: text(t, "uploadId"),
|
||||
preview_upload_id: text(t, "previewUploadId"),
|
||||
waveform: json_text(t, "waveform"),
|
||||
preview_waveform: json_text(t, "previewWaveform"),
|
||||
full_access: extra.full_access,
|
||||
saves: extra.saves,
|
||||
downloads: extra.downloads,
|
||||
purchases: extra.purchases,
|
||||
comments: extra.comments,
|
||||
related_of: None,
|
||||
handle,
|
||||
slug,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn item_flattens_and_skips_private() {
|
||||
let i = json!({
|
||||
"track": {"id": 5, "slug": "s", "title": "T", "price": 4.99, "bpm": 140,
|
||||
"waveform": [1, 2], "user": {"handle": "h", "isVerified": true}},
|
||||
"hasFullAccess": false, "saveCount": 3
|
||||
});
|
||||
let t = item(&i).unwrap();
|
||||
assert_eq!((t.id, t.handle.as_str(), t.artist_name.as_str()), (5, "h", "h"));
|
||||
assert_eq!((t.price, t.bpm, t.saves), (4.99, Some(140.0), 3));
|
||||
assert_eq!(t.waveform.as_deref(), Some("[1,2]"));
|
||||
let private = json!({"track": {"id": 6, "slug": "s", "isPublic": false, "user": {"handle": "h"}}});
|
||||
assert!(item(&private).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn related_reads_count_maps() {
|
||||
let d = json!({
|
||||
"tracks": [{"id": 9, "slug": "x", "user": {"handle": "a"}}],
|
||||
"access": {"9": true},
|
||||
"counts": {"saveCount": {"9": 4}}
|
||||
});
|
||||
let r = related(&d);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert!(r[0].full_access);
|
||||
assert_eq!(r[0].saves, 4);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
//! Pick the next batch of tracks for the scroll feed.
|
||||
//!
|
||||
//! Each unserved playable track gets a score:
|
||||
//!
|
||||
//! popularity log-scaled saves/downloads/purchases/comments, normalised to 0..1
|
||||
//! freshness exp(-age / 21 days)
|
||||
//! artist learned from your events on that artist's tracks (likes, full listens, skips)
|
||||
//! tempo / key closeness to the BPM and key of tracks you liked or finished
|
||||
//! related track came from the "related" page of something you liked
|
||||
//! exploration Gumbel noise, so the feed never freezes into one niche
|
||||
//!
|
||||
//! The batch is then picked greedily with a penalty for repeating an artist.
|
||||
//! A `Ranker` supplies the weights and may narrow the candidates; discovery modes
|
||||
//! are further `Ranker`s chosen with `?mode=`.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use rand::RngExt;
|
||||
use rusqlite::{Connection, params_from_iter, types::Value as Sql};
|
||||
use serde_json::{Value, json};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
use crate::db::Track;
|
||||
|
||||
pub struct Weights {
|
||||
pub popularity: f64,
|
||||
pub freshness: f64,
|
||||
pub artist: f64,
|
||||
pub tempo: f64,
|
||||
pub key: f64,
|
||||
pub related: f64,
|
||||
pub unknown_artist: f64,
|
||||
pub exploration: f64,
|
||||
}
|
||||
|
||||
pub trait Ranker: Send + Sync {
|
||||
fn weights(&self) -> &Weights;
|
||||
/// Extra SQL condition on `tracks t`, for modes that only want part of the catalogue.
|
||||
fn condition(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// The original mix.
|
||||
pub struct Classic;
|
||||
|
||||
const CLASSIC: Weights = Weights {
|
||||
popularity: 1.2,
|
||||
freshness: 0.8,
|
||||
artist: 1.5,
|
||||
tempo: 1.0,
|
||||
key: 0.3,
|
||||
related: 1.0,
|
||||
unknown_artist: 0.2,
|
||||
exploration: 0.6,
|
||||
};
|
||||
|
||||
impl Ranker for Classic {
|
||||
fn weights(&self) -> &Weights {
|
||||
&CLASSIC
|
||||
}
|
||||
}
|
||||
|
||||
pub fn by_name(mode: Option<&str>) -> Option<&'static dyn Ranker> {
|
||||
match mode.unwrap_or("") {
|
||||
"" | "classic" => Some(&Classic),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-event contributions to artist affinity.
|
||||
fn event_signal(kind: &str) -> f64 {
|
||||
match kind {
|
||||
"like" => 3.0,
|
||||
"unlike" => -3.0,
|
||||
"complete" => 1.5,
|
||||
"skip" => -1.0,
|
||||
"dislike" => -3.0,
|
||||
_ => 0.0,
|
||||
}
|
||||
}
|
||||
const REPEAT_ARTIST_PENALTY: f64 = 1.5;
|
||||
const RECENT_ARTIST_WINDOW: i64 = 6;
|
||||
|
||||
fn age_days(iso: Option<&str>) -> f64 {
|
||||
let Some(t) = iso.and_then(|s| OffsetDateTime::parse(s, &Rfc3339).ok()) else { return 365.0 };
|
||||
((OffsetDateTime::now_utc() - t).as_seconds_f64() / 86400.0).max(0.0)
|
||||
}
|
||||
|
||||
/// Gaussian closeness to any liked tempo, also counting half/double time as a match.
|
||||
fn tempo_match(bpm: Option<f64>, liked: &[f64]) -> f64 {
|
||||
let Some(bpm) = bpm.filter(|b| *b > 0.0) else { return 0.0 };
|
||||
liked
|
||||
.iter()
|
||||
.map(|b| {
|
||||
let d = (bpm - b).abs().min((bpm * 2.0 - b).abs()).min((bpm / 2.0 - b).abs());
|
||||
(-(d / 6.0).powi(2)).exp()
|
||||
})
|
||||
.fold(0.0, f64::max)
|
||||
}
|
||||
|
||||
pub struct Taste {
|
||||
pub artist: HashMap<String, f64>,
|
||||
pub bpms: Vec<f64>,
|
||||
pub keys: HashSet<String>,
|
||||
pub positive_ids: HashSet<i64>,
|
||||
pub liked_ids: Vec<i64>,
|
||||
}
|
||||
|
||||
/// Summarise a user's listening history into artist affinity, liked tempos/keys and liked track ids.
|
||||
pub fn taste(conn: &Connection, user_id: i64) -> rusqlite::Result<Taste> {
|
||||
let mut stmt = conn.prepare_cached(
|
||||
"SELECT e.track_id, e.kind, t.handle, t.bpm, t.key FROM events e JOIN tracks t ON t.id = e.track_id \
|
||||
WHERE e.user_id = ? AND e.kind IN ('like', 'unlike', 'complete', 'skip', 'dislike') ORDER BY e.ts",
|
||||
)?;
|
||||
let mut artist: HashMap<String, f64> = HashMap::new();
|
||||
// (track id, bpm, key), in the order they became positive.
|
||||
let mut liked: Vec<(i64, Option<f64>, Option<String>)> = vec![];
|
||||
let mut completed: Vec<(i64, Option<f64>, Option<String>)> = vec![];
|
||||
let rows = stmt.query_map([user_id], |r| {
|
||||
Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?, r.get::<_, Option<f64>>(3)?, r.get::<_, Option<String>>(4)?))
|
||||
})?;
|
||||
for row in rows {
|
||||
let (id, kind, handle, bpm, key) = row?;
|
||||
*artist.entry(handle).or_default() += event_signal(&kind);
|
||||
match kind.as_str() {
|
||||
"like" => {
|
||||
liked.retain(|l| l.0 != id);
|
||||
liked.push((id, bpm, key));
|
||||
}
|
||||
"unlike" => liked.retain(|l| l.0 != id),
|
||||
"complete" => {
|
||||
completed.retain(|l| l.0 != id);
|
||||
completed.push((id, bpm, key));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let liked_set: HashSet<i64> = liked.iter().map(|l| l.0).collect();
|
||||
let positive: Vec<_> = completed.iter().filter(|c| !liked_set.contains(&c.0)).chain(liked.iter()).collect();
|
||||
let bpms: Vec<f64> = positive.iter().filter_map(|p| p.1).collect();
|
||||
Ok(Taste {
|
||||
artist: artist.into_iter().map(|(h, v)| (h, (v / 4.0).tanh())).collect(),
|
||||
bpms: bpms[bpms.len().saturating_sub(50)..].to_vec(),
|
||||
keys: positive.iter().filter_map(|p| p.2.clone()).collect(),
|
||||
positive_ids: positive.iter().map(|p| p.0).collect(),
|
||||
liked_ids: liked.iter().rev().map(|l| l.0).collect(),
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Copy)]
|
||||
pub struct Filters {
|
||||
pub free_only: bool,
|
||||
pub bpm_min: Option<f64>,
|
||||
pub bpm_max: Option<f64>,
|
||||
}
|
||||
|
||||
pub fn pick(conn: &Connection, user_id: i64, n: usize, f: Filters, ranker: &dyn Ranker) -> rusqlite::Result<Vec<Value>> {
|
||||
let mut sql = String::from(
|
||||
"SELECT * FROM tracks t WHERE (t.full_access=1 OR t.preview_upload_id IS NOT NULL) \
|
||||
AND NOT EXISTS (SELECT 1 FROM events e WHERE e.user_id=? AND e.kind='served' AND e.track_id=t.id)",
|
||||
);
|
||||
let mut args = vec![Sql::Integer(user_id)];
|
||||
if f.free_only {
|
||||
sql.push_str(" AND t.price = 0");
|
||||
}
|
||||
if let Some(b) = f.bpm_min {
|
||||
sql.push_str(" AND t.bpm >= ?");
|
||||
args.push(Sql::Real(b));
|
||||
}
|
||||
if let Some(b) = f.bpm_max {
|
||||
sql.push_str(" AND t.bpm <= ?");
|
||||
args.push(Sql::Real(b));
|
||||
}
|
||||
if let Some(c) = ranker.condition() {
|
||||
sql.push_str(" AND (");
|
||||
sql.push_str(c);
|
||||
sql.push(')');
|
||||
}
|
||||
let cands: Vec<Track> = conn.prepare(&sql)?.query_map(params_from_iter(args), Track::from_row)?.collect::<Result<_, _>>()?;
|
||||
if cands.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
|
||||
let t = taste(conn, user_id)?;
|
||||
let recent: Vec<String> = conn
|
||||
.prepare_cached(
|
||||
"SELECT t.handle FROM events e JOIN tracks t ON t.id=e.track_id \
|
||||
WHERE e.user_id=? AND e.kind='served' ORDER BY e.ts DESC LIMIT ?",
|
||||
)?
|
||||
.query_map(rusqlite::params![user_id, RECENT_ARTIST_WINDOW], |r| r.get(0))?
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
let pop_raw = |r: &Track| {
|
||||
(r.saves as f64).ln_1p() + 0.7 * (r.downloads as f64).ln_1p() + (r.purchases as f64).ln_1p()
|
||||
+ 0.5 * (r.comments as f64).ln_1p()
|
||||
};
|
||||
let max_pop = cands.iter().map(pop_raw).fold(0.0, f64::max);
|
||||
let max_pop = if max_pop > 0.0 { max_pop } else { 1.0 };
|
||||
let w = ranker.weights();
|
||||
let mut rng = rand::rng();
|
||||
let mut scored: Vec<(f64, Track)> = cands
|
||||
.into_iter()
|
||||
.map(|r| {
|
||||
let aff = t.artist.get(&r.handle).copied();
|
||||
let u: f64 = rng.random::<f64>().max(1e-12);
|
||||
let s = w.popularity * pop_raw(&r) / max_pop
|
||||
+ w.freshness * (-age_days(r.created_at.as_deref()) / 21.0).exp()
|
||||
+ w.artist * aff.unwrap_or(0.0)
|
||||
+ w.tempo * tempo_match(r.bpm, &t.bpms)
|
||||
+ w.key * if r.key.as_ref().is_some_and(|k| t.keys.contains(k)) { 1.0 } else { 0.0 }
|
||||
+ w.related * if r.related_of.is_some_and(|id| t.positive_ids.contains(&id)) { 1.0 } else { 0.0 }
|
||||
+ w.unknown_artist * if aff.is_none() { 1.0 } else { 0.0 }
|
||||
+ w.exploration * -(-u.ln()).ln();
|
||||
(s, r)
|
||||
})
|
||||
.collect();
|
||||
|
||||
scored.sort_by(|a, b| b.0.total_cmp(&a.0));
|
||||
scored.truncate((n * 8).max(60));
|
||||
let mut seen = recent;
|
||||
let mut out = vec![];
|
||||
while !scored.is_empty() && out.len() < n {
|
||||
let penalised = |(s, r): &(f64, Track)| s - REPEAT_ARTIST_PENALTY * seen.iter().filter(|h| **h == r.handle).count() as f64;
|
||||
let best = (0..scored.len()).max_by(|&a, &b| penalised(&scored[a]).total_cmp(&penalised(&scored[b]))).unwrap_or(0);
|
||||
let (s, r) = scored.remove(best);
|
||||
seen.push(r.handle.clone());
|
||||
out.push(serialize(&r, Some(s)));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn serialize(r: &Track, score: Option<f64>) -> Value {
|
||||
let preview = !r.full_access;
|
||||
let wave = if preview && r.preview_waveform.is_some() { &r.preview_waveform } else { &r.waveform };
|
||||
let price = if r.price.fract() == 0.0 { json!(r.price as i64) } else { json!(r.price) };
|
||||
json!({
|
||||
"id": r.id,
|
||||
"handle": r.handle,
|
||||
"slug": r.slug,
|
||||
"title": r.title,
|
||||
"description": r.description,
|
||||
"artist_name": r.artist_name,
|
||||
"artist_img": r.artist_img,
|
||||
"artist_verified": r.artist_verified,
|
||||
"bpm": r.bpm,
|
||||
"key": r.key,
|
||||
"duration": if preview && r.preview_duration.is_some() { r.preview_duration } else { r.duration },
|
||||
"price": price,
|
||||
"created_at": r.created_at,
|
||||
"saves": r.saves,
|
||||
"downloads": r.downloads,
|
||||
"purchases": r.purchases,
|
||||
"comments": r.comments,
|
||||
"preview": preview,
|
||||
"upload_id": if preview { &r.preview_upload_id } else { &r.upload_id },
|
||||
"waveform": wave.as_deref().and_then(|w| serde_json::from_str::<Value>(w).ok()),
|
||||
"score": score.map(|s| (s * 1000.0).round() / 1000.0),
|
||||
})
|
||||
}
|
||||
|
||||
/// Liked tracks, newest like first.
|
||||
pub fn likes(conn: &Connection, user_id: i64) -> rusqlite::Result<Vec<Value>> {
|
||||
let ids = taste(conn, user_id)?.liked_ids;
|
||||
if ids.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let mut stmt = conn.prepare_cached("SELECT * FROM tracks WHERE id=?")?;
|
||||
let mut out = vec![];
|
||||
for id in ids {
|
||||
if let Some(t) = rusqlite::OptionalExtension::optional(stmt.query_row([id], Track::from_row))? {
|
||||
out.push(serialize(&t, None));
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tempo_counts_half_and_double_time() {
|
||||
assert!(tempo_match(Some(140.0), &[140.0]) > 0.99);
|
||||
assert!(tempo_match(Some(70.0), &[140.0]) > 0.99);
|
||||
assert!(tempo_match(Some(100.0), &[140.0]) < 0.01);
|
||||
assert_eq!(tempo_match(None, &[140.0]), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_dates_count_as_a_year_old() {
|
||||
assert_eq!(age_days(None), 365.0);
|
||||
assert_eq!(age_days(Some("garbage")), 365.0);
|
||||
assert!(age_days(Some("2020-01-01T00:00:00.000Z")) > 365.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
//! Sliding-window rate limits, kept in memory only.
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, VecDeque},
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
/// At most `limit` hits per key within `window`.
|
||||
pub struct RateLimiter {
|
||||
limit: usize,
|
||||
window: Duration,
|
||||
hits: Mutex<HashMap<String, VecDeque<Instant>>>,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
pub fn new(limit: usize, window_secs: u64) -> Self {
|
||||
RateLimiter { limit, window: Duration::from_secs(window_secs), hits: Mutex::new(HashMap::new()) }
|
||||
}
|
||||
|
||||
pub fn allow(&self, key: &str) -> bool {
|
||||
let now = Instant::now();
|
||||
let mut hits = self.hits.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if hits.len() > 10_000 {
|
||||
hits.retain(|_, q| q.back().is_some_and(|t| now.duration_since(*t) < self.window));
|
||||
}
|
||||
let q = hits.entry(key.to_string()).or_default();
|
||||
while q.front().is_some_and(|t| now.duration_since(*t) >= self.window) {
|
||||
q.pop_front();
|
||||
}
|
||||
if q.len() >= self.limit {
|
||||
return false;
|
||||
}
|
||||
q.push_back(now);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn limits_per_key() {
|
||||
let l = RateLimiter::new(2, 60);
|
||||
assert!(l.allow("a"));
|
||||
assert!(l.allow("a"));
|
||||
assert!(!l.allow("a"));
|
||||
assert!(l.allow("b"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex">
|
||||
<title>Datenschutzerklärung · pl8list ∞</title>
|
||||
<link rel="icon" href="/static/icons/icon.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
<body class="legal">
|
||||
<main>
|
||||
<p><a href="/">← zurück</a> · <a href="#english">English summary</a></p>
|
||||
<h1>Datenschutzerklärung</h1>
|
||||
<p>Stand: {{updated}}</p>
|
||||
|
||||
<h2>1. Verantwortlicher</h2>
|
||||
<p>
|
||||
{{name}}<br>
|
||||
{{address}}<br>
|
||||
E-Mail: {{email}}
|
||||
</p>
|
||||
<p>Eine Datenschutzbeauftragte oder ein Datenschutzbeauftragter ist nicht benannt, weil keine Pflicht dazu besteht.</p>
|
||||
|
||||
<h2>2. Kurz gesagt</h2>
|
||||
<ul>
|
||||
<li>Sie melden sich mit Ihrem pl8list-Konto an. Ihre E-Mail-Adresse und der Anmeldecode gehen dafür an pl8list.</li>
|
||||
<li>Wir speichern Ihr pl8list-Profil (ID, Nutzername, Anzeigename, Profilbild-ID) und was Sie hier hören, liken und überspringen, um Ihnen passende Tracks vorzuschlagen.</li>
|
||||
<li>Ihre pl8list-Anmeldung speichern wir nie. Wir nutzen sie einmal kurz im Arbeitsspeicher und melden sie danach bei pl8list ab.</li>
|
||||
<li>Keine Werbung, kein Tracking, keine Analyse-Tools, keine Inhalte von Drittanbietern im Browser.</li>
|
||||
<li>Sie können Ihre Daten jederzeit selbst herunterladen und Ihr Konto löschen.</li>
|
||||
</ul>
|
||||
|
||||
<h2>3. Aufruf der Website und Hosting</h2>
|
||||
<p>
|
||||
Beim Aufruf verarbeitet unser Server technisch zwingend Ihre IP-Adresse, die angefragte Adresse,
|
||||
Datum und Uhrzeit sowie die Angaben Ihres Browsers (User-Agent), um die Seite auszuliefern.
|
||||
Die Anwendung selbst schreibt keine Zugriffsprotokolle. Rechtsgrundlage ist Art. 6 Abs. 1 lit. f
|
||||
DSGVO; unser berechtigtes Interesse ist der sichere und funktionsfähige Betrieb.
|
||||
</p>
|
||||
<p>Der Server wird betrieben bei:<br>{{hosting}}</p>
|
||||
<p>
|
||||
Mit dem Hosting-Anbieter besteht ein Vertrag zur Auftragsverarbeitung nach Art. 28 DSGVO, soweit
|
||||
er Zugriff auf personenbezogene Daten haben kann.
|
||||
</p>
|
||||
|
||||
<h2>4. Anmeldung über pl8list</h2>
|
||||
<p>
|
||||
Ein eigenes Passwort gibt es hier nicht. Wenn Sie sich anmelden, passiert Folgendes:
|
||||
</p>
|
||||
<ol>
|
||||
<li>Ihre E-Mail-Adresse wird an pl8list.com übermittelt. pl8list sendet Ihnen daraufhin einen sechsstelligen Code.</li>
|
||||
<li>Den Code, den Sie hier eingeben, leiten wir an pl8list weiter. Ist er richtig, meldet pl8list unseren Server mit Ihrem Konto an.</li>
|
||||
<li>Mit dieser Anmeldung lesen wir einmalig Ihre persönliche pl8list-Übersicht (Künstler in Ihrer Bibliothek), um die Vorschläge zu verbessern. Das dauert höchstens zwei Minuten. Danach melden wir die Sitzung bei pl8list ab und verwerfen sie. Sie wird weder auf Datenträger geschrieben noch protokolliert.</li>
|
||||
<li>Von pl8list übernehmen wir Ihre pl8list-Nutzer-ID, Ihren Nutzernamen, Ihren Anzeigenamen und die ID Ihres Profilbilds.</li>
|
||||
</ol>
|
||||
<p>
|
||||
Ihre E-Mail-Adresse speichern wir nicht dauerhaft. Sie liegt nur während der Anmeldung (höchstens
|
||||
15 Minuten) im Arbeitsspeicher, zusätzlich zum Schutz vor Missbrauch (siehe Abschnitt 8).
|
||||
Rechtsgrundlage ist Art. 6 Abs. 1 lit. b DSGVO: Sie möchten die Anmeldung mit Ihrem pl8list-Konto
|
||||
nutzen. Für die Verarbeitung bei pl8list selbst ist pl8list verantwortlich; es gilt deren
|
||||
Datenschutzerklärung auf pl8list.com.
|
||||
</p>
|
||||
|
||||
<h2>5. Konto und Sitzung</h2>
|
||||
<p>
|
||||
Nach der Anmeldung legen wir ein Konto mit den Daten aus Abschnitt 4 an und merken uns den Zeitpunkt
|
||||
der Erstellung und der letzten Nutzung. Ihr Browser erhält ein Sitzungs-Cookie mit einer Zufallszahl.
|
||||
Bei uns liegt davon nur ein Hashwert, sodass auch wir das Cookie daraus nicht wiederherstellen können.
|
||||
Eine Sitzung endet nach 30 Tagen ohne Nutzung oder sofort, wenn Sie sich abmelden.
|
||||
Rechtsgrundlage ist Art. 6 Abs. 1 lit. b DSGVO.
|
||||
</p>
|
||||
|
||||
<h2>6. Vorschläge aus Ihrem Hörverhalten</h2>
|
||||
<p>
|
||||
Wir speichern, welche Tracks Ihnen gezeigt wurden und ob Sie sie gehört (mit Hördauer), zu Ende gehört,
|
||||
übersprungen, geliked oder als „nicht für mich“ markiert haben. Daraus berechnen wir automatisch, welche
|
||||
Tracks wir Ihnen als Nächstes zeigen. Das ist eine Form von Profiling. Es hat keine rechtliche oder
|
||||
ähnlich erhebliche Wirkung für Sie; es bestimmt nur die Reihenfolge der Musik. Rechtsgrundlage ist
|
||||
Art. 6 Abs. 1 lit. b DSGVO, denn personalisierte Vorschläge sind der Zweck dieses Dienstes.
|
||||
</p>
|
||||
<p>
|
||||
Der Eintrag „gezeigt“ wird nach 180 Tagen gelöscht. Alle anderen Einträge bleiben, bis Sie Ihr Konto
|
||||
löschen, weil sie Ihre Vorschläge und Ihre Like-Liste bilden.
|
||||
</p>
|
||||
|
||||
<h2>7. Audio und Bilder</h2>
|
||||
<p>
|
||||
Musik und Profilbilder laden Sie nicht direkt von pl8list, sondern über unseren Server. pl8list erfährt
|
||||
dadurch Ihre IP-Adresse nicht. Abgerufene Audiodateien und Bilder legen wir auf unserem Server
|
||||
zwischen, damit pl8list weniger belastet wird; dieser Zwischenspeicher enthält keine Angaben über Sie.
|
||||
Die Programmbibliothek zum Abspielen (hls.js) liefern wir selbst aus.
|
||||
</p>
|
||||
|
||||
<h2>8. Schutz vor Missbrauch</h2>
|
||||
<p>
|
||||
Jede Anmeldung lässt pl8list eine E-Mail verschicken. Damit niemand diesen Dienst nutzt, um fremde
|
||||
Postfächer zu überfluten oder Codes zu erraten, begrenzen wir Anmeldeversuche je IP-Adresse und je
|
||||
E-Mail-Adresse. Dafür halten wir IP-Adresse und E-Mail-Adresse höchstens 15 Minuten im Arbeitsspeicher;
|
||||
gespeichert werden sie nicht. Erfolgreiche Anmeldungen protokollieren wir mit Ihrem pl8list-Nutzernamen,
|
||||
um Fehler und Missbrauch erkennen zu können; diese Protokolle werden fortlaufend überschrieben
|
||||
(höchstens etwa 3 MB). Rechtsgrundlage ist Art. 6 Abs. 1 lit. f DSGVO.
|
||||
</p>
|
||||
|
||||
<h2>9. Cookies und lokaler Speicher</h2>
|
||||
<ul>
|
||||
<li><code>__Host-pl8inf</code>: Ihre Sitzung, bis zu 30 Tage ohne Nutzung.</li>
|
||||
<li><code>__Host-pl8inf_login</code>: verbindet die beiden Schritte der Anmeldung, 15 Minuten.</li>
|
||||
<li>Im lokalen Speicher Ihres Browsers: Ihre Filter (z. B. BPM-Bereich) und ob Sie den Bedienhinweis schon gesehen haben.</li>
|
||||
</ul>
|
||||
<p>
|
||||
Alle sind für den von Ihnen gewünschten Dienst unbedingt erforderlich (§ 25 Abs. 2 Nr. 2 TDDDG).
|
||||
Deshalb fragen wir nicht nach einer Einwilligung. Andere Cookies, Tracking oder Analyse-Tools gibt es nicht.
|
||||
</p>
|
||||
|
||||
<h2>10. Empfänger und Drittländer</h2>
|
||||
<p>
|
||||
Außer an pl8list (Abschnitt 4) und den Hosting-Anbieter (Abschnitt 3) geben wir keine Daten weiter.
|
||||
Wir selbst übermitteln keine Daten in Länder außerhalb der EU bzw. des EWR.
|
||||
</p>
|
||||
|
||||
<h2>11. Ihre Rechte</h2>
|
||||
<p>Sie haben das Recht auf</p>
|
||||
<ul>
|
||||
<li>Auskunft (Art. 15 DSGVO) und Datenübertragbarkeit (Art. 20 DSGVO): im Menü „Konto“ unter „Daten herunterladen“ erhalten Sie alle über Sie gespeicherten Daten als JSON-Datei;</li>
|
||||
<li>Löschung (Art. 17 DSGVO): im Menü „Konto“ unter „Konto löschen“ entfernen Sie Konto, Sitzungen und Hörverlauf sofort und vollständig;</li>
|
||||
<li>Berichtigung (Art. 16 DSGVO) und Einschränkung der Verarbeitung (Art. 18 DSGVO);</li>
|
||||
<li>Widerspruch (Art. 21 DSGVO) gegen Verarbeitungen, die auf Art. 6 Abs. 1 lit. f DSGVO beruhen, aus Gründen, die sich aus Ihrer besonderen Situation ergeben;</li>
|
||||
<li>Beschwerde bei einer Datenschutz-Aufsichtsbehörde (Art. 77 DSGVO), insbesondere in dem Mitgliedstaat Ihres Aufenthaltsorts oder Arbeitsplatzes.</li>
|
||||
</ul>
|
||||
<p>Für alle Anliegen genügt eine E-Mail an {{email}}.</p>
|
||||
|
||||
<h2>12. Pflicht zur Bereitstellung</h2>
|
||||
<p>
|
||||
Sie sind nicht verpflichtet, uns Daten zu geben. Ohne Anmeldung können Sie den Dienst aber nicht nutzen,
|
||||
weil die Vorschläge an Ihr Konto gebunden sind.
|
||||
</p>
|
||||
|
||||
<h2 id="english">English summary</h2>
|
||||
<p>
|
||||
This site is not affiliated with pl8list. To sign in, your email address and login code are passed to
|
||||
pl8list.com, which sends you the code. We keep your pl8list user id, handle, display name and avatar id,
|
||||
plus what you are shown, play, finish, skip, like or mark "not for me", to recommend tracks. Your
|
||||
pl8list login is never stored: we use it once, in memory, for at most two minutes to read your pl8list
|
||||
library, then log it out at pl8list. Audio and images are fetched through our server, so pl8list does not
|
||||
see your IP address. We use only the cookies listed above, which are strictly necessary; there is no
|
||||
tracking or analytics. IP and email address are held in memory for up to 15 minutes to rate-limit
|
||||
sign-ins. You can download all your data and delete your account in the "account" menu at any time.
|
||||
Legal bases: Art. 6(1)(b) GDPR for the account and recommendations, Art. 6(1)(f) for delivery and abuse
|
||||
protection. You may complain to a data protection authority.
|
||||
</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,52 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex">
|
||||
<title>Impressum · pl8list ∞</title>
|
||||
<link rel="icon" href="/static/icons/icon.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
</head>
|
||||
<body class="legal">
|
||||
<main>
|
||||
<p><a href="/">← zurück</a></p>
|
||||
<h1>Impressum</h1>
|
||||
|
||||
<h2>Angaben gemäß § 5 DDG</h2>
|
||||
<p>
|
||||
{{name}}<br>
|
||||
{{address}}
|
||||
</p>
|
||||
|
||||
<h2>Kontakt</h2>
|
||||
<p>
|
||||
E-Mail: {{email}}{{phone}}
|
||||
</p>
|
||||
|
||||
<h2>Verantwortlich für den Inhalt nach § 18 Abs. 2 MStV</h2>
|
||||
<p>
|
||||
{{name}}<br>
|
||||
{{address}}
|
||||
</p>
|
||||
|
||||
<h2>Hinweis zu pl8list</h2>
|
||||
<p>
|
||||
pl8list ∞ ist ein privates, nicht kommerzielles Projekt und steht in keiner Verbindung zu
|
||||
pl8list.com oder deren Betreibern. Alle Titel, Namen, Bilder und Audioinhalte stammen von
|
||||
pl8list.com und gehören den jeweiligen Künstlerinnen und Künstlern. Wenn Sie als Rechteinhaber
|
||||
möchten, dass Inhalte hier nicht mehr erscheinen, schreiben Sie bitte an die oben genannte
|
||||
E-Mail-Adresse.
|
||||
</p>
|
||||
|
||||
<h2>Haftung für Links</h2>
|
||||
<p>
|
||||
Diese Seite verlinkt auf pl8list.com. Für deren Inhalte ist allein der jeweilige Anbieter
|
||||
verantwortlich. Bei Bekanntwerden von Rechtsverletzungen entfernen wir entsprechende Links
|
||||
umgehend.
|
||||
</p>
|
||||
|
||||
<p><a href="/datenschutz">Datenschutzerklärung</a></p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
Reference in new issue
Block a user