mirror of
https://github.com/narrrl/pl8list-rewind.git
synced 2026-11-19 01:25:03 +01:00
Rewrite backend in Rust, add Impressum, privacy policy and account tools
New axum server in backend/ with the same HTTP API and database as the Python one. The pl8list session is only held in memory during the feed crawl and logged out afterwards. Local sessions use a __Host- cookie, 30-day idle expiry and weekly token rotation; the old cookie is migrated. The audio and image proxy only accepts known parameters, fixes the content type, follows redirects only to pl8list's file host and caches segments and images on disk. Strict security headers, origin checks on POST, body and concurrency limits. /impressum and /datenschutz are rendered from env variables. The account menu offers data export, sign out on all devices and account deletion. hls.js is self-hosted and artist images are proxied, so the browser only talks to this server. The Docker image builds a static binary on scratch.
This commit is contained in:
1 parent
61af1a6dd2
commit
54a436ae48
35 files changed
+6389
-42
No files matched your search
@@ -0,0 +1,162 @@
|
||||
//! pl8list's passwordless login: POST email, pl8list mails a 6-digit code, POST code.
|
||||
//!
|
||||
//! The site keeps the pending login in its `_session` cookie, so one flow holds that
|
||||
//! cookie between the two steps. After `verify` succeeds the cookie is a fully
|
||||
//! logged-in pl8list session. It is handed to the caller exactly once and is never
|
||||
//! logged, stored or kept in the flow.
|
||||
|
||||
use base64::{
|
||||
Engine,
|
||||
alphabet,
|
||||
engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig},
|
||||
};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::Pl8;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct LoginError(pub String);
|
||||
|
||||
impl std::fmt::Display for LoginError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn err(msg: impl Into<String>) -> LoginError {
|
||||
LoginError(msg.into())
|
||||
}
|
||||
|
||||
/// The pl8list account that just signed in.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Pl8User {
|
||||
pub id: i64,
|
||||
pub handle: String,
|
||||
pub name: Option<String>,
|
||||
pub img: Option<String>,
|
||||
}
|
||||
|
||||
pub struct LoginFlow {
|
||||
pub email: String,
|
||||
session: Option<SecretString>,
|
||||
}
|
||||
|
||||
struct Reply {
|
||||
status: u16,
|
||||
remix_redirect: Option<String>,
|
||||
payload: Option<Value>,
|
||||
}
|
||||
|
||||
impl LoginFlow {
|
||||
pub fn new(email: String) -> Self {
|
||||
LoginFlow { email, session: None }
|
||||
}
|
||||
|
||||
async fn call(&mut self, pl8: &Pl8, path: &str, query: &[(&str, &str)], route: &str, form: Option<&[(&str, &str)]>) -> Result<Reply, LoginError> {
|
||||
let mut url = pl8.url(&[path]);
|
||||
{
|
||||
let mut q = url.query_pairs_mut();
|
||||
for (k, v) in query {
|
||||
q.append_pair(k, v);
|
||||
}
|
||||
q.append_pair("_data", route);
|
||||
}
|
||||
let mut req = match form {
|
||||
Some(fields) => pl8.http().post(url).form(fields),
|
||||
None => pl8.http().get(url),
|
||||
};
|
||||
if let Some(s) = &self.session {
|
||||
req = req.header("Cookie", format!("_session={}", s.expose_secret()));
|
||||
}
|
||||
let res = req.send().await.map_err(|e| err(format!("pl8list unreachable: {}", e.without_url())))?;
|
||||
for c in res.headers().get_all("set-cookie") {
|
||||
if let Some(v) = c.to_str().ok().and_then(|c| c.strip_prefix("_session=")) {
|
||||
let v = v.split(';').next().unwrap_or_default();
|
||||
self.session = (!v.is_empty()).then(|| SecretString::from(v.to_string()));
|
||||
}
|
||||
}
|
||||
let status = res.status().as_u16();
|
||||
let remix_redirect = res.headers().get("x-remix-redirect").and_then(|v| v.to_str().ok()).map(String::from);
|
||||
let payload = res.bytes().await.ok().and_then(|b| serde_json::from_slice(&b).ok());
|
||||
Ok(Reply { status, remix_redirect, payload })
|
||||
}
|
||||
|
||||
/// The guest id pl8list stores in its cookie; the login forms echo it back.
|
||||
fn guest_id(&self) -> String {
|
||||
let Some(s) = &self.session else { return String::new() };
|
||||
let raw = percent_decode_str(s.expose_secret()).decode_utf8_lossy();
|
||||
let first = raw.split('.').next().unwrap_or_default();
|
||||
let engine = GeneralPurpose::new(
|
||||
&alphabet::STANDARD,
|
||||
GeneralPurposeConfig::new().with_decode_padding_mode(DecodePaddingMode::Indifferent),
|
||||
);
|
||||
engine
|
||||
.decode(first)
|
||||
.ok()
|
||||
.and_then(|b| serde_json::from_slice::<Value>(&b).ok())
|
||||
.and_then(|v| v.get("pl8list_guest_id").and_then(Value::as_str).map(String::from))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn payload_str(reply: &Reply, key: &str) -> Option<String> {
|
||||
reply.payload.as_ref()?.get(key)?.as_str().map(String::from)
|
||||
}
|
||||
|
||||
/// Ask pl8list to mail a login code.
|
||||
pub async fn start(&mut self, pl8: &Pl8) -> Result<(), LoginError> {
|
||||
self.call(pl8, "login", &[], "routes/login", None).await?;
|
||||
let guest = self.guest_id();
|
||||
let email = self.email.clone();
|
||||
let reply = self
|
||||
.call(pl8, "login", &[], "routes/login", Some(&[("email", &email), ("pl8list_guest_id", &guest)]))
|
||||
.await?;
|
||||
if !reply.remix_redirect.as_deref().unwrap_or_default().contains("/verify") {
|
||||
return Err(err(Self::payload_str(&reply, "error")
|
||||
.unwrap_or_else(|| format!("pl8list refused the login (HTTP {})", reply.status))));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn resend(&mut self, pl8: &Pl8) -> Result<(), LoginError> {
|
||||
let guest = self.guest_id();
|
||||
let email = self.email.clone();
|
||||
let query = [("email", email.as_str()), ("mode", "login")];
|
||||
let reply = self
|
||||
.call(pl8, "verify", &query, "routes/verify", Some(&[("action", "resend"), ("pl8list_guest_id", &guest)]))
|
||||
.await?;
|
||||
match Self::payload_str(&reply, "resendError") {
|
||||
Some(e) => Err(err(e)),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit the code. On success returns the account and its logged-in pl8list session;
|
||||
/// the flow no longer holds the session afterwards.
|
||||
pub async fn verify(&mut self, pl8: &Pl8, code: &str) -> Result<(Pl8User, SecretString), LoginError> {
|
||||
let guest = self.guest_id();
|
||||
let email = self.email.clone();
|
||||
let query = [("email", email.as_str()), ("mode", "login")];
|
||||
let reply = self
|
||||
.call(pl8, "verify", &query, "routes/verify", Some(&[("action", "verify"), ("code", code), ("pl8list_guest_id", &guest)]))
|
||||
.await?;
|
||||
if let Some(e) = Self::payload_str(&reply, "error") {
|
||||
return Err(err(e));
|
||||
}
|
||||
let session = self.session.take().ok_or_else(|| err("login failed, try again"))?;
|
||||
let root = pl8.root(Some(&session)).await.ok().flatten();
|
||||
let user = root.as_ref().and_then(|r| r.get("currentUser")).filter(|u| u.is_object());
|
||||
let Some(user) = user else {
|
||||
// Not logged in after all; nothing to end at pl8list.
|
||||
return Err(err("login failed, try again"));
|
||||
};
|
||||
let handle = user.get("handle").and_then(Value::as_str).filter(|h| !h.is_empty());
|
||||
let (Some(id), Some(handle)) = (user.get("id").and_then(Value::as_i64), handle) else {
|
||||
pl8.logout(&session).await;
|
||||
return Err(err("finish setting up your account on pl8list.com first (choose a handle)"));
|
||||
};
|
||||
let text = |k: &str| user.get(k).and_then(Value::as_str).map(String::from);
|
||||
Ok((Pl8User { id, handle: handle.to_string(), name: text("name"), img: text("img") }, session))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//! Client for pl8list.com.
|
||||
//!
|
||||
//! pl8list is a Remix app, so every page loader can be read as JSON by adding
|
||||
//! `?_data=<route id>` to the page URL. Resource routes under /api/ return JSON
|
||||
//! directly. All JSON requests share one throttle so we stay polite.
|
||||
//!
|
||||
//! Public data (artist pages, related tracks, previews, free tracks) works without
|
||||
//! an account, so the crawler runs as a guest. A user's own session is only needed
|
||||
//! for login and for reading their personal feed.
|
||||
|
||||
pub mod login;
|
||||
pub mod normalize;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::{StatusCode, redirect};
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use serde_json::Value;
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
use url::Url;
|
||||
|
||||
pub const USER_AGENT: &str = "pl8list-wrapper/0.3 (infinite-scroll client, https://pl8.narl.io)";
|
||||
/// Signed storage URLs for audio and images point here.
|
||||
const FILES_HOST: &str = "files.pl8list.com";
|
||||
|
||||
pub struct Pl8 {
|
||||
http: reqwest::Client,
|
||||
base: Url,
|
||||
min_interval: Duration,
|
||||
last: Mutex<Instant>,
|
||||
}
|
||||
|
||||
impl Pl8 {
|
||||
pub fn new(base: &str, min_interval: Duration) -> anyhow::Result<Self> {
|
||||
let base = Url::parse(base)?;
|
||||
let own_host = base.host_str().unwrap_or_default().to_string();
|
||||
let own_scheme = base.scheme().to_string();
|
||||
// Follow redirects only to pl8list and its storage host, never anywhere else.
|
||||
let policy = redirect::Policy::custom(move |attempt| {
|
||||
let url = attempt.url();
|
||||
let host = url.host_str().unwrap_or_default();
|
||||
let ok_host = host == own_host || host == FILES_HOST;
|
||||
let ok_scheme = url.scheme() == "https" || url.scheme() == own_scheme;
|
||||
if attempt.previous().len() >= 3 || !ok_host || !ok_scheme {
|
||||
attempt.stop()
|
||||
} else {
|
||||
attempt.follow()
|
||||
}
|
||||
});
|
||||
let http = reqwest::Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.redirect(policy)
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
Ok(Pl8 { http, base, min_interval, last: Mutex::new(Instant::now() - min_interval) })
|
||||
}
|
||||
|
||||
async fn throttle(&self) {
|
||||
let mut last = self.last.lock().await;
|
||||
let next = *last + self.min_interval;
|
||||
tokio::time::sleep_until(next).await;
|
||||
*last = Instant::now();
|
||||
}
|
||||
|
||||
/// URL on pl8list from already-decoded path segments.
|
||||
pub fn url(&self, segments: &[&str]) -> Url {
|
||||
let mut url = self.base.clone();
|
||||
url.path_segments_mut().expect("base is a URL with a path").clear().extend(segments);
|
||||
url
|
||||
}
|
||||
|
||||
pub fn request(&self, url: Url, session: Option<&SecretString>) -> reqwest::RequestBuilder {
|
||||
let req = self.http.get(url);
|
||||
match session {
|
||||
Some(s) => req.header("Cookie", format!("_session={}", s.expose_secret())),
|
||||
None => req,
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON GET with retries on overload. 403/404 give `None`.
|
||||
pub async fn get_json(
|
||||
&self,
|
||||
segments: &[&str],
|
||||
params: &[(&str, String)],
|
||||
route: Option<&str>,
|
||||
session: Option<&SecretString>,
|
||||
) -> anyhow::Result<Option<Value>> {
|
||||
let mut url = self.url(segments);
|
||||
{
|
||||
let mut q = url.query_pairs_mut();
|
||||
for (k, v) in params {
|
||||
q.append_pair(k, v);
|
||||
}
|
||||
if let Some(r) = route {
|
||||
q.append_pair("_data", r);
|
||||
}
|
||||
}
|
||||
for attempt in 0..4u32 {
|
||||
self.throttle().await;
|
||||
let res = self.request(url.clone(), session).send().await?;
|
||||
match res.status() {
|
||||
s if s.is_success() => {
|
||||
let body = res.bytes().await?;
|
||||
return Ok(if body.is_empty() { None } else { Some(serde_json::from_slice(&body)?) });
|
||||
}
|
||||
StatusCode::TOO_MANY_REQUESTS | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE
|
||||
| StatusCode::GATEWAY_TIMEOUT
|
||||
if attempt < 3 =>
|
||||
{
|
||||
tokio::time::sleep(Duration::from_secs(3 << attempt)).await;
|
||||
}
|
||||
StatusCode::FORBIDDEN | StatusCode::NOT_FOUND => return Ok(None),
|
||||
s => anyhow::bail!("pl8list {} for {}", s, url.path()),
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
// Loaders -------------------------------------------------------------
|
||||
//
|
||||
// Paginated loaders ignore `page` unless `pagination=true` is also set,
|
||||
// which is what the site's own useInfiniteLoading hook sends.
|
||||
|
||||
pub async fn root(&self, session: Option<&SecretString>) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&[], &[], Some("root"), session).await
|
||||
}
|
||||
|
||||
pub async fn home(&self) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&[], &[], Some("routes/_index"), None).await
|
||||
}
|
||||
|
||||
pub async fn feed(&self, session: &SecretString, page: i64) -> anyhow::Result<Option<Value>> {
|
||||
let p = [("page", page.to_string()), ("pagination", "true".into())];
|
||||
self.get_json(&["api", "feed"], &p, None, Some(session)).await
|
||||
}
|
||||
|
||||
pub async fn artist(&self, handle: &str, page: i64) -> anyhow::Result<Option<Value>> {
|
||||
let p = [("page", page.to_string()), ("pagination", "true".into())];
|
||||
self.get_json(&[handle], &p, Some("routes/$handle"), None).await
|
||||
}
|
||||
|
||||
pub async fn related(&self, handle: &str, slug: &str) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&[handle, slug, "related"], &[], Some("routes/$handle_.$slug.related"), None).await
|
||||
}
|
||||
|
||||
pub async fn best_selling_artists(&self) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&["best-selling-artists"], &[], Some("routes/best-selling-artists"), None).await
|
||||
}
|
||||
|
||||
pub async fn activity_ticker(&self) -> anyhow::Result<Option<Value>> {
|
||||
self.get_json(&["api", "activity-ticker"], &[], None, None).await
|
||||
}
|
||||
|
||||
pub async fn search_users(&self, q: &str, page: i64) -> anyhow::Result<Option<Value>> {
|
||||
let p = [("q", q.to_string()), ("page", page.to_string()), ("pagination", "true".into())];
|
||||
self.get_json(&["search", "users"], &p, Some("routes/search.users"), None).await
|
||||
}
|
||||
|
||||
/// End a session at pl8list. Errors are ignored: we drop the session either way.
|
||||
pub async fn logout(&self, session: &SecretString) {
|
||||
let mut url = self.url(&["logout"]);
|
||||
url.query_pairs_mut().append_pair("_data", "routes/logout");
|
||||
if let Err(e) = self.request(url, Some(session)).send().await {
|
||||
tracing::warn!("pl8list logout failed: {}", e.without_url());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn http(&self) -> &reqwest::Client {
|
||||
&self.http
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
//! Flatten pl8list loader JSON into `tracks` rows.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::db::Track;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Extra {
|
||||
full_access: bool,
|
||||
saves: i64,
|
||||
downloads: i64,
|
||||
purchases: i64,
|
||||
comments: i64,
|
||||
}
|
||||
|
||||
fn int(v: Option<&Value>) -> i64 {
|
||||
v.and_then(|v| v.as_i64().or_else(|| v.as_f64().map(|f| f as i64))).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn text(t: &Value, key: &str) -> Option<String> {
|
||||
t.get(key).and_then(Value::as_str).filter(|s| !s.is_empty()).map(String::from)
|
||||
}
|
||||
|
||||
fn num(t: &Value, key: &str) -> Option<f64> {
|
||||
t.get(key).and_then(Value::as_f64)
|
||||
}
|
||||
|
||||
fn json_text(t: &Value, key: &str) -> Option<String> {
|
||||
t.get(key).filter(|v| !v.is_null() && v.as_array().is_none_or(|a| !a.is_empty())).map(Value::to_string)
|
||||
}
|
||||
|
||||
/// A loader track item (`{track, hasFullAccess, saveCount, ...}`).
|
||||
pub fn item(item: &Value) -> Option<Track> {
|
||||
track(item.get("track")?, Extra {
|
||||
full_access: item.get("hasFullAccess").and_then(Value::as_bool).unwrap_or(false),
|
||||
saves: int(item.get("saveCount")),
|
||||
downloads: int(item.get("downloadCount")),
|
||||
purchases: int(item.get("purchaseCount")),
|
||||
comments: int(item.get("commentCount")),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn items(list: Option<&Value>) -> Vec<Track> {
|
||||
list.and_then(Value::as_array).map(|a| a.iter().filter_map(item).collect()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The related loader returns bare tracks plus per-id maps for access and counts.
|
||||
pub fn related(data: &Value) -> Vec<Track> {
|
||||
let counts = data.get("counts");
|
||||
let count = |name: &str, k: &str| int(counts.and_then(|c| c.get(name)).and_then(|m| m.get(k)));
|
||||
let Some(tracks) = data.get("tracks").and_then(Value::as_array) else { return vec![] };
|
||||
tracks
|
||||
.iter()
|
||||
.filter_map(|t| {
|
||||
let k = t.get("id")?.as_i64()?.to_string();
|
||||
track(t, Extra {
|
||||
full_access: data.get("access").and_then(|a| a.get(&k)).and_then(Value::as_bool).unwrap_or(false),
|
||||
saves: count("saveCount", &k),
|
||||
downloads: count("downloadCount", &k),
|
||||
purchases: count("purchaseCount", &k),
|
||||
comments: count("commentCount", &k),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn track(t: &Value, extra: Extra) -> Option<Track> {
|
||||
let user = t.get("user")?;
|
||||
let handle = text(user, "handle")?;
|
||||
if t.get("isDeleted").and_then(Value::as_bool) == Some(true) || t.get("isPublic").and_then(Value::as_bool) == Some(false) {
|
||||
return None;
|
||||
}
|
||||
let slug = text(t, "slug")?;
|
||||
Some(Track {
|
||||
id: t.get("id")?.as_i64()?,
|
||||
artist_name: text(user, "name").unwrap_or_else(|| handle.clone()),
|
||||
artist_img: text(user, "img"),
|
||||
artist_verified: user.get("isVerified").and_then(Value::as_bool).unwrap_or(false),
|
||||
title: text(t, "title").or_else(|| text(t, "filename")).unwrap_or_else(|| slug.clone()),
|
||||
description: text(t, "description"),
|
||||
bpm: num(t, "bpm"),
|
||||
key: text(t, "key"),
|
||||
duration: num(t, "duration"),
|
||||
preview_duration: num(t, "previewDuration"),
|
||||
price: num(t, "price").unwrap_or(0.0),
|
||||
created_at: text(t, "createdAt"),
|
||||
upload_id: text(t, "uploadId"),
|
||||
preview_upload_id: text(t, "previewUploadId"),
|
||||
waveform: json_text(t, "waveform"),
|
||||
preview_waveform: json_text(t, "previewWaveform"),
|
||||
full_access: extra.full_access,
|
||||
saves: extra.saves,
|
||||
downloads: extra.downloads,
|
||||
purchases: extra.purchases,
|
||||
comments: extra.comments,
|
||||
related_of: None,
|
||||
handle,
|
||||
slug,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn item_flattens_and_skips_private() {
|
||||
let i = json!({
|
||||
"track": {"id": 5, "slug": "s", "title": "T", "price": 4.99, "bpm": 140,
|
||||
"waveform": [1, 2], "user": {"handle": "h", "isVerified": true}},
|
||||
"hasFullAccess": false, "saveCount": 3
|
||||
});
|
||||
let t = item(&i).unwrap();
|
||||
assert_eq!((t.id, t.handle.as_str(), t.artist_name.as_str()), (5, "h", "h"));
|
||||
assert_eq!((t.price, t.bpm, t.saves), (4.99, Some(140.0), 3));
|
||||
assert_eq!(t.waveform.as_deref(), Some("[1,2]"));
|
||||
let private = json!({"track": {"id": 6, "slug": "s", "isPublic": false, "user": {"handle": "h"}}});
|
||||
assert!(item(&private).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn related_reads_count_maps() {
|
||||
let d = json!({
|
||||
"tracks": [{"id": 9, "slug": "x", "user": {"handle": "a"}}],
|
||||
"access": {"9": true},
|
||||
"counts": {"saveCount": {"9": 4}}
|
||||
});
|
||||
let r = related(&d);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert!(r[0].full_access);
|
||||
assert_eq!(r[0].saves, 4);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user