diff --git a/backend-rs/.env.example b/backend-rs/.env.example index e7876ad..d3cbe30 100644 --- a/backend-rs/.env.example +++ b/backend-rs/.env.example @@ -41,5 +41,20 @@ JWT_AUTH_COOKIE_SECURE=false # instances must leave it false or jobs fire twice. ENABLE_SCHEDULER=false +# --- Shop reviews (sync_reviews job) -------------------------------------- +# reviews.io store slug to pull anonymous shop reviews from. They feed the +# recommender as a prior under the community's own ratings — see +# src/recommend.rs. +REVIEWS_STORE=holy + +# Cap on pages fetched per run (100 reviews per page). The incremental sync +# stops on its own after a few pages; this only bounds a first-run backfill. +#REVIEWS_MAX_PAGES=2000 + +# Salt for the one-way hash of upstream reviewer ids. Defaults to SECRET_KEY. +# Changing it re-pseudonymises new rows only, so old rows stop joining up with +# them — truncate `external_review` and re-sync if you rotate it. +#REVIEWS_HASH_SALT=change-me + # tracing filter RUST_LOG=info,holy_backend_rs=debug,sea_orm=warn diff --git a/backend-rs/README.md b/backend-rs/README.md index 86d123c..3148f73 100644 --- a/backend-rs/README.md +++ b/backend-rs/README.md @@ -147,11 +147,16 @@ management command / Celery task. All endpoints require `IsAdminUser` (`is_staff layer with a single abstraction: - `BackgroundJob` async trait (`name`, `display_name`, `run`) + a `registry()` of - 5 jobs: `sync_flavors`, `cleanup_duplicates`, `backup_db`, `seed_legacy`, - `seed_banners`. `run_job()` is the lifecycle wrapper — it writes status / + 6 jobs: `sync_flavors`, `sync_reviews`, `cleanup_duplicates`, `backup_db`, + `seed_legacy`, `seed_banners`. `run_job()` is the lifecycle wrapper — it writes status / `last_run` / `last_output` / `error_message` into the `api_job` row (parity with Django's `_run_command_job`) and a per-process concurrency guard (`Arc>`) prevents the same job running twice. +- **CLI escape hatch**: `holy_backend_rs --run-job ` runs one job to + completion and exits without starting the server, for one-off operational runs + (a first backfill, a re-sync after a data fix). It bypasses the scheduler, so + it's safe on an instance that doesn't own `api_job` — just don't run a job that + is already running elsewhere. - **Scheduler** (`scheduler.rs`): an in-process 60 s poller replacing Celery beat. Each tick, any registry job whose row has `interval_hours > 0` and is due (`last_run` null, or `now - last_run >= interval_hours`) and not already running @@ -172,6 +177,40 @@ Django-data parity for the shared DB + media tree: `created_at`), re-points replies/notifications, inherits `external_id`, in a txn. - `sync_flavors` is a full port including the **Syrup** special-case (the single bundle product's variants expand into individual `Flavor` rows keyed by variant id). + It also records each flavor's Shopify `variant_ids`, which `sync_reviews` needs. + +### Shop reviews (`sync_reviews`) — no Django equivalent + +Pulls the public reviews.io timeline for the Holy store and keeps the product +reviews whose `sku` (a Shopify **variant** id) resolves to a flavor we track, +excluding the packs/merch category. About a quarter of product reviews map; the +rest are mix boxes, shakers and regional bundles that say nothing about a single +flavor. Newest-first crawl: deep on the first run, then stops after three +consecutive pages with nothing new (typically 2–3 requests), 250 ms apart. + +**Privacy:** the upstream payload carries the reviewer's name, city and review +text; none of it is stored. The `external_review` row keeps only a salted hash of +the upstream reviewer id, the flavor, the 1–5 rating and the date. The hash exists +solely so the recommender can distinguish "one buyer liked both of these" from +"two unrelated buyers". This is a ratings signal, not a mirror of their reviews. + +**How the recommender uses it** (`src/recommend.rs`) — deliberately *not* as extra +rows in the rating matrix, since the feed is 1–5 stars, heavily inflated (72% are +5★) and mostly single-review reviewers whose zero variance the correlation maths +discards anyway. Instead: + +- **Informed prior** in the popularity ranking: a flavor's community score is + shrunk toward the shop consensus for that flavor (recentred onto the 1–10 scale) + rather than toward the flat catalog mean. Prior strength is capped, so a solid + community verdict always outweighs it. Flavors with no community ratings at all + become candidates for the first time, tagged `reason: "shop"`. +- **Co-occurrence layer** in `/flavors/{id}/similar/`: cosine over "which anonymous + buyers liked both", blended with the community's adjusted cosine by confidence. + Meaningful even when every rating involved was 5★. A negative community + correlation vetoes the pair outright. + +Aggregates are cached in-process for 10 minutes (`src/reviews.rs`) and invalidated +by the job; scanning the whole table per request would be far too expensive. Verified live against a copy of the production SQLite: scheduler dispatched the due `sync_flavors` job which fetched `weareholy.com` and reported diff --git a/backend-rs/src/config.rs b/backend-rs/src/config.rs index 9b30303..da5f909 100644 --- a/backend-rs/src/config.rs +++ b/backend-rs/src/config.rs @@ -32,6 +32,17 @@ pub struct Config { /// Whether the in-process job scheduler runs. Production (docker compose) /// sets this to true on exactly one instance to own job scheduling. pub enable_scheduler: bool, + /// reviews.io store slug the `sync_reviews` job pulls shop reviews for. + pub reviews_store: String, + /// Hard cap on pages `sync_reviews` fetches in one run. Mainly an escape hatch + /// for bounding a first-run backfill (or a dry run) — the incremental sync + /// stops on its own long before this. + pub reviews_max_pages: usize, + /// Salt for the one-way hash of upstream reviewer ids (`REVIEWS_HASH_SALT`, + /// falling back to `SECRET_KEY`). Changing it re-pseudonymises new rows only — + /// existing `external_review` rows keep their old keys and would no longer + /// join up with them, so truncate the table if you ever rotate this. + pub reviews_hash_salt: String, } impl Config { @@ -127,6 +138,21 @@ impl Config { // Production (docker compose) sets ENABLE_SCHEDULER=true on the one // instance that should own `api_job` scheduling. enable_scheduler: env_bool("ENABLE_SCHEDULER", false), + reviews_store: env::var("REVIEWS_STORE") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "holy".to_string()), + reviews_max_pages: env::var("REVIEWS_MAX_PAGES") + .ok() + .and_then(|s| s.parse().ok()) + .filter(|n| *n > 0) + .unwrap_or(2_000), + reviews_hash_salt: env::var("REVIEWS_HASH_SALT") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| { + env::var("SECRET_KEY").unwrap_or_else(|_| "holy-reviews".to_string()) + }), }) } } diff --git a/backend-rs/src/db.rs b/backend-rs/src/db.rs index 65fc077..9e2cf95 100644 --- a/backend-rs/src/db.rs +++ b/backend-rs/src/db.rs @@ -94,6 +94,52 @@ async fn ensure_schema(conn: &DatabaseConnection) -> anyhow::Result<()> { .to_owned(), )) .await?; + // Shopify variant ids belonging to this flavor's product, as a JSON list of + // strings. Populated by `sync_flavors`; `sync_reviews` needs it because the + // reviews.io feed identifies a product by *variant* id (its `sku` field), + // while `external_id` holds the *product* id for everything but Syrup. + add_column_if_missing( + conn, + "api_flavor", + "variant_ids", + "ALTER TABLE api_flavor ADD COLUMN variant_ids TEXT", + ) + .await?; + // Rust-owned table (no Django model): shop reviews pulled from reviews.io by + // the `sync_reviews` job, used as an anonymous prior under the community + // recommender. Deliberately stores no personal data — the reviewer is a + // salted hash of the upstream reviewer id, and the review body, author name + // and city from the upstream payload are all discarded on ingest. + // UNIQUE(source, external_id) makes re-ingesting a page a no-op. + conn.execute(Statement::from_string( + DatabaseBackend::Sqlite, + "CREATE TABLE IF NOT EXISTS external_review (\ + id INTEGER PRIMARY KEY AUTOINCREMENT, \ + source VARCHAR(32) NOT NULL, \ + external_id VARCHAR(64) NOT NULL, \ + reviewer_key VARCHAR(64) NOT NULL, \ + flavor_id INTEGER NOT NULL, \ + rating INTEGER NOT NULL, \ + reviewed_at DATETIME NOT NULL, \ + created_at DATETIME NOT NULL, \ + UNIQUE(source, external_id))" + .to_owned(), + )) + .await?; + conn.execute(Statement::from_string( + DatabaseBackend::Sqlite, + "CREATE INDEX IF NOT EXISTS idx_external_review_flavor \ + ON external_review(flavor_id)" + .to_owned(), + )) + .await?; + conn.execute(Statement::from_string( + DatabaseBackend::Sqlite, + "CREATE INDEX IF NOT EXISTS idx_external_review_reviewer \ + ON external_review(reviewer_key)" + .to_owned(), + )) + .await?; Ok(()) } diff --git a/backend-rs/src/dto.rs b/backend-rs/src/dto.rs index 1556d79..5e1c106 100644 --- a/backend-rs/src/dto.rs +++ b/backend-rs/src/dto.rs @@ -235,8 +235,12 @@ pub struct RecommendationOut { pub predicted_score: f64, /// "N tasters like you" count (CF) or the flavor's rating count (popularity). pub contributing_neighbours: i64, - /// `"cf"` or `"popular"` — frontend picks the reason copy. Names are never - /// surfaced (privacy decision). + /// Anonymous shop reviews (reviews.io) backing this flavor. 0 when the feed + /// hasn't been synced or doesn't cover it. + pub external_reviews: i64, + /// `"cf"`, `"popular"` or `"shop"` (nobody here has rated it yet; the score + /// comes from shop reviews alone) — the frontend picks the reason copy. Names + /// are never surfaced (privacy decision). pub reason: String, } @@ -252,10 +256,12 @@ pub struct SimilarFlavorOut { pub average_rating: Option, pub is_legacy: bool, pub is_available: bool, - /// Adjusted-cosine similarity to the target flavor, in (0, 1]. + /// Blended similarity to the target flavor, in (0, 1]. pub similarity: f64, /// Users who rated both this flavor and the target (the confidence count). pub co_raters: i64, + /// Anonymous shop reviewers who liked both this flavor and the target. + pub external_co_reviewers: i64, } #[derive(Serialize, ToSchema)] diff --git a/backend-rs/src/entities/external_review.rs b/backend-rs/src/entities/external_review.rs new file mode 100644 index 0000000..4f52507 --- /dev/null +++ b/backend-rs/src/entities/external_review.rs @@ -0,0 +1,48 @@ +//! Shop reviews ingested from an external review platform (currently reviews.io) +//! by the `sync_reviews` job. No Django equivalent — this table is owned by the +//! Rust backend (created in `db::ensure_schema`). +//! +//! Privacy: this is deliberately *not* a copy of the upstream review. The author +//! name, city and review body present in the upstream payload are dropped at +//! ingest; `reviewer_key` is a salted SHA-256 of the upstream reviewer id, kept +//! only so the recommender can tell "the same anonymous buyer rated these two +//! flavors" apart from two unrelated buyers. +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "external_review")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + /// Upstream platform slug, e.g. `"reviews_io"`. Namespaces `external_id`. + pub source: String, + /// Upstream review id, unique per `source` — the incremental-sync stop marker. + pub external_id: String, + /// Salted hash of the upstream reviewer id (never the id itself). + pub reviewer_key: String, + pub flavor_id: i32, + /// Upstream star rating on its native 1–5 scale, stored unscaled. + pub rating: i32, + /// When the review was written upstream. + pub reviewed_at: ChronoDateTime, + /// When we ingested it. + pub created_at: ChronoDateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::flavor::Entity", + from = "Column::FlavorId", + to = "super::flavor::Column::Id" + )] + Flavor, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Flavor.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/backend-rs/src/entities/flavor.rs b/backend-rs/src/entities/flavor.rs index 1c42de1..6eb7016 100644 --- a/backend-rs/src/entities/flavor.rs +++ b/backend-rs/src/entities/flavor.rs @@ -29,6 +29,14 @@ pub struct Model { /// startup migration in `db.rs`; may be NULL on rows predating it. #[sea_orm(column_type = "Json", nullable)] pub aliases: Option, + /// Shopify variant ids of this flavor's product, as a JSON list of strings. + /// Written by `sync_flavors`; read by `sync_reviews`, which only ever sees a + /// variant id (reviews.io calls it `sku`) and has to resolve it back to a + /// flavor. For the Syrup special case the flavor *is* a variant, so this + /// holds that single id. Added by the guarded startup migration in `db.rs`; + /// NULL on rows that predate it or haven't been re-synced yet. + #[sea_orm(column_type = "Json", nullable)] + pub variant_ids: Option, /// Django stores naive UTC text ("YYYY-MM-DD HH:MM:SS.ffffff"); we treat it /// as UTC when formatting to DRF's ISO-8601 `...Z` output. pub created_at: ChronoDateTime, diff --git a/backend-rs/src/entities/mod.rs b/backend-rs/src/entities/mod.rs index 13ceebb..51f6aa3 100644 --- a/backend-rs/src/entities/mod.rs +++ b/backend-rs/src/entities/mod.rs @@ -7,6 +7,7 @@ pub mod banner; pub mod category; +pub mod external_review; pub mod flavor; pub mod job; pub mod notification; @@ -23,6 +24,7 @@ pub mod user_ip; pub mod prelude { pub use super::banner::Entity as Banner; pub use super::category::Entity as Category; + pub use super::external_review::Entity as ExternalReview; pub use super::flavor::Entity as Flavor; pub use super::job::Entity as Job; pub use super::notification::Entity as Notification; diff --git a/backend-rs/src/jobs/mod.rs b/backend-rs/src/jobs/mod.rs index 56cb827..1211999 100644 --- a/backend-rs/src/jobs/mod.rs +++ b/backend-rs/src/jobs/mod.rs @@ -18,6 +18,7 @@ mod scheduler; mod seed_banners; mod seed_legacy; mod sync_flavors; +mod sync_reviews; use std::sync::Arc; @@ -48,6 +49,7 @@ pub trait BackgroundJob: Send + Sync { pub fn registry() -> Vec> { vec![ Arc::new(sync_flavors::SyncFlavors), + Arc::new(sync_reviews::SyncReviews), Arc::new(cleanup_duplicates::CleanupDuplicates), Arc::new(backup_db::BackupDb), Arc::new(seed_legacy::SeedLegacy), diff --git a/backend-rs/src/jobs/sync_flavors.rs b/backend-rs/src/jobs/sync_flavors.rs index 228a8b6..c201f22 100644 --- a/backend-rs/src/jobs/sync_flavors.rs +++ b/backend-rs/src/jobs/sync_flavors.rs @@ -163,6 +163,13 @@ impl BackgroundJob for SyncFlavors { .and_then(|a| a.as_bool()) .unwrap_or(false) }); + // Every variant of this product maps back to this flavor — that's how + // `sync_reviews` resolves a reviews.io `sku` (a variant id) to a flavor. + let variant_ids: Vec = variants + .iter() + .filter_map(|v| v.get("id").and_then(|x| x.as_i64())) + .map(|id| id.to_string()) + .collect(); let images = p .get("images") .and_then(|v| v.as_array()) @@ -196,6 +203,7 @@ impl BackgroundJob for SyncFlavors { image_url, &image_urls, &pid.to_string(), + &variant_ids, ) .await?; if was_created { @@ -457,6 +465,8 @@ async fn sync_syrup_variants( image_url, &image_urls, &variant_id.to_string(), + // A syrup flavor *is* one variant, so it owns exactly that sku. + std::slice::from_ref(&variant_id.to_string()), ) .await?; if was_created { @@ -486,6 +496,7 @@ async fn upsert_flavor( image_url: Option, image_urls: &[String], dir_slug: &str, + variant_ids: &[String], ) -> anyhow::Result { let mut existing = Flavor::find() .filter(flavor::Column::ExternalId.eq(external_id)) @@ -561,6 +572,11 @@ async fn upsert_flavor( am.description = Set(description.to_string()); am.shop_url = Set(shop_url); am.is_available = Set(is_available); + // Keep the last known variant list when Shopify hands us none, so a transient + // empty payload can't blind `sync_reviews` to this flavor's skus. + if !variant_ids.is_empty() { + am.variant_ids = Set(Some(json!(variant_ids))); + } if !local_paths.is_empty() { am.local_image_paths = Set(Some(json!(local_paths))); diff --git a/backend-rs/src/jobs/sync_reviews.rs b/backend-rs/src/jobs/sync_reviews.rs new file mode 100644 index 0000000..cb178fe --- /dev/null +++ b/backend-rs/src/jobs/sync_reviews.rs @@ -0,0 +1,451 @@ +//! `sync_reviews` — pull anonymous shop reviews from reviews.io and store the +//! subset that maps to a real flavor, as an extra signal under the recommender +//! (see [`crate::recommend`] for how they're weighted). +//! +//! # What gets kept +//! +//! The upstream feed is a mixed timeline of store reviews ("great service") and +//! product reviews. Only product reviews matter here, and only those whose `sku` +//! resolves to a flavor we track — roughly a quarter of them. The rest are mix +//! boxes, shakers, merch and regional bundles, which say nothing about a flavor. +//! +//! # What gets discarded +//! +//! Everything identifying. The upstream payload carries the reviewer's name, city +//! and free-text review; none of it is written. All that survives is a salted hash +//! of the upstream reviewer id (see [`crate::reviews::reviewer_key`]), which exists +//! solely so the recommender can tell "one buyer liked both of these" apart from +//! "two unrelated buyers". This is a ratings signal, not a copy of their reviews. +//! +//! # Sync strategy +//! +//! The feed is walked newest-first. On the first run the table is empty so the +//! crawl runs deep (bounded by [`MAX_PAGES`]); afterwards it stops once +//! [`STOP_AFTER_KNOWN_PAGES`] consecutive pages contain nothing new, which is +//! typically two or three requests. Pages are fetched one at a time with a delay +//! between them — this is someone else's public endpoint. + +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +use async_trait::async_trait; +use sea_orm::sea_query::OnConflict; +use sea_orm::ActiveValue::Set; +use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QuerySelect}; +use serde_json::Value; + +use crate::entities::prelude::{Category, ExternalReview, Flavor}; +use crate::entities::{category, external_review, flavor}; +use crate::reviews::{reviewer_key, SOURCE_REVIEWS_IO}; +use crate::state::AppState; + +use super::download::http_client; +use super::BackgroundJob; + +/// Public widget feed. Returns a mixed store/product review timeline. +const TIMELINE_URL: &str = "https://api.reviews.io/timeline/data"; +/// Upstream page size. 100 is the largest the endpoint honours. +const PER_PAGE: usize = 100; +/// Default ceiling on pages per run (overridable via `REVIEWS_MAX_PAGES`). At +/// [`PER_PAGE`] this covers a feed of 200k reviews — enough for a full first-run +/// backfill, while bounding a run if upstream paging ever stops terminating. +const MAX_PAGES: usize = 2_000; +/// Stop after this many consecutive pages that contained mappable reviews but no +/// new ones. More than one because the feed is not strictly monotonic. +const STOP_AFTER_KNOWN_PAGES: usize = 3; +/// Politeness delay between page requests. +const PAGE_DELAY: Duration = Duration::from_millis(250); +/// Category slug holding bundles, shakers and merch. Reviews resolving here are +/// dropped: a review of a mixed sachet box tells us nothing about one flavor. +const PACKS_SLUG: &str = "packs-and-other"; + +pub struct SyncReviews; + +#[async_trait] +impl BackgroundJob for SyncReviews { + fn name(&self) -> &'static str { + "sync_reviews" + } + fn display_name(&self) -> &'static str { + "Sync Shop Reviews" + } + + async fn run(&self, state: &AppState) -> anyhow::Result { + let db = &state.db; + let mut log: Vec = Vec::new(); + + let sku_index = build_sku_index(state).await?; + if sku_index.is_empty() { + log.push( + "No flavor variant ids on record — run sync_flavors first so reviews \ + can be matched to flavors." + .to_string(), + ); + return Ok(log.join("\n")); + } + log.push(format!( + "Matching against {} variant skus across the catalog.", + sku_index.len() + )); + + let known_before = ExternalReview::find() + .filter(external_review::Column::Source.eq(SOURCE_REVIEWS_IO)) + .count(db) + .await?; + let backfill = known_before == 0; + log.push(if backfill { + "No reviews stored yet — running a full backfill.".to_string() + } else { + format!("{known_before} reviews already stored — incremental sync.") + }); + + let client = http_client(); + let store = &state.config.reviews_store; + let salt = &state.config.reviews_hash_salt; + + let mut inserted = 0usize; + let mut mapped = 0usize; + let mut skipped_unmapped = 0usize; + let mut pages_fetched = 0usize; + let mut quiet_pages = 0usize; + + let max_pages = state.config.reviews_max_pages.min(MAX_PAGES); + for page in 1..=max_pages { + let url = format!( + "{TIMELINE_URL}?store={store}&sort=date_desc&per_page={PER_PAGE}&page={page}" + ); + let body: Value = match fetch_page(&client, &url).await { + Ok(v) => v, + Err(e) => { + // Partial data is still useful — keep what we ingested and + // report where it stopped, rather than failing the whole run. + log.push(format!("Stopped at page {page}: {e}")); + break; + } + }; + pages_fetched += 1; + + let entries = body + .get("timeline") + .and_then(|v| v.as_array()) + .cloned() + .unwrap_or_default(); + if entries.is_empty() { + log.push(format!("Page {page} empty — end of feed.")); + break; + } + + // Product reviews on this page, paired with the flavor their sku + // resolves to. Unresolved ones are merch, bundles, or products we + // don't track — counted, then dropped. + let page_product_reviews: Vec = entries + .iter() + .filter_map(|e| e.get("_source")) + .filter_map(parse_product_review) + .collect(); + let product_review_count = page_product_reviews.len(); + let parsed: Vec<(ParsedReview, i32)> = page_product_reviews + .into_iter() + .filter_map(|r| sku_index.get(&r.sku).map(|&flavor_id| (r, flavor_id))) + .collect(); + skipped_unmapped += product_review_count - parsed.len(); + mapped += parsed.len(); + + // How many of this page's reviews we already had, before writing. + let already_known = count_known(db, &parsed).await?; + + let mut batch: Vec = Vec::new(); + let mut seen_in_batch: HashSet = HashSet::new(); + for (review, flavor_id) in parsed.iter() { + // The feed can repeat a review id within a page; UNIQUE handles it + // across pages, but one batch carrying a key twice would trip the + // on-conflict clause. + if !seen_in_batch.insert(review.external_id.clone()) { + continue; + } + batch.push(external_review::ActiveModel { + source: Set(SOURCE_REVIEWS_IO.to_string()), + external_id: Set(review.external_id.clone()), + reviewer_key: Set(reviewer_key(salt, SOURCE_REVIEWS_IO, &review.reviewer_id)), + flavor_id: Set(*flavor_id), + rating: Set(review.rating), + reviewed_at: Set(review.reviewed_at), + created_at: Set(crate::datetime::now_micros()), + ..Default::default() + }); + } + + if !batch.is_empty() { + ExternalReview::insert_many(batch) + .on_conflict( + OnConflict::columns([ + external_review::Column::Source, + external_review::Column::ExternalId, + ]) + .do_nothing() + .to_owned(), + ) + // Returns rows-affected rather than erroring when every row in + // the batch was already present (the steady state). + .exec_without_returning(db) + .await?; + } + + // Only pages that could have held something new count toward the stop + // condition — a page of pure merch reviews proves nothing either way. + if !parsed.is_empty() { + if already_known >= parsed.len() { + quiet_pages += 1; + } else { + quiet_pages = 0; + inserted += parsed.len() - already_known; + } + } + + if !backfill && quiet_pages >= STOP_AFTER_KNOWN_PAGES { + log.push(format!( + "Page {page}: {STOP_AFTER_KNOWN_PAGES} consecutive pages with nothing \ + new — caught up." + )); + break; + } + + tokio::time::sleep(PAGE_DELAY).await; + } + + // Recommendations read a cached aggregate; drop it so the new rows count. + state.external_signals.invalidate().await; + + let total = ExternalReview::find() + .filter(external_review::Column::Source.eq(SOURCE_REVIEWS_IO)) + .count(db) + .await?; + log.push(format!( + "Fetched {pages_fetched} pages. Flavor-mapped: {mapped}, new: {inserted}, \ + unmapped (packs/merch/untracked): {skipped_unmapped}. Stored total: {total}." + )); + Ok(log.join("\n")) + } +} + +/// One product review, reduced to the fields we're willing to store. +struct ParsedReview { + external_id: String, + reviewer_id: String, + sku: String, + rating: i32, + reviewed_at: chrono::NaiveDateTime, +} + +/// Pull a product review out of a timeline entry, or `None` if it isn't one or is +/// missing anything we need. Store reviews ("great service") carry no sku and are +/// filtered out here. +fn parse_product_review(src: &Value) -> Option { + if src.get("type").and_then(|v| v.as_str()) != Some("product_review") { + return None; + } + let sku = src.get("sku").and_then(json_as_string)?; + let external_id = src + .get("_id") + .and_then(json_as_string) + .or_else(|| src.get("product_review_id").and_then(json_as_string))?; + let reviewer_id = src.get("user_id").and_then(json_as_string)?; + let rating = src.get("rating").and_then(|v| v.as_i64())? as i32; + if !(1..=5).contains(&rating) { + return None; + } + let reviewed_at = src + .get("date_created") + .and_then(|v| v.as_str()) + .and_then(parse_upstream_datetime) + .unwrap_or_else(crate::datetime::now_micros); + Some(ParsedReview { + external_id, + reviewer_id, + sku, + rating, + reviewed_at, + }) +} + +/// The feed is inconsistent about quoting numeric ids — accept either form. +fn json_as_string(v: &Value) -> Option { + match v { + Value::String(s) if !s.is_empty() => Some(s.clone()), + Value::Number(n) => Some(n.to_string()), + _ => None, + } +} + +/// Upstream timestamps are `"YYYY-MM-DD HH:MM:SS"` (UTC, no zone marker); the +/// per-product endpoint uses ISO-8601 instead, so accept both. +fn parse_upstream_datetime(s: &str) -> Option { + chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") + .ok() + .or_else(|| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.fZ").ok()) +} + +/// Map every known Shopify variant id to its flavor, skipping the packs/merch +/// category. Reviews identify a product by variant id, which is what +/// `flavor.variant_ids` records. +async fn build_sku_index(state: &AppState) -> anyhow::Result> { + let packs_id = Category::find() + .filter(category::Column::Slug.eq(PACKS_SLUG)) + .one(&state.db) + .await? + .map(|c| c.id); + + let mut q = Flavor::find().filter(flavor::Column::VariantIds.is_not_null()); + if let Some(packs_id) = packs_id { + q = q.filter(flavor::Column::CategoryId.ne(packs_id)); + } + let flavors = q.all(&state.db).await?; + + let mut index = HashMap::new(); + for f in flavors { + let Some(Value::Array(ids)) = f.variant_ids else { + continue; + }; + for id in ids { + if let Some(sku) = json_as_string(&id) { + // First flavor wins: a sku belongs to exactly one product, so a + // clash means stale rows left by a merge — don't let it flap + // between runs. + index.entry(sku).or_insert(f.id); + } + } + } + Ok(index) +} + +/// How many of this page's mappable reviews are already stored. Drives the +/// "caught up" stop condition. +async fn count_known( + db: &sea_orm::DatabaseConnection, + parsed: &[(ParsedReview, i32)], +) -> anyhow::Result { + let ids: Vec = parsed + .iter() + .map(|(r, _)| r.external_id.clone()) + .collect::>() + .into_iter() + .collect(); + if ids.is_empty() { + return Ok(0); + } + let found: Vec = ExternalReview::find() + .filter(external_review::Column::Source.eq(SOURCE_REVIEWS_IO)) + .filter(external_review::Column::ExternalId.is_in(ids)) + .select_only() + .column(external_review::Column::ExternalId) + .into_tuple() + .all(db) + .await?; + Ok(found.len()) +} + +async fn fetch_page(client: &reqwest::Client, url: &str) -> anyhow::Result { + let resp = client.get(url).send().await?.error_for_status()?; + Ok(resp.json().await?) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn parses_a_product_review() { + let src = json!({ + "type": "product_review", + "_id": "product_review-55470773", + "user_id": 73972324, + "sku": "56879742288137", + "rating": 5, + "date_created": "2026-07-26 14:15:42", + "author": "Stefanie S", + "address": "Nuremberg, DE", + "comments": "sehr lecker", + }); + let r = parse_product_review(&src).expect("should parse"); + assert_eq!(r.external_id, "product_review-55470773"); + assert_eq!(r.reviewer_id, "73972324"); + assert_eq!(r.sku, "56879742288137"); + assert_eq!(r.rating, 5); + assert_eq!(r.reviewed_at.to_string(), "2026-07-26 14:15:42"); + } + + #[test] + fn store_reviews_are_not_product_reviews() { + let src = json!({ + "type": "store_review", + "_id": "store_review-30421911", + "user_id": 73972342, + "rating": 5, + "date_created": "2026-07-26 14:17:26", + }); + assert!(parse_product_review(&src).is_none()); + } + + #[test] + fn reviews_missing_required_fields_are_skipped() { + let base = json!({ + "type": "product_review", + "_id": "product_review-1", + "user_id": 1, + "sku": "123", + "rating": 5, + "date_created": "2026-07-26 14:15:42", + }); + for missing in ["sku", "user_id", "rating"] { + let mut src = base.clone(); + src.as_object_mut().unwrap().remove(missing); + assert!( + parse_product_review(&src).is_none(), + "missing {missing} should be skipped" + ); + } + } + + #[test] + fn out_of_range_ratings_are_rejected() { + let src = json!({ + "type": "product_review", + "_id": "product_review-1", + "user_id": 1, + "sku": "123", + "rating": 0, + "date_created": "2026-07-26 14:15:42", + }); + assert!(parse_product_review(&src).is_none()); + } + + #[test] + fn numeric_and_string_ids_both_parse() { + assert_eq!(json_as_string(&json!(12345)), Some("12345".to_string())); + assert_eq!(json_as_string(&json!("12345")), Some("12345".to_string())); + assert_eq!(json_as_string(&json!("")), None); + assert_eq!(json_as_string(&json!(null)), None); + } + + #[test] + fn parses_both_upstream_timestamp_shapes() { + assert!(parse_upstream_datetime("2026-07-26 14:15:42").is_some()); + assert!(parse_upstream_datetime("2026-07-26T14:15:42.000000Z").is_some()); + assert!(parse_upstream_datetime("last tuesday").is_none()); + } + + #[test] + fn a_missing_date_does_not_drop_the_review() { + // Ordering matters less than the rating itself — fall back to "now" rather + // than discarding a usable signal. + let src = json!({ + "type": "product_review", + "_id": "product_review-1", + "user_id": 1, + "sku": "123", + "rating": 4, + }); + assert!(parse_product_review(&src).is_some()); + } +} diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index ce205bc..4f579ee 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -18,6 +18,7 @@ mod pagination; mod password; mod password_policy; mod recommend; +mod reviews; mod routes; mod search; mod service; @@ -49,6 +50,20 @@ use crate::config::Config; use crate::openapi::ApiDoc; use crate::state::AppState; +/// The job name from `--run-job ` (or `--run-job=`), if present. +fn run_job_arg() -> Option { + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + if let Some(name) = arg.strip_prefix("--run-job=") { + return Some(name.to_string()); + } + if arg == "--run-job" { + return args.next(); + } + } + None +} + #[tokio::main] async fn main() -> anyhow::Result<()> { let _ = dotenvy::dotenv(); @@ -68,8 +83,31 @@ async fn main() -> anyhow::Result<()> { tokio::sync::Mutex::new(std::collections::HashSet::new()), ), security: std::sync::Arc::new(crate::throttle::Security::new()), + external_signals: std::sync::Arc::new(crate::reviews::SignalCache::default()), }; + // `--run-job `: run one background job to completion and exit, without + // starting the server. The scheduler and the admin trigger endpoint are the + // normal paths; this exists for one-off operational runs (a first backfill, a + // re-sync after a data fix) where neither is convenient. It bypasses the + // scheduler entirely, so it's safe to use on an instance that doesn't own + // `api_job` — but don't run a job here that's already running elsewhere. + if let Some(job_name) = run_job_arg() { + let Some(job) = jobs::lookup(&job_name) else { + let known: Vec<&str> = jobs::registry().iter().map(|j| j.name()).collect(); + anyhow::bail!("unknown job {job_name:?}; known jobs: {}", known.join(", ")); + }; + tracing::info!(job = %job_name, "running job from the command line"); + jobs::run_job(&state, job).await; + let row = jobs::get_or_create_row(&state.db, &job_name).await?; + println!("{}\n{}", row.status, row.last_output); + if !row.error_message.is_empty() { + eprintln!("{}", row.error_message); + std::process::exit(1); + } + return Ok(()); + } + // Background scheduler: runs jobs whose `interval_hours` has elapsed, // replacing Celery beat. Restart-safe (last_run persists in the DB). // Gated by ENABLE_SCHEDULER — exactly one process may own `api_job`, so diff --git a/backend-rs/src/recommend.rs b/backend-rs/src/recommend.rs index 1a8b7e2..3c37ab1 100644 --- a/backend-rs/src/recommend.rs +++ b/backend-rs/src/recommend.rs @@ -8,6 +8,27 @@ //! per request in well under a millisecond. If the dataset grows by orders of //! magnitude, move the call behind a scheduled job that precomputes per-user results //! — the math here is unchanged. +//! +//! # Two rating sources +//! +//! The community's own 1–10 ratings are the primary signal. On top of that sits +//! [`ExternalSignals`] — anonymous shop reviews ingested from reviews.io by the +//! `sync_reviews` job (~26k of them vs. our few hundred). They are *not* mixed into +//! the rating matrix, for two reasons: +//! +//! 1. They are 1–5 stars and heavily inflated (72% are 5★, store average 4.6), so +//! feeding them in raw would drag the global mean up and rank flavors by how +//! much shop traffic they get rather than how good they are. +//! 2. Most external reviewers leave one or two reviews, all 5★ — zero variance, so +//! the Pearson and adjusted-cosine maths below divide by zero and discard them +//! anyway. +//! +//! Instead they feed in two places where they're statistically sound: +//! - as an **informed Bayesian prior** in [`popularity`], replacing the flat global +//! mean, so a flavor with three community ratings is shrunk toward what a few +//! thousand buyers thought of it rather than toward the catalog average; +//! - as a **co-occurrence layer** in [`similar_flavors`], where "the same anonymous +//! buyer liked both" is meaningful even when every rating they left was 5★. use std::collections::{HashMap, HashSet}; @@ -27,6 +48,9 @@ pub enum RecSource { Cf, /// Cold-start / fallback Bayesian-popularity ranking. Popular, + /// Popularity ranking for a flavor nobody here has rated yet — the score comes + /// entirely from anonymous shop reviews. + Shop, } impl RecSource { @@ -34,6 +58,7 @@ impl RecSource { match self { RecSource::Cf => "cf", RecSource::Popular => "popular", + RecSource::Shop => "shop", } } } @@ -45,9 +70,218 @@ pub struct Recommendation { /// Number of similar users whose ratings backed this (CF), or the flavor's total /// rating count (popularity). Surfaced as the "N tasters" reason. pub contributing_neighbours: i64, + /// Anonymous shop reviews backing this flavor, if any. Zero when the external + /// feed hasn't been synced or doesn't cover the flavor. + pub external_reviews: i64, pub source: RecSource, } +// --- External (shop-review) signal ------------------------------------------- + +/// One anonymous shop review, already resolved to a flavor. +#[derive(Clone, Copy, Debug)] +pub struct ExternalInput { + /// Opaque per-reviewer key (a truncated salted hash of the upstream reviewer + /// id). Only equality matters — it exists so we can tell "one buyer rated both + /// of these" from "two unrelated buyers". + pub reviewer: u64, + pub flavor_id: i32, + /// Upstream rating on its native 1–5 star scale. + pub stars: i32, +} + +/// Aggregate stats for one flavor's shop reviews. +#[derive(Clone, Copy, Debug, Default)] +pub struct ExternalStat { + pub count: i64, + /// Mean rating, rescaled to the internal 1–10 scale. + pub mean: f64, +} + +/// Below this many shop reviews a flavor's external signal is treated as noise. +const EXT_MIN_REVIEWS: i64 = 3; +/// Pseudo-observations of prior strength contributed per shop review. Well under +/// 1: a shop review is much weaker evidence than a deliberate community rating. +const EXT_PRIOR_PER_REVIEW: f64 = 0.4; +/// Ceiling on external prior strength, in pseudo-observations. Without it a flavor +/// with 3,000 shop reviews would pin its own score and ignore the community. +const EXT_PRIOR_MAX: f64 = 12.0; +/// How much of a flavor's deviation from the external mean carries onto the +/// internal scale. 1.0 = as-is; shop ratings are compressed near the top, so this +/// deliberately does not try to stretch them out. +const EXT_SPREAD: f64 = 1.0; +/// Stars at or above this count as "liked" for co-occurrence. 4★+ is the standard +/// positive-sentiment cut for 5-star shop reviews. +const EXT_LIKE_STARS: i32 = 4; +/// Minimum shared likers before external co-occurrence similarity is trusted. +const EXT_MIN_COOC: i64 = 3; +/// Confidence ramp on shared likers: confidence is `min(shared, K)/K`. +const EXT_SIM_CONF_K: f64 = 8.0; +/// Confidence ramp on internal co-raters, same shape. +const INT_SIM_CONF_K: f64 = 5.0; +/// Cap on how much the external co-occurrence layer can weigh relative to the +/// community's own adjusted-cosine similarity, at equal confidence. +const EXT_SIM_WEIGHT: f64 = 0.6; + +/// Map an upstream 1–5 star rating onto the internal 1–10 scale, endpoints aligned +/// (1★→1.0, 3★→5.5, 5★→10.0). +fn stars_to_internal(stars: i32) -> f64 { + 1.0 + (stars.clamp(1, 5) as f64 - 1.0) * 9.0 / 4.0 +} + +/// Precomputed aggregates over the shop-review feed. Built once per request (or +/// once per cache refresh) and shared by [`recommend`] and [`similar_flavors`]. +#[derive(Clone, Debug, Default)] +pub struct ExternalSignals { + per_flavor: HashMap, + /// Mean of every shop review on the internal scale — the recentring anchor. + mean: f64, + /// flavor -> distinct reviewers who *liked* it (>= [`EXT_LIKE_STARS`]). + likers: HashMap, + /// Unordered flavor pair (low, high) -> reviewers who liked both. + cooc: HashMap<(i32, i32), i64>, + /// flavor -> the flavors it shares at least one liker with (co-occurrence + /// adjacency, so [`similar_flavors`] doesn't scan every pair). + neighbours: HashMap>, +} + +impl ExternalSignals { + /// Aggregate raw shop reviews. Duplicate (reviewer, flavor) pairs are collapsed + /// to that reviewer's mean for the flavor — the feed can carry several reviews + /// from one buyer for one product (repeat orders, per-variant reviews) and + /// counting them repeatedly would let a single buyer weight the co-occurrence + /// graph. + pub fn build(rows: &[ExternalInput]) -> Self { + // (reviewer, flavor) -> (sum of internal-scale scores, count) + let mut per_pair: HashMap<(u64, i32), (f64, i64)> = HashMap::new(); + for row in rows { + let e = per_pair + .entry((row.reviewer, row.flavor_id)) + .or_insert((0.0, 0)); + e.0 += stars_to_internal(row.stars); + e.1 += 1; + } + + let mut per_flavor: HashMap = HashMap::new(); + let mut likers: HashMap = HashMap::new(); + let mut liked_by_reviewer: HashMap> = HashMap::new(); + let mut total = 0.0; + let mut n = 0i64; + + // Same "liked" cut as EXT_LIKE_STARS, expressed on the internal scale. + let like_threshold = stars_to_internal(EXT_LIKE_STARS); + + for ((reviewer, flavor_id), (sum, count)) in &per_pair { + let score = sum / *count as f64; + let e = per_flavor.entry(*flavor_id).or_default(); + // Accumulate the sum in `mean` first, divide once at the end. + e.mean += score; + e.count += 1; + total += score; + n += 1; + if score >= like_threshold { + *likers.entry(*flavor_id).or_insert(0) += 1; + liked_by_reviewer + .entry(*reviewer) + .or_default() + .push(*flavor_id); + } + } + for stat in per_flavor.values_mut() { + if stat.count > 0 { + stat.mean /= stat.count as f64; + } + } + + let mut cooc: HashMap<(i32, i32), i64> = HashMap::new(); + let mut neighbours: HashMap> = HashMap::new(); + for liked in liked_by_reviewer.values() { + // A buyer who liked 40 products (a reviewer of whole bundles) says + // little about any specific pair, and contributes O(n²) pairs. Skip. + if liked.len() < 2 || liked.len() > MAX_BASKET { + continue; + } + for (i, &a) in liked.iter().enumerate() { + for &b in &liked[i + 1..] { + if a == b { + continue; + } + let key = if a < b { (a, b) } else { (b, a) }; + *cooc.entry(key).or_insert(0) += 1; + neighbours.entry(a).or_default().insert(b); + neighbours.entry(b).or_default().insert(a); + } + } + } + + Self { + per_flavor, + mean: if n == 0 { 0.0 } else { total / n as f64 }, + likers, + cooc, + neighbours, + } + } + + pub fn is_empty(&self) -> bool { + self.per_flavor.is_empty() + } + + /// Shop-review stats for one flavor, or `None` if it has none. + pub fn stat(&self, flavor_id: i32) -> Option { + self.per_flavor.get(&flavor_id).copied() + } + + /// Every flavor the feed covers with at least [`EXT_MIN_REVIEWS`] reviews. + fn ranked_flavors(&self) -> impl Iterator + '_ { + self.per_flavor + .iter() + .filter(|(_, s)| s.count >= EXT_MIN_REVIEWS) + .map(|(&f, _)| f) + } + + /// Cosine similarity over the binary "was liked by" vectors of two flavors, + /// plus the number of shared likers backing it. `None` below + /// [`EXT_MIN_COOC`] shared likers. + fn cooc_similarity(&self, a: i32, b: i32) -> Option<(f64, i64)> { + let key = if a < b { (a, b) } else { (b, a) }; + let shared = *self.cooc.get(&key)?; + if shared < EXT_MIN_COOC { + return None; + } + let na = *self.likers.get(&a)? as f64; + let nb = *self.likers.get(&b)? as f64; + if na <= 0.0 || nb <= 0.0 { + return None; + } + // Cosine (not raw count) so a flavor everyone buys doesn't look similar to + // everything simply by being everywhere. + Some((shared as f64 / (na * nb).sqrt(), shared)) + } +} + +/// Reviewers who liked more than this many distinct flavors are dropped from the +/// co-occurrence graph (bulk/bundle reviewers — high pair count, low signal). +const MAX_BASKET: usize = 25; + +/// The Bayesian prior to shrink a flavor's community score toward: normally the +/// catalog mean, but the shop-review consensus where we have enough of it. +/// Returns `(prior_mean, prior_strength, external_review_count)`. +fn prior_for(flavor_id: i32, internal_mean: f64, external: &ExternalSignals) -> (f64, f64, i64) { + match external.stat(flavor_id) { + Some(stat) if stat.count >= EXT_MIN_REVIEWS => { + // Recentre: what matters is how this flavor sits *relative to* other + // shop-reviewed flavors, not its absolute (inflated) star average. + let deviation = (stat.mean - external.mean) * EXT_SPREAD; + let prior_mean = (internal_mean + deviation).clamp(1.0, 10.0); + let strength = BAYES_C + (EXT_PRIOR_PER_REVIEW * stat.count as f64).min(EXT_PRIOR_MAX); + (prior_mean, strength, stat.count) + } + Some(stat) => (internal_mean, BAYES_C, stat.count), + None => (internal_mean, BAYES_C, 0), + } +} + // --- Tunables ---------------------------------------------------------------- /// Below this many of their own ratings a user is "cold" → popularity fallback. const MIN_PROFILE: usize = 5; @@ -60,8 +294,14 @@ const MIN_NEIGHBOURS: i64 = 2; /// Bayesian prior strength for the popularity fallback. const BAYES_C: f64 = 5.0; -/// Recommend up to `limit` flavors for `target`, given the full rating matrix. -pub fn recommend(target: i32, ratings: &[RatingInput], limit: usize) -> Vec { +/// Recommend up to `limit` flavors for `target`, given the full rating matrix and +/// the anonymous shop-review signal (pass `&ExternalSignals::default()` for none). +pub fn recommend( + target: i32, + ratings: &[RatingInput], + external: &ExternalSignals, + limit: usize, +) -> Vec { if limit == 0 { return Vec::new(); } @@ -80,7 +320,7 @@ pub fn recommend(target: i32, ratings: &[RatingInput], limit: usize) -> Vec = by_user @@ -146,28 +386,62 @@ pub fn recommend(target: i32, ratings: &[RatingInput], limit: usize) -> Vec = target_rated.clone(); + seen.extend(recs.iter().map(|r| r.flavor_id)); + for extra in popularity(&seen, ratings, external, limit - recs.len()) { + recs.push(extra); + } + } recs } /// Bayesian-average popularity ranking over flavors the target hasn't rated. Damps -/// flavors with few ratings toward the global mean so one 10/10 can't top the chart. +/// flavors with few ratings toward a prior so one 10/10 can't top the chart. +/// +/// The prior is the catalog mean when we know nothing else, and the shop-review +/// consensus for that specific flavor where the feed has enough of it — see +/// [`prior_for`]. That is where the external data earns its keep: a flavor with two +/// community ratings stops being shrunk toward "average catalog item" and starts +/// being shrunk toward what a few thousand buyers actually thought of it. +/// +/// Flavors with no community ratings at all are included too (ranked purely on +/// their prior, tagged [`RecSource::Shop`]) — otherwise a freshly-synced flavor +/// could never be discovered. fn popularity( target_rated: &HashSet, ratings: &[RatingInput], + external: &ExternalSignals, limit: usize, ) -> Vec { - let global_mean = mean(ratings.iter().map(|r| r.score)); + // With no community ratings at all, anchor on the external scale instead of + // `mean()`'s 0.0, which would put every prior at the bottom of the range. + let internal_mean = if ratings.is_empty() { + if external.is_empty() { + 0.0 + } else { + external.mean + } + } else { + mean(ratings.iter().map(|r| r.score)) + }; + let mut agg: HashMap = HashMap::new(); // flavor -> (sum, count) for r in ratings { if target_rated.contains(&r.flavor_id) { @@ -177,14 +451,31 @@ fn popularity( e.0 += r.score; e.1 += 1; } + // Shop-reviewed flavors nobody here has rated are candidates too. + for flavor_id in external.ranked_flavors() { + if target_rated.contains(&flavor_id) { + continue; + } + agg.entry(flavor_id).or_insert((0.0, 0)); + } let mut recs: Vec = agg .into_iter() - .map(|(flavor_id, (sum, count))| Recommendation { - flavor_id, - predicted_score: (BAYES_C * global_mean + sum) / (BAYES_C + count as f64), - contributing_neighbours: count, - source: RecSource::Popular, + .map(|(flavor_id, (sum, count))| { + let (prior_mean, prior_strength, external_reviews) = + prior_for(flavor_id, internal_mean, external); + Recommendation { + flavor_id, + predicted_score: (prior_strength * prior_mean + sum) + / (prior_strength + count as f64), + contributing_neighbours: count, + external_reviews, + source: if count == 0 { + RecSource::Shop + } else { + RecSource::Popular + }, + } }) .collect(); sort_recs(&mut recs); @@ -221,22 +512,36 @@ const MIN_CORATERS: usize = 2; #[derive(Clone, Debug)] pub struct SimilarFlavor { pub flavor_id: i32, - /// Adjusted-cosine similarity in (0, 1]. Higher = more alike. + /// Blended, confidence-discounted similarity in (0, 1]. Higher = more alike. + /// Not a raw correlation: a pair backed by little evidence scores low even if + /// the evidence it has agrees perfectly. Meant for ranking, not for display as + /// a "% match". pub similarity: f64, /// How many users rated both this flavor and the target (the confidence count). pub co_raters: i64, + /// How many anonymous shop reviewers liked both this flavor and the target. + pub external_co_reviewers: i64, } -/// Flavors most similar to `target_flavor`, by **adjusted cosine** over the user -/// rating vectors (each user's ratings mean-centred to cancel rater bias). Pure (no -/// DB) and computed on demand; at current scale this is sub-millisecond. +/// Flavors most similar to `target_flavor`, blending two independent signals: /// -/// Returns only positively-correlated flavors sharing at least [`MIN_CORATERS`] -/// raters with the target, ranked by similarity. Empty if the target has too few -/// raters to compare against. +/// - **adjusted cosine** over the community's rating vectors (each user's ratings +/// mean-centred to cancel rater bias), and +/// - **co-occurrence cosine** over anonymous shop reviewers who liked both flavors. +/// +/// Each contributes in proportion to its own confidence (how many raters back it), +/// with the external side capped at [`EXT_SIM_WEIGHT`] of the community's pull. A +/// candidate qualifies on either signal, so shop data extends coverage to flavor +/// pairs the community hasn't co-rated yet — but a *negative* community +/// correlation vetoes the pair outright, no matter what the shop data says: our own +/// tasters disagreeing is stronger evidence than strangers buying both. +/// +/// Pure (no DB) and computed on demand; at current scale this is sub-millisecond. +/// Empty if the target has too little of either signal to compare against. pub fn similar_flavors( target_flavor: i32, ratings: &[RatingInput], + external: &ExternalSignals, limit: usize, ) -> Vec { if limit == 0 { @@ -263,49 +568,90 @@ pub fn similar_flavors( .insert(r.user_id, centred); } - let Some(target_vec) = by_flavor.get(&target_flavor) else { - return Vec::new(); - }; + let empty_vec = HashMap::new(); + let target_vec = by_flavor.get(&target_flavor).unwrap_or(&empty_vec); + + // Candidates: anything the community co-rated with the target, plus anything a + // shop reviewer liked alongside it. + let mut candidates: HashSet = by_flavor.keys().copied().collect(); + if let Some(ext_neighbours) = external.neighbours.get(&target_flavor) { + candidates.extend(ext_neighbours.iter().copied()); + } + candidates.remove(&target_flavor); let mut sims: Vec = Vec::new(); - for (&f, f_vec) in &by_flavor { - if f == target_flavor { - continue; - } - // co-rated users - let (mut num, mut dt, mut df, mut co) = (0.0, 0.0, 0.0, 0i64); - for (&u, &a) in target_vec { - if let Some(&b) = f_vec.get(&u) { - num += a * b; - dt += a * a; - df += b * b; - co += 1; + for f in candidates { + // --- community side: adjusted cosine over mean-centred ratings --- + let mut internal: Option<(f64, i64)> = None; + let mut vetoed = false; + if let Some(f_vec) = by_flavor.get(&f) { + let (mut num, mut dt, mut df, mut co) = (0.0, 0.0, 0.0, 0i64); + for (&u, &a) in target_vec { + if let Some(&b) = f_vec.get(&u) { + num += a * b; + dt += a * a; + df += b * b; + co += 1; + } + } + let den = dt.sqrt() * df.sqrt(); + if (co as usize) >= MIN_CORATERS && den != 0.0 { + let sim = num / den; + if sim > 0.0 { + internal = Some((sim, co)); + } else { + // Our own tasters actively disagree about these two — that + // outranks any amount of shop co-purchase. + vetoed = true; + } } } - if (co as usize) < MIN_CORATERS { + if vetoed { continue; } - let den = dt.sqrt() * df.sqrt(); - if den == 0.0 { + + // --- shop side: co-occurrence cosine over "liked both" --- + let ext = external.cooc_similarity(target_flavor, f); + + let (int_sim, int_co) = internal.unwrap_or((0.0, 0)); + let (ext_sim, ext_co) = ext.unwrap_or((0.0, 0)); + if internal.is_none() && ext.is_none() { continue; } - let sim = num / den; - if sim <= 0.0 { - continue; // only positively-correlated flavors + + // Weight each side by its own confidence, so a pair backed by 8 community + // co-raters isn't overruled by 3 shop reviewers, and vice versa. + let int_conf = (int_co.min(INT_SIM_CONF_K as i64) as f64) / INT_SIM_CONF_K; + let ext_conf = EXT_SIM_WEIGHT * (ext_co.min(EXT_SIM_CONF_K as i64) as f64) / EXT_SIM_CONF_K; + if int_conf + ext_conf <= 0.0 { + continue; } + // Divide by the *maximum* achievable confidence, not the confidence we + // actually have. Normalising by the latter would let three shop buyers with + // a coincidentally perfect 1.0 co-occurrence outrank eight of our own + // tasters at 0.9 — thin evidence has to shrink the score toward zero, not + // just decide how the two sides are averaged. + let similarity = (int_sim * int_conf + ext_sim * ext_conf) / (1.0 + EXT_SIM_WEIGHT); + if similarity <= 0.0 { + continue; + } + sims.push(SimilarFlavor { flavor_id: f, - similarity: sim, - co_raters: co, + similarity, + co_raters: int_co, + external_co_reviewers: ext_co, }); } - // Deterministic: similarity desc, then more co-raters, then flavor id. + // Deterministic: similarity desc, then more co-raters, then more shop + // reviewers, then flavor id. sims.sort_by(|a, b| { b.similarity .partial_cmp(&a.similarity) .unwrap_or(std::cmp::Ordering::Equal) .then(b.co_raters.cmp(&a.co_raters)) + .then(b.external_co_reviewers.cmp(&a.external_co_reviewers)) .then(a.flavor_id.cmp(&b.flavor_id)) }); sims.truncate(limit); @@ -334,7 +680,7 @@ mod tests { r(3, 10, 8.0), r(3, 20, 5.0), ]; - let recs = recommend(1, &ratings, 10); + let recs = recommend(1, &ratings, &ExternalSignals::default(), 10); assert!(!recs.is_empty()); assert!(recs.iter().all(|x| x.source == RecSource::Popular)); // Flavor 10 (avg ~8.5) should outrank flavor 20 (avg ~4.5). @@ -373,7 +719,7 @@ mod tests { ratings.push(r(3, 3, 3.0)); ratings.push(r(3, 99, 1.0)); - let recs = recommend(1, &ratings, 10); + let recs = recommend(1, &ratings, &ExternalSignals::default(), 10); assert!(!recs.is_empty(), "expected CF recommendations"); assert!(recs.iter().any(|x| x.source == RecSource::Cf)); // The standout shared pick (99) should be recommended. @@ -402,7 +748,7 @@ mod tests { r(3, 6, 9.0), r(3, 7, 8.0), ]; - let recs = recommend(1, &ratings, 1); + let recs = recommend(1, &ratings, &ExternalSignals::default(), 1); assert_eq!(recs.len(), 1); // never recommend a flavor the target already rated assert!(recs.iter().all(|x| ![1, 2, 3, 4, 5].contains(&x.flavor_id))); @@ -423,7 +769,7 @@ mod tests { r(12, 2, 2.0), r(12, 3, 9.0), ]; - let sims = similar_flavors(1, &ratings, 10); + let sims = similar_flavors(1, &ratings, &ExternalSignals::default(), 10); assert!(!sims.is_empty(), "expected similar flavors"); assert_eq!(sims[0].flavor_id, 2, "flavor 2 moves with flavor 1"); assert!(sims[0].co_raters >= MIN_CORATERS as i64); @@ -435,12 +781,239 @@ mod tests { fn similar_flavors_needs_min_coraters() { // Only one shared rater between flavor 1 and 2 → below MIN_CORATERS → empty. let ratings = vec![r(10, 1, 9.0), r(10, 2, 9.0), r(11, 1, 8.0), r(12, 2, 7.0)]; - assert!(similar_flavors(1, &ratings, 10).is_empty()); + assert!(similar_flavors(1, &ratings, &ExternalSignals::default(), 10).is_empty()); } #[test] fn similar_flavors_unknown_target_is_empty() { let ratings = vec![r(10, 1, 9.0), r(11, 1, 8.0)]; - assert!(similar_flavors(999, &ratings, 10).is_empty()); + assert!(similar_flavors(999, &ratings, &ExternalSignals::default(), 10).is_empty()); + } + + // --- external (shop-review) signal --------------------------------------- + + fn e(reviewer: u64, flavor_id: i32, stars: i32) -> ExternalInput { + ExternalInput { + reviewer, + flavor_id, + stars, + } + } + + /// `n` reviewers each leaving one `stars` review for `flavor`, ids offset so + /// separate calls don't collide. + fn bulk(flavor: i32, stars: i32, n: u64, offset: u64) -> Vec { + (0..n).map(|i| e(offset + i, flavor, stars)).collect() + } + + #[test] + fn stars_map_onto_the_internal_scale_endpoints() { + assert_eq!(stars_to_internal(1), 1.0); + assert_eq!(stars_to_internal(3), 5.5); + assert_eq!(stars_to_internal(5), 10.0); + // Out-of-range input is clamped, never extrapolated. + assert_eq!(stars_to_internal(0), 1.0); + assert_eq!(stars_to_internal(9), 10.0); + } + + #[test] + fn external_prior_breaks_ties_between_thinly_rated_flavors() { + // Two flavors, one community rating each, identical score → the community + // data alone cannot separate them. The shop feed can: flavor 20 is loved, + // flavor 10 is mediocre. + let ratings = vec![r(2, 10, 8.0), r(3, 20, 8.0)]; + let mut rows = bulk(10, 3, 40, 0); + rows.extend(bulk(20, 5, 40, 1_000)); + let external = ExternalSignals::build(&rows); + + let recs = recommend(1, &ratings, &external, 10); + assert_eq!( + recs[0].flavor_id, 20, + "shop-loved flavor should win the tie" + ); + assert!(recs[0].predicted_score > recs[1].predicted_score); + assert_eq!(recs[0].external_reviews, 40); + } + + #[test] + fn external_prior_cannot_overturn_a_solid_community_verdict() { + // The shop adores flavor 10; twelve of our own tasters think it's poor. + // The prior is capped, so the community verdict must still win. + let mut ratings: Vec = (2..14).map(|u| r(u, 10, 2.0)).collect(); + ratings.extend((2..14).map(|u| r(u, 20, 8.0))); + let mut rows = bulk(10, 5, 3_000, 0); + rows.extend(bulk(20, 3, 5, 10_000)); + let external = ExternalSignals::build(&rows); + + let recs = recommend(1, &ratings, &external, 10); + assert_eq!( + recs[0].flavor_id, 20, + "12 community ratings must outweigh the shop prior" + ); + } + + #[test] + fn unrated_flavor_surfaces_from_shop_reviews_alone() { + // Flavor 99 has no community ratings at all — without external data it + // could never be recommended. + let ratings = vec![r(2, 10, 6.0), r(3, 10, 6.0)]; + let external = ExternalSignals::build(&bulk(99, 5, 50, 0)); + + let recs = recommend(1, &ratings, &external, 10); + let shop = recs + .iter() + .find(|x| x.flavor_id == 99) + .expect("shop-only flavor should be a candidate"); + assert_eq!(shop.source, RecSource::Shop); + assert_eq!(shop.contributing_neighbours, 0); + assert_eq!(shop.external_reviews, 50); + } + + #[test] + fn external_reviews_below_the_floor_are_ignored() { + // Two reviews is under EXT_MIN_REVIEWS → no prior, no candidacy. + let ratings = vec![r(2, 10, 6.0), r(3, 10, 6.0)]; + let external = ExternalSignals::build(&bulk(99, 5, 2, 0)); + let recs = recommend(1, &ratings, &external, 10); + assert!(recs.iter().all(|x| x.flavor_id != 99)); + } + + #[test] + fn repeat_reviews_from_one_buyer_count_once() { + // The feed carries several reviews from one buyer for one product (repeat + // orders, per-variant reviews); they must not inflate the count. + let rows = vec![e(1, 10, 5), e(1, 10, 5), e(1, 10, 5), e(2, 10, 5)]; + let external = ExternalSignals::build(&rows); + assert_eq!(external.stat(10).unwrap().count, 2); + } + + #[test] + fn shop_co_occurrence_finds_pairs_the_community_has_not_co_rated() { + // Nobody here has rated flavor 2 alongside flavor 1, so adjusted cosine has + // nothing to say. Five shop buyers liked both → similarity from the shop + // layer alone. + let ratings = vec![r(10, 1, 9.0), r(11, 1, 8.0)]; + let mut rows: Vec = Vec::new(); + for i in 0..5 { + rows.push(e(i, 1, 5)); + rows.push(e(i, 2, 5)); + } + // A flavor liked by unrelated buyers — shares no reviewer with the target. + rows.extend(bulk(3, 5, 5, 100)); + let external = ExternalSignals::build(&rows); + + let sims = similar_flavors(1, &ratings, &external, 10); + assert_eq!(sims.len(), 1, "only the co-liked flavor qualifies"); + assert_eq!(sims[0].flavor_id, 2); + assert_eq!(sims[0].co_raters, 0); + assert_eq!(sims[0].external_co_reviewers, 5); + } + + #[test] + fn shop_co_occurrence_needs_min_shared_likers() { + // Two shared likers is under EXT_MIN_COOC → not enough to claim similarity. + let ratings = vec![r(10, 1, 9.0)]; + let mut rows: Vec = Vec::new(); + for i in 0..2 { + rows.push(e(i, 1, 5)); + rows.push(e(i, 2, 5)); + } + let external = ExternalSignals::build(&rows); + assert!(similar_flavors(1, &ratings, &external, 10).is_empty()); + } + + #[test] + fn low_star_shop_reviews_do_not_create_co_occurrence() { + // "Bought both" is not "liked both": 2★ pairs must not link the flavors. + let ratings = vec![r(10, 1, 9.0)]; + let mut rows: Vec = Vec::new(); + for i in 0..6 { + rows.push(e(i, 1, 2)); + rows.push(e(i, 2, 2)); + } + let external = ExternalSignals::build(&rows); + assert!(similar_flavors(1, &ratings, &external, 10).is_empty()); + } + + #[test] + fn community_disagreement_vetoes_shop_co_occurrence() { + // Our tasters are anti-correlated on flavors 1 and 3 (see + // `similar_flavors_finds_co_liked`), yet plenty of shop buyers liked both. + // The community veto must hold. + let ratings = vec![ + r(10, 1, 9.0), + r(10, 3, 2.0), + r(11, 1, 8.0), + r(11, 3, 3.0), + r(12, 1, 3.0), + r(12, 3, 9.0), + ]; + let mut rows: Vec = Vec::new(); + for i in 0..20 { + rows.push(e(i, 1, 5)); + rows.push(e(i, 3, 5)); + } + let external = ExternalSignals::build(&rows); + let sims = similar_flavors(1, &ratings, &external, 10); + assert!(sims.iter().all(|s| s.flavor_id != 3)); + } + + #[test] + fn bulk_reviewers_are_excluded_from_the_co_occurrence_graph() { + // One buyer who liked 30 products says nothing about any specific pair. + let rows: Vec = (1..=30).map(|f| e(7, f, 5)).collect(); + let external = ExternalSignals::build(&rows); + assert!(external.cooc_similarity(1, 2).is_none()); + } + + #[test] + fn community_similarity_still_outranks_a_shop_only_pair() { + // Flavor 2 is backed by the community; flavor 4 only by shop co-purchase, + // where four buyers liked both and nothing else — a perfect 1.0 cosine. + // The community-backed pair must still rank first. + // + // (Each user rates a third flavor: with only two ratings each, mean-centring + // makes every user trivially anti-correlated and adjusted cosine degenerates + // to -1.) + let ratings = vec![ + r(10, 1, 9.0), + r(10, 2, 9.0), + r(10, 5, 3.0), + r(11, 1, 8.0), + r(11, 2, 8.0), + r(11, 5, 2.0), + r(12, 1, 3.0), + r(12, 2, 2.0), + r(12, 5, 9.0), + ]; + let mut rows: Vec = Vec::new(); + for i in 0..4 { + rows.push(e(i, 1, 5)); + rows.push(e(i, 4, 5)); + } + let external = ExternalSignals::build(&rows); + let sims = similar_flavors(1, &ratings, &external, 10); + assert_eq!(sims[0].flavor_id, 2); + assert!(sims.iter().any(|s| s.flavor_id == 4)); + } + + #[test] + fn empty_external_signal_changes_nothing() { + let ratings = vec![ + r(1, 100, 9.0), + r(2, 10, 9.0), + r(2, 20, 4.0), + r(3, 10, 8.0), + r(3, 20, 5.0), + ]; + let none = ExternalSignals::default(); + assert!(none.is_empty()); + let a = recommend(1, &ratings, &none, 10); + let b = recommend(1, &ratings, &ExternalSignals::build(&[]), 10); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(b.iter()) { + assert_eq!(x.flavor_id, y.flavor_id); + assert_eq!(x.predicted_score, y.predicted_score); + } } } diff --git a/backend-rs/src/reviews.rs b/backend-rs/src/reviews.rs new file mode 100644 index 0000000..06326fd --- /dev/null +++ b/backend-rs/src/reviews.rs @@ -0,0 +1,161 @@ +//! Glue between the `external_review` table and the pure scoring maths in +//! [`crate::recommend`]: pseudonymising reviewer ids on the way in, and loading + +//! caching the aggregated signal on the way out. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sea_orm::EntityTrait; +use sha2::{Digest, Sha256}; +use tokio::sync::RwLock; + +use crate::entities::prelude::ExternalReview; +use crate::error::ApiResult; +use crate::recommend::{ExternalInput, ExternalSignals}; +use crate::state::AppState; + +/// Source slug for reviews.io rows in `external_review.source`. +pub const SOURCE_REVIEWS_IO: &str = "reviews_io"; + +/// How long an aggregated snapshot is served before being rebuilt. The feed only +/// changes when `sync_reviews` runs (hours apart), so this is about bounding +/// staleness after a sync, not freshness per se — the job invalidates the cache +/// directly when it finishes. +const CACHE_TTL: Duration = Duration::from_secs(10 * 60); + +/// One-way, salted identifier for an upstream reviewer. +/// +/// We never store the upstream reviewer id itself — only enough to recognise that +/// two reviews came from the same anonymous buyer. Truncated to 32 hex chars (128 +/// bits), far beyond collision range for a feed of this size. +/// +/// The salt comes from `REVIEWS_HASH_SALT` (falling back to `SECRET_KEY`). Rotating +/// it re-pseudonymises *new* rows only, so existing rows would no longer join up +/// with them — truncate `external_review` and re-sync if you ever change it. +pub fn reviewer_key(salt: &str, source: &str, upstream_id: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(salt.as_bytes()); + hasher.update(b"\0"); + hasher.update(source.as_bytes()); + hasher.update(b"\0"); + hasher.update(upstream_id.as_bytes()); + let digest = hasher.finalize(); + let mut out = String::with_capacity(32); + for b in digest.iter().take(16) { + out.push_str(&format!("{b:02x}")); + } + out +} + +/// Collapse a stored `reviewer_key` into the `u64` bucket the scoring maths uses. +/// Only equality matters there, and 64 bits keeps collisions negligible. +fn reviewer_bucket(key: &str) -> u64 { + let hex: String = key.chars().take(16).collect(); + u64::from_str_radix(&hex, 16).unwrap_or_else(|_| { + // Non-hex keys shouldn't exist, but a stable fallback beats collapsing + // every malformed row onto one bucket. + let digest = Sha256::digest(key.as_bytes()); + u64::from_be_bytes(digest[..8].try_into().unwrap_or([0; 8])) + }) +} + +#[derive(Clone)] +struct Snapshot { + built_at: Instant, + signals: Arc, +} + +/// Process-local cache of the aggregated shop-review signal. +/// +/// Rebuilding scans the whole `external_review` table (tens of thousands of rows), +/// which is far too much to redo on every recommendation request. Per-process and +/// eventually consistent by design: a stale snapshot only ever means recommendations +/// lag a sync by up to [`CACHE_TTL`]. +#[derive(Default)] +pub struct SignalCache { + inner: RwLock>, +} + +impl SignalCache { + /// Drop the cached snapshot so the next read rebuilds. Called by `sync_reviews` + /// after it ingests new rows. + pub async fn invalidate(&self) { + *self.inner.write().await = None; + } +} + +/// The current aggregated shop-review signal, rebuilding it if the cache is cold or +/// stale. +pub async fn signals(state: &AppState) -> ApiResult> { + if let Some(snap) = state.external_signals.inner.read().await.as_ref() { + if snap.built_at.elapsed() < CACHE_TTL { + return Ok(snap.signals.clone()); + } + } + + let rows = ExternalReview::find().all(&state.db).await?; + let inputs: Vec = rows + .iter() + .map(|r| ExternalInput { + reviewer: reviewer_bucket(&r.reviewer_key), + flavor_id: r.flavor_id, + stars: r.rating, + }) + .collect(); + let signals = Arc::new(ExternalSignals::build(&inputs)); + + // A concurrent request may have rebuilt it meanwhile; last writer wins, and + // both snapshots are equally valid. + *state.external_signals.inner.write().await = Some(Snapshot { + built_at: Instant::now(), + signals: signals.clone(), + }); + tracing::debug!(reviews = rows.len(), "rebuilt external review signal"); + Ok(signals) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reviewer_key_is_stable_and_salted() { + let a = reviewer_key("salt-a", SOURCE_REVIEWS_IO, "12345"); + assert_eq!(a, reviewer_key("salt-a", SOURCE_REVIEWS_IO, "12345")); + assert_ne!(a, reviewer_key("salt-b", SOURCE_REVIEWS_IO, "12345")); + assert_ne!(a, reviewer_key("salt-a", SOURCE_REVIEWS_IO, "12346")); + assert_eq!(a.len(), 32); + assert!(a.chars().all(|c| c.is_ascii_hexdigit())); + // The upstream id must not be recoverable from (or visible in) the key. + assert!(!a.contains("12345")); + } + + #[test] + fn reviewer_key_is_namespaced_by_source() { + assert_ne!( + reviewer_key("s", "reviews_io", "1"), + reviewer_key("s", "other", "1") + ); + } + + #[test] + fn reviewer_bucket_separates_distinct_reviewers() { + let a = reviewer_bucket(&reviewer_key("s", SOURCE_REVIEWS_IO, "1")); + let b = reviewer_bucket(&reviewer_key("s", SOURCE_REVIEWS_IO, "2")); + assert_ne!(a, b); + // Same key in, same bucket out. + assert_eq!( + a, + reviewer_bucket(&reviewer_key("s", SOURCE_REVIEWS_IO, "1")) + ); + } + + #[test] + fn reviewer_bucket_tolerates_a_malformed_key() { + // Doesn't panic, and still separates two different malformed keys. + assert_ne!( + reviewer_bucket("not-hex-at-all"), + reviewer_bucket("also-not") + ); + } +} diff --git a/backend-rs/src/routes/flavors.rs b/backend-rs/src/routes/flavors.rs index b70e04e..25f620e 100644 --- a/backend-rs/src/routes/flavors.rs +++ b/backend-rs/src/routes/flavors.rs @@ -164,7 +164,11 @@ async fn similar( }) .collect(); - let sims = similar_flavors(id, &inputs, SIMILAR_LIMIT); + // Anonymous shop reviews widen the co-rating graph well past what our own + // tasters have covered (cached; see `crate::reviews`). + let external = crate::reviews::signals(&state).await?; + + let sims = similar_flavors(id, &inputs, &external, SIMILAR_LIMIT); if sims.is_empty() { return Ok(Json(Vec::new())); } @@ -192,6 +196,7 @@ async fn similar( is_available: f.is_available, similarity: s.similarity, co_raters: s.co_raters, + external_co_reviewers: s.external_co_reviewers, }) }) .collect(); @@ -268,7 +273,9 @@ async fn popular( }) .collect(); - let recs = recommend(POPULAR_SENTINEL, &inputs, POPULAR_LIMIT); + let external = crate::reviews::signals(&state).await?; + + let recs = recommend(POPULAR_SENTINEL, &inputs, &external, POPULAR_LIMIT); if recs.is_empty() { return Ok(Json(Vec::new())); } @@ -297,6 +304,7 @@ async fn popular( is_available: f.is_available, predicted_score: rec.predicted_score, contributing_neighbours: rec.contributing_neighbours, + external_reviews: rec.external_reviews, reason: rec.source.as_str().to_string(), }) }) diff --git a/backend-rs/src/routes/users.rs b/backend-rs/src/routes/users.rs index cf0d8c8..2bdb9fd 100644 --- a/backend-rs/src/routes/users.rs +++ b/backend-rs/src/routes/users.rs @@ -1467,7 +1467,11 @@ async fn recommendations( }) .collect(); - let recs = recommend(uid, &inputs, RECOMMENDATION_LIMIT); + // Anonymous shop reviews back the prior under the community's own ratings + // (cached; see `crate::reviews`). + let external = crate::reviews::signals(&state).await?; + + let recs = recommend(uid, &inputs, &external, RECOMMENDATION_LIMIT); if recs.is_empty() { return Ok(Json(Vec::new())); } @@ -1497,6 +1501,7 @@ async fn recommendations( is_available: f.is_available, predicted_score: rec.predicted_score, contributing_neighbours: rec.contributing_neighbours, + external_reviews: rec.external_reviews, reason: rec.source.as_str().to_string(), }) }) diff --git a/backend-rs/src/state.rs b/backend-rs/src/state.rs index 82cfae4..5033167 100644 --- a/backend-rs/src/state.rs +++ b/backend-rs/src/state.rs @@ -5,6 +5,7 @@ use sea_orm::DatabaseConnection; use tokio::sync::Mutex; use crate::config::Config; +use crate::reviews::SignalCache; use crate::throttle::Security; #[derive(Clone)] @@ -18,4 +19,7 @@ pub struct AppState { /// Rate limiting + confirmation-code TTL/attempt state (replaces /// `django_ratelimit`). Shared across handlers via `Arc`. pub security: Arc, + /// Cached aggregate of the ingested shop reviews that back the recommender. + /// Too expensive to rebuild per request; see [`crate::reviews::signals`]. + pub external_signals: Arc, } diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index a6734c4..70850ad 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -59,8 +59,10 @@ export interface Recommendation { predicted_score: number; /** "N tasters like you" count (cf) or rating count (popular). */ contributing_neighbours: number; - /** "cf" | "popular" — frontend picks the reason copy. */ - reason: 'cf' | 'popular'; + /** Anonymous shop reviews (reviews.io) backing this flavor; 0 if none. */ + external_reviews: number; + /** "cf" | "popular" | "shop" (no community ratings yet — shop reviews only). */ + reason: 'cf' | 'popular' | 'shop'; } export interface DashboardData { @@ -81,8 +83,10 @@ export interface SimilarFlavor { average_rating: number | null; is_legacy: boolean; is_available: boolean; - /** Adjusted-cosine similarity to the target flavor, in (0, 1]. */ + /** Blended, confidence-discounted similarity. Ranking only — not a "% match". */ similarity: number; /** Users who rated both this flavor and the target. */ co_raters: number; + /** Anonymous shop reviewers who liked both this flavor and the target. */ + external_co_reviewers: number; } diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index e9a76f5..79ba196 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -40,6 +40,7 @@ const resources = { recHeading: 'Tasters like you loved', recReasonCf: '{{count}} tasters like you', recReasonPopular: 'Popular pick', + recReasonShop: '{{count}} shop buyers', predictedForYou: 'Predicted for you', myRatings: 'My Ratings', missing: 'Missing', @@ -179,6 +180,9 @@ const resources = { similarHeading: 'People who liked this also liked', similarCoRaters: '{{count}} rated both', similarTooltip: 'Tasters who rated both this flavor and the one above', + similarShopCoReviewers: '{{count}} shop buyers liked both', + similarShopTooltip: + 'Nobody here has rated both yet — this pairing comes from anonymous shop reviews', rateThisFlavor: 'Rate this flavor', scoreLabel: 'Your score, 1 to 10', submitReview: 'Submit Review', @@ -603,6 +607,7 @@ const resources = { recHeading: 'Tester wie du mochten', recReasonCf: '{{count}} Tester wie du', recReasonPopular: 'Beliebte Wahl', + recReasonShop: '{{count}} Shop-Käufer', predictedForYou: 'Für dich vorhergesagt', myRatings: 'Meine Bewertungen', missing: 'Fehlend', @@ -742,6 +747,9 @@ const resources = { similarHeading: 'Wer das mochte, mochte auch', similarCoRaters: '{{count}} bewerteten beide', similarTooltip: 'Tester, die diese Sorte und die obige bewertet haben', + similarShopCoReviewers: '{{count}} Shop-Käufer mochten beide', + similarShopTooltip: + 'Hier hat noch niemand beide bewertet — diese Paarung stammt aus anonymen Shop-Bewertungen', rateThisFlavor: 'Diese Sorte bewerten', scoreLabel: 'Deine Bewertung, 1 bis 10', submitReview: 'Bewertung abschicken', diff --git a/frontend/src/pages/dashboard/RecommendationCard.tsx b/frontend/src/pages/dashboard/RecommendationCard.tsx index a7de5ed..7a714f1 100644 --- a/frontend/src/pages/dashboard/RecommendationCard.tsx +++ b/frontend/src/pages/dashboard/RecommendationCard.tsx @@ -15,7 +15,9 @@ export interface RecommendationCardProps { /** * A single recommended flavor. The reason chip is **count-only** (no neighbour - * names, per product decision): CF → "N tasters like you", popularity → "Popular". + * names, per product decision): CF → "N tasters like you", popularity → + * "Popular", shop → "N shop buyers" for flavors nobody here has rated yet, whose + * score comes from the anonymous reviews.io feed alone. */ const RecommendationCard: React.FC = ({ rec }) => { const { t } = useTranslation(); @@ -23,7 +25,9 @@ const RecommendationCard: React.FC = ({ rec }) => { const reasonLabel = rec.reason === 'cf' ? t('dashboard.recReasonCf', { count: rec.contributing_neighbours }) - : t('dashboard.recReasonPopular'); + : rec.reason === 'shop' + ? t('dashboard.recReasonShop', { count: rec.external_reviews }) + : t('dashboard.recReasonPopular'); return ( = ({ item }) => { {t(`categories.${item.category_slug}`, { defaultValue: item.category_name })} - + {/* Pairs the community hasn't co-rated yet can still surface, backed by + anonymous shop reviewers — don't claim "0 rated both" for those. */} + 0 + ? 'flavorDetail.similarTooltip' + : 'flavorDetail.similarShopTooltip', + )} + > } - label={t('flavorDetail.similarCoRaters', { count: item.co_raters })} - color="primary" + label={ + item.co_raters > 0 + ? t('flavorDetail.similarCoRaters', { count: item.co_raters }) + : t('flavorDetail.similarShopCoReviewers', { + count: item.external_co_reviewers, + }) + } + color={item.co_raters > 0 ? 'primary' : 'default'} variant="outlined" sx={{ fontWeight: 700, maxWidth: '100%' }} /> diff --git a/scripts/install-hooks.sh b/scripts/install-hooks.sh old mode 100644 new mode 100755 diff --git a/scripts/pre-push.sh b/scripts/pre-push.sh index f8ee16f..e633ea6 100755 --- a/scripts/pre-push.sh +++ b/scripts/pre-push.sh @@ -11,7 +11,7 @@ echo "========================================" echo "" echo "Step 1: Checking Frontend Formatting..." cd "$ROOT_DIR/frontend" -if ! npm run format:check; then +if ! bun run format:check; then echo "" echo "❌ ERROR: Frontend formatting issues found. Run 'npm run format' in frontend/." exit 1 @@ -19,7 +19,7 @@ fi echo "" echo "Step 2: Checking Frontend Build..." -if ! npm run build; then +if ! bun run build; then echo "" echo "❌ ERROR: Frontend build failed." echo " Please fix build errors before pushing."