updated generic adapter

This commit is contained in:
2026-06-17 17:16:04 +02:00
parent e38f0a3e22
commit 148e441425
+20 -1
View File
@@ -52,7 +52,26 @@ pub async fn fetch(
) -> anyhow::Result<Option<FetchedProduct>> { ) -> anyhow::Result<Option<FetchedProduct>> {
let resp = client.get(raw_url).send().await?; let resp = client.get(raw_url).send().await?;
if !resp.status().is_success() { if !resp.status().is_success() {
anyhow::bail!("product page returned HTTP {}", resp.status()); let status = resp.status();
// Cloudflare/Akamai bot managers answer 403/503 with an active
// challenge a plain HTTP client can't solve (needs a real browser).
// Name it so the failure is actionable, not a mystery "403".
let challenged = resp
.headers()
.get("cf-mitigated")
.is_some_and(|v| v.to_str().map(|s| s.contains("challenge")).unwrap_or(false))
|| matches!(status.as_u16(), 403 | 503)
&& resp
.headers()
.get("server")
.and_then(|v| v.to_str().ok())
.is_some_and(|s| s.eq_ignore_ascii_case("cloudflare"));
if challenged {
anyhow::bail!(
"this shop is behind anti-bot protection (Cloudflare) and can't be price-checked automatically"
);
}
anyhow::bail!("product page returned HTTP {status}");
} }
let body = resp.text().await?; let body = resp.text().await?;