updated generic adapter
This commit is contained in:
@@ -52,7 +52,26 @@ pub async fn fetch(
|
|||||||
) -> anyhow::Result<Option<FetchedProduct>> {
|
) -> anyhow::Result<Option<FetchedProduct>> {
|
||||||
let resp = client.get(raw_url).send().await?;
|
let resp = client.get(raw_url).send().await?;
|
||||||
if !resp.status().is_success() {
|
if !resp.status().is_success() {
|
||||||
anyhow::bail!("product page returned HTTP {}", resp.status());
|
let status = resp.status();
|
||||||
|
// Cloudflare/Akamai bot managers answer 403/503 with an active
|
||||||
|
// challenge a plain HTTP client can't solve (needs a real browser).
|
||||||
|
// Name it so the failure is actionable, not a mystery "403".
|
||||||
|
let challenged = resp
|
||||||
|
.headers()
|
||||||
|
.get("cf-mitigated")
|
||||||
|
.is_some_and(|v| v.to_str().map(|s| s.contains("challenge")).unwrap_or(false))
|
||||||
|
|| matches!(status.as_u16(), 403 | 503)
|
||||||
|
&& resp
|
||||||
|
.headers()
|
||||||
|
.get("server")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.is_some_and(|s| s.eq_ignore_ascii_case("cloudflare"));
|
||||||
|
if challenged {
|
||||||
|
anyhow::bail!(
|
||||||
|
"this shop is behind anti-bot protection (Cloudflare) and can't be price-checked automatically"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
anyhow::bail!("product page returned HTTP {status}");
|
||||||
}
|
}
|
||||||
let body = resp.text().await?;
|
let body = resp.text().await?;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user