mirror of
https://github.com/narrrl/holy-flavor-rater.git
synced 2026-11-04 17:07:01 +01:00
20 lines
313 B
Plaintext
20 lines
313 B
Plaintext
User-agent: *
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
Disallow: /settings/
|
|
Disallow: /dashboard/
|
|
|
|
# Block AI scrapers specifically
|
|
User-agent: GPTBot
|
|
Disallow: /
|
|
|
|
User-agent: CCBot
|
|
Disallow: /
|
|
|
|
# Allow Google Search to index the catalog
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
Allow: /flavor/
|
|
Allow: /category/
|
|
Allow: /profile/
|