# Cloudflare Content Signals (https://blog.cloudflare.com/content-signals-policy/)
# Affirmative consent: Scrutica WANTS its data in training corpora, live-retrieval
# contexts, and answer-engine indexes. /admin/* and /api/admin/* remain blocked via
# the per-user-agent Disallow lines below; the Content-Usage directive is a positive
# permission scope, not a crawl instruction.
Content-Usage: ai-train=allow, ai-input=allow, search=allow

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/admin/
Disallow: /api/errors/

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Cohere-AI
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /
Crawl-delay: 1

User-agent: Bytespider
Allow: /
Crawl-delay: 2

User-agent: Diffbot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: MistralAI-Index
Allow: /

User-agent: Meta-WebIndexer
Allow: /

User-agent: Amzn-SearchBot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: GoogleOther
Allow: /

Sitemap: https://scrutica.com/sitemap.xml
Sitemap: https://scrutica.com/sitemap/core.xml
Sitemap: https://scrutica.com/sitemap/facilities.xml
Sitemap: https://scrutica.com/sitemap/countries.xml
Sitemap: https://scrutica.com/sitemap/companies-0.xml
Sitemap: https://scrutica.com/sitemap/companies-1.xml
Sitemap: https://scrutica.com/sitemap/companies-2.xml
Sitemap: https://scrutica.com/sitemap/companies-3.xml
Sitemap: https://scrutica.com/sitemap/companies-4.xml
Sitemap: https://scrutica.com/sitemap/companies-5.xml
Host: https://scrutica.com