# Soljo: content is meant to be found and cited, by search engines and by AI
# answer engines (ChatGPT, Gemini, Claude, Perplexity, Copilot, Apple). Blocked:
# SEO scrapers and a few crawlers without citation value. Personal reports (/report/) and /admin/ are
# never crawled. Machine-readable summaries: /llms.txt, /llms-full.txt

# Search engines
User-agent: Googlebot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Bingbot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: DuckDuckBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Slurp
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Applebot
Allow: /
Disallow: /report/
Disallow: /admin/

# AI search and answer engines (live citations)
User-agent: OAI-SearchBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: ChatGPT-User
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: PerplexityBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Perplexity-User
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Claude-SearchBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Claude-User
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: DuckAssistBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: MistralAI-User
Allow: /
Disallow: /report/
Disallow: /admin/

# AI model crawlers: allowed, so the models know Soljo and its sources
# (Google-Extended governs Gemini, Applebot-Extended Apple Intelligence)
User-agent: GPTBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Google-Extended
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: ClaudeBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: anthropic-ai
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Applebot-Extended
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: CCBot
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: cohere-ai
Allow: /
Disallow: /report/
Disallow: /admin/

User-agent: Amazonbot
Allow: /
Disallow: /report/
Disallow: /admin/

# Crawlers without a citation benefit: blocked
User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# SEO scrapers: blocked
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Default
User-agent: *
Allow: /
Disallow: /report/
Disallow: /admin/

Sitemap: https://soljo.ch/sitemap-ch.xml