# robots.txt for Spork (www.sporkai.co)
# Policy: OPEN. Full crawl access for search, answer engines, user-triggered
# fetchers, AND training crawlers. Named Allow blocks per verified agent.
# No Crawl-delay: full-speed crawling is intended.
# Crawler tokens verified against vendor documentation on 2026-08-14.
# Google-Extended / Applebot-Extended are intentionally omitted (control
# tokens, not crawlers). anthropic-ai / Claude-Web are excluded (deprecated).
# Generated at build time from scripts/seo/crawlers.js — do not edit by hand.

# ── Search engines ──
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# ── Answer engines / retrieval indexes ──
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: MistralAI-Index
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# ── User-triggered fetchers (vendor robots.txt policy diverges here) ──
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# ── AI training crawlers (allowed under open policy) ──
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: Diffbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

User-agent: MistralAI-Training
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# UNVERIFIED: no vendor robots.txt documentation for the token below.
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

# Default for all other agents — open, with the shared exclusions.
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /draft/
Disallow: /preview/
Disallow: /search
Disallow: /*?*

Sitemap: https://www.sporkai.co/sitemap.xml