# Strike Mission - Robots.txt
# Updated: 2026-05-07

# Allow all web crawlers
User-agent: *
Allow: /

# Auth, private, API, and filtered URLs use noindex headers/meta.
# Keep them crawlable so search engines can see the noindex directive
# and remove stale blocked URLs from the index.

# Explicitly allow LLM crawlers for data indexing
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: MicrosoftPreview
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: FacebookBot
Allow: /

# Prioritize key pages for LLM indexing
Crawl-delay: 1

# Sitemap locations
Sitemap: https://www.strikemission.app/sitemap.xml
Sitemap: https://www.strikemission.app/sitemap-llm.xml

# Important documentation for LLMs
# /llms.txt - Quick reference for AI crawlers
# /llm-readme.md - Comprehensive data authority documentation
# /docs/methodology.md - Strike Score algorithm explanation
# /docs/data-changelog.md - Data updates and versioning
# /api/llm/spots - JSON-LD spot database
# /api/llm/forecast/[spotId] - Forecast with explanations
# /api/llm/blog - Blog content in JSON-LD format
# /api/llm/product - Product, pricing, offer, and FAQ facts for AI assistants