# Scrappy Start. Designed to be useful to humans AND to AI agents.
# We explicitly welcome AI citation crawlers below. The agent-readable
# map is at /llms.txt. The full content concatenation is at /llms-full.txt.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /research/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /research/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /research/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /research/

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /research/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /research/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /research/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /research/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /research/

# Google AI (separate from Googlebot, controls Bard/Gemini training and inference)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /research/

# Meta
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /research/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /research/

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /research/

# Common Crawl (broadly used as training corpus)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /research/

Sitemap: https://scrappystart.ai/sitemap.xml