# AgentField — AI-friendly robots.txt
# We welcome AI crawlers for citation and discovery.

User-agent: *
Allow: /
Disallow: /api/
# Read-only documentation API — safe and useful for any crawler or agent.
Allow: /api/v1/
Allow: /api/agent/
Disallow: /_next/
# Release binaries — nothing to index, and they are large.
Disallow: /downloads/

# AI Search Bots — explicitly allowed
User-agent: GPTBot
Allow: /
Allow: /api/v1/
Allow: /api/agent/
Allow: /agents.md
Allow: /pricing.md
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /docs-ai.json
Allow: /openapi.json
Allow: /.well-known/ai-plugin.json

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Block training-only crawlers (not search/citation bots)
User-agent: CCBot
Disallow: /

User-agent: img2dataset
Disallow: /

Sitemap: https://agentfield.ai/sitemap.xml

# Documentation index for AI agents
AI-Docs: /llms.txt
AI-Instructions: /agents.md
AI-Pricing: /pricing.md
AI-Manifest: /docs-ai.json
AI-Markdown: /llm/docs/<slug>
AI-OpenAPI: /openapi.json
# Any page also serves markdown via `Accept: text/markdown` content negotiation.

# LLM-readable site description
# See https://agentfield.ai/llms.txt