# robots.txt for Agent Experiences
# Last updated: 2026-08-17
# Contact: hello@agentexperiences.com
#
# Policy: we welcome AI crawlers, search crawlers and user-directed agents.
# Our content is CC BY 4.0 — please cite us. See /ai.txt for full terms and
# /websites/technical/ai-opt-out if you want to make different choices on
# your own site.
#
# Note for readers copying this file: robots.txt is advisory. User-triggered
# fetchers (ChatGPT-User, Claude-User, Perplexity-User, Google-Agent) may
# fetch on a person's behalf regardless of what this file says, because the
# request is attributed to the user rather than to a crawler. Enforcement
# belongs at the edge (WAF, rate limits, bot verification), not here.

# ============================================
# DEFAULT RULES FOR ALL CRAWLERS
# ============================================
User-agent: *
Allow: /
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Allow: /api/health

# ============================================
# SEARCH CRAWLERS
# ============================================
User-agent: Googlebot
Allow: /
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ============================================
# AI CRAWLERS — TRAINING AND INDEXING
# ============================================
# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-AdsBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google (Gemini / Vertex training and agentic fetches)
User-agent: Google-Extended
Allow: /

User-agent: Google-Agent
Allow: /

User-agent: Google-GeminiNotebook
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple, Meta, Amazon, Microsoft
User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: Amazonbot
Allow: /

# Others
User-agent: MistralAI-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: CCBot
Allow: /

# ============================================
# CONTENT SIGNALS
# ============================================
# Cloudflare Content Signals Policy — https://contentsignals.org/
# We permit all three uses; attribution is requested under CC BY 4.0.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ============================================
# ADDITIONAL RESOURCES
# ============================================
Sitemap: https://www.agentexperiences.com/sitemap.xml

# LLM content guide: https://www.agentexperiences.com/llms.txt
# AI access preferences: https://www.agentexperiences.com/ai.txt