# robots.txt for ai.killerxpress.com, the machine readable files for https://killerxpress.com.
# Generated from content/seo/crawl.json and content/seo/llms/. Every documented search and AI crawler is welcome.

# OpenAI: Surfaces websites in ChatGPT search results (honors robots.txt: yes)
User-agent: OAI-SearchBot
Allow: /

# OpenAI: Crawls content that may be used to train OpenAI foundation models (honors robots.txt: yes)
User-agent: GPTBot
Allow: /

# OpenAI: Fetches a page when a ChatGPT user or Custom GPT asks for it; OpenAI says robots.txt rules may not apply (honors robots.txt: limited)
User-agent: ChatGPT-User
Allow: /

# Anthropic: Improves search result quality for Claude users (honors robots.txt: yes)
User-agent: Claude-SearchBot
Allow: /

# Anthropic: Collects web content that may contribute to model training (honors robots.txt: yes)
User-agent: ClaudeBot
Allow: /

# Anthropic: Fetches a page when a Claude user asks a question (honors robots.txt: yes)
User-agent: Claude-User
Allow: /

# Google: Google Search, including AI Overviews and AI Mode (honors robots.txt: yes)
User-agent: Googlebot
Allow: /

# Google: Control token, not a crawler: governs Gemini model training and grounding. Does not affect Google Search inclusion or ranking
User-agent: Google-Extended
Allow: /

# Google: Generic Google crawler used by product teams; does not affect Search (honors robots.txt: yes)
User-agent: GoogleOther
Allow: /

# Perplexity: Surfaces and links websites in Perplexity results; not used for model training (honors robots.txt: yes)
User-agent: PerplexityBot
Allow: /

# Perplexity: Fetches a page a user asked about; Perplexity says it generally ignores robots.txt (honors robots.txt: no)
User-agent: Perplexity-User
Allow: /

# Microsoft: Bing index, which also grounds Microsoft Copilot answers (honors robots.txt: yes)
User-agent: bingbot
Allow: /

# Apple: Apple search features across Siri, Spotlight and Safari (honors robots.txt: yes)
User-agent: Applebot
Allow: /

# Apple: Control token, not a crawler: governs use of content to train Apple foundation models
User-agent: Applebot-Extended
Allow: /

# Meta: Improves Meta AI search result quality (honors robots.txt: yes)
User-agent: meta-webindexer
Allow: /

# Meta: Training foundation models and indexing content for Meta products (honors robots.txt: yes)
User-agent: meta-externalagent
Allow: /

# Meta: Fetches a link at a user's request; Meta says it may bypass robots.txt (honors robots.txt: limited)
User-agent: meta-externalfetcher
Allow: /

# Amazon: Search experiences in Amazon products; not used for model training (honors robots.txt: yes)
User-agent: Amzn-SearchBot
Allow: /

# Amazon: May be used to train Amazon AI models (honors robots.txt: yes)
User-agent: Amazonbot
Allow: /

# Amazon: User triggered fetches such as Alexa questions; may not follow all robots.txt rules (honors robots.txt: limited)
User-agent: Amzn-User
Allow: /

# DuckDuckGo: Fetches pages live for DuckAssist answers; not used to train AI models (honors robots.txt: yes)
User-agent: DuckAssistBot
Allow: /

# Mistral AI: Indexing for Mistral assistant answers; not used for training (honors robots.txt: yes)
User-agent: MistralAI-Index
Allow: /

# Mistral AI: User triggered fetches in Mistral assistants; not used for training (honors robots.txt: yes)
User-agent: MistralAI-User
Allow: /

# Mistral AI: Builds training datasets (honors robots.txt: yes)
User-agent: MistralAI-Training
Allow: /

# Common Crawl: Open web corpus used widely in AI training and research (honors robots.txt: yes)
User-agent: CCBot
Allow: /

User-agent: *
Allow: /

Sitemap: https://ai.killerxpress.com/sitemap.xml