# AutoSpa.work robots.txt
# Policy: Allow search and cite-and-link AI answer crawlers; block training-only crawlers.
# Operator decision 2026-11-02 (Terms section 3 carve-out).
# Rationale: AI citation bots (PerplexityBot, OAI-SearchBot) drive referral
# traffic and surface the site in AI-assisted search. Training-only crawlers
# (GPTBot, ClaudeBot, CCBot, Google-Extended) consume content without
# attribution. We allow the search-serving variants and block the
# pure-training variants per operator preference.

# Traditional search engines - full access
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI search / citation bots - full access (these drive referral traffic)
User-agent: PerplexityBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: YouBot
Allow: /

# User-triggered fetches and AI search crawlers (cite and link; see Terms section 3)
User-agent: ChatGPT-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Applebot
Allow: /

# OpenAI GPTBot - training-only, blocked
User-agent: GPTBot
Disallow: /

# Anthropic ClaudeBot - training crawler (search is Claude-SearchBot, allowed above)
User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Google-Extended - Gemini/Bard training; search handled by Googlebot above
User-agent: Google-Extended
Disallow: /

# Common Crawl - dataset builder, no attribution
User-agent: CCBot
Disallow: /

# Bytespider (ByteDance/TikTok) - training crawler
User-agent: Bytespider
Disallow: /

# Meta external agent - training crawler
User-agent: meta-externalagent
Disallow: /

# Apple AI training - block, allow regular Applebot for search
User-agent: Applebot-Extended
Disallow: /

# Amazon Alexa training
User-agent: Amazonbot
Disallow: /

# Default - allow everything not explicitly addressed above
# Content signals: search and AI answers with attribution yes; model training no.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /admin/

Sitemap: https://autospa.work/sitemap-index.xml