# Spexo — robots.txt
# Full crawling is intentionally open: this is a free public tool and we
# want it indexed by general search engines, and discovered/cited by AI
# assistants and answer engines. See /llms.txt (and compatibility aliases /llm.txt and /LLM.txt) for an AI-oriented summary
# of this site's purpose and key pages.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# ---- Major AI assistants & answer engines (explicitly allowed) ----
# OpenAI — ChatGPT / web browsing plugin / search
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Google — Gemini / AI features (separate from classic Googlebot below)
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: GoogleOther
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Meta / Llama
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: meta-externalfetcher
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Apple
User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# ByteDance
User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# Common Crawl (used to train many LLMs)
User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# You.com
User-agent: YouBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

# ---- Classic search engine crawlers ----
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/
User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /blog/admin/

Sitemap: https://spexo.xyz/sitemap.xml