# Crawl policy. Some agents only honor groups that name them explicitly
# (e.g. AdsBot, AI/LLM crawlers), so each is listed by name below.

# Block all crawlers (AdsBot crawlers must be named explicitly)
User-agent: *
User-agent: AdsBot-Google
Disallow: /

# Block AI/LLM crawlers and training/search agents (named explicitly)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: CCBot
User-agent: Bytespider
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
Disallow: /