# AI crawlers and agents are welcome here.
# This site measures how AI systems read the web; blocking them would defeat it.
# Nothing is cloaked: every client receives identical bytes for the same URL.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Timpibot
User-agent: YouBot
Allow: /
Crawl-delay: 10

User-agent: *
Allow: /
Crawl-delay: 10
Disallow: /internal/
Disallow: /no-crawl/
Disallow: /private-preview/

# The disallowed paths above serve ordinary content and return 200. They are not
# traps and contain nothing sensitive. They exist so that robots.txt compliance
# is measurable rather than assumed. Results: https://agentshieldaidefense.com/lab
#
# Crawl-delay is published for the same reason: a rate nobody was given cannot be
# honoured or ignored, only guessed at. It is set loose on purpose, it is not
# enforced, and no request is ever refused for exceeding it. Crawl-delay is not
# part of the original specification and several crawlers state that they ignore
# it — that is a measurement, not a grievance.

Sitemap: https://agentshieldaidefense.com/sitemap.xml