# robots.txt — managed by Rankbeam (AI crawler directives)

# Content signals (https://contentsignals.org) — how this content may be
# used, independent of the crawl-access rules above. Advisory: honoured by
# participating crawlers.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow:

# --- AI crawlers (managed by Rankbeam) ---

# GPTBot — OpenAI (AI training)
User-agent: GPTBot
Disallow: /

# ClaudeBot — Anthropic (AI training)
User-agent: ClaudeBot
Disallow: /

# anthropic-ai — Anthropic (AI training)
User-agent: anthropic-ai
Disallow: /

# Google-Extended — Google (AI training)
User-agent: Google-Extended
Disallow: /

# Applebot-Extended — Apple (AI training)
User-agent: Applebot-Extended
Disallow: /

# CCBot — Common Crawl (AI training)
User-agent: CCBot
Disallow: /

# Bytespider — ByteDance (AI training) — advisory: this bot may not honour robots.txt
User-agent: Bytespider
Disallow: /

# meta-externalagent — Meta (AI training)
User-agent: meta-externalagent
Disallow: /

# DeepSeekBot — DeepSeek (AI training) — advisory: this bot may not honour robots.txt
User-agent: DeepSeekBot
Disallow: /

# AI2Bot — Ai2 (AI training)
User-agent: AI2Bot
Disallow: /

# Timpibot — Timpi (AI training)
User-agent: Timpibot
Disallow: /

# omgili — Webz.io (AI training)
User-agent: omgili
Disallow: /

# ImagesiftBot — ImageSift (AI training)
User-agent: ImagesiftBot
Disallow: /

# Diffbot — Diffbot (AI training)
User-agent: Diffbot
Disallow: /

Sitemap: https://blog.rankbeam.dev/sitemap.xml
# AI content index: https://blog.rankbeam.dev/llms.txt