# Content Signals Policy: https://contentsignals.org/
# search: search index + snippets (not AI summaries)
# ai-input: real-time AI answers / RAG / grounding
# ai-train: training or fine-tuning models
# Last-Updated: 2026-08-25
# Explicit Allow list (+ wildcard * for everyone else).
# List is curated, not exhaustive.
# AI discovery (kanonisch):
#   /llms.txt · /llms-full.txt · /llms-index.txt · /llms-keywords.txt · /llms-serp.txt
#   /llms-impressum-kontakt.txt · /llms-orte-geo.txt · /llms-urheberrecht.txt · /llms-copyright.txt
#   /llms-mcp-server.txt · /llms-mcp-web.txt
#   /.well-known/agent-skills/index.json
#   /ai-pages/ · /.well-known/ai.txt · /.well-known/ai-plugin.json · /.well-known/answer-engine.json
#   /.well-known/ai-catalog.json (ARD)
#   /sitemap.txt · /sitemap.xml
# Root-Aliase (Rewrite, kein zweites File): /ai.txt · /ai-plugin.json · /answer-engine.json · /ai-catalog.json
Sitemap: https://www.ikeytz.com/sitemap.xml
# Plain-text URL list for AI: https://www.ikeytz.com/sitemap.txt (not a Google XML sitemap)
Agentmap: https://www.ikeytz.com/.well-known/ai-catalog.json

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
Disallow: /devsecops

# --- Search + AI crawlers (curated; Cloudflare Radar / common UA tokens) ---
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: bingbot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: TikTokSpider
Allow: /

User-agent: CCBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: PetalBot
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: Ai2Bot-Dolma
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: FirecrawlAgent
Allow: /

User-agent: archive.org_bot
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: Cloudflare-Crawler
Allow: /

User-agent: Cloudflare-AutoRAG
Allow: /

User-agent: Timpibot
Allow: /

User-agent: ProRataInc
Allow: /

User-agent: Manus-Bot
Allow: /

User-agent: ManusBot
Allow: /

User-agent: Novellum-AI-Crawl
Allow: /

User-agent: Anchor
Allow: /

User-agent: Terracotta
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: aiHitBot
Allow: /

User-agent: Magpie-Crawler
Allow: /

User-agent: adscanner
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: omgili
Allow: /

User-agent: omgilibot
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: Grok
Allow: /

User-agent: GitHub-Copilot
Allow: /

User-agent: Webzio-Extended
Allow: /

User-agent: iaskspider
Allow: /

# --- Social / link preview (OG, cards, chat unfurl) ---
User-agent: claude-web
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /