# Robots Exclusion · Clementine Media
# https://clementine-media.de

# Standard crawler — explicit allow
User-agent: *
Allow: /
Disallow: /_template/
Disallow: /bewerbung/
Disallow: /kunden/
Disallow: /*?nocache=
Disallow: /karriereseiten-kommunen
Disallow: /wie-wir-arbeiten
Disallow: /unser-team

# AI / LLM crawlers — explicit welcome for the right ones
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: cohere-ai
Allow: /

# Spam / harmful scrapers — block
User-agent: AhrefsBot
Crawl-delay: 30
User-agent: SemrushBot
Crawl-delay: 30
User-agent: MJ12bot
Disallow: /

# Sitemaps & feeds
Sitemap: https://clementine-media.de/sitemap.xml
Sitemap: https://clementine-media.de/blog/rss.xml

# Host
Host: https://clementine-media.de