# Robots.txt for kubernetes.recipes
# https://kubernetes.recipes/
# Last updated: 2026-01-28

# ============================================
# MAIN CRAWLERS - Full access
# Note: do NOT block /_astro/ — Google needs the fingerprinted CSS/JS to
# render pages. Do NOT block query strings (/*?*) — they serve the same static
# page and are already de-duplicated via rel=canonical; blocking them only
# generates "Blocked by robots.txt" noise in Search Console.
# ============================================
User-agent: Googlebot
Allow: /
Disallow: /api/
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /
Disallow: /api/
Crawl-delay: 1

User-agent: Slurp
Allow: /
Disallow: /api/
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Crawl-delay: 1

User-agent: Applebot
Allow: /
Disallow: /api/

# ============================================
# AI CRAWLERS - Full access for training
# ============================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Cohere-AI
Allow: /

# ============================================
# DEFAULT - All other crawlers
# ============================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_astro/
Crawl-delay: 2

# Gated lead magnet — keep out of search indexes on every host.
# (GitHub Pages cannot send the X-Robots-Tag header in public/_headers.)
Disallow: /KubernetesRecipes-sample.pdf

# ============================================
# Sitemaps
# ============================================
Sitemap: https://kubernetes.recipes/sitemap-index.xml

# RSS feed (latest recipes)
# https://kubernetes.recipes/rss.xml

# ============================================
# Canonical Host
# ============================================
Host: https://kubernetes.recipes
# LLMs.txt - AI-friendly content index
# https://llmstxt.org/