# ─────────────────────────────────────────
# robots.txt - cutthewaffle.nz
# Static site. Fully open for crawling, indexing, AI training, and social sharing.
# AI engines: see /llms.txt for a structured summary of this site.
# ─────────────────────────────────────────

User-agent: *
Allow: /

# Housekeeping: never index local backup copies if one is ever uploaded
Disallow: /BACKUP/

# Tool endpoint, not a page
Disallow: /waffle-check.php


# ─────────────────────────────────────────
# AI search & answer engines - explicitly welcomed
# (these inherit the User-agent: * rule above, but listing them
# documents intent and signals that AEO is wanted, not opted-out of)
# ─────────────────────────────────────────

# OpenAI (ChatGPT, GPT-4, GPT search)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI training (separate from search indexing - Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Intelligence (Siri, Apple search)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Common Crawl (feeds many AI training datasets)
User-agent: CCBot
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Amazon AI / Alexa
User-agent: Amazonbot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# ByteDance (TikTok / Doubao)
User-agent: Bytespider
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /


# ─────────────────────────────────────────
# Search engines
# ─────────────────────────────────────────

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /


# ─────────────────────────────────────────
# Social previews & link unfurls
# ─────────────────────────────────────────

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Pinterestbot
Allow: /


# ─────────────────────────────────────────
# Sitemap location
# ─────────────────────────────────────────
Sitemap: https://cutthewaffle.nz/sitemap.xml


# ─────────────────────────────────────────
# Notes for future maintenance
# ─────────────────────────────────────────
# - To exclude a page (e.g. a draft or template), add to the User-agent: * block:
#     Disallow: /blog/_template.html
# - To opt OUT of a specific AI training crawler, change its Allow: / to:
#     Disallow: /
#   (Not recommended for this site - we want AI citations.)
# - Robots.txt is a polite request, not enforcement. Good crawlers respect it.