# =====================================================
# robots.txt — Production
# Purpose: allow full indexing of the public bill-check
# tool and content pages; block only internal/non-indexable
# paths. Explicitly whitelists AI answer-engine crawlers
# for GEO visibility (Google AI Overviews, ChatGPT, Perplexity,
# Claude, Bing Copilot).
# =====================================================

# ---- Core search engines ----
User-agent: Googlebot
Disallow: /api/
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /*?*session=
Disallow: /*?*token=
Disallow: /account/
Disallow: /checkout/
Allow: /

User-agent: Bingbot
Disallow: /api/
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /account/
Disallow: /checkout/
Allow: /

# ---- AI answer-engine / grounding crawlers (GEO) ----
# Google's AI training/grounding crawler (feeds AI Overviews & Gemini)
User-agent: Google-Extended
Allow: /

# OpenAI - live retrieval for ChatGPT browsing/answers
User-agent: OAI-SearchBot
Allow: /

# OpenAI - training crawler
User-agent: GPTBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Bing's AI / Copilot grounding
User-agent: BingPreview
Allow: /

# Common Crawl (feeds many downstream LLM datasets)
User-agent: CCBot
Allow: /

# ---- Everything else (fallback) ----
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /account/
Disallow: /checkout/
Disallow: /*?*session=
Disallow: /*?*token=
Allow: /

# ---- Sitemap ----
Sitemap: https://yourdomain.com/sitemap.xml