# ==============================================================================
# Kanpur Metro Guide (https://metrokanpur.in) - robots.txt
# Optimized for Google Search, Google AI Overviews, AEO, GEO & Generative Search
# ==============================================================================

# 1. Primary Google Crawlers
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image
Disallow: /login
Disallow: /404.shtml

User-agent: Googlebot-Mobile
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image
Disallow: /login

User-agent: Googlebot-Image
Allow: /images/
Allow: /uploads/
Allow: /storage/
Allow: /build/
Disallow: /admin

User-agent: Mediapartners-Google
Allow: /

# 2. Google AI Overviews & Gemini Generative Search Engine (AEO/GEO)
User-agent: Google-Extended
Allow: /
Allow: /news/
Allow: /news/*
Allow: /category/
Allow: /category/*
Allow: /tag/
Allow: /tag/*
Allow: /author/
Allow: /author/*
Allow: /route
Allow: /stations
Allow: /stations/*
Allow: /fare
Allow: /timings
Allow: /map
Allow: /ticket
Allow: /phase-2
Allow: /about
Allow: /contact
Allow: /privacy-policy
Allow: /terms
Allow: /disclaimer
Allow: /images/
Allow: /uploads/
Allow: /storage/
Allow: /build/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image

# 3. OpenAI / ChatGPT & SearchGPT (AEO/GEO)
User-agent: GPTBot
Allow: /
Allow: /news/
Allow: /news/*
Allow: /category/
Allow: /category/*
Allow: /tag/
Allow: /tag/*
Allow: /author/
Allow: /author/*
Allow: /route
Allow: /stations
Allow: /stations/*
Allow: /fare
Allow: /timings
Allow: /map
Allow: /ticket
Allow: /phase-2
Allow: /images/
Allow: /uploads/
Allow: /storage/
Allow: /build/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image

User-agent: ChatGPT-User
Allow: /
Allow: /news/
Allow: /news/*
Allow: /category/
Allow: /category/*
Allow: /tag/
Allow: /tag/*
Allow: /author/
Allow: /author/*
Allow: /route
Allow: /stations
Allow: /stations/*
Allow: /fare
Allow: /timings
Allow: /map
Allow: /ticket
Allow: /phase-2
Allow: /images/
Allow: /uploads/
Allow: /storage/
Allow: /build/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image

# 4. Perplexity AI Search Engine (AEO/GEO)
User-agent: PerplexityBot
Allow: /
Allow: /news/
Allow: /news/*
Allow: /category/
Allow: /category/*
Allow: /tag/
Allow: /tag/*
Allow: /author/
Allow: /author/*
Allow: /route
Allow: /stations
Allow: /stations/*
Allow: /fare
Allow: /timings
Allow: /map
Allow: /ticket
Allow: /phase-2
Allow: /images/
Allow: /uploads/
Allow: /storage/
Allow: /build/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image

# 5. Anthropic Claude Search & Grounding (AEO/GEO)
User-agent: ClaudeBot
Allow: /
Allow: /news/
Allow: /news/*
Allow: /category/
Allow: /category/*
Allow: /tag/
Allow: /tag/*
Allow: /author/
Allow: /author/*
Allow: /route
Allow: /stations
Allow: /stations/*
Allow: /fare
Allow: /timings
Allow: /map
Allow: /ticket
Allow: /phase-2
Allow: /images/
Allow: /uploads/
Allow: /storage/
Allow: /build/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image

# 6. Apple Intelligence & Siri Search
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Allow: /news/
Allow: /news/*
Allow: /category/
Allow: /category/*
Allow: /tag/
Allow: /tag/*
Allow: /author/
Allow: /author/*
Allow: /route
Allow: /stations
Allow: /stations/*
Allow: /fare
Allow: /timings
Allow: /map
Allow: /images/
Allow: /uploads/
Allow: /llms.txt
Disallow: /admin
Disallow: /admin/*

# 7. Microsoft Bing & Copilot
User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image

# 8. All Other Web Crawlers
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /captcha-image
Disallow: /login
Disallow: /404.shtml

# Sitemaps Declaration
Sitemap: https://metrokanpur.in/sitemap.xml
Sitemap: https://metrokanpur.in/sitemap_news.xml
Sitemap: https://metrokanpur.in/sitemap_pages.xml
Sitemap: https://metrokanpur.in/sitemap_tags.xml
Sitemap: https://metrokanpur.in/sitemap_authors.xml
Sitemap: https://metrokanpur.in/sitemap_images.xml