# ==============================================================================
# PROLOAN INDIA — ROBOTS.TXT
# Optimized for Traditional Search Engines (Google, Bing) and Generative AI (LLMs)
# Website: https://proloansindia.com
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. Standard Search Engine Crawlers (Google, Bing, Yahoo, DuckDuckGo, Yandex, etc.)
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /assets/
Allow: /assets/banners/
Allow: /assets/banks/
Allow: /assets/fonts/
Allow: /sitemap.xml
Allow: /sitemap.html
Allow: /llms.txt
Allow: /llms-full.txt

# Disallow Internal Admin Portals, Scratch & Data directories
Disallow: /portal.html
Disallow: /portal
Disallow: /scratch/
Disallow: /data/
Disallow: /api/
Disallow: /*.bak*
Disallow: /*.json$

# ------------------------------------------------------------------------------
# 2. Generative Engine Optimization (GEO) & AI Search Crawlers
# Explicitly allowing LLM citation and search indexers
# ------------------------------------------------------------------------------

# OpenAI SearchGPT Indexer (Real-Time Search Results)
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /portal.html
Disallow: /portal
Disallow: /data/
Disallow: /scratch/

# OpenAI GPTBot (Foundation Model Training & Ingestion)
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /portal.html
Disallow: /portal
Disallow: /data/
Disallow: /scratch/

# OpenAI ChatGPT User Browse
User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Perplexity AI Search Engine
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /portal.html
Disallow: /portal
Disallow: /data/
Disallow: /scratch/

# Google AI & Gemini Models
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Google Other
User-agent: GoogleOther
Allow: /

# Anthropic Claude Crawlers
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /portal.html
Disallow: /portal

User-agent: anthropic-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Meta AI Crawlers
User-agent: Meta-ExternalAgent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# DuckDuckGo AI Assist
User-agent: DuckAssistBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# ByteDance (Doubao / TikTok AI)
User-agent: Bytespider
Allow: /

# Common Crawl (Foundation model pre-training)
User-agent: CCBot
Allow: /
Disallow: /portal.html
Disallow: /portal
Disallow: /data/

# ------------------------------------------------------------------------------
# 3. Sitemaps and LLM Documentation References
# ------------------------------------------------------------------------------
Sitemap: https://proloansindia.com/sitemap.xml
Host: https://proloansindia.com