# ==============================================================================
# Robots.txt for Nequi PSE Framework
# Prepared with guidance from Cloudflare's Robots.txt and AI agent policies.
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. General Search Crawlers
# ------------------------------------------------------------------------------
# Allow standard search engines (Google, Bing, Yahoo, etc.) to discover the site.
User-agent: *
Allow: /

# ------------------------------------------------------------------------------
# 2. AI & LLM Scrapers / Crawler Controls
# ------------------------------------------------------------------------------
# Below are directives for major AI, LLM, and dataset training crawlers.
# Currently set to Disallow to protect the framework assets and mockup views.
# Toggle to "Allow: /" if you explicitly want to allow AI training or features.

# OpenAI GPTBot (ChatGPT's web crawler)
User-agent: GPTBot
Disallow: /

# OpenAI ChatGPT User Assistant (Direct requests from ChatGPT users)
User-agent: ChatGPT-User
Disallow: /

# Google Gemini / Bard Extended Crawler
User-agent: Google-Extended
Disallow: /

# Anthropic Claude Crawler
User-agent: Anthropic-AI
Disallow: /

# Claude Web crawler
User-agent: Claude-Web
Disallow: /

# Common Crawl Bot (Primary builder of LLM training datasets)
User-agent: CCBot
Disallow: /

# Perplexity AI Crawler
User-agent: PerplexityBot
Disallow: /

# Cohere AI Crawler
User-agent: Cohere-ai
Disallow: /

# Applebot Extended (Apple's AI training crawler)
User-agent: Applebot-Extended
Disallow: /

# Facebook Bot
User-agent: facebookexternalhit
Disallow: /

# ------------------------------------------------------------------------------
# 3. Content Signals Policy
# ------------------------------------------------------------------------------
# Cloudflare supports declaring machine-readable content usage signals.
# Reference: https://contentsignals.org/
# Settings: ai-train=no (opt-out of AI training), search=yes, ai-input=no
Content-Signal: ai-train=no, search=yes, ai-input=no

# ------------------------------------------------------------------------------
# 4. Sitemaps
# ------------------------------------------------------------------------------
# Define the path to your sitemap when deployed.
# Sitemap: https://yourdomain.com/sitemap.xml

# ------------------------------------------------------------------------------
# CLOUDFLARE BOT MANAGEMENT NOTE:
# Since robots.txt is voluntary and "polite" bots can choose to ignore it,
# if this project is proxied behind Cloudflare, you should also navigate to:
# Cloudflare Dashboard -> Security -> Bots
# -> Toggle "Instruct AI bot traffic with robots.txt" (auto-prepends AI blocks)
# -> Toggle "Block AI Bots" (enforces blocks at the network edge)
# ------------------------------------------------------------------------------
