User-agent: *
# Content Signals — https://contentsignals.org/ (IETF draft-romm-aipref-contentsignals)
# search: build a search index and return links/excerpts.
# ai-input: real-time retrieval/grounding for generative answers and agents.
# ai-train: training or fine-tuning AI models.
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Block authentication and authenticated product surfaces (intentional).
Disallow: /login
Disallow: /reset-password
Disallow: /app/*

# Cloudflare email-obfuscation stubs return 404 to crawlers.
Disallow: /cdn-cgi/

# Explicitly allow high-value public routes (redundant with Allow: / but documents intent).
Allow: /llms.txt
Allow: /sitemap.xml
Allow: /docs/
Allow: /docs/rfp-ai-rest-api-documentation
Allow: /docs/how-to-install-rfp-ai-browser-extension
Allow: /docs/claude-desktop-mcp-integration
Allow: /docs/how-to-connect-claude-desktop-to-rfp-ai
Allow: /openapi.json
Allow: /pricing
Allow: /support
Allow: /security
Allow: /agents
Allow: /ai-rfp-software
Allow: /rfp-platform
Allow: /integrations
Allow: /cookie-policy
Allow: /why/
Allow: /use-cases/
Allow: /compare/
Allow: /industries/
Allow: /blog/
Allow: /resources/

# Block common bot traps and scanner targets
Disallow: /admin
Disallow: /wp-admin
Disallow: /.env
Disallow: /config
Disallow: /backup
Disallow: /.git

# Explicit AI crawler policy: allow public pages while preserving blocked paths above.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /login
Disallow: /reset-password
Disallow: /app/*
Disallow: /cdn-cgi/
Disallow: /admin
Disallow: /wp-admin
Disallow: /.env
Disallow: /config
Disallow: /backup
Disallow: /.git

# Block Common Crawl training-only scraping while keeping answer engines reachable.
User-agent: CCBot
Content-Signal: ai-train=no, search=no, ai-input=no
Disallow: /

# Bing still honors Crawl-delay; keep it out of the * group so strict Google-focused validators stay quiet.
User-agent: Bingbot
Crawl-delay: 1

# LLM-friendly question → URL map for AI crawlers and assistants
# https://rfp.ai/llms.txt

# Sitemap
Sitemap: https://rfp.ai/sitemap.xml