# Search engines may crawl everything; app pages carry noindex and need a login.
# AI crawlers and agents may read the public pages, the agent discovery files
# (llms.txt, llms-full.txt, the MCP server card) and the sitemap, but not the app.
# Keep this list in step with sitemap.xml and exponentially_selective_indexing()
# in the theme's functions.php.

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: GPTBot
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: ChatGPT-User
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: Claude-SearchBot
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: ClaudeBot
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: Claude-User
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: PerplexityBot
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: Perplexity-User
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

User-agent: *
Allow: /login
Allow: /register
Allow: /idea-validator
Allow: /rapidly-mcp
Allow: /faq
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /.well-known/mcp/
Disallow: /

Sitemap: https://www.rapidly.co/sitemap.xml