# Public marketing/research/product pages are crawlable.
# Authenticated dashboard, APIs, admin, vault, swarm, onboarding and developer
# secrets remain disallowed. AI crawlers inherit the same Allow/Disallow rules
# for public content; they do not unlock private routes.
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /api/
Disallow: /auth/
Disallow: /admin/
Disallow: /advisor/
Disallow: /vault/
Disallow: /swarm/
Disallow: /onboarding/
Disallow: /developer/keys
Disallow: /developer/sandbox
Disallow: /give-ai-agent-10k/result/
Disallow: /give-ai-agent-10k/challenge/
Disallow: /give-ai-agent-10k/onboarding

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Allow AI search and discovery crawlers on public pages
User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# On-demand / user-triggered AI answer-engine fetchers
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Sitemap
Sitemap: https://www.neufin.ai/sitemap.xml