# robots.txt for https://varg.ai
#
# Public pages may be crawled for search, used as input for AI answers and
# used for AI training. Signed-in app and operational paths stay out.
# Index for AI agents: https://varg.ai/llms.txt
# Every model's spec for AI agents: https://varg.ai/models/llms.txt
# Docs index for AI agents: https://docs.varg.ai/llms.txt
#
# One group for every crawler on purpose: a crawler that gets its own group
# (for example "User-agent: GPTBot") ignores the rules of the "*" group.
# No "Allow: /" line: parsers that apply the first matching rule would let
# it override every Disallow below.
#
# Cloudflare may prepend its own content-signals block to this file. If it
# does, keep its Content-Signal values in line with the one below.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
# The dashboard pages a guest can open (GUEST_DASHBOARD_PAGES in
# src/middleware.ts) are where the Home's Pricing, API, MCP, Academy,
# What's new and model (Playground) links lead. They carry noindex; allowing
# the crawl is what lets crawlers see it, and lets agents follow those
# links. They come before the
# /dashboard/ Disallow so that first-match parsers read them first as well.
Allow: /dashboard/academy
Allow: /dashboard/api
Allow: /dashboard/changelog
Allow: /dashboard/mcp
Allow: /dashboard/pricing
Allow: /dashboard/tools/playground
Disallow: /dashboard/
Disallow: /project/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /onboarding
Disallow: /oauth/
Disallow: /auth/
Disallow: /alpha-gate
Disallow: /debug/
Disallow: /dev/
Disallow: /cdn-cgi/

Sitemap: https://varg.ai/sitemap.xml
Sitemap: https://docs.varg.ai/sitemap.xml