# Aiter robots.txt
#
# Rule of thumb applied here: Disallow only what must never be *fetched*.
# Anything that should merely stay out of the index carries <meta name="robots"
# content="noindex"> instead, and MUST stay crawlable — a blocked URL is a URL
# Google cannot learn to drop, so it keeps the address and indexes it without
# content. That is exactly what happened to /app/*: 31,700 impressions at a
# 0.44% CTR from pages that have carried noindex all along.

User-agent: *
Allow: /

# Authenticated product surfaces. Real content lives behind a login, so there
# is nothing here for a crawler to fetch.
Disallow: /dashboard
Disallow: /admin
Disallow: /profile
Disallow: /reset-password
Disallow: /campaign/

# Per-campaign app routes. Unbounded ID space, no public content.
Disallow: /app/campaigns
Disallow: /app/profile

# Firebase auth handler. Appeared in Search Console with the API key visible in
# the query string; it is machinery, not a page.
Disallow: /__/

# NOTE: /app/ as a whole is deliberately NOT blocked any more. /app/cookie-policy,
# /app/privacy-policy and /app/terms-and-conditions now 301 to their public
# equivalents (public/_redirects), and the rest of /app/* renders noindex.
# Google has to be allowed to fetch them to act on either signal.

# AI answer engines (GPTBot, OAI-SearchBot, ClaudeBot, PerplexityBot,
# Google-Extended, Applebot-Extended, CCBot) are deliberately allowed, and are
# covered by the `*` group above.
#
# They are NOT given their own groups on purpose. A named user-agent group
# replaces the `*` group outright rather than adding to it, so writing
# "User-agent: GPTBot / Allow: /" would hand those crawlers the login-walled
# routes this file just excluded. One group, one set of rules.

Sitemap: https://aiter.io/sitemap.xml