# https://www.getxapi.com

# One group for every crawler. Groups do not merge: a crawler obeys only the
# most specific group naming it, so separate "User-agent: X / Allow: /" stanzas
# silently dropped every Disallow below for the bots they named. Listing the
# search and AI crawlers here keeps the explicit welcome (blocking the
# search/retrieval bots while meaning to block only training is the classic
# silent AEO own-goal) while they all get the same rules.
#
# Do not disallow /_next/static/: crawlers need the JS and CSS to render pages.
# That rule was live from 2026-05-18 and kept Googlebot from rendering the site
# until 2026-06-13.
#
# robots.txt is crawl guidance, not access control or an indexing guarantee.
# Private routes stay protected by auth, and noindex belongs in page metadata.
User-agent: *
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Yeti
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /payment/
Disallow: /signup
Disallow: /login
# Block only blog search results (?q=, in any parameter position). Pagination
# (?page=N) stays crawlable: it is the blog's internal navigation.
Disallow: /blogs?q=
Disallow: /blogs?*&q=

# Content Signals, declare AI/search preferences per https://contentsignals.org
# search: allow indexing in search results
# ai-train: allow use of content in AI model training
# ai-input: allow retrieval for AI-generated answers (RAG / grounding)
Content-Signal: search=yes, ai-train=yes, ai-input=yes

Sitemap: https://www.getxapi.com/sitemap.xml