User-agent: *
Allow: /
Allow: /blog/
Allow: /blog/category/
Allow: /payment-gateways/

# Please pace requests.
Crawl-delay: 2

# Disallow admin and private areas
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

# Primary Sitemap Index (contains all sub-sitemaps with pagination)
# Google will discover all sub-sitemaps automatically from the index
Sitemap: https://www.useaxra.com/sitemap-index.xml

# AI-answer-engine sitemap (Plan 56)
Sitemap: https://www.useaxra.com/sitemap-ai.xml

# --- Retrieval / answer bots + classic search bots ---
# These fetch a page to answer a live user question and cite it with a link,
# or crawl for a classic search index. Allow, unambiguously, and do not rely
# on the wildcard: several engines look for their own token before falling
# back to '*'. No Crawl-delay — some answer engines skip the fetch entirely
# rather than honour a delay inside their latency budget.
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: DuckAssistBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: MistralAI-User
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Slurp
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/

# --- Training crawlers (Plan 56, CEO decision Q1: APPROVED, allow) ---
# Ingest content into model weights. NEXT_PUBLIC_ALLOW_TRAINING_CRAWLERS is a
# kill switch defaulted ON (unset = allowed); set it to 'false' to revert to
# blocking without a code change.
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2

User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2

User-agent: Bytespider
Allow: /
Disallow: /admin/
Disallow: /api/cron/
Disallow: /api/admin/
Disallow: /api/og/
Disallow: /_next/
Disallow: /api/blog/
Disallow: /scripts/
Disallow: /api/ai/admin/
Disallow: /interview/
Crawl-delay: 2