# tryrankwise.com
#
# We want AI assistants to read, quote and cite this site. Everything below
# says so explicitly, in the two vocabularies crawlers actually parse.
#
# Generated by lib/seo/robots.ts. Do not hand-edit the served file: every
# group repeats the same Disallow rules on purpose (RFC 9309 2.2.1 means a
# named group does NOT inherit them from the wildcard group), and dropping
# them from one group would expose /dashboard/ and /api/ to that crawler.

# Content Signals (https://contentsignals.org): yes to all three. Use this
# content for search, for grounding AI answers, and for training.
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# OpenAI
# Surfaces pages in ChatGPT search. Block this and you leave ChatGPT results.
User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Fetches a page when a ChatGPT user's question needs it.
User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Collects training corpus for OpenAI foundation models.
User-agent: GPTBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Anthropic
# Indexes pages for Claude's search results.
User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Retrieves a page when a Claude user's question needs it.
User-agent: Claude-User
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Collects training corpus for Anthropic models.
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Older Anthropic token, not in Anthropic's current crawler docs.
User-agent: Claude-Web
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Older Anthropic token, not in Anthropic's current crawler docs.
User-agent: anthropic-ai
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Google
# Controls Gemini training and grounding use. Does not affect Google Search.
User-agent: Google-Extended
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Builds the Search index AI Overviews and AI Mode draw from.
User-agent: Googlebot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Perplexity
# Indexes pages for Perplexity answers and their citations.
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Fetches a page for a live Perplexity question.
User-agent: Perplexity-User
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Microsoft
# Bing's crawler. Microsoft Copilot can ground answers in Bing search.
User-agent: bingbot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Apple
# Powers search in Spotlight, Siri and Safari.
User-agent: Applebot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Gates Apple foundation-model training on this content.
User-agent: Applebot-Extended
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Meta
# Collects content for Meta AI training and product indexing.
User-agent: meta-externalagent
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Indexes pages so Meta AI can cite and link them.
User-agent: meta-webindexer
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Fetches a link when a Meta AI user asks for it.
User-agent: meta-externalfetcher
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Amazon
# Improves Amazon products and may train Amazon AI models.
User-agent: Amazonbot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Indexes pages for search in Amazon products. Not used for training.
User-agent: Amzn-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Fetches a page for a live Alexa question. Not used for training.
User-agent: Amzn-User
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# ByteDance
# ByteDance crawler. ByteDance publishes no documentation for it.
User-agent: Bytespider
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# You.com
# You.com crawler. Its documentation page is no longer published.
User-agent: YouBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# DuckDuckGo
# Fetches pages for DuckDuckGo's AI assisted answers.
User-agent: DuckAssistBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Mistral
# Fetches a page for a live Mistral question.
User-agent: MistralAI-User
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Indexes pages for Mistral. Not used for training.
User-agent: MistralAI-Index
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Collects content to train Mistral models.
User-agent: MistralAI-Training
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Allen Institute for AI
# Collects content to train Ai2's open language models.
User-agent: AI2Bot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Cohere
# Older Cohere token. Cohere documents no crawler under this name.
User-agent: cohere-ai
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# xAI
# Grok fetcher token. xAI publishes no crawler documentation.
User-agent: GrokBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# xAI token. xAI publishes no crawler documentation.
User-agent: xAI-Grok
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

# Everyone else, including any AI crawler not named above.
User-agent: *
Allow: /
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /sign-in
Disallow: /sign-up

Sitemap: https://tryrankwise.com/sitemap-index.xml