# https://hyperresearch.ai/robots.txt
# Hyperresearch. These are preferences, not enforcement.
# Search, real-time AI answers and model training: yes.
# Contact: support@hyperresearch.ai

# Cloudflare Content Signals Policy (contentsignals.org)
# As a condition of accessing this website, you agree to abide by the
# following content signals:
# (a) If a content-signal = yes, you may collect content for the
#     corresponding use.
# (b) If a content-signal = no, you may not collect content for the
#     corresponding use.
# Any use not listed is not granted.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes, use=reference
Content-Usage: search=y, ai-use=y, train-ai=y
Allow: /
Disallow: /api/
Disallow: /*?*token=

# Search indexes and AI answer engines: ALLOW
User-agent: Googlebot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: bingbot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: OAI-AdsBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: MistralAI-Index
User-agent: DuckDuckBot
User-agent: ExaBot
Allow: /

# Live, user-triggered fetchers: ALLOW
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: meta-externalfetcher
User-agent: facebookexternalhit
Allow: /

# Coding agents fetching our docs: ALLOW
User-agent: Claude-Code
User-agent: Cursor
User-agent: opencode
User-agent: Google-Gemini-CLI
Allow: /

# Model-training crawlers and usage flags: ALLOW
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: MistralAI-Training
User-agent: cohere-training-data-crawler
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: DeepSeekBot
User-agent: CCBot
Allow: /

# Third-party scrape-and-resell: DISALLOW
User-agent: Diffbot
User-agent: Timpibot
User-agent: omgili
User-agent: Omgilibot
User-agent: ApifyBot
User-agent: ApifyWebsiteContentCrawler
Disallow: /

Sitemap: https://hyperresearch.ai/sitemap-index.xml