# Content-Signal: how AI systems may use this site's content. # ai-train=yes → may be used for AI model training # search=yes → may be used to build search indexes # ai-input=yes → may be used as input for AI answers / RAG # # Per-bot blocks below carry a Content-Signal matching what that bot does: # live-fetch / answer bots → ai-input=yes (used to answer a user now) # search / index crawlers → search=yes (used to build an index) # training / dataset bots → ai-train=yes (used for model training) # Advertising & link-preview fetchers are allowed but carry no AI signal. User-agent: * # Content-Signal: ai-train=yes, search=yes, ai-input=yes Allow: / Allow: /product Allow: /pricing Allow: /download Allow: /use-cases Allow: /changelog Allow: /comparisons Allow: /terms Allow: /privacy Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # OpenAI User-agent: ChatGPT-User # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: OAI-SearchBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: GPTBot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Anthropic / Claude User-agent: Claude-User # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Claude-SearchBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: ClaudeBot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: anthropic-ai # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Perplexity User-agent: Perplexity-User # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: PerplexityBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Google / Gemini User-agent: Googlebot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Google-Extended # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: GoogleOther # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Microsoft / Bing / Copilot User-agent: Copilot # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Bingbot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: msnbot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Apple User-agent: Applebot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Applebot-Extended # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # xAI / Grok User-agent: Grok-DeepSearch # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: xAI-SearchBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: xAI-Web-Crawler # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Mistral User-agent: MistralAI-User # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: MistralAI-Index # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Meta AI User-agent: Meta-ExternalFetcher # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Meta-ExternalAgent # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Amazon User-agent: Amzn-User # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Amazonbot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # DuckDuckGo User-agent: DuckAssistBot # Content-Signal: ai-input=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: DuckDuckBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # You.com User-agent: YouBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # ByteDance User-agent: Bytespider # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Other search indexers User-agent: YandexBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: SeznamBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: PetalBot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Qwantbot # Content-Signal: search=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Training / dataset & AI-research crawlers User-agent: CCBot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: cohere-ai # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Diffbot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: Timpibot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: LinerBot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: QualifiedBot # Content-Signal: ai-train=yes Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo # ───────────────────────────────────────────────────────────────────────────── # Advertising & link-preview fetchers — allowed, but not AI content consumers, # so they carry no AI Content-Signal. User-agent: OAI-AdsBot Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: meta-externalads Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo User-agent: facebookexternalhit Allow: / Disallow: /dashboard Disallow: /login Disallow: /onboarding Disallow: /auth/callback Disallow: /api/ Disallow: /affiliate Disallow: /demo Sitemap: https://aiola.app/sitemap.xml