# robots.txt for bryq.com
# Strategy: Maximum AI visibility — Bryq wants to be crawled,
# indexed, cited, and recommended by all major AI engines.
# Last updated: 2026-04-03

# ============================================================
# Default: Allow all well-behaved crawlers
# ============================================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/static/
Disallow: /_next/image/
Disallow: /_next/data/

# ============================================================
# Traditional Search Engines
# ============================================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ============================================================
# OpenAI Crawlers
# ============================================================
# GPTBot — model training. Allowing ensures Bryq content
# is represented in future GPT model knowledge.
User-agent: GPTBot
Allow: /

# OAI-SearchBot — powers ChatGPT search results.
# Critical for appearing in ChatGPT answers.
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User — real-time retrieval when a user asks
# ChatGPT a question. Must be allowed for live citation.
User-agent: ChatGPT-User
Allow: /

# ============================================================
# Anthropic / Claude Crawlers
# ============================================================
# ClaudeBot — training data collection. Allowing ensures
# Bryq is in Claude's knowledge base.
User-agent: ClaudeBot
Allow: /

# Claude-SearchBot — indexes content for Claude search.
# Required for appearing in Claude search results.
User-agent: Claude-SearchBot
Allow: /

# Claude-User — fetches pages when a user asks Claude
# a question. Must be allowed for real-time answers.
User-agent: Claude-User
Allow: /

# ============================================================
# Google AI / Gemini
# ============================================================
# Google-Extended — controls Gemini/Vertex AI training.
# Allowing ensures Bryq appears in Google AI Overviews
# and Gemini responses.
User-agent: Google-Extended
Allow: /

# ============================================================
# Perplexity
# ============================================================
# PerplexityBot — search indexing and retrieval.
# Critical for appearing in Perplexity AI answers.
User-agent: PerplexityBot
Allow: /

# ============================================================
# Apple Intelligence
# ============================================================
# Applebot-Extended — trains Apple Intelligence features.
# Allowing ensures visibility in Apple AI surfaces
# (Siri, Safari suggestions, Apple Intelligence).
User-agent: Applebot-Extended
Allow: /

# ============================================================
# Meta AI
# ============================================================
# Meta-ExternalAgent — search crawling and training for
# Meta AI. Allowing ensures Bryq appears in Meta AI answers
# across WhatsApp, Instagram, and Facebook.
User-agent: Meta-ExternalAgent
Allow: /

# ============================================================
# Microsoft Copilot
# ============================================================
User-agent: CopilotBot
Allow: /

# ============================================================
# Cohere
# ============================================================
# cohere-ai — retrieval and training for Cohere language models.
User-agent: cohere-ai
Allow: /

# ============================================================
# You.com
# ============================================================
# YouBot — search indexing for You.com AI assistant.
User-agent: YouBot
Allow: /

# ============================================================
# Aggressive / Low-Value Scrapers — Block
# ============================================================
# Bytespider (ByteDance) — aggressive crawler with
# inconsistent robots.txt compliance. No search visibility
# benefit for Western markets.
User-agent: Bytespider
Disallow: /

# CCBot (Common Crawl) — bulk scraping for open datasets.
# No direct search or AI citation benefit.
User-agent: CCBot
Allow: /
# Allowed — Common Crawl is upstream training data for many LLMs.

# ============================================================
# Sitemaps & Structured AI Content
# ============================================================
Sitemap: https://www.bryq.com/sitemap.xml

# LLMs.txt — structured content guide for AI/LLM crawlers.
# See https://www.bryq.com/llms.txt for Bryq's AI-readable
# site overview, product descriptions, and resource index.