# fictivebox.com - crawler policy
#
# Reviewed 2026-09-05. The directives below are UNCHANGED from the previous
# policy; only this documentation was added. Two distinct things are being
# granted here and they should be decided separately:
#
#   SEARCH / RETRIEVAL - the crawler fetches a page so it can be indexed,
#     ranked, or cited with a link back. This is what earns visibility.
#
#   MODEL TRAINING - the crawler collects pages into a corpus used to train a
#     model. It does NOT affect search ranking or AI-search citation, and it is
#     a licensing decision about the company's content, not an SEO one.
#
# Blocking the training-only agents below would cost nothing in search or
# AI-search visibility. They are currently allowed. If that is not a deliberate
# choice, change it here - see VERIFICATION-REQUIRED.md.
#
# Note: pages that should not be indexed use a per-page `noindex` robots meta
# tag rather than a Disallow here. A page blocked in robots.txt cannot be
# crawled, so its noindex would never be seen.

User-agent: *
Allow: /

# ---------------------------------------------------------------------------
# SEARCH / RETRIEVAL - keep these allowed. They produce indexing and citations.
# ---------------------------------------------------------------------------

# Google Search, and eligibility for AI Overviews / AI Mode.
User-agent: Googlebot
Allow: /

# Bing indexing, and Microsoft Copilot answers.
User-agent: Bingbot
Allow: /

# ChatGPT Search retrieval and citation. This is the agent that matters for
# appearing in ChatGPT results - it is not the training crawler.
User-agent: OAI-SearchBot
Allow: /

# Live fetch when a user asks ChatGPT to open a URL.
User-agent: ChatGPT-User
Allow: /

# Perplexity indexing and citation.
User-agent: PerplexityBot
Allow: /

# ---------------------------------------------------------------------------
# MODEL TRAINING / CORPUS COLLECTION - allowed, but this is a content-licensing
# decision. Disallowing any of these would not reduce search or AI-search
# visibility.
# ---------------------------------------------------------------------------

# OpenAI training corpus. Separate from OAI-SearchBot above.
User-agent: GPTBot
Allow: /

# Google's Gemini training / grounding corpus. Separate from Googlebot: blocking
# this does NOT remove the site from Google Search or AI Overviews eligibility.
User-agent: Google-Extended
Allow: /

# Common Crawl. Feeds many third-party training datasets. Not a search signal.
User-agent: CCBot
Allow: /

# Anthropic crawling. Claude-Web is a legacy token retained for compatibility.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

Sitemap: https://fictivebox.com/sitemap.xml