# Straitway Truth Ministry — production robots policy
# Goal: MAXIMUM AI SEARCH / ANSWER discoverability, while NOT opening pure model-training
# crawlers. Public teaching/fellowship/testimony pages are open; private member, admin,
# moderation, auth, event-operations (CountMeIn) and API areas are disallowed.
#
# IMPORTANT — EDGE-MANAGED AI RULES (Cloudflare):
# On the PREVIEW edge, Cloudflare prepends a "Cloudflare Managed Content" robots block that
# currently BLOCKS several crawlers site-wide (GPTBot, ClaudeBot, CCBot, Applebot-Extended,
# Amazonbot, Bytespider, Google-Extended, meta-externalagent) and sets Content-Signal
# ai-train=no. Whether the SAME block applies to production straitway.com must be verified
# in the Cloudflare dashboard and via real User-Agent tests — an app-level robots.txt CANNOT
# override an edge block. Where our intent below differs from Cloudflare (notably we ALLOW
# Google-Extended), the Cloudflare zone settings must be updated for it to take effect.
#
# Crawler facts (do not conflate):
#   * OAI-SearchBot = OpenAI ChatGPT SEARCH crawler.  GPTBot = OpenAI model-TRAINING crawler.
#   * Claude-SearchBot / Claude-User = Anthropic search-index / user-initiated fetch.
#     ClaudeBot = Anthropic model-TRAINING crawler.
#   * PerplexityBot / Perplexity-User = Perplexity search-index / user-initiated fetch.
#   * Google-Extended is NOT a crawler — it is a robots.txt product-control TOKEN governing
#     Gemini Apps grounding + Gemini/Vertex generative & future training uses. It does NOT
#     determine ordinary Google Search inclusion or ranking.
#   * Normal Googlebot and Bingbot are allowed for search via the default "*" group below.

# --- Default: all crawlers (search allowed; private areas protected) -------
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# ===========================================================================
#  ALLOWED — AI SEARCH / ANSWER / USER-RETRIEVAL crawlers (public pages only)
# ===========================================================================

# OpenAI ChatGPT search (distinct from GPTBot training crawler)
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# Anthropic Claude search index (distinct from ClaudeBot training crawler)
User-agent: Claude-SearchBot
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# Anthropic Claude user-initiated fetch (when a user asks Claude about a page)
User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# Perplexity search index
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# Perplexity user-initiated fetch
User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# Google-Extended — CONTROL TOKEN (not a crawler). ALLOWED on public content so Gemini can
# ground/represent Straitway accurately. Tradeoff accepted: this can permit public content
# for Gemini Apps grounding, Gemini/Vertex generative uses, and future Gemini training. It
# does NOT affect ordinary Google Search. NOTE: Cloudflare currently blocks Google-Extended
# at the edge — update the Cloudflare zone for this allowance to actually take effect.
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /admin-login
Disallow: /dashboard
Disallow: /moderation
Disallow: /members
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /join/
Disallow: /superchat/
Disallow: /thank-you
Disallow: /countmein
Disallow: /api/

# ===========================================================================
#  BLOCKED — pure/general model-TRAINING crawlers (kept out site-wide)
#  (This agrees with Cloudflare's current managed block for these agents.)
# ===========================================================================
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

# LLM discovery: /llms.txt is a plain-language site guide following the emerging, open
# "llms.txt" convention (llmstxt.org) — a proposed convention, not a guaranteed standard, and
# no system is guaranteed to read it. Google has stated llms.txt is not required for Google
# Search and does not improve rankings. Also linked from the HTML <head> via rel="describedby".
LLM-Content: https://straitway.com/llms.txt

Sitemap: https://straitway.com/sitemap.xml