# robots.txt - SNworks default
# Updated 2026-10-19
#
# Policy: no AI model training on our content, and no Meta crawlers except
# facebookexternalhit, which builds link previews (decision tied to Meta's
# Oct 30, 2026 Commercial Terms).
#
# robots.txt is a request, not a lock. Bots that ignore it (Bytespider,
# Perplexity-User, ChatGPT-User, meta-externalfetcher) need CDN/WAF rules.
# Each host needs its own copy; this file governs only the host serving it.
# Crawlers cache this file for up to 24 hours.

# ---------------------------------------------------------------------------
# Default for everyone else (unchanged)
# ---------------------------------------------------------------------------
User-agent: *
Crawl-delay: 10
Allow: /

# ---------------------------------------------------------------------------
# SEO tooling (existing rule, plus Semrush's AI-tooling bots)
# ---------------------------------------------------------------------------
User-agent: SemrushBot
Disallow: /

# ===========================================================================
# META - ALL CRAWLERS EXCEPT LINK PREVIEWS
# ===========================================================================

# ALLOWED ON PURPOSE: facebookexternalhit. It builds the headline/image/
# description preview when a statenews.com URL is shared on Facebook,
# Instagram, Messenger, WhatsApp or Threads, including our own posts. It is
# not in any group below, so it falls under the default "User-agent: *"
# rules above and is allowed. Do not add it to the block list.

# meta-externalagent   AI model training and direct product indexing
# meta-externalads     "advertising and business products" - the purpose named
#                      in the revised Commercial Terms
# meta-webindexer      Meta AI search index (blocking removes Meta AI citations)
# meta-externalfetcher user-requested / agentic fetches - Meta says it may
#                      bypass robots.txt
# meta-externaltest    integration test requests (new, Oct 2026)
# muse-                Meta's Muse agent client (new, Oct 2026, per Cloudflare's
#                      bot directory)
# FacebookBot          legacy Meta training crawler (speech/language models)
User-agent: meta-externalagent
User-agent: meta-externalads
User-agent: meta-webindexer
User-agent: meta-externalfetcher
User-agent: meta-externaltest
User-agent: muse-
User-agent: FacebookBot
Disallow: /

# ===========================================================================
# AI MODEL-TRAINING CRAWLERS
# ===========================================================================

# OpenAI - training crawler (OAI-SearchBot and ChatGPT-User are separate; see below)
User-agent: GPTBot
Disallow: /

# Anthropic - training crawler. anthropic-ai and Claude-Web are legacy tokens
# no longer in Anthropic's docs; harmless to keep.
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
Disallow: /

# Google - Google-Extended is a usage token, not a crawler: it opts out of
# Gemini training/grounding and does not affect Google Search. It will never
# appear in logs. GoogleOther is Google's R&D crawler (not used for Search).
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-CloudVertexBot
Disallow: /

# Apple - usage token for Apple foundation-model training. Applebot itself
# (Siri/Spotlight/Safari search) stays allowed and never appears as this token.
User-agent: Applebot-Extended
Disallow: /

# Amazon - Alexa / Nova training
User-agent: Amazonbot
Disallow: /

# Common Crawl - the open corpus most models train on
User-agent: CCBot
Disallow: /

# ByteDance / TikTok - no published robots.txt policy; widely reported to
# ignore it. Block at the CDN/WAF as well.
User-agent: Bytespider
User-agent: TikTokSpider
Disallow: /

# Cohere
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
Disallow: /

# Other model builders and training-corpus crawlers
# KimiBot (Moonshot AI), Reflectionbot (Reflection AI), PetalBot and PanguBot
# (Huawei), AI2Bot / Ai2Bot-Dolma (Allen Institute, Dolma dataset)
User-agent: KimiBot
User-agent: Reflectionbot
User-agent: PetalBot
User-agent: PanguBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
Disallow: /

# Data brokers and dataset tools that resell crawled content, including to
# AI companies: Diffbot, Webz.io (omgili / webzio-extended), Timpi,
# Hive (ImagesiftBot), img2dataset
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: webzio-extended
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: img2dataset
Disallow: /

# ===========================================================================
# NOT BLOCKED - AI search / answer engines
# ===========================================================================
# These index pages so an assistant can cite and link us; their operators say
# they are not used for training. Blocking them removes us from those answers.
# We do block Meta's equivalent (meta-webindexer) above. To block these too,
# remove the leading # on each line.
#
# User-agent: OAI-SearchBot
# User-agent: Claude-SearchBot
# User-agent: PerplexityBot
# User-agent: DuckAssistBot
# User-agent: YouBot
# Disallow: /

# ===========================================================================
# NOT BLOCKED - user-triggered fetchers
# ===========================================================================
# These fetch one page because a person asked an assistant about it. OpenAI
# and Perplexity state that robots.txt may not apply to these; Anthropic says
# it honors it. Enforce at the CDN if it matters.
#
# User-agent: ChatGPT-User
# User-agent: Claude-User
# User-agent: Perplexity-User
# User-agent: MistralAI-User
# User-agent: Google-Agent
# User-agent: GrokAgent
# Disallow: /

# Search engines deliberately untouched: Googlebot, Bingbot, Applebot, DuckDuckBot.
# Bing/Copilot has no robots.txt training token; Microsoft's controls are the
# noarchive / nocache robots meta tags in page markup.