# PressAudit — robots policy
#
# Google, OpenAI, Claude and Perplexity may crawl eligible public corpus pages
# because they send attributable referral traffic. Other named discovery, Meta,
# and SEO crawlers may read only the small promotional surface that explains
# PressAudit.
#
# Policy (2026-09-17, Claude and Perplexity added 2026-10-05): Google and AI
# answer-engine search get eligible corpus access.
# Other named discovery crawlers get promotional pages only. Link-preview bots
# may fetch a URL a person shares. Training crawlers are blocked. Raw data
# surfaces (/api, /snap, /newsapi, /lexicon) are off limits to everyone: the
# pages are the product, the JSON behind them is not.

User-agent: *
# Unnamed crawlers (web-filter and reputation categorizers such as Fortinet,
# alphaMountain, Symantec, Zscaler, Palo Alto) get the same promotional surface
# as Bingbot so they can classify PressAudit as News and Media instead of
# guessing from registration data. Corpus and raw surfaces stay closed.
Allow: /$
Allow: /llms.txt$
Allow: /docs$
Allow: /docs/
Allow: /how-it-works$
Allow: /ground-news-alternatives$
Allow: /features$
Allow: /faq$
Allow: /pressauditfaq$
Allow: /about-me$
Allow: /contact$
Allow: /privacy$
Allow: /terms$
Allow: /AI-ethics$
Allow: /changelog$
Allow: /extension$
Allow: /outlets$
Allow: /regulations$
Allow: /categories$
Allow: /interests$
Disallow: /

# --- Search engines ---------------------------------------------------------
# Google-Extended (Paul, 2026-10-06) shares Google Search's access so Gemini can
# read PressAudit when people ask it about the site.

User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /newsapi/
Disallow: /snap/
Disallow: /lexicon/
Disallow: /share-image
Disallow: /__
Disallow: /local
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /onboarding
Disallow: /account
Disallow: /profile
Disallow: /settings
Disallow: /saved
Disallow: /following
Disallow: /foryou
Disallow: /you
Disallow: /scorecard
Disallow: /trackrecord
Disallow: /regsqa
Disallow: /standings-audit
Disallow: /signup
Disallow: /signup/
Disallow: /preferences

# --- AI answer engines: search discovery + user fetches ---------------------
# OpenAI, Claude and Perplexity are allowed the same eligible public corpus as
# Google (Claude and Perplexity since 2026-10-05: their answers link readers
# here). GPTBot, ClaudeBot and anthropic-ai remain blocked below, so this does
# not grant model-training access.

User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /newsapi/
Disallow: /snap/
Disallow: /lexicon/
Disallow: /share-image
Disallow: /__
Disallow: /local
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /onboarding
Disallow: /account
Disallow: /profile
Disallow: /settings
Disallow: /saved
Disallow: /following
Disallow: /foryou
Disallow: /you
Disallow: /scorecard
Disallow: /trackrecord
Disallow: /regsqa
Disallow: /standings-audit
Disallow: /signup
Disallow: /signup/
Disallow: /preferences

# --- Promotional surface only ----------------------------------------------
# These crawlers may understand and promote PressAudit, but may not crawl the
# story, regulation-detail, finding, faceoff, prediction, timeline, outlet-
# detail, feed, search-result, or other corpus surfaces. The trailing $ keeps
# directory landing pages from granting access to their detail descendants.

User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Meta-ExternalFetcher
User-agent: meta-webindexer
User-agent: FacebookBot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DuckAssistBot
Allow: /$
Allow: /llms.txt$
Allow: /docs$
Allow: /docs/
Allow: /how-it-works$
Allow: /ground-news-alternatives$
Allow: /features$
Allow: /faq$
Allow: /pressauditfaq$
Allow: /about-me$
Allow: /contact$
Allow: /privacy$
Allow: /terms$
Allow: /AI-ethics$
Allow: /changelog$
Allow: /extension$
Allow: /outlets$
Allow: /regulations$
Allow: /categories$
Allow: /interests$
Disallow: /

# --- Human-triggered link previews -----------------------------------------
# These fetch the particular URL a person shares. They receive public HTML for
# a useful preview, but raw/private surfaces remain off limits.

User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: redditbot
User-agent: Discordbot
User-agent: TelegramBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /newsapi/
Disallow: /snap/
Disallow: /lexicon/
Disallow: /share-image
Disallow: /__
Disallow: /local
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /onboarding
Disallow: /account
Disallow: /profile
Disallow: /settings
Disallow: /saved
Disallow: /following
Disallow: /foryou
Disallow: /you
Disallow: /scorecard
Disallow: /trackrecord
Disallow: /regsqa
Disallow: /standings-audit
Disallow: /signup
Disallow: /signup/
Disallow: /preferences

# --- AI training crawlers — blocked ----------------------------------------
# These collect corpora for model training and send nobody back.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: TimpiBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: YouBot
Disallow: /

# --- Sitemap ---

Sitemap: https://pressaudit.org/sitemap.xml