# robots.txt - https://www.wownews24x7.com
#
# Archive articles (published before config('seo.index_from')) use a
# "noindex, follow" meta tag. They stay crawlable so Google can see it.

# ---------------------------------------------------------------------------
# Google Search, Google News, Discover, Images, Video, Ads, Search Console
# Googlebot-News has no separate HTTP UA - it uses Googlebot. The News token
# still belongs here so Publisher Center / news.google.com honour these rules.
# AdsBot / Mediapartners-Google / APIs-Google ignore User-agent: * so they
# must be listed explicitly.
# ---------------------------------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Google-InspectionTool
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Mediapartners-Google
User-agent: APIs-Google
User-agent: FeedFetcher-Google
User-agent: GoogleProducer
User-agent: Google-Safety
Allow: /
Disallow: /admin
Disallow: /administrator
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /user
Disallow: /search
Disallow: /api
Disallow: /load-more
Disallow: /*/load-more
Disallow: /*?app=
Disallow: /*?theme=

# ---------------------------------------------------------------------------
# Other search engines (same public-content policy)
# ---------------------------------------------------------------------------
User-agent: bingbot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: YandexBot
Allow: /
Disallow: /admin
Disallow: /administrator
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /user
Disallow: /search
Disallow: /api
Disallow: /load-more
Disallow: /*/load-more
Disallow: /*?app=
Disallow: /*?theme=

# ---------------------------------------------------------------------------
# AI answer / search engines - fetch pages to cite them in answers ("What is
# WowNews?"). Same public-content policy as search engines. Not model training.
# ---------------------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /administrator
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /user
Disallow: /search
Disallow: /api
Disallow: /load-more
Disallow: /*/load-more
Disallow: /*?app=
Disallow: /*?theme=

# ---------------------------------------------------------------------------
# AI training crawlers - does not affect Google Search ranking.
# Google-Extended is Gemini/Vertex training only, not Search / News / Discover.
# ---------------------------------------------------------------------------
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: GPTBot
User-agent: CCBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Bytespider
User-agent: Amazonbot
Disallow: /

# ---------------------------------------------------------------------------
# Everyone else
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /admin
Disallow: /administrator
Disallow: /login
Disallow: /logout
Disallow: /account
Disallow: /user
Disallow: /search
Disallow: /api
Disallow: /load-more
Disallow: /*/load-more
Disallow: /*?app=
Disallow: /*?theme=

# Sitemaps
Sitemap: https://www.wownews24x7.com/sitemap.xml
Sitemap: https://www.wownews24x7.com/news-sitemap.xml
Sitemap: https://www.wownews24x7.com/sitemap-pages.xml