# robots.txt for News Today / نيوز توداي
# Optimized for 2026 SEO + AI Search Engines

# Google Bots
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Storebot-Google
Allow: /

# Google AI / SGE / Gemini
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

User-agent: GoogleOther-Video
Allow: /

# Bing Bots
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: BingPreview
Allow: /

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /

# Bytedance / TikTok
User-agent: Bytespider
Allow: /

# Social Media Bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

# Search Engine Bots
User-agent: YandexBot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# Default for all other bots
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow admin and private pages
Disallow: /admin
Disallow: /admin-auth
Disallow: /*.json$
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /preview/

# Allow important paths
Allow: /article/
Allow: /category/
Allow: /search
Allow: /about
Allow: /contact
Allow: /editorial-policy
Allow: /privacy
Allow: /terms
Allow: /gold-prices-today
Allow: /sports-today
Allow: /tech-crypto-news
Allow: /author/
Allow: /saved
Allow: /search-console
Allow: /sitemap.xml
Allow: /sitemap-news.xml
Allow: /rss
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/llms
Allow: /api/sitemap

# Sitemaps
Sitemap: https://www.newstoday.lol/sitemap.xml
Sitemap: https://www.newstoday.lol/api/sitemap
Sitemap: https://www.newstoday.lol/api/sitemap?type=news

# Host
Host: www.newstoday.lol