# robots.txt for https://ngoagency.gov.az

###########################
# Universal Rules
###########################

User-agent: *
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/
Disallow: /test/
Disallow: /dev/
Disallow: /staging/
Disallow: /old/
Disallow: /legacy/

# Prevent crawling/search results/filtered pages (for SEO hygiene; uncomment as needed)
# Disallow: /search
# Disallow: /results
# Disallow: /*?*sort=
# Disallow: /*?*filter=
# Disallow: /*?*page=

# Specify your main sitemap for discoverability
Sitemap: https://ngoagency.gov.az/az/sitemap.xml
Sitemap: https://ngoagency.gov.az/en/sitemap.xml
Sitemap: https://ngoagency.gov.az/ru/sitemap.xml

# (Optional) Crawl-delay for less aggressive crawling by non-Google bots
Crawl-delay: 10

###########################
# Major Search Engines (explicit, for clarity)
###########################

# Googlebot (Google Search)
User-agent: Googlebot
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/

# Googlebot-Image (Google Image Search)
User-agent: Googlebot-Image
Allow: /

# Bingbot (Microsoft Bing)
User-agent: Bingbot
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/

# Baiduspider (China’s largest engine)
User-agent: Baiduspider
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/

# Yandex (Russia’s largest engine)
User-agent: Yandex
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/

# Applebot (Siri & Spotlight)
User-agent: Applebot
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/

# DuckDuckBot (DuckDuckGo search)
User-agent: DuckDuckBot
Allow: /
Disallow: /_next/
Disallow: /api/
Disallow: /server/
Disallow: /private/
Disallow: /tmp/
Disallow: /drafts/

###########################
# AI Crawlers (trusted — allowed to index public content)
###########################

# OpenAI ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /private/

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /private/

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /private/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /private/

# Common Crawl (used by many open LLMs)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /private/

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /private/

###########################
# AI Crawlers (untrusted — blocked)
###########################

# Amazon Alexa
User-agent: Amazonbot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

User-agent: SeekrBot
Disallow: /

###########################
# Blocked Bots (scrapers, SEO tools, non-beneficial)
###########################

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Sogou spider
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: spbot
Disallow: /

###########################
# Optional: Allow only homepage for certain bots (example)
###########################

# User-agent: Pinterestbot
# Allow: /
# Disallow: /_next/
# Disallow: /api/
# Disallow: /server/
# Disallow: /private/
# Disallow: /tmp/
# Disallow: /drafts/
# Disallow: /static/

###########################
# Optional: International & Mobile (uncomment if needed)
###########################

# User-agent: Googlebot-Mobile
# Allow: /
# Disallow: /_next/
# Disallow: /api/
# Disallow: /server/
# Disallow: /private/
# Disallow: /tmp/
# Disallow: /drafts/
# Disallow: /static/

###########################
# Comments/Instructions for developers:
#
# - Always update Disallow rules if you add/remove private folders or test environments
# - Make sure your sitemap is current and reachable
# - Remove 'Crawl-delay' if not needed or adjust per your bandwidth
# - Add 'Noindex' meta tags on private/test pages for double protection
# - Always test robots.txt at https://www.google.com/webmasters/tools/robots-testing-tool
###########################

# END robots.txt