# ============================================================
# 🤖 Robots.txt - SEO Friendly + AI Bot Block Version
# Domain: https://matka-tips.org/
# ============================================================

# ==========================
# Allow Major Search Engines
# ==========================
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: Slurp
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Baiduspider
Disallow:

User-agent: YandexBot
Disallow:

# ==========================
# Block AI Crawlers & Data Scrapers
# ==========================
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OpenAI-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

# ==========================
# Default Rules (All Other Bots)
# ==========================
User-agent: *
Disallow: /admin/
Disallow: /includes/
Disallow: /tools/
Disallow: /cgi-bin/
Disallow: /mod/
Disallow: /admins/
Disallow: /demo/
Disallow: /login.php/
Disallow: /_pages/

Allow: /*.css$
Allow: /*.js$
Allow: /images/
Allow: /assets/

# ==========================
# Sitemap
# ==========================
Sitemap: https://matka-tips.org/sitemap.xml
Sitemap: https://matka-tips.org/pages.xml
Sitemap: https://matka-tips.org/page.xml