# ==============================================================
# robots.txt — AI Bizname
# Optimized for Maximum Search Engine Traffic
# Last updated: June 2026
# Validator: https://www.google.com/webmasters/tools/robots-testing-tool
# ==============================================================


# --------------------------------------------------------------
# SECTION 1 — Block Bad Bots & Aggressive Scrapers
# (Must be listed FIRST — specific rules override wildcard)
# --------------------------------------------------------------

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Amazonbot
Disallow: /


# --------------------------------------------------------------
# SECTION 2 — Google Bots (Primary Traffic Source)
# --------------------------------------------------------------

User-agent: Googlebot
Allow: /
Allow: /_next/static/
Disallow: /_next/
Disallow: /.next/
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /*?*utm_source=
Disallow: /*?*ref=
Disallow: /*?*session=

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Google-Extended
Allow: /


# --------------------------------------------------------------
# SECTION 3 — Bing / Microsoft Bots
# --------------------------------------------------------------

User-agent: Bingbot
Allow: /
Allow: /_next/static/
Disallow: /_next/
Disallow: /.next/
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Crawl-delay: 1

User-agent: MSNBot
Allow: /

User-agent: MSNBot-Media
Allow: /

User-agent: AdIdxBot
Allow: /

User-agent: BingPreview
Allow: /


# --------------------------------------------------------------
# SECTION 4 — Other Major Search Engines
# --------------------------------------------------------------

User-agent: Slurp
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: YandexImages
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: NaverBot
Allow: /


# --------------------------------------------------------------
# SECTION 5 — AI Bots (Indexing & Training)
# --------------------------------------------------------------

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /


# --------------------------------------------------------------
# SECTION 6 — Default Rule for All Other Bots
# --------------------------------------------------------------

User-agent: *
Allow: /
Allow: /_next/static/
Disallow: /_next/
Disallow: /.next/
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /*?*utm_source=
Disallow: /*?*ref=
Disallow: /*?*session=


# --------------------------------------------------------------
# SECTION 7 — Sitemap
# ==============================================================

Sitemap: https://www.aibizname.com/sitemap.xml
Sitemap: https://www.aibizname.com/sitemap-posts.xml
Sitemap: https://www.aibizname.com/sitemap-categories.xml