# =============================================
# robots.txt - Optimized for WordPress
# =============================================

User-agent: *

# --- Block Admin & Login Areas ---
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php

# --- Block Core WP Files & Directories ---
Disallow: /wp-content/upgrade/
Disallow: /wp-content/backup-db/
Disallow: /wp-content/mu-plugins/

# --- Block Utility & System Pages ---
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /?s=                  # Block search result pages
Disallow: /search/

# --- Block Query Strings (Duplicate Content) ---
Disallow: /*?*replytocom=
Disallow: /*?*attachment_id=

# --- Allow Critical Assets ---
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/plugins/
Allow: /wp-content/themes/
Allow: /wp-content/cache/
Allow: /wp-includes/

# =============================================
# Googlebot Specific
# =============================================

User-agent: Googlebot-Image
Allow: /wp-content/uploads/


# =============================================
# OpenAI Specific
# =============================================

User-agent: OAI-SearchBot 
Allow: / 

User-agent: OAI-AdsBot 
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Google-Extended
Allow: /

# =============================================
# Block Bad Bots
# =============================================

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# =============================================
# Sitemap
# =============================================

Sitemap: https://robstopicopizza.com/sitemap_index.xml