# robots.txt — mrc-roofing.com (Martin Roofing & Construction)
# WordPress + Rank Math. Updated 2026-06-26.

# ----------------------------------------------------------
# DEFAULT — all standard crawlers (Googlebot, Bingbot, etc.)
# ----------------------------------------------------------
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# Do NOT disallow /wp-content/ or /wp-includes/ — blocking
# CSS/JS/images breaks rendering for Google and AI crawlers.

# ----------------------------------------------------------
# AI SEARCH & RETRIEVAL CRAWLERS — ALLOW (drive AI visibility)
# ----------------------------------------------------------
# OpenAI
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Google AI (separate from Googlebot; controls Gemini / AI Overviews training)
User-agent: Google-Extended
Allow: /

# Apple (Siri/Spotlight = Applebot; Apple Intelligence training = Applebot-Extended)
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
User-agent: FacebookBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Common Crawl (feeds many open-source LLMs)
User-agent: CCBot
Allow: /

# ----------------------------------------------------------
# BLOCKED — aggressive scrapers / no US AI-search value
# ----------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: cohere-ai
Disallow: /

# ----------------------------------------------------------
# SITEMAP
# ----------------------------------------------------------
Sitemap: https://mrc-roofing.com/sitemap_index.xml