# Dive With Frank robots.txt

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Light crawl control for compliant bots
Crawl-delay: 5

# Block obvious junk/scanner patterns
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp

# Sitemaps
Sitemap: https://divewithfrank.com/sitemap_index.xml


# --- Specific Bots ---

# Google intentionally uses the global rules above.
# Google does not support Crawl-delay, so no Googlebot-specific group is needed.

# Bing
User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp
Crawl-delay: 5

# GPTBot intentionally uses the global rules above.

# CCBot
User-agent: CCBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp
Crawl-delay: 10

# Bytespider
User-agent: Bytespider
Disallow: /

# Amazonbot
User-agent: Amazonbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp
Crawl-delay: 10

# Anthropic / Claude

# User-requested retrieval
User-agent: Claude-User
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp
Crawl-delay: 5

# Claude search indexing
User-agent: Claude-SearchBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp
Crawl-delay: 5

# General Anthropic crawler
User-agent: ClaudeBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?replytocom=
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /*?amp
Crawl-delay: 5