{
    "title": "AI crawlers in robots.txt: the top 1,000 websites",
    "crawl_date": "2026-10-12",
    "tranco_list": "94GG2",
    "tranco_url": "https://tranco-list.eu/list/94GG2/1000000",
    "user_agent": "SEOpeckBot/1.0 (robots.txt study; +https://seopeck.com/page/robots-txt-study)",
    "sample": {
        "domains_checked": 2364,
        "websites": 1000,
        "skipped": 1364,
        "skipped_reasons": {
            "no_connection": 694,
            "redirects_to_another_domain": 221,
            "homepage_not_2xx": 189,
            "robots_refused": 168,
            "homepage_disallowed_and_not_in_search": 37,
            "homepage_no_connection": 23,
            "robots_server_error": 12,
            "homepage_not_html": 9,
            "timeout": 9,
            "robots_rate_limited": 2
        }
    },
    "robots_fetch": {
        "ok": 890,
        "html_not_robots": 49,
        "not_found": 59,
        "other_4xx": 2
    },
    "robots_alt_host": 72,
    "evaluated": 1000,
    "has_valid_robots_txt": 890,
    "googlebot_blocked": 2,
    "crawlers": {
        "CCBot": {
            "category": "training",
            "named": 170,
            "blocked": 155,
            "blocked_by_name": 129,
            "blocked_by_star": 26,
            "ai_specific_block": 155,
            "partial_by_name": 25
        },
        "Bytespider": {
            "category": "training",
            "named": 148,
            "blocked": 152,
            "blocked_by_name": 124,
            "blocked_by_star": 28,
            "ai_specific_block": 152,
            "partial_by_name": 19
        },
        "GPTBot": {
            "category": "training",
            "named": 195,
            "blocked": 138,
            "blocked_by_name": 121,
            "blocked_by_star": 17,
            "ai_specific_block": 138,
            "partial_by_name": 44
        },
        "ClaudeBot": {
            "category": "training",
            "named": 175,
            "blocked": 131,
            "blocked_by_name": 109,
            "blocked_by_star": 22,
            "ai_specific_block": 131,
            "partial_by_name": 38
        },
        "Meta-ExternalAgent": {
            "category": "training",
            "named": 116,
            "blocked": 118,
            "blocked_by_name": 94,
            "blocked_by_star": 24,
            "ai_specific_block": 118,
            "partial_by_name": 20
        },
        "Google-Extended": {
            "category": "training",
            "named": 162,
            "blocked": 112,
            "blocked_by_name": 95,
            "blocked_by_star": 17,
            "ai_specific_block": 112,
            "partial_by_name": 39
        },
        "PerplexityBot": {
            "category": "search",
            "named": 164,
            "blocked": 105,
            "blocked_by_name": 88,
            "blocked_by_star": 17,
            "ai_specific_block": 105,
            "partial_by_name": 50
        },
        "Amazonbot": {
            "category": "training",
            "named": 104,
            "blocked": 98,
            "blocked_by_name": 74,
            "blocked_by_star": 24,
            "ai_specific_block": 98,
            "partial_by_name": 22
        },
        "Applebot-Extended": {
            "category": "training",
            "named": 104,
            "blocked": 97,
            "blocked_by_name": 73,
            "blocked_by_star": 24,
            "ai_specific_block": 97,
            "partial_by_name": 22
        },
        "ChatGPT-User": {
            "category": "user",
            "named": 145,
            "blocked": 90,
            "blocked_by_name": 74,
            "blocked_by_star": 16,
            "ai_specific_block": 90,
            "partial_by_name": 49
        },
        "Perplexity-User": {
            "category": "user",
            "named": 91,
            "blocked": 84,
            "blocked_by_name": 59,
            "blocked_by_star": 25,
            "ai_specific_block": 84,
            "partial_by_name": 23
        },
        "Claude-User": {
            "category": "user",
            "named": 86,
            "blocked": 79,
            "blocked_by_name": 56,
            "blocked_by_star": 23,
            "ai_specific_block": 79,
            "partial_by_name": 22
        },
        "Claude-SearchBot": {
            "category": "search",
            "named": 89,
            "blocked": 75,
            "blocked_by_name": 52,
            "blocked_by_star": 23,
            "ai_specific_block": 75,
            "partial_by_name": 29
        },
        "OAI-SearchBot": {
            "category": "search",
            "named": 134,
            "blocked": 70,
            "blocked_by_name": 54,
            "blocked_by_star": 16,
            "ai_specific_block": 70,
            "partial_by_name": 53
        },
        "Amzn-SearchBot": {
            "category": "search",
            "named": 23,
            "blocked": 51,
            "blocked_by_name": 19,
            "blocked_by_star": 32,
            "ai_specific_block": 51,
            "partial_by_name": 2
        },
        "Amzn-User": {
            "category": "user",
            "named": 20,
            "blocked": 48,
            "blocked_by_name": 16,
            "blocked_by_star": 32,
            "ai_specific_block": 48,
            "partial_by_name": 3
        },
        "Googlebot": {
            "category": "engine",
            "named": 122,
            "blocked": 2,
            "blocked_by_name": 2,
            "blocked_by_star": 0,
            "ai_specific_block": null,
            "partial_by_name": 94
        },
        "Bingbot": {
            "category": "engine",
            "named": 107,
            "blocked": 7,
            "blocked_by_name": 4,
            "blocked_by_star": 3,
            "ai_specific_block": null,
            "partial_by_name": 73
        }
    },
    "kinds": {
        "blocks_any_ai": 209,
        "blocks_training": 206,
        "blocks_ai_search": 124,
        "blocks_user_triggered": 118,
        "blocks_every_training_crawler": 48,
        "blocks_every_ai_crawler": 20,
        "names_any_ai": 271
    },
    "strategies": {
        "open_to_ai": 789,
        "blocks_all_ai": 110,
        "blocks_training_only": 78,
        "blocks_training_and_search": 11,
        "mixed": 10,
        "blocks_all_crawlers": 2
    },
    "operators": {
        "OpenAI": {
            "training_blocked": 138,
            "training_blocked_search_allowed": 70,
            "search_blocked": 70
        },
        "Anthropic": {
            "training_blocked": 131,
            "training_blocked_search_allowed": 58,
            "search_blocked": 75
        },
        "Perplexity": {
            "training_blocked": null,
            "training_blocked_search_allowed": null,
            "search_blocked": 105
        }
    },
    "tiers": {
        "1-100": {
            "evaluated": 100,
            "blocks_any_ai": 27,
            "blocks_gptbot": 20
        },
        "101-500": {
            "evaluated": 400,
            "blocks_any_ai": 95,
            "blocks_gptbot": 62
        },
        "501-1000": {
            "evaluated": 500,
            "blocks_any_ai": 87,
            "blocks_gptbot": 56
        }
    },
    "mistakes": {
        "legacy_names_listed": {
            "cohere-ai": 115,
            "anthropic-ai": 109,
            "diffbot": 106,
            "claude-web": 94,
            "youbot": 94,
            "omgilibot": 80,
            "timpibot": 74,
            "imagesiftbot": 71,
            "ai2bot": 66,
            "google-cloudvertexbot": 49
        },
        "legacy_only_anthropic": 7,
        "sites_with_ignored_lines": 9,
        "ignored_line_types": {
            "versioned_token": 5,
            "rule_before_user_agent": 4
        },
        "user_fetcher_blocked_search_allowed": {
            "Perplexity": 7,
            "OpenAI": 22,
            "Anthropic": 7
        },
        "html_served_as_robots_txt": 49
    },
    "signals": {
        "content_signal_sites": 30,
        "content_signal_ai_train_no": 15,
        "content_signal_values": {
            "ai-train=yes,search=yes,ai-input=yes": 8,
            "search=yes,ai-input=yes,ai-train=yes": 7,
            "search=yes,ai-input=yes,ai-train=no": 5,
            "ai-train=no,search=yes,ai-input=yes": 5,
            "search=no,ai-input=no,ai-train=no": 2,
            "ai-train=no,search=yes,ai-input=no": 2,
            "ai-train=no,search=yes": 1,
            "search=yes,ai-input=yes": 1,
            "search=yes,ai-train=yes,ai-input=no": 1,
            "search=yes,ai-train=no,ai-input=no": 1,
            "ai-train=yes,ai-input=yes,search=yes": 1,
            "search=yes,ai-train=no,use=reference": 1
        },
        "cloudflare_policy_text": 7,
        "llms_txt": 192
    },
    "sensitivity": {
        "robots_refused": 168,
        "blocks_any_ai_if_refused_counted_open": [
            209,
            1168
        ]
    },
    "homepage_disallowed_for_seopeckbot": 35,
    "opted_out": 0,
    "removed_on_request": 0,
    "llms_txt_not_checked_disallowed": 42,
    "percent_of_evaluated": {
        "blocks_any_ai": 20.9,
        "blocks_training": 20.6,
        "blocks_ai_search": 12.4,
        "blocks_user_triggered": 11.8,
        "blocks_every_ai_crawler": 2,
        "names_any_ai": 27.1,
        "blocks_any_ai_if_refused_counted_open": 17.9,
        "crawlers": {
            "CCBot": {
                "blocked": 15.5,
                "ai_specific_block": 15.5,
                "named": 17
            },
            "Bytespider": {
                "blocked": 15.2,
                "ai_specific_block": 15.2,
                "named": 14.8
            },
            "GPTBot": {
                "blocked": 13.8,
                "ai_specific_block": 13.8,
                "named": 19.5
            },
            "ClaudeBot": {
                "blocked": 13.1,
                "ai_specific_block": 13.1,
                "named": 17.5
            },
            "Meta-ExternalAgent": {
                "blocked": 11.8,
                "ai_specific_block": 11.8,
                "named": 11.6
            },
            "Google-Extended": {
                "blocked": 11.2,
                "ai_specific_block": 11.2,
                "named": 16.2
            },
            "PerplexityBot": {
                "blocked": 10.5,
                "ai_specific_block": 10.5,
                "named": 16.4
            },
            "Amazonbot": {
                "blocked": 9.8,
                "ai_specific_block": 9.8,
                "named": 10.4
            },
            "Applebot-Extended": {
                "blocked": 9.7,
                "ai_specific_block": 9.7,
                "named": 10.4
            },
            "ChatGPT-User": {
                "blocked": 9,
                "ai_specific_block": 9,
                "named": 14.5
            },
            "Perplexity-User": {
                "blocked": 8.4,
                "ai_specific_block": 8.4,
                "named": 9.1
            },
            "Claude-User": {
                "blocked": 7.9,
                "ai_specific_block": 7.9,
                "named": 8.6
            },
            "Claude-SearchBot": {
                "blocked": 7.5,
                "ai_specific_block": 7.5,
                "named": 8.9
            },
            "OAI-SearchBot": {
                "blocked": 7,
                "ai_specific_block": 7,
                "named": 13.4
            },
            "Amzn-SearchBot": {
                "blocked": 5.1,
                "ai_specific_block": 5.1,
                "named": 2.3
            },
            "Amzn-User": {
                "blocked": 4.8,
                "ai_specific_block": 4.8,
                "named": 2
            },
            "Googlebot": {
                "blocked": 0.2,
                "ai_specific_block": null,
                "named": 12.2
            },
            "Bingbot": {
                "blocked": 0.7,
                "ai_specific_block": null,
                "named": 10.7
            }
        },
        "strategies": {
            "open_to_ai": 78.9,
            "blocks_all_ai": 11,
            "blocks_training_only": 7.8,
            "blocks_training_and_search": 1.1,
            "mixed": 1,
            "blocks_all_crawlers": 0.2
        }
    }
}