position,tranco_rank,domain,host,robots_txt_url,http_status,fetch_class,strategy,GPTBot,OAI-SearchBot,ChatGPT-User,ClaudeBot,Claude-SearchBot,Claude-User,Google-Extended,Applebot-Extended,PerplexityBot,Perplexity-User,Meta-ExternalAgent,CCBot,Bytespider,Amazonbot,Amzn-SearchBot,Amzn-User,Googlebot,Bingbot,legacy_names_listed,ignored_lines,inconsistent_rules,content_signal,cloudflare_policy_text,llms_txt,crawl_date
1,1,google.com,www.google.com,https://www.google.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
2,2,cloudflare.com,www.cloudflare.com,https://www.cloudflare.com/robots.txt,200,ok,open_to_ai,"allowed (named)",allowed,"allowed (named)",allowed,allowed,allowed,"allowed (named)",allowed,"allowed (named)",allowed,allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,"anthropic-ai claude-web cohere-ai",,,"ai-train=yes, search=yes, ai-input=yes",no,yes,2026-10-12
3,3,facebook.com,facebook.com,https://facebook.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
4,8,microsoft.com,www.microsoft.com,https://www.microsoft.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
5,9,youtube.com,www.youtube.com,https://www.youtube.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
6,10,apple.com,www.apple.com,https://www.apple.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
7,11,instagram.com,www.instagram.com,https://www.instagram.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
8,16,twitter.com,twitter.com,https://twitter.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
9,18,linkedin.com,www.linkedin.com,https://www.linkedin.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (named)","blocked (named)","blocked (named)","partial (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai diffbot omgilibot timpibot google-cloudvertexbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed; Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,"not checked (disallowed)",2026-10-12
10,23,live.com,outlook.live.com,https://outlook.live.com/mail/,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
11,26,amazon.com,www.amazon.com,https://www.amazon.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
12,29,github.com,github.com,https://github.com/robots.txt,200,ok,blocks_training_only,"partial (named)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",anthropic-ai,,,,no,yes,2026-10-12
13,30,wikipedia.org,en.wikipedia.org,https://en.wikipedia.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
14,32,bing.com,www.bing.com,https://www.bing.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
15,39,cloud.microsoft,m365.cloud.microsoft,https://m365.cloud.microsoft/,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
16,45,netflix.com,www.netflix.com,https://www.netflix.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
17,48,x.com,x.com,https://x.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
18,49,wordpress.org,wordpress.org,https://wordpress.org/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",anthropic-ai,,,,no,yes,2026-10-12
19,50,digicert.com,www.digicert.com,https://www.digicert.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
20,52,icloud.com,www.icloud.com,https://www.icloud.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)",,,,,no,no,2026-10-12
21,53,pinterest.com,www.pinterest.com,https://www.pinterest.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
22,54,tiktok.com,www.tiktok.com,https://www.tiktok.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","anthropic-ai ai2bot",,,,no,no,2026-10-12
23,55,whatsapp.com,www.whatsapp.com,https://www.whatsapp.com/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
24,60,yahoo.com,www.yahoo.com,https://www.yahoo.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (named)","blocked (named)","blocked (named)","partial (named)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
25,62,roblox.com,www.roblox.com,https://www.roblox.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
26,65,adobe.com,www.adobe.com,https://www.adobe.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
27,68,spotify.com,www.spotify.com,https://www.spotify.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
28,70,myfritz.net,myfritz.net,https://myfritz.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,"not checked (disallowed)",2026-10-12
29,72,msn.com,www.msn.com,https://www.msn.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
30,73,ntp.org,www.ntp.org,https://www.ntp.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
31,74,qq.com,www.qq.com,https://www.qq.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
32,76,nginx.org,nginx.org,https://nginx.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
33,80,baidu.com,www.baidu.com,https://www.baidu.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)",,,,,no,"not checked (disallowed)",2026-10-12
34,83,opera.com,www.opera.com,https://www.opera.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
35,87,vimeo.com,vimeo.com,https://vimeo.com/robots.txt,200,ok,blocks_training_only,"partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","anthropic-ai claude-web diffbot imagesiftbot omgilibot",,,,no,no,2026-10-12
36,88,f5.com,www.f5.com,https://www.f5.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
37,94,sentry.io,sentry.io,https://sentry.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"search=yes, ai-input=yes, ai-train=yes",no,yes,2026-10-12
38,96,mozilla.org,www.mozilla.org,https://www.mozilla.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
39,99,yandex.ru,yandex.ru,https://yandex.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
40,100,wordpress.com,wordpress.com,https://wordpress.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
41,104,yuga.com,yuga.com,https://yuga.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
42,106,samsung.com,www.samsung.com,https://www.samsung.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
43,112,nic.ru,nic.ru,https://nic.ru/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
44,113,ui.com,ui.com,https://ui.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,claude-web,,,,no,no,2026-10-12
45,115,europa.eu,europa.eu,https://europa.eu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,no,2026-10-12
46,128,reg.ru,reg.ru,https://reg.ru/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
47,129,snapchat.com,www.snapchat.com,https://www.snapchat.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","blocked (named)","partial (named)","partial (named)","blocked (named)","blocked (named)","partial (named)","partial (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
48,131,apache.org,apache.org,https://apache.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
49,132,dropbox.com,www.dropbox.com,https://www.dropbox.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
50,133,intuit.com,www.intuit.com,https://www.intuit.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
51,138,afternic.com,www.afternic.com,https://www.afternic.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
52,143,criteo.com,criteo.com,https://criteo.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
53,145,triplinkintl.com,www.triplinkintl.com,https://www.triplinkintl.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
54,146,kaspersky.com,www.kaspersky.com,https://www.kaspersky.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web diffbot youbot",,,,no,yes,2026-10-12
55,148,archive.org,archive.org,https://archive.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
56,154,gravatar.com,gravatar.com,https://gravatar.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",omgilibot,,,,no,yes,2026-10-12
57,161,mts.ru,moskva.mts.ru,https://moskva.mts.ru/robotstxt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
58,164,hichina.com,www.hichina.com,https://www.hichina.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
59,166,example.com,example.com,https://example.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
60,169,discord.com,discord.com,https://discord.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
61,170,paypal.com,www.paypal.com,https://www.paypal.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
62,171,shopify.com,shopify.com,https://shopify.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
63,173,dns.google,dns.google,https://dns.google/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
64,178,applovin.com,applovin.com,https://applovin.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
65,183,base.org,www.base.org,https://www.base.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
66,189,w3.org,www.w3.org,https://www.w3.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
67,191,flickr.com,flickr.com,https://flickr.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","partial (named)","partial (named)","blocked (from *)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
68,194,soundcloud.com,soundcloud.com,https://soundcloud.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai",,,,no,no,2026-10-12
69,196,creativecommons.org,creativecommons.org,https://creativecommons.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
70,197,duckdns.org,www.duckdns.org,https://www.duckdns.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
71,199,pki.goog,pki.goog,https://pki.goog/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
72,202,android.com,www.android.com,https://www.android.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai diffbot youbot",,,,no,no,2026-10-12
73,205,cnn.com,www.cnn.com,https://www.cnn.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
74,206,theguardian.com,www.theguardian.com,https://www.theguardian.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai youbot google-cloudvertexbot",,,,no,no,2026-10-12
75,213,t-online.de,www.t-online.de,https://www.t-online.de/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
76,214,miit.gov.cn,www.miit.gov.cn,https://www.miit.gov.cn/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
77,215,doi.org,www.doi.org,https://www.doi.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
78,219,ubuntu.com,ubuntu.com,https://ubuntu.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai",,,,no,yes,2026-10-12
79,221,forbes.com,www.forbes.com,https://www.forbes.com/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","allowed (named)","partial (from *)","blocked (named)","allowed (named)","allowed (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai diffbot imagesiftbot omgilibot",,,,no,no,2026-10-12
80,222,bbc.com,www.bbc.com,https://www.bbc.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot google-cloudvertexbot",,,,no,no,2026-10-12
81,224,ipify.org,www.ipify.org,https://www.ipify.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
82,229,mit.edu,web.mit.edu,https://web.mit.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
83,230,twitch.tv,www.twitch.tv,https://www.twitch.tv/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
84,232,scorecardresearch.com,scorecardresearch.com,https://scorecardresearch.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
85,234,cisco.com,www.cisco.com,https://www.cisco.com/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,yes,2026-10-12
86,239,bbc.co.uk,www.bbc.co.uk,https://www.bbc.co.uk/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot google-cloudvertexbot",,,,no,no,2026-10-12
87,241,stripe.com,stripe.com,https://stripe.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
88,242,nist.gov,www.nist.gov,https://www.nist.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
89,244,taboola.com,www.taboola.com,https://www.taboola.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
90,245,t.co,t.co,https://t.co/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","allowed (named)","allowed (named)",,,,,no,"not checked (disallowed)",2026-10-12
91,246,wikimedia.org,www.wikimedia.org,https://www.wikimedia.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
92,248,sourceforge.net,sourceforge.net,https://sourceforge.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
93,249,alidns.com,alidns.com,https://alidns.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
94,250,cpanel.net,www.cpanel.net,https://www.cpanel.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
95,260,hubspot.com,www.hubspot.com,https://www.hubspot.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
96,261,roku.com,www.roku.com,https://www.roku.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
97,263,samsungcloud.com,www.samsungcloud.com,https://www.samsungcloud.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
98,266,who.int,www.who.int,https://www.who.int/robots.txt,200,ok,blocks_training_only,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"blocked (named)",allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
99,269,imdb.com,www.imdb.com,https://www.imdb.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
100,270,miui.com,www.miui.com,https://www.miui.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
101,272,linktr.ee,linktr.ee,https://linktr.ee/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","allowed (named)","allowed (named)","partial (named)","partial (named)","blocked (from *)","partial (named)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","claude-web google-cloudvertexbot",versioned_token,,"ai-train=no, search=yes",yes,"not checked (disallowed)",2026-10-12
102,277,hcaptcha.com,www.hcaptcha.com,https://www.hcaptcha.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
103,279,forter.com,forter.com,https://forter.com/robots.txt,200,ok,blocks_training_only,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","blocked (named)","blocked (named)","blocked (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai diffbot imagesiftbot omgilibot",,,"search=yes, ai-input=yes, ai-train=yes | search=yes, ai-input=yes | search=no, ai-input=no, ai-train=no | search=yes, ai-input=yes, ai-train=no",no,yes,2026-10-12
104,280,mtgglobals.com,www.mtgglobals.com,https://www.mtgglobals.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
105,286,booking.com,booking.com,https://booking.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
106,287,ibm.com,www.ibm.com,https://www.ibm.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
107,290,reuters.com,www.reuters.com,https://www.reuters.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
108,291,appsflyer.com,www.appsflyer.com,https://www.appsflyer.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
109,293,salesforce.com,www.salesforce.com,https://www.salesforce.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
110,297,tinyurl.com,tinyurl.com,https://tinyurl.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
111,298,springer.com,www.springer.com,https://www.springer.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
112,301,amplitude.com,amplitude.com,https://amplitude.com/robots.txt,200,ok,blocks_training_only,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai cohere-ai",,,"search=yes, ai-input=yes, ai-train=yes | search=no, ai-input=no, ai-train=no",no,yes,2026-10-12
113,303,hp.com,www.hp.com,https://www.hp.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
114,306,nature.com,www.nature.com,https://www.nature.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
115,307,www.gov.uk,www.gov.uk,https://www.gov.uk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
116,309,steampowered.com,store.steampowered.com,https://store.steampowered.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
117,311,weibo.com,weibo.com,https://weibo.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (named)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","allowed (named)","allowed (named)",,,,"search=yes, ai-input=yes, ai-train=no",no,"not checked (disallowed)",2026-10-12
118,312,inmobi.com,www.inmobi.com,https://www.inmobi.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
119,316,ok.ru,ok.ru,https://ok.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
120,317,debian.org,www.debian.org,https://www.debian.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
121,318,harvard.edu,www.harvard.edu,https://www.harvard.edu/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
122,320,alibaba.com,www.alibaba.com,https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
123,325,weather.com,weather.com,https://weather.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","diffbot imagesiftbot omgilibot youbot google-cloudvertexbot",,,,no,yes,2026-10-12
124,328,wiley.com,www.wiley.com,https://www.wiley.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
125,330,php.net,www.php.net,https://www.php.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
126,331,crpt.ru,crpt.ru,https://crpt.ru/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
127,332,dell.com,www.dell.com,https://www.dell.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
128,333,drom.ru,www.drom.ru,https://www.drom.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
129,337,name.com,www.name.com,https://www.name.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
130,340,amazon.co.uk,www.amazon.co.uk,https://www.amazon.co.uk/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
131,342,ip-api.com,ip-api.com,https://ip-api.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
132,344,weebly.com,www.weebly.com,https://www.weebly.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
133,346,gnu.org,www.gnu.org,https://www.gnu.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
134,347,bilibili.com,www.bilibili.com,https://www.bilibili.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
135,349,slack.com,slack.com,https://slack.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
136,350,doubleverify.com,doubleverify.com,https://doubleverify.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
137,352,naver.com,www.naver.com,https://www.naver.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,"not checked (disallowed)",2026-10-12
138,354,trustpilot.com,www.trustpilot.com,https://www.trustpilot.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (named)","partial (named)","blocked (from *)","partial (named)","partial (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","anthropic-ai cohere-ai diffbot imagesiftbot omgilibot youbot timpibot",,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
139,355,ea.com,www.ea.com,https://www.ea.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai ai2bot",,,,no,yes,2026-10-12
140,359,checkpoint.com,www.checkpoint.com,https://www.checkpoint.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",claude-web,,,,no,yes,2026-10-12
141,360,media.net,www.media.net,https://www.media.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
142,361,cdc.gov,www.cdc.gov,https://www.cdc.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
143,362,un.org,www.un.org,https://www.un.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
144,363,selectel.ru,selectel.ru,https://selectel.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
145,368,nasa.gov,www.nasa.gov,https://www.nasa.gov/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
146,369,tradingview.com,www.tradingview.com,https://www.tradingview.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","omgilibot ai2bot google-cloudvertexbot",,,,no,no,2026-10-12
147,372,gitlab.com,gitlab.com,https://gitlab.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
148,373,opendns.com,www.opendns.com,https://www.opendns.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
149,374,zendesk.com,www.zendesk.com,https://www.zendesk.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
150,376,issuu.com,issuu.com,https://issuu.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
151,377,nvidia.com,www.nvidia.com,https://www.nvidia.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=yes, search=yes, ai-input=yes",no,yes,2026-10-12
152,378,tp-link.com,www.tp-link.com,https://www.tp-link.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
153,380,stanford.edu,www.stanford.edu,https://www.stanford.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
154,385,wsj.com,www.wsj.com,https://www.wsj.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",google-cloudvertexbot,,,,no,"not checked (disallowed)",2026-10-12
155,386,datadoghq.com,www.datadoghq.com,https://www.datadoghq.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
156,387,plesk.com,www.plesk.com,https://www.plesk.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
157,391,calendly.com,calendly.com,https://calendly.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
158,396,yandex.com,yandex.com,https://yandex.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
159,398,amazontrust.com,www.amazontrust.com,https://www.amazontrust.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
160,399,amazon.de,www.amazon.de,https://www.amazon.de/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
161,400,bsky.app,bsky.app,https://bsky.app/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
162,402,espn.com,www.espn.com,https://www.espn.com/robots.txt,200,ok,mixed,"blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai omgilibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
163,407,quickconnect.to,quickconnect.to,https://quickconnect.to/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
164,413,globo.com,www.globo.com,https://www.globo.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
165,416,dropcatch.com,www.dropcatch.com,https://www.dropcatch.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
166,417,163.com,www.163.com,https://www.163.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
167,419,line.me,line.me,https://line.me/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
168,420,brave.com,brave.com,https://brave.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
169,423,netease.com,www.netease.com,https://www.netease.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
170,424,go.com,www.go.com,https://www.go.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
171,429,dailymotion.com,www.dailymotion.com,https://www.dailymotion.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
172,430,steamcommunity.com,steamcommunity.com,https://steamcommunity.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
173,432,google.cn,google.cn,https://google.cn/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
174,433,temu.com,www.temu.com,https://www.temu.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
175,436,huawei.com,www.huawei.com,https://www.huawei.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
176,437,supercell.com,supercell.com,https://supercell.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
177,438,duckduckgo.com,duckduckgo.com,https://duckduckgo.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
178,443,taobao.com,www.taobao.com,https://www.taobao.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
179,446,paloaltonetworks.com,www.paloaltonetworks.com,https://www.paloaltonetworks.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
180,448,substack.com,substack.com,https://substack.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
181,451,duolingo.com,www.duolingo.com,https://www.duolingo.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
182,453,vivoglobal.com,vivoglobal.com,https://vivoglobal.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
183,454,aol.com,www.aol.com,https://www.aol.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot",,,,no,no,2026-10-12
184,455,businessinsider.com,www.businessinsider.com,https://www.businessinsider.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai diffbot timpibot",,,,no,no,2026-10-12
185,456,yahoo.co.jp,www.yahoo.co.jp,https://www.yahoo.co.jp/robots.txt,200,ok,blocks_training_only,"blocked (named)",allowed,allowed,allowed,allowed,allowed,"blocked (named)",allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
186,460,cloudns.net,www.cloudns.net,https://www.cloudns.net/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
187,462,arxiv.org,arxiv.org,https://arxiv.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
188,464,launchdarkly.com,launchdarkly.com,https://launchdarkly.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
189,465,walmart.com,www.walmart.com,https://www.walmart.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
190,466,speedtest.net,www.speedtest.net,https://www.speedtest.net/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
191,469,goodreads.com,www.goodreads.com,https://www.goodreads.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",,,,,no,no,2026-10-12
192,473,mikrotik.com,mikrotik.com,https://mikrotik.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
193,480,okta.com,www.okta.com,https://www.okta.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
194,482,cnbc.com,www.cnbc.com,https://www.cnbc.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot",,"Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,no,2026-10-12
195,484,kwai.net,www.kwai.net,https://www.kwai.net/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","allowed (named)","allowed (named)",,,,,no,"not checked (disallowed)",2026-10-12
196,485,statista.com,www.statista.com,https://www.statista.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
197,487,markmonitor.com,www.markmonitor.com,https://www.markmonitor.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
198,488,ietf.org,www.ietf.org,https://www.ietf.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
199,489,sina.com.cn,www.sina.com.cn,https://www.sina.com.cn/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
200,493,sohu.com,www.sohu.com,https://www.sohu.com/robots.txt,200,ok,mixed,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai diffbot youbot google-cloudvertexbot",,,,no,"not checked (disallowed)",2026-10-12
201,498,addtoany.com,www.addtoany.com,https://www.addtoany.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
202,499,amazon.co.jp,www.amazon.co.jp,https://www.amazon.co.jp/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
203,501,cloudflareinsights.com,cloudflareinsights.com,https://cloudflareinsights.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
204,503,shein.com,www.shein.com,https://www.shein.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
205,505,dynatrace.com,www.dynatrace.com,https://www.dynatrace.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
206,506,amazon.co.za,www.amazon.co.za,https://www.amazon.co.za/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
207,508,ipinfo.io,ipinfo.io,https://ipinfo.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
208,509,ivi.ru,www.ivi.ru,https://www.ivi.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
209,511,gosuslugi.ru,www.gosuslugi.ru,https://www.gosuslugi.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,"not checked (disallowed)",2026-10-12
210,513,newrelic.com,newrelic.com,https://newrelic.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
211,514,mediatek.com,www.mediatek.com,https://www.mediatek.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
212,516,docker.com,www.docker.com,https://www.docker.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
213,520,scribd.com,www.scribd.com,https://www.scribd.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web diffbot imagesiftbot ai2bot",,,,no,no,2026-10-12
214,524,indiatimes.com,www.indiatimes.com,https://www.indiatimes.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
215,527,adriver.ru,adriver.ru,https://adriver.ru/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
216,528,foxnews.com,www.foxnews.com,https://www.foxnews.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
217,533,eventbrite.com,eventbrite.com,https://eventbrite.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
218,534,mysql.com,www.mysql.com,https://www.mysql.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
219,537,share.google,share.google,https://share.google/error,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
220,538,tplinkcloud.com,www.tplinkcloud.com,https://www.tplinkcloud.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
221,540,slideshare.net,www.slideshare.net,https://www.slideshare.net/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web diffbot imagesiftbot ai2bot",,,,no,no,2026-10-12
222,543,bitrix24.ru,www.bitrix24.ru,https://www.bitrix24.ru/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
223,547,capcut.com,www.capcut.com,https://www.capcut.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)",allowed,"partial (named)",allowed,allowed,"partial (named)","partial (named)","partial (named)","partial (named)",,,,,no,yes,2026-10-12
224,548,oup.com,global.oup.com,http://global.oup.com/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
225,549,publicnode.com,publicnode.com,https://publicnode.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
226,550,trendmicro.com,www.trendmicro.com,https://www.trendmicro.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","claude-web cohere-ai",,,"search=yes, ai-input=yes, ai-train=no",no,yes,2026-10-12
227,552,squarespace.com,www.squarespace.com,https://www.squarespace.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai omgilibot youbot ai2bot",,,,no,yes,2026-10-12
228,553,blogger.com,www.blogger.com,https://www.blogger.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
229,555,google.com.hk,www.google.com.hk,https://www.google.com.hk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
230,557,firefox.com,www.firefox.com,https://www.firefox.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
231,560,wired.com,www.wired.com,https://www.wired.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot youbot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
232,562,digitalocean.com,www.digitalocean.com,https://www.digitalocean.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
233,563,fast.com,fast.com,https://fast.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
234,564,jquery.com,jquery.com,https://jquery.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
235,565,grammarly.com,www.grammarly.com,https://www.grammarly.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
236,566,openstreetmap.org,www.openstreetmap.org,https://www.openstreetmap.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
237,570,uol.com.br,www.uol.com.br,https://www.uol.com.br/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
238,571,time.com,time.com,https://time.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
239,575,free.fr,www.free.fr,https://www.free.fr/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)",,,,,no,no,2026-10-12
240,576,rakuten.co.jp,www.rakuten.co.jp,https://www.rakuten.co.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
241,577,branch.io,www.branch.io,https://www.branch.io/robots.txt,200,ok,open_to_ai,"allowed (named)",allowed,allowed,"allowed (named)",allowed,allowed,"allowed (named)","partial (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,anthropic-ai,,,,no,yes,2026-10-12
242,580,cornell.edu,www.cornell.edu,https://www.cornell.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
243,581,eset.com,www.eset.com,https://www.eset.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
244,589,dreamhost.com,www.dreamhost.com,https://www.dreamhost.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
245,591,teamviewer.com,www.teamviewer.com,https://www.teamviewer.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
246,592,palmplaystore.com,palmplaystore.com,https://palmplaystore.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
247,599,ca.gov,www.ca.gov,https://www.ca.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
248,601,threads.com,www.threads.com,https://www.threads.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
249,603,braze.com,braze.com,https://braze.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
250,607,klaviyo.com,www.klaviyo.com,https://www.klaviyo.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",youbot,,,,no,yes,2026-10-12
251,608,target.com,www.target.com,https://www.target.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
252,611,avito.ru,www.avito.ru,https://www.avito.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
253,612,anthropic.com,www.anthropic.com,https://www.anthropic.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
254,617,ovh.net,proof.ovh.net,https://proof.ovh.net,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
255,618,ieee.org,www.ieee.org,https://www.ieee.org/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
256,619,atlassian.com,www.atlassian.com,https://www.atlassian.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
257,620,berkeley.edu,www.berkeley.edu,https://www.berkeley.edu/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
258,622,onetrust.com,www.onetrust.com,https://www.onetrust.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
259,627,amazon.fr,www.amazon.fr,https://www.amazon.fr/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
260,629,nintendo.com,www.nintendo.com,https://www.nintendo.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot omgilibot ai2bot timpibot",,,,no,no,2026-10-12
261,630,wix.com,www.wix.com,https://www.wix.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
262,634,ikea.com,www.ikea.com,https://www.ikea.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
263,636,ups.com,www.ups.com,https://www.ups.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
264,638,linode.com,www.linode.com,https://www.linode.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
265,642,cdnvideo.ru,cdnvideo.ru,https://cdnvideo.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
266,643,amazon.ca,www.amazon.ca,https://www.amazon.ca/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
267,644,tencent.com,www.tencent.com,https://www.tencent.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
268,645,nike.com,www.nike.com,https://www.nike.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
269,646,techcrunch.com,techcrunch.com,https://techcrunch.com/robots.txt,200,ok,mixed,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
270,649,jotform.com,www.jotform.com,https://www.jotform.com/robots.txt,200,ok,open_to_ai,"partial (named)","allowed (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","anthropic-ai claude-web cohere-ai diffbot",,,,no,no,2026-10-12
271,651,sagepub.com,www.sagepub.com,https://www.sagepub.com/robots.txt,200,ok,blocks_training_only,"allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","anthropic-ai cohere-ai diffbot imagesiftbot omgilibot",,,,no,no,2026-10-12
272,653,loc.gov,www.loc.gov,https://www.loc.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
273,655,mega.co.nz,mega.co.nz,https://mega.co.nz/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
274,661,shopee.com.br,shopee.com.br,https://shopee.com.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
275,666,stackadapt.com,www.stackadapt.com,https://www.stackadapt.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
276,669,redhat.com,www.redhat.com,https://www.redhat.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
277,670,primevideo.com,www.primevideo.com,https://www.primevideo.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed; Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,no,2026-10-12
278,672,verisign.com,www.verisign.com,https://www.verisign.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
279,673,elpais.com,elpais.com,https://elpais.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)",omgilibot,,,,no,no,2026-10-12
280,675,360.cn,360.cn,https://360.cn/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
281,677,surveymonkey.com,www.surveymonkey.com,https://www.surveymonkey.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
282,679,yelp.com,www.yelp.com,https://www.yelp.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","anthropic-ai claude-web",,,,no,"not checked (disallowed)",2026-10-12
283,681,sophos.com,www.sophos.com,https://www.sophos.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,"allowed (named)",allowed,allowed,allowed,allowed,"allowed (named)",allowed,allowed,"allowed (named)","allowed (named)","anthropic-ai youbot",rule_before_user_agent,,,no,yes,2026-10-12
284,683,optimizely.com,www.optimizely.com,https://www.optimizely.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
285,685,porkbun.com,porkbun.com,https://porkbun.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
286,687,amazon.in,www.amazon.in,https://www.amazon.in/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
287,690,consultant.ru,www.consultant.ru,https://www.consultant.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
288,692,dailymail.com,www.dailymail.com,https://www.dailymail.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
289,694,viber.com,www.viber.com,https://www.viber.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
290,695,outbrain.com,www.outbrain.com,https://www.outbrain.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,no,2026-10-12
291,696,uk.com,uk.com,https://uk.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
292,698,kick.com,kick.com,https://kick.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
293,701,cbsnews.com,www.cbsnews.com,https://www.cbsnews.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
294,702,parklogic.com,parklogic.com,https://parklogic.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
295,703,lemonde.fr,www.lemonde.fr,https://www.lemonde.fr/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
296,707,wps.com,www.wps.com,https://www.wps.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
297,711,fb.com,fb.com,https://fb.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
298,715,ring.com,ring.com,https://ring.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai",,,,no,yes,2026-10-12
299,716,oxylabs.io,oxylabs.io,https://oxylabs.io/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,"search=yes, ai-input=yes, ai-train=yes",no,yes,2026-10-12
300,718,wpguardian.com,wpguardian.com,https://wpguardian.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
301,719,amazon.es,www.amazon.es,https://www.amazon.es/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
302,721,seznam.cz,www.seznam.cz,https://www.seznam.cz/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
303,722,ya.ru,ya.ru,https://ya.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
304,725,itch.io,itch.io,https://itch.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
305,726,independent.co.uk,www.independent.co.uk,https://www.independent.co.uk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
306,729,cnet.com,www.cnet.com,https://www.cnet.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
307,731,uber.com,www.uber.com,https://www.uber.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
308,733,asus.com,www.asus.com,https://www.asus.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
309,734,amazon.it,www.amazon.it,https://www.amazon.it/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
310,738,fontawesome.com,fontawesome.com,https://fontawesome.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
311,739,mailchimp.com,mailchimp.com,https://mailchimp.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
312,741,globalsign.com,www.globalsign.com,https://www.globalsign.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","blocked (named)","partial (named)","partial (from *)","blocked (named)","blocked (named)","partial (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
313,743,bugsnag.com,www.bugsnag.com,https://www.bugsnag.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,yes,no,2026-10-12
314,744,shopee.co.id,shopee.co.id,https://shopee.co.id/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
315,746,imgur.com,imgur.com,https://imgur.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
316,748,airbnb.com,www.airbnb.com,https://www.airbnb.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai cohere-ai imagesiftbot ai2bot",,,,no,no,2026-10-12
317,749,python.org,www.python.org,https://www.python.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
318,757,fda.gov,www.fda.gov,https://www.fda.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
319,758,warnerbros.com,www.warnerbros.com,https://www.warnerbros.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
320,759,meta.com,www.meta.com,https://www.meta.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
321,761,wp.pl,www.wp.pl,https://www.wp.pl/robots.txt,200,ok,blocks_training_only,"blocked (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
322,763,g.co,g.co,https://g.co/robots.txt,400,other_4xx,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
323,766,mediafire.com,mediafire.com,https://mediafire.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
324,769,mlb.com,www.mlb.com,https://www.mlb.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
325,771,gumgum.com,gumgum.com,https://gumgum.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)","allowed (named)",anthropic-ai,,,,no,yes,2026-10-12
326,775,bild.de,www.bild.de,https://www.bild.de/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
327,777,box.com,www.box.com,https://www.box.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
328,783,nikkei.com,www.nikkei.com,https://www.nikkei.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
329,788,sedo.com,sedo.com,https://sedo.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
330,791,bestbuy.com,www.bestbuy.com,https://www.bestbuy.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai claude-web",,,,no,no,2026-10-12
331,792,singular.net,www.singular.net,https://www.singular.net/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",anthropic-ai,,,,no,yes,2026-10-12
332,796,craigslist.org,craigslist.org,https://craigslist.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
333,805,ameblo.jp,ameblo.jp,https://ameblo.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
334,806,welt.de,www.welt.de,https://www.welt.de/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
335,810,mixpanel.com,mixpanel.com,https://mixpanel.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
336,811,prnewswire.com,www.prnewswire.com,https://www.prnewswire.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
337,812,disneyplus.com,www.disneyplus.com,https://www.disneyplus.com/welcome/unavailable,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
338,813,moloco.com,www.moloco.com,https://www.moloco.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
339,814,rt.ru,www.rt.ru,https://www.rt.ru/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
340,815,rakuten.com,www.rakuten.com,https://www.rakuten.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
341,817,xbox.com,www.xbox.com,https://www.xbox.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
342,819,zoho.com,www.zoho.com,https://www.zoho.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
343,820,patreon.com,patreon.com,https://patreon.com/robots.txt,200,ok,mixed,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","timpibot google-cloudvertexbot",,,"search=yes, ai-input=yes, ai-train=no",no,no,2026-10-12
344,826,vivo.com.cn,www.vivo.com.cn,https://www.vivo.com.cn/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
345,827,lefigaro.fr,www.lefigaro.fr,https://www.lefigaro.fr/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot google-cloudvertexbot",,,,no,no,2026-10-12
346,832,ryanair.com,www.ryanair.com,https://www.ryanair.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
347,835,odoo.com,www.odoo.com,https://www.odoo.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
348,838,appcenter.ms,appcenter.ms,https://appcenter.ms/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
349,839,samsungosp.com,www.samsungosp.com,https://www.samsungosp.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
350,841,deepl.com,www.deepl.com,https://www.deepl.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
351,842,usps.com,www.usps.com,https://www.usps.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
352,844,unpkg.com,unpkg.com,https://unpkg.com/robots.txt@1.1.0/index.js,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
353,848,blackberry.com,www.blackberry.com,https://www.blackberry.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
354,849,agoda.com,www.agoda.com,https://www.agoda.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
355,851,academia.edu,www.academia.edu,https://www.academia.edu/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (named)","partial (named)","blocked (from *)","partial (named)","partial (named)","blocked (named)","blocked (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",youbot,,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
356,852,theverge.com,theverge.com,https://theverge.com/robots.txt,200,ok,blocks_all_ai,"allowed (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
357,853,pendo.io,www.pendo.io,https://www.pendo.io/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
358,856,abovedomains.com,abovedomains.com,https://abovedomains.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
359,860,onetag-sys.com,onetag-sys.com,https://onetag-sys.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
360,862,livejournal.com,www.livejournal.com,https://www.livejournal.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
361,863,amazon.com.br,www.amazon.com.br,https://www.amazon.com.br/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
362,866,jetbrains.com,www.jetbrains.com,https://www.jetbrains.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
363,867,amazon.com.au,www.amazon.com.au,https://www.amazon.com.au/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
364,870,daum.net,daum.net,https://daum.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,"not checked (disallowed)",2026-10-12
365,871,figma.com,www.figma.com,https://www.figma.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","blocked (named)","partial (named)","partial (named)","blocked (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai omgilibot",,,,no,no,2026-10-12
366,873,chess.com,www.chess.com,https://www.chess.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
367,874,frontiersin.org,www.frontiersin.org,https://www.frontiersin.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
368,876,cloudinary.com,cloudinary.com,https://cloudinary.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
369,881,www.gov.br,www.gov.br,https://www.gov.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
370,882,synology.com,www.synology.com,https://www.synology.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
371,883,pikabu.ru,pikabu.ru,https://pikabu.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
372,886,kleinanzeigen.de,www.kleinanzeigen.de,https://www.kleinanzeigen.de/robots.txt,200,ok,open_to_ai,"partial (named)","allowed (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
373,888,biblegateway.com,www.biblegateway.com,https://www.biblegateway.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
374,890,hbr.org,hbr.org,https://hbr.org/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
375,894,lwsdns.com,lwsdns.com,https://lwsdns.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
376,896,disqus.com,disqus.com,https://disqus.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
377,897,nbcnews.com,www.nbcnews.com,https://www.nbcnews.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot",,,,no,no,2026-10-12
378,902,worldbank.org,www.worldbank.org,https://www.worldbank.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
379,904,yieldmo.com,yieldmo.com,https://yieldmo.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
380,908,lg.com,www.lg.com,https://www.lg.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
381,910,ys7.com,www.ys7.com,https://www.ys7.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
382,911,binance.com,www.binance.com,https://www.binance.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
383,914,bankofamerica.com,www.bankofamerica.com,https://www.bankofamerica.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
384,917,agora.io,www.agora.io,https://www.agora.io/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
385,918,arcgis.com,www.arcgis.com,https://www.arcgis.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
386,923,thenai.org,thenai.org,https://thenai.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
387,925,latimes.com,www.latimes.com,https://www.latimes.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","diffbot imagesiftbot omgilibot",,,,no,no,2026-10-12
388,927,starlink.com,starlink.com,https://starlink.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
389,929,epa.gov,www.epa.gov,https://www.epa.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
390,931,seedtag.com,www.seedtag.com,https://www.seedtag.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
391,932,qualtrics.com,www.qualtrics.com,https://www.qualtrics.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
392,935,mercadolivre.com.br,www.mercadolivre.com.br,https://www.mercadolivre.com.br/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed; Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,no,2026-10-12
393,937,deloitte.com,www.deloitte.com,https://www.deloitte.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
394,938,genius.com,genius.com,https://genius.com/robots.txt,200,ok,mixed,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai diffbot imagesiftbot omgilibot youbot timpibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
395,942,fidelity.com,www.fidelity.com,https://www.fidelity.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
396,945,smartthings.com,partners.smartthings.com,https://partners.smartthings.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
397,947,youku.com,www.youku.com,https://www.youku.com/index/y404,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
398,948,miwifi.com,www.miwifi.com,https://www.miwifi.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
399,949,mercadolibre.com.ar,www.mercadolibre.com.ar,https://www.mercadolibre.com.ar/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed; Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,no,2026-10-12
400,950,tailscale.com,tailscale.com,https://tailscale.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
401,954,note.com,note.com,https://note.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
402,955,leiniao.com,www.leiniao.com,https://www.leiniao.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
403,956,onet.pl,www.onet.pl,https://www.onet.pl/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","cohere-ai diffbot omgilibot youbot timpibot",,,,no,no,2026-10-12
404,961,otto.de,www.otto.de,https://www.otto.de/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,yes,2026-10-12
405,966,corriere.it,www.corriere.it,https://www.corriere.it/robots.txt,200,ok,mixed,"blocked (named)","allowed (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","allowed (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web youbot timpibot google-cloudvertexbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed","search=yes, ai-train=yes, ai-input=no",no,no,2026-10-12
406,967,webmd.com,www.webmd.com,https://www.webmd.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
407,972,change.org,www.change.org,https://www.change.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
408,974,auvik.com,www.auvik.com,https://www.auvik.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
409,979,rfc-editor.org,www.rfc-editor.org,https://www.rfc-editor.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
410,982,xerox.com,www.xerox.com,https://www.xerox.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
411,985,playrix.com,playrix.com,https://playrix.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
412,986,kwai.com,www.kwai.com,https://www.kwai.com/robots.txt,200,ok,blocks_training_only,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"blocked (named)",allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
413,993,elmundo.es,www.elmundo.es,https://www.elmundo.es/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai omgilibot",,,,no,no,2026-10-12
414,997,heytapmobi.com,heytapmobi.com,https://heytapmobi.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
415,1000,nationalgeographic.com,www.nationalgeographic.com,https://www.nationalgeographic.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
416,1003,poki.com,poki.com,https://poki.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
417,1005,costco.com,www.costco.com,https://www.costco.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
418,1006,shopee.ph,shopee.ph,https://shopee.ph/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
419,1007,flashscore.com,www.flashscore.com,https://www.flashscore.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
420,1008,flipkart.com,www.flipkart.com,https://www.flipkart.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
421,1009,statcounter.com,statcounter.com,https://statcounter.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
422,1010,onesignal.com,onesignal.com,https://onesignal.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
423,1013,healthline.com,www.healthline.com,https://www.healthline.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai diffbot timpibot",,,,no,no,2026-10-12
424,1016,jimdo.com,www.jimdo.com,https://www.jimdo.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot ai2bot google-cloudvertexbot",,,,no,yes,2026-10-12
425,1019,xing.com,www.xing.com,https://www.xing.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
426,1020,spamhaus.org,www.spamhaus.org,https://www.spamhaus.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
427,1023,eu.com,eu.com,https://eu.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
428,1025,youronlinechoices.com,www.youronlinechoices.com,https://www.youronlinechoices.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
429,1027,amazon.com.mx,www.amazon.com.mx,https://www.amazon.com.mx/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
430,1028,nextcloud.com,nextcloud.com,https://nextcloud.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
431,1029,infobae.com,www.infobae.com,https://www.infobae.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
432,1031,orange.fr,www.orange.fr,https://www.orange.fr/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
433,1034,whitehouse.gov,www.whitehouse.gov,https://www.whitehouse.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
434,1035,bund.de,verwaltung.bund.de,https://verwaltung.bund.de/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
435,1041,zhihu.com,www.zhihu.com,https://www.zhihu.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
436,1044,garmin.com,www.garmin.com,https://www.garmin.com/robots.txt,200,ok,mixed,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",omgilibot,,,,no,no,2026-10-12
437,1045,typeform.com,www.typeform.com,https://www.typeform.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
438,1048,kontur.ru,kontur.ru,https://kontur.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)",,,,,no,yes,2026-10-12
439,1052,wise.com,wise.com,https://wise.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
440,1053,n-tv.de,www.n-tv.de,https://www.n-tv.de/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
441,1054,themeforest.net,themeforest.net,https://themeforest.net/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
442,1055,pbs.org,www.pbs.org,https://www.pbs.org/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
443,1056,instructure.com,www.instructure.com,https://www.instructure.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
444,1059,ouest-france.fr,www.ouest-france.fr,https://www.ouest-france.fr/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot youbot timpibot",,,,no,"not checked (disallowed)",2026-10-12
445,1062,adblockplus.org,adblockplus.org,https://adblockplus.org/robots-function,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
446,1065,inkuai.com,www.inkuai.com,https://www.inkuai.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
447,1066,maricopa.gov,www.maricopa.gov,https://www.maricopa.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
448,1068,ameba.jp,www.ameba.jp,https://www.ameba.jp/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"partial (named)",allowed,,,,,no,no,2026-10-12
449,1069,classlink.com,www.classlink.com,https://www.classlink.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
450,1070,netgear.com,www.netgear.com,https://www.netgear.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
451,1072,mioffice.cn,cas.mioffice.cn,https://cas.mioffice.cn/login,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
452,1074,goal.com,www.goal.com,https://www.goal.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
453,1076,archiveofourown.org,archiveofourown.org,https://archiveofourown.org/robots.txt,200,ok,mixed,"blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
454,1077,unrulymedia.com,unrulymedia.com,https://unrulymedia.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
455,1078,terra.com.br,www.terra.com.br,https://www.terra.com.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
456,1079,alphonso.tv,alphonso.tv,https://alphonso.tv/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
457,1080,interia.pl,www.interia.pl,https://www.interia.pl/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
458,1082,redfin.com,www.redfin.com,https://www.redfin.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,no,2026-10-12
459,1084,huffpost.com,www.huffpost.com,https://www.huffpost.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot timpibot",,,,no,no,2026-10-12
460,1086,bfmtv.com,www.bfmtv.com,https://www.bfmtv.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
461,1087,ilovepdf.com,www.ilovepdf.com,https://www.ilovepdf.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
462,1088,crazygames.com,www.crazygames.com,https://www.crazygames.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
463,1090,marca.com,www.marca.com,https://www.marca.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai omgilibot",,,,no,no,2026-10-12
464,1092,nsone.net,my.nsone.net,https://my.nsone.net/,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
465,1096,tabelog.com,tabelog.com,https://tabelog.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",youbot,,,,no,no,2026-10-12
466,1099,safety.google,safety.google,https://safety.google/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
467,1101,kslawin.com,www.kslawin.com,https://www.kslawin.com/robots.txt,200,ok,blocks_training_only,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"blocked (named)",allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
468,1105,aparat.com,www.aparat.com,https://www.aparat.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
469,1108,skysports.com,www.skysports.com,https://www.skysports.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
470,1109,gofundme.com,www.gofundme.com,https://www.gofundme.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
471,1110,huggingface.co,huggingface.co,https://huggingface.co/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
472,1111,riotgames.com,www.riotgames.com,https://www.riotgames.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
473,1113,superhosting.bg,www.superhosting.bg,https://www.superhosting.bg/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
474,1114,nexon.com,www.nexon.com,https://www.nexon.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
475,1116,rustore.ru,www.rustore.ru,https://www.rustore.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
476,1118,aftonbladet.se,www.aftonbladet.se,https://www.aftonbladet.se/robots.txt,200,ok,blocks_training_and_search,"partial (from *)","partial (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
477,1119,gsmarena.com,www.gsmarena.com,https://www.gsmarena.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)","anthropic-ai diffbot imagesiftbot youbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
478,1121,cookpad.com,cookpad.com,https://cookpad.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot ai2bot timpibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
479,1122,finn.no,www.finn.no,https://www.finn.no/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",anthropic-ai,,,,no,no,2026-10-12
480,1123,wetteronline.de,www.wetteronline.de,https://www.wetteronline.de/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
481,1127,alipay.com,alipay.com,https://alipay.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
482,1128,auth0.com,auth0.com,https://auth0.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
483,1132,mercadolibre.com.mx,www.mercadolibre.com.mx,https://www.mercadolibre.com.mx/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed; Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,no,2026-10-12
484,1135,lanacion.com.ar,www.lanacion.com.ar,https://www.lanacion.com.ar/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
485,1141,one.com,www.one.com,https://www.one.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
486,1142,vivo.com,www.vivo.com,https://www.vivo.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
487,1143,locaweb.com.br,www.locaweb.com.br,https://www.locaweb.com.br/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"partial (named)",allowed,,,,,no,no,2026-10-12
488,1144,apa.org,www.apa.org,https://www.apa.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
489,1149,nypost.com,nypost.com,https://nypost.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot",,,,no,no,2026-10-12
490,1150,he.net,he.net,https://he.net/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
491,1153,letsencrypt.org,letsencrypt.org,https://letsencrypt.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
492,1154,hotpepper.jp,www.hotpepper.jp,https://www.hotpepper.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",,,,,no,no,2026-10-12
493,1156,clarin.com,www.clarin.com,https://www.clarin.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
494,1157,clever.com,clever.com,https://clever.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
495,1158,yale.edu,www.yale.edu,https://www.yale.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
496,1159,ultimate-guitar.com,www.ultimate-guitar.com,https://www.ultimate-guitar.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,no,2026-10-12
497,1163,inven.co.kr,www.inven.co.kr,https://www.inven.co.kr/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
498,1165,smilewanted.com,www.smilewanted.com,https://www.smilewanted.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
499,1167,dafont.com,www.dafont.com,https://www.dafont.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
500,1169,plos.org,plos.org,https://plos.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
501,1170,ad.nl,www.ad.nl,https://www.ad.nl/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot youbot timpibot",,,,no,no,2026-10-12
502,1171,theconversation.com,theconversation.com,https://theconversation.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (named)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
503,1177,gismeteo.ru,www.gismeteo.ru,https://www.gismeteo.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
504,1178,tbs.co.jp,www.tbs.co.jp,https://www.tbs.co.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
505,1179,wetter.com,www.wetter.com,https://www.wetter.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
506,1180,jw.org,www.jw.org,https://www.jw.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
507,1181,itmedia.co.jp,www.itmedia.co.jp,https://www.itmedia.co.jp/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
508,1185,upenn.edu,upenn.edu,https://upenn.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
509,1186,nu.nl,www.nu.nl,https://www.nu.nl/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot youbot timpibot",,,,no,no,2026-10-12
510,1196,vg.no,vg.no,https://vg.no/robots.txt,200,ok,blocks_training_and_search,"partial (from *)","partial (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
511,1197,dcinside.com,www.dcinside.com,https://www.dcinside.com/robots.txt,200,ok,blocks_training_and_search,"blocked (named)",allowed,allowed,"blocked (named)",allowed,allowed,"blocked (named)","blocked (named)","blocked (named)",allowed,"blocked (named)","blocked (named)","blocked (named)","blocked (named)",allowed,allowed,allowed,allowed,"anthropic-ai claude-web cohere-ai",,,,no,no,2026-10-12
512,1202,siemens.com,www.siemens.com,https://www.siemens.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
513,1208,julydns.com,www.julydns.com,https://www.julydns.com,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
514,1214,ay267.com,ay267.com,https://ay267.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
515,1216,remove.bg,www.remove.bg,https://www.remove.bg/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)",allowed,,,,,no,no,2026-10-12
516,1218,acesso.gov.br,servicos.acesso.gov.br,https://servicos.acesso.gov.br,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
517,1219,coursera.org,www.coursera.org,https://www.coursera.org/robots.txt,200,ok,blocks_training_only,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (from *)","blocked (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
518,1220,lichess.org,lichess.org,https://lichess.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
519,1221,hostpoint.ch,www.hostpoint.ch,https://www.hostpoint.ch/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
520,1222,tabor.ru,tabor.ru,https://tabor.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
521,1223,rightmove.co.uk,www.rightmove.co.uk,https://www.rightmove.co.uk/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
522,1224,tori.fi,www.tori.fi,https://www.tori.fi/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
523,1227,marktplaats.nl,www.marktplaats.nl,https://www.marktplaats.nl/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
524,1228,pandora.com,pandora.com,https://pandora.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","ai2bot timpibot",,,,no,no,2026-10-12
525,1229,douyin.com,www.douyin.com,https://www.douyin.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)",allowed,allowed,allowed,"partial (named)",allowed,"partial (named)","partial (named)",allowed,allowed,"partial (named)",allowed,allowed,allowed,"partial (named)","partial (named)",,,,,no,no,2026-10-12
526,1230,sectigo.com,www.sectigo.com,https://www.sectigo.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
527,1231,wiktionary.org,en.wiktionary.org,https://en.wiktionary.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
528,1233,gazzetta.it,gazzetta.it,https://gazzetta.it/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",google-cloudvertexbot,,,,no,"not checked (disallowed)",2026-10-12
529,1235,admin.ch,www.admin.ch,https://www.admin.ch/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
530,1236,tradplusad.com,tradplusad.com,https://tradplusad.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
531,1237,digikala.com,www.digikala.com,https://www.digikala.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
532,1238,savefrom.net,savefrom.net,https://savefrom.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
533,1239,amap.com,amap.com,https://amap.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)","partial (named)","blocked (from *)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai youbot",,"Perplexity: Perplexity-User blocked, PerplexityBot allowed",,no,"not checked (disallowed)",2026-10-12
534,1242,rutube.ru,rutube.ru,https://rutube.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
535,1243,ovhcloud.com,www.ovhcloud.com,https://www.ovhcloud.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
536,1248,divar.ir,divar.ir,https://divar.ir/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
537,1250,canada.ca,www.canada.ca,https://www.canada.ca/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
538,1251,hln.be,www.hln.be,https://www.hln.be/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot youbot timpibot",,,,no,no,2026-10-12
539,1252,deepseek.com,deepseek.com,https://deepseek.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
540,1253,furaffinity.net,www.furaffinity.net,https://www.furaffinity.net/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
541,1257,livedoor.com,www.livedoor.com,https://www.livedoor.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
542,1259,us.com,us.com,https://us.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
543,1262,mundodeportivo.com,www.mundodeportivo.com,https://www.mundodeportivo.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
544,1265,abc.net.au,www.abc.net.au,https://www.abc.net.au/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot timpibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
545,1266,livescore.com,www.livescore.com,https://www.livescore.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
546,1272,cdiscount.com,www.cdiscount.com,https://www.cdiscount.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
547,1274,litnet.com,litnet.com,https://litnet.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
548,1275,themoviedb.org,www.themoviedb.org,https://www.themoviedb.org/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)",allowed,allowed,allowed,allowed,"anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
549,1276,dribbble.com,dribbble.com,https://dribbble.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
550,1277,ladepeche.fr,www.ladepeche.fr,https://www.ladepeche.fr/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","allowed (named)","allowed (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot",,,,no,no,2026-10-12
551,1278,psu.edu,www.psu.edu,https://www.psu.edu/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",anthropic-ai,,,"ai-train=yes, search=yes, ai-input=yes",no,no,2026-10-12
552,1281,psychologytoday.com,www.psychologytoday.com,https://www.psychologytoday.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
553,1289,ruliweb.com,bbs.ruliweb.com,https://bbs.ruliweb.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
554,1290,luugy.com,luugy.com,https://luugy.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
555,1291,postgresql.org,www.postgresql.org,https://www.postgresql.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
556,1293,actu.fr,actu.fr,https://actu.fr/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai",,,,no,no,2026-10-12
557,1298,fortune.com,fortune.com,https://fortune.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",omgilibot,,,,no,no,2026-10-12
558,1300,polybuzz.ai,polybuzz.ai,https://polybuzz.ai/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
559,1304,ilfattoquotidiano.it,www.ilfattoquotidiano.it,https://www.ilfattoquotidiano.it/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",,,,,no,no,2026-10-12
560,1305,grow.google,grow.google,https://grow.google/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)",allowed,"allowed (named)","allowed (named)",allowed,"allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,"anthropic-ai cohere-ai diffbot youbot",,,,no,no,2026-10-12
561,1306,kaspi.kz,kaspi.kz,https://kaspi.kz/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
562,1307,rajasthan.gov.in,rajasthan.gov.in,https://rajasthan.gov.in/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
563,1312,hai8g.com,hai8g.com,https://hai8g.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
564,1314,netkeiba.com,netkeiba.com,https://netkeiba.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
565,1318,playhop.com,playhop.com,https://playhop.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
566,1320,labs.google,labs.google,https://labs.google/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
567,1323,meteofrance.com,meteofrance.com,https://meteofrance.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
568,1325,edna.ru,edna.ru,https://edna.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
569,1326,jino.ru,jino.ru,https://jino.ru/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
570,1329,sapo.pt,sapo.pt,https://sapo.pt/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
571,1331,suumo.jp,suumo.jp,https://suumo.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","allowed (named)",,,,,no,no,2026-10-12
572,1332,calculator.net,calculator.net,https://calculator.net/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
573,1333,jstor.org,www.jstor.org,https://www.jstor.org/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
574,1334,jalan.net,www.jalan.net,https://www.jalan.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
575,1338,funpay.com,funpay.com,https://funpay.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
576,1344,irs.gov,www.irs.gov,https://www.irs.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
577,1345,tenki.jp,tenki.jp,https://tenki.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
578,1346,mongodb.com,www.mongodb.com,https://www.mongodb.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,yes,2026-10-12
579,1347,weathernews.jp,weathernews.jp,https://weathernews.jp/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
580,1349,ucla.edu,www.ucla.edu,https://www.ucla.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
581,1350,varzesh3.com,www.varzesh3.com,https://www.varzesh3.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
582,1352,asurascans.com,asurascans.com,https://asurascans.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
583,1354,home.pl,home.pl,https://home.pl/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
584,1359,spiegel.de,www.spiegel.de,https://www.spiegel.de/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
585,1361,turkiye.gov.tr,turkiye.gov.tr,https://turkiye.gov.tr/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
586,1363,oricon.co.jp,www.oricon.co.jp,https://www.oricon.co.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
587,1367,haberler.com,www.haberler.com,https://www.haberler.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
588,1368,sportybet.com,sportybet.com,https://sportybet.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
589,1372,tapad.com,tapad.com,https://tapad.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
590,1373,nikkansports.com,www.nikkansports.com,https://www.nikkansports.com/robots.txt,200,ok,mixed,"blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai omgilibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
591,1375,nesine.com,www.nesine.com,https://www.nesine.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)",anthropic-ai,,,,no,yes,2026-10-12
592,1377,novinky.cz,www.novinky.cz,https://www.novinky.cz/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
593,1378,cardgames.io,cardgames.io,https://cardgames.io/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
594,1380,tgju.org,www.tgju.org,https://www.tgju.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)",,,,,no,no,2026-10-12
595,1382,twilio.com,www.twilio.com,https://www.twilio.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=no, search=yes, ai-input=yes",no,yes,2026-10-12
596,1383,stackexchange.com,stackexchange.com,https://stackexchange.com/robots.txt,418,other_4xx,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
597,1385,mbga.jp,mbga.jp,https://mbga.jp/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
598,1386,dagbladet.no,www.dagbladet.no,https://www.dagbladet.no/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,"search=yes, ai-train=no, ai-input=no",yes,no,2026-10-12
599,1387,okdiario.com,okdiario.com,https://okdiario.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)",,,,,no,no,2026-10-12
600,1392,syosetu.com,syosetu.com,https://syosetu.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)",,versioned_token,,,no,no,2026-10-12
601,1397,ilmeteo.it,www.ilmeteo.it,https://www.ilmeteo.it/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","blocked (named)","partial (named)","partial (named)","partial (named)","blocked (named)","partial (named)","partial (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot timpibot",,,,no,yes,2026-10-12
602,1398,mangabuff.ru,mangabuff.ru,https://mangabuff.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
603,1401,carsensor.net,www.carsensor.net,https://www.carsensor.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
604,1402,filmweb.pl,www.filmweb.pl,https://www.filmweb.pl/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"partial (named)",allowed,,,,,no,no,2026-10-12
605,1403,property24.com,www.property24.com,https://www.property24.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
606,1404,dw.com,www.dw.com,https://www.dw.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",versioned_token,,,no,no,2026-10-12
607,1406,elsevier.com,www.elsevier.com,https://www.elsevier.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
608,1407,inps.it,www.inps.it,https://www.inps.it/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
609,1408,iltalehti.fi,www.iltalehti.fi,https://www.iltalehti.fi/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
610,1410,cronista.com,www.cronista.com,https://www.cronista.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
611,1411,protothema.gr,www.protothema.gr,https://www.protothema.gr/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
612,1412,wordwall.net,wordwall.net,https://wordwall.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
613,1413,zedge.net,www.zedge.net,https://www.zedge.net/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
614,1418,otomoto.pl,www.otomoto.pl,https://www.otomoto.pl/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=yes, search=yes, ai-input=yes",no,no,2026-10-12
615,1419,max.ru,max.ru,https://max.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
616,1420,kernel.org,kernel.org,https://kernel.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
617,1421,cmoa.jp,www.cmoa.jp,https://www.cmoa.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
618,1423,blic.rs,www.blic.rs,https://www.blic.rs/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
619,1426,jutarnji.hr,www.jutarnji.hr,https://www.jutarnji.hr/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
620,1428,24h.com.vn,www.24h.com.vn,https://www.24h.com.vn/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
621,1429,index.hr,www.index.hr,https://www.index.hr/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
622,1431,3bmeteo.com,www.3bmeteo.com,https://www.3bmeteo.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
623,1433,iana.org,www.iana.org,https://www.iana.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
624,1434,loobygamesonline.com,www.loobygamesonline.com,http://www.loobygamesonline.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
625,1436,betway.co.za,www.betway.co.za,https://www.betway.co.za/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
626,1437,omg10.com,omg10.com,https://omg10.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
627,1439,gamewith.jp,gamewith.jp,https://gamewith.jp/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
628,1444,myinstants.com,www.myinstants.com,https://www.myinstants.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
629,1445,jagran.com,www.jagran.com,https://www.jagran.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
630,1446,wisc.edu,www.wisc.edu,https://www.wisc.edu/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
631,1450,gcash.com,gcash.com,https://gcash.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
632,1456,ssstik.io,ssstik.io,https://ssstik.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
633,1460,klikbca.com,www.klikbca.com,https://www.klikbca.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
634,1464,amazon.eg,www.amazon.eg,https://www.amazon.eg/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
635,1467,admanmedia.com,admanmedia.com,https://admanmedia.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
636,1469,rustdesk.com,rustdesk.com,https://rustdesk.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
637,1473,markt.de,markt.de,https://markt.de/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
638,1477,bblaa.com,bblaa.com,https://bblaa.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
639,1478,ptt.cc,www.ptt.cc,https://www.ptt.cc/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
640,1480,eenadu.net,www.eenadu.net,https://www.eenadu.net/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot google-cloudvertexbot",,,,no,no,2026-10-12
641,1482,ninisite.com,www.ninisite.com,https://www.ninisite.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
642,1484,torob.com,torob.com,https://torob.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
643,1491,sondakika.com,www.sondakika.com,https://www.sondakika.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
644,1493,ioh.co.id,ioh.co.id,https://ioh.co.id/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
645,1495,kooora.com,www.kooora.com,https://www.kooora.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
646,1496,simpcity.cr,simpcity.cr,https://simpcity.cr/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
647,1498,crn77.com,crn77.com,https://crn77.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
648,1502,notion.com,www.notion.com,https://www.notion.com/robots.txt,200,ok,blocks_training_and_search,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
649,1503,cam.ac.uk,www.cam.ac.uk,https://www.cam.ac.uk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
650,1504,fedoraproject.org,fedoraproject.org,https://fedoraproject.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
651,1505,dogdrip.net,www.dogdrip.net,https://www.dogdrip.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
652,1516,bt.dk,www.bt.dk,https://www.bt.dk/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
653,1520,programme-tv.net,www.programme-tv.net,https://www.programme-tv.net/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
654,1523,aajtak.in,www.aajtak.in,https://www.aajtak.in/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
655,1527,musicdel.ir,musicdel.ir,https://musicdel.ir/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
656,1529,utorrent.com,www.utorrent.com,https://www.utorrent.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",anthropic-ai,,,,no,yes,2026-10-12
657,1534,kakuyomu.jp,kakuyomu.jp,https://kakuyomu.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
658,1535,indianrail.gov.in,indianrail.gov.in,https://indianrail.gov.in/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
659,1538,tjk.org,www.tjk.org,https://www.tjk.org/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","allowed (named)","blocked (named)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot google-cloudvertexbot",,,,no,"not checked (disallowed)",2026-10-12
660,1540,deezer.com,www.deezer.com,https://www.deezer.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot ai2bot",,,,no,no,2026-10-12
661,1541,uci.edu,uci.edu,https://uci.edu/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
662,1543,rakuten-sec.co.jp,www.rakuten-sec.co.jp,https://www.rakuten-sec.co.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
663,1547,manulife-sinochem.com,www.manulife-sinochem.com,https://www.manulife-sinochem.com/404,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
664,1548,afip.gob.ar,www.afip.gob.ar,https://www.afip.gob.ar/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
665,1549,bikewale.com,www.bikewale.com,https://www.bikewale.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
666,1551,hover.com,www.hover.com,https://www.hover.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=yes, search=yes, ai-input=yes",no,no,2026-10-12
667,1552,snaptik.app,snaptik.app,https://snaptik.app/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
668,1556,ftc.gov,www.ftc.gov,https://www.ftc.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
669,1559,trilltrill.jp,trilltrill.jp,https://trilltrill.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
670,1562,meteociel.fr,meteociel.fr,https://meteociel.fr/robots.txt,200,ok,blocks_training_only,allowed,allowed,allowed,"blocked (named)","partial (named)",allowed,allowed,allowed,allowed,allowed,allowed,allowed,"blocked (named)","partial (named)",allowed,allowed,"partial (named)","partial (named)",,,,,no,no,2026-10-12
671,1563,osym.gov.tr,osym.gov.tr,https://osym.gov.tr/Home/Index,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
672,1565,fastdl.app,fastdl.app,https://fastdl.app/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
673,1566,therapservices.net,www.therapservices.net,https://www.therapservices.net/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"search=yes, ai-input=yes, ai-train=yes",no,yes,2026-10-12
674,1568,verify.hn,verify.hn,https://verify.hn/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
675,1575,buzzfeed.com,www.buzzfeed.com,https://www.buzzfeed.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot timpibot",,,,no,no,2026-10-12
676,1579,oddspark.com,www.oddspark.com,https://www.oddspark.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
677,1581,nya2.com,nya2.com,https://nya2.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
678,1583,screener.in,www.screener.in,https://www.screener.in/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
679,1584,mamastar.jp,mamastar.jp,https://mamastar.jp/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",,,,,no,no,2026-10-12
680,1585,imgsrc.ru,imgsrc.ru,https://imgsrc.ru/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=no, search=yes, ai-input=no",no,no,2026-10-12
681,1586,betika.com,www.betika.com,https://www.betika.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
682,1589,mechacomic.jp,mechacomic.jp,https://mechacomic.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
683,1595,girlschannel.net,girlschannel.net,https://girlschannel.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
684,1597,tupi.fm,www.tupi.fm,https://www.tupi.fm/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)",,,,,no,no,2026-10-12
685,1603,lk21official.cc,tv12.lk21official.cc,https://tv12.lk21official.cc/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
686,1604,ss.com,www.ss.com,https://www.ss.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
687,1606,plex.tv,plex.tv,https://plex.tv/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
688,1607,site.com,site.com,https://site.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
689,1618,rayjump.com,www.rayjump.com,https://www.rayjump.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
690,1620,bakusai.com,bakusai.com,https://bakusai.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
691,1621,jra.go.jp,www.jra.go.jp,https://www.jra.go.jp/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
692,1626,alphapolis.co.jp,www.alphapolis.co.jp,https://www.alphapolis.co.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","allowed (named)",,,,,no,yes,2026-10-12
693,1627,piccoma.com,piccoma.com,https://piccoma.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
694,1634,tiktokio.com,tiktokio.com,https://tiktokio.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
695,1635,vinted.fr,www.vinted.fr,https://www.vinted.fr/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=no, search=yes, ai-input=yes | ai-train=no, search=yes, ai-input=no",yes,no,2026-10-12
696,1637,ey43.com,ey43.com,https://ey43.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
697,1645,goojara.to,ww1.goojara.to,https://ww1.goojara.to/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,yes,no,2026-10-12
698,1646,flashscore.fr,www.flashscore.fr,https://www.flashscore.fr/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
699,1649,an1.com,an1.com,https://an1.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
700,1650,smadex.com,smadex.com,https://smadex.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
701,1655,wpastra.com,wpastra.com,https://wpastra.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=no, search=yes, ai-input=yes",no,yes,2026-10-12
702,1658,jra.jp,jra.jp,https://jra.jp/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
703,1661,kemkes.go.id,kemkes.go.id,https://kemkes.go.id/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
704,1664,xosodaiphat.com,xosodaiphat.com,https://xosodaiphat.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,rule_before_user_agent,,,no,no,2026-10-12
705,1666,rambler.ru,www.rambler.ru,https://www.rambler.ru/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
706,1672,flashscore.mobi,www.flashscore.mobi,https://www.flashscore.mobi/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
707,1676,goal7.co,goal7.co,https://goal7.co/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
708,1678,ecnavi.jp,ecnavi.jp,https://ecnavi.jp/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
709,1680,betking.com,www.betking.com,https://www.betking.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
710,1683,tjsp.jus.br,www.tjsp.jus.br,https://www.tjsp.jus.br/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,rule_before_user_agent,,,no,no,2026-10-12
711,1685,livescore.cz,www.livescore.cz,https://www.livescore.cz/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
712,1687,sarkariresult.com.cm,sarkariresult.com.cm,https://sarkariresult.com.cm/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
713,1688,turbo.az,turbo.az,https://turbo.az/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
714,1689,eff.org,www.eff.org,https://www.eff.org/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",anthropic-ai,,,,no,no,2026-10-12
715,1694,betpawa.ug,www.betpawa.ug,https://www.betpawa.ug/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai",,,,no,no,2026-10-12
716,1696,bancodevenezuela.com,www.bancodevenezuela.com,https://www.bancodevenezuela.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
717,1697,espn.com.ve,www.espn.com.ve,https://www.espn.com.ve/robots.txt,200,ok,mixed,"blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai omgilibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
718,1698,humoruniv.com,www.humoruniv.com,https://www.humoruniv.com/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
719,1700,choiceadvantage.com,www.choiceadvantage.com,https://www.choiceadvantage.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
720,1704,xsmn.mobi,xsmn.mobi,https://xsmn.mobi/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
721,1706,minhngoc.net.vn,www.minhngoc.net.vn,https://www.minhngoc.net.vn/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
722,1708,thscore99.com,www.thscore99.com,https://www.thscore99.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,"not checked (disallowed)",2026-10-12
723,1711,xoso.com.vn,xoso.com.vn,https://xoso.com.vn/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
724,1712,packzy.com,packzy.com,https://packzy.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
725,1715,kqbd.mobi,kqbd.mobi,https://kqbd.mobi/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
726,1716,mobiledokan.com,www.mobiledokan.com,https://www.mobiledokan.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
727,1723,albumaty.com,www.albumaty.com,https://www.albumaty.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
728,1724,az24.vn,az24.vn,https://az24.vn/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
729,1725,ahacdn.me,ahacdn.me,https://ahacdn.me/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
730,1727,linkmake.in,linkmake.in,https://linkmake.in/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,yes,no,2026-10-12
731,1729,pitc.com.pk,pitc.com.pk,https://pitc.com.pk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
732,1736,cbl.gov.ly,cbl.gov.ly,https://cbl.gov.ly/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
733,1737,ico.org.uk,ico.org.uk,https://ico.org.uk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
734,1739,loteriadehoy.com,loteriadehoy.com,https://loteriadehoy.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
735,1746,newtoki1.org,newtoki1.org,https://newtoki1.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
736,1752,33apk.com,33apk.com,https://33apk.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
737,1753,staseraintv.com,staseraintv.com,https://staseraintv.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
738,1755,desitelugusex.com,desitelugusex.com,https://desitelugusex.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
739,1757,xn--12c1ezaww.com,xn--12c1ezaww.com,https://xn--12c1ezaww.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
740,1758,xn--42cf7cgd7gxbd4m7c.cc,www.xn--42cf7cgd7gxbd4m7c.cc,https://www.xn--42cf7cgd7gxbd4m7c.cc/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
741,1760,mobcup.fm,mobcup.fm,https://mobcup.fm/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
742,1761,sambad.com,sambad.com,https://sambad.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
743,1762,sexmupxinh.net,sexmupxinh.net,https://sexmupxinh.net/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
744,1765,yasyadong02.tv,yasyadong02.tv,https://yasyadong02.tv/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
745,1767,mangaraw.ac,mangaraw.ac,https://mangaraw.ac/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
746,1769,ad-contents.jp,ad-contents.jp,https://ad-contents.jp/robots.txt,200,ok,blocks_all_crawlers,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"blocked (named)","blocked (named)",,,,,no,no,2026-10-12
747,1771,playinboss.com,playinboss.com,https://playinboss.com/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
748,1772,sattamatkano1.me,sattamatkano1.me,https://sattamatkano1.me/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
749,1773,xvideo.vlog.br,xvideo.vlog.br,https://xvideo.vlog.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
750,1774,hostinger.com,www.hostinger.com,https://www.hostinger.com/robots.txt,200,ok,blocks_training_only,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","youbot google-cloudvertexbot",,,"search=yes, ai-input=yes, ai-train=no",no,yes,2026-10-12
751,1776,kolkataff.tv,kolkataff.tv,https://kolkataff.tv/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
752,1778,moe.video,moe.video,https://moe.video/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,yes,2026-10-12
753,1784,enable-javascript.com,www.enable-javascript.com,https://www.enable-javascript.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
754,1786,hik-connect.com,www.hik-connect.com,https://www.hik-connect.com/views/login/index.html,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
755,1788,akismet.com,akismet.com,https://akismet.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
756,1789,blizzard.com,www.blizzard.com,https://www.blizzard.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
757,1790,cmu.edu,www.cmu.edu,https://www.cmu.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
758,1798,fao.org,www.fao.org,https://www.fao.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
759,1799,overwolf.com,www.overwolf.com,https://www.overwolf.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
760,1800,loopia.se,www.loopia.se,https://www.loopia.se/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
761,1801,tawk.to,www.tawk.to,https://www.tawk.to/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
762,1804,intercom.com,www.intercom.com,https://www.intercom.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=yes, ai-input=yes, search=yes",no,yes,2026-10-12
763,1806,amocrm.ru,www.amocrm.ru,https://www.amocrm.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
764,1807,wayground.com,wayground.com,https://wayground.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
765,1808,googleapis.cn,www.googleapis.cn,https://www.googleapis.cn/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
766,1809,ixl.com,www.ixl.com,https://www.ixl.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
767,1810,kargo.com,kargo.com,https://kargo.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai omgilibot youbot ai2bot",,,,no,no,2026-10-12
768,1815,beeline.ru,www.beeline.ru,https://www.beeline.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
769,1817,merkur.de,merkur.de,https://merkur.de/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","partial (named)","partial (named)","blocked (from *)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",google-cloudvertexbot,,,,no,"not checked (disallowed)",2026-10-12
770,1819,contextweb.com,contextweb.com,https://contextweb.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
771,1820,bitdefender.com,www.bitdefender.com,https://www.bitdefender.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=yes, search=yes, ai-input=yes",no,yes,2026-10-12
772,1822,vmware.com,www.vmware.com,https://www.vmware.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
773,1824,newsweek.com,www.newsweek.com,https://www.newsweek.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
774,1825,w3schools.com,www.w3schools.com,https://www.w3schools.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
775,1826,waze.com,www.waze.com,https://www.waze.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
776,1827,gamebanana.com,gamebanana.com,https://gamebanana.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
777,1828,medallia.com,www.medallia.com,https://www.medallia.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
778,1829,acs.org,www.acs.org,https://www.acs.org/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
779,1830,fc2.com,fc2.com,https://fc2.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
780,1831,fedex.com,www.fedex.com,https://www.fedex.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
781,1833,newyorker.com,www.newyorker.com,https://www.newyorker.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot youbot timpibot google-cloudvertexbot",,,,no,no,2026-10-12
782,1834,rapid7.com,www.rapid7.com,https://www.rapid7.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
783,1836,trello.com,trello.com,https://trello.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
784,1841,hola.com,www.hola.com,https://www.hola.com/robots.txt,200,ok,blocks_all_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai omgilibot",,,,no,no,2026-10-12
785,1843,1c.ru,1c.ru,https://1c.ru/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)",,,,,no,no,2026-10-12
786,1845,va.gov,www.va.gov,https://www.va.gov/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
787,1849,trustarc.com,trustarc.com,https://trustarc.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
788,1850,pewresearch.org,www.pewresearch.org,https://www.pewresearch.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,"ai-train=yes, search=yes, ai-input=yes",no,yes,2026-10-12
789,1852,cloud.com,www.cloud.com,https://www.cloud.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
790,1853,pluto.tv,pluto.tv,https://pluto.tv/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
791,1855,wps.cn,www.wps.cn,https://www.wps.cn/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
792,1856,auone.jp,portal.auone.jp,https://portal.auone.jp,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
793,1857,zdnet.com,www.zdnet.com,https://www.zdnet.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
794,1858,dantri.com.vn,dantri.com.vn,https://dantri.com.vn/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","cohere-ai omgilibot",,,,no,yes,2026-10-12
795,1859,astrumu.io,astrumu.io,https://astrumu.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
796,1864,sciencedaily.com,sciencedaily.com,https://sciencedaily.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
797,1866,nps.gov,www.nps.gov,https://www.nps.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",,,,,no,no,2026-10-12
798,1868,iscorp.com,www.iscorp.com,https://www.iscorp.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
799,1871,yimg.com,yimg.com,https://yimg.com/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
800,1872,sweb.ru,sweb.ru,https://sweb.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
801,1875,pusher.com,pusher.com,https://pusher.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
802,1878,ucsd.edu,ucsd.edu,https://ucsd.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
803,1879,cpmstar.com,www.cpmstar.com,https://www.cpmstar.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
804,1880,gdemoideti.ru,gdemoideti.ru,https://gdemoideti.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
805,1885,marketwatch.com,www.marketwatch.com,https://www.marketwatch.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",google-cloudvertexbot,,,,no,"not checked (disallowed)",2026-10-12
806,1886,sky.com,www.sky.com,https://www.sky.com/robots.txt,200,ok,open_to_ai,"partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
807,1890,uchicago.edu,www.uchicago.edu,https://www.uchicago.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
808,1893,vlr.gg,www.vlr.gg,https://www.vlr.gg/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
809,1895,russianfood.com,www.russianfood.com,https://www.russianfood.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
810,1896,elegantthemes.com,www.elegantthemes.com,https://www.elegantthemes.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
811,1899,scientificamerican.com,www.scientificamerican.com,https://www.scientificamerican.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai cohere-ai",,,,no,no,2026-10-12
812,1901,beget.com,beget.com,https://beget.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
813,1902,utexas.edu,www.utexas.edu,https://www.utexas.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
814,1908,mega.nz,mega.nz,https://mega.nz/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,"not checked (disallowed)",2026-10-12
815,1911,1password.com,1password.com,https://1password.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
816,1917,clevelandclinic.org,my.clevelandclinic.org,https://my.clevelandclinic.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
817,1918,eltiempo.es,www.eltiempo.es,https://www.eltiempo.es/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
818,1919,scryfall.com,scryfall.com,https://scryfall.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
819,1920,xinhuanet.com,www.xinhuanet.com,https://www.xinhuanet.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
820,1927,orcid.org,orcid.org,https://orcid.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
821,1928,vice.com,www.vice.com,https://www.vice.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
822,1931,sprinthost.ru,sprinthost.ru,https://sprinthost.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
823,1933,usgs.gov,www.usgs.gov,https://www.usgs.gov/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
824,1934,clickpetroleoegas.com.br,clickpetroleoegas.com.br,https://clickpetroleoegas.com.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
825,1938,alura.com.br,www.alura.com.br,https://www.alura.com.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
826,1940,caliente.mx,www.caliente.mx,https://www.caliente.mx/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
827,1942,www.nhs.uk,www.nhs.uk,https://www.nhs.uk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
828,1944,flashscore.com.ng,www.flashscore.com.ng,https://www.flashscore.com.ng/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
829,1948,www.gob.mx,www.gob.mx,https://www.gob.mx/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)",allowed,,,,,no,no,2026-10-12
830,1949,diretta.it,www.diretta.it,https://www.diretta.it/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
831,1950,mapbox.com,www.mapbox.com,https://www.mapbox.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
832,1951,samsungknox.com,www.samsungknox.com,https://www.samsungknox.com/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
833,1953,iqiyi.com,www.iqiyi.com,http://www.iqiyi.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",cohere-ai,,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
834,1954,sport24.gr,sport24.gr,https://sport24.gr/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
835,1958,deepintent.com,deepintent.com,https://deepintent.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
836,1959,git-scm.com,git-scm.com,https://git-scm.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
837,1962,wikihow.com,www.wikihow.com,https://www.wikihow.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot",,,,no,no,2026-10-12
838,1964,easytrader.ir,easytrader.ir,https://easytrader.ir/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
839,1965,oreilly.com,www.oreilly.com,https://www.oreilly.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai claude-web cohere-ai diffbot youbot ai2bot timpibot",versioned_token,,,no,yes,2026-10-12
840,1966,iubenda.com,www.iubenda.com,https://www.iubenda.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
841,1967,hetzner.com,www.hetzner.com,https://www.hetzner.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
842,1968,nontondrama.my,tv9.nontondrama.my,https://tv9.nontondrama.my/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
843,1970,football.com,www.football.com,https://www.football.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
844,1973,takarakuji-official.jp,www.takarakuji-official.jp,https://www.takarakuji-official.jp/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
845,1976,themeisle.com,themeisle.com,https://themeisle.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
846,1980,mercadolibre.com,mercadolibre.com,https://mercadolibre.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed; Anthropic: Claude-User blocked, Claude-SearchBot allowed",,no,no,2026-10-12
847,1984,moengage.com,www.moengage.com,https://www.moengage.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
848,1988,bigcommerce.com,www.bigcommerce.com,https://www.bigcommerce.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
849,1989,betway.co.bw,www.betway.co.bw,https://www.betway.co.bw/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
850,1993,av19t.com,av19t.com,https://av19t.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
851,1994,xn--l3ca4b6dsc1b.com,xn--l3ca4b6dsc1b.com,https://xn--l3ca4b6dsc1b.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
852,1999,thenkiri.com,thenkiri.com,https://thenkiri.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
853,2002,tvmon1.com,tvmon1.com,https://tvmon1.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
854,2005,i993.com,i993.com,https://i993.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
855,2006,educationportal3.in,educationportal3.in,https://educationportal3.in/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
856,2012,euronews.com,www.euronews.com,https://www.euronews.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",youbot,,,,no,no,2026-10-12
857,2020,wa.link,wa.link,https://wa.link/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
858,2022,mindbox.ru,mindbox.ru,https://mindbox.ru/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (named)","allowed (named)","anthropic-ai claude-web cohere-ai youbot ai2bot",versioned_token,,,no,yes,2026-10-12
859,2023,bootstrapcdn.com,www.bootstrapcdn.com,https://www.bootstrapcdn.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
860,2025,strava.com,www.strava.com,https://www.strava.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
861,2028,disdarrummers.shop,disdarrummers.shop,https://disdarrummers.shop/robots.txt,200,ok,blocks_all_crawlers,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)",,,,,no,no,2026-10-12
862,2032,richaudience.com,richaudience.com,https://richaudience.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
863,2038,redditstatic.com,www.redditstatic.com,https://www.redditstatic.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","allowed (named)","allowed (named)",,,,,no,"not checked (disallowed)",2026-10-12
864,2041,sec.gov,www.sec.gov,https://www.sec.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
865,2045,rackspace.com,www.rackspace.com,https://www.rackspace.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
866,2047,techtarget.com,www.techtarget.com,https://www.techtarget.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,no,2026-10-12
867,2049,hdhub4u.bi,hdhub4u.bi,https://hdhub4u.bi/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
868,2050,dondominio.com,www.dondominio.com,https://www.dondominio.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
869,2051,sedoparking.com,sedoparking.com,https://sedoparking.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
870,2052,wpengine.com,wpengine.com,https://wpengine.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
871,2054,squareup.com,squareup.com,https://squareup.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
872,2055,videeyss.shop,videeyss.shop,https://videeyss.shop/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
873,2059,meetup.com,www.meetup.com,https://www.meetup.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
874,2085,fifa.com,www.fifa.com,https://www.fifa.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
875,2089,logitech.com,www.logitech.com,https://www.logitech.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
876,2091,freebsd.org,www.freebsd.org,https://www.freebsd.org/robots.txt,200,ok,open_to_ai,"partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
877,2095,sogou.com,www.sogou.com,https://www.sogou.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","blocked (from *)",,,,,no,"not checked (disallowed)",2026-10-12
878,2096,intentiq.com,www.intentiq.com,https://www.intentiq.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
879,2101,hh.ru,hh.ru,https://hh.ru/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)","anthropic-ai claude-web",,,,no,no,2026-10-12
880,2105,romarg.com,romarg.com,https://romarg.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
881,2106,umeng.com,www.umeng.com,https://www.umeng.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,"allowed (named)",allowed,allowed,allowed,"allowed (named)",allowed,allowed,allowed,"allowed (named)","allowed (named)",anthropic-ai,,,,no,yes,2026-10-12
882,2108,iponweb.net,iam.iponweb.net,https://iam.iponweb.net/auth/realms/iponweb/protocol/openid-connect/auth,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
883,2109,apple.news,apple.news,https://apple.news/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
884,2113,wechat.com,www.wechat.com,https://www.wechat.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
885,2115,getbootstrap.com,getbootstrap.com,https://getbootstrap.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
886,2120,wunderground.com,www.wunderground.com,https://www.wunderground.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
887,2121,ign.com,www.ign.com,https://www.ign.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot",,,,no,no,2026-10-12
888,2123,transip.nl,www.transip.nl,https://www.transip.nl/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
889,2125,bls.gov,www.bls.gov,https://www.bls.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
890,2126,matterport.com,matterport.com,https://matterport.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
891,2128,gizmodo.com,gizmodo.com,https://gizmodo.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","cohere-ai diffbot imagesiftbot omgilibot ai2bot timpibot",,,,no,no,2026-10-12
892,2129,salesforceliveagent.com,salesforceliveagent.com,https://salesforceliveagent.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","allowed (named)","blocked (from *)",,,,,no,"not checked (disallowed)",2026-10-12
893,2133,broadcom.com,www.broadcom.com,https://www.broadcom.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
894,2139,acronis.com,www.acronis.com,https://www.acronis.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
895,2142,automattic.com,automattic.com,https://automattic.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
896,2146,bitbucket.org,bitbucket.org,https://bitbucket.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
897,2153,eskimi.com,www.eskimi.com,https://www.eskimi.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
898,2155,sonobi.com,sonobi.com,https://sonobi.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
899,2156,revenuecat.com,www.revenuecat.com,https://www.revenuecat.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
900,2158,cafe24.com,www.cafe24.com,https://www.cafe24.com/,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
901,2161,snowflake.com,www.snowflake.com,https://www.snowflake.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
902,2162,trip.com,www.trip.com,https://www.trip.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
903,2163,evernote.com,evernote.com,https://evernote.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)",,,,,no,yes,2026-10-12
904,2164,blog.google,blog.google,https://blog.google/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
905,2168,primis.tech,www.primis.tech,https://www.primis.tech/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
906,2169,meitu.com,www.meitu.com,https://www.meitu.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
907,2170,yotpo.com,www.yotpo.com,https://www.yotpo.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","allowed (named)",anthropic-ai,,,"search=yes, ai-input=yes, ai-train=yes",no,yes,2026-10-12
908,2171,ahrefs.com,ahrefs.com,https://ahrefs.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
909,2174,nic.io,nic.io,https://nic.io/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
910,2178,mashable.com,mashable.com,https://mashable.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot youbot ai2bot timpibot",,,,no,no,2026-10-12
911,2180,aniview.com,aniview.com,https://aniview.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
912,2181,ticketmaster.com,www.ticketmaster.com,https://www.ticketmaster.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,no,2026-10-12
913,2182,unicef.org,www.unicef.org,https://www.unicef.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
914,2184,asu.edu,www.asu.edu,https://www.asu.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
915,2189,thesun.co.uk,www.thesun.co.uk,https://www.thesun.co.uk/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
916,2191,metamask.io,metamask.io,https://metamask.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=yes, search=yes, ai-input=yes",no,yes,2026-10-12
917,2193,ufl.edu,www.ufl.edu,https://www.ufl.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
918,2201,datto.com,www.datto.com,https://www.datto.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
919,2203,shodan.io,www.shodan.io,https://www.shodan.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
920,2204,house.gov,www.house.gov,https://www.house.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
921,2205,trustedstack.com,trustedstack.com,https://trustedstack.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
922,2208,bytedance.com,bytedance.com,https://bytedance.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
923,2210,posthog.com,posthog.com,https://posthog.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"partial (named)","partial (named)",,,,,no,yes,2026-10-12
924,2211,aboutcookies.org,www.aboutcookies.org,https://www.aboutcookies.org/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
925,2212,av380.net,av380.net,https://av380.net/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
926,2215,proofpoint.com,www.proofpoint.com,https://www.proofpoint.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","diffbot ai2bot",,,,no,no,2026-10-12
927,2216,nubank.com.br,nubank.com.br,https://nubank.com.br/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
928,2220,sap.com,www.sap.com,https://www.sap.com/robots.txt,200,ok,blocks_all_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","blocked (from *)","partial (named)","partial (named)","blocked (from *)","partial (named)","blocked (from *)","partial (named)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
929,2222,unity.com,unity.com,https://unity.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
930,2225,mirtesen.ru,mirtesen.ru,https://mirtesen.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (from *)",,,,,no,no,2026-10-12
931,2230,mmcdn.com,mmcdn.com,https://mmcdn.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
932,2232,engadget.com,engadget.com,https://engadget.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
933,2233,adkernel.com,adkernel.com,https://adkernel.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,yes,2026-10-12
934,2236,americanexpress.com,www.americanexpress.com,https://www.americanexpress.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
935,2237,census.gov,www.census.gov,https://www.census.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
936,2238,mparticle.com,www.mparticle.com,https://www.mparticle.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot",,,,no,no,2026-10-12
937,2241,nodejs.org,nodejs.org,https://nodejs.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
938,2242,medicalnewstoday.com,www.medicalnewstoday.com,https://www.medicalnewstoday.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","anthropic-ai diffbot timpibot",,,,no,no,2026-10-12
939,2243,didiglobal.com,didiglobal.com,https://didiglobal.com/404,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
940,2245,itu.int,www.itu.int,https://www.itu.int/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
941,2246,ay.delivery,ay.delivery,https://ay.delivery/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,yes,no,2026-10-12
942,2247,gallup.com,www.gallup.com,https://www.gallup.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",cohere-ai,,,,no,no,2026-10-12
943,2248,semrush.com,www.semrush.com,https://www.semrush.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)",,,,,no,yes,2026-10-12
944,2249,asana.com,asana.com,https://asana.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
945,2250,mgid.com,www.mgid.com,https://www.mgid.com/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (from *)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","partial (from *)","allowed (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web diffbot imagesiftbot omgilibot youbot",,,,no,yes,2026-10-12
946,2253,freefiremobile.com,www.freefiremobile.com,https://www.freefiremobile.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
947,2255,justice.gov,www.justice.gov,https://www.justice.gov/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
948,2259,variety.com,variety.com,https://variety.com/robots.txt,200,ok,blocks_all_ai,"blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot omgilibot youbot timpibot",,,,no,no,2026-10-12
949,2260,mirror.co.uk,www.mirror.co.uk,https://www.mirror.co.uk/robots.txt,200,ok,blocks_training_and_search,"blocked (named)","blocked (named)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","anthropic-ai claude-web youbot",,,,no,no,2026-10-12
950,2262,verizon.com,verizon.com,https://verizon.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
951,2263,duke.edu,www.duke.edu,https://www.duke.edu/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
952,2264,laravel.com,laravel.com,https://laravel.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"ai-train=no, search=yes, ai-input=yes",no,no,2026-10-12
953,2265,sharethis.com,sharethis.com,https://sharethis.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
954,2270,made-in-china.com,www.made-in-china.com,https://www.made-in-china.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
955,2271,elastic.co,www.elastic.co,https://www.elastic.co/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
956,2272,semanticscholar.org,www.semanticscholar.org,https://www.semanticscholar.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
957,2273,rspamd.com,rspamd.com,https://rspamd.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)","allowed (named)",,,,,no,no,2026-10-12
958,2276,atom.com,www.atom.com,https://www.atom.com/robots.txt,200,ok,open_to_ai,"allowed (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
959,2280,cctv.com,www.cctv.com,https://www.cctv.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
960,2282,customer.io,customer.io,https://customer.io/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,"search=yes, ai-input=yes, ai-train=yes",no,yes,2026-10-12
961,2284,generatepress.com,generatepress.com,https://generatepress.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (named)","partial (named)",,,,,no,no,2026-10-12
962,2285,websupport.sk,www.websupport.sk,https://www.websupport.sk/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
963,2289,purdue.edu,purdue.edu,https://purdue.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
964,2293,adswizz.com,www.adswizz.com,https://www.adswizz.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
965,2295,garena.com,intl.garena.com,https://intl.garena.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
966,2297,quantummetric.com,www.quantummetric.com,https://www.quantummetric.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
967,2298,drweb.ru,www.drweb.ru,https://www.drweb.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
968,2301,messenger.com,www.messenger.com,https://www.messenger.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
969,2302,cookiebot.com,www.cookiebot.com,https://www.cookiebot.com/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,"ai-train=no, search=yes, ai-input=yes",no,yes,2026-10-12
970,2309,ionos.de,www.ionos.de,https://www.ionos.de/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
971,2310,photobucket.com,photobucket.com,https://photobucket.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
972,2311,1688.com,www.1688.com,https://www.1688.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","allowed (named)","blocked (named)",,,,,no,no,2026-10-12
973,2312,secu100.net,www.secu100.net,https://www.secu100.net/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
974,2313,innovid.com,www.innovid.com,https://www.innovid.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)","allowed (named)",allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,"allowed (named)",,,,,no,yes,2026-10-12
975,2314,weather.gov,www.weather.gov,https://www.weather.gov/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
976,2315,www.gov.cn,www.gov.cn,https://www.gov.cn/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,rule_before_user_agent,,,no,no,2026-10-12
977,2318,khanacademy.org,www.khanacademy.org,https://www.khanacademy.org/robots.txt,200,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
978,2320,msu.edu,msu.edu,https://msu.edu/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
979,2323,paramountplus.com,www.paramountplus.com,https://www.paramountplus.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
980,2326,techradar.com,www.techradar.com,https://www.techradar.com/robots.txt,200,ok,blocks_training_only,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","blocked (named)","partial (from *)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","diffbot youbot ai2bot",,,,no,no,2026-10-12
981,2328,infomaniak.com,www.infomaniak.com,https://www.infomaniak.com/robots.txt,200,ok,open_to_ai,"partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot youbot",,,,no,no,2026-10-12
982,2332,kasserver.com,kasserver.com,https://kasserver.com/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
983,2333,sendbird.com,sendbird.com,https://sendbird.com/robots.txt,200,ok,blocks_training_only,"blocked (named)","allowed (named)","allowed (named)","blocked (named)","partial (from *)","partial (from *)","blocked (named)","blocked (named)","allowed (named)","allowed (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","partial (from *)","partial (from *)","allowed (named)","allowed (named)","anthropic-ai claude-web cohere-ai diffbot",,,"search=yes,ai-train=no,use=reference",no,yes,2026-10-12
984,2335,nyu.edu,www.nyu.edu,https://www.nyu.edu/robots.txt,202,html_not_robots,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
985,2336,screenconnect.com,www.screenconnect.com,https://www.screenconnect.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
986,2337,ycombinator.com,www.ycombinator.com,https://www.ycombinator.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
987,2339,instacart.com,www.instacart.com,https://www.instacart.com/robots.txt,200,ok,blocks_all_ai,"blocked (from *)","partial (named)","partial (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","blocked (from *)","partial (named)","partial (named)",,,,,no,"not checked (disallowed)",2026-10-12
988,2341,grafana.com,grafana.com,https://grafana.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
989,2347,example.org,example.org,https://example.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
990,2349,verkada.com,www.verkada.com,https://www.verkada.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
991,2350,unc.edu,www.unc.edu,https://www.unc.edu/robots.txt,200,ok,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
992,2352,vox.com,www.vox.com,https://www.vox.com/robots.txt,200,ok,blocks_all_ai,"allowed (named)","partial (from *)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","blocked (named)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","anthropic-ai claude-web cohere-ai diffbot imagesiftbot omgilibot youbot ai2bot timpibot google-cloudvertexbot",,"OpenAI: ChatGPT-User blocked, OAI-SearchBot allowed",,no,no,2026-10-12
993,2354,signal.org,signal.org,https://signal.org/robots.txt,404,not_found,open_to_ai,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,allowed,,,,,no,no,2026-10-12
994,2355,magnit.ru,magnit.ru,https://magnit.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
995,2356,fullstory.com,www.fullstory.com,https://www.fullstory.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
996,2358,ucoz.ru,ucoz.ru,https://ucoz.ru/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,yes,2026-10-12
997,2359,drweb.com,www.drweb.com,https://www.drweb.com/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
998,2360,geocomply.com,www.geocomply.com,https://www.geocomply.com/robots.txt,200,ok,open_to_ai,"allowed (named)","allowed (named)","allowed (named)","allowed (named)",allowed,allowed,"allowed (named)",allowed,"allowed (named)",allowed,allowed,"allowed (named)",allowed,allowed,allowed,allowed,allowed,allowed,claude-web,,,,no,yes,2026-10-12
999,2362,kde.org,kde.org,https://kde.org/robots.txt,200,ok,open_to_ai,"partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12
1000,2364,libero.it,www.libero.it,https://www.libero.it/robots.txt,200,ok,open_to_ai,"partial (from *)","allowed (named)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)","partial (from *)",,,,,no,no,2026-10-12