{ "title": "Robots.txt says yes, the firewall says no: AI search-bot access on the top 5,000 websites (October 2026)", "publisher": "GEO Audit", "license": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/). Credit: GEO Audit, AI crawler access study, October 2026.", "csv": "/data/ai-crawler-access-2026-10.csv", "run": { "times_utc": [ "start 2026-10-13T16:40:14Z", "end 2026-10-13T16:47:20Z", "resume1 2026-10-13T16:48:20Z", "end-r1 2026-10-13T16:53:43Z", "resume2 2026-10-13T16:54:58Z", "supervisor-start 2026-10-13T16:58:31Z", "crawl-complete 2026-10-13T17:08:15Z", "recheck-start 2026-10-13T17:10:48Z", "recheck-complete 2026-10-13T17:13:32Z" ], "vantage_point": "A single fixed-line connection on an Indian ISP (not a datacenter or cloud IP), one run, on 13 October 2026. Results from other networks, especially datacenter IPs, and other countries can differ.", "concurrency": "at most 16 requests in flight; requests to any one domain sequential", "timeout_ms": 10000, "retries": "robots.txt and browser home page: at most one retry, only on 5xx/429/network errors; probes: none", "robots_user_agent": "GEOAuditBot/1.0 (+https://geo-audit-saas-eight.vercel.app)", "browser_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", "probe_user_agents": { "OAI-SearchBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot", "PerplexityBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", "Claude-SearchBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/)" }, "probe_caveat": "Simulated user agent, not IP-verified. The probes send the bot's user-agent string from our connection, not from the vendor's published IP ranges, so a CDN that verifies crawler IPs may treat the real crawler differently in either direction.", "engine": "GEO Audit engine: lib/robots.ts (parseRobots, BOTS; RFC 9309 exact-token matching), lib/engine/fetcher.ts (createFetcher), lib/audit-engine.ts (PROBE_BOTS, judgeProbe, detectCdn, usageSignals, isTextFile, fetchPage)", "segments_note": "The crawl ran 16:40–17:08 UTC on 13 Oct 2026 in five process segments: the Node.js HTTP client (undici 7.29 on Windows) crashed natively several times; each restart resumed after the last completed domain and re-ran the domains that were in flight from scratch. No domain was counted twice." }, "sample": { "source": "Tranco list 64VKX (https://tranco-list.eu/list/64VKX), generated 2026-10-12; daily list combining CrUX, Farsight, Majestic, Cloudflare Radar and Cisco Umbrella over 2026-09-13..2026-10-12 (Dowdall rule), pay-level domains only", "ranks": "1-5000", "domains": 5000, "live_sites_home_2xx": 3023, "live_sites_with_known_robots_outcome": 2976, "live_sites_with_robots_file": 2414, "live_sites_with_clean_browser_baseline": 3007 }, "definitions": { "live_site": "The home page (https:///, following up to 5 redirects; www/bare sibling tried only if the host did not answer) returned 2xx to the browser user agent.", "robots_states": { "rules": "robots.txt returned 2xx with plain-text directives (engine isTextFile)", "not_found": "4xx other than 401/403/429 (RFC 9309: unavailable, crawler may access everything)", "forbidden": "401/403 to our own GEOAuditBot user agent (RFC 9309 treats this as unavailable; a firewall rule may be involved)", "html_or_nontext": "2xx but an HTML page or other non-robots content (no directives)", "empty": "2xx with an empty body", "server_error": "5xx or 429 (RFC 9309: crawler must assume complete disallow)", "redirect_loop": "more than 5 redirects", "no_response": "no HTTP answer (dns, network, timeout)" }, "blocked_robots": "The bot's governing group (exact product-token match per RFC 9309, vendor-documented fallback for Applebot→Googlebot, else '*') disallows '/' under longest-match, Allow-wins-ties. 'named' = a group naming the bot decided it; 'wildcard' = the '*' group decided it.", "bot_column_codes": "An = allowed by a group naming the bot; Aw = allowed by the * group; A- = no group applies (allowed); Dn = disallowed by a group naming the bot; Dw = disallowed by the * group; empty = no robots.txt rules were read", "probe_verdicts": "engine judgeProbe: ok; refused (non-2xx without challenge markup); challenge (cf-mitigated: challenge header or challenge markup, on a non-2xx, or on a 2xx with under max(40, 30% of the browser page's words) words); paywall (HTTP 402); unknown (no HTTP answer); not_run (home page did not load for the browser UA, so probes were skipped as in the engine)", "browser_baseline": "judgeProbe applied to the browser-UA response itself; only sites with baseline 'ok' are used for edge-blocking rates, so generic anti-automation that also challenges our browser request is not counted as AI-bot blocking", "cdn": "engine detectCdn on the home-page response headers (falls back to robots.txt response headers only if the home page got no HTTP answer): cf-ray/server cloudflare, x-vercel-id, x-amz-cf-id, x-nf-request-id/netlify, fastly headers, akamai headers. Sites behind CDNs that send none of these are 'none detected'.", "bytes": "decoded body length in UTF-8 bytes, capped: robots.txt 512,000; browser home page 2,000,000; probes 96,000" }, "robots_state": { "all_domains": { "rules": 2743, "no_response:dns": 899, "forbidden": 369, "not_found": 270, "html_or_nontext": 248, "no_response:timeout": 240, "no_response:network": 165, "empty": 37, "server_error": 21, "redirect_loop": 8 }, "live_sites": { "rules": 2414, "html_or_nontext": 239, "not_found": 170, "forbidden": 117, "no_response:timeout": 42, "empty": 35, "server_error": 3, "no_response:network": 2, "redirect_loop": 1 } }, "refused_browser_but_publish_robots": { "definition": "Domains whose home page answered our browser-UA request with a 4xx other than 404/410 (usually 403 from bot management) but which serve a robots.txt file. Excluded from 'live sites'; reported here because they block AI crawlers more often.", "n": 287, "home_status": { "400": 9, "401": 8, "402": 10, "403": 242, "405": 3, "406": 2, "418": 2, "429": 8, "439": 2, "451": 1 }, "cdn": { "Cloudflare": 186, "none detected": 62, "Amazon CloudFront": 21, "Akamai": 17, "Vercel": 1 } }, "all_robots_files": 2743, "cloudflare_managed_block_all_robots_files": { "n": 7, "of": 2743, "pct": 0.3 }, "content_signal_all_robots_files": { "n": 101, "of": 2743, "pct": 3.7 }, "no_robots_txt": { "live_sites_without_robots_file": { "n": 562, "of": 3023, "pct": 18.6 }, "live_sites_with_robots_file": { "n": 2414, "of": 3023, "pct": 79.9 }, "all_domains_with_robots_file": { "n": 2743, "of": 5000, "pct": 54.9 } }, "per_bot": [ { "token": "oai-searchbot", "label": "OAI-SearchBot", "vendor": "OpenAI", "cls": "retrieval", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 178, "of": 2976, "pct": 6 }, "blocked_of_robots_files": { "n": 178, "of": 2414, "pct": 7.4 }, "named_block_of_robots_files": { "n": 110, "of": 2414, "pct": 4.6 }, "named_block_of_live_sites": { "n": 110, "of": 2976, "pct": 3.7 }, "wildcard_only_block_of_robots_files": { "n": 68, "of": 2414, "pct": 2.8 }, "blocked_of_all_robots_files": { "n": 221, "of": 2743, "pct": 8.1 }, "named_block_of_all_robots_files": { "n": 129, "of": 2743, "pct": 4.7 }, "blocked_of_refusers": { "n": 30, "of": 287, "pct": 10.5 }, "blocked_of_live_plus_refusers": { "n": 208, "of": 3263, "pct": 6.4 } }, { "token": "chatgpt-user", "label": "ChatGPT-User", "vendor": "OpenAI", "cls": "user-fetch", "mayIgnoreRobots": true, "blocked_of_live_sites": { "n": 225, "of": 2976, "pct": 7.6 }, "blocked_of_robots_files": { "n": 225, "of": 2414, "pct": 9.3 }, "named_block_of_robots_files": { "n": 157, "of": 2414, "pct": 6.5 }, "named_block_of_live_sites": { "n": 157, "of": 2976, "pct": 5.3 }, "wildcard_only_block_of_robots_files": { "n": 68, "of": 2414, "pct": 2.8 }, "blocked_of_all_robots_files": { "n": 281, "of": 2743, "pct": 10.2 }, "named_block_of_all_robots_files": { "n": 189, "of": 2743, "pct": 6.9 }, "blocked_of_refusers": { "n": 42, "of": 287, "pct": 14.6 }, "blocked_of_live_plus_refusers": { "n": 267, "of": 3263, "pct": 8.2 } }, { "token": "gptbot", "label": "GPTBot", "vendor": "OpenAI", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 345, "of": 2976, "pct": 11.6 }, "blocked_of_robots_files": { "n": 345, "of": 2414, "pct": 14.3 }, "named_block_of_robots_files": { "n": 278, "of": 2414, "pct": 11.5 }, "named_block_of_live_sites": { "n": 278, "of": 2976, "pct": 9.3 }, "wildcard_only_block_of_robots_files": { "n": 67, "of": 2414, "pct": 2.8 }, "blocked_of_all_robots_files": { "n": 437, "of": 2743, "pct": 15.9 }, "named_block_of_all_robots_files": { "n": 349, "of": 2743, "pct": 12.7 }, "blocked_of_refusers": { "n": 77, "of": 287, "pct": 26.8 }, "blocked_of_live_plus_refusers": { "n": 422, "of": 3263, "pct": 12.9 } }, { "token": "oai-adsbot", "label": "OAI-AdsBot", "vendor": "OpenAI", "cls": "other", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 95, "of": 2976, "pct": 3.2 }, "blocked_of_robots_files": { "n": 95, "of": 2414, "pct": 3.9 }, "named_block_of_robots_files": { "n": 2, "of": 2414, "pct": 0.1 }, "named_block_of_live_sites": { "n": 2, "of": 2976, "pct": 0.1 }, "wildcard_only_block_of_robots_files": { "n": 93, "of": 2414, "pct": 3.9 }, "blocked_of_all_robots_files": { "n": 132, "of": 2743, "pct": 4.8 }, "named_block_of_all_robots_files": { "n": 6, "of": 2743, "pct": 0.2 }, "blocked_of_refusers": { "n": 24, "of": 287, "pct": 8.4 }, "blocked_of_live_plus_refusers": { "n": 119, "of": 3263, "pct": 3.6 } }, { "token": "claude-searchbot", "label": "Claude-SearchBot", "vendor": "Anthropic", "cls": "retrieval", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 169, "of": 2976, "pct": 5.7 }, "blocked_of_robots_files": { "n": 169, "of": 2414, "pct": 7 }, "named_block_of_robots_files": { "n": 93, "of": 2414, "pct": 3.9 }, "named_block_of_live_sites": { "n": 93, "of": 2976, "pct": 3.1 }, "wildcard_only_block_of_robots_files": { "n": 76, "of": 2414, "pct": 3.1 }, "blocked_of_all_robots_files": { "n": 227, "of": 2743, "pct": 8.3 }, "named_block_of_all_robots_files": { "n": 122, "of": 2743, "pct": 4.4 }, "blocked_of_refusers": { "n": 44, "of": 287, "pct": 15.3 }, "blocked_of_live_plus_refusers": { "n": 213, "of": 3263, "pct": 6.5 } }, { "token": "claude-user", "label": "Claude-User", "vendor": "Anthropic", "cls": "user-fetch", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 177, "of": 2976, "pct": 5.9 }, "blocked_of_robots_files": { "n": 177, "of": 2414, "pct": 7.3 }, "named_block_of_robots_files": { "n": 102, "of": 2414, "pct": 4.2 }, "named_block_of_live_sites": { "n": 102, "of": 2976, "pct": 3.4 }, "wildcard_only_block_of_robots_files": { "n": 75, "of": 2414, "pct": 3.1 }, "blocked_of_all_robots_files": { "n": 234, "of": 2743, "pct": 8.5 }, "named_block_of_all_robots_files": { "n": 130, "of": 2743, "pct": 4.7 }, "blocked_of_refusers": { "n": 43, "of": 287, "pct": 15 }, "blocked_of_live_plus_refusers": { "n": 220, "of": 3263, "pct": 6.7 } }, { "token": "claudebot", "label": "ClaudeBot", "vendor": "Anthropic", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 316, "of": 2976, "pct": 10.6 }, "blocked_of_robots_files": { "n": 316, "of": 2414, "pct": 13.1 }, "named_block_of_robots_files": { "n": 243, "of": 2414, "pct": 10.1 }, "named_block_of_live_sites": { "n": 243, "of": 2976, "pct": 8.2 }, "wildcard_only_block_of_robots_files": { "n": 73, "of": 2414, "pct": 3 }, "blocked_of_all_robots_files": { "n": 417, "of": 2743, "pct": 15.2 }, "named_block_of_all_robots_files": { "n": 317, "of": 2743, "pct": 11.6 }, "blocked_of_refusers": { "n": 86, "of": 287, "pct": 30 }, "blocked_of_live_plus_refusers": { "n": 402, "of": 3263, "pct": 12.3 } }, { "token": "perplexitybot", "label": "PerplexityBot", "vendor": "Perplexity", "cls": "retrieval", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 246, "of": 2976, "pct": 8.3 }, "blocked_of_robots_files": { "n": 246, "of": 2414, "pct": 10.2 }, "named_block_of_robots_files": { "n": 178, "of": 2414, "pct": 7.4 }, "named_block_of_live_sites": { "n": 178, "of": 2976, "pct": 6 }, "wildcard_only_block_of_robots_files": { "n": 68, "of": 2414, "pct": 2.8 }, "blocked_of_all_robots_files": { "n": 325, "of": 2743, "pct": 11.8 }, "named_block_of_all_robots_files": { "n": 232, "of": 2743, "pct": 8.5 }, "blocked_of_refusers": { "n": 65, "of": 287, "pct": 22.6 }, "blocked_of_live_plus_refusers": { "n": 311, "of": 3263, "pct": 9.5 } }, { "token": "perplexity-user", "label": "Perplexity-User", "vendor": "Perplexity", "cls": "user-fetch", "mayIgnoreRobots": true, "blocked_of_live_sites": { "n": 179, "of": 2976, "pct": 6 }, "blocked_of_robots_files": { "n": 179, "of": 2414, "pct": 7.4 }, "named_block_of_robots_files": { "n": 100, "of": 2414, "pct": 4.1 }, "named_block_of_live_sites": { "n": 100, "of": 2976, "pct": 3.4 }, "wildcard_only_block_of_robots_files": { "n": 79, "of": 2414, "pct": 3.3 }, "blocked_of_all_robots_files": { "n": 235, "of": 2743, "pct": 8.6 }, "named_block_of_all_robots_files": { "n": 126, "of": 2743, "pct": 4.6 }, "blocked_of_refusers": { "n": 44, "of": 287, "pct": 15.3 }, "blocked_of_live_plus_refusers": { "n": 223, "of": 3263, "pct": 6.8 } }, { "token": "googlebot", "label": "Googlebot", "vendor": "Google", "cls": "search", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 51, "of": 2976, "pct": 1.7 }, "blocked_of_robots_files": { "n": 51, "of": 2414, "pct": 2.1 }, "named_block_of_robots_files": { "n": 7, "of": 2414, "pct": 0.3 }, "named_block_of_live_sites": { "n": 7, "of": 2976, "pct": 0.2 }, "wildcard_only_block_of_robots_files": { "n": 44, "of": 2414, "pct": 1.8 }, "blocked_of_all_robots_files": { "n": 71, "of": 2743, "pct": 2.6 }, "named_block_of_all_robots_files": { "n": 10, "of": 2743, "pct": 0.4 }, "blocked_of_refusers": { "n": 8, "of": 287, "pct": 2.8 }, "blocked_of_live_plus_refusers": { "n": 59, "of": 3263, "pct": 1.8 } }, { "token": "google-extended", "label": "Google-Extended", "vendor": "Google", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 277, "of": 2976, "pct": 9.3 }, "blocked_of_robots_files": { "n": 277, "of": 2414, "pct": 11.5 }, "named_block_of_robots_files": { "n": 208, "of": 2414, "pct": 8.6 }, "named_block_of_live_sites": { "n": 208, "of": 2976, "pct": 7 }, "wildcard_only_block_of_robots_files": { "n": 69, "of": 2414, "pct": 2.9 }, "blocked_of_all_robots_files": { "n": 363, "of": 2743, "pct": 13.2 }, "named_block_of_all_robots_files": { "n": 269, "of": 2743, "pct": 9.8 }, "blocked_of_refusers": { "n": 70, "of": 287, "pct": 24.4 }, "blocked_of_live_plus_refusers": { "n": 347, "of": 3263, "pct": 10.6 } }, { "token": "google-cloudvertexbot", "label": "Google-CloudVertexBot", "vendor": "Google", "cls": "other", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 163, "of": 2976, "pct": 5.5 }, "blocked_of_robots_files": { "n": 163, "of": 2414, "pct": 6.8 }, "named_block_of_robots_files": { "n": 80, "of": 2414, "pct": 3.3 }, "named_block_of_live_sites": { "n": 80, "of": 2976, "pct": 2.7 }, "wildcard_only_block_of_robots_files": { "n": 83, "of": 2414, "pct": 3.4 }, "blocked_of_all_robots_files": { "n": 223, "of": 2743, "pct": 8.1 }, "named_block_of_all_robots_files": { "n": 110, "of": 2743, "pct": 4 }, "blocked_of_refusers": { "n": 47, "of": 287, "pct": 16.4 }, "blocked_of_live_plus_refusers": { "n": 210, "of": 3263, "pct": 6.4 } }, { "token": "google-agent", "label": "Google-Agent", "vendor": "Google", "cls": "agent", "mayIgnoreRobots": true, "blocked_of_live_sites": { "n": 101, "of": 2976, "pct": 3.4 }, "blocked_of_robots_files": { "n": 101, "of": 2414, "pct": 4.2 }, "named_block_of_robots_files": { "n": 9, "of": 2414, "pct": 0.4 }, "named_block_of_live_sites": { "n": 9, "of": 2976, "pct": 0.3 }, "wildcard_only_block_of_robots_files": { "n": 92, "of": 2414, "pct": 3.8 }, "blocked_of_all_robots_files": { "n": 135, "of": 2743, "pct": 4.9 }, "named_block_of_all_robots_files": { "n": 10, "of": 2743, "pct": 0.4 }, "blocked_of_refusers": { "n": 21, "of": 287, "pct": 7.3 }, "blocked_of_live_plus_refusers": { "n": 122, "of": 3263, "pct": 3.7 } }, { "token": "google-gemininotebook", "label": "Google-GeminiNotebook", "vendor": "Google", "cls": "user-fetch", "mayIgnoreRobots": true, "blocked_of_live_sites": { "n": 94, "of": 2976, "pct": 3.2 }, "blocked_of_robots_files": { "n": 94, "of": 2414, "pct": 3.9 }, "named_block_of_robots_files": { "n": 1, "of": 2414, "pct": 0 }, "named_block_of_live_sites": { "n": 1, "of": 2976, "pct": 0 }, "wildcard_only_block_of_robots_files": { "n": 93, "of": 2414, "pct": 3.9 }, "blocked_of_all_robots_files": { "n": 128, "of": 2743, "pct": 4.7 }, "named_block_of_all_robots_files": { "n": 2, "of": 2743, "pct": 0.1 }, "blocked_of_refusers": { "n": 21, "of": 287, "pct": 7.3 }, "blocked_of_live_plus_refusers": { "n": 115, "of": 3263, "pct": 3.5 } }, { "token": "bingbot", "label": "Bingbot", "vendor": "Microsoft", "cls": "search", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 55, "of": 2976, "pct": 1.8 }, "blocked_of_robots_files": { "n": 55, "of": 2414, "pct": 2.3 }, "named_block_of_robots_files": { "n": 9, "of": 2414, "pct": 0.4 }, "named_block_of_live_sites": { "n": 9, "of": 2976, "pct": 0.3 }, "wildcard_only_block_of_robots_files": { "n": 46, "of": 2414, "pct": 1.9 }, "blocked_of_all_robots_files": { "n": 76, "of": 2743, "pct": 2.8 }, "named_block_of_all_robots_files": { "n": 12, "of": 2743, "pct": 0.4 }, "blocked_of_refusers": { "n": 10, "of": 287, "pct": 3.5 }, "blocked_of_live_plus_refusers": { "n": 65, "of": 3263, "pct": 2 } }, { "token": "applebot", "label": "Applebot", "vendor": "Apple", "cls": "mixed", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 83, "of": 2976, "pct": 2.8 }, "blocked_of_robots_files": { "n": 83, "of": 2414, "pct": 3.4 }, "named_block_of_robots_files": { "n": 39, "of": 2414, "pct": 1.6 }, "named_block_of_live_sites": { "n": 39, "of": 2976, "pct": 1.3 }, "wildcard_only_block_of_robots_files": { "n": 44, "of": 2414, "pct": 1.8 }, "blocked_of_all_robots_files": { "n": 105, "of": 2743, "pct": 3.8 }, "named_block_of_all_robots_files": { "n": 44, "of": 2743, "pct": 1.6 }, "blocked_of_refusers": { "n": 10, "of": 287, "pct": 3.5 }, "blocked_of_live_plus_refusers": { "n": 93, "of": 3263, "pct": 2.9 } }, { "token": "applebot-extended", "label": "Applebot-Extended", "vendor": "Apple", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 250, "of": 2976, "pct": 8.4 }, "blocked_of_robots_files": { "n": 250, "of": 2414, "pct": 10.4 }, "named_block_of_robots_files": { "n": 168, "of": 2414, "pct": 7 }, "named_block_of_live_sites": { "n": 168, "of": 2976, "pct": 5.6 }, "wildcard_only_block_of_robots_files": { "n": 82, "of": 2414, "pct": 3.4 }, "blocked_of_all_robots_files": { "n": 346, "of": 2743, "pct": 12.6 }, "named_block_of_all_robots_files": { "n": 238, "of": 2743, "pct": 8.7 }, "blocked_of_refusers": { "n": 80, "of": 287, "pct": 27.9 }, "blocked_of_live_plus_refusers": { "n": 330, "of": 3263, "pct": 10.1 } }, { "token": "meta-webindexer", "label": "Meta-WebIndexer", "vendor": "Meta", "cls": "search", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 150, "of": 2976, "pct": 5 }, "blocked_of_robots_files": { "n": 150, "of": 2414, "pct": 6.2 }, "named_block_of_robots_files": { "n": 61, "of": 2414, "pct": 2.5 }, "named_block_of_live_sites": { "n": 61, "of": 2976, "pct": 2 }, "wildcard_only_block_of_robots_files": { "n": 89, "of": 2414, "pct": 3.7 }, "blocked_of_all_robots_files": { "n": 194, "of": 2743, "pct": 7.1 }, "named_block_of_all_robots_files": { "n": 74, "of": 2743, "pct": 2.7 }, "blocked_of_refusers": { "n": 31, "of": 287, "pct": 10.8 }, "blocked_of_live_plus_refusers": { "n": 181, "of": 3263, "pct": 5.5 } }, { "token": "meta-externalfetcher", "label": "Meta-ExternalFetcher", "vendor": "Meta", "cls": "user-fetch", "mayIgnoreRobots": true, "blocked_of_live_sites": { "n": 187, "of": 2976, "pct": 6.3 }, "blocked_of_robots_files": { "n": 187, "of": 2414, "pct": 7.7 }, "named_block_of_robots_files": { "n": 106, "of": 2414, "pct": 4.4 }, "named_block_of_live_sites": { "n": 106, "of": 2976, "pct": 3.6 }, "wildcard_only_block_of_robots_files": { "n": 81, "of": 2414, "pct": 3.4 }, "blocked_of_all_robots_files": { "n": 244, "of": 2743, "pct": 8.9 }, "named_block_of_all_robots_files": { "n": 133, "of": 2743, "pct": 4.8 }, "blocked_of_refusers": { "n": 44, "of": 287, "pct": 15.3 }, "blocked_of_live_plus_refusers": { "n": 231, "of": 3263, "pct": 7.1 } }, { "token": "meta-externalagent", "label": "Meta-ExternalAgent", "vendor": "Meta", "cls": "mixed", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 291, "of": 2976, "pct": 9.8 }, "blocked_of_robots_files": { "n": 291, "of": 2414, "pct": 12.1 }, "named_block_of_robots_files": { "n": 211, "of": 2414, "pct": 8.7 }, "named_block_of_live_sites": { "n": 211, "of": 2976, "pct": 7.1 }, "wildcard_only_block_of_robots_files": { "n": 80, "of": 2414, "pct": 3.3 }, "blocked_of_all_robots_files": { "n": 383, "of": 2743, "pct": 14 }, "named_block_of_all_robots_files": { "n": 274, "of": 2743, "pct": 10 }, "blocked_of_refusers": { "n": 76, "of": 287, "pct": 26.5 }, "blocked_of_live_plus_refusers": { "n": 367, "of": 3263, "pct": 11.2 } }, { "token": "meta-externalads", "label": "Meta-ExternalAds", "vendor": "Meta", "cls": "other", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 92, "of": 2976, "pct": 3.1 }, "blocked_of_robots_files": { "n": 92, "of": 2414, "pct": 3.8 }, "named_block_of_robots_files": { "n": 3, "of": 2414, "pct": 0.1 }, "named_block_of_live_sites": { "n": 3, "of": 2976, "pct": 0.1 }, "wildcard_only_block_of_robots_files": { "n": 89, "of": 2414, "pct": 3.7 }, "blocked_of_all_robots_files": { "n": 123, "of": 2743, "pct": 4.5 }, "named_block_of_all_robots_files": { "n": 3, "of": 2743, "pct": 0.1 }, "blocked_of_refusers": { "n": 18, "of": 287, "pct": 6.3 }, "blocked_of_live_plus_refusers": { "n": 110, "of": 3263, "pct": 3.4 } }, { "token": "amzn-searchbot", "label": "Amzn-SearchBot", "vendor": "Amazon", "cls": "search", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 120, "of": 2976, "pct": 4 }, "blocked_of_robots_files": { "n": 120, "of": 2414, "pct": 5 }, "named_block_of_robots_files": { "n": 31, "of": 2414, "pct": 1.3 }, "named_block_of_live_sites": { "n": 31, "of": 2976, "pct": 1 }, "wildcard_only_block_of_robots_files": { "n": 89, "of": 2414, "pct": 3.7 }, "blocked_of_all_robots_files": { "n": 161, "of": 2743, "pct": 5.9 }, "named_block_of_all_robots_files": { "n": 40, "of": 2743, "pct": 1.5 }, "blocked_of_refusers": { "n": 28, "of": 287, "pct": 9.8 }, "blocked_of_live_plus_refusers": { "n": 148, "of": 3263, "pct": 4.5 } }, { "token": "amzn-user", "label": "Amzn-User", "vendor": "Amazon", "cls": "user-fetch", "mayIgnoreRobots": true, "blocked_of_live_sites": { "n": 115, "of": 2976, "pct": 3.9 }, "blocked_of_robots_files": { "n": 115, "of": 2414, "pct": 4.8 }, "named_block_of_robots_files": { "n": 25, "of": 2414, "pct": 1 }, "named_block_of_live_sites": { "n": 25, "of": 2976, "pct": 0.8 }, "wildcard_only_block_of_robots_files": { "n": 90, "of": 2414, "pct": 3.7 }, "blocked_of_all_robots_files": { "n": 156, "of": 2743, "pct": 5.7 }, "named_block_of_all_robots_files": { "n": 35, "of": 2743, "pct": 1.3 }, "blocked_of_refusers": { "n": 28, "of": 287, "pct": 9.8 }, "blocked_of_live_plus_refusers": { "n": 143, "of": 3263, "pct": 4.4 } }, { "token": "amazonbot", "label": "Amazonbot", "vendor": "Amazon", "cls": "mixed", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 234, "of": 2976, "pct": 7.9 }, "blocked_of_robots_files": { "n": 234, "of": 2414, "pct": 9.7 }, "named_block_of_robots_files": { "n": 159, "of": 2414, "pct": 6.6 }, "named_block_of_live_sites": { "n": 159, "of": 2976, "pct": 5.3 }, "wildcard_only_block_of_robots_files": { "n": 75, "of": 2414, "pct": 3.1 }, "blocked_of_all_robots_files": { "n": 322, "of": 2743, "pct": 11.7 }, "named_block_of_all_robots_files": { "n": 218, "of": 2743, "pct": 7.9 }, "blocked_of_refusers": { "n": 73, "of": 287, "pct": 25.4 }, "blocked_of_live_plus_refusers": { "n": 307, "of": 3263, "pct": 9.4 } }, { "token": "duckassistbot", "label": "DuckAssistBot", "vendor": "DuckDuckGo", "cls": "user-fetch", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 188, "of": 2976, "pct": 6.3 }, "blocked_of_robots_files": { "n": 188, "of": 2414, "pct": 7.8 }, "named_block_of_robots_files": { "n": 107, "of": 2414, "pct": 4.4 }, "named_block_of_live_sites": { "n": 107, "of": 2976, "pct": 3.6 }, "wildcard_only_block_of_robots_files": { "n": 81, "of": 2414, "pct": 3.4 }, "blocked_of_all_robots_files": { "n": 241, "of": 2743, "pct": 8.8 }, "named_block_of_all_robots_files": { "n": 129, "of": 2743, "pct": 4.7 }, "blocked_of_refusers": { "n": 39, "of": 287, "pct": 13.6 }, "blocked_of_live_plus_refusers": { "n": 227, "of": 3263, "pct": 7 } }, { "token": "mistralai-index", "label": "MistralAI-Index", "vendor": "Mistral", "cls": "retrieval", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 97, "of": 2976, "pct": 3.3 }, "blocked_of_robots_files": { "n": 97, "of": 2414, "pct": 4 }, "named_block_of_robots_files": { "n": 5, "of": 2414, "pct": 0.2 }, "named_block_of_live_sites": { "n": 5, "of": 2976, "pct": 0.2 }, "wildcard_only_block_of_robots_files": { "n": 92, "of": 2414, "pct": 3.8 }, "blocked_of_all_robots_files": { "n": 141, "of": 2743, "pct": 5.1 }, "named_block_of_all_robots_files": { "n": 16, "of": 2743, "pct": 0.6 }, "blocked_of_refusers": { "n": 31, "of": 287, "pct": 10.8 }, "blocked_of_live_plus_refusers": { "n": 128, "of": 3263, "pct": 3.9 } }, { "token": "mistralai-user", "label": "MistralAI-User", "vendor": "Mistral", "cls": "user-fetch", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 168, "of": 2976, "pct": 5.6 }, "blocked_of_robots_files": { "n": 168, "of": 2414, "pct": 7 }, "named_block_of_robots_files": { "n": 84, "of": 2414, "pct": 3.5 }, "named_block_of_live_sites": { "n": 84, "of": 2976, "pct": 2.8 }, "wildcard_only_block_of_robots_files": { "n": 84, "of": 2414, "pct": 3.5 }, "blocked_of_all_robots_files": { "n": 224, "of": 2743, "pct": 8.2 }, "named_block_of_all_robots_files": { "n": 108, "of": 2743, "pct": 3.9 }, "blocked_of_refusers": { "n": 42, "of": 287, "pct": 14.6 }, "blocked_of_live_plus_refusers": { "n": 210, "of": 3263, "pct": 6.4 } }, { "token": "mistralai-training", "label": "MistralAI-Training", "vendor": "Mistral", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 99, "of": 2976, "pct": 3.3 }, "blocked_of_robots_files": { "n": 99, "of": 2414, "pct": 4.1 }, "named_block_of_robots_files": { "n": 7, "of": 2414, "pct": 0.3 }, "named_block_of_live_sites": { "n": 7, "of": 2976, "pct": 0.2 }, "wildcard_only_block_of_robots_files": { "n": 92, "of": 2414, "pct": 3.8 }, "blocked_of_all_robots_files": { "n": 136, "of": 2743, "pct": 5 }, "named_block_of_all_robots_files": { "n": 11, "of": 2743, "pct": 0.4 }, "blocked_of_refusers": { "n": 24, "of": 287, "pct": 8.4 }, "blocked_of_live_plus_refusers": { "n": 123, "of": 3263, "pct": 3.8 } }, { "token": "youbot", "label": "YouBot", "vendor": "You.com", "cls": "retrieval", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 237, "of": 2976, "pct": 8 }, "blocked_of_robots_files": { "n": 237, "of": 2414, "pct": 9.8 }, "named_block_of_robots_files": { "n": 154, "of": 2414, "pct": 6.4 }, "named_block_of_live_sites": { "n": 154, "of": 2976, "pct": 5.2 }, "wildcard_only_block_of_robots_files": { "n": 83, "of": 2414, "pct": 3.4 }, "blocked_of_all_robots_files": { "n": 302, "of": 2743, "pct": 11 }, "named_block_of_all_robots_files": { "n": 188, "of": 2743, "pct": 6.9 }, "blocked_of_refusers": { "n": 51, "of": 287, "pct": 17.8 }, "blocked_of_live_plus_refusers": { "n": 288, "of": 3263, "pct": 8.8 } }, { "token": "bytespider", "label": "Bytespider", "vendor": "ByteDance", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 354, "of": 2976, "pct": 11.9 }, "blocked_of_robots_files": { "n": 354, "of": 2414, "pct": 14.7 }, "named_block_of_robots_files": { "n": 273, "of": 2414, "pct": 11.3 }, "named_block_of_live_sites": { "n": 273, "of": 2976, "pct": 9.2 }, "wildcard_only_block_of_robots_files": { "n": 81, "of": 2414, "pct": 3.4 }, "blocked_of_all_robots_files": { "n": 457, "of": 2743, "pct": 16.7 }, "named_block_of_all_robots_files": { "n": 344, "of": 2743, "pct": 12.5 }, "blocked_of_refusers": { "n": 86, "of": 287, "pct": 30 }, "blocked_of_live_plus_refusers": { "n": 440, "of": 3263, "pct": 13.5 } }, { "token": "ccbot", "label": "CCBot", "vendor": "Common Crawl", "cls": "training", "mayIgnoreRobots": false, "blocked_of_live_sites": { "n": 357, "of": 2976, "pct": 12 }, "blocked_of_robots_files": { "n": 357, "of": 2414, "pct": 14.8 }, "named_block_of_robots_files": { "n": 281, "of": 2414, "pct": 11.6 }, "named_block_of_live_sites": { "n": 281, "of": 2976, "pct": 9.4 }, "wildcard_only_block_of_robots_files": { "n": 76, "of": 2414, "pct": 3.1 }, "blocked_of_all_robots_files": { "n": 467, "of": 2743, "pct": 17 }, "named_block_of_all_robots_files": { "n": 360, "of": 2743, "pct": 13.1 }, "blocked_of_refusers": { "n": 94, "of": 287, "pct": 32.8 }, "blocked_of_live_plus_refusers": { "n": 451, "of": 3263, "pct": 13.8 } } ], "by_class": [ { "cls": "search", "label": "Search-engine crawlers (Googlebot, Bingbot, Meta-WebIndexer, Amzn-SearchBot)", "bots": [ "googlebot", "bingbot", "meta-webindexer", "amzn-searchbot" ], "blocks_any_of_live_sites": { "n": 176, "of": 2976, "pct": 5.9 }, "blocks_all_of_live_sites": { "n": 45, "of": 2976, "pct": 1.5 }, "names_and_blocks_any_of_live_sites": { "n": 91, "of": 2976, "pct": 3.1 } }, { "cls": "retrieval", "label": "AI search indexes (OAI-SearchBot, Claude-SearchBot, PerplexityBot, MistralAI-Index, YouBot)", "bots": [ "oai-searchbot", "claude-searchbot", "perplexitybot", "mistralai-index", "youbot" ], "blocks_any_of_live_sites": { "n": 298, "of": 2976, "pct": 10 }, "blocks_all_of_live_sites": { "n": 73, "of": 2976, "pct": 2.5 }, "names_and_blocks_any_of_live_sites": { "n": 218, "of": 2976, "pct": 7.3 } }, { "cls": "user-fetch", "label": "User-triggered fetchers (ChatGPT-User, Claude-User, Perplexity-User, …)", "bots": [ "chatgpt-user", "claude-user", "perplexity-user", "google-gemininotebook", "meta-externalfetcher", "amzn-user", "duckassistbot", "mistralai-user" ], "blocks_any_of_live_sites": { "n": 285, "of": 2976, "pct": 9.6 }, "blocks_all_of_live_sites": { "n": 71, "of": 2976, "pct": 2.4 }, "names_and_blocks_any_of_live_sites": { "n": 204, "of": 2976, "pct": 6.9 } }, { "cls": "agent", "label": "Agents (Google-Agent)", "bots": [ "google-agent" ], "blocks_any_of_live_sites": { "n": 101, "of": 2976, "pct": 3.4 }, "blocks_all_of_live_sites": { "n": 101, "of": 2976, "pct": 3.4 }, "names_and_blocks_any_of_live_sites": { "n": 9, "of": 2976, "pct": 0.3 } }, { "cls": "mixed", "label": "Mixed-purpose (Applebot, Meta-ExternalAgent, Amazonbot)", "bots": [ "applebot", "meta-externalagent", "amazonbot" ], "blocks_any_of_live_sites": { "n": 329, "of": 2976, "pct": 11.1 }, "blocks_all_of_live_sites": { "n": 74, "of": 2976, "pct": 2.5 }, "names_and_blocks_any_of_live_sites": { "n": 253, "of": 2976, "pct": 8.5 } }, { "cls": "training", "label": "Training crawlers and tokens (GPTBot, ClaudeBot, Google-Extended, CCBot, …)", "bots": [ "gptbot", "claudebot", "google-extended", "applebot-extended", "mistralai-training", "bytespider", "ccbot" ], "blocks_any_of_live_sites": { "n": 461, "of": 2976, "pct": 15.5 }, "blocks_all_of_live_sites": { "n": 82, "of": 2976, "pct": 2.8 }, "names_and_blocks_any_of_live_sites": { "n": 384, "of": 2976, "pct": 12.9 } }, { "cls": "other", "label": "Other (ads, Vertex, …)", "bots": [ "oai-adsbot", "google-cloudvertexbot", "meta-externalads" ], "blocks_any_of_live_sites": { "n": 170, "of": 2976, "pct": 5.7 }, "blocks_all_of_live_sites": { "n": 87, "of": 2976, "pct": 2.9 }, "names_and_blocks_any_of_live_sites": { "n": 85, "of": 2976, "pct": 2.9 } } ], "training_vs_search": { "pairs": [ { "vendor": "OpenAI", "training_bot": "gptbot", "search_bot": "oai-searchbot", "training_blocked": { "n": 345, "of": 2976, "pct": 11.6 }, "search_blocked": { "n": 178, "of": 2976, "pct": 6 }, "training_only": { "n": 168, "of": 2976, "pct": 5.6 }, "both": { "n": 177, "of": 2976, "pct": 5.9 }, "search_only": { "n": 1, "of": 2976, "pct": 0 }, "of_training_blockers_still_allow_search": { "n": 168, "of": 345, "pct": 48.7 } }, { "vendor": "Anthropic", "training_bot": "claudebot", "search_bot": "claude-searchbot", "training_blocked": { "n": 316, "of": 2976, "pct": 10.6 }, "search_blocked": { "n": 169, "of": 2976, "pct": 5.7 }, "training_only": { "n": 148, "of": 2976, "pct": 5 }, "both": { "n": 168, "of": 2976, "pct": 5.6 }, "search_only": { "n": 1, "of": 2976, "pct": 0 }, "of_training_blockers_still_allow_search": { "n": 148, "of": 316, "pct": 46.8 } }, { "vendor": "Mistral", "training_bot": "mistralai-training", "search_bot": "mistralai-index", "training_blocked": { "n": 99, "of": 2976, "pct": 3.3 }, "search_blocked": { "n": 97, "of": 2976, "pct": 3.3 }, "training_only": { "n": 5, "of": 2976, "pct": 0.2 }, "both": { "n": 94, "of": 2976, "pct": 3.2 }, "search_only": { "n": 3, "of": 2976, "pct": 0.1 }, "of_training_blockers_still_allow_search": { "n": 5, "of": 99, "pct": 5.1 } } ], "class_split": { "blocks_any_training": { "n": 461, "of": 2976, "pct": 15.5 }, "blocks_any_ai_search": { "n": 298, "of": 2976, "pct": 10 }, "training_only_blockers": { "n": 166, "of": 2976, "pct": 5.6 }, "blocks_both": { "n": 295, "of": 2976, "pct": 9.9 }, "ai_search_only_blockers": { "n": 3, "of": 2976, "pct": 0.1 }, "of_training_blockers_allow_all_ai_search": { "n": 166, "of": 461, "pct": 36 }, "blocks_all_three_probed_search_bots": { "n": 142, "of": 2976, "pct": 4.8 } } }, "wildcard": { "robots_files_with_wildcard_group": { "n": 2324, "of": 2414, "pct": 96.3 }, "robots_files_wildcard_disallow_root": { "n": 93, "of": 2414, "pct": 3.9 }, "live_sites_wildcard_disallow_root": { "n": 93, "of": 2976, "pct": 3.1 } }, "edge": { "per_bot": [ { "bot": "OAI-SearchBot", "robots_allows_and_probe_answered": 2758, "robots_allows_but_edge_blocks": { "n": 226, "of": 2758, "pct": 8.2 }, "verdict_mix": { "challenge": 152, "refused": 66, "paywall": 8 }, "status_mix": { "402": 8, "403": 53, "404": 2, "405": 1, "406": 2, "410": 1, "418": 2, "429": 3, "503": 2, "challenge (403)": 147, "challenge (200)": 2, "challenge (444)": 2, "challenge (429)": 1 }, "unknown_probe_timeouts": 26, "robots_file_allows_but_edge_blocks": { "n": 124, "of": 2202, "pct": 5.6 }, "no_robots_file_but_edge_blocks": { "n": 102, "of": 556, "pct": 18.3 }, "robots_names_and_allows_but_edge_blocks": { "n": 10, "of": 173, "pct": 5.8 }, "robots_disallows_and_probe_answered": 174, "robots_disallows_but_edge_serves": { "n": 127, "of": 174, "pct": 73 }, "robots_names_and_disallows_but_edge_serves": { "n": 73, "of": 106, "pct": 68.9 }, "by_cdn": { "Cloudflare": { "n": 18, "of": 691, "pct": 2.6 }, "Akamai": { "n": 98, "of": 191, "pct": 51.3 }, "Fastly": { "n": 1, "of": 6, "pct": 16.7 }, "Amazon CloudFront": { "n": 23, "of": 394, "pct": 5.8 }, "Vercel": { "n": 1, "of": 56, "pct": 1.8 }, "Netlify": { "n": 0, "of": 26, "pct": 0 }, "none detected": { "n": 85, "of": 1394, "pct": 6.1 } } }, { "bot": "PerplexityBot", "robots_allows_and_probe_answered": 2698, "robots_allows_but_edge_blocks": { "n": 245, "of": 2698, "pct": 9.1 }, "verdict_mix": { "challenge": 151, "refused": 86, "paywall": 8 }, "status_mix": { "402": 8, "403": 71, "404": 1, "405": 1, "406": 2, "412": 1, "418": 2, "429": 3, "502": 1, "503": 2, "504": 1, "520": 1, "challenge (403)": 144, "challenge (200)": 4, "challenge (444)": 2, "challenge (429)": 1 }, "unknown_probe_timeouts": 19, "robots_file_allows_but_edge_blocks": { "n": 142, "of": 2142, "pct": 6.6 }, "no_robots_file_but_edge_blocks": { "n": 103, "of": 556, "pct": 18.5 }, "robots_names_and_allows_but_edge_blocks": { "n": 12, "of": 176, "pct": 6.8 }, "robots_disallows_and_probe_answered": 241, "robots_disallows_but_edge_serves": { "n": 172, "of": 241, "pct": 71.4 }, "robots_names_and_disallows_but_edge_serves": { "n": 118, "of": 173, "pct": 68.2 }, "by_cdn": { "Cloudflare": { "n": 37, "of": 681, "pct": 5.4 }, "Akamai": { "n": 96, "of": 189, "pct": 50.8 }, "Fastly": { "n": 0, "of": 6, "pct": 0 }, "Amazon CloudFront": { "n": 20, "of": 375, "pct": 5.3 }, "Vercel": { "n": 1, "of": 54, "pct": 1.9 }, "Netlify": { "n": 0, "of": 26, "pct": 0 }, "none detected": { "n": 91, "of": 1367, "pct": 6.7 } } }, { "bot": "Claude-SearchBot", "robots_allows_and_probe_answered": 2771, "robots_allows_but_edge_blocks": { "n": 200, "of": 2771, "pct": 7.2 }, "verdict_mix": { "challenge": 143, "refused": 45, "paywall": 12 }, "status_mix": { "402": 12, "403": 36, "404": 1, "405": 1, "410": 1, "418": 2, "429": 1, "503": 1, "504": 1, "520": 1, "challenge (403)": 134, "challenge (200)": 5, "challenge (429)": 2, "challenge (444)": 2 }, "unknown_probe_timeouts": 23, "robots_file_allows_but_edge_blocks": { "n": 108, "of": 2214, "pct": 4.9 }, "no_robots_file_but_edge_blocks": { "n": 92, "of": 557, "pct": 16.5 }, "robots_names_and_allows_but_edge_blocks": { "n": 8, "of": 91, "pct": 8.8 }, "robots_disallows_and_probe_answered": 166, "robots_disallows_but_edge_serves": { "n": 129, "of": 166, "pct": 77.7 }, "robots_names_and_disallows_but_edge_serves": { "n": 64, "of": 90, "pct": 71.1 }, "by_cdn": { "Cloudflare": { "n": 22, "of": 690, "pct": 3.2 }, "Akamai": { "n": 95, "of": 197, "pct": 48.2 }, "Fastly": { "n": 0, "of": 6, "pct": 0 }, "Amazon CloudFront": { "n": 12, "of": 389, "pct": 3.1 }, "Vercel": { "n": 1, "of": 57, "pct": 1.8 }, "Netlify": { "n": 0, "of": 26, "pct": 0 }, "none detected": { "n": 70, "of": 1406, "pct": 5 } } } ], "any_of_three": { "definition": "Live sites whose browser-UA fetch got a normal 2xx page, whose robots.txt allows all three probed bots at '/', and where all three probes got an HTTP answer", "eligible": 2672, "at_least_one_blocked": { "n": 263, "of": 2672, "pct": 9.8 }, "all_three_blocked": { "n": 166, "of": 2672, "pct": 6.2 }, "by_verdict_at_least_one": { "refused": 99, "challenge": 157, "paywall_402": 10 }, "status_429_involved": 7, "examples": [ { "rank": 52, "domain": "digicert.com", "cdn": null, "probes": { "OAI-SearchBot": "ok/200", "PerplexityBot": "challenge/200", "Claude-SearchBot": "challenge/200" } }, { "rank": 63, "domain": "adobe.com", "cdn": null, "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 65, "domain": "endpoints.news", "cdn": "Amazon CloudFront", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "ok/200" } }, { "rank": 100, "domain": "wordpress.com", "cdn": null, "probes": { "OAI-SearchBot": "refused/403", "PerplexityBot": "refused/403", "Claude-SearchBot": "refused/403" } }, { "rank": 110, "domain": "openai.com", "cdn": "Cloudflare", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 132, "domain": "intuit.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "refused/429", "PerplexityBot": "ok/200", "Claude-SearchBot": "ok/200" } }, { "rank": 154, "domain": "tumblr.com", "cdn": null, "probes": { "OAI-SearchBot": "refused/406", "PerplexityBot": "ok/200", "Claude-SearchBot": "ok/200" } }, { "rank": 161, "domain": "unity3d.com", "cdn": null, "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 162, "domain": "webex.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 178, "domain": "xiaomi.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 197, "domain": "medium.com", "cdn": "Cloudflare", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 228, "domain": "cisco.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 243, "domain": "meraki.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 274, "domain": "ibm.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "refused/403", "PerplexityBot": "refused/403", "Claude-SearchBot": "refused/403" } }, { "rank": 289, "domain": "pubmatic.com", "cdn": "Amazon CloudFront", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "ok/200" } }, { "rank": 297, "domain": "springer.com", "cdn": null, "probes": { "OAI-SearchBot": "ok/200", "PerplexityBot": "refused/406", "Claude-SearchBot": "ok/200" } }, { "rank": 300, "domain": "autodesk.com", "cdn": "Amazon CloudFront", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 305, "domain": "salesforce.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 320, "domain": "mi.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 361, "domain": "cdc.gov", "cdn": null, "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 442, "domain": "intel.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 477, "domain": "wp.com", "cdn": null, "probes": { "OAI-SearchBot": "refused/403", "PerplexityBot": "refused/403", "Claude-SearchBot": "refused/403" } }, { "rank": 546, "domain": "trendmicro.com", "cdn": null, "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 569, "domain": "cornell.edu", "cdn": null, "probes": { "OAI-SearchBot": "refused/403", "PerplexityBot": "ok/200", "Claude-SearchBot": "ok/200" } }, { "rank": 593, "domain": "amazonalexa.com", "cdn": "Amazon CloudFront", "probes": { "OAI-SearchBot": "refused/503", "PerplexityBot": "refused/503", "Claude-SearchBot": "ok/200" } }, { "rank": 626, "domain": "mdpi.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 636, "domain": "accuweather.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 645, "domain": "ups.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/200", "PerplexityBot": "challenge/200", "Claude-SearchBot": "challenge/200" } }, { "rank": 658, "domain": "chatgpt.site", "cdn": "Cloudflare", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 665, "domain": "nike.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 713, "domain": "shopee.com.br", "cdn": null, "probes": { "OAI-SearchBot": "refused/403", "PerplexityBot": "refused/403", "Claude-SearchBot": "ok/200" } }, { "rank": 725, "domain": "airbnb.com", "cdn": null, "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 734, "domain": "itch.io", "cdn": "Cloudflare", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "ok/200", "Claude-SearchBot": "challenge/403" } }, { "rank": 738, "domain": "independent.co.uk", "cdn": null, "probes": { "OAI-SearchBot": "ok/200", "PerplexityBot": "paywall/402", "Claude-SearchBot": "ok/200" } }, { "rank": 743, "domain": "shopee.co.id", "cdn": null, "probes": { "OAI-SearchBot": "refused/403", "PerplexityBot": "refused/403", "Claude-SearchBot": "ok/200" } }, { "rank": 753, "domain": "fda.gov", "cdn": "Akamai", "probes": { "OAI-SearchBot": "refused/404", "PerplexityBot": "refused/404", "Claude-SearchBot": "refused/404" } }, { "rank": 772, "domain": "homedepot.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 795, "domain": "zillow.com", "cdn": "Amazon CloudFront", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 808, "domain": "lenovo.com", "cdn": "Akamai", "probes": { "OAI-SearchBot": "challenge/403", "PerplexityBot": "challenge/403", "Claude-SearchBot": "challenge/403" } }, { "rank": 817, "domain": "xvideos.com", "cdn": null, "probes": { "OAI-SearchBot": "ok/200", "PerplexityBot": "refused/403", "Claude-SearchBot": "ok/200" } } ] } }, "by_cdn": [ { "cdn": "Cloudflare", "live_sites": 723, "share_of_live_sites": 23.9, "browser_baseline_challenged_or_refused": { "n": 8, "of": 723, "pct": 1.1 }, "robots_blocks_gptbot": { "n": 53, "of": 720, "pct": 7.4 }, "robots_blocks_oai_searchbot": { "n": 19, "of": 720, "pct": 2.6 }, "robots_blocks_any_ai_search": { "n": 37, "of": 720, "pct": 5.1 }, "edge_eligible": 675, "edge_blocks_at_least_one_despite_robots_allow": { "n": 42, "of": 675, "pct": 6.2 }, "edge_verdicts": { "refused": 25, "challenge": 13, "paywall_402": 5 }, "clean_baseline_sites": 715, "cf_managed_robots": { "n": 3, "of": 720, "pct": 0.4 } }, { "cdn": "Akamai", "live_sites": 249, "share_of_live_sites": 8.2, "browser_baseline_challenged_or_refused": { "n": 4, "of": 249, "pct": 1.6 }, "robots_blocks_gptbot": { "n": 27, "of": 221, "pct": 12.2 }, "robots_blocks_oai_searchbot": { "n": 17, "of": 221, "pct": 7.7 }, "robots_blocks_any_ai_search": { "n": 21, "of": 221, "pct": 9.5 }, "edge_eligible": 188, "edge_blocks_at_least_one_despite_robots_allow": { "n": 97, "of": 188, "pct": 51.6 }, "edge_verdicts": { "refused": 11, "challenge": 85, "paywall_402": 1 }, "clean_baseline_sites": 245 }, { "cdn": "Fastly", "live_sites": 6, "share_of_live_sites": 0.2, "browser_baseline_challenged_or_refused": { "n": 0, "of": 6, "pct": 0 }, "robots_blocks_gptbot": { "n": 0, "of": 6, "pct": 0 }, "robots_blocks_oai_searchbot": { "n": 0, "of": 6, "pct": 0 }, "robots_blocks_any_ai_search": { "n": 0, "of": 6, "pct": 0 }, "edge_eligible": 6, "edge_blocks_at_least_one_despite_robots_allow": { "n": 1, "of": 6, "pct": 16.7 }, "edge_verdicts": { "refused": 1, "challenge": 0, "paywall_402": 0 }, "clean_baseline_sites": 6 }, { "cdn": "Amazon CloudFront", "live_sites": 436, "share_of_live_sites": 14.4, "browser_baseline_challenged_or_refused": { "n": 3, "of": 436, "pct": 0.7 }, "robots_blocks_gptbot": { "n": 76, "of": 435, "pct": 17.5 }, "robots_blocks_oai_searchbot": { "n": 37, "of": 435, "pct": 8.5 }, "robots_blocks_any_ai_search": { "n": 67, "of": 435, "pct": 15.4 }, "edge_eligible": 374, "edge_blocks_at_least_one_despite_robots_allow": { "n": 22, "of": 374, "pct": 5.9 }, "edge_verdicts": { "refused": 7, "challenge": 15, "paywall_402": 1 }, "clean_baseline_sites": 433 }, { "cdn": "Vercel", "live_sites": 57, "share_of_live_sites": 1.9, "browser_baseline_challenged_or_refused": { "n": 0, "of": 57, "pct": 0 }, "robots_blocks_gptbot": { "n": 3, "of": 57, "pct": 5.3 }, "robots_blocks_oai_searchbot": { "n": 1, "of": 57, "pct": 1.8 }, "robots_blocks_any_ai_search": { "n": 4, "of": 57, "pct": 7 }, "edge_eligible": 54, "edge_blocks_at_least_one_despite_robots_allow": { "n": 1, "of": 54, "pct": 1.9 }, "edge_verdicts": { "refused": 0, "challenge": 1, "paywall_402": 0 }, "clean_baseline_sites": 57 }, { "cdn": "Netlify", "live_sites": 26, "share_of_live_sites": 0.9, "browser_baseline_challenged_or_refused": { "n": 0, "of": 26, "pct": 0 }, "robots_blocks_gptbot": { "n": 1, "of": 26, "pct": 3.8 }, "robots_blocks_oai_searchbot": { "n": 0, "of": 26, "pct": 0 }, "robots_blocks_any_ai_search": { "n": 0, "of": 26, "pct": 0 }, "edge_eligible": 26, "edge_blocks_at_least_one_despite_robots_allow": { "n": 0, "of": 26, "pct": 0 }, "edge_verdicts": { "refused": 0, "challenge": 0, "paywall_402": 0 }, "clean_baseline_sites": 26 }, { "cdn": "none detected", "live_sites": 1526, "share_of_live_sites": 50.5, "browser_baseline_challenged_or_refused": { "n": 1, "of": 1526, "pct": 0.1 }, "robots_blocks_gptbot": { "n": 185, "of": 1511, "pct": 12.2 }, "robots_blocks_oai_searchbot": { "n": 104, "of": 1511, "pct": 6.9 }, "robots_blocks_any_ai_search": { "n": 169, "of": 1511, "pct": 11.2 }, "edge_eligible": 1349, "edge_blocks_at_least_one_despite_robots_allow": { "n": 100, "of": 1349, "pct": 7.4 }, "edge_verdicts": { "refused": 55, "challenge": 43, "paywall_402": 3 }, "clean_baseline_sites": 1525 } ], "content_signals": { "robots_files_with_content_signal": { "n": 88, "of": 2414, "pct": 3.6 }, "live_sites_with_content_signal_in_robots": { "n": 88, "of": 3023, "pct": 2.9 }, "robots_files_with_content_usage": { "n": 8, "of": 2414, "pct": 0.3 }, "live_sites_with_signal_response_header": { "n": 2, "of": 3023, "pct": 0.1 }, "content_signal_values_by_site": { "ai-train": { "yes": 47, "no": 44 }, "search": { "yes": 88, "no": 2 }, "ai-input": { "yes": 76, "no": 13 }, "use": { "reference": 7 } }, "top_content_signal_lines": [ [ "ai-train=yes,search=yes,ai-input=yes", 25 ], [ "search=yes,ai-input=yes,ai-train=yes", 17 ], [ "ai-train=no,search=yes,ai-input=yes", 14 ], [ "search=yes,ai-input=yes,ai-train=no", 11 ], [ "ai-train=no,search=yes,ai-input=no", 8 ], [ "search=yes,ai-input=yes,ai-train=no,use=reference", 4 ], [ "ai-train=yes,ai-input=yes,search=yes", 3 ], [ "search=yes,ai-train=no,use=reference", 3 ], [ "search=yes,ai-train=no", 3 ], [ "search=no,ai-input=no,ai-train=no", 2 ], [ "ai-train=no,search=yes", 1 ], [ "search=yes,ai-input=yes", 1 ] ], "top_content_usage_lines": [ [ "train-ai=n", 5 ], [ "ai=n", 2 ], [ "ai=y", 1 ] ], "header_examples": [ { "domain": "wpastra.com", "signals": [ "Content-Signal header: ai-train=no, search=yes, ai-input=yes" ] }, { "domain": "freshdesk.com", "signals": [ "Content-Signal header: search=yes, ai-input=yes, ai-train=no, use=reference" ] } ], "robots_files_with_cloudflare_managed_block": { "n": 3, "of": 2414, "pct": 0.1 }, "ai_train_no_on_cloudflare": { "n": 24, "of": 638, "pct": 3.8 } }, "errors": { "script_errors": 0, "robots_no_response": { "n": 1304, "of": 5000, "pct": 26.1 }, "robots_no_response_kinds": { "no_response:dns": 899, "no_response:timeout": 240, "no_response:network": 165 }, "home_not_2xx_or_no_answer": { "n": 1977, "of": 5000, "pct": 39.5 }, "home_no_answer_kinds": { "dns": 902, "timeout": 197, "network": 168 }, "home_non_2xx_status": { "301": 3, "302": 11, "307": 4, "308": 1, "400": 18, "401": 9, "402": 10, "403": 525, "404": 76, "405": 4, "406": 2, "412": 1, "417": 1, "418": 5, "429": 16, "439": 2, "451": 1, "498": 2, "500": 1, "502": 4, "503": 10, "520": 1, "522": 1, "526": 2 }, "browser_baseline_on_live_sites": { "ok": 3007, "challenge": 16 }, "probe_requests": 9069, "probe_no_answer": { "n": 158, "of": 9069, "pct": 1.7 } }, "rank_tiers": [ { "ranks": "1-1000", "live_sites": 579, "robots_blocks_gptbot": { "n": 81, "of": 569, "pct": 14.2 }, "robots_blocks_oai_searchbot": { "n": 47, "of": 569, "pct": 8.3 }, "edge_blocks_at_least_one_despite_robots_allow": { "n": 48, "of": 496, "pct": 9.7 } }, { "ranks": "1001-5000", "live_sites": 2444, "robots_blocks_gptbot": { "n": 264, "of": 2407, "pct": 11 }, "robots_blocks_oai_searchbot": { "n": 131, "of": 2407, "pct": 5.4 }, "edge_blocks_at_least_one_despite_robots_allow": { "n": 215, "of": 2176, "pct": 9.9 } } ], "columns": { "rank": "Tranco rank (list 64VKX)", "domain": "registrable domain from the Tranco list", "robots_state": "see definitions.robots_states (no_response_ when no HTTP answer)", "robots_status": "final HTTP status of /robots.txt (after redirects; re-fetched on the home page's final origin if the home page redirected to another origin)", "robots_error": "fetch error kind when no HTTP answer: dns, network, timeout, redirects, unsafe", "robots_bytes": "robots.txt body size in bytes (capped at 512,000)", "robots_cf_managed_block": "1 if robots.txt contains Cloudflare's '# BEGIN Cloudflare Managed content' block", "wildcard_disallow_root": "1 if the User-agent: * group disallows '/'", "content_signal": "Content-Signal lines in robots.txt (values only, ' | ' separated)", "content_usage": "Content-Usage (IETF aipref) lines in robots.txt", "signal_headers": "Content-Signal / Content-Usage HTTP response headers on the home page", "cdn": "CDN detected from response headers (see definitions.cdn)", "home_final_host": "host the browser-UA home page request landed on", "home_status": "final HTTP status for the browser-UA home page request", "home_error": "fetch error kind when no HTTP answer", "home_bytes": "browser-UA home page size in bytes (capped at 2,000,000)", "home_words": "visible words in the browser-UA HTML (scripts, styles, noscript, template, svg removed)", "home_baseline": "judgeProbe verdict for the browser-UA response (ok / challenge / refused)", "oai_searchbot_status": "HTTP status for the simulated bot-UA home page request (simulated user agent, not IP-verified)", "oai_searchbot_bytes": "bot-UA response size in bytes (capped at 96,000)", "oai_searchbot_verdict": "engine judgeProbe verdict (see definitions.probe_verdicts)", "oai_searchbot_error": "fetch error kind for the bot-UA request when no HTTP answer", "perplexitybot_status": "HTTP status for the simulated bot-UA home page request (simulated user agent, not IP-verified)", "perplexitybot_bytes": "bot-UA response size in bytes (capped at 96,000)", "perplexitybot_verdict": "engine judgeProbe verdict (see definitions.probe_verdicts)", "perplexitybot_error": "fetch error kind for the bot-UA request when no HTTP answer", "claude_searchbot_status": "HTTP status for the simulated bot-UA home page request (simulated user agent, not IP-verified)", "claude_searchbot_bytes": "bot-UA response size in bytes (capped at 96,000)", "claude_searchbot_verdict": "engine judgeProbe verdict (see definitions.probe_verdicts)", "claude_searchbot_error": "fetch error kind for the bot-UA request when no HTTP answer", "bot_oai_searchbot": "robots.txt verdict at '/' for OAI-SearchBot (retrieval); codes in definitions.bot_column_codes", "bot_chatgpt_user": "robots.txt verdict at '/' for ChatGPT-User (user-fetch); codes in definitions.bot_column_codes", "bot_gptbot": "robots.txt verdict at '/' for GPTBot (training); codes in definitions.bot_column_codes", "bot_oai_adsbot": "robots.txt verdict at '/' for OAI-AdsBot (other); codes in definitions.bot_column_codes", "bot_claude_searchbot": "robots.txt verdict at '/' for Claude-SearchBot (retrieval); codes in definitions.bot_column_codes", "bot_claude_user": "robots.txt verdict at '/' for Claude-User (user-fetch); codes in definitions.bot_column_codes", "bot_claudebot": "robots.txt verdict at '/' for ClaudeBot (training); codes in definitions.bot_column_codes", "bot_perplexitybot": "robots.txt verdict at '/' for PerplexityBot (retrieval); codes in definitions.bot_column_codes", "bot_perplexity_user": "robots.txt verdict at '/' for Perplexity-User (user-fetch); codes in definitions.bot_column_codes", "bot_googlebot": "robots.txt verdict at '/' for Googlebot (search); codes in definitions.bot_column_codes", "bot_google_extended": "robots.txt verdict at '/' for Google-Extended (training); codes in definitions.bot_column_codes", "bot_google_cloudvertexbot": "robots.txt verdict at '/' for Google-CloudVertexBot (other); codes in definitions.bot_column_codes", "bot_google_agent": "robots.txt verdict at '/' for Google-Agent (agent); codes in definitions.bot_column_codes", "bot_google_gemininotebook": "robots.txt verdict at '/' for Google-GeminiNotebook (user-fetch); codes in definitions.bot_column_codes", "bot_bingbot": "robots.txt verdict at '/' for Bingbot (search); codes in definitions.bot_column_codes", "bot_applebot": "robots.txt verdict at '/' for Applebot (mixed); codes in definitions.bot_column_codes", "bot_applebot_extended": "robots.txt verdict at '/' for Applebot-Extended (training); codes in definitions.bot_column_codes", "bot_meta_webindexer": "robots.txt verdict at '/' for Meta-WebIndexer (search); codes in definitions.bot_column_codes", "bot_meta_externalfetcher": "robots.txt verdict at '/' for Meta-ExternalFetcher (user-fetch); codes in definitions.bot_column_codes", "bot_meta_externalagent": "robots.txt verdict at '/' for Meta-ExternalAgent (mixed); codes in definitions.bot_column_codes", "bot_meta_externalads": "robots.txt verdict at '/' for Meta-ExternalAds (other); codes in definitions.bot_column_codes", "bot_amzn_searchbot": "robots.txt verdict at '/' for Amzn-SearchBot (search); codes in definitions.bot_column_codes", "bot_amzn_user": "robots.txt verdict at '/' for Amzn-User (user-fetch); codes in definitions.bot_column_codes", "bot_amazonbot": "robots.txt verdict at '/' for Amazonbot (mixed); codes in definitions.bot_column_codes", "bot_duckassistbot": "robots.txt verdict at '/' for DuckAssistBot (user-fetch); codes in definitions.bot_column_codes", "bot_mistralai_index": "robots.txt verdict at '/' for MistralAI-Index (retrieval); codes in definitions.bot_column_codes", "bot_mistralai_user": "robots.txt verdict at '/' for MistralAI-User (user-fetch); codes in definitions.bot_column_codes", "bot_mistralai_training": "robots.txt verdict at '/' for MistralAI-Training (training); codes in definitions.bot_column_codes", "bot_youbot": "robots.txt verdict at '/' for YouBot (retrieval); codes in definitions.bot_column_codes", "bot_bytespider": "robots.txt verdict at '/' for Bytespider (training); codes in definitions.bot_column_codes", "bot_ccbot": "robots.txt verdict at '/' for CCBot (training); codes in definitions.bot_column_codes", "edge_block_class_pass2": "For sites in the headline base whose simulated AI search-bot request was blocked: second-pass class (see second_pass.class_definitions); empty otherwise" }, "second_pass": { "when_utc": { "first": "2026-10-13T17:10:50.521Z", "last": "2026-10-13T17:13:21.844Z" }, "what": "Every site where a simulated bot request was refused, challenged or got 402 (plus a comparison sample of 300 sites where all three were served) was requested once more with the browser UA, the three AI search-bot UAs and two controls: our own honest bot UA (GEOAuditBot/1.0) and a spoofed Googlebot UA. Same politeness rules.", "reproducibility": { "blocked_pairs": 667, "reproduced": { "n": 658, "of": 667, "pct": 98.7 }, "by_bot": { "OAI-SearchBot": { "n": 222, "of": 225, "pct": 98.7 }, "PerplexityBot": { "n": 241, "of": 244, "pct": 98.8 }, "Claude-SearchBot": { "n": 195, "of": 198, "pct": 98.5 } } }, "headline_base": { "eligible": 2672, "blocked_pass1": 263, "classes": { "generic_bot_block": 95, "verifies_bot_identity": 85, "ai_ua_specific": 77, "not_reproduced": 5, "no_usable_recheck": 1 }, "classes_by_cdn": { "Cloudflare": { "ai_ua_specific": 31, "generic_bot_block": 5, "verifies_bot_identity": 5, "not_reproduced": 1 }, "Akamai": { "generic_bot_block": 51, "verifies_bot_identity": 42, "ai_ua_specific": 3, "not_reproduced": 1 }, "Amazon CloudFront": { "generic_bot_block": 11, "ai_ua_specific": 7, "verifies_bot_identity": 3, "no_usable_recheck": 1 }, "none detected": { "ai_ua_specific": 36, "verifies_bot_identity": 34, "generic_bot_block": 28, "not_reproduced": 2 } }, "ai_specific_share_of_eligible": { "n": 77, "of": 2672, "pct": 2.9 }, "ai_specific_which_bots": { "PerplexityBot": 38, "OAI-SearchBot + PerplexityBot": 13, "OAI-SearchBot + PerplexityBot + Claude-SearchBot": 10, "OAI-SearchBot": 9, "OAI-SearchBot + Claude-SearchBot": 5, "Claude-SearchBot": 2 }, "paywall_402_sites": 10 }, "class_definitions": { "generic_bot_block": "the site also refused our honest, unknown bot UA: it blocks non-browser clients in general", "verifies_bot_identity": "the site served our unknown bot but also refused a spoofed Googlebot: it likely checks that a request claiming to be a known crawler comes from that crawler's IPs, so the real, verified AI crawler may be let through", "ai_ua_specific": "the site served both our unknown bot and a spoofed Googlebot, but refused the AI search-bot UA: a rule aimed at AI crawlers by name, which the real crawler would probably also meet unless the rule exempts verified IPs", "not_reproduced": "the block did not repeat on the second pass", "inconclusive": "a control request timed out" }, "markers_on_reproduced_blocks": { "access denied": 137, "(no challenge marker)": 117, "request blocked": 11, "attention required! | cloudflare": 5, "just a moment...": 5, "challenge-platform": 4, "incapsula incident": 3, "sucuri website firewall": 1 }, "server_headers_on_reproduced_blocks": { "akamaighost": 101, "(none)": 63, "cloudflare": 47, "nginx": 23, "cloudfront": 11, "sgw": 9, "akamainetstorage": 5, "ddos-guard": 4, "varnish": 3, "amazons3": 2, "apache": 2, "server": 1, "blb": 1, "ring publishing - accelerator": 1, "eagleshield": 1, "akamai": 1, "web cache": 1, "ats": 1, "qrator": 1, "cep": 1, "webserver": 1, "ycalb": 1, "awselb": 1, "sucuri": 1 }, "ok_sample": { "sites": 298, "probe_now_blocked": { "n": 1, "of": 298, "pct": 0.3 }, "own_bot_blocked": { "n": 4, "of": 297, "pct": 1.3 }, "googlebot_spoof_blocked": { "n": 21, "of": 297, "pct": 7.1 } } } }