| 2026-09-17 | Applebot | observed_paths_fetched | new | | robots_txt_and_home_only |
| 2026-09-17 | Googlebot | observed_verified_request_ratio_census | new | | 0.991 |
| 2026-09-17 | OAI-SearchBot | observed_arrival_before_sitemap | new | | yes |
| 2026-09-17 | GoogleOther | observed_fetches_robots_txt | new | | no |
| 2026-09-17 | GoogleOther | observed_requested_mcp_endpoint | new | | yes |
| 2026-09-17 | ClaudeBot | observed_format_pass_order | new | | html_json_md |
| 2026-09-17 | ClaudeBot | observed_first_request_path | new | | /sitemap.xml |
| 2026-09-17 | ClaudeBot | observed_fetches_json | new | | yes |
| 2026-09-17 | ClaudeBot | observed_fetches_markdown | new | | yes |
| 2026-09-14 | Googlebot | reads_llms_txt | new | | no |
| 2026-09-14 | Bingbot | follows_noarchive | new | | yes |
| 2026-09-14 | Bingbot | opt_out_mechanism | new | | yes |
| 2026-09-14 | Bingbot | user_agent_string_full | new | | Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) |
| 2026-09-14 | Bingbot | reads_llms_txt | new | | not_documented |
| 2026-09-14 | Bingbot | cites_sources | new | | yes |
| 2026-09-14 | Bingbot | separate_search_and_training_tokens | new | | no |
| 2026-09-14 | Bingbot | purpose_documented | new | | yes |
| 2026-09-14 | Bingbot | verifiable_by_rdns | new | | yes |
| 2026-09-14 | Bingbot | publishes_ip_list | new | | yes |
| 2026-09-14 | Bingbot | honours_crawl_delay | new | | yes |
| 2026-09-14 | Bingbot | respects_robots_txt | new | | yes |
| 2026-09-13 | PerplexityBot | user_agent_string_full | new | | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot) |
| 2026-09-13 | PerplexityBot | reads_llms_txt | new | | not_documented |
| 2026-09-13 | PerplexityBot | cites_sources | new | | yes |
| 2026-09-13 | PerplexityBot | separate_search_and_training_tokens | new | | not_documented |
| 2026-09-13 | PerplexityBot | purpose_documented | new | | search |
| 2026-09-13 | PerplexityBot | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | PerplexityBot | publishes_ip_list | new | | yes |
| 2026-09-13 | PerplexityBot | honours_crawl_delay | new | | not_documented |
| 2026-09-13 | PerplexityBot | respects_robots_txt | new | | yes |
| 2026-09-13 | Perplexity-User | user_agent_string_full | new | | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user) |
| 2026-09-13 | Perplexity-User | reads_llms_txt | new | | not_documented |
| 2026-09-13 | Perplexity-User | cites_sources | new | | yes |
| 2026-09-13 | Perplexity-User | separate_search_and_training_tokens | new | | not_documented |
| 2026-09-13 | Perplexity-User | purpose_documented | new | | user_fetch |
| 2026-09-13 | Perplexity-User | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | Perplexity-User | publishes_ip_list | new | | yes |
| 2026-09-13 | Perplexity-User | honours_crawl_delay | new | | not_documented |
| 2026-09-13 | Perplexity-User | respects_robots_txt | new | | no |
| 2026-09-13 | OAI-SearchBot | user_agent_string_full | new | | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot |
| 2026-09-13 | OAI-SearchBot | reads_llms_txt | new | | not_documented |
| 2026-09-13 | OAI-SearchBot | cites_sources | new | | yes |
| 2026-09-13 | OAI-SearchBot | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | OAI-SearchBot | purpose_documented | new | | search |
| 2026-09-13 | OAI-SearchBot | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | OAI-SearchBot | publishes_ip_list | new | | yes |
| 2026-09-13 | OAI-SearchBot | honours_crawl_delay | new | | not_documented |
| 2026-09-13 | OAI-SearchBot | respects_robots_txt | new | | yes |
| 2026-09-13 | OAI-AdsBot | scope_of_crawling | new | | limited to submitted ad landing pages |
| 2026-09-13 | OAI-AdsBot | user_agent_string_full | new | | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot |
| 2026-09-13 | OAI-AdsBot | separate_search_and_training_tokens | new | | not_documented |
| 2026-09-13 | OAI-AdsBot | purpose_documented | new | | ads |
| 2026-09-13 | OAI-AdsBot | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | OAI-AdsBot | publishes_ip_list | new | | yes |
| 2026-09-13 | OAI-AdsBot | honours_crawl_delay | new | | not_documented |
| 2026-09-13 | OAI-AdsBot | respects_robots_txt | new | | not_documented |
| 2026-09-13 | GPTBot | robots_txt_fetch_marker | new | | yes |
| 2026-09-13 | GPTBot | user_agent_string_full | new | | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot |
| 2026-09-13 | GPTBot | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | GPTBot | purpose_documented | new | | training |
| 2026-09-13 | GPTBot | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | GPTBot | publishes_ip_list | new | | yes |
| 2026-09-13 | GPTBot | honours_crawl_delay | new | | not_documented |
| 2026-09-13 | GPTBot | respects_robots_txt | new | | yes |
| 2026-09-13 | GoogleOther | robots_txt_token | new | | GoogleOther |
| 2026-09-13 | GoogleOther | separate_search_and_training_tokens | new | | not_documented |
| 2026-09-13 | GoogleOther | purpose_documented | new | | mixed |
| 2026-09-13 | GoogleOther | verifiable_by_rdns | new | | yes |
| 2026-09-13 | GoogleOther | publishes_ip_list | new | | yes |
| 2026-09-13 | GoogleOther | honours_crawl_delay | new | | no |
| 2026-09-13 | GoogleOther | respects_robots_txt | new | | yes |
| 2026-09-13 | Googlebot | user_agent_string_full | new | | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) |
| 2026-09-13 | Googlebot | cites_sources | new | | yes |
| 2026-09-13 | Googlebot | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | Googlebot | purpose_documented | new | | search |
| 2026-09-13 | Googlebot | verifiable_by_rdns | new | | yes |
| 2026-09-13 | Googlebot | publishes_ip_list | new | | yes |
| 2026-09-13 | Googlebot | honours_crawl_delay | new | | no |
| 2026-09-13 | Googlebot | respects_robots_txt | new | | yes |
| 2026-09-13 | Google-Extended | robots_token_differs_from_ua | new | | yes |
| 2026-09-13 | Google-Extended | what_it_controls | new | | Gemini model training and grounding, not Search inclusion |
| 2026-09-13 | Google-Extended | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | Google-Extended | purpose_documented | new | | policy_only |
| 2026-09-13 | Google-Extended | verifiable_by_rdns | new | | n/a |
| 2026-09-13 | Google-Extended | publishes_ip_list | new | | no |
| 2026-09-13 | Google-Extended | honours_crawl_delay | new | | n/a |
| 2026-09-13 | Google-Extended | respects_robots_txt | new | | n/a |
| 2026-09-13 | ClaudeBot | user_agent_string_full | new | | not_documented |
| 2026-09-13 | ClaudeBot | anti_circumvention_stance | new | | documented |
| 2026-09-13 | ClaudeBot | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | ClaudeBot | purpose_documented | new | | training |
| 2026-09-13 | ClaudeBot | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | ClaudeBot | publishes_ip_list | new | | yes |
| 2026-09-13 | ClaudeBot | honours_crawl_delay | new | | yes |
| 2026-09-13 | ClaudeBot | respects_robots_txt | new | | yes |
| 2026-09-13 | Claude-User | user_agent_string_full | new | | not_documented |
| 2026-09-13 | Claude-User | reads_llms_txt | new | | not_documented |
| 2026-09-13 | Claude-User | cites_sources | new | | not_documented |
| 2026-09-13 | Claude-User | separate_search_and_training_tokens | new | | not_documented |
| 2026-09-13 | Claude-User | purpose_documented | new | | user_fetch |
| 2026-09-13 | Claude-User | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | Claude-User | publishes_ip_list | new | | yes |
| 2026-09-13 | Claude-User | honours_crawl_delay | new | | yes |
| 2026-09-13 | Claude-User | user_fetch_exception_documented | new | | no |
| 2026-09-13 | Claude-User | respects_robots_txt | new | | yes |
| 2026-09-13 | Claude-SearchBot | user_agent_string_full | new | | not_documented |
| 2026-09-13 | Claude-SearchBot | reads_llms_txt | new | | not_documented |
| 2026-09-13 | Claude-SearchBot | cites_sources | new | | not_documented |
| 2026-09-13 | Claude-SearchBot | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | Claude-SearchBot | purpose_documented | new | | search |
| 2026-09-13 | Claude-SearchBot | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | Claude-SearchBot | publishes_ip_list | new | | yes |
| 2026-09-13 | Claude-SearchBot | honours_crawl_delay | new | | yes |
| 2026-09-13 | Claude-SearchBot | respects_robots_txt | new | | yes |
| 2026-09-13 | ChatGPT-User | user_agent_string_full | new | | Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot |
| 2026-09-13 | ChatGPT-User | reads_llms_txt | new | | not_documented |
| 2026-09-13 | ChatGPT-User | cites_sources | new | | not_documented |
| 2026-09-13 | ChatGPT-User | separate_search_and_training_tokens | new | | not_documented |
| 2026-09-13 | ChatGPT-User | purpose_documented | new | | user_fetch |
| 2026-09-13 | ChatGPT-User | verifiable_by_rdns | new | | not_documented |
| 2026-09-13 | ChatGPT-User | publishes_ip_list | new | | yes |
| 2026-09-13 | ChatGPT-User | honours_crawl_delay | new | | not_documented |
| 2026-09-13 | ChatGPT-User | respects_robots_txt | new | | partial |
| 2026-09-13 | Applebot | fallback_to_googlebot_rules | new | | yes |
| 2026-09-13 | Applebot | user_agent_string_full | new | | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot) |
| 2026-09-13 | Applebot | reads_llms_txt | new | | not_documented |
| 2026-09-13 | Applebot | cites_sources | new | | yes |
| 2026-09-13 | Applebot | separate_search_and_training_tokens | new | | partial |
| 2026-09-13 | Applebot | purpose_documented | new | | mixed |
| 2026-09-13 | Applebot | verifiable_by_rdns | new | | yes |
| 2026-09-13 | Applebot | publishes_ip_list | new | | yes |
| 2026-09-13 | Applebot | honours_crawl_delay | new | | no |
| 2026-09-13 | Applebot | respects_robots_txt | new | | yes |
| 2026-09-13 | Applebot-Extended | robots_token_differs_from_ua | new | | yes |
| 2026-09-13 | Applebot-Extended | what_it_controls | new | | AI training use of content, not search indexing |
| 2026-09-13 | Applebot-Extended | separate_search_and_training_tokens | new | | yes |
| 2026-09-13 | Applebot-Extended | purpose_documented | new | | policy_only |
| 2026-09-13 | Applebot-Extended | verifiable_by_rdns | new | | n/a |
| 2026-09-13 | Applebot-Extended | publishes_ip_list | new | | no |
| 2026-09-13 | Applebot-Extended | honours_crawl_delay | new | | n/a |
| 2026-09-13 | Applebot-Extended | respects_robots_txt | new | | n/a |