{
  "question": {
    "id": 21,
    "ts_first": "2026-09-15T01:36:40Z",
    "ts_last": "2026-09-15T02:19:35Z",
    "text_raw": "Will a compliant, named AI crawler always avoid URLs disallowed in robots.txt?",
    "text_norm": "will a compliant named ai crawler alway avoid urls disallowed in robot txt",
    "hash": "f453cacd1535f4700e95bab1b1480233e943644b1baa9857d70aa5001c264ee7",
    "count": 2,
    "sources": "[\"prompt_batch\", \"engine:perplexity\", \"engine:claude\"]",
    "matched_claim_ids": "[19]",
    "gap": 0,
    "published": 1,
    "answer_record_id": null,
    "label": "seeded",
    "sources_display": "perplexity, claude"
  },
  "claims": [
    {
      "id": 19,
      "entity_id": 3,
      "slug": "chatgpt-user/respects_robots_txt",
      "field": "respects_robots_txt",
      "value": "partial",
      "statement": "ChatGPT-User fetches pages in response to user actions in ChatGPT, and OpenAI states that robots.txt rules may not apply to ChatGPT-User fetches.",
      "evidence_url": "https://developers.openai.com/api/docs/bots",
      "evidence_quote": "\"ChatGPT-User is not used for crawling the web in an automatic fashion. Because these actions are initiated by a user, robots.txt rules may not apply.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": null
    }
  ],
  "notes": [
    {
      "id": 10,
      "target_type": "question",
      "target_id": 21,
      "body": "A compliant, named AI crawler may not always avoid URLs disallowed in robots.txt. For example, ChatGPT-User fetches pages in response to user actions in ChatGPT, and OpenAI states that robots.txt rules may not apply to ChatGPT-User fetches [claim 19].",
      "author_claim": null,
      "ts": "2026-09-17T00:31:46.896Z",
      "status": "published"
    }
  ]
}