{
 "collected_utc": [
  "2026-11-13T23:39:53.893923+00:00",
  "2026-11-13T23:42:04.842204+00:00"
 ],
 "sample": "E3 50 (Nov 6) + first 50 not-already-included companies of the Forbes Cloud 100 2025 in rank order (ranks 1-65)",
 "all_100": {
  "sites_attempted": 100,
  "robots_parsed": 96,
  "robots_not_parsed": {
   "freshworks.com": "HTTP 404",
   "gusto.com": "HTTP 403",
   "cohesity.com": "HTTP 403",
   "cribl.io": "SSLError: HTTPSConnectionPool(host='cribl.io', port=443): Max retries exceeded with url: /robots.txt (Caused by SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WH"
  },
  "bots": {
   "GPTBot": {
    "named": 16,
    "homepage_allowed": 93,
    "homepage_disallowed_count": 3,
    "homepage_disallowed": [
     "figma.com",
     "canva.com",
     "loom.com"
    ]
   },
   "OAI-SearchBot": {
    "named": 13,
    "homepage_allowed": 96,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "ChatGPT-User": {
    "named": 14,
    "homepage_allowed": 96,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "ClaudeBot": {
    "named": 16,
    "homepage_allowed": 94,
    "homepage_disallowed_count": 2,
    "homepage_disallowed": [
     "figma.com",
     "canva.com"
    ]
   },
   "Claude-SearchBot": {
    "named": 5,
    "homepage_allowed": 96,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "anthropic-ai": {
    "named": 8,
    "homepage_allowed": 95,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "canva.com"
    ]
   },
   "PerplexityBot": {
    "named": 15,
    "homepage_allowed": 96,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "Google-Extended": {
    "named": 14,
    "homepage_allowed": 95,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "figma.com"
    ]
   },
   "CCBot": {
    "named": 10,
    "homepage_allowed": 92,
    "homepage_disallowed_count": 4,
    "homepage_disallowed": [
     "figma.com",
     "canva.com",
     "calendly.com",
     "forter.com"
    ]
   },
   "Applebot-Extended": {
    "named": 8,
    "homepage_allowed": 95,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "canva.com"
    ]
   },
   "Bytespider": {
    "named": 8,
    "homepage_allowed": 91,
    "homepage_disallowed_count": 5,
    "homepage_disallowed": [
     "figma.com",
     "canva.com",
     "postman.com",
     "amplitude.com",
     "forter.com"
    ]
   },
   "meta-externalagent": {
    "named": 7,
    "homepage_allowed": 94,
    "homepage_disallowed_count": 2,
    "homepage_disallowed": [
     "canva.com",
     "forter.com"
    ]
   }
  },
  "named_any_ai_bot": 19,
  "homepage_open_to_all_12": 89,
  "homepage_disallows_at_least_one": [
   "figma.com",
   "canva.com",
   "calendly.com",
   "loom.com",
   "postman.com",
   "amplitude.com",
   "forter.com"
  ],
  "blocks_GPTBot_but_allows_OAI-SearchBot": [
   "figma.com",
   "canva.com",
   "loom.com"
  ],
  "llms_txt_valid": 68,
  "llms_txt_h1": 62,
  "llms_full_valid": 11,
  "sitemap_in_robots": 90
 },
 "original_50": {
  "sites_attempted": 50,
  "robots_parsed": 48,
  "robots_not_parsed": {
   "freshworks.com": "HTTP 404",
   "gusto.com": "HTTP 403"
  },
  "bots": {
   "GPTBot": {
    "named": 8,
    "homepage_allowed": 45,
    "homepage_disallowed_count": 3,
    "homepage_disallowed": [
     "figma.com",
     "canva.com",
     "loom.com"
    ]
   },
   "OAI-SearchBot": {
    "named": 7,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "ChatGPT-User": {
    "named": 6,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "ClaudeBot": {
    "named": 8,
    "homepage_allowed": 46,
    "homepage_disallowed_count": 2,
    "homepage_disallowed": [
     "figma.com",
     "canva.com"
    ]
   },
   "Claude-SearchBot": {
    "named": 3,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "anthropic-ai": {
    "named": 4,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "canva.com"
    ]
   },
   "PerplexityBot": {
    "named": 7,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "Google-Extended": {
    "named": 7,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "figma.com"
    ]
   },
   "CCBot": {
    "named": 6,
    "homepage_allowed": 45,
    "homepage_disallowed_count": 3,
    "homepage_disallowed": [
     "figma.com",
     "canva.com",
     "calendly.com"
    ]
   },
   "Applebot-Extended": {
    "named": 4,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "canva.com"
    ]
   },
   "Bytespider": {
    "named": 5,
    "homepage_allowed": 44,
    "homepage_disallowed_count": 4,
    "homepage_disallowed": [
     "figma.com",
     "canva.com",
     "postman.com",
     "amplitude.com"
    ]
   },
   "meta-externalagent": {
    "named": 4,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "canva.com"
    ]
   }
  },
  "named_any_ai_bot": 11,
  "homepage_open_to_all_12": 42,
  "homepage_disallows_at_least_one": [
   "figma.com",
   "canva.com",
   "calendly.com",
   "loom.com",
   "postman.com",
   "amplitude.com"
  ],
  "blocks_GPTBot_but_allows_OAI-SearchBot": [
   "figma.com",
   "canva.com",
   "loom.com"
  ],
  "llms_txt_valid": 37,
  "llms_txt_h1": 34,
  "llms_full_valid": 7,
  "sitemap_in_robots": 43
 },
 "added_50": {
  "sites_attempted": 50,
  "robots_parsed": 48,
  "robots_not_parsed": {
   "cohesity.com": "HTTP 403",
   "cribl.io": "SSLError: HTTPSConnectionPool(host='cribl.io', port=443): Max retries exceeded with url: /robots.txt (Caused by SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WH"
  },
  "bots": {
   "GPTBot": {
    "named": 8,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "OAI-SearchBot": {
    "named": 6,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "ChatGPT-User": {
    "named": 8,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "ClaudeBot": {
    "named": 8,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "Claude-SearchBot": {
    "named": 2,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "anthropic-ai": {
    "named": 4,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "PerplexityBot": {
    "named": 8,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "Google-Extended": {
    "named": 7,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "CCBot": {
    "named": 4,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "forter.com"
    ]
   },
   "Applebot-Extended": {
    "named": 4,
    "homepage_allowed": 48,
    "homepage_disallowed_count": 0,
    "homepage_disallowed": []
   },
   "Bytespider": {
    "named": 3,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "forter.com"
    ]
   },
   "meta-externalagent": {
    "named": 3,
    "homepage_allowed": 47,
    "homepage_disallowed_count": 1,
    "homepage_disallowed": [
     "forter.com"
    ]
   }
  },
  "named_any_ai_bot": 8,
  "homepage_open_to_all_12": 47,
  "homepage_disallows_at_least_one": [
   "forter.com"
  ],
  "blocks_GPTBot_but_allows_OAI-SearchBot": [],
  "llms_txt_valid": 31,
  "llms_txt_h1": 28,
  "llms_full_valid": 4,
  "sitemap_in_robots": 47
 },
 "changes_vs_2026-11-06_original_50": {
  "field_changes": [
   {
    "domain": "figma.com",
    "field": "anthropic-ai_disallow_rules",
    "nov6": "17",
    "nov13": "16"
   },
   {
    "domain": "figma.com",
    "field": "Applebot-Extended_disallow_rules",
    "nov6": "17",
    "nov13": "16"
   },
   {
    "domain": "figma.com",
    "field": "meta-externalagent_disallow_rules",
    "nov6": "17",
    "nov13": "16"
   },
   {
    "domain": "miro.com",
    "field": "GPTBot_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "OAI-SearchBot_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "ChatGPT-User_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "ClaudeBot_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "Claude-SearchBot_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "anthropic-ai_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "PerplexityBot_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "Google-Extended_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "CCBot_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "Applebot-Extended_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "Bytespider_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "miro.com",
    "field": "meta-externalagent_disallow_rules",
    "nov6": "64",
    "nov13": "63"
   },
   {
    "domain": "zapier.com",
    "field": "GPTBot_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "OAI-SearchBot_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "ChatGPT-User_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "ClaudeBot_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "Claude-SearchBot_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "anthropic-ai_disallow_rules",
    "nov6": "24",
    "nov13": "25"
   },
   {
    "domain": "zapier.com",
    "field": "PerplexityBot_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "Google-Extended_disallow_rules",
    "nov6": "0",
    "nov13": "11"
   },
   {
    "domain": "zapier.com",
    "field": "CCBot_disallow_rules",
    "nov6": "24",
    "nov13": "25"
   },
   {
    "domain": "zapier.com",
    "field": "Applebot-Extended_disallow_rules",
    "nov6": "24",
    "nov13": "25"
   },
   {
    "domain": "zapier.com",
    "field": "Bytespider_disallow_rules",
    "nov6": "24",
    "nov13": "25"
   },
   {
    "domain": "zapier.com",
    "field": "meta-externalagent_disallow_rules",
    "nov6": "24",
    "nov13": "25"
   },
   {
    "domain": "ramp.com",
    "field": "GPTBot_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "OAI-SearchBot_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "ChatGPT-User_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "ClaudeBot_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "Claude-SearchBot_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "anthropic-ai_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "PerplexityBot_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "Google-Extended_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "CCBot_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "Applebot-Extended_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "Bytespider_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   },
   {
    "domain": "ramp.com",
    "field": "meta-externalagent_disallow_rules",
    "nov6": "10",
    "nov13": "11"
   }
  ],
  "domains_changed": [
   "figma.com",
   "miro.com",
   "ramp.com",
   "zapier.com"
  ],
  "nov6_summary_quick": {
   "robots_parsed": 48,
   "named_any_ai_bot": 11,
   "homepage_open_to_all_12": 42,
   "llms_txt_valid": 37,
   "llms_full_valid": 7,
   "sitemap_in_robots": 43,
   "homepage_disallowed_counts": {
    "GPTBot": 3,
    "OAI-SearchBot": 0,
    "ChatGPT-User": 0,
    "ClaudeBot": 2,
    "Claude-SearchBot": 0,
    "PerplexityBot": 0,
    "Google-Extended": 1,
    "CCBot": 3,
    "Applebot-Extended": 1,
    "Bytespider": 4,
    "meta-externalagent": 1
   }
  }
 }
}