From 27916b8bbbc7df30a5b78869f1e6580b189b5417 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 19:28:04 +0000 Subject: [PATCH 1/6] feat: add bots and pattern updates from forks and upstream Compare getarcis/well-known-bots, official GitHub forks, and monperrus/crawler-user-agents against this list. Official forks had no unique commits. getarcis's full corpus matched ours; its slim file added browser-automation tools. Upstream added hundreds of entries since the hard-fork; add the documented, well-known ones and widen a few existing patterns that missed live UAs. Co-authored-by: david --- well-known-bots.json | 1345 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1332 insertions(+), 13 deletions(-) diff --git a/well-known-bots.json b/well-known-bots.json index 346d6e6..d330ee5 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -4196,18 +4196,20 @@ ], "pattern": { "accepted": [ - "Qwantify" + "Qwant(?:ify|bot)" ], "forbidden": [] }, - "url": "https://www.qwant.com/", + "url": "https://help.qwant.com/bot/", "verification": [], "instances": { "accepted": [ "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", - "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" + "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)", + "Mozilla/5.0 (compatible; Qwantbot/1.0_800166; +https://help.qwant.com/bot/)", + "Mozilla/5.0 (compatible; Qwantbot-news/2.0; +https://help.qwant.com/bot/)" ], "rejected": [] }, @@ -13588,7 +13590,7 @@ ], "pattern": { "accepted": [ - "Datadog\\/{0,1}Synthetics" + "Datadog(?:\\/{0,1}Synthetics| Synthetic)" ], "forbidden": [] }, @@ -13608,7 +13610,8 @@ "instances": { "accepted": [ "Datadog/Synthetics", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 DatadogSynthetics" + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0 DatadogSynthetics", + "Datadog Synthetic" ], "rejected": [] }, @@ -14065,7 +14068,7 @@ ], "pattern": { "accepted": [ - "AmazonAdBot\\/" + "AmazonAdBot" ], "forbidden": [] }, @@ -14080,7 +14083,8 @@ ], "instances": { "accepted": [ - "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)" + "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)", + "AmazonAdBot" ], "rejected": [] }, @@ -14185,7 +14189,7 @@ ], "pattern": { "accepted": [ - "Perplexity-User\\/" + "Perplexity-?User" ], "forbidden": [] }, @@ -14204,7 +14208,8 @@ ], "instances": { "accepted": [ - "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)" + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityUser/1.0; +https://perplexity.ai)" ], "rejected": [] }, @@ -14217,7 +14222,7 @@ ], "pattern": { "accepted": [ - "NewRelicbot\\/" + "NewRelic(?:bot\\/|Synthetics)" ], "forbidden": [] }, @@ -14236,7 +14241,8 @@ ], "instances": { "accepted": [ - "Mozilla/5.0 (compatible; NewRelicbot/2.1; +http://www.newrelic.com)" + "Mozilla/5.0 (compatible; NewRelicbot/2.1; +http://www.newrelic.com)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36 NewRelicSynthetics/1.0" ], "rejected": [] }, @@ -14282,7 +14288,7 @@ ], "pattern": { "accepted": [ - "Checkly\\/" + "Checkly" ], "forbidden": [] }, @@ -14301,7 +14307,8 @@ ], "instances": { "accepted": [ - "Checkly/1.0" + "Checkly/1.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.6998.35 Safari/537.36 (Checkly, https://www.checklyhq.com)" ], "rejected": [] }, @@ -14791,5 +14798,1317 @@ "rejected": [] }, "url": "https://sitebulb.com/" + }, + { + "id": "google-cws", + "categories": [ + "google" + ], + "pattern": { + "accepted": [ + "Google-CWS" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + }, + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Google-CWS)" + ], + "rejected": [] + }, + "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", + "aliases": [ + "GoogleCWS" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-gemini-notebook", + "categories": [ + "ai", + "google" + ], + "pattern": { + "accepted": [ + "Google-(?:GeminiNotebook|NotebookLM)" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + }, + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", + "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36 (compatible; Google-GeminiNotebook; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook)", + "Mozilla/5.0 (compatible; Google-NotebookLM; +https://notebooklm.google.com/)" + ], + "rejected": [] + }, + "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-gemininotebook", + "aliases": [ + "GoogleNotebookLM", + "GoogleGeminiNotebook" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-agent", + "categories": [ + "ai", + "google" + ], + "pattern": { + "accepted": [ + "Google-Agent" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/137.0.0.0 Safari/537.36", + "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Mobile Safari/537.36 (compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent)" + ], + "rejected": [] + }, + "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent", + "aliases": [ + "GoogleAgent" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-messages", + "categories": [ + "google", + "preview" + ], + "pattern": { + "accepted": [ + "GoogleMessages" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + }, + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], + "instances": { + "accepted": [ + "GoogleMessages" + ], + "rejected": [] + }, + "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", + "aliases": [ + "GoogleMessages" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-pinpoint", + "categories": [ + "google" + ], + "pattern": { + "accepted": [ + "Google-Pinpoint" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + }, + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], + "instances": { + "accepted": [ + "Google-Pinpoint" + ], + "rejected": [] + }, + "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers", + "aliases": [ + "GooglePinpoint" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-producer", + "categories": [ + "google", + "feedfetcher" + ], + "pattern": { + "accepted": [ + "GoogleProducer" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + }, + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" + ] + } + ], + "instances": { + "accepted": [ + "GoogleProducer; (+https://developers.google.com/search/docs/crawling-indexing/google-producer)" + ], + "rejected": [] + }, + "url": "https://developers.google.com/search/docs/crawling-indexing/google-producer", + "aliases": [ + "GoogleProducer" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-gemini-deep-research", + "categories": [ + "ai", + "google" + ], + "pattern": { + "accepted": [ + "Gemini-Deep-Research" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://gemini.google/overview/deep-research/", + "aliases": [ + "GeminiDeepResearch" + ], + "addition_date": "2026/08/18" + }, + { + "id": "google-image-proxy", + "categories": [ + "google", + "preview" + ], + "pattern": { + "accepted": [ + "GoogleImageProxy" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)" + ], + "rejected": [] + }, + "url": "https://support.google.com/mail/answer/1311182", + "addition_date": "2026/08/18" + }, + { + "id": "google-association-service", + "categories": [ + "google" + ], + "pattern": { + "accepted": [ + "GoogleAssociationService" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "GoogleAssociationService", + "GoogleAssociationService/" + ], + "rejected": [] + }, + "url": "https://developers.google.com/digital-asset-links", + "addition_date": "2026/08/18" + }, + { + "id": "google-stackdriver-monitoring", + "categories": [ + "google", + "monitor" + ], + "pattern": { + "accepted": [ + "GoogleStackdriverMonitoring" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "GoogleStackdriverMonitoring-UptimeChecks(https://cloud.google.com/monitoring)" + ], + "rejected": [] + }, + "url": "https://cloud.google.com/monitoring", + "addition_date": "2026/08/18" + }, + { + "id": "meta-webindexer", + "categories": [ + "meta", + "search-engine" + ], + "pattern": { + "accepted": [ + "meta-webindexer" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-csv", + "url": "http://www.facebook.com/peering/geofeed" + } + ] + } + ], + "instances": { + "accepted": [ + "meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)" + ], + "rejected": [] + }, + "url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers#meta-webindexer", + "addition_date": "2026/08/18" + }, + { + "id": "meta-externalhit", + "categories": [ + "meta", + "preview", + "social" + ], + "pattern": { + "accepted": [ + "Meta-ExternalHit" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-csv", + "url": "http://www.facebook.com/peering/geofeed" + } + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Meta-ExternalHit/1.1; +http://www.facebook.com/externalhit_uatext.php)" + ], + "rejected": [] + }, + "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/", + "addition_date": "2026/08/18" + }, + { + "id": "anthropic-ai", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "anthropic-ai" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "anthropic-ai" + ], + "rejected": [] + }, + "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", + "aliases": [ + "AnthropicAI" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "mistral-ai-user", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "MistralAI-User" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)" + ], + "rejected": [] + }, + "url": "https://docs.mistral.ai/robots", + "aliases": [ + "MistralAIUser" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "mistral-ai-index", + "categories": [ + "ai", + "search-engine" + ], + "pattern": { + "accepted": [ + "MistralAI-Index" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)" + ], + "rejected": [] + }, + "url": "https://docs.mistral.ai/robots", + "aliases": [ + "MistralAIIndex" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "mistral-ai-training", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "MistralAI-Training" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)" + ], + "rejected": [] + }, + "url": "https://docs.mistral.ai/robots", + "aliases": [ + "MistralAITraining" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "duckassist-bot", + "categories": [ + "ai", + "search-engine" + ], + "pattern": { + "accepted": [ + "DuckAssistBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)" + ], + "rejected": [] + }, + "url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", + "aliases": [ + "DuckAssistBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "deepseek-crawler", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "DeepSeekBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; DeepSeekBot/1.0; +https://www.deepseek.com/bot)" + ], + "rejected": [] + }, + "url": "https://www.deepseek.com/bot", + "aliases": [ + "DeepSeekBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "firecrawl-agent", + "categories": [ + "ai", + "tool" + ], + "pattern": { + "accepted": [ + "FirecrawlAgent" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; FirecrawlAgent; +https://firecrawl.dev/)" + ], + "rejected": [] + }, + "url": "https://firecrawl.dev/", + "aliases": [ + "FirecrawlAgent" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "cloudflare-autorag", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Cloudflare-AutoRAG" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Cloudflare-AutoRAG (https://developers.cloudflare.com/autorag; autorag@cloudflare.com)" + ], + "rejected": [] + }, + "url": "https://developers.cloudflare.com/autorag", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "tavily-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "TavilyBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; TavilyBot; +https://tavily.com/)" + ], + "rejected": [] + }, + "url": "https://tavily.com/", + "aliases": [ + "TavilyBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "shapbot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "ShapBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ShapBot/0.1.0" + ], + "rejected": [] + }, + "url": "https://docs.parallel.ai/resources/crawler", + "aliases": [ + "ShapBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "zanista-bot", + "categories": [ + "ai", + "search-engine" + ], + "pattern": { + "accepted": [ + "ZanistaBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ZanistaBot/1.0; +https://zanista.ai/crawler-info) Chrome/W.X.Y.Z Safari/537.36" + ], + "rejected": [] + }, + "url": "https://zanista.ai/crawler-info", + "aliases": [ + "ZanistaBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "kagi-fetcher", + "categories": [ + "ai", + "search-engine" + ], + "pattern": { + "accepted": [ + "kagi-fetcher" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "kagi-fetcher" + ], + "rejected": [] + }, + "url": "https://help.kagi.com/kagi/ai/kagi-ai.html", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "newsai", + "categories": [ + "ai", + "feedfetcher" + ], + "pattern": { + "accepted": [ + "newsai\\/" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 newsai/1.0 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/newsai", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "brightbot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Brightbot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Brightbot 1.0" + ], + "rejected": [] + }, + "url": "https://brightdata.com/brightbot", + "aliases": [ + "Brightbot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "atlassian-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "atlassian-bot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; atlassian-bot; +https://www.atlassian.com/)" + ], + "rejected": [] + }, + "url": "https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "azure-ai-searchbot", + "categories": [ + "ai", + "microsoft", + "search-engine" + ], + "pattern": { + "accepted": [ + "AzureAI-SearchBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; AzureAI-SearchBot/1.0;" + ], + "rejected": [] + }, + "url": "https://azure.microsoft.com/en-us/products/ai-services", + "aliases": [ + "AzureAISearchBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "amazon-product-discovery", + "categories": [ + "amazon", + "search-engine" + ], + "pattern": { + "accepted": [ + "AmazonProductDiscovery" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 (compatible; AmazonProductDiscovery/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)", + "Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 (compatible; AmazonProductDiscovery/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)" + ], + "rejected": [] + }, + "url": "https://vendorcentral.amazon.com/support/amazonproductbot", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "amazon-seller-initiated-listing", + "categories": [ + "amazon", + "search-engine" + ], + "pattern": { + "accepted": [ + "AmazonSellerInitiatedListing" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 (compatible; AmazonSellerInitiatedListing/1.0; https://vendorcentral.amazon.com/support/amazonproductbot)" + ], + "rejected": [] + }, + "url": "https://vendorcentral.amazon.com/support/amazonproductbot", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "amazon-searchbot", + "categories": [ + "ai", + "amazon", + "search-engine" + ], + "pattern": { + "accepted": [ + "Amzn-SearchBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/119.0.6045.214 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://developer.amazon.com/amazonbot", + "aliases": [ + "AmznSearchBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "amazon-user", + "categories": [ + "ai", + "amazon" + ], + "pattern": { + "accepted": [ + "Amzn-User" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Amzn-User; +https://developer.amazon.com/)" + ], + "rejected": [] + }, + "url": "https://developer.amazon.com/amazonbot", + "aliases": [ + "AmznUser" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "amazon-bedrock-agentcore-browser", + "categories": [ + "ai", + "amazon", + "tool" + ], + "pattern": { + "accepted": [ + "Amazon-Bedrock-AgentCore-Browser" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Amazon-Bedrock-AgentCore-Browser; +https://aws.amazon.com/bedrock/)" + ], + "rejected": [] + }, + "url": "https://docs.aws.amazon.com/bedrock-agentcore/", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "geedo-crawler-shop", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "GeedoShopProductFinder" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "dns", + "masks": [ + "@.geedo.com" + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; GeedoShopProductFinder) Chrome/142.0.0.0 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://geedo.com/product-finder/", + "addition_date": "2026/08/18" + }, + { + "id": "naver-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "NaverBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; NaverBot/1.0; nhnbot@naver.com)" + ], + "rejected": [] + }, + "url": "https://www.naver.com/", + "aliases": [ + "NaverBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "aasa-bot", + "categories": [ + "apple" + ], + "pattern": { + "accepted": [ + "AASA-Bot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "AASA-Bot/1.0.0" + ], + "rejected": [] + }, + "url": "https://developer.apple.com/documentation/xcode/allowing-apps-and-websites-to-link-to-your-content", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "watchfor-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "WatchForBot" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://watchfor.io/ips.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "@.probe.watchfor.io" + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; WatchForBot/1.0; +https://watchfor.io/bot)" + ], + "rejected": [] + }, + "url": "https://watchfor.io/bot", + "aliases": [ + "WatchForBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "observepoint-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "ObservePoint" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "ObservePoint" + ], + "rejected": [] + }, + "url": "https://help.observepoint.com/en/articles/9101465-allow-exclude-observepoint-traffic", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "prtg-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "PRTG(?:CloudBot\\/| Network Monitor)" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; PRTGCloudBot/1.0; +http://www.paessler.com/prtgcloudbot; for_[edf7e62f223b268942b7efa36b6be1e305fcdadb])", + "Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com ); Windows)" + ], + "rejected": [] + }, + "url": "https://www.paessler.com/manuals/prtg/http_transaction_sensor", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "thousandeyes-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "ThousandEyes" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 (ThousandEyes Agent)" + ], + "rejected": [] + }, + "url": "https://docs.thousandeyes.com/product-documentation/global-vantage-points/cloud-and-enterprise-agents", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "alertsite-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "AlertSite" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; AlertSite; +https://www.alertsite.com/)" + ], + "rejected": [] + }, + "url": "https://smartbear.com/product/alertsite/", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "listgem-bot", + "categories": [ + "preview", + "social" + ], + "pattern": { + "accepted": [ + "ListGemBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "ListGemBot/1.0 (+https://listgem.com/bot)" + ], + "rejected": [] + }, + "url": "https://listgem.com/bot", + "aliases": [ + "ListGemBot" + ], + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "bluesky-preview", + "categories": [ + "preview", + "social" + ], + "pattern": { + "accepted": [ + "BlueskyPreviewBot" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "BlueskyPreviewBot/1.0" + ], + "rejected": [] + }, + "url": "https://docs.bsky.app", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "github-camo", + "categories": [ + "preview" + ], + "pattern": { + "accepted": [ + "github-camo" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "github-camo (c006e452)" + ], + "rejected": [] + }, + "url": "https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-anonymized-urls", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "playwright", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "[Pp]laywright" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Playwright/1.40.0" + ], + "rejected": [] + }, + "url": "https://playwright.dev", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "puppeteer", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "[Pp]uppeteer" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36 Puppeteer" + ], + "rejected": [] + }, + "url": "https://pptr.dev", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "selenium", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "[Ss]elenium" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36; Selenium" + ], + "rejected": [] + }, + "url": "https://www.selenium.dev", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "cypress", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "[Cc]ypress" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "Cypress/13.0.0" + ], + "rejected": [] + }, + "url": "https://www.cypress.io", + "addition_date": "2026/08/18", + "verification": [] + }, + { + "id": "httpie", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "^HTTPie\\/" + ], + "forbidden": [] + }, + "instances": { + "accepted": [ + "HTTPie/3.2.2" + ], + "rejected": [] + }, + "url": "https://httpie.io", + "addition_date": "2026/08/18", + "verification": [] } ] From 33d1059ddebdb4354875adf84a02f5b507956219 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 19:36:38 +0000 Subject: [PATCH 2/6] feat: add verified monperrus bots with accessible documentation Review the remaining upstream bulk-import entries and keep only those with an accessible docs URL, a distinctive User-Agent token mentioned on that page, and at least one matching instance. Skip scanners, generic clients, homepage-only listings, and dead documentation URLs. Co-authored-by: david --- well-known-bots.json | 2129 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2129 insertions(+) diff --git a/well-known-bots.json b/well-known-bots.json index d330ee5..01b129c 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -16110,5 +16110,2134 @@ "url": "https://httpie.io", "addition_date": "2026/08/18", "verification": [] + }, + { + "id": "anomura-crawler", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Anomura" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Anomura/1.2 (+https://www.direqt.ai)" + ], + "rejected": [] + }, + "url": "https://docs.direqt-search.com/direqt-bots/direqt-crawlers-and-user-agents" + }, + { + "id": "apify-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "ApifyBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; ApifyBot/1.0)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/apifybot" + }, + { + "id": "archive-it", + "categories": [ + "archive" + ], + "pattern": { + "accepted": [ + "Archive-It" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "UM-Bentley-Archive-It", + "Mozilla/5.0 (compatible; special_archiver; Archive-It; +http://archive-it.org/files/site-owners-special.html)" + ], + "rejected": [] + }, + "url": "https://archive-it.org/files/site-owners-special.html" + }, + { + "id": "artemis-web-reader", + "categories": [ + "feedfetcher" + ], + "pattern": { + "accepted": [ + "artemis web reader" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "artemis web reader/1.0 - https://artemis.jamesg.blog/bot" + ], + "rejected": [] + }, + "url": "https://artemis.jamesg.blog/bot" + }, + { + "id": "audigent-adbot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "AudigentAdBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; AudigentAdBot; +http://www.audigent.com/bot.html) Chrome/W.X.Y.Z Safari/537.36" + ], + "rejected": [] + }, + "url": "https://audigent.com/bot.html" + }, + { + "id": "buffer-link-preview", + "categories": [ + "preview", + "social" + ], + "pattern": { + "accepted": [ + "BufferLinkPreviewBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "BufferLinkPreviewBot/1.0 (+https://scraper.buffer.com/about/bots/link-preview-bot)" + ], + "rejected": [] + }, + "url": "https://scraper.buffer.com/about/bots/link-preview-bot" + }, + { + "id": "channel3-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Channel3Bot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Channel3Bot/1.0; +https://trychannel3.com/channel3bot)" + ], + "rejected": [] + }, + "url": "https://trychannel3.com/channel3bot" + }, + { + "id": "ezoic-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "EzoicBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-UptimeOwl; +http://www.ezoic.com/bot)", + "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36 (compatible; EzoicBot-Nicheiq; +http://www.ezoic.com/bot)" + ], + "rejected": [] + }, + "url": "https://www.ezoic.com/bot" + }, + { + "id": "findfiles-crawler", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "FindFiles\\.net" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "FindFiles.net-FaviconFetcher/1.0 (+https://findfiles.net/bot)" + ], + "rejected": [] + }, + "url": "https://findfiles.net/en/bot" + }, + { + "id": "honeybadger-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "HoneybadgerBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; HoneybadgerBot; +https://www.honeybadger.io/)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/honeybadgerbot" + }, + { + "id": "ibou-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "IbouBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; IbouBot/1.0; +bot@ibou.io; +https://ibou.io/iboubot.html)" + ], + "rejected": [] + }, + "url": "https://ibou.io/iboubot.html" + }, + { + "id": "imagespider", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "imageSpider" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; imageSpider; +https://www.bytedance.com/)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/imagespider" + }, + { + "id": "laion-huggingface-processor", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "laion-huggingface-processor" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; laion-huggingface-processor; +https://laion.ai/)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/laion-huggingface-processor" + }, + { + "id": "linkup-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "LinkupBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "LinkupBot/1.0 (LinkupBot for web indexing; https://linkup.so/bot; bot@linkup.so)" + ], + "rejected": [] + }, + "url": "https://www.linkup.so/bot" + }, + { + "id": "lyonl-crawler", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "lyonl-crawler" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "lyonl-crawler/0.1 (+https://lyonl.com/crawler; crawler@lyonl.com)" + ], + "rejected": [] + }, + "url": "https://lyonl.com/bot.html" + }, + { + "id": "mcontextual-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "McontextualBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; McontextualBot/0.1; +http://mcontextual.net/mcontextual-bot)" + ], + "rejected": [] + }, + "url": "https://mcontextual.net/mcontextual-bot" + }, + { + "id": "mixrank-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "MixrankBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/mixrankbot" + }, + { + "id": "montastic-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "MontasticMonitor" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; MontasticMonitor; +https://montastic.com/)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/montasticmonitor" + }, + { + "id": "nanointeractive-crawler", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "NanoInteractive" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (iPhone; CPU iPhone OS 16_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Mobile/15E148 Safari/604.1 (compatible; NanoInteractive/1.0; +https://nanointeractive.com/crawler/)" + ], + "rejected": [] + }, + "url": "https://www.nanointeractive.com/resources/legal/crawler-policy/" + }, + { + "id": "nestdaddy-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "NestDaddybot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "NestDaddybot/1.0.0 (+https://nestdaddy.com/bot)" + ], + "rejected": [] + }, + "url": "https://nestdaddy.com/bot" + }, + { + "id": "netseer-crawler", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "NetSeer crawler" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; NetSeer crawler/2.0; +http://www.netseer.com/crawler.html; crawler@netseer.com)" + ], + "rejected": [] + }, + "url": "https://inuvo.com/" + }, + { + "id": "nitro-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "NitroBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; NitroBot; +https://www.nitropack.io/)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/nitrobot" + }, + { + "id": "phind-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "PhindBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; PhindBot; +https://www.phind.com/)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/phindbot" + }, + { + "id": "productsup-crawler", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "productsup\\.io\\/crawler" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "productsup.io/crawler" + ], + "rejected": [] + }, + "url": "https://help.productsup.com/docs/help-center/import-data-feeds/import-data-from-internal-systems/import-web-data-with-website-crawler" + }, + { + "id": "scope3-crawler", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "Scope3" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Scope3/2.0 (scope3.com)" + ], + "rejected": [] + }, + "url": "https://docs.scope3.com/docs/scope3-crawler" + }, + { + "id": "scrunch-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "Scrunchbot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Scrunchbot/1.0; +https://scrunchai.com/bots)" + ], + "rejected": [] + }, + "url": "https://scrunch.com/bots/" + }, + { + "id": "statusnest-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "StatusNestBacklinkSpider" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; StatusNestBacklinkSpider; +https://statusnest.com/)" + ], + "rejected": [] + }, + "url": "https://statusnest.com/bot" + }, + { + "id": "stepstone-crawler", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "stepstoneCrawlBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; stepstoneCrawlBot; +https://www.thestepstonegroup.com/crawler/)" + ], + "rejected": [] + }, + "url": "https://www.thestepstonegroup.com/english/crawler/" + }, + { + "id": "uip-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "uipbot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "uipbot/1.0 (uipbot@semasio.com)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/uipbot" + }, + { + "id": "wepch-crawler", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "WepchSearchEngine" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; WepchSearchEngine; +https://www.wepch.com/)" + ], + "rejected": [] + }, + "url": "https://www.wepch.com/search-engine" + }, + { + "id": "cybaa-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "CybaaBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 CybaaBot" + ], + "rejected": [] + }, + "url": "https://cybaa.io/bot-policy" + }, + { + "id": "fleebs-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "FleebsBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; FleebsBot/1~w; +https://fleebs.com/bot)" + ], + "rejected": [] + }, + "url": "https://fleebs.com/bot" + }, + { + "id": "flyriver-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Flyriverbot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Flyriverbot/1.1 (+https://www.flyriver.com/; AI Content Source Check)" + ], + "rejected": [] + }, + "url": "https://www.flyriver.com/crawler" + }, + { + "id": "noah-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "NoahBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; NoahBot/1.0; +https://noahwire.com/bot-info)" + ], + "rejected": [] + }, + "url": "https://noahwire.com/bot-info" + }, + { + "id": "plagaware-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "PlagAwareBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; PlagAwareBot/3.2; +https://www.plagaware.com/bot)" + ], + "rejected": [] + }, + "url": "https://www.plagaware.com/bot" + }, + { + "id": "sirdata-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "SirdataBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "SirdataBot (+https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction)" + ], + "rejected": [] + }, + "url": "https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction" + }, + { + "id": "trafilatura", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "trafilatura" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "trafilatura/2.0.0 (+https://github.com/adbar/trafilatura)" + ], + "rejected": [] + }, + "url": "https://github.com/adbar/trafilatura" + }, + { + "id": "website-info-robot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Website-info\\.net-Robot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Website-info.net-Robot; https://website-info.net/robot)" + ], + "rejected": [] + }, + "url": "https://website-info.net/robot" + }, + { + "id": "crawl4ai", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "crawl4ai" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "crawl4ai-adapter/1.0" + ], + "rejected": [] + }, + "url": "https://github.com/unclecode/crawl4ai" + }, + { + "id": "abeval-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "ABEvalBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ABEvalBot/0.1) Version/11.1.2 Safari/605.1.15" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/abevalbot" + }, + { + "id": "carbon-umbrella-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "carbon-umbrella-bot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "carbon-umbrella-bot/1.0" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/carbon-umbrella-bot" + }, + { + "id": "dazzle-bluesky-bot", + "categories": [ + "preview", + "social" + ], + "pattern": { + "accepted": [ + "Dazzle BlueSky Bot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Dazzle BlueSky Bot/1.1" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/dazzle-bluesky-bot" + }, + { + "id": "fedistats-crawler", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "fedistatsCrawler" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "fedistatsCrawler/1.0" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/fedistatscrawler" + }, + { + "id": "goparser-bot", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "GoParserBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; GoParserBot/1.0)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/goparserbot" + }, + { + "id": "hanalei-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "HanaleiBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "HanaleiBot runid=beta-stage-integration-test" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/hanaleibot" + }, + { + "id": "openthebox-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "OpenTheBoxBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "OpenTheBoxBot/1.0" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/opentheboxbot" + }, + { + "id": "poduptime", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "Poduptime" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Poduptime; +fediverse.observer)" + ], + "rejected": [] + }, + "url": "https://fediverse.observer/" + }, + { + "id": "pomothy-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "Pomothy-Bot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Pomothy-Bot/1.0)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/pomothy-bot" + }, + { + "id": "swiftbot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "Swiftbot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Swiftbot/1.0; UID/649dce5512ab734096a50277; +http://swiftype.com/swiftbot)" + ], + "rejected": [] + }, + "url": "https://swiftype.com/swiftbot" + }, + { + "id": "synthesi-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "SynthesiBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; SynthesiBot/1.0)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/synthesibot" + }, + { + "id": "tsm-bot", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "TSMbot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; TSMbot) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/tsmbot" + }, + { + "id": "urlsuma-crawler", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "UrlSuMa\\.de crawler" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; UrlSuMa.de crawler)" + ], + "rejected": [] + }, + "url": "https://urlsuma.de/" + }, + { + "id": "yandori-rss-bot", + "categories": [ + "feedfetcher" + ], + "pattern": { + "accepted": [ + "YandoriRSSBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "YandoriRSSBot/3.0 (Go)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/yandorirssbot" + }, + { + "id": "zeal-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "Zealbot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/4.0 (compatible; Zealbot 1.0)" + ], + "rejected": [] + }, + "url": "https://knownagents.com/agents/zealbot" + }, + { + "id": "aranea-crawler", + "categories": [ + "academic" + ], + "pattern": { + "accepted": [ + "Aranea" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Aranea Web-Crawled Corpora Project (+http://unesco.uniba.sk/guest (Hebrew 2024 Spring Crawl))" + ], + "rejected": [] + }, + "url": "http://unesco.uniba.sk/guest/" + }, + { + "id": "archivebox", + "categories": [ + "archive" + ], + "pattern": { + "accepted": [ + "ArchiveBox" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.2 (+https://github.com/ArchiveBox/ArchiveBox/)" + ], + "rejected": [] + }, + "url": "https://archivebox.io/" + }, + { + "id": "classla-web-crawler", + "categories": [ + "academic" + ], + "pattern": { + "accepted": [ + "CLASSLA-web" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; CLASSLA-web; +https://www.clarin.si/info/classla-web-crawler/)" + ], + "rejected": [] + }, + "url": "https://www.clarin.si/info/classla-web-crawler/" + }, + { + "id": "cotoyogi-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Cotoyogi" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Cotoyogi/4.0; +https://ds.rois.ac.jp/center8/crawler/)" + ], + "rejected": [] + }, + "url": "https://ds.rois.ac.jp/center8/crawler/" + }, + { + "id": "repo-lookout-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "RepoLookoutBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "RepoLookoutBot/v1.1.0-297-gcf436d3 (abuse reports to abuse@repo-lookout.org)" + ], + "rejected": [] + }, + "url": "https://www.repo-lookout.org/" + }, + { + "id": "domcop-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "DomCopBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "DomCopBot (https://www.domcop.com/bot)" + ], + "rejected": [] + }, + "url": "https://www.domcop.com/bot" + }, + { + "id": "easycron-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "EasyCron\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "EasyCron/1.0 (https://www.easycron.com/)" + ], + "rejected": [] + }, + "url": "https://www.easycron.com/" + }, + { + "id": "crusty-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "crusty\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "crusty/0.12.0" + ], + "rejected": [] + }, + "url": "https://github.com/let4be/crusty" + }, + { + "id": "deskyo-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Deskyobot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Deskyobot/1.0 (+https://www.deskyo.com/bot)" + ], + "rejected": [] + }, + "url": "https://www.deskyo.com/bot" + }, + { + "id": "discourse-onebox", + "categories": [ + "preview", + "social" + ], + "pattern": { + "accepted": [ + "Discourse Forum Onebox" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Discourse Forum Onebox v3.3.0.beta1-dev" + ], + "rejected": [] + }, + "url": "https://www.discourse.org/" + }, + { + "id": "edgewatch-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "EdgeWatch" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; EdgeWatch/1.1; https://about.edgewatch.com/)" + ], + "rejected": [] + }, + "url": "https://about.edgewatch.com/" + }, + { + "id": "erepublik-tools-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "eRepublik\\.tools" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "eRepublik.tools - Multithreaded Crawler - " + ], + "rejected": [] + }, + "url": "https://erepublik.tools/" + }, + { + "id": "exodus-movement-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "ExodusMovement" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "ExodusMovement/1.0 GlobalCoinHeight/1.0" + ], + "rejected": [] + }, + "url": "https://www.exodus.com/" + }, + { + "id": "fastmail-preview", + "categories": [ + "preview", + "social" + ], + "pattern": { + "accepted": [ + "FastmailUA" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "FastmailUA/1.0" + ], + "rejected": [] + }, + "url": "https://www.fastmail.com/policies/bots/" + }, + { + "id": "fedica-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Fedicabot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Fedicabot / FedicaApp" + ], + "rejected": [] + }, + "url": "https://fedica.com/info/fedicabot" + }, + { + "id": "fedreporter-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "FedReporterDataBot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "FedReporterDataBot/1.0 (+https://fedreporter.net/FedReporterBotDocumentation;)" + ], + "rejected": [] + }, + "url": "https://fedreporter.net/FedReporterBotDocumentation/Readme.txt" + }, + { + "id": "feedburner", + "categories": [ + "feedfetcher" + ], + "pattern": { + "accepted": [ + "FeedBurner" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "FeedBurner/1.0 (http://www.FeedBurner.com)" + ], + "rejected": [] + }, + "url": "http://www.feedburner.com/" + }, + { + "id": "feeder-co", + "categories": [ + "feedfetcher" + ], + "pattern": { + "accepted": [ + "feeder\\.co" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (feeder.co; Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://feeder.co/crawler" + }, + { + "id": "feedwind", + "categories": [ + "feedfetcher" + ], + "pattern": { + "accepted": [ + "Feedwind" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Feedwind/3.0; +http://feed.mikle.com/support/description/)" + ], + "rejected": [] + }, + "url": "http://feed.mikle.com/support/description/" + }, + { + "id": "freespoke-crawler", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "Freespoke\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Freespoke/2.0; +https://crawler.freespoke.com)" + ], + "rejected": [] + }, + "url": "https://docs.freespoke.com/search/bot/" + }, + { + "id": "ghost-inspector", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "Ghost Inspector" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0 Safari/537.36 Ghost Inspector (63c85ddbde52d6697f57c623)" + ], + "rejected": [] + }, + "url": "https://ghostinspector.com/" + }, + { + "id": "google-apps-script", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Google-Apps-Script" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Google-Apps-Script; beanserver; +https://script.google.com; id: UAEmdDd_XLoqpGxtGfu5uvUaQlW77VLYz-w)" + ], + "rejected": [] + }, + "url": "https://developers.google.com/apps-script/" + }, + { + "id": "gtmetrix-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "GTmetrix" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 GTmetrix" + ], + "rejected": [] + }, + "url": "https://gtmetrix.com/" + }, + { + "id": "hcard-validator", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "hCardValidator" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "hCardValidator/1 PHP/5 (http://hcard.geekhood.net)" + ], + "rejected": [] + }, + "url": "http://hcard.geekhood.net/" + }, + { + "id": "hetrixtools-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "HetrixTools" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "HetrixTools Uptime Monitoring Bot. https://hetrix.tools/uptime-monitoring-bot.html" + ], + "rejected": [] + }, + "url": "https://hetrixtools.com/uptime-monitor-bot/" + }, + { + "id": "hlidam-robot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Hlidam\\.to robot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Hlidam.to robot" + ], + "rejected": [] + }, + "url": "https://hlidam.to/" + }, + { + "id": "innguma-fetcher", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Innguma\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Innguma/1.0 (+https://factory.innguma.com/fetcher/)" + ], + "rejected": [] + }, + "url": "https://factory.innguma.com/fetcher/" + }, + { + "id": "instapaper", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Instapaper\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Instapaper/4.0Instaparser/1.0" + ], + "rejected": [] + }, + "url": "https://www.instapaper.com/about-legal/about/publishers" + }, + { + "id": "jobswithgpt-bot", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "jobswithgptcom-bot" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "jobswithgptcom-bot" + ], + "rejected": [] + }, + "url": "https://jobswithgpt.com/bot.html" + }, + { + "id": "linksindexer-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "LinksIndexerBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; LinksIndexerBot/1.0; +http://linksindexer.com/bot)" + ], + "rejected": [] + }, + "url": "https://linksindexer.com/bot" + }, + { + "id": "madb-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "MADBbot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "MADBbot/0.1 (Gathering webpages for data analytics; https://madb.zapto.org/bot.html; ma-db-crawl@googlegroups.com)" + ], + "rejected": [] + }, + "url": "https://madb.zapto.org/bot.html" + }, + { + "id": "magnet-me-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Magnet\\.me-web\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Magnet.me-web/1.0 (+https://magnet.me/bot.html)" + ], + "rejected": [] + }, + "url": "https://magnet.me/bot.html" + }, + { + "id": "media-monitoring-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "MediaMonitoringBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "MediaMonitoringBot/1.1 (+https://mediamonitoringbot.com/crawler; crawler@mediamonitoringbot.com)" + ], + "rejected": [] + }, + "url": "https://mediamonitoringbot.com/" + }, + { + "id": "motominer-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "MotoMinerBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; MotoMinerBot/1.0; +https://motominer.com/Bot)" + ], + "rejected": [] + }, + "url": "https://motominer.com/Bot" + }, + { + "id": "paqle-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Paqlebot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Paqlebot/2.0; +http://www.paqle.dk/about/paqlebot)" + ], + "rejected": [] + }, + "url": "https://www.paqle.dk/about/paqlebot" + }, + { + "id": "qualified-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "QualifiedBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "QualifiedBot/1.0", + "Mozilla/5.0 (compatible; QualifiedBot/1.0; +https://www.qualified.com/legal/qualified-crawler-user-agent)" + ], + "rejected": [] + }, + "url": "https://www.qualified.com/legal/qualified-crawler-user-agent" + }, + { + "id": "rakuten-usa-image-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "rakutenusabot-image\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; rakutenusabot-image/1.0) Chrome/114.0.0.0 Safari/537.36" + ], + "rejected": [] + }, + "url": "https://product-image.ebates.com/item-gsp/rakutenusabot.html" + }, + { + "id": "retroliste-crawler", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "RetroListeCOM\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; RetroListeCOM/1.0)" + ], + "rejected": [] + }, + "url": "https://retroliste.com/" + }, + { + "id": "scour-rss-bot", + "categories": [ + "feedfetcher" + ], + "pattern": { + "accepted": [ + "ScourRSSBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "ScourRSSBot/1.0 (+https://scour.ing/bot)" + ], + "rejected": [] + }, + "url": "https://scour.ing/bot" + }, + { + "id": "clarity-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "ClarityBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; ClarityBot/9.0; +https://www.seoclarity.net/bot.html)" + ], + "rejected": [] + }, + "url": "https://www.seoclarity.net/bot.html" + }, + { + "id": "smartology-bot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "SmartologyBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "SmartologyBot/1.0 (+http://www.smartology.net/smartologybot)" + ], + "rejected": [] + }, + "url": "https://smartology.net/smartologybot/" + }, + { + "id": "spectate-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "Spectate\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Spectate/1.0 (+https://docs.spectate.net/faq/uptime-monitor-bot)" + ], + "rejected": [] + }, + "url": "https://docs.spectate.net/faq/uptime-monitor-bot" + }, + { + "id": "ttd-content", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "TTD-Content" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; TTD-Content; +https://www.thetradedesk.com/general/ttd-content)" + ], + "rejected": [] + }, + "url": "https://www.thetradedesk.com/legal/ttd-content" + }, + { + "id": "uas-link-checker", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "UASlinkChecker\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; UASlinkChecker/2.1; +https://udger.com/support/UASlinkChecker)" + ], + "rejected": [] + }, + "url": "https://udger.com/support/UASlinkChecker" + }, + { + "id": "ward-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "WARDBot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; WARDBot/1.0; http://ward.ai/robot)" + ], + "rejected": [] + }, + "url": "https://ward.ai/robot" + }, + { + "id": "xy-archive-compliance", + "categories": [ + "archive" + ], + "pattern": { + "accepted": [ + "XY-Archive-Compliance" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; XY-Archive-Compliance-Crawler; +https://archive.xyplanningnetwork.com/)", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 (compatible; XY-Archive-Compliance-Archiver; +https://archive.xyplanningnetwork.com/)" + ], + "rejected": [] + }, + "url": "https://xy-archive.helpscoutdocs.com/article/61-does-xy-archive-have-a-dedicated-ip-address" + }, + { + "id": "zombie-bot", + "categories": [ + "optimizer" + ], + "pattern": { + "accepted": [ + "Zombiebot\\/" + ], + "forbidden": [] + }, + "addition_date": "2026/08/18", + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv 11.0) like Gecko (compatible; Zombiebot/2.1; +http://www.zombiedomain.net/robot/)" + ], + "rejected": [] + }, + "url": "http://www.zombiedomain.net/robot/" } ] From 25862af07ea0dcce90a2bda4cdd00cc05dc9c00b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 19:46:32 +0000 Subject: [PATCH 3/6] feat: add documented bots found in the no-proof and arXiv source review Re-search the 170 monperrus leftovers whose homepages lacked a token, then mine the identifier sources cited by arXiv 2603.28546v1. Add only entries with first-party User-Agent documentation (Kagi, Snap Ads, Yahoo, Grafana, Jetpack, LinkChecker, Tabstack, Exa, Kimi, Crawlspace) and widen Montastic to the observed lowercase UA. Co-authored-by: david --- well-known-bots.json | 387 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 384 insertions(+), 3 deletions(-) diff --git a/well-known-bots.json b/well-known-bots.json index 01b129c..0f588cc 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -16478,7 +16478,7 @@ ], "pattern": { "accepted": [ - "MontasticMonitor" + "MontasticMonitor|montastic-monitor" ], "forbidden": [] }, @@ -16486,11 +16486,12 @@ "verification": [], "instances": { "accepted": [ - "Mozilla/5.0 (compatible; MontasticMonitor; +https://montastic.com/)" + "Mozilla/5.0 (compatible; MontasticMonitor; +https://montastic.com/)", + "montastic-monitor www.montastic.com" ], "rejected": [] }, - "url": "https://knownagents.com/agents/montasticmonitor" + "url": "https://montastic.com/" }, { "id": "nanointeractive-crawler", @@ -18239,5 +18240,385 @@ "rejected": [] }, "url": "http://www.zombiedomain.net/robot/" + }, + { + "id": "kagi-crawler", + "categories": [ + "search-engine" + ], + "pattern": { + "accepted": [ + "Kagibot" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "ip", + "ips": [ + "216.18.205.234", + "35.212.27.76", + "104.254.65.50", + "209.151.156.194" + ] + }, + { + "type": "dns", + "masks": [ + "@.kagibot.org" + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)" + ], + "rejected": [ + "kagi-fetcher" + ] + }, + "url": "https://kagi.com/bot", + "aliases": [ + "Kagibot", + "KagiBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "snapchat-adsbot", + "categories": [ + "advertising" + ], + "pattern": { + "accepted": [ + "SnapchatAds" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; SnapchatAds/1.0; +https://businesshelp.snapchat.com/s/article/adsbot-crawler)" + ], + "rejected": [ + "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" + ] + }, + "url": "https://businesshelp.snapchat.com/s/article/adsbot-crawler", + "aliases": [ + "SnapchatAds", + "SnapchatAdsBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "yahoo-ad-monitoring", + "categories": [ + "advertising", + "yahoo" + ], + "pattern": { + "accepted": [ + "Yahoo Ad [Mm]onitoring" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)", + "Mozilla/5.0 (compatible; Yahoo Ad Monitoring https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) " + ], + "rejected": [] + }, + "url": "https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html", + "aliases": [ + "Yahoo Ad monitoring", + "Yahoo Ad Monitoring" + ], + "addition_date": "2026/08/18" + }, + { + "id": "yahoo-mail-proxy", + "categories": [ + "preview", + "yahoo" + ], + "pattern": { + "accepted": [ + "YahooMailProxy" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "YahooMailProxy; https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html", + "YahooMailProxy; https://senders.yahooinc.com/mail-proxy-servers" + ], + "rejected": [] + }, + "url": "https://senders.yahooinc.com/mail-proxy-servers/", + "aliases": [ + "YahooMailProxy" + ], + "addition_date": "2026/08/18" + }, + { + "id": "grafana-synthetic-monitoring-agent", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "synthetic-monitoring-agent" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "synthetic-monitoring-agent/v0.41.3 (linux amd64; 1522bdcd162c24088daf19d0aa6667773e69825a; 2025-09-02T13:45:18Z; +https://github.com/grafana/synthetic-monitoring-agent)" + ], + "rejected": [] + }, + "url": "https://grafana.com/docs/grafana-cloud/testing/synthetic-monitoring/create-checks/checks/http/", + "aliases": [ + "synthetic-monitoring-agent" + ], + "addition_date": "2026/08/18" + }, + { + "id": "jetpack-monitor", + "categories": [ + "monitor" + ], + "pattern": { + "accepted": [ + "jetmon" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "jetmon/1.0 (Jetpack Site Uptime Monitor by WordPress.com)" + ], + "rejected": [] + }, + "url": "https://jetpack.com/support/monitor/", + "aliases": [ + "jetmon" + ], + "addition_date": "2026/08/18" + }, + { + "id": "linkchecker-pro", + "categories": [ + "tool" + ], + "pattern": { + "accepted": [ + "LinkCheckerBot" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; LinkCheckerBot/2.0; +https://linkchecker.pro/robot/)" + ], + "rejected": [] + }, + "url": "https://linkchecker.pro/robot/", + "aliases": [ + "LinkCheckerBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "mozilla-tabstack", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Mozilla-Tabstack" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla-Tabstack/1.0 (+https://tabstack.ai)" + ], + "rejected": [] + }, + "url": "https://docs.tabstack.ai/trust/controlling-access", + "aliases": [ + "Mozilla-Tabstack" + ], + "addition_date": "2026/08/18" + }, + { + "id": "exa-searchbot", + "categories": [ + "search-engine", + "ai" + ], + "pattern": { + "accepted": [ + "ExaSearchBot" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Mozilla/5.0 (compatible; ExaSearchBot/1.0; +https://crawler.exa.ai/)" + ], + "rejected": [ + "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)" + ] + }, + "url": "https://crawler.exa.ai/", + "aliases": [ + "ExaSearchBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "kimi-bot", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "KimiBot" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://www.kimi.ai/policies/kimibot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; KimiBot/1.0; +https://www.kimi.com/policies/kimi-crawlers" + ], + "rejected": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-User/1.0; +https://www.kimi.com/policies/kimi-crawlers", + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-SearchBot/1.0; +https://www.kimi.com/policies/kimi-crawlers" + ] + }, + "url": "https://www.kimi.com/policies/kimi-crawlers", + "aliases": [ + "KimiBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "kimi-user", + "categories": [ + "ai" + ], + "pattern": { + "accepted": [ + "Kimi-User" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://www.kimi.ai/policies/kimi-user.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-User/1.0; +https://www.kimi.com/policies/kimi-crawlers" + ], + "rejected": [] + }, + "url": "https://www.kimi.com/policies/kimi-crawlers", + "aliases": [ + "Kimi-User" + ], + "addition_date": "2026/08/18" + }, + { + "id": "kimi-searchbot", + "categories": [ + "ai", + "search-engine" + ], + "pattern": { + "accepted": [ + "Kimi-SearchBot" + ], + "forbidden": [] + }, + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://www.kimi.ai/policies/kimi-searchbot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + } + ], + "instances": { + "accepted": [ + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; Kimi-SearchBot/1.0; +https://www.kimi.com/policies/kimi-crawlers" + ], + "rejected": [] + }, + "url": "https://www.kimi.com/policies/kimi-crawlers", + "aliases": [ + "Kimi-SearchBot" + ], + "addition_date": "2026/08/18" + }, + { + "id": "crawlspace", + "categories": [ + "ai", + "tool" + ], + "pattern": { + "accepted": [ + "Crawlspace" + ], + "forbidden": [] + }, + "verification": [], + "instances": { + "accepted": [ + "Crawlspace" + ], + "rejected": [] + }, + "url": "https://crawlspace.dev", + "aliases": [ + "Crawlspace" + ], + "addition_date": "2026/08/18" } ] From 4e7973b4d49b980c1822a5b368fc3f2db53f88a3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 20:04:04 +0000 Subject: [PATCH 4/6] fix: review and refresh bot verification methods Re-check every published verification source, switch Meta geofeed URLs to HTTPS, add first-party CIDR/IP/DNS feeds found via GoodBots and provider docs, update Censys ranges, and drop unpublished Adagio IPs. Co-authored-by: david --- well-known-bots.json | 191 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 167 insertions(+), 24 deletions(-) diff --git a/well-known-bots.json b/well-known-bots.json index 0f588cc..c882c85 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -60,7 +60,25 @@ ], "forbidden": [] }, - "verification": [], + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", + "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" + ] + } + ], "instances": { "accepted": [ "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", @@ -2707,7 +2725,24 @@ }, "addition_date": "2012/02/05", "url": "http://www.commoncrawl.org/bot.html", - "verification": [], + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://index.commoncrawl.org/ccbot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + }, + { + "type": "dns", + "masks": [ + "@.crawl.commoncrawl.org" + ] + } + ], "instances": { "accepted": [ "CCBot/2.0 (http://commoncrawl.org/faq/)", @@ -2862,7 +2897,7 @@ "sources": [ { "type": "http-csv", - "url": "http://www.facebook.com/peering/geofeed" + "url": "https://www.facebook.com/peering/geofeed" } ] } @@ -3091,7 +3126,18 @@ }, "addition_date": "2012/09/19", "url": "http://duckduckgo.com/duckduckbot.html", - "verification": [], + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://duckduckgo.com/duckduckbot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + } + ], "instances": { "accepted": [ "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", @@ -3976,7 +4022,7 @@ "sources": [ { "type": "http-csv", - "url": "http://www.facebook.com/peering/geofeed" + "url": "https://www.facebook.com/peering/geofeed" } ] } @@ -4267,6 +4313,16 @@ "masks": [ "@.applebot.apple.com" ] + }, + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://search.developer.apple.com/applebot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] } ], "instances": { @@ -4761,7 +4817,24 @@ }, "addition_date": "2016/01/23", "url": "https://www.mojeek.com/bot.html", - "verification": [], + "verification": [ + { + "type": "dns", + "masks": [ + "@.mojeek.com" + ] + }, + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://www.mojeek.com/mojeekbot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + } + ], "instances": { "accepted": [ "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", @@ -5806,7 +5879,17 @@ "forbidden": [] }, "addition_date": "2017/10/01", - "verification": [], + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-text", + "url": "https://core.telegram.org/resources/cidr.txt" + } + ] + } + ], "instances": { "accepted": [ "TelegramBot (like TwitterBot)" @@ -7387,6 +7470,19 @@ "masks": [ "ip.uptimerobot.com" ] + }, + { + "type": "ip", + "sources": [ + { + "type": "http-text", + "url": "https://uptimerobot.com/inc/files/ips/IPv4.txt" + }, + { + "type": "http-text", + "url": "https://uptimerobot.com/inc/files/ips/IPv6.txt" + } + ] } ], "instances": { @@ -11438,7 +11534,24 @@ }, "addition_date": "2022/10/17", "url": "https://www.freshworks.com/website-monitoring/", - "verification": [], + "verification": [ + { + "type": "ip", + "ips": [ + "35.173.69.86", + "50.17.185.102", + "18.179.133.14", + "34.246.131.0", + "13.251.205.206", + "52.60.140.174", + "13.55.57.184", + "52.42.49.200", + "13.232.175.73", + "18.228.60.182", + "18.130.156.195" + ] + } + ], "instances": { "accepted": [ "FreshpingBot/1.0 (+https://freshping.io/)" @@ -11523,7 +11636,18 @@ }, "addition_date": "2022/11/13", "url": "https://betteruptime.com/", - "verification": [], + "verification": [ + { + "type": "ip", + "sources": [ + { + "type": "http-json", + "url": "https://uptime.betterstack.com/ips-by-cluster.json", + "selector": "$.*[*]" + } + ] + } + ], "instances": { "accepted": [ "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" @@ -13410,7 +13534,7 @@ "sources": [ { "type": "http-csv", - "url": "http://www.facebook.com/peering/geofeed" + "url": "https://www.facebook.com/peering/geofeed" } ] } @@ -13952,22 +14076,14 @@ "forbidden": [] }, "addition_date": "2026/02/07", - "verification": [ - { - "type": "ip", - "ips": [ - "35.204.201.174", - "34.125.202.46" - ] - } - ], + "verification": [], "instances": { "accepted": [ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Adagiobot/1.0) Chrome/112.0.5615.137 Safari/537.36" ], "rejected": [] }, - "url": "https://adagio-io.gitbook.io/adagio-documentation/general-configuration/update-your-app-ads.txt-file" + "url": "https://adagio.io/" }, { "id": "algolia-crawler", @@ -14108,6 +14224,16 @@ "masks": [ "@.crawl.commoncrawl.org" ] + }, + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://index.commoncrawl.org/ccbot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] } ], "instances": { @@ -14396,7 +14522,7 @@ "sources": [ { "type": "http-csv", - "url": "http://www.facebook.com/peering/geofeed" + "url": "https://www.facebook.com/peering/geofeed" } ] } @@ -14732,6 +14858,12 @@ "type": "ip", "ips": [ "66.132.159.0/24", + "66.132.148.0/24", + "66.132.153.0/24", + "66.132.224.0/24", + "66.132.186.0/24", + "66.132.195.0/24", + "66.132.172.0/24", "162.142.125.0/24", "167.94.138.0/24", "167.94.145.0/24", @@ -15193,7 +15325,7 @@ "sources": [ { "type": "http-csv", - "url": "http://www.facebook.com/peering/geofeed" + "url": "https://www.facebook.com/peering/geofeed" } ] } @@ -15226,7 +15358,7 @@ "sources": [ { "type": "http-csv", - "url": "http://www.facebook.com/peering/geofeed" + "url": "https://www.facebook.com/peering/geofeed" } ] } @@ -15360,7 +15492,18 @@ "DuckAssistBot" ], "addition_date": "2026/08/18", - "verification": [] + "verification": [ + { + "type": "cidr", + "sources": [ + { + "type": "http-json", + "url": "https://duckduckgo.com/duckassistbot.json", + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" + } + ] + } + ] }, { "id": "deepseek-crawler", From 6ce66d769346a8b6edb2037a39872f0025fd7b90 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 20:12:12 +0000 Subject: [PATCH 5/6] fix: address review comments on selectors and patterns Use README-correct JSONPath selectors on new Google and WatchFor entries, tighten Anomura and Poduptime patterns, and point Freshping at the official IP article. Co-authored-by: david --- well-known-bots.json | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/well-known-bots.json b/well-known-bots.json index c882c85..007a5e5 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -67,7 +67,7 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -11533,7 +11533,7 @@ "forbidden": [] }, "addition_date": "2022/10/17", - "url": "https://www.freshworks.com/website-monitoring/", + "url": "https://support.freshping.io/support/solutions/articles/237620-why-are-my-checks-always-down-even-when-my-website-is-up-", "verification": [ { "type": "ip", @@ -14949,12 +14949,12 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -14997,12 +14997,12 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -15048,7 +15048,7 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -15092,12 +15092,12 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -15139,12 +15139,12 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -15187,12 +15187,12 @@ { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" }, { "type": "http-json", "url": "https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -15973,7 +15973,7 @@ { "type": "http-json", "url": "https://watchfor.io/ips.json", - "selector": "$.prefixes[*][\\\"ipv6Prefix\\\",\\\"ipv4Prefix\\\"]" + "selector": "$.prefixes[*][\"ipv4Prefix\",\"ipv6Prefix\"]" } ] }, @@ -16261,7 +16261,7 @@ ], "pattern": { "accepted": [ - "Anomura" + "Anomura\\/" ], "forbidden": [] }, @@ -17232,7 +17232,7 @@ ], "pattern": { "accepted": [ - "Poduptime" + "Poduptime;\\s*\\+fediverse\\.observer" ], "forbidden": [] }, From 25513cfb8585b8d18f4626a62b58e920c21d98ab Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 18 Aug 2026 20:23:00 +0000 Subject: [PATCH 6/6] fix: remove unsupported bot identities Drop stale or fabricated bot entries, update Cloudflare AI Search and Amazon User metadata from current provider docs, remove discontinued Freshping IP verification, and use a representative Cypress Electron UA. Co-authored-by: david --- well-known-bots.json | 140 +++---------------------------------------- 1 file changed, 8 insertions(+), 132 deletions(-) diff --git a/well-known-bots.json b/well-known-bots.json index 007a5e5..47dc7dc 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -11534,24 +11534,7 @@ }, "addition_date": "2022/10/17", "url": "https://support.freshping.io/support/solutions/articles/237620-why-are-my-checks-always-down-even-when-my-website-is-up-", - "verification": [ - { - "type": "ip", - "ips": [ - "35.173.69.86", - "50.17.185.102", - "18.179.133.14", - "34.246.131.0", - "13.251.205.206", - "52.60.140.174", - "13.55.57.184", - "52.42.49.200", - "13.232.175.73", - "18.228.60.182", - "18.130.156.195" - ] - } - ], + "verification": [], "instances": { "accepted": [ "FreshpingBot/1.0 (+https://freshping.io/)" @@ -15505,30 +15488,6 @@ } ] }, - { - "id": "deepseek-crawler", - "categories": [ - "ai" - ], - "pattern": { - "accepted": [ - "DeepSeekBot" - ], - "forbidden": [] - }, - "instances": { - "accepted": [ - "Mozilla/5.0 (compatible; DeepSeekBot/1.0; +https://www.deepseek.com/bot)" - ], - "rejected": [] - }, - "url": "https://www.deepseek.com/bot", - "aliases": [ - "DeepSeekBot" - ], - "addition_date": "2026/08/18", - "verification": [] - }, { "id": "firecrawl-agent", "categories": [ @@ -15555,23 +15514,24 @@ "verification": [] }, { - "id": "cloudflare-autorag", + "id": "cloudflare-ai-search", "categories": [ "ai" ], "pattern": { "accepted": [ - "Cloudflare-AutoRAG" + "Cloudflare-(?:AI-Search|AutoRAG)" ], "forbidden": [] }, "instances": { "accepted": [ + "Cloudflare-AI-Search", "Cloudflare-AutoRAG (https://developers.cloudflare.com/autorag; autorag@cloudflare.com)" ], "rejected": [] }, - "url": "https://developers.cloudflare.com/autorag", + "url": "https://developers.cloudflare.com/ai-search/platform/release-note/", "addition_date": "2026/08/18", "verification": [] }, @@ -15848,7 +15808,7 @@ }, "instances": { "accepted": [ - "Mozilla/5.0 (compatible; Amzn-User; +https://developer.amazon.com/)" + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36" ], "rejected": [] }, @@ -16039,27 +15999,6 @@ "addition_date": "2026/08/18", "verification": [] }, - { - "id": "thousandeyes-monitor", - "categories": [ - "monitor" - ], - "pattern": { - "accepted": [ - "ThousandEyes" - ], - "forbidden": [] - }, - "instances": { - "accepted": [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 (ThousandEyes Agent)" - ], - "rejected": [] - }, - "url": "https://docs.thousandeyes.com/product-documentation/global-vantage-points/cloud-and-enterprise-agents", - "addition_date": "2026/08/18", - "verification": [] - }, { "id": "alertsite-monitor", "categories": [ @@ -16149,69 +16088,6 @@ "addition_date": "2026/08/18", "verification": [] }, - { - "id": "playwright", - "categories": [ - "tool" - ], - "pattern": { - "accepted": [ - "[Pp]laywright" - ], - "forbidden": [] - }, - "instances": { - "accepted": [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Playwright/1.40.0" - ], - "rejected": [] - }, - "url": "https://playwright.dev", - "addition_date": "2026/08/18", - "verification": [] - }, - { - "id": "puppeteer", - "categories": [ - "tool" - ], - "pattern": { - "accepted": [ - "[Pp]uppeteer" - ], - "forbidden": [] - }, - "instances": { - "accepted": [ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36 Puppeteer" - ], - "rejected": [] - }, - "url": "https://pptr.dev", - "addition_date": "2026/08/18", - "verification": [] - }, - { - "id": "selenium", - "categories": [ - "tool" - ], - "pattern": { - "accepted": [ - "[Ss]elenium" - ], - "forbidden": [] - }, - "instances": { - "accepted": [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36; Selenium" - ], - "rejected": [] - }, - "url": "https://www.selenium.dev", - "addition_date": "2026/08/18", - "verification": [] - }, { "id": "cypress", "categories": [ @@ -16225,11 +16101,11 @@ }, "instances": { "accepted": [ - "Cypress/13.0.0" + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Cypress/13.0.0 Chrome/114.0.5735.289 Electron/25.8.0 Safari/537.36" ], "rejected": [] }, - "url": "https://www.cypress.io", + "url": "https://docs.cypress.io/app/references/launching-browsers#Electron-Browser", "addition_date": "2026/08/18", "verification": [] },