diff --git a/fixtures/downloaded/downloaded b/fixtures/downloaded/downloaded
index 053b5b0..b510b01 100644
--- a/fixtures/downloaded/downloaded
+++ b/fixtures/downloaded/downloaded
@@ -1 +1 @@
-Mon, 09 Sep 2024 15:39:05 GMT
\ No newline at end of file
+Mon, 23 Sep 2024 20:27:17 GMT
\ No newline at end of file
diff --git a/fixtures/downloaded/matomo-org.json b/fixtures/downloaded/matomo-org.json
index 04faa89..a20e2bc 100644
--- a/fixtures/downloaded/matomo-org.json
+++ b/fixtures/downloaded/matomo-org.json
@@ -50,6 +50,7 @@
   "msnbot/2.0b v1394720947.6703",
   "MSNBot/Nutch-1.5.1",
   "bitlybot/3.0",
+  "Blackbox Exporter/0.24.0",
   "Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)",
   "Bloglovin/1.0 (http://www.bloglovin.com; 1 subscribers)",
   "Blogtrottr/2.0",
@@ -710,6 +711,7 @@
   "cyberscan.io",
   "Mozilla/5.0 (l9scan/2.0.xxxxxxxxxxxxxxxxxxxxxxxx; +https://leakix.net)",
   "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)",
+  "Mozilla/5.0 (compatible; vuhuvRBT/2.0; +https://vuhuv.com/rbt.html)",
   "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com",
   "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Image size by Siteimprove.com",
   "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Probe by Siteimprove.com",
@@ -1018,5 +1020,12 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) https://deepcrawl.com/bot",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) https://deepcrawl.com/testing",
   "Mozilla/7.0 (compatible; Golfe/1.1; +http://www.goo-olfe.ae/bot.html)",
-  "Mozilla/5.0 (compatible; SpiderLing; +https://nlp.fi.muni.cz/projects/biwec/)"
+  "Mozilla/5.0 (compatible; SpiderLing; +https://nlp.fi.muni.cz/projects/biwec/)",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bravebot/1.0; +https://search.brave.com/help/brave-search-crawler) Chrome/W.X.Y.Z Safari/537.36",
+  "Mozilla/5.0 (compatible; 1001FirmsBot/1.0; +https://www.1001firms.com/1001firmsbot.php)",
+  "Google-CloudVertexBot",
+  "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)",
+  "Oh Dear Sitemap check https://ohdear.app",
+  "Mozilla/5.0 (compatible; OhDear/1.1; +https://ohdear.app/checker; brokenLinks)",
+  "Mozilla/5.0 (compatible; OhDear/1.1; +https://ohdear.app/checker; mixedContent)"
 ]
diff --git a/fixtures/downloaded/myip.ms.json b/fixtures/downloaded/myip.ms.json
index 934c8f2..5bee8a3 100644
--- a/fixtures/downloaded/myip.ms.json
+++ b/fixtures/downloaded/myip.ms.json
@@ -23,13 +23,13 @@
   "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
   "Mozilla/5.0 (compatible; Cliqzbot/1.0; +http://cliqz.com/company/cliqzbot)",
   "Googlebot-Video/1.0",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
   "Mozilla/5.0 (compatible; Googlebot/2.1;+http://www.google.com/bot.html)",
   "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.tropicdesigns.net)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)",
   "Mozilla/5.0 (compatible; 008/0.85; http://www.80legs.com/webcrawler.html) Gecko/2008032620",
   "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943)",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
   "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (iPhone; U; CPU iPhone OS 41 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
   "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",
@@ -58,10 +58,10 @@
   "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.268",
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36",
+  "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
   "Mozilla/5.0 (iPhone; U; CPU iPhone OS 41 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider",
   "Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)",
-  "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
   "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
   "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/112.0.0.0 Safari/537.36",
@@ -105,17 +105,17 @@
   "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",
   "Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.0.0 Safari/537.36",
   "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php+)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (X11; Linux x8664) AppleWebKit/537.36 (KHTML like Gecko; Google Page Speed Insights) Chrome/27.0.1453 Safari/537.36 GoogleBot/2.1",
   "pyspider/0.3.7 (+http://pyspider.org/)",
   "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/)",
-  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.71 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )",
   "Mozilla/5.0 (compatible; SputnikBot/2.3; +http://corp.sputnik.ru/webmaster)",
   "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )",
   "Mozilla/5.0 (Linux;u;Android 2.3.7;zh-cn;) AppleWebKit/533.1 (KHTMLlike Gecko) Version/4.0 Mobile Safari/533.1 (compatible; +http://www.baidu.com/search/spider.html)",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)",
   "NerdyBot",
   "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)",
   "Mozilla/5.0 (compatible;PetalBot;+",
@@ -131,8 +131,8 @@
   "Mozilla/5.0 (compatible) Feedfetcher-Google;(+http://www.google.com/feedfetcher.html)",
   "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36",
-  "Scrapy/1.4.0 (+http://scrapy.org)",
   "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)",
+  "Scrapy/1.4.0 (+http://scrapy.org)",
   "Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)",
   "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)",
   "Baiduspider+(+http://www.baidu.com/search/spider.htm)",
@@ -148,9 +148,9 @@
   "Mozilla/5.0 (YahooYSMcm/3.0.0; http://help.yahoo.com)",
   "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider",
   "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/aid/42716/)",
+  "FeedBurner/1.0 (http://www.FeedBurner.com)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "FeedBurner/1.0 (http://www.FeedBurner.com)",
   "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)",
   "Mozilla/5.0 (iPhone; CPU iPhone OS 601 like Mac OS X) AppleWebKit/537.36 (KHTML like Gecko; Google Page Speed Insights) Version/6.0 Mobile/10A525 Safari/8536.25 GoogleBot/2.1",
   "Mozilla/5.0 (iPhone; CPU iPhone OS 70 like Mac OS X) AppleWebKit/537.51.1 (KHTML like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",
@@ -166,6 +166,7 @@
   "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)",
   "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 Prerender (+https://github.com/prerender/prerender)",
   "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+http://aspiegel.com/petalbot)",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.97 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.90 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)",
@@ -178,9 +179,9 @@
   "Mozilla/5.0 (compatible; DnyzBot/1.0)",
   "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.5563.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/100.0.4896.127 Safari/537.36",
   "Screaming Frog SEO Spider/16.7",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.5563.146 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/100.0.4896.127 Safari/537.36",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5414.74 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -201,9 +202,9 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.85 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (X11; Linux x8664) AppleWebKit/537.4 (KHTML like Gecko; Google Page Speed Insights) Chrome/22.0.1229 Safari/537.4 GoogleBot/2.1",
   "1st ZipCommander (Net) - http://www.zipcommander.com",
+  "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.6045.123 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Scrapy/1.5.0 (+https://scrapy.org)",
@@ -246,10 +247,10 @@
   "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.216 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17",
+  "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
   "JOC Web Spider",
   "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)",
   "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
-  "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
   "annotate_google; http://ponderer.org/download/annotate_google.user.js",
   "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25(compatible; Sogou wap spider/4.0; +http://www.sogou.com/docs/help/webmasters.htm#07)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -284,15 +285,16 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.5195.102 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )",
   "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20110814 Firefox/6.0 Google (+https://developers.google.com/+/web/snippet/)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.132 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19 moatbot",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.6045.105 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "CRAZYWEBCRAWLER 0.9.0 http://www.crazywebcrawler.com",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.126 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "spider-ads",
   "ZumBot/1.0 (ZUM Search; http://help.zum.com/inquiry)",
+  "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
   "Mozilla/5.0 (iPhone; CPU iPhone OS 81 like Mac OS X) AppleWebKit/600.1.4 (KHTML like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)",
   "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.175 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -303,10 +305,9 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.78 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)",
-  "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
+  "Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)",
   "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.126 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.92 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.199 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -320,10 +321,11 @@
   "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)",
   "Mozilla/5.0 (compatible; musobot/1.0; info@muso.com; +http://www.muso.com)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.130 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)",
   "Mozilla/5.0 (compatible; archive.orgbot +http://www.archive.org/details/archive.orgbot)",
+  "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.127 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible; DnyzBot/1.0) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/64.0.3282.167 Safari/537.36",
-  "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)",
   "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)",
   "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)",
   "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)",
@@ -335,7 +337,6 @@
   "FreeWebMonitoring SiteChecker/0.1 (+http://www.freewebmonitoring.com)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.133 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.92 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.5993.117 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.102 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -353,17 +354,18 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.84 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "LinkWalker/3.0 (http://www.brandprotect.com)",
   "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.99 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.113 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
   "Scrapy/1.6.0 (+https://scrapy.org)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.5563.110 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.99 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "WeSEE:Search/0.1 (Alpha http://www.wesee.com/bot/)",
+  "Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "demandbase-bot/0.1 (http://www.demandbase.com; info@demandbase.com)",
   "Mozilla/4.0 (vBSEO; http://www.vbseo.com)",
   "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )",
-  "Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)",
   "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.121 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.155 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -375,7 +377,9 @@
   "Mozilla/5.0 (YunSecurityBot/1.0) Gecko/20100101 Firefox/36.0",
   "Mozilla/4.0 (compatible; HostTracker.com/1.0;+http://host-tracker.com/)",
   "https://dulkarneen666group.us.su?hl=en&noredirect=I Us sU I http://dulkarneen666group.us.su?hl=en&noredirect=I Us sU I ftp://dulkarneen666group.us.su?hl=en&noredirect=I Us sU I https://groups.google.com/group/wwwdulkarneen666group?hl=en&noredirect=I Us sU",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.137 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)",
   "facebookscraper/1.0( http://www.facebook.com/sharescraperhelp.php)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.109 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.141 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -385,7 +389,6 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.5993.88 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.6045.159 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible; Y!J-WSC/1.0; +https://yahoo.jp/3BSZgF)",
-  "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (compatible) Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)",
   "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)",
@@ -396,11 +399,10 @@
   "Mozilla/5.0 (compatible; SputnikBot/2.3)",
   "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0 ; Claritybot)",
   "Mozilla/5.0 (compatible; Codewisebot/2.0; +http://www.nosite.com/somebot.htm)",
-  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.113 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +https://help.mail.ru/webmaster/indexing/robots)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.138 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Scrapy/1.0.3 (+http://scrapy.org)",
   "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id3276502715230424062)",
-  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +https://help.mail.ru/webmaster/indexing/robots)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.66 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "AdsTxtCrawler/1.0; +https://github.com/InteractiveAdvertisingBureau/adstxtcrawler",
@@ -460,11 +462,11 @@
   "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)",
   "Pinterest/0.1 +http://pinterest.com",
   "ias-va/3.1 (+https://www.admantx.com/service-fetcher.html)",
+  "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)",
   "Mozilla/5.0 (compatible; Qwantify/1.0; +https://www.qwant.com/)",
   "Mozilla/5.0 (compatible; Qwantify/2.3w; +https://www.qwant.com/)/2.3w",
   "http://www.spreadtrum.com/wap/6600LDORADOUAProfile.xml",
   "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)",
-  "Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)",
   "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)",
   "Mozilla/5.0 (compatible; uCrawler/1.0 ; +https://blog.ucoz.ru/upolicy)",
   "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0",
@@ -483,6 +485,7 @@
   "Affectv Robot v1.0/Nutch-1.6 (http://www.affectv.co.uk; austin at affectv dot co dot uk)",
   "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.5672.63 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/114.0.5735.179 Safari/537.36",
   "Pinterest/0.2 (+http://www.pinterest.com/)",
   "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)",
@@ -522,13 +525,13 @@
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/111.0.5563.64 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/112.0.5615.142 Safari/537.36",
   "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1 (compatible; Sogou web spider/4.0; +http://www.sogou.com/docs/help/webmasters.htm#07)",
+  "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)",
   "Sogou Web Spider",
   "woobot/2.0",
   "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id4797747767309071643)",
   "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; feed-id7935049554773385424)",
   "Mozilla/5.0 (compatible; AhrefsBot/2.0; +http://ahrefs.com/robot/)",
   "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )",
-  "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)",
   "WeSEE:Search/0.1 (Alpha http://www.wesee.com/en/support/bot/)",
   "Mozilla/5.0 (compatible; AhrefsBot/2.0; +http://ahrefs.com/robot/)gzip(gfe) (via docs.google.com/viewer)",
   "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",
@@ -557,6 +560,7 @@
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/101.0.4951.64 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/85.0.4183.140 Safari/537.36",
   "admantx-usastn/2.4 (+http://www.admantx.com/service-fetcher.html)",
+  "Buck/2.4.2; (+https://app.hypefactors.com/media-monitoring/about.html)",
   "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)",
   "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com)",
   "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)",
@@ -571,7 +575,7 @@
   "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/109.0.5414.101 Safari/537.36",
   "Riddler (http://riddler.io/about)",
-  "Buck/2.4.2; (+https://app.hypefactors.com/media-monitoring/about.html)",
+  "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "Mozilla/5.0 (compatible; Qwantify/2.1w; +https://www.qwant.com/)/*",
   "Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)",
   "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)",
@@ -612,6 +616,7 @@
   "Mozilla/5.0 (Windows NT 6.2) Insitesbot/1.0",
   "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)",
   "tbot-nutch/Nutch-1.10",
+  "colly - https://github.com/gocolly/colly/v2",
   "CoralWebPrx/0.1.20 (See http://coralcdn.org/)",
   "CRAZYWEBCRAWLER 0.9.4 http://www.crazywebcrawler.com",
   "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id8137258620597103195)",
@@ -624,6 +629,7 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.139 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.207 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/102.0.5005.115 Safari/537.36",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/127.0.6533.99 Safari/537.36",
   "semanticbot (info@semanticaudience.com)",
   "WCH Web Spider",
   "Yahoo!+Slurp/3.0;+http://help.yahoo.com/help/us/ysearch/slurp)",
@@ -649,7 +655,6 @@
   "admantx-euastn/2.5 (+http://www.admantx.com/service-fetcher.html)",
   "annotategoogle; http://ponderer.org/download/annotategoogle.user.js",
   "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG",
-  "colly - https://github.com/gocolly/colly/v2",
   "Mozilla/5.0 (compatible; Baiduspider-cpro; +http://www.baidu.com/search/spider.html)",
   "Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)",
   "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/support/robots.html)",
@@ -680,7 +685,6 @@
   "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)",
   "Mozilla/5.0 (compatible; Yahoo Pipes 2.0; +http://developer.yahoo.com/yql/provider) Gecko/20090729 Firefox/3.5.2",
   "Mozilla/5.0 (Linux; Android 4.4.4; Nexus 5 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.102 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/127.0.6533.99 Safari/537.36",
   "Scrapy/1.1.1 (+http://scrapy.org)",
   "Screaming Frog SEO Spider/9.1",
   "bitlybot",
@@ -734,11 +738,11 @@
   "Mozilla/5.0 (Linux; U; Android 4.2.2; en-us; CUBOT P9 Build/JDQ39) AppleWebKit/534.30 (KHTML like Gecko) Version/4.0 Mobile Safari/534.30",
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)",
   "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
   "New-Sogou-Spider/1.0 (compatible; MSIE 5.5; Windows 98)",
   "Slackbot-LinkExpanding (+https://api.slack.com/robots)",
   "Spiderbot/Nutch-1.7",
   "admantx-usasab/3.0 (+http://www.admantx.com/service-fetcher.html)",
-  "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
   "Mozilla/5.0 (compatible; AddSearchBot; +http://www.addsearch.com/bot/)",
   "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot help@moz.com)",
   "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)",
@@ -756,6 +760,7 @@
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/126.0.6478.182 Safari/537.36",
   "Mozilla/5.0+(compatible;+YodaoBot/1.0;+http://www.youdao.com/help/webmaster/spider/;+)",
   "My Nutch Spider/Nutch-1.11",
+  "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)",
   "SeznamBot/3.0 (+http://fulltext.sblog.cz/)",
   "suma spider",
   "crawler4j (https://github.com/yasserg/crawler4j/)",
@@ -771,11 +776,9 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.164 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)",
   "Mozilla/5.0 (Windows; U; Windows NT 5.1; ar; rv:1.9.1) Gecko/20090624 Firefox/3.5%https://dulkarneen666group.us.suhlen&noredirectI UssU I http://dulkarneen666group.us.suhlen&noredirectI UssU I ftp://dulkarneen666group.us.suhlen&noredirectI UssU I https://",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/105.0.5195.102 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/111.0.5563.146 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36",
-  "serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)",
   "admantx-ussy04/3.2 (+http://www.admantx.com/service-fetcher.html)",
   "http://www.checkprivacy.or.kr:6600/RS/PRIVACYENFAQ.jsp",
   "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)",
@@ -802,6 +805,7 @@
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.63 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/119.0.6045.123 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/125.0.6422.154 Safari/537.36",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.6613.113 Safari/537.36",
   "Scrapy/0.24.4 (+http://scrapy.org)",
   "SemrushBot/0.91",
   "SEOENGWorldBot/1.0 (+http://www.seoengine.com/seoengbot.htm)",
@@ -903,7 +907,6 @@
   "Mozilla/5.0 (Linux; Android 6.0; CUBOT_NOTE_S Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36",
   "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://support.google.com/webmasters/answer/1061943),gzip(gfe)",
   "Mozilla/5.0 (Linux; U; Android 4.2.2; fa-ir; GLX SpiderI Build/JDQ39) AppleWebKit/534.30 (KHTML like Gecko) Version/4.0 Mobile Safari/534.30",
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/106.0.5249.91 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/118.0.5993.70 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/120.0.6099.71 Safari/537.36",
@@ -926,6 +929,7 @@
   "Googlebot-Mobile",
   "Grammarly/1.0 (http://www.grammarly.com)",
   "ias-ir/3.1 (+https://www.admantx.com/service-fetcher.html)",
+  "ias-va/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)",
   "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)",
   "Mechanize/2.7.3 Ruby/1.9.3p551 (http://github.com/sparklemotion/mechanize/)",
   "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)",
@@ -968,7 +972,6 @@
   "GridBot/1.0 crawler@sysomos.com",
   "http://www.checkprivacy.or.kr:6600/RS/PRIVACYFAQ.jsp",
   "Huaweisymantecspider (compatible; MSIE 8.0; DSE-support@huaweisymantec.com)",
-  "ias-va/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)",
   "Mechanize/2.7.2 Ruby/2.1.3p242 (http://github.com/sparklemotion/mechanize/)",
   "MixBot (+http://t.co/GSRLLKex24)",
   "Mozilla/5.0 (compatible; 4SeoHuntBot; +http://4seohunt.com/bot.html)",
@@ -1084,6 +1087,7 @@
   "DomainCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/steelads.com)",
   "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id13966453542643020755)",
   "GentleSource Short URL Checker ( http://www.gentlesource.com/ )",
+  "ias-ie/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)",
   "LivelapBot/0.2 (http://site.livelap.com/crawler)",
   "Mechanize/2.7.2 Ruby/2.1.2p95 (http://github.com/sparklemotion/mechanize/)",
   "MLBot (www.metadatalabs.com/mlbot)",
@@ -1154,7 +1158,6 @@
   "EasyBib AutoCite (http://autocite-info.citation-api.com/)",
   "GarlikCrawler/1.2 (http://garlik.com/ crawler@garlik.com)",
   "HTTPRequest2/2.1.1 (http://pear.php.net/package/httprequest2) PHP/5.3.2-1ubuntu4.18",
-  "ias-ie/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)",
   "ias-sg/3.1 (+https://www.admantx.com/service-fetcher.html)",
   "InAGist URL Resolver (http://inagist.com)",
   "Keybot Translation-Search-Machine",
@@ -1453,6 +1456,7 @@
   "Scrapy/1.0.1 (+http://scrapy.org)",
   "Scrapy/2.4.1 (+https://scrapy.org)",
   "Screaming Frog SEO Spider/10.2",
+  "Screaming Frog SEO Spider/19.4",
   "Screaming Frog SEO Spider/3.3",
   "Screaming Frog SEO Spider/7.2",
   "semanticbot",
@@ -1460,6 +1464,7 @@
   "SNK Screenshot Bot/0.20 ( abuse at savenkeep dotcom )",
   "Spiderbot",
   "sqlmap/1.2.1.11#dev (http://sqlmap.org)",
+  "StartmeBot/1.0 (+https://start.me/bot)",
   "TurnitinBot/3.0 (http://www.turnitin.com/robot/crawlerinfo.html)",
   "User-Agent:Mozilla/5.0 (SaidwotBot)",
   "W3CValidator/1.3 http://validator.w3.org/services",
@@ -1626,6 +1631,7 @@
   "Mozilla/5.0 Annuaire gratuit generaliste multiprise.NET (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0.1 AlexaToolbar/alxf-2.15 (http://www.multiprise.net)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.101 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/110.0.5481.77 Safari/537.36",
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.6613.119 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/75.0.3761.0 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/83.0.4103.100 Safari/537.36",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/83.0.4103.108 Safari/537.36",
@@ -1650,12 +1656,12 @@
   "scrapyproject (+http://www.profound.net)",
   "Screaming Frog SEO Spider/10.4",
   "Screaming Frog SEO Spider/2.20",
+  "Screaming Frog SEO Spider/20.2",
   "Screaming Frog SEO Spider/7.1",
   "SimpleBrowser (http://github.com/axefrog/SimpleBrowser)",
   "SiteCheckerBot/1.0 (+http://sitechecker.pro)",
   "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
   "sqlmap/1.0.8.2#dev (http://sqlmap.org)",
-  "StartmeBot/1.0 (+https://start.me/bot)",
   "Surferbot/1.0 (https://surferseo.com)",
   "TasBot (http://www.httpool.com/)",
   "test bot",
@@ -1740,6 +1746,7 @@
   "GoodBot",
   "Google Bot",
   "google-bot",
+  "got (https://github.com/sindresorhus/got)",
   "grub-client-1.5.3; (grub-client-1.5.3; Crawl your own stuff with http://grub.org)",
   "GVC-SEARCH-BOT (+http://computernetworkscctvservices.wordpress.com) ZSEBOT",
   "Hatena Antenna/0.5 (http://a.hatena.ne.jp/help)",
@@ -1967,7 +1974,6 @@
   "Scrapy/2.5.1 (+https://scrapy.org)",
   "Scrapy/2.6.1 (+https://scrapy.org)",
   "Screaming Frog SEO Spider/15.2",
-  "Screaming Frog SEO Spider/19.4",
   "Screaming Frog SEO Spider/19.8",
   "Screaming Frog SEO Spider/2.50",
   "Screaming Frog SEO Spider/3.2",
@@ -2190,7 +2196,6 @@
   "Gigabot",
   "Gigabot/1.0",
   "gonzo2P +http://www.suchen.de/faq.html",
-  "got (https://github.com/sindresorhus/got)",
   "got/8.3.1 (https://github.com/sindresorhus/got)",
   "grub crawler(http://www.grub.org)",
   "gschaid bot/1.0 (+http://www.gschaid.de/bot.html)",
@@ -2456,6 +2461,8 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.110 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.6045.159 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.113 Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.137 Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.94 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.205 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
@@ -2639,7 +2646,6 @@
   "Screaming Frog SEO Spider/2.00",
   "Screaming Frog SEO Spider/2.22",
   "Screaming Frog SEO Spider/2.40",
-  "Screaming Frog SEO Spider/20.2",
   "Screaming Frog SEO Spider/230",
   "Screaming Frog SEO Spider/5.0",
   "SearchBot v1.1.0",
@@ -2757,6 +2763,7 @@
   "DigitalPebble Crawler/1.14 (StormCrawler Benchmark; http://digitalpebble.com/; crawler@digitalpebble.com)",
   "DingTalkBot-SecurityService/1.0",
   "Discordbot/2.0; ( https://discordapp.com)",
+  "dopamina katecholaminy potasu https://i.pinimg.com/originals/64/e5/44/64e544072049b0ef38cc04ac604259c5.png",
   "DowntimeDetector/1.0 (+http://downforeveryoneorjustme.com)",
   "Dy robot 1.0",
   "elefent/1.4 (web crawler :: robots.txt exclude elefent; https://crawler.elefent.net; webmaster at elefent dot net)",
@@ -2798,6 +2805,7 @@
   "HTTP_Request2/2.3.0 (http://pear.php.net/package/http_request2) PHP/7.0.32-0ubuntu0.16.04.1",
   "HybridBot (hybrid.ru/about. If our bot caused problems please contact us. Contact email: abuse@hybrid.ru)",
   "ias-au/3.1 (+https://www.admantx.com/service-fetcher.html)",
+  "ias-sg/3.3 (former https://www.admantx.com + https://integralads.com/about-ias/)",
   "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)",
   "Iframely/1.3.1 (+https://iframely.com/docs/about) Notion",
   "Is is up? (+https://isitup.org)",
@@ -3754,6 +3762,33 @@
   "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1725505542-0",
   "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1725505927-0",
   "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1725664330-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1725883649-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1725948661-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726043824-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726046241-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726053568-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726063716-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726084898-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726084965-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726085029-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726085040-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726092687-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726285804-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726474649-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726483502-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726602760-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726671225-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726721379-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726734777-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726740043-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726743552-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726745239-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726752995-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726771361-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726771374-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726788680-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726912150-0",
+  "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) cnv.aws-prod---sieve.hlfs-rest_client/1726987605-0",
   "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1603028876-0",
   "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1603256091-0",
   "Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html) tands-prod-eng.hlfs-prod---sieve.hlfs-desktop/1603374815-0",
@@ -10285,7 +10320,7 @@
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.65 Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.6478.182 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.99 Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
-  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.113 Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)",
+  "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.6613.113 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.103 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.118 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
   "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.123 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
diff --git a/package.json b/package.json
index 6c41ff3..d61e065 100644
--- a/package.json
+++ b/package.json
@@ -60,7 +60,7 @@
   },
   "devDependencies": {
     "@types/jest": "^29.5.12",
-    "esbuild": "^0.23.0",
+    "esbuild": "~0.24.0",
     "jest": "^29.7.0",
     "prettier": "^3.2.5",
     "pug": "^3.0.2",