@@ -27,6 +27,7 @@ final class Bots
27
27
|AmorankSpider
28
28
|ApacheBench
29
29
|Applebot
30
+ |iTMS
30
31
|AppSignalBot
31
32
|Arachni
32
33
|AspiegelBot
@@ -40,6 +41,7 @@ final class Bots
40
41
|BazQux
41
42
|Better\sUptime\sBot
42
43
|MSNBot|msrbot|bingbot|bingadsbot|BingPreview|msnbot-(UDiscovery|NewsBlogs)|adidxbot
44
+ |Blackbox\sExporter
43
45
|Blekkobot
44
46
|BLEXBot
45
47
|Bloglovin
@@ -58,6 +60,8 @@ final class Bots
58
60
|Cloudflare-SSLDetector
59
61
|Cloudflare\sCustom\sHostname\sVerification
60
62
|Cloudflare-Traffic-Manager
63
+ |Cloudflare-Smart-Transit
64
+ |CloudflareObservatory
61
65
|https://developers\.cloudflare\.com/security-center/
62
66
|coccoc\.com
63
67
|collectd
@@ -72,14 +76,17 @@ final class Bots
72
76
|Domain\sRe-Animator\sBot|support@domainreanimator\.com
73
77
|DotBot
74
78
|DuckDuck(?:Go-Favicons-)?Bot
79
+ |DuckAssistBot
75
80
|EasouSpider
76
81
|eCairn-Grabber
77
82
|EMail\sExractor
78
83
|evc-batch
79
84
|Exabot|ExaleadCloudview
80
85
|ExactSeek\sCrawler
81
86
|Ezooms
82
- |facebookexternalhit|facebookplatform|facebookexternalua|facebookcatalog
87
+ |facebook(?:catalog|externalhit|externalua|platform|scraper)
88
+ |meta-externalagent
89
+ |meta-externalfetcher
83
90
|FacebookBot/[\d.]+
84
91
|Feedbin
85
92
|FeedBurner
@@ -104,25 +111,32 @@ final class Bots
104
111
|Google-Structured-Data-Testing-Tool
105
112
|GoogleStackdriverMonitoring
106
113
|Google-Transparency-Report
114
+ |Google-CloudVertexBot
107
115
|via\sggpht\.com\sGoogleImageProxy
116
+ |Google-Document-Conversion
117
+ |GoogleDocs;\sapps-spreadsheets
118
+ |GoogleDocs;\sapps-presentations
119
+ |GoogleDocs;
108
120
|SeznamEmailProxy
109
121
|Seznam-Zbozi-robot
110
122
|Heurekabot-Feed
111
123
|ShopAlike
124
+ |deepcrawl\.com
112
125
|Googlebot-News
113
- |Adwords-(?:DisplayAds|Express|Instant)|Google\sWeb\sPreview|Google[\s-]Publisher[\s-]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Read-Aloud|Safety| Shopping-Quality|Site-Verification|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|DuplexWeb |Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\+/web/snippet
126
+ |Adwords-(?:DisplayAds|Express|Instant)|Google\sWeb\sPreview|Google[\s-]Publisher[\s-]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Read-Aloud|Shopping-Quality|Site-Verification|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\+/web/snippet
114
127
|^Google$
128
+ |Google-Safety
129
+ |DuplexWeb-Google
115
130
|Google-Area120-PrivacyPolicyFetcher
116
- |heritrix
117
131
|HubSpot\s
118
- |vuhuvBot
132
+ |vuhuv(?:Bot|RBT)
119
133
|HTTPMon/[\d.]+
120
134
|ICC-Crawler
121
135
|inoreader\.com
122
136
|iisbot
123
137
|ips-agent
124
138
|IP-Guide\.com
125
- |k6/[0-9\ .]+
139
+ |k6/[0-9.]+
126
140
|kouio
127
141
|larbin
128
142
|[A-z0-9]*-Lighthouse
@@ -134,7 +148,7 @@ final class Bots
134
148
|magpie-crawler
135
149
|MagpieRSS
136
150
|masscan-ng/[\d.]+
137
- |masscan
151
+ |.* masscan
138
152
|Mastodon/
139
153
|meanpathbot
140
154
|MetaJobBot
@@ -155,6 +169,7 @@ final class Bots
155
169
|nlcrawler
156
170
|Nmap\sScripting\sEngine
157
171
|Nuzzel
172
+ |NodePing
158
173
|Octopus\s[0-9]
159
174
|OnlineOrNot\.com_bot
160
175
|omgili
@@ -174,7 +189,7 @@ final class Bots
174
189
|Quora-Bot
175
190
|RamblerMail
176
191
|QuerySeekerSpider
177
- |Qwantify
192
+ |Qwantify|Qwantbot
178
193
|Rainmeter
179
194
|redditbot
180
195
|Riddler
@@ -185,6 +200,7 @@ final class Bots
185
200
|Screaming\sFrog\sSEO\sSpider
186
201
|ScreenerBot
187
202
|SemrushBot
203
+ |BacklinksExtendedBot
188
204
|SerpReputationManagementAgent/[\d.]+
189
205
|SplitSignalBot
190
206
|SiteAuditBot/[\d.]+
@@ -206,7 +222,6 @@ final class Bots
206
222
|Sogou[\s-](?:head|inst|Orion|Pic|Test|web)[\s-]spider|New-Sogou-Spider
207
223
|Sosospider|Sosoimagespider
208
224
|Sprinklr
209
- |sqlmap/
210
225
|SSL\sLabs
211
226
|StatusCake
212
227
|Superfeedr\sbot
@@ -252,6 +267,8 @@ final class Bots
252
267
|websitepulse[+\s]checker
253
268
|WordPress.+isitwp\.com
254
269
|Automattic\sAnalytics\sCrawler/[\d.]+
270
+ |WordPress\.com\smShots
271
+ |wp\.com\sfeedbot
255
272
|WordPress
256
273
|Wotbox
257
274
|XenForo
@@ -332,6 +349,7 @@ final class Bots
332
349
|Startpagina-Linkchecker
333
350
|MoodleBot-Linkchecker
334
351
|GTmetrix
352
+ |CyberFind\s?Crawler
335
353
|Nutch
336
354
|Seobility
337
355
|Vercelbot
@@ -346,7 +364,7 @@ final class Bots
346
364
|Barkrowler
347
365
|BDCbot
348
366
|adbeat
349
- |BW /[\d.]+
367
+ |(?:BuiltWith|BW) /[\d.]+
350
368
|https://whatis\.contentkingapp\.com
351
369
|MicroAdBot
352
370
|PingAdmin\.Ru
@@ -403,6 +421,7 @@ final class Bots
403
421
|seolyt/[\d.]+
404
422
|YaK/[\d.]+
405
423
|KomodiaBot/[\d.]+
424
+ |KStandBot/[\d.]+
406
425
|Neevabot/[\d.]+
407
426
|LinkPreview/[\d.]+
408
427
|JungleKeyThumbnail/[\d.]+
@@ -426,7 +445,7 @@ final class Bots
426
445
|OnalyticaBot
427
446
|deepnoc
428
447
|Newslitbot/[\d.]+
429
- |um-LN /[\d.]+
448
+ |um-(?:ANS|CC|FC|IC|LN) /[\d.]+
430
449
|Abonti/[\d.]+
431
450
|collection@infegy\.com
432
451
|HTTP\sBanner\sDetection\s\(https://security\.ipip\.net\)
@@ -452,6 +471,7 @@ final class Bots
452
471
|CriteoBot/
453
472
|PayPal\sIPN
454
473
|MaCoCu
474
+ |CLASSLA
455
475
|dnt-policy@eff\.org
456
476
|InfoTigerBot
457
477
|(?:Birdcrawlerbot|CrawlaDeBot)
@@ -477,7 +497,7 @@ final class Bots
477
497
|Sublinq
478
498
|Gregarius/[\d.]+
479
499
|COMODO\sDCV
480
- |Sectigo\sDCV
500
+ |Sectigo\sDCV|acme\.sectigo\.com
481
501
|KlarnaBot-(?:DownloadProductImage|EnrichProducts|PriceWatcher)/[\d.]+
482
502
|Taboolabot/[\d.]+
483
503
|Asana/[\d.]+
@@ -502,14 +522,14 @@ final class Bots
502
522
|AdsTxtCrawler/[\d.]+
503
523
|Morningscore
504
524
|Uptime-Kuma/[\d.]+
525
+ |OAI-SearchBot
526
+ |GPTBot/[\d.]+
505
527
|ChatGPT-User
506
528
|BrightEdge\sCrawler/[\d.]+
507
529
|sfFeedReader/[\d.]+
508
530
|cyberscan\.io
509
- |deepcrawl\.com
510
531
|researchscan\.comsys\.rwth-aachen\.de
511
532
|newspaper/[\d.]+
512
- |GPTBot/[\d.]+
513
533
|Ant(?:\.com\sbeta|Bot)(?:/([\d+.]+))?
514
534
|WebwikiBot/[\d.]+
515
535
|phpMyAdmin
@@ -518,7 +538,7 @@ final class Bots
518
538
|ArchiveTeam\sArchiveBot
519
539
|MADBbot/[\d.]+
520
540
|MeltwaterNews
521
- |(?:Owler@ows\.eu|OWLer)/[\d.]+
541
+ |owler
522
542
|bbc\.co\.uk/display/men/Page\+Monitor
523
543
|BBC-Forge-URL-Monitor-Twisted
524
544
|ClaudeBot
@@ -528,6 +548,7 @@ final class Bots
528
548
|DaspeedBot/([\d+.]+)
529
549
|StractBot(?:/([\d+.]+))?
530
550
|GeedoBot(?:/([\d+.]+))?
551
+ |GeedoProductSearch
531
552
|BackupLand(?:/([\d+.]+))?
532
553
|Konturbot(?:/([\d+.]+))?
533
554
|keys-so-bot
@@ -585,7 +606,6 @@ final class Bots
585
606
|Monsidobot/[\d.]+
586
607
|AccompanyBot
587
608
|Ghost\sInspector
588
- |Cypress/[\d.]+
589
609
|Google-Apps-Script
590
610
|SiteOne-Crawler/[\d.]+
591
611
|Detectify
@@ -619,7 +639,63 @@ final class Bots
619
639
|abuse\.xmco\.fr
620
640
|CheckHost
621
641
|LAC_IAHarvester/[\d.]+
622
- |nuhk|grub-client|Download\sDemon|SearchExpress|Microsoft\sURL\sControl|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?:\sBuild|Plus))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google\sSketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib\sAutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re\sStudio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle\sClient|Hello,?\sworld|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M\sCODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux\sGnu\s\(cow\)|Test\sCertificate\sInfo|iplabel|Magellan|TheSafex?Internetx?Search|Searcherweb|kirkland-signature|LinkChain|survey-security-dot-txt|^xenu|^ZmEu|^(?:chrome|firefox|Zeus)$
623
- |[a-z0-9_-]*(?:(?<!cu|power[\s_]|m[\s_])bot(?![\s_]TAB|[\s_]?5[0-9]|[\s_]Senior|[\s_]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|fetcher|indexer|inspector|monitor|project(?!or)|(?<!Google\sWap\s)proxy|research|resolver|robots|scanner|scraper|script|searcher|(?<!-)security|spider|study|transcoder|uptime|user[\s_]?agent|validator)(?:[^a-z]|$)
642
+ |InsytfulBot/[\d.]+
643
+ |statista\.com
644
+ |SubstackContentFetch/[\d.]+
645
+ |^ds9
646
+ |LiveJournal\.com
647
+ |bitdiscovery
648
+ |Castopod/[\d.]+
649
+ |Elastic/Synthetics
650
+ |WDG_Validator/[\d.]+
651
+
652
+ |CrawlyProjectCrawler/[\d.]+
653
+ |BDFetch
654
+ |PunkMap
655
+ |GenomeCrawlerd/[\d.]+
656
+ |Gaisbot/[\d.]+
657
+ |FAST-WebCrawler/[\d.]+
658
+ |ducks\.party
659
+ |DepSpid/[\d.]+
660
+ |Website-info\.net
661
+ |RedekenBot
662
+ |semaltbot
663
+ |MakeMerryBot
664
+ |Timpibot
665
+ |Validbot
666
+ |NPBot
667
+ |domaincodex\.com
668
+ |Swisscows\sFavicons
669
+ |leak\.info
670
+ |workona
671
+ |Bloglines
672
+ |heritrix
673
+ |search\.marginalia\.nu
674
+ |vu-server-health-scanner/[\d.]+
675
+ |Functionize
676
+ |Prerender
677
+ |bl\.uk_ldfc_bot
678
+ |Miniature\.io
679
+ |Convertify
680
+ |ZoteroTranslationServer
681
+ |MuckRack
682
+ |Golfe
683
+ |SpiderLing
684
+ |Bravebot
685
+ |1001FirmsBot
686
+ |SteamChatURLLookup
687
+ |ohdear\.app
688
+ |Inspici
689
+ |peer39_crawler
690
+ |Pandalytics
691
+ |CloudServerMarketSpider
692
+ |Pigafetta
693
+ |Cotoyogi
694
+ |SuggestBot
695
+ |cms-experiment
696
+ |SiteCheckerBotCrawler
697
+ |SBIder
698
+ |nuhk|grub-client|Download\sDemon|SearchExpress|Microsoft\sURL\sControl|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?:\sBuild|Plus|\sCM62|\sHD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google\sSketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib\sAutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re\sStudio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle\sClient|Hello,?\sworld|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M\sCODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux\sGnu\s\(cow\)|Test\sCertificate\sInfo|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root\sSlut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report\sRunner|url|Zeus|ZmEu)$
699
+ |[a-z0-9_-]*(?:(?<!cu|power[\s_]|m[\s_])bot(?![\s_]TAB|[\s_]?5[0-9]|[\s_]Senior|[\s_]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|(?<!node-|uclient-|Mikrotik/\d\.[x\d]\s|electron-)fetch(?:er)?|indexer|inspector|monitor|(?<!Microsoft\s|banshee-)project(?!or)|(?<!Google\sWap\s|Blue\s|SpeedMode;\s)proxy|(?<!P)research|resolver|robots|(?<!Cam)scanner|scraper|script|searcher|(?<!-)security|spider(?!\s8)|study|transcoder|uptime|user[\s_]?agent|validator)(?:[^a-z]|$)
624
700
#x " ;
625
701
}
0 commit comments