V4.2024.08.4715 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
-404checker
-Apexoo
-BacklinkCrawler
-Badass
-BetaBot
-Buddy
-Bytespider
+80legs
+ADmantX
+Aboundexbot
+Aipbot
+Attach
+BDFetch
+BLEXBot
+BackStreet
+BacklinksExtendedBot
+BlackWidow
+BlowFish
+Brandprotect
 CSHttp
-Claritybot
-Cloud mapping
-Collector
+CheeseBot
+Cliqzbot
+Cocolyzebot
 Copier
-CyotekWebCopy
+CopyRightCheck
+Curious
 Discobot
-Dispatch
+DomainCrawler
+DomainStatsBot
+EMail Siphon
 Ebingbong
-Extreme Picture Finder
-FemtosearchBot
-Firefox/7.0
-FlashGet
-Freeuploader
-Gigabot
-Humanlinks
-IDBot
-IndeedBot
-Indy Library
-JOC Web Spider
+EroCrawler
+Exabot
+FDM
+GPTBot
+GalaxyBot
+HTTrack
+HonoluluBot
+HybridBot
+Image Fetch
+ImagesiftBot
+InfraSec Scanner
+InterGET
+Iskanie
+Kenjin Spider
 Keyword Density
-Larbin
-Likse
-LinkpadBot
-Ltx71
-MFC_Tear_Sample
-Majestic SEO
-MarkWatch
-Mass Downloader
-MeanPath Bot
-MicroMessenger
-Mozlila
-Needle
-NetLyzer
-Nmap
-Piepmatz
-PleaseCrawl
-ProPowerBot
-Pu_iN
-RSSingBot
-RankActive
-RankingBot2
+Kinza
+LibWeb
+Libwhisker
+LinqiaRSSBot
+MIDown tool
+MSIECrawler
+MarkMonitor
+NearSite
+Nettrack
+Openfind
+Page Analyzer
+PageScorer
+Picscout
+Psbot
+QueryN Metasearch
+Rankivabot
+RankurBot
+Reaper
 RepoMonkey
-Rogerbot
+SEOkicks-Robot
 SEOlyticsCrawler
-SEOstats
-ScoutJet
-ScreenerBot
+ScanAlert
 ScrepyBot
-SearchmetricsBot
 SemrushBot
-SeobilityBot
+SiteCheckerBotCrawler
+SmartDownload
+Sociscraper
 Spanner
-Stripper
-Titan
+Steeler
+SuperBot
+Teleport
+TightTwatBot
+Tracemyfile
+URLy.Warning
 Virusdie
-Voil
-WWW::Mechanize
-WWWOFFLE
-Web Sauger
-WebCopier
-WebFetch
-WebReaper
-WebSucker
-WebWhacker
-Whack
-Whacker
-WinHTTrack
-Wotbox
-YoudaoBot
-Zade
-Zermelo
+WeSEE
+Web Pix
+WebStripper
+WebsiteQuester
+WiseGuys Robot
+Woobot
+Xaldon_WebSpider
+Xenu
 Zeus
-ZmEu
-ZoominfoBot
-ZumBot
-ZyBorg
+adscanner
 anthropic-ai
+arquivo-web-crawler
 awario.com
-coccocbot
-cohere-ai
-dataforseo.com
+com.plumanalytics
+demandbase-bot
+domainsproject.org
+facebookscraper
 internetVista monitor
+ips-agent
 magpie-crawler
-meanpathbot
-satoristudio.net
-scan.lol
-seocompany.store
-seoscanners
-seostar
-webpros.com
-webprosbot
+page scorer
+siteripz
+tAkeOut
+x09Mozilla
+zauba.io
+zgrab