V4.2023.03.3622 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
-AIBOT
-AhrefsBot
+360Spider
+ALittle Client
+Alexibot
+Anarchy
+Anarchy99
 Aspiegel
-Atomseobot
-AwarioRssBot
+AwarioSmartBot
+BackDoorBot
+BackWeb
+Backlink-Ceck
 BacklinkCrawler
 Badass
-Black Hole
-Bolt
-Buddy
-Citoid
-Collector
-CrazyWebCrawler
-Discobot
+Bitacle
+BotALot
+Brandwatch
+BuiltWith
+Bullseye
+CATExplorador
+CensysInspect
+Crescent
+Demon
+Dispatch
 DnBCrawler-Analytics
-EMail Wolf
-Ebingbong
-Ecxi
-EroCrawler
-Fyrebot
-GetWeb
-Gigabot
-Gotit
-Grabber
-HaosouSpider
-HonoluluBot
-Image Sucker
-InterGET
-Iria
-JetCar
-Jetty
-JustView
-Kozmosbot
-LeechGet
-LieBaoFast
-LinkScan
-Majestic12
-Masscan
-MegaIndex.ru
-Microsoft Data Access
-NICErsPRO
-NPbot
-Nameprotect
-Nutch
-OpenVAS
-OutfoxBot
-Pandalytics
-PeoplePal
+DomainStatsBot
+Ezooms
+Fimap
+FlashGet
+GrabNet
+HTTrack
+IDBTE4M
+Image Fetch
+InfoTekies
+JOC Web Spider
+JamesBOT
+Larbin
+Lightspeedsystems
+LinqiaRSSBot
+LinqiaScrapeBot
+MicroMessenger
+Minefield
+Moblie Safari
+Mozlila
+Msrabot
+NearSite
+Netvibes
+Nikto
+Offline Explorer
+Openvas
+Papa Foto
+Pavuk
+Petalbot
 Picsearch
-Piepmatz
-ProWebWalker
-Pump
-RocketCrawler
+PleaseCrawl
+Pockey
+RankFlex
+RankurBot
 SBIder
-SEOstats
-Scrapy
-SearchmetricsBot
-Seekport
-Seomoz
-SiteExplorer
-SiteLockSpider
-Sitevigil
-Snapbot
-Sociscraper
-SputnikBot
+SEOlyticsCrawler
+SISTRIX
+SeoSiteCheckup
+SeobilityBot
+Site Sucker
+SiteCheckerBotCrawler
+Sogou web spider
+Sosospider
+Spanner
 Sqworm
-Surfbot
+Stripper
+Sucker
+SuperHTTP
 Teleport
-TeleportPro
-TightTwatBot
-Turingos
-TwengaBot
-Voil
-WISENutbot
-WWW-Mechanize
-WWW::Mechanize
-Web Auto
-Web Enhancer
+The Intraformant
+Thumbor
+Toata
+Tracemyfile
+Turnitin
+V-BOT
+VeriCiteCrawler
+VidibleScraper
+WBSearchBot
 Web Sauger
+WebAuto
+WebImageCollector
+Webshag
 WebsiteExtractor
-WiseGuys Robot
-Wotbox
+WebsiteQuester
+Widow
 Xenu
-YoudaoBot
 Zermelo
-Zitebot
-ZyBorg
-autoemailspider
-com.plumanalytics
-eCatch
+ZoomBot
+crawl.sogou.com
+crawler4j
+demandbase-bot
 gopher
-magpie-crawler
-mediawords
+lwp-trivial
+polaris version
 ripz
-seobility
-spyfu
-tAkeOut
-webpros.com
-x09Mozilla
+satoristudio.net
+seoscanners
+zgrab