V4.2021.10.2644 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
 404checker
-ADmantX
-AIBOT
+404enemy
 ASPSeek
-Aboundexbot
+Aboundex
 Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-Alexibot
-AlphaBot
-Anthill
+Aipbot
+Ankit
+Apexoo
+Asterias
 Attach
-BDFetch
+AwarioRssBot
+AwarioSmartBot
 BackWeb
-Bandit
-Black Hole
-Blackboard
-Bolt
-Brandwatch
+Badass
+Barkrowler
+BatchFTP
+Bitacle
+Boardreader
+Brandprotect
 Buck
-Buddy
-BuiltWith
+BuiltBotTough
 Bullseye
 BunnySlippers
-Calculon
-CazoodleBot
+BuzzSumo
+CATExplorador
+CCBot
+CODE87
 Cegbfeieh
-CensysInspect
-CheTeam
-CherryPicker
+CheeseBot
 Chlooe
-Cliqzbot
-Cocolyzebot
-Collector
-Copier
+Claritybot
 CopyRightCheck
 Copyscape
 Cosmos
 Craftbot
-CrazyWebCrawler
+Crawling at Home Project
 Crescent
-CrunchBot
-DIIbot
-DSearch
-DatabaseDriverMysqli
-Demon
-Dirbuster
-Discobot
+Curious
+DBLBot
+DTS Agent
+DataCha0s
+Devil
 Discoverybot
-Dispatch
-DnyzBot
+DomCopBot
 DomainAppender
+DomainCrawler
+DomainStatsBot
+Domains Project
+Dotbot
 Download Wonder
-Dragonfly
-Drip
-EMail Wolf
-EirGrabber
+ECCP/1.0
+Ecxi
+Evil
 Exabot
-Express WebPictures
 ExtractorPro
 FDM
-Firefox/7.0
+FHscan
+FemtosearchBot
 FlashGet
 Flunky
-Foobot
 Freeuploader
-FrontPage
-Fyrebot
-GT::WWW
-Getintent
-Gotit
+G-i-g-a-b-o-t
+GalaxyBot
+Genieo
+GetRight
+Go!Zilla
+Go-Ahead-Got-It
+GoZilla
+GrabNet
 Grafula
-GrapeFX
-HTTP::Lite
+GridBot
+HTMLparser
+Haansoft
+HaosouSpider
 Harvest
-Hloader
-IDBot
-IRLbot
-Iblog
-IlseBot
+Havij
+Image Fetch
 Indy Library
+InfoNaviRobot
 InterGET
-Iria
+Internet Ninja
 JamesBOT
-JetCar
-Jetty
+JikeSpider
+Joomla
+Jorgee
 JustView
+Jyxobot
 Kenjin Spider
-Keyword Density
 Kinza
-Lanshanbot
+Kozmosbot
+LWP::Simple
 Leap
+LeechFTP
 LeechGet
+LibWeb
 LieBaoFast
 Lightspeedsystems
-LinkpadBot
+Likse
+Linkbot
+Linkdexbot
 LinqiaScrapeBot
-Lipperhey Spider
-Lmspider
-MFC_Tear_Sample
-MIIxpc
-MSIECrawler
-Magnet
-Majestic SEO
-Majestic-SEO
-Mass Downloader
-Masscan
-MauiBot
+Lipperhey
+Litemage_walker
+MJ12bot
+MQQBrowser
+Mail.RU_Bot
+Majestic12
+Mb2345Browser
 MeanPath Bot
-Meanpathbot
+MegaIndex.ru
 Metauri
-MicroMessenger
 Microsoft URL Control
 Mister PiX
-Moblie Safari
+Mojolicious
+Mr.4x3
 Msrabot
-Musobot
-NICErsPRO
-Navroad
-NearSite
+NPbot
+Name Intelligence
+Net Vampire
+NetLyzer
 NetMechanic
-Netcraft
-Netvibes
+NetZIP
 Nibbler
-NimbleCrawler
-Nimbostratus
-Nmap
+Nikto
+Ninja
 Octopus
 Offline Explorer
+OpenVAS
 Openvas
-OrangeSpider
-PECL::HTTP
-PHPCrawl
+OutclicksBot
 POE-Component-Client-HTTP
-Page Analyzer
-PageScorer
+PageAnalyzer
 Pandalytics
-Panscient
 Petalbot
-Picsearch
 PictureFinder
-Pixray
 PleaseCrawl
 Pockey
-Pump
-PxBroker
-RankFlex
-Re-re
-Recorder
-RocketCrawler
-SEOkicks-Robot
+Pu_iN
+PyCurl
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
+RankActive
+RankActiveLinkBot
+Reaper
+RepoMonkey
+SEOkicks
 SEOlyticsCrawler
-SalesIntelligent
+SEOprofiler
+SMTBot
+Scrapy
 ScrepyBot
-Searchestate
-SentiBot
+Semrush
+SemrushBot
 SeoSiteCheckup
 Seomoz
-Shodan
-Site Sucker
+Siphon
 SiteCheckerBotCrawler
+SiteExplorer
 SiteLockSpider
-Siteimprove
+SiteSucker
 Sitevigil
 SlySearch
 SmartDownload
-Sosospider
+SocialRankIOBot
+Sociscraper
+Sottopop
 SpaceBison
 Spammen
-SpankBot
 Spbot
-SputnikBot
-Sqlworm
+Steeler
+Stripper
+Sucuri
 SuperBot
-SuperHTTP
-Surfbot
-Suzuran
-T0PHackTeam
-TeleportPro
-Telesphorep
+T8Abot
+Teleport
 The Intraformant
 TheNomad
-Titan
-Toweyabot
-Tracemyfile
-Trendictionbot
+Thumbor
+Trendiction
 True_Robot
-TurnitinBot
-URLy Warning
-URLy.Warning
-UnisterBot
+Turingos
 Upflow
+VB Project
+Vagabondo
 VelenPublicWebCrawler
-VeriCiteCrawler
-Virusdie
-VoidEYE
+WBSearchBot
 WISENutbot
 WPScan
-WWW-Mechanize
+WWW-Collector-E
 WWWOFFLE
-Wallpapers
-Wallpapers/3.0
 WallpapersHD
 Web Collage
 Web Pix
-WebCollage
-WebCopier
-WebLeacher
+Web Sucker
+WebAuto
+WebFetch
+WebImageCollector
 WebSucker
-WebZIP
-WebmasterWorldForumBot
+WebWhacker
+Webalta
 Webshag
-Website Quester
-WebsiteExtractor
+WebsiteQuester
 Webster
 Whack
-Widow
-Woobot
+Whacker
+Who.is Bot
+WinHTTrack
+Wonderbot
 Wotbox
-Wprecon
-Xenu
-YoudaoBot
+Xaldon WebSpider
 Zade
 Zauba
-Zermelo
-Zeus
 Zitebot
+ZoomBot
 ZoominfoBot
-ZyBorg
-arquivo-web-crawler
-cognitiveseo
-dataforseo.com
-domainsproject.org
-eCatch
-gopher
+ZumBot
+arquivo.pt
+cah.io.community
+check1.exe
+clark-crawler
+coccocbot-web
+com.plumanalytics
+crawl.sogou.com
+crawler.feedback
+evc-batch
+facebookscraper
 instabid
-ips-agent
-lwp-trivial
+internetVista monitor
+meanpathbot
+mediawords
 muhstik-scan
-polaris version
+oBot
 ripz
 s1z.ru
+satoristudio.net
+scalaj-http
 scan.lol
+seobility
 sitechecker.pro
-spyfu
+siteripz
+sp_auditbot
+sysscan
 trendiction.com
-trendiction.de
+webpros.com
 webprosbot
+x09Mozilla
 x22Mozilla
+zgrab