V4.2020.12.2196 [ci skip]

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
+360Spider
+404enemy
 Aboundex
-Aboundexbot
 ADmantX
 AfD-Verbotsverfahren
-Ankit
+AhrefsBot
+Alexibot
+AllSubmitter
 Anthill
 Apexoo
-ASPSeek
-Asterias
-autoemailspider
+arquivo.pt
+arquivo-web-crawler
 AwarioSmartBot
 BackDoorBot
+Backlink-Ceck
 backlink-check
 BackStreet
-Bandit
-Barkrowler
-BatchFTP
-Battleztar Bazinga
-BBBike
-BDCbot
+BackWeb
+Badass
 BDFetch
-Bigfoot
+Bitacle
 Blackboard
 Black Hole
-BlackWidow
-Boardreader
-Bolt
-Brandprotect
+BLEXBot
+Blow
+BlowFish
+Brandwatch
 Buddy
 BuiltWith
 Bullseye
-BuzzSumo
-CazoodleBot
+CCBot
 Cegbfeieh
-CheeseBot
-ChinaClaw
-coccocbot-web
-Collector
+CheTeam
+CODE87
+cognitiveseo
 com.plumanalytics
-Copier
 CopyRightCheck
-Cosmos
+Craftbot
 crawler4j
 crawler.feedback
 crawl.sogou.com
-CrazyWebCrawler
-CSHttp
+Crescent
 Curious
 CyotekWebCopy
+DatabaseDriverMysqli
 DataCha0s
-DBLBot
-Demon
+demandbase-bot
+Deusu
 Digincore
-Dirbuster
-Discobot
+DigitalPebble
+Discoverybot
+Dispatch
+DnyzBot
 DomainAppender
+DomainCrawler
 Domains Project
 domainsproject.org
 DomainStatsBot
-Dotbot
-Download Wonder
+Dragonfly
+Ebingbong
 eCatch
-ECCP/1.0
+Ecxi
 EirGrabber
+EMail Wolf
 EroCrawler
 evc-batch
-Evil
-Exabot
-Express WebPictures
 ExtLinksBot
-facebookscraper
-FDM
+FemtosearchBot
 FHscan
 Fimap
-FlashGet
-Flunky
-Foobot
-Freeuploader
 FrontPage
-Fyrebot
 Genieo
-GermCrawler
-GetRight
-GetWeb
 Gigabot
-Go-Ahead-Got-It
-GoZilla
-Go!Zilla
+G-i-g-a-b-o-t
+Grabber
+Grafula
 GrapeFX
-GrapeshotCrawler
-Haansoft
-heritrix
+GT::WWW
 Heritrix
-Hloader
-HMView
 HTMLparser
 HTTrack
-Iblog
-IDBot
-Id-search
-IlseBot
-InfoNaviRobot
+Humanlinks
+IDBTE4M
+Image Sucker
+Indy Library
+InfoTekies
 instabid
 Intelliseek
+InterGET
 internetVista monitor
 Iria
-IRLbot
-IstellaBot
 JamesBOT
-JennyBot
+Jbrofuzz
 JetCar
 Jetty
 JikeSpider
+JOC Web Spider
 Joomla
+JustView
 Jyxobot
-LeechGet
+Kenjin Spider
+Kinza
+Lanshanbot
+LexiBot
 Lftp
-LibWeb
 Libwhisker
-Lightspeedsystems
-Linkdexbot
-Lipperhey
+LieBaoFast
+Likse
+LinkpadBot
+LinkScan
+LinqiaScrapeBot
 Lmspider
-lwp-request
-Magnet
+LWP::Simple
+lwp-trivial
+magpie-crawler
 Mail.RU_Bot
-MarkMonitor
-MarkWatch
-Mb2345Browser
-Meanpathbot
-Mediatoolkitbot
-mediawords
-MegaIndex.ru
-Microsoft Data Access
-MIIxpc
-MJ12bot
-Morfeus Fucking Scanner
-Mozlila
-Mr.4x3
+Majestic SEO
+Majestic-SEO
+Masscan
+Mass Downloader
+Mata Hari
+MauiBot
+MFC_Tear_Sample
+MIDown tool
+MQQBrowser
+MSIECrawler
 Name Intelligence
 NearSite
+Needle
 Nessus
-Netvibes
-NextGenSearchBot
+NetMechanic
 Nibbler
+Niki-bot
 Nikto
-Nmap
-oBot
-Openfind
-OpenLinkProfiler
-Openvas
+NPbot
+Nutch
+Octopus
+OnCrawl
 OpenVAS
-OrangeSpider
-OutclicksBot
+OPPO A33
+OrangeBot
+OutfoxBot
 PageAnalyzer
-PageGrabber
 page scorer
-PageScorer
 Papa Foto
-pcBrowser
+PECL::HTTP
 Picsearch
-Pixray
-PleaseCrawl
+PictureFinder
 plumanalytics
-Pockey
-Probethenet
-Psbot
-PxBroker
+POE-Component-Client-HTTP
+polaris version
+ProPowerBot
+ProWebWalker
+PyCurl
 QueryN Metasearch
 RankActive
-RankActiveLinkBot
 RankFlex
-RankingBot
-RankingBot2
 Rankivabot
 RankurBot
+Recorder
 RedesScrapy
+ReGet
 RepoMonkey
 RocketCrawler
+Rogerbot
+RSSingBot
+s1z.ru
+SalesIntelligent
+satoristudio.net
 SBIder
+Scanbot
+scan.lol
 ScoutJet
-Searchestate
-SearchmetricsBot
+ScreenerBot
 Semrush
-SemrushBot
+SEOkicks
+SEOkicks-Robot
 SEOlyticsCrawler
-Seomoz
-Shodan
-Siphon
-SISTRIX
-SiteCheckerBotCrawler
+seoscanners
+SeoSiteCheckup
+SEOstats
+serpstatbot
 sitechecker.pro
+SiteExplorer
+Siteimprove
+SiteSnagger
 SiteSucker
 SlySearch
-SmartDownload
-Sottopop
-SpankBot
-Spanner
-sp_auditbot
+Snoopy
+SocialRankIOBot
+Sociscraper
+sogouspider
+SpaceBison
 Spinn3r
-Sqlworm
-Sqworm
+spyfu
+Sqlmap
 Sucker
 Sucuri
+SuperBot
+SuperHTTP
 Surfbot
-SurveyBot
 Suzuran
-Swiftbot
 T8Abot
-Telesphorep
-The Intraformant
-TheNomad
+TeleportPro
 TightTwatBot
+Titan
+Toweyabot
 Tracemyfile
+Trendiction
 Trendictionbot
-trendiction.de
-TwengaBot
+Turnitin
+TurnitinBot
 Twice
-URLy.Warning
-VeriCiteCrawler
+Typhoeus
+Upflow
+VidibleScraper
+Virusdie
 Voil
-Voltron
-Wallpapers/3.0
 WallpapersHD
 WASALive-Bot
 WBSearchBot
-Web Auto
-Web Collage
-WEBDAV
+Webalta
+WebAuto
+WebBandit
+WebCopier
+Web Enhancer
+Web Fuck
 WebGo IS
 WebLeacher
-WebmasterWorldForumBot
 webmeup-crawler
 WebPix
 Web Pix
-Webster
+WebSauger
+Webshag
+WebsiteQuester
+WebStripper
 WebSucker
+Web Sucker
 Whack
-Whacker
-WISENutbot
+WPScan
 WWW-Collector-E
 WWW-Mechanize
-WWWOFFLE
-x22Mozilla
-Xaldon WebSpider
+WWW::Mechanize
+Xaldon_WebSpider
+Xenu
 xpymep1.exe
+YoudaoBot
 Zade
-Zermelo
-ZmEu
+zauba.io
+ZumBot