V4.2020.04.2053 [ci skip]

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
+360Spider
+404checker
 Aboundexbot
-ADmantX
+Acunetix
+AhrefsBot
+AIBOT
 AiHitBot
 Aipbot
-Alexibot
-AllSubmitter
+Alligator
 AlphaBot
-arquivo.pt
+Anarchie
 autoemailspider
-AwarioRssBot
-AwarioSmartBot
 BackDoorBot
 backlink-check
 BacklinkCrawler
-BackStreet
-Badass
-BatchFTP
+BackWeb
+Barkrowler
+BBBike
+Bigfoot
+BlackWidow
 Boardreader
-Brandwatch
 Buddy
 BuiltBotTough
-BuiltWith
-BunnySlippers
-CATExplorador
-CherryPicker
-CheTeam
-Claritybot
+Calculon
+CazoodleBot
+CCBot
+Cegbfeieh
 Cliqzbot
-coccocbot-web
+Cloud mapping
 Cogentbot
 Collector
-CopyRightCheck
+Copier
 Copyscape
+Craftbot
 crawler4j
 crawler.feedback
-CrazyWebCrawler
+crawl.sogou.com
 CSHttp
-Curious
-Custo
-DataCha0s
+DatabaseDriverMysqli
 Demon
-Devil
-Digincore
-DIIbot
+Deusu
+Dirbuster
 Disco
 Discoverybot
-Dispatch
 DittoSpyder
+DnyzBot
+DomainAppender
+DomainSigmaCrawler
+DomainStatsBot
 Download Wonder
-Dragonfly
+Drip
+DSearch
 DTS Agent
 EasyDL
-Ebingbong
+eCatch
 EMail Wolf
-evc-batch
-Exabot
-EyeNetIE
+Evil
+ExtLinksBot
+Extractor
+ExtractorPro
 facebookscraper
 FDM
+FemtosearchBot
+Fimap
 Firefox/7.0
-FlashGet
 Flunky
 Foobot
 FyberSpider
-Fyrebot
-GalaxyBot
-Genieo
-GermCrawler
+Getintent
 GetRight
-GetWeb
+Gigabot
 G-i-g-a-b-o-t
-Go-Ahead-Got-It
 Gotit
-GoZilla
+Go!Zilla
+Grabber
 Grafula
-GT::WWW
+GrapeFX
+GrapeshotCrawler
+HaosouSpider
 Harvest
-HEADMasterSEO
-Hloader
+HMView
+HTMLparser
 HTTP::Lite
-HTTrack
 Humanlinks
-HybridBot
 Iblog
 IDBot
-Id-search
-IlseBot
 Image Sucker
+IndeedBot
 Indy Library
+InfoNaviRobot
 instabid
 Intelliseek
-InterGET
-InternetSeer
-IRLbot
-Iskanie
-IstellaBot
+Iria
 JamesBOT
-JennyBot
+Jbrofuzz
+JetCar
 Jetty
-JOC Web Spider
-JustView
-Kinza
-LeechGet
+Joomla
+Kozmosbot
 LexiBot
-Lightspeedsystems
-LinkScan
+LibWeb
+Linkdexbot
+LinkextractorPro
+LinkpadBot
 LinksManager
+LinkWalker
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
 Lipperhey
-Litemage_walker
+Lipperhey Spider
+Lmspider
 LNSpiderguy
 Ltx71
 lwp-request
-lwp-trivial
+LWP::Simple
 Mag-Net
 magpie-crawler
+Mail.RU_Bot
 Majestic12
-MarkWatch
-Masscan
-Mass Downloader
-MauiBot
+Majestic-SEO
 Mb2345Browser
-meanpathbot
-MegaIndex.ru
-Microsoft URL Control
-MQQBrowser
-MSFrontPage
+Meanpathbot
+MFC_Tear_Sample
+MicroMessenger
+MIDown tool
+Mister PiX
+Mr.4x3
 Msrabot
-muhstik-scan
+Musobot
 Name Intelligence
-Nessus
-NetMechanic
+Needle
+Netcraft
+NetLyzer
 NetSpider
+Nettrack
 Net Vampire
 Netvibes
-NetZIP
-NextGenSearchBot
-Nimbostratus
+NICErsPRO
+Nikto
+NimbleCrawler
 Ninja
+Nmap
+NPbot
 oBot
-OnCrawl
+Octopus
+Openfind
 OpenLinkProfiler
-Openvas
-OPPO A33
+OrangeBot
 OrangeSpider
-OutclicksBot
-OutfoxBot
-PageAnalyzer
-Page Analyzer
-PageGrabber
+page scorer
 PageScorer
-Pandalytics
-Panscient
-pcBrowser
+Papa Foto
 PECL::HTTP
 PHPCrawl
-Pi-Monster
-Pixray
+PictureFinder
 PleaseCrawl
-Pockey
-polaris version
+POE-Component-Client-HTTP
+Probethenet
 ProPowerBot
-ProWebWalker
 Psbot
-Pump
 PxBroker
-RankActive
-RankActiveLinkBot
-RankingBot
+PyCurl
+QueryN Metasearch
+RankFlex
 RankingBot2
-RedesScrapy
+RankurBot
+RealDownload
+Reaper
 ReGet
 RocketCrawler
-RSSingBot
-s1z.ru
 SalesIntelligent
-scan.lol
-ScoutJet
-Screaming
-SearchmetricsBot
 Semrush
-SemrushBot
 SEOkicks
 SEOkicks-Robot
+SEOlyticsCrawler
 Seomoz
-Siphon
+SeoSiteCheckup
+sexsearcher
+Shodan
+SiteExplorer
+Siteimprove
+SiteLockSpider
 SiteSnagger
 SiteSucker
-Snoopy
-SocialRankIOBot
-sogouspider
+Site Sucker
+SlySearch
+SMTBot
+Snake
 Sogou web spider
 Sosospider
 Sottopop
-spyfu
-Sqlworm
-Steeler
-Stripper
-Sucuri
-SurveyBot
-Suzuran
-Swiftbot
-T8Abot
+SpankBot
+Spbot
+Spinn3r
+SputnikBot
+Sqlmap
+Sqworm
+SuperBot
+SuperHTTP
+Surfbot
 tAkeOut
-The Intraformant
+TeleportPro
+Telesoft
+Telesphorep
+Thumbor
 TightTwatBot
-Tracemyfile
-Trendictionbot
-True_Robot
-Turingos
-Twice
-URLy Warning
-URLy.Warning
-Vagabondo
-VCI
+Titan
+Toweyabot
+Trendiction
+trendiction.com
+trendiction.de
+VB Project
 VeriCiteCrawler
-VidibleScraper
+Virusdie
+VoidEYE
 Voltron
-Wallpapers/3.0
-WASALive-Bot
+WallpapersHD
+WBSearchBot
+WebAuto
+Web Auto
 WebBandit
-Web Collage
+WebCopier
+WebEnhancer
 Web Enhancer
-Web Fetch
+WebFetch
+WebGo IS
+WebLeacher
 WebmasterWorldForumBot
 webmeup-crawler
 WebPix
-Web Sauger
+Web Pix
+WebReaper
+WebsiteQuester
 Webster
+WebSucker
 Web Sucker
-WebWhacker
-Whack
 Whacker
-WinHTTrack
-WiseGuys Robot
-WISENutbot
-Wonderbot
-Woobot
+Whatweb
+Who.is Bot
 WPScan
-WWW-Mechanize
-WWW::Mechanize
+WWWOFFLE
 Xaldon WebSpider
-Xaldon_WebSpider
+Xenu
 YoudaoBot
-Zauba
+zauba.io
 Zermelo
 Zeus
-ZumBot
+zgrab
+ZmEu
 ZyBorg