V4.2020.03.2031 [ci skip]

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
+404enemy
 80legs
+Aboundex
+Acunetix
+ADmantX
+AIBOT
 AiHitBot
-Alligator
+Alexibot
 Anarchie
-Ankit
-Apexoo
-archive.org_bot
-Aspeigel
+arquivo-web-crawler
 ASPSeek
-Attach
-AwarioRssBot
-backlink-check
-BacklinkCrawler
-BackStreet
-Bandit
-Barkrowler
-Battleztar Bazinga
+AwarioSmartBot
+Backlink-Ceck
+BackWeb
+BatchFTP
+BBBike
 BDCbot
-BDFetch
+BetaBot
 Bigfoot
-BlackWidow
-Bolt
+Bitacle
+Black Hole
+BlowFish
+Boardreader
 BuzzSumo
-CATExplorador
+Calculon
+CazoodleBot
+CCBot
+Cegbfeieh
 CheeseBot
-Chlooe
+CheTeam
+ChinaClaw
 Claritybot
-Cliqzbot
 Cloud mapping
-Cogentbot
 cognitiveseo
 Collector
-Copier
-CopyRightCheck
 Copyscape
-Cosmos
-Craftbot
 crawler4j
-crawl.sogou.com
-CSHttp
-DatabaseDriverMysqli
+CrazyWebCrawler
+CrunchBot
+Curious
+Custo
+DataCha0s
 DBLBot
-demandbase-bot
 Deusu
-Dotbot
+Devil
+Digincore
+DigitalPebble
+DIIbot
+Discobot
+DomainAppender
 Download Wonder
+Dragonfly
 Drip
-DSearch
+DTS Agent
+EasyDL
 Ebingbong
 ECCP/1.0
-EirGrabber
-EMail Wolf
-Exabot
+EMail Siphon
+Express WebPictures
 ExtLinksBot
 Extractor
 Extreme Picture Finder
 EyeNetIE
-facebookscraper
-FDM
+FemtosearchBot
+FHscan
+Fimap
 Firefox/7.0
-Flunky
-Foobot
 Freeuploader
 FyberSpider
-GalaxyBot
-Genieo
 GermCrawler
 GetRight
-GetWeb
-Gigablast
-Gigabot
-G-i-g-a-b-o-t
+Go-Ahead-Got-It
 Gotit
+GoZilla
+Go!Zilla
 Grabber
-GrabNet
+Grafula
 GrapeshotCrawler
-GridBot
-GT::WWW
-Harvest
-Havij
-Heritrix
-Hloader
+HaosouSpider
+heritrix
 HMView
-HTMLparser
-HTTP::Lite
 HTTrack
+Humanlinks
 HybridBot
-IlseBot
-IndeedBot
-InfoNaviRobot
+Iblog
+Id-search
+Image Fetch
+Image Sucker
+Indy Library
 InfoTekies
+instabid
 InternetSeer
+internetVista monitor
 ips-agent
-JamesBOT
+Iria
+Iskanie
+Jbrofuzz
 JennyBot
 JetCar
-Jetty
-Joomla
-Jorgee
+JOC Web Spider
 JustView
-Jyxobot
-Kozmosbot
-LeechFTP
-LeechGet
-LexiBot
-Lftp
+Keyword Density
+Kinza
+Lanshanbot
+Larbin
+Libwhisker
 Lightspeedsystems
-LinkScan
+Likse
+LinkextractorPro
+LinkpadBot
 LinkWalker
 LinqiaRSSBot
-LinqiaScrapeBot
-Lipperhey
 Litemage_walker
 Lmspider
-LNSpiderguy
-lwp-request
 LWP::Simple
+lwp-trivial
+Majestic12
 Majestic SEO
 Mass Downloader
-Mata Hari
 MauiBot
-Mb2345Browser
-Meanpathbot
-Mediatoolkitbot
-mediawords
+MeanPath Bot
+MegaIndex.ru
+Metauri
+MFC_Tear_Sample
+MicroMessenger
+Microsoft Data Access
+Microsoft URL Control
 MIDown tool
+MIIxpc
+Mister PiX
 MJ12bot
 Mojeek
-Mojolicious
-Morfeus Fucking Scanner
 MSFrontPage
+MSIECrawler
+muhstik-scan
+Name Intelligence
 Navroad
 Nessus
 Netcraft
 NetSpider
-Nettrack
-Niki-bot
+NextGenSearchBot
+Nibbler
+NICErsPRO
 Nikto
-Ninja
+Nimbostratus
 Nmap
-NPbot
 oBot
-Octopus
-Offline Explorer
-Openfind
+Openvas
 OrangeBot
-OutfoxBot
-PageAnalyzer
+OutclicksBot
+Page Analyzer
 PageGrabber
-Panscient
-pcBrowser
-PECL::HTTP
-Picsearch
+page scorer
+Papa Foto
+Pavuk
+PHPCrawl
+Picscout
 PictureFinder
-Pimonster
-Pi-Monster
-Pixray
 PleaseCrawl
 Pockey
+POE-Component-Client-HTTP
+polaris version
 ProPowerBot
-Pump
-PyCurl
-RankFlex
+RankActive
 RankingBot
-Rankivabot
-Reaper
+RankingBot2
+RankurBot
+RealDownload
+Recorder
 RedesScrapy
-ReGet
 RepoMonkey
-RocketCrawler
-RSSingBot
+Ripper
 s1z.ru
 SalesIntelligent
-Scanbot
-scan.lol
-Scrapy
-Screaming
+SBIder
 ScreenerBot
-SEOkicks
-SEOlyticsCrawler
+SearchmetricsBot
+Semrush
+SemrushBot
+SEOkicks-Robot
+SEOprofiler
 seoscanners
-SeoSiteCheckup
+serpstatbot
 sexsearcher
-Shodan
-Sitebeam
+Siphon
+SISTRIX
 SiteExplorer
+Siteimprove
 SiteSucker
-Site Sucker
 Sitevigil
+SlySearch
 SMTBot
 Snake
-Snoopy
+Snapbot
+SocialRankIOBot
+sogouspider
 Sogou web spider
-Spammen
+Sosospider
+Spanner
+Spbot
+Spinn3r
+SputnikBot
 Sqlmap
-Sqworm
-Steeler
-Stripper
-Sucker
+Sucuri
+SuperBot
 Swiftbot
-sysscan
+Szukacz
+T0PHackTeam
+T8Abot
+Teleport
 Telesoft
 Telesphoreo
 The Intraformant
-TightTwatBot
-Tracemyfile
+Toata
 Trendictionbot
+trendiction.com
 True_Robot
-Turingos
 Turnitin
+Twice
 Upflow
-URLy Warning
-Vagabondo
-VCI
-VidibleScraper
-Virusdie
-Voil
-Webalta
-Web Auto
-WebCopier
-WebGo IS
-WebImageCollector
+VeriCiteCrawler
+WBSearchBot
+WebAuto
+WebCollage
+WebEnhancer
+Web Enhancer
+Web Fetch
+WebFuck
 WebLeacher
-webmeup-crawler
-Webshag
-WebsiteQuester
-Website Quester
-Webster
-WebStripper
+WebmasterWorldForumBot
+Web Pix
+WebReaper
+WebSauger
+Web Sauger
+WebsiteExtractor
 WebSucker
 Web Sucker
-Whack
-Who.is Bot
+WebWhacker
+WeSEE
+Whacker
+Widow
+WiseGuys Robot
 WISENutbot
-Woobot
-Wotbox
-WPScan
-WWWOFFLE
+Wonderbot
+x09Mozilla
 x22Mozilla
-Xaldon WebSpider
-Xaldon_WebSpider
 Xenu
-xpymep1.exe
-Zauba
-zauba.io
+Zade
 Zermelo
-Zeus
 ZmEu
-ZyBorg