V4.2020.04.2047 [ci skip]

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
+360Spider
+80legs
 Abonti
 Aboundex
+Aboundexbot
+ADmantX
 AhrefsBot
+AIBOT
 AiHitBot
-AllSubmitter
+Aipbot
+Alexibot
+Alligator
 Anarchie
-Apexoo
+Ankit
 archive.org_bot
 arquivo.pt
 Aspiegel
+ASPSeek
 Asterias
 autoemailspider
-BackDoorBot
+AwarioRssBot
 Backlink-Ceck
-BacklinkCrawler
+backlink-check
+BackStreet
 BackWeb
 Badass
 Bandit
 BatchFTP
 BDCbot
-BetaBot
+BDFetch
+Black Hole
 Blow
-BlowFish
-BotALot
-Brandprotect
-Buddy
+Bolt
+Brandwatch
 Bullseye
+BunnySlippers
 BuzzSumo
-Calculon
 CATExplorador
 CazoodleBot
-Cegbfeieh
-CheeseBot
+CherryPicker
+CheTeam
 Chlooe
-Cogentbot
-com.plumanalytics
-Copier
-Copyscape
-crawler4j
+Cloud mapping
+Craftbot
+crawl.sogou.com
+Crescent
 CrunchBot
+CSHttp
 Curious
-DataCha0s
-DBLBot
+Custo
 demandbase-bot
 Demon
-Devil
-Digincore
-Dirbuster
-DittoSpyder
+Disco
+Discobot
+Dispatch
+DnyzBot
+DomainAppender
+DomainCrawler
 DomainStatsBot
-Download Wonder
-Dragonfly
+Dotbot
 Drip
-DSearch
 DTS Agent
-EirGrabber
-EMail Siphon
+eCatch
+ECCP/1.0
+Ecxi
+EMail Wolf
 Evil
 Exabot
-EyeNetIE
+ExtLinksBot
+Extractor
 Ezooms
+FemtosearchBot
 FHscan
 Firefox/7.0
-Flunky
-Foobot
-Getintent
-GetRight
+FlashGet
+Freeuploader
+FrontPage
+GalaxyBot
+Genieo
+GermCrawler
 GetWeb
 Gigabot
+Gotit
 Go!Zilla
 Grabber
-GrapeshotCrawler
-GridBot
+GrabNet
 GT::WWW
-HaosouSpider
-Harvest
-HMView
+Haansoft
+HEADMasterSEO
+Hloader
 HTMLparser
-HTTrack
-Humanlinks
+HTTP::Lite
+HybridBot
+Iblog
 IDBot
-Id-search
 IlseBot
-Image Fetch
-Image Sucker
 IndeedBot
-InfoNaviRobot
-instabid
-Internet Ninja
-internetVista monitor
-ips-agent
-Iria
-Iskanie
+IstellaBot
 JamesBOT
 Jbrofuzz
+Jetty
 JOC Web Spider
 Joomla
 Jorgee
+JustView
 Kinza
-Kozmosbot
 Lanshanbot
-LeechGet
-LexiBot
+Lftp
 LibWeb
-LieBaoFast
+Libwhisker
 Lightspeedsystems
 Likse
-LinkScan
-Lipperhey
-Lmspider
-Ltx71
-Magnet
-Mag-Net
-magpie-crawler
-Mail.RU_Bot
+Linkdexbot
+LinkpadBot
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+LinqiaScrapeBot
+lwp-trivial
 Majestic12
-Majestic SEO
-Mass Downloader
-Mb2345Browser
+Majestic-SEO
+MarkMonitor
+Mata Hari
+MauiBot
 meanpathbot
 Meanpathbot
+mediawords
 MegaIndex.ru
 Metauri
-Microsoft URL Control
-MIDown tool
-Mojolicious
+Microsoft Data Access
+MJ12bot
+Mojeek
 Morfeus Fucking Scanner
-MQQBrowser
-Mr.4x3
 MSIECrawler
 Msrabot
 Musobot
-Name Intelligence
-Needle
-Nessus
-NetAnts
+NetSpider
 Nettrack
-Net Vampire
+Netvibes
 NextGenSearchBot
+Nibbler
 Niki-bot
+Nikto
 NimbleCrawler
-Nimbostratus
-Ninja
-Nmap
-Nutch
-OnCrawl
+Octopus
+Offline Explorer
 Openvas
 OpenVAS
 OPPO A33
 OrangeSpider
-OutclicksBot
+OutfoxBot
 PageAnalyzer
 Page Analyzer
-PageScorer
-Pandalytics
+PageGrabber
+Panscient
 pcBrowser
 PECL::HTTP
 PeoplePal
-Picscout
-PictureFinder
-Pixray
+Picsearch
+Pimonster
+PleaseCrawl
 Pockey
-Probethenet
-ProPowerBot
-Psbot
 Pump
-PxBroker
 QueryN Metasearch
-Quick-Crawler
-RankFlex
+RankActive
+RankActiveLinkBot
 RankingBot2
-Rankivabot
 RankurBot
+RealDownload
 Reaper
-Ripper
-Rogerbot
-RSSingBot
+SalesIntelligent
+SBIder
 Scanbot
-ScreenerBot
+Scrapy
 Searchestate
 SearchmetricsBot
-SemrushBot
-SEOlyticsCrawler
-SEOprofiler
-SEOstats
+SEOkicks
+SEOkicks-Robot
+seoscanners
 serpstatbot
+sexsearcher
+Siphon
 SISTRIX
-SiteExplorer
 SiteSnagger
-Site Sucker
+SlySearch
 SmartDownload
 SocialRankIOBot
-Sogou web spider
-Sosospider
-Spammen
+Sociscraper
+SpankBot
 Spanner
-sp_auditbot
+Spinn3r
 spyfu
+Sqlmap
 Sqlworm
+Sqworm
+Steeler
 Stripper
-Sucker
-Sucuri
+Surfbot
 SurveyBot
-Swiftbot
+Suzuran
 sysscan
 tAkeOut
 TeleportPro
+Telesoft
 Telesphoreo
-TheNomad
-TightTwatBot
-Titan
-Trendiction
-trendiction.com
-Turingos
+Telesphorep
+Thumbor
+trendiction.de
 Turnitin
 TurnitinBot
-UnisterBot
-Upflow
-VeriCiteCrawler
+TwengaBot
+URLy Warning
+VB Project
+VidibleScraper
+Virusdie
 VoidEYE
+Voil
+Voltron
 Wallpapers/3.0
+WBSearchBot
+Webalta
 WebAuto
-WebCopier
+WebCollage
+Web Collage
 WebEnhancer
+Web Enhancer
 WebFetch
-WebGo IS
+WebFuck
 WebImageCollector
+WebLeacher
 WebmasterWorldForumBot
 webmeup-crawler
+WebReaper
 Web Sauger
 Webshag
-WebsiteExtractor
+WebsiteQuester
 WebStripper
 Web Sucker
+WebWhacker
 WebZIP
-WeSEE
-Whack
-WISENutbot
-Wprecon
+Who.is Bot
+WinHTTrack
+Woobot
 WPScan
 WWW-Collector-E
+WWWOFFLE
 x22Mozilla
-Xenu
-YoudaoBot
 Zauba
-Zermelo
 zgrab
+Zitebot
 ZumBot
-ZyBorg