V4.2021.12.2761 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
-404enemy
-ADmantX
-ALittle Client
 AfD-Verbotsverfahren
-AlphaBot
-Anarchy99
-Anthill
-BDCbot
-BDFetch
+Alligator
+Ankit
+Apexoo
+Asterias
+AwarioSmartBot
 BLEXBot
-Backlink-Ceck
-Bandit
+BackStreet
+BacklinkCrawler
+Badass
+BetaBot
+Bigfoot
+Bitacle
 Black Hole
-Bolt
-BotALot
+Blow
+BlowFish
+Boardreader
 Buck
 Buddy
 BuiltWith
 BunnySlippers
-CODE87
-Calculon
-CazoodleBot
-CheeseBot
-Claritybot
+Cegbfeieh
+CherryPicker
+Citoid
 Cliqzbot
-Cloud mapping
-Collector
-Copier
+Cocolyzebot
+Cogentbot
+CopyRightCheck
 Copyscape
+Craftbot
+Crawling at Home Project
 CrazyWebCrawler
+Crescent
+Custo
 CyotekWebCopy
+DSearch
 DataCha0s
+Demon
 Deusu
-Devil
-Discobot
-DittoSpyder
+DigitalPebble
 DnBCrawler-Analytics
-DomCopBot
-DomainCrawler
+DnyzBot
+DomainAppender
 DomainStatsBot
-Dotbot
+Download Wonder
 Dragonfly
-EMail Siphon
+Drip
+EMail Wolf
 EasyDL
+Ebingbong
 Ecxi
 EirGrabber
-Evil
-ExtLinksBot
-ExtractorPro
+Exabot
+Extractor
+Extreme Picture Finder
+EyeNetIE
+Fimap
+Firefox/7.0
+Foobot
+FrontPage
 Fuzz
-FyberSpider
+Fyrebot
 G-i-g-a-b-o-t
 GT::WWW
-GalaxyBot
 Genieo
 GermCrawler
 GetRight
-GetWeb
-GrabNet
+GoZilla
+Grafula
+GrapeFX
 GridBot
 HMView
-HTTP::Lite
+Haansoft
 HaosouSpider
+Harvest
 Heritrix
-Humanlinks
-HybridBot
+IDBTE4M
+IDBot
 IRLbot
-Iblog
+Id-search
 IlseBot
-Image Fetch
-Image Sucker
-InfoNaviRobot
-InfoTekies
-Internet Ninja
-Iria
+InternetSeer
 Iskanie
-JOC Web Spider
-Jbrofuzz
-Jetty
-JustView
-Kenjin Spider
-Kozmosbot
-LeechGet
-LexiBot
-Lftp
+JikeSpider
+Jyxobot
+Keyword Density
+LWP::Simple
+Libwhisker
+Likse
 Linkdexbot
 LinkpadBot
-Lipperhey Spider
+LinqiaScrapeBot
+Lipperhey
 Litemage_walker
-Ltx71
-MFC_Tear_Sample
+Lmspider
 MIDown tool
+MJ12bot
 MQQBrowser
-MarkMonitor
+MSIECrawler
+Mail.RU_Bot
+Majestic SEO
+Majestic-SEO
 MarkWatch
-Mata Hari
+Mass Downloader
+MauiBot
+Mb2345Browser
+Meanpathbot
+Mediatoolkitbot
 MegaIndex.ru
 MicroMessenger
-Microsoft URL Control
 Minefield
-Mojeek
-Mozlila
+Mister PiX
+Moblie Safari
 Mr.4x3
+NICErsPRO
+NPbot
 Name Intelligence
-Nameprotect
-Needle
+Navroad
 Nessus
 Net Vampire
-NetMechanic
-NetSpider
 NetZIP
 Netcraft
-Nettrack
 Netvibes
 NextGenSearchBot
-Nikto
-Nimbostratus
-Ninja
 Nmap
 Nuclei
-Octopus
-Offline Navigator
-OpenLinkProfiler
+OnCrawl
 OpenVAS
 Openfind
-OrangeBot
+Openvas
 OrangeSpider
 OutclicksBot
+OutfoxBot
+PECL::HTTP
 PHPCrawl
-POE-Component-Client-HTTP
+PageAnalyzer
 PageScorer
-Pavuk
+PageThing.com
+Pandalytics
+PeoplePal
+Petalbot
 Pi-Monster
-ProPowerBot
+Piepmatz
+Pockey
+Psbot
 PxBroker
-PyCurl
-QueryN Metasearch
+Quick-Crawler
 RSSingBot
 RankActive
-RankFlex
-RankingBot2
-Rankivabot
-Reaper
-RedesScrapy
+Re-re
+RealDownload
+Recorder
+RepoMonkey
+RocketCrawler
 Rogerbot
 SBIder
+SEOkicks
+SEOlyticsCrawler
 SEOprofiler
 SEOstats
+SISTRIX
 SMTBot
 Scrapy
-ScrepyBot
-Searchestate
-SearchmetricsBot
-SeoSiteCheckup
+SemanticJuice
+SemrushBot
 SeobilityBot
-Shodan
-SiteLockSpider
-SiteSucker
-Sitevigil
-Snapbot
-Snoopy
-SocialRankIOBot
-Sosospider
-Spanner
-Sqlmap
-Sqworm
-SurveyBot
-T8Abot
-Thumbor
+Seomoz
+Site Sucker
+Siteimprove
+SlySearch
+Sociscraper
+Sogou web spider
+Sottopop
+SpaceBison
+Spbot
+Spinn3r
+SputnikBot
+Sqlworm
+Steeler
+Stripper
+Sucker
+Sucuri
+SuperBot
+SuperHTTP
+Surfbot
+Suzuran
+Teleport
+TeleportPro
+Telesoft
+Telesphorep
+TheNomad
 TightTwatBot
-Toata
-Toweyabot
+Titan
 Tracemyfile
-Trendictionbot
 True_Robot
 Turingos
-TurnitinBot
-URLy Warning
-UnisterBot
-VelenPublicWebCrawler
-VeriCiteCrawler
-VidibleScraper
+TwengaBot
+V-BOT
+Vagabondo
 WASALive-Bot
 WBSearchBot
-WISENutbot
-WPScan
 WWW-Collector-E
-WWW-Mechanize
-WWW::Mechanize
 WWWOFFLE
-Web Collage
-Web Sauger
+Wallpapers
+Wallpapers/3.0
+WallpapersHD
+Web Fuck
+Web Pix
 WebBandit
-WebEnhancer
-WebFuck
-WebGo IS
+WebImageCollector
+WebLeacher
 WebReaper
-WebStripper
+WebSauger
 WebSucker
-WebWhacker
+WebZIP
 WebmasterWorldForumBot
-Webshag
-Website Quester
-Webster
-Whack
-Whacker
-Whatweb
-Widow
-WinHTTrack
+WebsiteQuester
+WiseGuys Robot
+Wotbox
 Xaldon WebSpider
 Xaldon_WebSpider
 Xenu
-Zade
-Zauba
+YoudaoBot
 Zeus
+ZoomBot
+ZoominfoBot
 ZumBot
-adscanner
 arquivo-web-crawler
-coccocbot-web
+arquivo.pt
+autoemailspider
+check1.exe
+clark-crawler
 cognitiveseo
 com.plumanalytics
-crawl.sogou.com
 crawler.feedback
 crawler4j
-domainsproject.org
-heritrix
-internetVista monitor
-isitwp.com
-lwp-request
-lwp-trivial
-magpie-crawler
-meanpathbot
-muhstik-scan
-probe-image-size
+facebookscraper
+ripz
+satoristudio.net
 scalaj-http
-seoscanners
-seostar
-serpstatbot
+scan.lol
+sitechecker.pro
 siteripz
-sp_auditbot
+spyfu
 tAkeOut
-trendiction.com
-ubermetrics-technologies.com
-webgains-bot
+trendiction.de
 webpros.com
 webprosbot
-x22Mozilla
-zauba.io
+zgrab