V4.2021.09.2615 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
+01h4x.com
+80legs
 AIBOT
-ASPSeek
-Abonti
+Aboundex
 Aboundexbot
 AfD-Verbotsverfahren
 Aipbot
-Alligator
+AlphaBot
+Anarchie
+Anarchy
 Anarchy99
-Ankit
-Anthill
-Apexoo
+Aspiegel
 Asterias
 AwarioRssBot
-BackDoorBot
-Backlink-Ceck
-BacklinkCrawler
+BDFetch
 Badass
-BetaBot
+Barkrowler
 Bigfoot
-Bitacle
-Black Hole
 Blackboard
-BlowFish
-Boardreader
-Bolt
-BotALot
 Brandprotect
+Brandwatch
 BuiltBotTough
-CSHttp
-Calculon
-Cegbfeieh
-Chlooe
-Copier
-Cosmos
-CrazyWebCrawler
+Bullseye
+BunnySlippers
+CCBot
+CODE87
+CheTeam
+Claritybot
+Cloud mapping
+Cocolyzebot
+Cogentbot
+Craftbot
 CrunchBot
 Custo
 CyotekWebCopy
-DBLBot
-DSearch
-DTS Agent
+DataCha0s
 DatabaseDriverMysqli
-Demon
 Devil
+Digincore
 DigitalPebble
 Dirbuster
 Discobot
 DittoSpyder
-DomainAppender
-DomainCrawler
-Domains Project
-Dragonfly
+Drip
+EirGrabber
+EroCrawler
+Evil
 Exabot
 Express WebPictures
 ExtLinksBot
-FHscan
-FemtosearchBot
+ExtractorPro
+EyeNetIE
+FDM
 Fimap
-Firefox/7.0
-Flunky
+Foobot
 Freeuploader
-FyberSpider
+Fuzz
+Fyrebot
 G-i-g-a-b-o-t
+GT::WWW
+GermCrawler
 GetRight
+Getintent
 Gigabot
-Go!Zilla
+GoZilla
 Gotit
+GrabNet
+Grabber
+Grafula
+GrapeFX
+GridBot
 HEADMasterSEO
+HMView
+HTMLparser
 Haansoft
-Heritrix
-Hloader
 Humanlinks
-IRLbot
-Iblog
-InfoNaviRobot
-InfoTekies
+HybridBot
+IlseBot
+IndeedBot
+Intelliseek
 InterGET
 Internet Ninja
-InternetSeer
-Iria
-Iskanie
 JOC Web Spider
-Jbrofuzz
-JennyBot
+JamesBOT
 Jetty
-JustView
-Keyword Density
-Kinza
-Leap
+Kenjin Spider
+Kozmosbot
+Lanshanbot
+LeechFTP
 LeechGet
 LexiBot
 LibWeb
-Lightspeedsystems
-Linkbot
+Libwhisker
+LieBaoFast
+LinkScan
+Linkdexbot
 LinkextractorPro
+LinkpadBot
 LinqiaMetadataDownloaderBot
 LinqiaRSSBot
 Lipperhey
-Lipperhey Spider
 Litemage_walker
-MIIxpc
-MQQBrowser
+Lmspider
+MIDown tool
 MSFrontPage
-Mag-Net
 Majestic SEO
-Majestic12
-MarkMonitor
+Majestic-SEO
 MarkWatch
-Mass Downloader
-Mb2345Browser
-Meanpathbot
-MegaIndex.ru
-Metauri
-Microsoft URL Control
-Mister PiX
-Mozlila
+Masscan
+MauiBot
+Mediatoolkitbot
+Microsoft Data Access
+Mojolicious
+Musobot
 NICErsPRO
-Navroad
+Nameprotect
+NearSite
 Needle
+Nessus
+NetLyzer
 NetMechanic
-Netcraft
 Nettrack
-Netvibes
 NextGenSearchBot
-Nibbler
-Ninja
-Nutch
-Octopus
-OpenLinkProfiler
+Nimbostratus
 OpenVAS
-PECL::HTTP
-POE-Component-Client-HTTP
+Openvas
+OrangeSpider
+OutfoxBot
 PageAnalyzer
-Panscient
-Petalbot
-Pockey
-ProPowerBot
+PageGrabber
+Pavuk
+PictureFinder
+PleaseCrawl
 Probethenet
 Pump
-QueryN Metasearch
-RSSingBot
+Quick-Crawler
 RankActive
-RankingBot
+RankFlex
 Rankivabot
-RankurBot
+Re-re
+ReGet
+Reaper
+RebelMouse
 Recorder
-RepoMonkey
-RocketCrawler
+Rogerbot
 SBIder
+SEOkicks
+SEOkicks-Robot
 SEOlyticsCrawler
 SEOprofiler
+SalesIntelligent
 ScanAlert
 ScoutJet
-Screaming
-Searchestate
+ScreenerBot
+SearchmetricsBot
 Semrush
+SeoSiteCheckup
 Seomoz
-Siphon
+Shodan
+Site Sucker
 SiteCheckerBotCrawler
-SiteExplorer
-SiteLockSpider
 SiteSucker
-Sitebeam
 Sitevigil
-Sociscraper
-Sogou web spider
-Sosospider
-SpankBot
-Spanner
-Spbot
+SmartDownload
+Snoopy
+SocialRankIOBot
+SpaceBison
+Spammen
 Spinn3r
 Sqlmap
-Steeler
-Stripper
-SuperBot
-Suzuran
+Sqworm
+Surfbot
+SurveyBot
 Swiftbot
 Szukacz
 Teleport
-Telesphorep
-TheNomad
-Toata
+Telesphoreo
+TightTwatBot
+Titan
+Toweyabot
 Trendictionbot
 True_Robot
-TurnitinBot
+Turnitin
 Typhoeus
-UnisterBot
-Upflow
-VeriCiteCrawler
-VoidEYE
-WASALive-Bot
-WBSearchBot
+URLy Warning
+V-BOT
+Vagabondo
+VelenPublicWebCrawler
+Virusdie
+Voil
+WISENutbot
 WPScan
 WWW::Mechanize
 WWWOFFLE
-Wallpapers
-Wallpapers/3.0
-WeSEE
-Web Fuck
-Web Pix
-WebAuto
-WebBandit
+Web Sauger
+Web Sucker
 WebCopier
-WebEnhancer
+WebFetch
 WebFuck
-WebGo IS
+WebImageCollector
 WebLeacher
-WebPix
-WebSauger
+WebWhacker
 WebZIP
 WebmasterWorldForumBot
-Webshag
-Whacker
-Whatweb
+WebsiteExtractor
+Whack
+Who.is Bot
 Widow
-Woobot
-Wotbox
+WinHTTrack
+Wonderbot
 Wprecon
+Xaldon WebSpider
+Xaldon_WebSpider
+Xenu
+YoudaoBot
+Zade
 Zauba
+Zermelo
 Zeus
 Zitebot
 ZmEu
 ZoomBot
+ZumBot
+archive.org_bot
+autoemailspider
 backlink-check
-check1.exe
+cah.io.community
+clark-crawler
 com.plumanalytics
-crawler.feedback
-dataforseo.com
-demandbase-bot
-domainsproject.org
+crawl.sogou.com
+eCatch
 instabid
-ips-agent
-isitwp.com
-meanpathbot
-mediawords
+internetVista monitor
+lwp-trivial
+magpie-crawler
+muhstik-scan
 page scorer
-pcBrowser
-polaris version
+plumanalytics
+probe-image-size
 ripz
 s1z.ru
-scalaj-http
-scan.lol
+satoristudio.net
 seobility
 sitechecker.pro
-siteripz
+sogouspider
 spyfu
+tAkeOut
 trendiction.com
+voyagerx.com
 webprosbot
-zgrab