V4.2021.10.2636 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
+360Spider
+404checker
+80legs
+ADmantX
 ALittle Client
-Aboundexbot
-Acunetix
+ASPSeek
+Aboundex
 AfD-Verbotsverfahren
-AhrefsBot
-AllSubmitter
+Aipbot
 Alligator
-Ankit
-Aspiegel
-Backlink-Ceck
-BacklinkCrawler
-BetaBot
+AlphaBot
+Anarchie
+Anarchy
+Anarchy99
+Apexoo
+AwarioSmartBot
+BBBike
+BDCbot
+BDFetch
+BackDoorBot
+Barkrowler
+Battleztar Bazinga
 Bigfoot
-Black Hole
 Blackboard
-BlowFish
-Bolt
+Buddy
 BuiltBotTough
-BuiltWith
-BuzzSumo
-CCBot
-CODE87
-CSHttp
+Bullseye
+BunnySlippers
+CATExplorador
+Calculon
+CazoodleBot
 Cegbfeieh
-CheeseBot
+CheTeam
+CherryPicker
 ChinaClaw
-Chlooe
-Cloud mapping
-Cosmos
+Collector
 CrunchBot
+CyotekWebCopy
 DIIbot
-DTS Agent
-DataCha0s
-DatabaseDriverMysqli
+Demon
+Devil
 Digincore
+DigitalPebble
+Dirbuster
+Disco
+Discobot
 Discoverybot
-Dispatch
-DomainCrawler
-DomainSigmaCrawler
+DomCopBot
 DomainStatsBot
-Download Wonder
 Dragonfly
-Drip
-ECCP/1.0
-EasyDL
-Ebingbong
+Ecxi
+EirGrabber
 EroCrawler
-Express WebPictures
-Extreme Picture Finder
+Exabot
+ExtLinksBot
+Extractor
+ExtractorPro
 EyeNetIE
+Ezooms
 FDM
-FemtosearchBot
-Fimap
 FlashGet
-Foobot
 FrontPage
-Fuzz
+FyberSpider
+G-i-g-a-b-o-t
 GT::WWW
-Genieo
 GermCrawler
 GetRight
-Gigabot
-Gotit
-Grafula
-HEADMasterSEO
+GetWeb
+Go-Ahead-Got-It
+GrabNet
+Grabber
+GridBot
 HMView
 HTTP::Lite
 HTTrack
-Haansoft
-Hloader
-Humanlinks
-IDBot
+Harvest
 Iblog
-Id-search
 IlseBot
-Intelliseek
-Iria
+Image Fetch
+Image Sucker
+InfoNaviRobot
+Internet Ninja
+InternetSeer
 Iskanie
+IstellaBot
 JOC Web Spider
+JamesBOT
+Jbrofuzz
 JetCar
 Jetty
-Kenjin Spider
+Joomla
+JustView
 Keyword Density
 Kinza
+Kozmosbot
+LNSpiderguy
+Leap
+LeechGet
 LexiBot
-Lftp
-LieBaoFast
+Lightspeedsystems
+LinkScan
 Linkbot
-LinkextractorPro
-LinkpadBot
+LinksManager
 LinqiaRSSBot
-LinqiaScrapeBot
+Litemage_walker
 Lmspider
-MSIECrawler
+Ltx71
+MTRobot
+Mail.RU_Bot
+Majestic-SEO
 Mass Downloader
-MauiBot
-Mb2345Browser
-MeanPath Bot
-Meanpathbot
 Mediatoolkitbot
-Mister PiX
-NICErsPRO
-NPbot
+Metauri
+Microsoft URL Control
+Mojolicious
+Morfeus Fucking Scanner
+Mozlila
+Musobot
+Nameprotect
 Navroad
-NetLyzer
+Nessus
+NetAnts
 NetSpider
 NetZIP
+Netvibes
+NextGenSearchBot
+Nibbler
+Nikto
 NimbleCrawler
 Nimbostratus
-Ninja
+Nmap
 Nuclei
-Offline Navigator
+Nutch
 OpenLinkProfiler
-OpenVAS
-Openfind
 OrangeBot
-PHPCrawl
+OrangeSpider
+OutfoxBot
+POE-Component-Client-HTTP
 Page Analyzer
-PageAnalyzer
-PageGrabber
-PageThing.com
 Pandalytics
-Panscient
-Papa Foto
-Picscout
+Pavuk
+PeoplePal
+Pi-Monster
 Picsearch
-PictureFinder
-Piepmatz
 Pimonster
 Pixray
-Probethenet
-Psbot
+PleaseCrawl
+ProWebWalker
 Pump
-QueryN Metasearch
-Quick-Crawler
-RankActive
-RankActiveLinkBot
-RankFlex
-Re-re
-RebelMouse
 Recorder
-RedesScrapy
+RocketCrawler
 Rogerbot
-SBIder
-SEOkicks-Robot
-SEOlyticsCrawler
 SEOprofiler
-SEOstats
-SalesIntelligent
-Scanbot
-ScoutJet
+SISTRIX
+Scrapy
 ScreenerBot
-Searchestate
-Semrush
+SearchmetricsBot
+SemrushBot
 SentiBot
 Seomoz
 Shodan
-SlySearch
-Snake
-Snapbot
-SocialRankIOBot
-Sogou web spider
-Sottopop
-Spammen
-SpankBot
+Site Sucker
+SiteCheckerBotCrawler
+SiteSucker
+Sitebeam
+Siteimprove
+Sitevigil
+SmartDownload
+SpaceBison
+Spanner
 Spbot
-SputnikBot
 Sqlmap
-Steeler
-Sucuri
-SuperBot
-Surfbot
+Stripper
+SuperHTTP
+SurveyBot
 Suzuran
-Swiftbot
-TeleportPro
-Thumbor
-TightTwatBot
-Tracemyfile
+T0PHackTeam
+T8Abot
+Teleport
+Telesoft
+Telesphoreo
+Telesphorep
+TheNomad
+Toata
+Toweyabot
+Trendictionbot
 True_Robot
-Twice
-URLy Warning
-URLy.Warning
+Typhoeus
 VB Project
 Vacuum
-VidibleScraper
-Virusdie
+VeriCiteCrawler
+VoidEYE
+WBSearchBot
 WEBDAV
-WWW-Mechanize
-Wallpapers
+WISENutbot
+WWW-Collector-E
+WWW::Mechanize
+WWWOFFLE
 WallpapersHD
 WeSEE
-Web Enhancer
+Web Collage
 Web Fetch
 Web Sucker
-WebCopier
+WebBandit
+WebCollage
 WebEnhancer
 WebFetch
-WebFuck
+WebImageCollector
 WebLeacher
 WebSauger
-WebStripper
-WebWhacker
-WebmasterWorldForumBot
-Webshag
 Whack
 Whacker
 Whatweb
-WinHTTrack
+Who.is Bot
+WiseGuys Robot
+Wonderbot
+Woobot
 Wotbox
 Xaldon_WebSpider
-Zade
+Zauba
 Zermelo
-Zitebot
 ZoomBot
-ZoominfoBot
-ZumBot
 archive.org_bot
-arquivo-web-crawler
-backlink-check
-coccocbot-web
+clark-crawler
+cognitiveseo
 crawler.feedback
 crawler4j
-dataforseo.com
 demandbase-bot
-domainsproject.org
-eCatch
-evc-batch
+facebookscraper
+internetVista monitor
+isitwp.com
 lwp-request
 lwp-trivial
+meanpathbot
 mediawords
-muhstik-scan
 netEstate NE Crawler
-oBot
 page scorer
 pcBrowser
-polaris version
 ripz
 satoristudio.net
-scan.lol
-seobility
+seoscanners
 seostar
-sogouspider
+sexsearcher
+sitechecker.pro
+spyfu
 tAkeOut
+trendiction.com
 webmeup-crawler
-webpros.com
-webprosbot
-x09Mozilla
 zauba.io
-zgrab