V4.2021.10.2675 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+360Spider
 404checker
-ASPSeek
+ALittle Client
 Abonti
-Acunetix
+Aboundex
+Aboundexbot
+AhrefsBot
+Alexibot
+AllSubmitter
+Alligator
+AlphaBot
+Anarchy
 Ankit
-Anthill
-Apexoo
-Aspiegel
-Asterias
-BackDoorBot
-BackWeb
-BacklinkCrawler
+BBBike
+BDCbot
+Backlink-Ceck
+Bandit
 BatchFTP
-BetaBot
-Blackboard
+Battleztar Bazinga
+Bigfoot
+Bitacle
+Black Hole
 BlowFish
-Bolt
-Buck
-Buddy
+BuiltWith
+BuzzSumo
 CATExplorador
-Calculon
-CazoodleBot
+CODE87
+CSHttp
 Cegbfeieh
 CensysInspect
-CherryPicker
 ChinaClaw
+Chlooe
 Claritybot
 Cliqzbot
-Cloud mapping
-Cosmos
+Copier
+Copyscape
+Craftbot
 Crawling at Home Project
-CrazyWebCrawler
-CrunchBot
-Custo
-DIIbot
+Crescent
+Curious
+DSearch
 DTS Agent
 DataCha0s
+DatabaseDriverMysqli
+Demon
+Deusu
 Dirbuster
-Disco
 Discoverybot
 Dispatch
-DittoSpyder
-DomCopBot
+DnyzBot
 DomainAppender
-Dotbot
-Download Wonder
-ECCP/1.0
-EroCrawler
+Domains Project
+EMail Siphon
+Evil
+Exabot
+Express WebPictures
+ExtLinksBot
 Extractor
+Extreme Picture Finder
+EyeNetIE
 FHscan
 FlashGet
 Flunky
-Foobot
-Freeuploader
 Fuzz
-Fyrebot
-G-i-g-a-b-o-t
+FyberSpider
 GalaxyBot
 Genieo
-GetRight
-Go-Ahead-Got-It
-GoZilla
-Gotit
-Grafula
+GermCrawler
+Getintent
+Go!Zilla
+GrapeshotCrawler
+GridBot
 HEADMasterSEO
-HMView
-HTMLparser
 HTTP::Lite
-HTTrack
 Haansoft
+HaosouSpider
+Havij
 Heritrix
-Humanlinks
-Iblog
+Id-search
 IlseBot
 Image Fetch
-IndeedBot
-InfoTekies
 Intelliseek
-InternetSeer
-IstellaBot
+InterGET
+Iria
+Iskanie
 JOC Web Spider
-JamesBOT
-Jbrofuzz
-JetCar
-Jetty
 JikeSpider
+Joomla
 Jorgee
 JustView
-Kenjin Spider
-Keyword Density
-Leap
-LexiBot
+LNSpiderguy
+LWP::Simple
+Lanshanbot
+LeechFTP
+LeechGet
 Lftp
 Lightspeedsystems
-Likse
 LinkScan
-Linkdexbot
-LinkextractorPro
-LinkpadBot
+LinkWalker
 LinksManager
-LinqiaRSSBot
 LinqiaScrapeBot
-Lipperhey Spider
 Litemage_walker
-Ltx71
-MIDown tool
-MJ12bot
+Lmspider
 MQQBrowser
-MSIECrawler
-MTRobot
-Mag-Net
-Masscan
-Mata Hari
-MeanPath Bot
-Microsoft URL Control
+Majestic-SEO
+MarkWatch
+Meanpathbot
+MegaIndex.ru
+MicroMessenger
+Microsoft Data Access
 Mister PiX
-Moblie Safari
-NPbot
+Mojolicious
+Mr.4x3
+NICErsPRO
+Nameprotect
+Navroad
 NearSite
-Net Vampire
 NetAnts
-NetLyzer
-NetZIP
-Nettrack
-Niki-bot
-Octopus
-Openvas
-OutclicksBot
-POE-Component-Client-HTTP
-PageAnalyzer
-PageGrabber
-Papa Foto
-Petalbot
+NetMechanic
+NetSpider
+Netvibes
+Ninja
+Offline Explorer
+Offline Navigator
+OnCrawl
+Openfind
+OrangeBot
+OrangeSpider
+PageThing.com
+Pavuk
 Pi-Monster
+Picscout
 Picsearch
-Piepmatz
-Pixray
+PictureFinder
 Pockey
-ProPowerBot
+Probethenet
+Psbot
+Pu_iN
 Pump
-PxBroker
 PyCurl
-QueryN Metasearch
-RankingBot
+RSSingBot
+RankFlex
 RankingBot2
 Rankivabot
-Re-re
-ReGet
-RedesScrapy
-Ripper
+Recorder
 RocketCrawler
 Rogerbot
+SEOkicks-Robot
 SEOlyticsCrawler
+SEOprofiler
+SEOstats
+SISTRIX
 SMTBot
-Scanbot
-ScoutJet
-Scrapy
+SalesIntelligent
+Screaming
+ScreenerBot
 ScrepyBot
-Searchestate
 SearchmetricsBot
+SemrushBot
+SentiBot
+SeoSiteCheckup
+Shodan
 Site Sucker
+SiteSnagger
+Sitebeam
 Siteimprove
-Sitevigil
+SlySearch
 SmartDownload
 Snake
-Snoopy
-Sogou web spider
+Sottopop
+SpaceBison
 SpankBot
 Spinn3r
 SputnikBot
-Sqlworm
-Sqworm
-Steeler
+Sucker
 Sucuri
-SuperHTTP
 Surfbot
-Suzuran
-Swiftbot
-Thumbor
-Titan
+SurveyBot
+Szukacz
+T0PHackTeam
+Teleport
+TheNomad
+Toata
+Toweyabot
 Tracemyfile
-Trendiction
 Trendictionbot
 True_Robot
 Turnitin
-VB Project
-VCI
+TwengaBot
+Twice
+UnisterBot
+V-BOT
+Vacuum
+Vagabondo
 VelenPublicWebCrawler
 VeriCiteCrawler
 Virusdie
 VoidEYE
-WWW::Mechanize
+Voil
+WASALive-Bot
+WISENutbot
+WPScan
+WWW-Collector-E
+WWW-Mechanize
 WWWOFFLE
 Wallpapers/3.0
 WallpapersHD
 Web Auto
 Web Collage
-Web Enhancer
-Web Pix
-Web Sucker
-WebAuto
+Web Fetch
 WebCollage
+WebCopier
+WebEnhancer
+WebGo IS
+WebImageCollector
 WebSauger
 WebSucker
-WebWhacker
 WebmasterWorldForumBot
+WebsiteExtractor
 Webster
-Who.is Bot
-WinHTTrack
-WiseGuys Robot
-Wprecon
-Xaldon_WebSpider
+Widow
+Wotbox
 Xenu
-YoudaoBot
-Zauba
+Zermelo
 Zitebot
-ZoomBot
 ZoominfoBot
-ZyBorg
-archive.org_bot
-arquivo-web-crawler
-backlink-check
-cah.io.community
+ZumBot
+autoemailspider
 check1.exe
-eCatch
-evc-batch
-heritrix
-instabid
-ips-agent
+clark-crawler
+cognitiveseo
+dataforseo.com
+gopher
+internetVista monitor
 magpie-crawler
-meanpathbot
+netEstate NE Crawler
 oBot
-page scorer
 plumanalytics
 polaris version
-probe-image-size
-ripz
-s1z.ru
-satoristudio.net
 scan.lol
-seoscanners
 serpstatbot
 sogouspider
+sp_auditbot
 spyfu
 tAkeOut
 trendiction.de
-voyagerx.com
-webmeup-crawler
-xpymep1.exe
-zauba.io
+webprosbot
+x09Mozilla
+x22Mozilla
+zgrab