V4.2021.10.2671 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
-80legs
-Aboundexbot
+ALittle Client
 Acunetix
 AfD-Verbotsverfahren
+AhrefsBot
 AiHitBot
-Aipbot
-Alligator
-AlphaBot
+Anarchy99
 Apexoo
+AwarioSmartBot
+BBBike
+BDCbot
 BLEXBot
-BackDoorBot
-BacklinkCrawler
+BackWeb
 Badass
-Bandit
-Bigfoot
+Barkrowler
+Battleztar Bazinga
+BetaBot
 Bitacle
+Black Hole
 BlackWidow
 Blow
-BlowFish
-Bolt
+BotALot
 Brandprotect
-BuiltBotTough
-BuiltWith
-Bullseye
-BuzzSumo
-CODE87
+Buck
+CATExplorador
+CCBot
 Calculon
 Cegbfeieh
-ChinaClaw
+CheTeam
+CheeseBot
+CherryPicker
+Cloud mapping
 Cocolyzebot
-Copier
 CopyRightCheck
-Crescent
+CrunchBot
+CyotekWebCopy
 DBLBot
-DTS Agent
-DataCha0s
+DSearch
 Demon
+Deusu
 Devil
+Digincore
+Dirbuster
 Disco
-Discobot
 Discoverybot
 Dispatch
-DittoSpyder
-DnBCrawler-Analytics
 DnyzBot
-DomCopBot
-DomainStatsBot
-Dotbot
-Download Wonder
-EirGrabber
+DomainCrawler
+Dragonfly
+Drip
+ECCP/1.0
+Ebingbong
+Ecxi
 Evil
 Exabot
-Extractor
-Ezooms
-FHscan
-Freeuploader
+ExtLinksBot
+ExtractorPro
+EyeNetIE
+FemtosearchBot
+FlashGet
+FrontPage
 Fuzz
-FyberSpider
 Fyrebot
-GT::WWW
+G-i-g-a-b-o-t
 GetRight
+Getintent
 Go!Zilla
-GrabNet
-GrapeshotCrawler
+Go-Ahead-Got-It
+Grabber
+GrapeFX
 GridBot
-HEADMasterSEO
 HTMLparser
-Haansoft
-HybridBot
+HTTP::Lite
+Harvest
+Humanlinks
 IDBTE4M
-IRLbot
-Image Sucker
-IndeedBot
-InfoNaviRobot
+Iblog
+Id-search
+Indy Library
+Intelliseek
+Internet Ninja
 InternetSeer
-Iria
-IstellaBot
 JamesBOT
-Jbrofuzz
-JustView
+JennyBot
+Jetty
+Joomla
+Jyxobot
 Kenjin Spider
 Keyword Density
-Kinza
 Kozmosbot
-LeechFTP
-Lftp
-LibWeb
+LNSpiderguy
+Lanshanbot
+Leap
+LeechGet
+LexiBot
 Libwhisker
 LieBaoFast
-Likse
-LinkWalker
+Lightspeedsystems
 Linkbot
-Linkdexbot
+LinkextractorPro
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
 LinqiaScrapeBot
-Lipperhey
 MFC_Tear_Sample
 MIDown tool
-MIIxpc
 MJ12bot
-Mail.RU_Bot
-Majestic-SEO
+Mag-Net
+Magnet
+MarkWatch
 Mata Hari
-Meanpathbot
-Mediatoolkitbot
+MeanPath Bot
+Metauri
 Microsoft URL Control
-Mojeek
+Mister PiX
 Msrabot
-NPbot
-Name Intelligence
+Musobot
+Nameprotect
+NearSite
 Needle
 Net Vampire
-Nibbler
+NetLyzer
+NetSpider
+NetZIP
+Nettrack
+Netvibes
 Nikto
 NimbleCrawler
-Nimbostratus
+Nuclei
 Offline Explorer
 Openfind
-OrangeBot
-OrangeSpider
-OutclicksBot
-OutfoxBot
-PECL::HTTP
 POE-Component-Client-HTTP
-PageGrabber
-PageThing.com
+Page Analyzer
+PageScorer
+Pandalytics
+Panscient
+Papa Foto
+Pavuk
 PeoplePal
-Pi-Monster
 Picscout
-Piepmatz
+Pimonster
 PleaseCrawl
-Pockey
-PxBroker
-PyCurl
-RankActiveLinkBot
-RankingBot
-RankingBot2
+ProPowerBot
+Probethenet
+Psbot
+RankFlex
+Rankivabot
 RankurBot
+Re-re
 ReGet
 RealDownload
 RebelMouse
-Recorder
+RedesScrapy
 RepoMonkey
-Ripper
-RocketCrawler
-SEOlyticsCrawler
-SMTBot
 SalesIntelligent
-Screaming
-ScreenerBot
-Searchestate
-SearchmetricsBot
 SemrushBot
-SiteCheckerBotCrawler
-SiteSnagger
+SeoSiteCheckup
+SeobilityBot
+Shodan
+Site Sucker
+SiteExplorer
+SiteLockSpider
 SiteSucker
+SlySearch
+Snapbot
 Snoopy
 Sosospider
 Sottopop
 SpaceBison
-SpankBot
-Spanner
-Spinn3r
-SputnikBot
-Stripper
-Sucuri
+Sucker
 SuperBot
-Surfbot
-Szukacz
+SurveyBot
+Suzuran
+Swiftbot
+T0PHackTeam
 Teleport
-Telesphoreo
+TeleportPro
+Telesoft
 TheNomad
-TightTwatBot
-Titan
-Toweyabot
+Toata
 Tracemyfile
-Trendiction
 True_Robot
-Turingos
-Turnitin
 TurnitinBot
-Twice
-UnisterBot
-Vacuum
-Vagabondo
-VelenPublicWebCrawler
+TwengaBot
+Typhoeus
+URLy.Warning
+Upflow
+VeriCiteCrawler
 Virusdie
-VoidEYE
-Voil
 WISENutbot
-WWW-Collector-E
 WWW-Mechanize
+WWW::Mechanize
 WWWOFFLE
 Wallpapers
-Web Auto
+WallpapersHD
+WeSEE
 Web Collage
-Web Fetch
-Web Sauger
+Web Enhancer
+Web Pix
 WebBandit
 WebFetch
 WebFuck
-WebGo IS
-WebImageCollector
-WebLeacher
-WebPix
+WebReaper
 WebStripper
-Webalta
+WebZIP
 Website Quester
+WebsiteQuester
 Whack
 Whatweb
 Widow
+WinHTTrack
+WiseGuys Robot
 Woobot
 Wotbox
-Xaldon WebSpider
-Xaldon_WebSpider
+Wprecon
 Xenu
-Zade
+Zauba
+Zermelo
 Zitebot
 ZmEu
-ZoomBot
 ZoominfoBot
+ZumBot
 arquivo-web-crawler
+arquivo.pt
 cah.io.community
-coccocbot-web
+clark-crawler
+com.plumanalytics
+crawl.sogou.com
 demandbase-bot
-domainsproject.org
-gopher
-internetVista monitor
-lwp-request
-magpie-crawler
+eCatch
+evc-batch
+facebookscraper
+instabid
+isitwp.com
 meanpathbot
 mediawords
 muhstik-scan
-netEstate NE Crawler
 page scorer
-pcBrowser
 plumanalytics
+ripz
+s1z.ru
 scalaj-http
-seobility
-seoscanners
-serpstatbot
+sexsearcher
+sogouspider
 sp_auditbot
-spyfu
-trendiction.com
+sysscan
+trendiction.de
+voyagerx.com
+webmeup-crawler
+webpros.com
 x09Mozilla
 xpymep1.exe
 zauba.io