V4.2021.12.2820 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,117 +1,121 @@
-360Spider
-404enemy
 80legs
-ADmantX
+ALittle Client
 Aboundex
-Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
+Aboundexbot
+Aipbot
+AllSubmitter
 Alligator
-AlphaBot
-Anarchy
+Ankit
 Anthill
-AwarioRssBot
+Apexoo
 BBBike
+BLEXBot
+BackDoorBot
 BackStreet
+BackWeb
 Backlink-Ceck
-Barkrowler
+Bandit
 BatchFTP
-Battleztar Bazinga
 Bigfoot
-Bitacle
 Black Hole
-Blackboard
-Blow
-BlowFish
-Bolt
-Brandprotect
+Brandwatch
 Buddy
 BuiltBotTough
-CheTeam
-Chlooe
+CATExplorador
+CSHttp
+CazoodleBot
+CensysInspect
 Citoid
-Cogentbot
+Cliqzbot
+Copier
 CopyRightCheck
 Crescent
-Custo
-CyotekWebCopy
-DBLBot
+Curious
 DSearch
-Digincore
+DTS Agent
+Demon
+DigitalPebble
+Discobot
 Dispatch
+DittoSpyder
 DnBCrawler-Analytics
 DomainCrawler
-Domains Project
-Download Wonder
+DomainStatsBot
 Dragonfly
-EMail Siphon
+ECCP/1.0
 EMail Wolf
-EasyDL
 Ebingbong
-Ecxi
-Evil
-ExtractorPro
-EyeNetIE
+Exabot
+Extractor
+Ezooms
 FDM
+FHscan
 FemtosearchBot
-Fimap
-Firefox/7.0
-Flunky
-Freeuploader
+Foobot
+FrontPage
+Fuzz
 FyberSpider
-GermCrawler
-GetRight
-Gigabot
+Fyrebot
+Getintent
+Go-Ahead-Got-It
 GrabNet
-Grabber
 Grafula
-HTTP::Lite
-Haansoft
-HaosouSpider
-Harvest
-Heritrix
+GrapeFX
+HEADMasterSEO
+Havij
 Hloader
+HybridBot
 IDBTE4M
-IlseBot
-Image Fetch
-Indy Library
-InfoTekies
-Iskanie
-JOC Web Spider
-JennyBot
-JetCar
+Iblog
+Id-search
+InfoNaviRobot
+InterGET
 Joomla
+JustView
+Jyxobot
 Kinza
+LNSpiderguy
+Lanshanbot
+Larbin
+LeechFTP
+LeechGet
+LexiBot
 Lftp
-Libwhisker
-Linkbot
-Lipperhey Spider
+LibWeb
+Likse
+LinkScan
+Linkdexbot
+LinkextractorPro
+LinqiaMetadataDownloaderBot
+Lipperhey
 Litemage_walker
-Ltx71
-MFC_Tear_Sample
-MSIECrawler
+MJ12bot
 MTRobot
-Majestic SEO
-MarkWatch
+Mag-Net
+Magnet
+Majestic12
 Mass Downloader
-MeanPath Bot
+MauiBot
+Mb2345Browser
+Meanpathbot
 Mediatoolkitbot
-MicroMessenger
-Mojolicious
+Microsoft Data Access
+Microsoft URL Control
+Minefield
+Mojeek
+Morfeus Fucking Scanner
+Mozlila
 Mr.4x3
-Msrabot
+Musobot
 NICErsPRO
 NPbot
-Nameprotect
-Needle
 Nessus
-NetAnts
-NetLyzer
-NetMechanic
-NetSpider
 Netvibes
-NextGenSearchBot
-NimbleCrawler
+Nimbostratus
+Nmap
+Not
+Nuclei
+Nutch
 Offline Navigator
 OpenLinkProfiler
 OpenVAS
@@ -119,132 +123,128 @@ Openfind
 Openvas
 OrangeBot
 OrangeSpider
-OutfoxBot
-POE-Component-Client-HTTP
-PageGrabber
-PageThing.com
+OutclicksBot
+PECL::HTTP
+PHPCrawl
+Page Analyzer
+Pandalytics
 Papa Foto
-Pavuk
-PeoplePal
-Picsearch
+Petalbot
+Picscout
+PictureFinder
+Piepmatz
 Pockey
 ProPowerBot
+ProWebWalker
+Probethenet
 Pu_iN
+PxBroker
 PyCurl
+QueryN Metasearch
 Quick-Crawler
-RSSingBot
 RankActive
+RankActiveLinkBot
 RankingBot
-RankurBot
+RankingBot2
+Rankivabot
 RealDownload
 Recorder
-RepoMonkey
-RocketCrawler
+RedesScrapy
 SEOkicks-Robot
-SEOstats
+SEOlyticsCrawler
+SEOprofiler
 SMTBot
+ScanAlert
 Scrapy
 Screaming
 ScreenerBot
-SearchmetricsBot
-SentiBot
+SemanticJuice
 SeoSiteCheckup
-Shodan
-Siphon
-Site Sucker
 SiteSnagger
-SiteSucker
-SmartDownload
+Sitevigil
 Snake
 Snapbot
-SocialRankIOBot
-Sosospider
-SpaceBison
+Snoopy
+Sogou web spider
 Spammen
-Spanner
-Spinn3r
-SputnikBot
+Spbot
+Sqworm
 Steeler
-Stripper
+Sucker
+Sucuri
+SuperHTTP
 Surfbot
 SurveyBot
-Swiftbot
+Suzuran
 Szukacz
-TheNomad
+T0PHackTeam
+T8Abot
+Teleport
+Telesphorep
+The Intraformant
+Thumbor
+Titan
+Toata
 Toweyabot
-True_Robot
+Trendictionbot
 Turingos
 Turnitin
 TwengaBot
-UnisterBot
-Upflow
+URLy.Warning
 VCI
-Vacuum
-VidibleScraper
+Vagabondo
+VelenPublicWebCrawler
+VeriCiteCrawler
 Voil
-Voltron
+WEBDAV
 WISENutbot
-WWW-Collector-E
-WWW-Mechanize
-WWWOFFLE
 Wallpapers
-WallpapersHD
+WeSEE
 Web Enhancer
 Web Fetch
-Web Fuck
-Web Pix
-Web Sucker
 WebFetch
-WebPix
+WebLeacher
+WebReaper
 WebSauger
-WebSucker
-Webalta
+WebStripper
+WebWhacker
+WebmasterWorldForumBot
 Website Quester
-WebsiteExtractor
-Webster
 Whack
-Who.is Bot
-Widow
+Whatweb
 WinHTTrack
-Wonderbot
+WiseGuys Robot
 Woobot
 Wprecon
-Xenu
 YoudaoBot
 Zauba
-Zitebot
+Zermelo
+ZmEu
 ZoomBot
-ZoominfoBot
-ZumBot
-adscanner
-archive.org_bot
-backlink-check
-clark-crawler
-coccocbot-web
-crawl.sogou.com
-crawler.feedback
-crawler4j
-domainsproject.org
-eCatch
-evc-batch
+autoemailspider
+cah.io.community
+demandbase-bot
 facebookscraper
 gopher
 heritrix
-instabid
 internetVista monitor
 lwp-trivial
+magpie-crawler
+meanpathbot
 muhstik-scan
 netEstate NE Crawler
-oBot
-polaris version
-probe-image-size
-s1z.ru
-scan.lol
-seocompany.store
+page scorer
+pcBrowser
+ripz
+scalaj-http
+seoscanners
 seostar
-spyfu
+sexsearcher
+sitechecker.pro
+siteripz
+sp_auditbot
 tAkeOut
-webpros.com
+webgains-bot
+webmeup-crawler
 webprosbot
 x09Mozilla
-zgrab