V4.2021.09.2596 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
-404enemy
-ADmantX
 AIBOT
-Abonti
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
-Alexibot
+ALittle Client
+Aboundexbot
+Acunetix
+Aipbot
 Alligator
+AlphaBot
+Anarchy
+Anarchy99
 Ankit
 Anthill
+Aspiegel
+Asterias
 Attach
 AwarioRssBot
 AwarioSmartBot
+BBBike
+BDCbot
 BLEXBot
-BackStreet
-BackWeb
-Backlink-Ceck
 BacklinkCrawler
 Badass
-Battleztar Bazinga
-BetaBot
+Bandit
+Barkrowler
+BatchFTP
 BlackWidow
-BlowFish
-Brandwatch
-BuiltWith
-CATExplorador
-CCBot
-CazoodleBot
-Cegbfeieh
+BotALot
+Buck
+BunnySlippers
+BuzzSumo
+CensysInspect
 CherryPicker
-ChinaClaw
-Claritybot
-Cocolyzebot
-Cogentbot
+Cliqzbot
+Copier
 CopyRightCheck
 Craftbot
-Curious
-Custo
-CyotekWebCopy
-DBLBot
-DIIbot
-DSearch
-DTS Agent
-Deusu
-Digincore
-Dirbuster
-Discobot
+Crawling at Home Project
+CrazyWebCrawler
+CrunchBot
+DatabaseDriverMysqli
+Demon
+Devil
+Disco
 Discoverybot
-Dispatch
-DittoSpyder
-DomCopBot
-DomainAppender
 DomainCrawler
-DomainSigmaCrawler
 DomainStatsBot
+Domains Project
 Dotbot
-Drip
+Dragonfly
 EMail Siphon
-EirGrabber
-EroCrawler
+EMail Wolf
 Evil
 Exabot
-ExtLinksBot
-Extractor
-ExtractorPro
-Extreme Picture Finder
-FDM
-FHscan
-Fimap
-Fuzz
-FyberSpider
+Express WebPictures
+EyeNetIE
+Freeuploader
+FrontPage
 Fyrebot
 GetWeb
-Getintent
-Gotit
+Gigabot
+Go!Zilla
+Grabber
 Grafula
 GrapeFX
-GrapeshotCrawler
 HMView
-Hloader
-IDBTE4M
+HTTrack
+Haansoft
+HybridBot
+IDBot
 IRLbot
-IlseBot
-Image Fetch
-Image Sucker
-InfoTekies
-Iria
+Iblog
+IndeedBot
+Indy Library
+InfoNaviRobot
+InterGET
+Internet Ninja
+Iskanie
 JOC Web Spider
-JamesBOT
-JetCar
-Joomla
+Jbrofuzz
+Jetty
+JikeSpider
+Jorgee
+Kenjin Spider
 Keyword Density
-Kozmosbot
-Lanshanbot
-LeechFTP
-LeechGet
-LexiBot
+Kinza
+LNSpiderguy
+Larbin
+Leap
 LibWeb
 Lightspeedsystems
+Likse
 LinkScan
 LinkWalker
-LinqiaMetadataDownloaderBot
+Linkbot
+LinkextractorPro
+LinkpadBot
 LinqiaScrapeBot
-MFC_Tear_Sample
-MIDown tool
-MQQBrowser
+MIIxpc
+MJ12bot
 MSFrontPage
+MTRobot
 Magnet
-Majestic SEO
+MarkWatch
+Mata Hari
 MauiBot
-MegaIndex.ru
-Microsoft Data Access
-Mister PiX
-Moblie Safari
-Mojeek
-Mojolicious
+Mozlila
 Msrabot
+Musobot
+NPbot
 Nameprotect
-Navroad
 Nessus
-Net Vampire
-NetSpider
+NetAnts
+NetLyzer
+NetMechanic
 NetZIP
 Netcraft
-Nettrack
 Netvibes
-Nibbler
-Niki-bot
+NimbleCrawler
 Nimbostratus
-Nutch
-Offline Navigator
-OpenLinkProfiler
-OutfoxBot
-POE-Component-Client-HTTP
-Pandalytics
+Ninja
+Nuclei
+OnCrawl
+OpenVAS
+Openfind
+PECL::HTTP
+PHPCrawl
+Page Analyzer
+PageAnalyzer
+PageThing.com
 Panscient
-PeoplePal
+Pavuk
+Petalbot
+Pi-Monster
 Picscout
-Piepmatz
-Pockey
-Psbot
+PictureFinder
+ProWebWalker
 Pump
-Quick-Crawler
+QueryN Metasearch
 RankActive
 RankActiveLinkBot
-RankingBot
-RankurBot
+RankFlex
+RankingBot2
+Re-re
 ReGet
 RealDownload
-Recorder
-RedesScrapy
-Rogerbot
+Ripper
+SBIder
 SEOkicks
-Scrapy
-Searchestate
+SEOkicks-Robot
+SMTBot
+Scanbot
+ScoutJet
+Screaming
+ScrepyBot
+Semrush
+SemrushBot
 SentiBot
 SeoSiteCheckup
-Site Sucker
-SiteExplorer
-SiteSnagger
+SeobilityBot
+SiteCheckerBotCrawler
 SiteSucker
-Sitebeam
 Siteimprove
-SmartDownload
+Sitevigil
+SlySearch
 Snake
+Snapbot
 Sociscraper
-Sottopop
-Spanner
-SputnikBot
-Sqlmap
+Sogou web spider
+SpankBot
+Spbot
+Spinn3r
+Sqlworm
 Steeler
+Sucuri
+SuperHTTP
 Surfbot
 SurveyBot
-T0PHackTeam
+T8Abot
+Teleport
 Telesoft
 Telesphoreo
 TheNomad
 Thumbor
-Toata
-Tracemyfile
-True_Robot
-TurnitinBot
+TightTwatBot
+Trendiction
+Trendictionbot
+Turingos
+Turnitin
+TwengaBot
+Twice
+Typhoeus
+URLy.Warning
+Upflow
+V-BOT
 VB Project
-VCI
-Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
-WBSearchBot
-WPScan
+Vacuum
+Voil
+Voltron
+WASALive-Bot
 WWW-Collector-E
-WWW-Mechanize
+WWW::Mechanize
 Wallpapers
-Wallpapers/3.0
-WallpapersHD
-Web Enhancer
-WebBandit
-WebEnhancer
-WebImageCollector
+WeSEE
+Web Auto
+Web Collage
+Web Fetch
+Web Fuck
+Web Sauger
+Web Sucker
+WebAuto
+WebCopier
+WebFuck
 WebLeacher
+WebPix
 WebReaper
-WebStripper
-WebSucker
-WebZIP
-Website Quester
-Webster
-Whacker
 Whatweb
-Who.is Bot
-WinHTTrack
 WiseGuys Robot
 Wonderbot
 Woobot
-Wotbox
-Xaldon WebSpider
-Xaldon_WebSpider
+Xenu
+YoudaoBot
+Zade
+Zermelo
 Zitebot
+ZmEu
 ZoomBot
-ZoominfoBot
-arquivo-web-crawler
-backlink-check
+ZyBorg
 coccocbot-web
-com.plumanalytics
-dataforseo.com
 evc-batch
 gopher
-heritrix
-instabid
-lwp-request
-lwp-trivial
-mediawords
+internetVista monitor
+magpie-crawler
+meanpathbot
+muhstik-scan
+netEstate NE Crawler
+oBot
+polaris version
 probe-image-size
-ripz
 scalaj-http
-seostar
+seobility
+seoscanners
 serpstatbot
-siteripz
+sexsearcher
+sogouspider
 tAkeOut
 trendiction.com
 ubermetrics-technologies.com
-voyagerx.com
 webmeup-crawler
-webpros.com
-webprosbot
 x22Mozilla
-xpymep1.exe
 zauba.io
-zgrab