V4.2021.08.2527 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
+01h4x.com
 404enemy
-AIBOT
+Aboundex
 Aboundexbot
-Aipbot
+AhrefsBot
 AllSubmitter
 Alligator
-Anarchie
-Anarchy
-Anarchy99
+AlphaBot
+Ankit
 Anthill
 Aspiegel
-BDCbot
+BBBike
+BLEXBot
+BackDoorBot
+BackStreet
 BackWeb
-Badass
+Backlink-Ceck
+Battleztar Bazinga
 BetaBot
 Bigfoot
-Bitacle
-Black Hole
 BlackWidow
-Bolt
-BotALot
-Brandprotect
-Brandwatch
-Buck
-Buddy
+BuiltBotTough
+BuiltWith
 CATExplorador
-CCBot
-Calculon
+CODE87
 CazoodleBot
-CensysInspect
-CherryPicker
+Cegbfeieh
+CheTeam
+CheeseBot
 ChinaClaw
-Claritybot
-Cloud mapping
-Copier
-CopyRightCheck
+Chlooe
+Cocolyzebot
+Collector
+Cosmos
+Craftbot
 CrunchBot
-Curious
-Custo
-DBLBot
-DIIbot
+DSearch
+DTS Agent
 DataCha0s
-DatabaseDriverMysqli
-Demon
-DigitalPebble
+Deusu
+Digincore
 Dirbuster
 Discobot
 DomCopBot
-DomainAppender
+DomainCrawler
 DomainStatsBot
-Download Wonder
-Dragonfly
-EasyDL
+Domains Project
+EMail Wolf
+EirGrabber
 Express WebPictures
-ExtLinksBot
 ExtractorPro
-EyeNetIE
+Ezooms
+FHscan
 FemtosearchBot
-Firefox/7.0
-Flunky
+FlashGet
+FyberSpider
 Fyrebot
-GalaxyBot
-GetWeb
-Gigablast
+GT::WWW
+GetRight
+Getintent
 Go!Zilla
-Go-Ahead-Got-It
+GoZilla
+Gotit
+GrabNet
+Grabber
+Grafula
 HEADMasterSEO
 HMView
-HTTP::Lite
-HTTrack
-Haansoft
+HTMLparser
 HaosouSpider
-Heritrix
-Id-search
-Image Sucker
-InfoTekies
+Havij
+Humanlinks
+HybridBot
+IRLbot
+Iblog
+IndeedBot
+InfoNaviRobot
 Internet Ninja
-Iria
-JamesBOT
+Iskanie
+JOC Web Spider
 Jbrofuzz
 JennyBot
-JikeSpider
+JetCar
+Joomla
 Jorgee
-Kenjin Spider
+JustView
+Kinza
 LNSpiderguy
-Lanshanbot
-LeechFTP
 LeechGet
+LexiBot
 Lftp
-LibWeb
-Libwhisker
-Lightspeedsystems
 LinkWalker
 Linkbot
+LinkpadBot
+LinksManager
+LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-LinqiaScrapeBot
-Lmspider
-Ltx71
-MFC_Tear_Sample
-MIDown tool
-MQQBrowser
-MTRobot
+MJ12bot
+MSFrontPage
+Mag-Net
 Magnet
-Majestic-SEO
-Majestic12
-MarkMonitor
 MarkWatch
-Mass Downloader
 MauiBot
 Mb2345Browser
 MeanPath Bot
-MicroMessenger
-Microsoft URL Control
-Mister PiX
+Microsoft Data Access
 Mojeek
-NICErsPRO
-NearSite
-Needle
+Mojolicious
+Mozlila
+Mr.4x3
+Musobot
 Nessus
-Net Vampire
-NetMechanic
-NextGenSearchBot
+NetAnts
+NetSpider
+Netcraft
+Netvibes
+Nibbler
+Niki-bot
 Nikto
 NimbleCrawler
+Nimbostratus
 Ninja
-Octopus
+Nmap
+Nutch
 Offline Navigator
-Openvas
+OpenLinkProfiler
+Openfind
 OrangeBot
-OutfoxBot
-PHPCrawl
-PeoplePal
-Piepmatz
-Pockey
-Probethenet
-Pu_iN
+OrangeSpider
+PECL::HTTP
+POE-Component-Client-HTTP
+PageAnalyzer
+PageScorer
+PageThing.com
+Picsearch
+Pixray
+PleaseCrawl
+ProPowerBot
+ProWebWalker
+Psbot
 Pump
 QueryN Metasearch
 Quick-Crawler
-RSSingBot
-RankActiveLinkBot
+RankFlex
 RankingBot
 Rankivabot
+RankurBot
 RealDownload
-RebelMouse
 RedesScrapy
 Ripper
 Rogerbot
-SBIder
-SEOkicks
-SEOlyticsCrawler
-SEOprofiler
+SEOkicks-Robot
 SEOstats
-SISTRIX
-ScoutJet
+SMTBot
+ScanAlert
+Scrapy
 Screaming
 ScrepyBot
-Searchestate
-SearchmetricsBot
-Semrush
-SemrushBot
-Seomoz
+SentiBot
+SeoSiteCheckup
+SeobilityBot
 Shodan
-Siphon
-SiteExplorer
-SiteSucker
-Siteimprove
-SlySearch
-SocialRankIOBot
-Sociscraper
-Sottopop
-Spinn3r
+Site Sucker
+SiteCheckerBotCrawler
+Sitebeam
+SmartDownload
+Sogou web spider
+Sosospider
+Spanner
+Spbot
+Sqlmap
+Sqlworm
 Sqworm
-Steeler
+Stripper
 SuperBot
-SurveyBot
+Surfbot
+Swiftbot
 T0PHackTeam
+Telesoft
 Telesphoreo
-TheNomad
+The Intraformant
 Thumbor
 TightTwatBot
-Toata
+Tracemyfile
+Trendictionbot
 True_Robot
 Turingos
-Turnitin
 TurnitinBot
 TwengaBot
-URLy Warning
 URLy.Warning
-UnisterBot
-VB Project
-VCI
+Upflow
 Vacuum
 VeriCiteCrawler
-VoidEYE
-Voil
+VidibleScraper
 WASALive-Bot
-WISENutbot
-Wallpapers
+WWW-Collector-E
+WWW::Mechanize
+WWWOFFLE
+WeSEE
 Web Collage
-WebCollage
+Web Enhancer
+Web Fuck
+Web Pix
+Web Sauger
+Web Sucker
 WebCopier
-WebEnhancer
+WebFetch
 WebGo IS
-WebLeacher
-WebPix
-WebReaper
 WebStripper
-WebSucker
+WebWhacker
 Webalta
+WebmasterWorldForumBot
 WebsiteExtractor
-WebsiteQuester
 Whack
-Widow
-WinHTTrack
-Wotbox
-Xaldon_WebSpider
-Xenu
+Whacker
+Wonderbot
+Xaldon WebSpider
+Zade
+Zermelo
+Zeus
 ZoomBot
 ZumBot
 ZyBorg
 arquivo.pt
 backlink-check
-clark-crawler
-coccocbot-web
-cognitiveseo
-com.plumanalytics
-crawl.sogou.com
-crawler4j
 demandbase-bot
-domainsproject.org
-eCatch
-facebookscraper
+evc-batch
 gopher
+heritrix
+instabid
 internetVista monitor
+lwp-trivial
 meanpathbot
+mediawords
 netEstate NE Crawler
-page scorer
+pcBrowser
 plumanalytics
-polaris version
+probe-image-size
+satoristudio.net
 scan.lol
+sexsearcher
+sogouspider
+sp_auditbot
+sysscan
 tAkeOut
-trendiction.de
+ubermetrics-technologies.com
 voyagerx.com
 webmeup-crawler
-webpros.com
-x22Mozilla
-xpymep1.exe
+zauba.io
 zgrab