V4.2021.10.2640 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-ALittle Client
-Aboundex
-AhrefsBot
+404checker
+AIBOT
+Aboundexbot
+Acunetix
 AiHitBot
 Aipbot
-AlphaBot
-Anarchy
+Alligator
 Aspiegel
 Attach
 AwarioRssBot
+AwarioSmartBot
 BBBike
 BDFetch
-BLEXBot
-Backlink-Ceck
+BackWeb
 BacklinkCrawler
-Badass
 Bandit
-BlackWidow
+Barkrowler
+Bitacle
+Black Hole
+Blackboard
 BlowFish
+Boardreader
 Brandprotect
-Buck
-Buddy
-BuiltWith
+Brandwatch
+BuiltBotTough
 Bullseye
-BunnySlippers
 CATExplorador
-CODE87
-CSHttp
+CCBot
 Calculon
+CazoodleBot
 CensysInspect
-Chlooe
-Cocolyzebot
-Cogentbot
-Collector
+CheTeam
+Claritybot
+Cliqzbot
+Cloud mapping
 Copier
-CopyRightCheck
-Cosmos
+Copyscape
 Crawling at Home Project
+CrazyWebCrawler
 Crescent
-CrunchBot
 DIIbot
 DSearch
 DTS Agent
-Demon
+DatabaseDriverMysqli
 DigitalPebble
-Dirbuster
-Discobot
+Discoverybot
 Dispatch
 DnyzBot
-DomainCrawler
+DomCopBot
 DomainStatsBot
 Dotbot
+Drip
 ECCP/1.0
+EMail Siphon
+EMail Wolf
 EasyDL
-Ecxi
-Evil
-Exabot
-Extractor
-Extreme Picture Finder
-EyeNetIE
-Ezooms
-FDM
-FlashGet
-Foobot
+EroCrawler
+Firefox/7.0
 FrontPage
-FyberSpider
-G-i-g-a-b-o-t
+GT::WWW
+GalaxyBot
+Genieo
+GermCrawler
+GetWeb
+Getintent
 Go!Zilla
 GoZilla
 Grafula
 GrapeFX
-HEADMasterSEO
-HTMLparser
+GridBot
+HTTrack
 Haansoft
-Havij
-Hloader
-HybridBot
+HaosouSpider
+Harvest
 IDBTE4M
-IRLbot
 Id-search
-IlseBot
 Image Fetch
 Image Sucker
-IndeedBot
-InfoNaviRobot
+Indy Library
+Intelliseek
+InterGET
+Internet Ninja
+InternetSeer
 JOC Web Spider
 JennyBot
-JustView
-Kinza
+JetCar
+Jetty
+Joomla
+Jorgee
+Jyxobot
 Kozmosbot
-Lanshanbot
-LexiBot
-LibWeb
+LWP::Simple
+LeechFTP
+Lftp
 Libwhisker
+Lightspeedsystems
 Likse
 LinkScan
-LinkpadBot
-LinqiaRSSBot
-Lipperhey Spider
-Litemage_walker
-MSFrontPage
+LinkWalker
+Linkbot
+Linkdexbot
+LinkextractorPro
+Lmspider
+Ltx71
+MIIxpc
+MJ12bot
 Magnet
 Majestic SEO
 Majestic12
-MarkWatch
+MarkMonitor
+Masscan
+Mb2345Browser
 MeanPath Bot
+Meanpathbot
+Mediatoolkitbot
+Microsoft Data Access
 Microsoft URL Control
 Mister PiX
-Mr.4x3
-Musobot
+Mojeek
+Msrabot
 NICErsPRO
-Name Intelligence
-Navroad
-NearSite
-Needle
-Nessus
-Net Vampire
 NetAnts
+NetLyzer
+NetMechanic
+NetSpider
 NetZIP
-Nettrack
-Nibbler
-NimbleCrawler
-Nimbostratus
-Ninja
-Nmap
-Octopus
+Netcraft
+NextGenSearchBot
+Niki-bot
+Nikto
+Nutch
+Offline Explorer
 OpenLinkProfiler
-OrangeBot
+OpenVAS
+Openfind
 OrangeSpider
-PECL::HTTP
-POE-Component-Client-HTTP
+PHPCrawl
 Page Analyzer
-PageAnalyzer
 PageGrabber
-Papa Foto
-Picscout
+PageThing.com
+Panscient
+Pi-Monster
 Picsearch
-Probethenet
-PxBroker
-Quick-Crawler
+PictureFinder
+Piepmatz
+Pockey
+ProPowerBot
+ProWebWalker
+Pu_iN
 RSSingBot
-RankActiveLinkBot
-RankingBot2
-RealDownload
-Reaper
+RankActive
+RankFlex
+ReGet
 RebelMouse
-RedesScrapy
 Rogerbot
+SBIder
 SEOkicks-Robot
 SEOlyticsCrawler
 SEOprofiler
 SISTRIX
-SalesIntelligent
+ScanAlert
 Scanbot
-ScoutJet
-Scrapy
 Screaming
-Searchestate
-SearchmetricsBot
-SemrushBot
+ScrepyBot
+Semrush
 SentiBot
-SeoSiteCheckup
-SeobilityBot
-Shodan
-Siphon
-Sitebeam
-Snapbot
+Site Sucker
+SiteExplorer
+Siteimprove
+Sitevigil
+SlySearch
 Sociscraper
+Sottopop
 SpaceBison
-Spinn3r
+SpankBot
+Spanner
 SputnikBot
-Sqlmap
+Sqlworm
 Steeler
-Stripper
-Sucuri
-SuperBot
-Swiftbot
+Suzuran
 Szukacz
-T0PHackTeam
-Teleport
-Telesphoreo
-Telesphorep
-The Intraformant
-Thumbor
-Toata
+TightTwatBot
+Titan
+Toweyabot
 Tracemyfile
-Trendiction
-Turingos
-TwengaBot
-V-BOT
+True_Robot
+TurnitinBot
+Typhoeus
+VB Project
 VCI
-VelenPublicWebCrawler
-VeriCiteCrawler
-Voil
-WASALive-Bot
+VidibleScraper
+Virusdie
+VoidEYE
 WPScan
-WWW-Collector-E
-WallpapersHD
+WWWOFFLE
+Web Enhancer
 Web Fuck
-Web Sauger
-Web Sucker
 WebAuto
 WebBandit
 WebCollage
-WebFuck
+WebCopier
+WebEnhancer
+WebFetch
 WebGo IS
 WebImageCollector
-WebReaper
-WebSauger
-WebWhacker
-Webalta
-WebmasterWorldForumBot
-WebsiteQuester
-Whack
-Who.is Bot
-WinHTTrack
+WebPix
+WebStripper
+Webster
+Whacker
+Whatweb
 WiseGuys Robot
-Wonderbot
-Woobot
 Wotbox
-Xenu
-YoudaoBot
-Zauba
+Xaldon WebSpider
+Xaldon_WebSpider
 ZmEu
 arquivo-web-crawler
 arquivo.pt
+cah.io.community
 check1.exe
-clark-crawler
+coccocbot-web
 cognitiveseo
 crawler.feedback
-demandbase-bot
+crawler4j
+dataforseo.com
 facebookscraper
-gopher
-instabid
+heritrix
 internetVista monitor
+ips-agent
+lwp-request
+lwp-trivial
 magpie-crawler
-meanpathbot
-muhstik-scan
-plumanalytics
+mediawords
+netEstate NE Crawler
+oBot
+pcBrowser
+polaris version
 probe-image-size
-scalaj-http
-sexsearcher
+seoscanners
+serpstatbot
 sitechecker.pro
-sogouspider
+siteripz
+sp_auditbot
 spyfu
+tAkeOut
 trendiction.de
-voyagerx.com
+webmeup-crawler
+webpros.com
+x22Mozilla
 xpymep1.exe
 zauba.io