V4.2021.09.2587 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-ADmantX
-ALittle Client
 ASPSeek
-Abonti
 Aboundex
-Aboundexbot
+AhrefsBot
 AiHitBot
 Aipbot
-Alligator
 AlphaBot
-Anarchie
-Ankit
+Apexoo
+Aspiegel
 Asterias
 Attach
+AwarioRssBot
+AwarioSmartBot
+BBBike
 BDCbot
-BDFetch
+BLEXBot
+BackStreet
+BackWeb
+Backlink-Ceck
 Badass
+BatchFTP
+Battleztar Bazinga
+BetaBot
+Bigfoot
+Bitacle
 Black Hole
 BlowFish
-Boardreader
-Bolt
+Brandprotect
 Buck
-BuiltWith
+Buddy
 CATExplorador
-CSHttp
+CODE87
 Cegbfeieh
-CheTeam
-CheeseBot
 Chlooe
+Claritybot
 Cliqzbot
-Cloud mapping
+Cocolyzebot
 Collector
-CopyRightCheck
-Cosmos
-Custo
+Craftbot
+Crawling at Home Project
 CyotekWebCopy
+DBLBot
 DIIbot
+DSearch
 DTS Agent
 DatabaseDriverMysqli
+Demon
+Deusu
 Digincore
-Dispatch
-DnyzBot
+Dirbuster
 DomainAppender
-DomainSigmaCrawler
+DomainStatsBot
 Domains Project
-Dragonfly
+Dotbot
 ECCP/1.0
-Ebingbong
-Evil
-Express WebPictures
-EyeNetIE
+EasyDL
+EirGrabber
+Exabot
+ExtLinksBot
+Extractor
+Extreme Picture Finder
 Firefox/7.0
 FlashGet
+Foobot
+Freeuploader
 FrontPage
-FyberSpider
+Fyrebot
 G-i-g-a-b-o-t
-Genieo
-GermCrawler
+GT::WWW
 GetRight
+Getintent
+GoZilla
 Gotit
-Grafula
+GrapeFX
+GrapeshotCrawler
 GridBot
-HMView
-Haansoft
+HTMLparser
+HTTP::Lite
+HTTrack
+Humanlinks
 HybridBot
-IDBTE4M
 IDBot
+IRLbot
 Iblog
-Image Fetch
-Image Sucker
-Indy Library
+Id-search
+InfoNaviRobot
 Intelliseek
 Iria
-IstellaBot
 JOC Web Spider
 Jbrofuzz
-JikeSpider
-JustView
+JennyBot
+JetCar
 Kenjin Spider
 Keyword Density
+LNSpiderguy
 Lanshanbot
-Larbin
 LeechFTP
-LexiBot
-Libwhisker
-Lightspeedsystems
-LinkScan
-LinkWalker
+LibWeb
+LinkextractorPro
+LinkpadBot
 LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
-Ltx71
-MIIxpc
-MJ12bot
-MQQBrowser
-MSFrontPage
-MSIECrawler
+Lipperhey Spider
+Mag-Net
 Magnet
-Mail.RU_Bot
-Majestic SEO
-Majestic-SEO
+Majestic12
+MarkMonitor
+MarkWatch
+Mass Downloader
+Masscan
 MauiBot
+Mb2345Browser
+MeanPath Bot
+Meanpathbot
+Mediatoolkitbot
 Metauri
-Microsoft URL Control
-Mozlila
-Mr.4x3
+Mister PiX
 Msrabot
-Navroad
-Needle
-Net Vampire
+Musobot
+NPbot
+Nameprotect
 NetAnts
-NetMechanic
 NetSpider
-Nettrack
-Niki-bot
-Nikto
-Ninja
-OpenLinkProfiler
+NextGenSearchBot
+NimbleCrawler
+Nuclei
+Nutch
+Offline Navigator
 Openfind
-OrangeBot
-OrangeSpider
 OutclicksBot
-Page Analyzer
-PageGrabber
-PageScorer
+PageThing.com
 Pandalytics
-Pavuk
+Panscient
+Papa Foto
 Petalbot
-Picscout
-Pixray
-Pockey
+Pi-Monster
+PictureFinder
+PleaseCrawl
 ProPowerBot
-Probethenet
-Psbot
-Pump
+ProWebWalker
 PxBroker
 QueryN Metasearch
-Quick-Crawler
-RSSingBot
+RankActive
+RankActiveLinkBot
+RankingBot
 RankingBot2
+Rankivabot
+Re-re
 ReGet
+RealDownload
+Reaper
 RebelMouse
 Recorder
-Rogerbot
-SBIder
-SEOkicks
+Ripper
 SEOlyticsCrawler
-SEOprofiler
+SEOstats
+SISTRIX
+SalesIntelligent
 ScanAlert
-ScoutJet
+Screaming
 ScreenerBot
-SearchmetricsBot
+ScrepyBot
+Searchestate
 Semrush
-SemrushBot
-Shodan
-Site Sucker
 SiteCheckerBotCrawler
+SiteExplorer
 SiteLockSpider
-SiteSnagger
 Sitevigil
+SmartDownload
 Snapbot
-SocialRankIOBot
-Sogou web spider
+Sociscraper
+Sosospider
 SpaceBison
 Spammen
 Spanner
-Spbot
-SputnikBot
 Sqlmap
-Sqlworm
-Steeler
-Stripper
-Sucker
+Sucuri
 SuperBot
+SuperHTTP
 Surfbot
 SurveyBot
-Suzuran
-Swiftbot
+T0PHackTeam
 T8Abot
-Telesphoreo
-TheNomad
 Thumbor
-Titan
-Toata
 Tracemyfile
+Trendictionbot
 True_Robot
 Turnitin
+TurnitinBot
 TwengaBot
 Twice
+Typhoeus
+URLy.Warning
 UnisterBot
-V-BOT
+Vacuum
+Vagabondo
 VeriCiteCrawler
-VoidEYE
+VidibleScraper
 Voil
-Voltron
-WASALive-Bot
-WBSearchBot
-WEBDAV
-WISENutbot
 WPScan
+WWW-Collector-E
 WWW-Mechanize
-Wallpapers/3.0
-Web Auto
-Web Collage
+Wallpapers
+WallpapersHD
 Web Pix
-WebBandit
-WebCollage
+WebAuto
 WebFetch
-WebGo IS
-WebLeacher
-WebSucker
-Webalta
+WebPix
+WebSauger
 WebmasterWorldForumBot
-Website Quester
+Webshag
 WebsiteQuester
+Whacker
+Whatweb
 Who.is Bot
+WinHTTrack
+WiseGuys Robot
 Wonderbot
 Woobot
 Wotbox
-Xenu
+Zade
+Zauba
+Zeus
+Zitebot
 ZmEu
 ZumBot
+ZyBorg
+archive.org_bot
 arquivo.pt
-autoemailspider
-cognitiveseo
+backlink-check
+clark-crawler
 com.plumanalytics
-dataforseo.com
-demandbase-bot
+eCatch
 facebookscraper
-instabid
-internetVista monitor
+ips-agent
 isitwp.com
-magpie-crawler
-page scorer
-pcBrowser
+mediawords
+muhstik-scan
+plumanalytics
 probe-image-size
-ripz
-s1z.ru
-satoristudio.net
-scalaj-http
-seostar
-serpstatbot
+sitechecker.pro
 siteripz
-sogouspider
 sp_auditbot
+spyfu
 sysscan
+tAkeOut
+trendiction.com
+trendiction.de
 webmeup-crawler
-xpymep1.exe
-zauba.io
+webprosbot
 zgrab