V4.2021.07.2476 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 360Spider
 404enemy
-80legs
+ADmantX
 AIBOT
 Aboundexbot
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
 Aipbot
 Alligator
 AlphaBot
 Anarchie
-Anarchy99
-Apexoo
-Asterias
+Ankit
+Aspiegel
+Attach
 AwarioSmartBot
-BDFetch
-BLEXBot
+BBBike
 BackStreet
-BackWeb
-Backlink-Ceck
 BacklinkCrawler
-Bandit
-Bitacle
-BotALot
+Barkrowler
+Battleztar Bazinga
+BetaBot
+Bigfoot
+BlowFish
 Brandprotect
-Buddy
-CCBot
-CSHttp
+Brandwatch
+Buck
+Bullseye
+BuzzSumo
+CATExplorador
 CazoodleBot
-Cegbfeieh
-CheeseBot
-CherryPicker
-Chlooe
-Cliqzbot
+CheTeam
+ChinaClaw
 Cloud mapping
-Cocolyzebot
-Collector
-Cosmos
-CrazyWebCrawler
-DBLBot
-DIIbot
+Crescent
+Curious
+CyotekWebCopy
+DSearch
 DataCha0s
-Devil
-Digincore
-Discoverybot
+DatabaseDriverMysqli
+Dirbuster
+Disco
+Discobot
 Dispatch
 DnyzBot
 DomCopBot
 DomainAppender
-DomainSigmaCrawler
+DomainStatsBot
 Domains Project
 Dotbot
-Dragonfly
-ECCP/1.0
-EMail Wolf
+Download Wonder
+EasyDL
+Ebingbong
 Ecxi
 EirGrabber
-EroCrawler
 Exabot
-ExtractorPro
-Fuzz
+Express WebPictures
+FDM
+FHscan
+FemtosearchBot
+Flunky
+Freeuploader
 FyberSpider
 Fyrebot
-GT::WWW
-GalaxyBot
 GetRight
-GetWeb
-Gigabot
+Go-Ahead-Got-It
+Gotit
+GrabNet
+Grabber
+Grafula
 GrapeFX
-GrapeshotCrawler
+GridBot
 HEADMasterSEO
-Haansoft
-Hloader
+HTTP::Lite
+HaosouSpider
+Heritrix
+Humanlinks
 IRLbot
-Iblog
-Image Sucker
-Indy Library
-InfoNaviRobot
-InfoTekies
+Id-search
+Image Fetch
+Intelliseek
 InterGET
+Internet Ninja
 InternetSeer
-Iskanie
+Iria
 IstellaBot
-JOC Web Spider
 Jbrofuzz
 Jetty
+Joomla
+Jorgee
 JustView
-Kenjin Spider
-Keyword Density
-Kinza
+LWP::Simple
 Lanshanbot
-LexiBot
-LibWeb
-Lightspeedsystems
+Larbin
+LeechGet
+Libwhisker
 Likse
+LinkScan
 Linkbot
 Linkdexbot
 LinkpadBot
 LinksManager
+LinqiaRSSBot
 Lipperhey Spider
+Lmspider
+Ltx71
 MFC_Tear_Sample
-MJ12bot
+MIDown tool
 MSFrontPage
 MSIECrawler
-MTRobot
-Mail.RU_Bot
-Majestic SEO
+Mag-Net
 Majestic-SEO
-Majestic12
-MarkMonitor
+MarkWatch
 Masscan
-Mb2345Browser
-MeanPath Bot
-MegaIndex.ru
-Metauri
-MicroMessenger
-Microsoft Data Access
+MauiBot
+Meanpathbot
+Mister PiX
+Mojolicious
 Morfeus Fucking Scanner
-Mozlila
 Msrabot
+Musobot
 Name Intelligence
 Nameprotect
 Navroad
-NearSite
+Needle
 Net Vampire
 NetMechanic
-NetSpider
-NetZIP
-Netcraft
 Nettrack
-Netvibes
 NextGenSearchBot
-Nibbler
-Niki-bot
-NimbleCrawler
-Nimbostratus
+Ninja
 Nuclei
-OPPO A33
+Nutch
+Octopus
 Offline Explorer
+OpenLinkProfiler
 Openfind
-Openvas
 OrangeSpider
-OutfoxBot
+PECL::HTTP
 PHPCrawl
-Page Analyzer
+POE-Component-Client-HTTP
+PageAnalyzer
+PageScorer
 PageThing.com
-Pi-Monster
-PictureFinder
-Pixray
-ProPowerBot
+Panscient
+Pavuk
+Picscout
+Picsearch
+Piepmatz
+Pimonster
 ProWebWalker
-Pu_iN
-QueryN Metasearch
+Probethenet
+Pump
+PyCurl
+Quick-Crawler
 RankActive
-RankActiveLinkBot
-RankingBot
-Rankivabot
+RankingBot2
+Re-re
 ReGet
 RealDownload
-RedesScrapy
-RepoMonkey
+RebelMouse
+Ripper
 Rogerbot
-SEOkicks-Robot
-SISTRIX
+SEOlyticsCrawler
 ScoutJet
+Scrapy
 ScreenerBot
-Semrush
-Site Sucker
-SiteExplorer
-SiteLockSpider
+SemrushBot
+SentiBot
+SeoSiteCheckup
+SiteCheckerBotCrawler
 SiteSucker
+Sitebeam
 Siteimprove
-Sitevigil
-Snapbot
-Sogou web spider
-Sosospider
-SpaceBison
+Snake
+Snoopy
+SocialRankIOBot
+Sottopop
 Spammen
+Spanner
 Spbot
-Sqlworm
+Sqlmap
 Sqworm
-Stripper
-Sucuri
-SuperBot
 Suzuran
-Swiftbot
-Teleport
+T0PHackTeam
+Telesoft
+Telesphorep
 The Intraformant
-TightTwatBot
-Toata
+TheNomad
+Titan
+Trendiction
+Trendictionbot
 True_Robot
 Turnitin
-Twice
-UnisterBot
+TwengaBot
+Typhoeus
+URLy.Warning
 Upflow
-VCI
-VoidEYE
-Voil
+V-BOT
+Vagabondo
+VelenPublicWebCrawler
+Voltron
 WASALive-Bot
-WISENutbot
+WEBDAV
+WWW-Collector-E
 WWW-Mechanize
+WWW::Mechanize
 WWWOFFLE
-WeSEE
-Web Fuck
+Wallpapers/3.0
+Web Auto
+Web Collage
+Web Enhancer
+Web Fetch
 Web Pix
-Web Sauger
-WebBandit
-WebCollage
-WebCopier
+Web Sucker
 WebEnhancer
-WebFetch
 WebFuck
 WebImageCollector
 WebLeacher
-WebSucker
-WebWhacker
-Webshag
-Website Quester
+WebZIP
+WebmasterWorldForumBot
 WebsiteQuester
-Widow
+Whack
 WinHTTrack
 WiseGuys Robot
-Wonderbot
 Xaldon WebSpider
-Xaldon_WebSpider
-ZoomBot
+Xenu
+Zeus
 arquivo-web-crawler
 arquivo.pt
-autoemailspider
+crawl.sogou.com
 crawler.feedback
-gopher
-internetVista monitor
+crawler4j
+evc-batch
+facebookscraper
+heritrix
+instabid
+isitwp.com
 lwp-request
-lwp-trivial
-mediawords
-pcBrowser
+magpie-crawler
+page scorer
 plumanalytics
+polaris version
 ripz
-s1z.ru
 scalaj-http
-seobility
-seoscanners
 seostar
+sitechecker.pro
 siteripz
+sogouspider
 sp_auditbot
-trendiction.com
+spyfu
+sysscan
+xpymep1.exe
 zauba.io
-zgrab