V4.2022.11.3446 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
-404enemy
 ADmantX
 Abonti
+Aboundex
 Aboundexbot
 AfD-Verbotsverfahren
 AiHitBot
-AllSubmitter
-Alligator
+Aipbot
+AlphaBot
 Anarchie
-Anarchy
-Anarchy99
-Anthill
+Ankit
 Aspiegel
-Asterias
-Attach
-AwarioRssBot
-BBBike
-BDFetch
-BackStreet
+Atomseobot
 BackWeb
-Bandit
 Battleztar Bazinga
 Bigfoot
+Black Hole
 Blow
 BlowFish
+Boardreader
+BotALot
 Brandwatch
-Buck
 Buddy
-BunnySlippers
+BuiltBotTough
 BuzzSumo
 CATExplorador
 CCBot
-CSHttp
+CODE87
+Calculon
+CazoodleBot
 Cegbfeieh
-CheTeam
-CheeseBot
-CherryPicker
-Claritybot
+ChinaClaw
+Chlooe
+Citoid
+Cloud mapping
 Cocolyzebot
-Cogentbot
-Collector
-CopyRightCheck
-Copyscape
-Craftbot
-Crescent
+Cosmos
+Crawling at Home Project
 CrunchBot
-Curious
-Custo
-DSearch
-DigitalPebble
+DataCha0s
+Deusu
+Digincore
+Dirbuster
 Discobot
-Dispatch
-DittoSpyder
 DnBCrawler-Analytics
-DnyzBot
+DomCopBot
+DomainAppender
+DomainCrawler
+DomainSigmaCrawler
+DomainStatsBot
+Domains Project
 Download Wonder
-Drip
-EMail Wolf
-Ebingbong
+Dragonfly
+EasyDL
 Ecxi
-Evil
-Express WebPictures
+EirGrabber
+Exabot
+ExtLinksBot
+EyeNetIE
 FemtosearchBot
 Flunky
 Foobot
-GetRight
+Fyrebot
+G-i-g-a-b-o-t
+GT::WWW
+GetWeb
 Gigabot
+Go-Ahead-Got-It
 Gotit
-Grabber
+Grafula
+GrapeFX
 GridBot
-HMView
-Haansoft
-HaosouSpider
-Harvest
+Havij
+Heritrix
 HonoluluBot
+Humanlinks
+HybridBot
+IDBTE4M
 IDBot
-Image Fetch
-IndeedBot
+InfoNaviRobot
 Intelliseek
 Internet Ninja
-InternetSeer
-JennyBot
-Jetty
+Iskanie
+IstellaBot
+JOC Web Spider
+Jbrofuzz
+JetCar
+JikeSpider
+Jorgee
 Keybot Translation-Search-Machine
 Keyword Density
-Kinza
-Kozmosbot
 LWP::Simple
-Larbin
-Libwhisker
-LieBaoFast
+LeechFTP
+LeechGet
+Lftp
 Lightspeedsystems
-LinkextractorPro
+LinkWalker
+LinkpadBot
 LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-LinqiaScrapeBot
+Lipperhey
 Lipperhey Spider
-MFC_Tear_Sample
-MIIxpc
-MJ12bot
-MSFrontPage
-MTRobot
-Magnet
-Mail.RU_Bot
-MarkWatch
+Lmspider
+MSIECrawler
+Mag-Net
+Masscan
+MauiBot
 Mb2345Browser
-Meanpathbot
-Microsoft URL Control
+MeanPath Bot
+MegaIndex.ru
 Mister PiX
 Moblie Safari
+Mojeek
+Mojolicious
 MolokaiBot
-Mr.4x3
-Nameprotect
-NearSite
-Nessus
-NetLyzer
+Morfeus Fucking Scanner
+Mozlila
+Msrabot
+Needle
 NetMechanic
-Nettrack
+NetSpider
+Netcraft
 NextGenSearchBot
-Nibbler
+Niki-bot
+Nikto
 NimbleCrawler
-Nutch
-OpenLinkProfiler
+Nmap
+Not
+Offline Explorer
+Offline Navigator
 Openfind
-OutclicksBot
-Page Analyzer
-PageGrabber
-Pandalytics
+PageScorer
+Panscient
 Papa Foto
-Pavuk
-Pi-Monster
 Picscout
 Picsearch
+PictureFinder
 Piepmatz
-Pixray
-ProWebWalker
+Pimonster
 Probethenet
+Psbot
+Pump
 PxBroker
-RankFlex
+PyCurl
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
+RankActive
+RankActiveLinkBot
+RankingBot2
+Rankivabot
+RankurBot
+Re-re
+RealDownload
+Reaper
+Recorder
 RedesScrapy
 RepoMonkey
-RocketCrawler
 Rogerbot
 SBIder
-SEOstats
-SISTRIX
-SMTBot
-Scanbot
-Scrapy
-Searchestate
+SEOkicks
+SEOlyticsCrawler
+SEOprofiler
+ScanAlert
+ScoutJet
+ScreenerBot
+ScrepyBot
+SemanticJuice
 SemrushBot
-SeobilityBot
-Shodan
-Siphon
-Site Sucker
-SiteCheckerBotCrawler
-SiteExplorer
-SiteLockSpider
-SiteSnagger
+Seomoz
 SiteSucker
-Sitebeam
-Siteimprove
-SmartDownload
-Snake
-Snoopy
-Spanner
-Spbot
+SlySearch
+Sogou web spider
+Spammen
 SputnikBot
-Stripper
-Sucker
+Sqlmap
+Steeler
 Sucuri
 SuperBot
-SuperHTTP
-Suzuran
+Swiftbot
 Szukacz
-Telesphoreo
-TheNomad
-Thumbor
-TightTwatBot
+Telesoft
+Titan
+Toata
+Toweyabot
 Trendiction
+Trendictionbot
+True_Robot
 Turnitin
-TwengaBot
-URLy Warning
+Twice
+URLy.Warning
+UnisterBot
+Upflow
 V-BOT
-VCI
-Vacuum
-Vagabondo
+VB Project
+Virusdie
 VoidEYE
 Voltron
 WEBDAV
-WISENutbot
-WWW::Mechanize
-WeSEE
-Web Enhancer
+WWW-Collector-E
+WWW-Mechanize
+WWWOFFLE
+Wallpapers
+WallpapersHD
+Web Collage
+Web Fetch
 Web Fuck
-Web Pix
 Web Sucker
-WebBandit
-WebCollage
+WebAuto
 WebEnhancer
-WebFuck
-WebGo IS
 WebLeacher
+WebPix
 WebStripper
 WebWhacker
-Webshag
-Who.is Bot
+WebsiteQuester
+Whacker
+Whatweb
 Wonderbot
-Woobot
 Wotbox
-Wprecon
 Xaldon WebSpider
-Xenu
+Xaldon_WebSpider
+YoudaoBot
+Zade
+Zitebot
 ZoominfoBot
-ZyBorg
-archive.org_bot
+arquivo-web-crawler
 arquivo.pt
 cah.io.community
-check1.exe
-clark-crawler
-crawl.sogou.com
-dataforseo.com
-domainsproject.org
-eCatch
-evc-batch
-instabid
+crawler.feedback
+crawler4j
+facebookscraper
+gopher
 ips-agent
-isitwp.com
 linkdexbot
+lwp-trivial
 magpie-crawler
 muhstik-scan
+netEstate NE Crawler
+page scorer
 pcBrowser
 polaris version
 satoristudio.net
+scalaj-http
 seobility
-serpstatbot
-sexsearcher
-sp_auditbot
-spyfu
-sysscan
+seostar
 tAkeOut
+trendiction.de
 voyagerx.com
-webmeup-crawler
-x09Mozilla
-x22Mozilla
-xpymep1.exe
+webprosbot
+zauba.io
 zgrab