V4.2022.01.2851 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
-404checker
 ADmantX
-AIBOT
+ALittle Client
 ASPSeek
 Aboundex
-Aboundexbot
+Acunetix
 AfD-Verbotsverfahren
 AiHitBot
-Alexibot
-AlphaBot
-Anarchie
+Aipbot
+Anarchy
 Anarchy99
-Ankit
-Attach
-AwarioRssBot
-Backlink-Ceck
+Anthill
+Apexoo
+Asterias
+BBBike
+BDCbot
+BDFetch
+BLEXBot
+BacklinkCrawler
+Barkrowler
 BatchFTP
 Battleztar Bazinga
 Bigfoot
-Bitacle
 Blackboard
-Bolt
+BlowFish
+Boardreader
+BotALot
 Brandprotect
-Buck
+Buddy
+BuiltBotTough
+BuiltWith
 Bullseye
-CODE87
-CazoodleBot
-CensysInspect
+BuzzSumo
+CSHttp
 CheTeam
-CherryPicker
-Cloud mapping
+CheeseBot
+Chlooe
+Claritybot
 Cocolyzebot
-Copier
-Copyscape
-Crawling at Home Project
-CrazyWebCrawler
-Crescent
+Collector
+CopyRightCheck
+Curious
 Custo
-DBLBot
+DSearch
 DataCha0s
+Demon
+Deusu
 Devil
 DigitalPebble
-Discoverybot
-Dispatch
+Dirbuster
+Discobot
 DittoSpyder
-DnBCrawler-Analytics
-DomCopBot
-DomainCrawler
-Download Wonder
-Dragonfly
+DomainSigmaCrawler
+Domains Project
+Dotbot
+Drip
 ECCP/1.0
-EMail Siphon
 EasyDL
-Ebingbong
-Ecxi
-EroCrawler
-Evil
 Exabot
-Extreme Picture Finder
+Extractor
+EyeNetIE
+FDM
 FHscan
 Fimap
-Freeuploader
+FlashGet
+Foobot
+FrontPage
+Fuzz
+Fyrebot
 G-i-g-a-b-o-t
+GT::WWW
 GalaxyBot
-GermCrawler
-GetWeb
 Getintent
-Gotit
+GoZilla
 Grabber
-GrapeFX
-HEADMasterSEO
-HMView
 HTTP::Lite
-Humanlinks
-HybridBot
-IDBTE4M
-IRLbot
+HTTrack
+Harvest
+Hloader
+Iblog
+Id-search
 Image Fetch
 IndeedBot
-Indy Library
 InfoNaviRobot
 Intelliseek
-Iskanie
+Internet Ninja
+InternetSeer
+Iria
 JOC Web Spider
-JamesBOT
-Jbrofuzz
-JennyBot
 Jyxobot
-LNSpiderguy
 LWP::Simple
-Leap
+Larbin
 LeechGet
 LexiBot
-LibWeb
+LieBaoFast
+Lightspeedsystems
 Likse
+LinkScan
+LinkWalker
+Linkbot
 Linkdexbot
 LinkextractorPro
+LinksManager
 LinqiaRSSBot
-Litemage_walker
-MIDown tool
-MIIxpc
-MTRobot
+Lmspider
+Ltx71
+MFC_Tear_Sample
+MJ12bot
+MSIECrawler
 Mag-Net
-Magnet
-Mail.RU_Bot
 Majestic-SEO
-Mass Downloader
-Masscan
-MeanPath Bot
+Majestic12
+MarkWatch
+Mb2345Browser
 Meanpathbot
-MegaIndex.ru
-Metauri
-MicroMessenger
 Minefield
-Mister PiX
-Moblie Safari
-Mojeek
-Morfeus Fucking Scanner
 Mr.4x3
 Msrabot
-Name Intelligence
-Nameprotect
+Musobot
 Navroad
-Needle
+NearSite
 NetAnts
-NetMechanic
-NetSpider
-NetZIP
-Nimbostratus
-Ninja
-Not
-Nutch
+NetLyzer
+Nibbler
+Nikto
+Nmap
+Nuclei
 OnCrawl
-OpenVAS
+Openfind
 Openvas
-Pavuk
-PeoplePal
+OrangeBot
+PECL::HTTP
+PHPCrawl
+Page Analyzer
+PageScorer
+PageThing.com
 Picscout
 Piepmatz
-PyCurl
-QueryN Metasearch
+Pimonster
+Pockey
+PxBroker
 Quick-Crawler
 RSSingBot
-RankActiveLinkBot
+RankFlex
 RankurBot
+ReGet
 RealDownload
 RebelMouse
-Ripper
-Rogerbot
-SEOprofiler
+Recorder
+RepoMonkey
+SBIder
+SEOkicks
 SEOstats
-SISTRIX
-ScanAlert
+SalesIntelligent
 Scanbot
-ScoutJet
-Scrapy
-Screaming
-ScreenerBot
 ScrepyBot
 SearchmetricsBot
 Seekport
-SemanticJuice
 Semrush
+SemrushBot
+SentiBot
+SeobilityBot
 Seomoz
 Shodan
-Siphon
+Site Sucker
 SiteCheckerBotCrawler
-SiteExplorer
 SiteLockSpider
-Siteimprove
-Sitevigil
-Snake
-Snoopy
-Sosospider
-SpaceBison
-Spbot
+SiteSucker
+Sitebeam
+SocialRankIOBot
+Sogou web spider
+SpankBot
+Spanner
+Spinn3r
 Sqworm
-Stripper
-SuperBot
-Surfbot
 Suzuran
-Swiftbot
 Szukacz
-Telesphoreo
+T8Abot
+TeleportPro
+Telesoft
 Telesphorep
-TheNomad
-Titan
-Tracemyfile
-Trendictionbot
-True_Robot
+The Intraformant
+Toweyabot
+Trendiction
+Turnitin
+URLy.Warning
 UnisterBot
-Upflow
 V-BOT
 VB Project
-Vacuum
-Vagabondo
-Virusdie
-WEBDAV
-WPScan
-WWW-Mechanize
-Wallpapers
+VelenPublicWebCrawler
+VeriCiteCrawler
+WBSearchBot
+WISENutbot
 Wallpapers/3.0
-WeSEE
 Web Enhancer
-Web Fetch
 Web Fuck
 Web Pix
+Web Sauger
+WebBandit
+WebCollage
 WebGo IS
-WebLeacher
+WebImageCollector
 WebWhacker
-WebmasterWorldForumBot
 Webshag
+Website Quester
 WebsiteExtractor
-Webster
+Whack
 WinHTTrack
-Wprecon
-Xaldon WebSpider
+Woobot
+Wotbox
+Xaldon_WebSpider
+Zeus
 Zitebot
+ZmEu
 ZoomBot
-ZumBot
-arquivo.pt
-cah.io.community
+ZoominfoBot
+archive.org_bot
+arquivo-web-crawler
+check1.exe
+clark-crawler
+cognitiveseo
+crawl.sogou.com
 crawler.feedback
-dataforseo.com
 domainsproject.org
 evc-batch
-facebookscraper
-heritrix
+gopher
+instabid
 ips-agent
 isitwp.com
+lwp-request
+lwp-trivial
 magpie-crawler
+meanpathbot
+mediawords
+muhstik-scan
+netEstate NE Crawler
 oBot
-page scorer
-ripz
-satoristudio.net
-scalaj-http
-scan.lol
-seobility
 seocompany.store
-seoscanners
-seostar
-siteripz
+serpstatbot
+sitechecker.pro
+sogouspider
 sp_auditbot
+sysscan
 tAkeOut
-webgains-bot
-x22Mozilla
+voyagerx.com
+webmeup-crawler
+webpros.com
+webprosbot
+x09Mozilla
 xpymep1.exe
-zgrab
+zauba.io