V4.2022.01.2829 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-404checker
+80legs
 ALittle Client
-Aboundex
+ASPSeek
+Abonti
 Acunetix
-Anarchy
+AfD-Verbotsverfahren
+AiHitBot
 Anarchy99
-Ankit
-AwarioSmartBot
-BLEXBot
-BackDoorBot
-BackWeb
+BDCbot
+BDFetch
+BackStreet
 Backlink-Ceck
 Bandit
 Barkrowler
-BatchFTP
-BetaBot
+Battleztar Bazinga
 Bigfoot
+Bitacle
 Black Hole
-BlackWidow
+BlowFish
 Boardreader
-Brandprotect
+Bolt
 Brandwatch
 Buck
 BuiltBotTough
-Bullseye
-Cegbfeieh
-CheeseBot
+BuiltWith
+CATExplorador
+CheTeam
+ChinaClaw
 Chlooe
-Citoid
-Claritybot
-Cloud mapping
-Collector
+Cliqzbot
+Cocolyzebot
+Cogentbot
 Copier
 Copyscape
-Cosmos
 Craftbot
-Crawling at Home Project
-Crescent
 CrunchBot
-DIIbot
-DSearch
-DatabaseDriverMysqli
+DTS Agent
+Deusu
 Devil
-Digincore
+DigitalPebble
+Dirbuster
 Discoverybot
-Dispatch
-DittoSpyder
-DnBCrawler-Analytics
-DomainCrawler
-DomainSigmaCrawler
-DomainStatsBot
+DomainAppender
+Dotbot
 Download Wonder
-EirGrabber
-Evil
-Exabot
+EMail Siphon
+EMail Wolf
+EasyDL
+Ebingbong
+EroCrawler
 ExtLinksBot
-Extreme Picture Finder
-Ezooms
-FemtosearchBot
-Fimap
+FHscan
 FlashGet
-Flunky
 Foobot
-Fuzz
+FrontPage
 FyberSpider
 G-i-g-a-b-o-t
-Getintent
-Grabber
-HMView
-Havij
-Hloader
-Iblog
+GetRight
+Go-Ahead-Got-It
+GrapeFX
+HTMLparser
+HaosouSpider
+IRLbot
+IndeedBot
 InfoNaviRobot
 InterGET
-Iskanie
 JOC Web Spider
+JennyBot
+JetCar
 Jetty
-JikeSpider
-Joomla
 Jorgee
-Jyxobot
+JustView
 Kenjin Spider
-Keyword Density
+Kinza
 LNSpiderguy
+LWP::Simple
 Lanshanbot
-Larbin
 LibWeb
 Lightspeedsystems
-LinkScan
+Likse
 LinkWalker
 Linkbot
-Linkdexbot
-LinkextractorPro
 LinkpadBot
-LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-Lipperhey
-Lipperhey Spider
-Litemage_walker
+LinqiaScrapeBot
 Lmspider
 MFC_Tear_Sample
-MSFrontPage
-MSIECrawler
-MTRobot
-Majestic SEO
+MQQBrowser
+Mag-Net
+Mail.RU_Bot
 Majestic-SEO
 Majestic12
-MarkMonitor
-MarkWatch
+Masscan
 Mata Hari
 Mb2345Browser
+MeanPath Bot
 Meanpathbot
 Mediatoolkitbot
-MegaIndex.ru
-Metauri
-MicroMessenger
 Microsoft Data Access
 Minefield
+Mister PiX
 Moblie Safari
-Msrabot
-Musobot
+Mojolicious
+Mozlila
+Mr.4x3
 Name Intelligence
-Netcraft
+NearSite
+Nessus
+NetLyzer
+NetZIP
+Nettrack
+Netvibes
+NextGenSearchBot
+Nibbler
+Niki-bot
+NimbleCrawler
+Ninja
 Nutch
-OpenLinkProfiler
+Offline Navigator
+OpenVAS
+Openfind
 Openvas
 OrangeBot
+PECL::HTTP
 PHPCrawl
-Page Analyzer
-PageAnalyzer
-PageThing.com
+POE-Component-Client-HTTP
+PageScorer
 Panscient
-Pi-Monster
 PictureFinder
-Pockey
-ProPowerBot
-QueryN Metasearch
+PleaseCrawl
+ProWebWalker
+Psbot
+Pump
+PyCurl
 Quick-Crawler
-RankingBot2
-RealDownload
+RankActive
+RankActiveLinkBot
+Re-re
+ReGet
 Reaper
+RebelMouse
 Recorder
 RepoMonkey
 Ripper
+RocketCrawler
 Rogerbot
+SEOkicks
+SEOkicks-Robot
+SEOstats
+SISTRIX
+SMTBot
 SalesIntelligent
 ScanAlert
-Scanbot
-Screaming
-ScrepyBot
+Scrapy
+ScreenerBot
 Searchestate
-SearchmetricsBot
-SeobilityBot
+SemanticJuice
+SentiBot
+Seomoz
 Siphon
-Site Sucker
-SiteExplorer
-SiteSucker
-Siteimprove
-SlySearch
-Snapbot
-Sociscraper
-Sogou web spider
+SiteLockSpider
+SiteSnagger
+Sitebeam
+SmartDownload
+SocialRankIOBot
+Sosospider
 Spanner
-Sqworm
+Spbot
+Spinn3r
 Stripper
+Sucker
+SuperBot
+Surfbot
 SurveyBot
+Swiftbot
 Szukacz
-T0PHackTeam
-The Intraformant
-TheNomad
+Telesphoreo
 Thumbor
 Titan
 Trendiction
-Trendictionbot
 True_Robot
 Turingos
 TwengaBot
 Twice
-URLy.Warning
-V-BOT
-Vacuum
-Vagabondo
+UnisterBot
+Upflow
+VB Project
 VelenPublicWebCrawler
-VidibleScraper
-Virusdie
-Voil
+VeriCiteCrawler
+Voltron
 WASALive-Bot
+WBSearchBot
 WISENutbot
 WPScan
-WWW-Collector-E
+WWW-Mechanize
+WWW::Mechanize
 WWWOFFLE
+Wallpapers
 Wallpapers/3.0
 WallpapersHD
-Web Enhancer
+WeSEE
 Web Fuck
-Web Pix
-WebBandit
-WebCollage
+WebAuto
 WebEnhancer
-WebFetch
-WebGo IS
-WebLeacher
-WebReaper
-WebSucker
+WebFuck
+WebImageCollector
+WebSauger
+WebStripper
 WebWhacker
+WebZIP
 Webalta
-WebmasterWorldForumBot
-Website Quester
-WebsiteExtractor
+Webshag
 WebsiteQuester
-Webster
-Whack
-Whatweb
+Whacker
 Woobot
 Wprecon
-YoudaoBot
+Xaldon WebSpider
+Xaldon_WebSpider
+Xenu
+Zitebot
 ZoomBot
-ZumBot
-arquivo.pt
-cognitiveseo
-crawl.sogou.com
-crawler.feedback
-dataforseo.com
-demandbase-bot
-evc-batch
-instabid
+ZyBorg
+autoemailspider
+backlink-check
+cah.io.community
+check1.exe
+clark-crawler
+coccocbot-web
+com.plumanalytics
+crawler4j
+eCatch
+facebookscraper
+gopher
+heritrix
 isitwp.com
-lwp-request
-lwp-trivial
-meanpathbot
-netEstate NE Crawler
+magpie-crawler
 page scorer
 ripz
 s1z.ru
-satoristudio.net
-scalaj-http
 scan.lol
+seobility
+seocompany.store
 seoscanners
-seostar
+serpstatbot
 siteripz
-sogouspider
-spyfu
-sysscan
+sp_auditbot
 tAkeOut
 trendiction.com
-trendiction.de
+webgains-bot
+webmeup-crawler
 xpymep1.exe
-zauba.io