V4.2022.05.3099 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
-404checker
+80legs
+AIBOT
 ASPSeek
 Abonti
-Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-Aipbot
+AiHitBot
 Alexibot
-AllSubmitter
-Alligator
-Anarchy
+Anarchie
 Anarchy99
-Aspiegel
-AwarioRssBot
+Ankit
+Apexoo
+Asterias
+AwarioSmartBot
+BackDoorBot
+BackStreet
 BacklinkCrawler
-Bigfoot
-Black Hole
+Badass
+Barkrowler
+BetaBot
+BlackWidow
 BlowFish
-Boardreader
-Buck
+Bolt
 BuiltBotTough
 BuiltWith
-Bullseye
 BunnySlippers
 BuzzSumo
-CODE87
+CATExplorador
+CCBot
 Calculon
-CazoodleBot
-Cegbfeieh
-Cocolyzebot
-Cogentbot
+CheeseBot
+ChinaClaw
 Copier
-CopyRightCheck
+Cosmos
 Craftbot
-CyotekWebCopy
-DTS Agent
+CrunchBot
+DIIbot
 DatabaseDriverMysqli
-Deusu
 Disco
-DomainCrawler
-Domains Project
-Download Wonder
+DomCopBot
+Dotbot
 Dragonfly
 Drip
-Ebingbong
-Ecxi
-EirGrabber
-Evil
+ECCP/1.0
+EMail Siphon
+Exabot
+ExtLinksBot
 Extractor
-ExtractorPro
+EyeNetIE
 Ezooms
 FDM
-FemtosearchBot
 Firefox/7.0
 Freeuploader
-FrontPage
+Fuzz
 FyberSpider
-G-i-g-a-b-o-t
 GalaxyBot
-GermCrawler
-Getintent
-Gigabot
+Genieo
+GetRight
+Go!Zilla
 Go-Ahead-Got-It
 Gotit
-GrapeFX
-HEADMasterSEO
-HTMLparser
-Harvest
+Grabber
+Grafula
+GridBot
+HMView
+HTTP::Lite
+HaosouSpider
+Havij
 HonoluluBot
 Humanlinks
-HybridBot
-IDBTE4M
-Iblog
-Id-search
-Image Fetch
+IlseBot
 IndeedBot
 Indy Library
-InfoNaviRobot
 InfoTekies
-Internet Ninja
-Jbrofuzz
+Intelliseek
+JOC Web Spider
+JamesBOT
 Jetty
-Joomla
-Jorgee
-Kinza
-LWP::Simple
-Leap
-Lftp
-Libwhisker
-Lightspeedsystems
+JikeSpider
+JustView
+Kenjin Spider
+Keybot Translation-Search-Machine
+Keyword Density
+Kozmosbot
+LNSpiderguy
+LibWeb
+LieBaoFast
 Likse
-LinkScan
-LinksManager
-LinqiaScrapeBot
+Linkbot
+LinqiaRSSBot
 Lipperhey
-Lipperhey Spider
 Litemage_walker
+Lmspider
 Ltx71
+MFC_Tear_Sample
 MJ12bot
-Mail.RU_Bot
-Majestic12
+MQQBrowser
+MTRobot
+Mag-Net
+Majestic SEO
+Majestic-SEO
 MarkWatch
+Masscan
 Mata Hari
-MeanPath Bot
+MauiBot
+MegaIndex.ru
 Metauri
 Minefield
-Mojolicious
-MolokaiBot
+Mojeek
+Morfeus Fucking Scanner
 Mr.4x3
-Name Intelligence
-Navroad
-NearSite
-NetLyzer
+Msrabot
+NICErsPRO
+Nessus
+Net Vampire
 NetSpider
-Nikto
-Nimbostratus
+NetZIP
+Netvibes
+Nibbler
+Niki-bot
+Ninja
 Nuclei
-Octopus
-Offline Navigator
-OpenLinkProfiler
-OrangeSpider
-OutclicksBot
+Offline Explorer
+OnCrawl
+Openfind
+Openvas
+OrangeBot
 OutfoxBot
-PECL::HTTP
-PHPCrawl
 POE-Component-Client-HTTP
-Page Analyzer
-PageScorer
 PageThing.com
 Pandalytics
-Papa Foto
+Pavuk
 Pi-Monster
-Pimonster
+Picsearch
+Piepmatz
 Pixray
-PleaseCrawl
-ProWebWalker
-Probethenet
-Psbot
+Pockey
 Pu_iN
+Pump
+PyCurl
 QueryN Metasearch
+Quick-Crawler
 RSSingBot
-RankActiveLinkBot
-Rankivabot
-RankurBot
-RebelMouse
-RedesScrapy
-RepoMonkey
-RocketCrawler
-SBIder
-SEOkicks-Robot
+RankingBot
+Re-re
+SEOkicks
+SEOlyticsCrawler
+SEOstats
 SISTRIX
+SMTBot
 SalesIntelligent
-ScanAlert
+Scanbot
+ScoutJet
 Scrapy
-ScrepyBot
 Searchestate
-Seekport
-SemrushBot
+SearchmetricsBot
+Semrush
+SentiBot
 SeoSiteCheckup
+Seomoz
 Shodan
-SlySearch
-Snoopy
-SocialRankIOBot
-Sogou web spider
-Sottopop
-Spammen
-Spbot
+Site Sucker
+SiteCheckerBotCrawler
+SiteExplorer
+SiteSnagger
+Sitebeam
+Siteimprove
+SmartDownload
+Snapbot
+Sosospider
+Spanner
 Sqlworm
-Steeler
-Sucker
+Sucuri
 SuperBot
-Surfbot
+SuperHTTP
+Suzuran
 T8Abot
-TeleportPro
 Telesoft
 Telesphoreo
-Telesphorep
 Thumbor
-Titan
+Toata
 Toweyabot
-Trendiction
-Trendictionbot
-Typhoeus
-URLy Warning
+Tracemyfile
+Turnitin
+TwengaBot
 URLy.Warning
-Upflow
-Vacuum
+UnisterBot
+V-BOT
+VB Project
+Vagabondo
+VeriCiteCrawler
+VidibleScraper
+Virusdie
 VoidEYE
-Voltron
-WASALive-Bot
-WEBDAV
-Wallpapers/3.0
-WallpapersHD
-Web Enhancer
-Web Pix
-Web Sauger
+WBSearchBot
+WISENutbot
+WPScan
+WWW-Collector-E
+WWWOFFLE
+Wallpapers
+WeSEE
 WebBandit
-WebImageCollector
-WebPix
-WebReaper
+WebCollage
+WebFetch
+WebSauger
 WebStripper
-WebSucker
-WebWhacker
 WebZIP
-Website Quester
-WebsiteExtractor
+Webalta
+Webster
 Whack
-Whacker
 Whatweb
-Who.is Bot
 WiseGuys Robot
 Wonderbot
-Xaldon WebSpider
-Xaldon_WebSpider
-Zeus
-ZmEu
+Woobot
+Wprecon
+YoudaoBot
+Zade
+Zitebot
 ZoomBot
-ZoominfoBot
+ZumBot
 arquivo.pt
-autoemailspider
-coccocbot
-cognitiveseo
+cah.io.community
+clark-crawler
 com.plumanalytics
-crawl.sogou.com
-demandbase-bot
-eCatch
+crawler.feedback
+crawler4j
+gopher
+internetVista monitor
+linkdexbot
 magpie-crawler
+meanpathbot
 mediawords
-netEstate NE Crawler
-page scorer
-polaris version
-ripz
+muhstik-scan
+oBot
+pcBrowser
+plumanalytics
 s1z.ru
 satoristudio.net
-scalaj-http
 seobility
-spyfu
-sysscan
+seostar
+sexsearcher
+siteripz
 tAkeOut
 trendiction.com
-ubermetrics-technologies.com
-voyagerx.com
 webgains-bot
 webmeup-crawler
-x09Mozilla
-zauba.io
+webpros.com
+xpymep1.exe
 zgrab