V4.2021.12.2770 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
+01h4x.com
 80legs
+AIBOT
 ALittle Client
-ASPSeek
 Aboundex
-Aboundexbot
-AllSubmitter
-Alligator
-Anarchy
-Anarchy99
-Ankit
+Acunetix
+AiHitBot
+Alexibot
+Anarchie
+Anthill
+Aspiegel
+Asterias
+Attach
+AwarioRssBot
 AwarioSmartBot
+BDCbot
 BDFetch
-BLEXBot
-BackStreet
-Bandit
-Barkrowler
-BatchFTP
+Backlink-Ceck
 BlackWidow
 Blow
-BlowFish
-Brandwatch
-BunnySlippers
+Boardreader
+BotALot
+Brandprotect
+Buck
 BuzzSumo
+CCBot
+CODE87
+CSHttp
+Calculon
 CazoodleBot
 Cegbfeieh
+CheTeam
 CheeseBot
-CherryPicker
+ChinaClaw
 Chlooe
+Citoid
+Cliqzbot
+Cocolyzebot
 Copier
-Craftbot
-Crescent
 CrunchBot
-DBLBot
+DIIbot
 DTS Agent
 DataCha0s
 Demon
 Deusu
-Devil
-Discoverybot
-DittoSpyder
+Dirbuster
+Discobot
+Dispatch
 DnBCrawler-Analytics
-DnyzBot
-DomainAppender
-DomainStatsBot
+Domains Project
+Dotbot
 Download Wonder
-Evil
+ECCP/1.0
+EMail Wolf
+Ebingbong
 Exabot
 Express WebPictures
-ExtractorPro
-Extreme Picture Finder
-EyeNetIE
+ExtLinksBot
+Extractor
+FHscan
 FemtosearchBot
 Fimap
-Foobot
-Freeuploader
+Flunky
+FrontPage
 Fuzz
 FyberSpider
-Fyrebot
-GT::WWW
+G-i-g-a-b-o-t
 GalaxyBot
+GermCrawler
 GetRight
-Go!Zilla
-Gotit
-Grabber
-Grafula
-GridBot
-HMView
-HTMLparser
-HTTrack
-Havij
+Go-Ahead-Got-It
+GoZilla
+GrabNet
+GrapeshotCrawler
+HEADMasterSEO
+HTTP::Lite
+Haansoft
+Harvest
 Heritrix
-Hloader
-HybridBot
+Humanlinks
 IDBot
 Iblog
 Id-search
-Intelliseek
-IstellaBot
+Indy Library
+InfoNaviRobot
+InterGET
+Internet Ninja
+Iria
+Iskanie
+JOC Web Spider
 JamesBOT
 Jbrofuzz
 JetCar
-JikeSpider
-Jorgee
 Jyxobot
 Keyword Density
-Kozmosbot
-LNSpiderguy
-Lanshanbot
-LeechFTP
-LeechGet
-LibWeb
-LieBaoFast
-Lightspeedsystems
-Likse
+Larbin
+LexiBot
 LinkScan
-LinkWalker
+Linkbot
+Linkdexbot
 LinkpadBot
-LinqiaScrapeBot
-Litemage_walker
-Lmspider
-MIDown tool
-MIIxpc
-Majestic-SEO
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+Lipperhey Spider
+Ltx71
+MSFrontPage
+Mag-Net
 Majestic12
 MarkMonitor
-MarkWatch
+Mass Downloader
+Masscan
 Mata Hari
-MauiBot
 Mb2345Browser
+MeanPath Bot
+Mediatoolkitbot
+MegaIndex.ru
 Metauri
-MicroMessenger
-Mojeek
-Mozlila
-Msrabot
-Name Intelligence
-Nameprotect
-NearSite
-Nessus
-NetLyzer
+Microsoft Data Access
+Microsoft URL Control
+Moblie Safari
+Mojolicious
+Morfeus Fucking Scanner
+Mr.4x3
+NICErsPRO
+Navroad
+Net Vampire
+NetMechanic
+Netcraft
+Niki-bot
 NimbleCrawler
+Nimbostratus
+Nmap
 Nuclei
 Octopus
-Offline Explorer
-Offline Navigator
 OnCrawl
-OrangeBot
-OutfoxBot
+OpenLinkProfiler
+OpenVAS
+Openfind
+OrangeSpider
+POE-Component-Client-HTTP
+Page Analyzer
+PageGrabber
+PageThing.com
 Pandalytics
-Papa Foto
-Pavuk
 PeoplePal
+Picscout
 PictureFinder
 Piepmatz
-Pixray
 Pockey
 ProWebWalker
-Probethenet
-Psbot
-RankActive
+PxBroker
+PyCurl
+QueryN Metasearch
 RankActiveLinkBot
-RankingBot2
-Re-re
+RankFlex
+RankingBot
+Rankivabot
+RankurBot
 Reaper
-Recorder
+RebelMouse
+RedesScrapy
+Ripper
+Rogerbot
 SEOkicks
+SEOkicks-Robot
 SEOlyticsCrawler
+SEOstats
 SISTRIX
-SMTBot
+ScanAlert
+ScoutJet
 Scrapy
 ScreenerBot
-ScrepyBot
+Searchestate
 SearchmetricsBot
-SemanticJuice
-SemrushBot
-SeoSiteCheckup
+Semrush
 SeobilityBot
-Seomoz
 Shodan
-SiteCheckerBotCrawler
+Siphon
+Site Sucker
+SiteExplorer
+SiteSucker
 Sitebeam
 Sitevigil
-SlySearch
-SmartDownload
 SocialRankIOBot
+Sociscraper
 Sottopop
+SpaceBison
 Spanner
 Spinn3r
-SputnikBot
-Sqlmap
-Sqlworm
-Sucker
-SuperBot
-Szukacz
-T0PHackTeam
-Telesphoreo
-Telesphorep
-Tracemyfile
-Trendictionbot
-Turingos
+Steeler
+SurveyBot
+Suzuran
+Swiftbot
+The Intraformant
+Titan
+TwengaBot
 Typhoeus
-URLy.Warning
-V-BOT
+Upflow
 VB Project
-VCI
-Vacuum
-VelenPublicWebCrawler
-VeriCiteCrawler
+VidibleScraper
 Virusdie
-Voltron
+Voil
 WASALive-Bot
 WEBDAV
-WISENutbot
-WWW-Mechanize
-WWWOFFLE
-Web Collage
-WebFetch
+WallpapersHD
+WeSEE
+Web Fuck
+WebEnhancer
+WebLeacher
+WebReaper
 WebSauger
-WebSucker
+WebStripper
+WebWhacker
+WebZIP
 Webalta
-WebmasterWorldForumBot
 Webshag
+WebsiteQuester
 Webster
-Widow
-WinHTTrack
-Wonderbot
-Woobot
-Wprecon
-Xaldon_WebSpider
+Whatweb
+WiseGuys Robot
+Wotbox
 YoudaoBot
-Zade
+Zauba
+Zermelo
 Zeus
 Zitebot
-ZoominfoBot
 ZyBorg
 arquivo.pt
-check1.exe
 clark-crawler
 com.plumanalytics
 dataforseo.com
 eCatch
-evc-batch
-instabid
-internetVista monitor
+gopher
+heritrix
 ips-agent
-isitwp.com
-lwp-request
-lwp-trivial
-mediawords
-netEstate NE Crawler
-page scorer
+meanpathbot
+muhstik-scan
 pcBrowser
 plumanalytics
 polaris version
-probe-image-size
-ripz
-scalaj-http
-scan.lol
-seobility
-seostar
-sitechecker.pro
-siteripz
-sysscan
-tAkeOut
+satoristudio.net
+serpstatbot
+trendiction.com
+trendiction.de
 ubermetrics-technologies.com
-webgains-bot
+voyagerx.com
 webmeup-crawler
-webpros.com
 webprosbot
 x09Mozilla
+zgrab