V4.2021.07.2470 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,156 +1,156 @@
-360Spider
+01h4x.com
 404checker
-AIBOT
-ASPSeek
+ADmantX
+Abonti
 Acunetix
+AfD-Verbotsverfahren
+AiHitBot
 Aipbot
+Alexibot
 AllSubmitter
-Anarchie
-Anarchy
-Anthill
-Apexoo
-Aspiegel
+AlphaBot
 Asterias
-Attach
 AwarioRssBot
+AwarioSmartBot
+BDCbot
 BDFetch
+BLEXBot
 BackDoorBot
+BackStreet
 BackWeb
-Backlink-Ceck
 BacklinkCrawler
-Badass
-Barkrowler
-BetaBot
 Bigfoot
-Blackboard
+Black Hole
+BlackWidow
+Blow
+Bolt
+BotALot
+Brandwatch
 Buck
-Buddy
+BuiltBotTough
 BuiltWith
-BunnySlippers
-CODE87
+Bullseye
+BuzzSumo
+CATExplorador
 CSHttp
-Calculon
-CheTeam
-ChinaClaw
+CazoodleBot
+CensysInspect
+CherryPicker
 Chlooe
-Cliqzbot
 Cloud mapping
-CrunchBot
-Custo
+Cocolyzebot
+CopyRightCheck
+CrazyWebCrawler
 DTS Agent
 DataCha0s
-Devil
+Deusu
 Digincore
 DigitalPebble
-DittoSpyder
+DnyzBot
+DomainCrawler
+DomainSigmaCrawler
 DomainStatsBot
-Dragonfly
-EMail Wolf
-EasyDL
-Ebingbong
+Domains Project
+Download Wonder
+Drip
 EroCrawler
-ExtractorPro
-Ezooms
+ExtLinksBot
+Extreme Picture Finder
+FDM
 FHscan
-FlashGet
+FemtosearchBot
 Freeuploader
 FrontPage
-Fyrebot
-GT::WWW
+Fuzz
 GalaxyBot
-Genieo
-GermCrawler
+GetWeb
+Getintent
 Gigabot
-Go-Ahead-Got-It
+Gotit
+Grafula
+GrapeFX
 GridBot
-HaosouSpider
+HMView
+Heritrix
+Hloader
 Humanlinks
-HybridBot
-IDBot
+IRLbot
 Iblog
-Image Fetch
 Image Sucker
 IndeedBot
 Indy Library
-InfoTekies
-Intelliseek
-Internet Ninja
-InternetSeer
+InfoNaviRobot
 Iskanie
 IstellaBot
-JennyBot
+JOC Web Spider
 JetCar
+Jetty
+JikeSpider
+Joomla
+Jorgee
+JustView
 Kenjin Spider
-Kinza
+Keyword Density
+Kozmosbot
 LNSpiderguy
+LWP::Simple
 Lanshanbot
 Larbin
-LeechFTP
-LexiBot
+Lftp
+LibWeb
 Libwhisker
-Likse
-LinkScan
+LieBaoFast
 Linkdexbot
-Lipperhey
-Lipperhey Spider
+LinkextractorPro
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+Ltx71
 MFC_Tear_Sample
 MJ12bot
-MQQBrowser
-MSIECrawler
+MTRobot
 Mag-Net
-MarkMonitor
-MarkWatch
 Mass Downloader
 Masscan
 MauiBot
 Mb2345Browser
 MeanPath Bot
-Mediatoolkitbot
-MegaIndex.ru
 Microsoft Data Access
-Microsoft URL Control
-Mojolicious
-Mozlila
+Mojeek
+Morfeus Fucking Scanner
 Musobot
-NICErsPRO
-NPbot
 Name Intelligence
-Navroad
-NearSite
 Needle
+Nessus
 Net Vampire
-Netvibes
+NetMechanic
 NextGenSearchBot
-Nibbler
-Nikto
-Ninja
-Nmap
+Niki-bot
 Nuclei
-OPPO A33
-Offline Explorer
-OnCrawl
+Nutch
 OpenLinkProfiler
-PageGrabber
+OrangeSpider
+OutfoxBot
+PHPCrawl
+PageAnalyzer
 PageScorer
+Pandalytics
 Panscient
-Papa Foto
-PeoplePal
 Pi-Monster
-Picsearch
 Pimonster
-Pixray
 Pockey
-ProWebWalker
+Pu_iN
 PyCurl
 QueryN Metasearch
 Quick-Crawler
+RSSingBot
+RankActive
 RankActiveLinkBot
-RankFlex
-RankingBot2
 RankurBot
-Re-re
-RealDownload
-Ripper
+Reaper
+RebelMouse
+Recorder
+RocketCrawler
+Rogerbot
 SEOkicks
 SEOlyticsCrawler
 SEOprofiler
@@ -158,93 +158,93 @@ SEOstats
 SalesIntelligent
 ScanAlert
 Scrapy
-Screaming
+ScreenerBot
 Searchestate
-SearchmetricsBot
 Semrush
 SemrushBot
 SentiBot
-SeoSiteCheckup
 Seomoz
+Shodan
+Siphon
 Site Sucker
 SiteLockSpider
+SiteSucker
 Sitebeam
 SlySearch
+SmartDownload
+Snapbot
+Sociscraper
 Sottopop
+Spammen
 Spanner
-Spbot
-Sqworm
-Steeler
-Sucuri
+Spinn3r
+SputnikBot
+Sqlmap
+Sucker
 SuperBot
-SurveyBot
-T0PHackTeam
-Teleport
+SuperHTTP
+Surfbot
+Suzuran
+Swiftbot
+Szukacz
+T8Abot
 TeleportPro
-Telesphoreo
-The Intraformant
-Toweyabot
-Tracemyfile
-Turnitin
-TurnitinBot
+Telesoft
+Trendiction
+Trendictionbot
+True_Robot
 TwengaBot
-Typhoeus
-URLy.Warning
-Upflow
-V-BOT
-VCI
-Vacuum
-VeriCiteCrawler
+Twice
+UnisterBot
+VB Project
+Virusdie
 VoidEYE
-WASALive-Bot
-WEBDAV
-WallpapersHD
-Web Auto
+Voil
+WBSearchBot
+WISENutbot
+WPScan
+WWW-Mechanize
 Web Enhancer
 Web Fetch
 Web Fuck
-WebEnhancer
-WebFuck
-WebGo IS
+WebAuto
+WebBandit
+WebCopier
+WebFetch
 WebImageCollector
-WebLeacher
-WebPix
-WebStripper
-WebWhacker
-WebZIP
-Webalta
-WebmasterWorldForumBot
+WebSauger
+WebSucker
 Webshag
-WebsiteExtractor
-WebsiteQuester
 Webster
 Whatweb
 Widow
 WinHTTrack
 Wotbox
-Zauba
-Zermelo
+Wprecon
+Zade
 ZmEu
+ZoomBot
 ZoominfoBot
+arquivo-web-crawler
+arquivo.pt
+check1.exe
 coccocbot-web
 crawl.sogou.com
-crawler.feedback
 crawler4j
-demandbase-bot
-domainsproject.org
-ips-agent
-lwp-trivial
-mediawords
-muhstik-scan
+eCatch
+evc-batch
+isitwp.com
+lwp-request
+magpie-crawler
+meanpathbot
+netEstate NE Crawler
 oBot
 pcBrowser
-polaris version
+ripz
 satoristudio.net
 seobility
-sitechecker.pro
-siteripz
-spyfu
-sysscan
-trendiction.com
+serpstatbot
+sogouspider
+sp_auditbot
 webmeup-crawler
-x09Mozilla
+x22Mozilla