V4.2021.08.2530 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
-404checker
-ASPSeek
-Abonti
-Aboundex
+80legs
+ALittle Client
 Acunetix
 AiHitBot
 Alexibot
-Alligator
+AllSubmitter
+AlphaBot
 Anarchie
 Anarchy
 Ankit
 Anthill
-Aspiegel
-Asterias
-Attach
+Apexoo
 AwarioRssBot
-BDFetch
-BackWeb
+AwarioSmartBot
+BBBike
+BDCbot
+BLEXBot
+BackDoorBot
 Barkrowler
-BetaBot
-Black Hole
-BlowFish
-BotALot
+Battleztar Bazinga
+Bigfoot
+Blackboard
+Blow
+Boardreader
 Brandprotect
+Brandwatch
+Buck
+Buddy
+BuiltBotTough
+BuiltWith
 Bullseye
-Calculon
+CATExplorador
+CazoodleBot
+CensysInspect
+CheeseBot
 Chlooe
-Cliqzbot
 Cloud mapping
-Cogentbot
+Cocolyzebot
 Collector
 Copier
-Copyscape
+Cosmos
 Craftbot
-Crescent
-CyotekWebCopy
-DIIbot
-DatabaseDriverMysqli
-Deusu
-Dirbuster
+Crawling at Home Project
+CrazyWebCrawler
+Custo
+DBLBot
+DSearch
+DTS Agent
+DataCha0s
+Demon
+DigitalPebble
 Disco
-Dispatch
+Discobot
 DnyzBot
-DomCopBot
 DomainAppender
-DomainStatsBot
-Dotbot
-Drip
-EMail Siphon
-EasyDL
-EirGrabber
-EroCrawler
+DomainCrawler
 Evil
 Extractor
+Extreme Picture Finder
 EyeNetIE
-FDM
-Flunky
-Fuzz
-FyberSpider
+Ezooms
+FemtosearchBot
+FlashGet
+Foobot
+FrontPage
 GT::WWW
-GalaxyBot
-GetRight
-Getintent
+GermCrawler
+Gigablast
 Gigabot
 Go!Zilla
-Go-Ahead-Got-It
-Gotit
 GrabNet
-GrapeFX
+Grafula
+GrapeshotCrawler
 GridBot
-HTMLparser
-Haansoft
-Harvest
-Havij
-Humanlinks
+HEADMasterSEO
+HTTrack
+Heritrix
+Hloader
+IDBot
 IRLbot
-IndeedBot
-Indy Library
+Id-search
+IlseBot
+Image Fetch
+Image Sucker
 InfoNaviRobot
 Internet Ninja
-InternetSeer
 Iria
+Iskanie
+JamesBOT
 Jbrofuzz
 JetCar
-Jyxobot
+Jetty
+Joomla
+JustView
+Kenjin Spider
+Keyword Density
+Kozmosbot
 LNSpiderguy
-LWP::Simple
 Leap
-LeechFTP
 LexiBot
-Lftp
-LibWeb
-Lightspeedsystems
-LinkWalker
-Linkbot
-LinkextractorPro
+LinkScan
+Linkdexbot
 LinkpadBot
-LinksManager
-LinqiaScrapeBot
-MIDown tool
-MIIxpc
-MJ12bot
+LinqiaMetadataDownloaderBot
+Lipperhey
+Litemage_walker
+Lmspider
+Ltx71
 MQQBrowser
 MSIECrawler
+Mag-Net
 Magnet
-Majestic SEO
-Majestic-SEO
-Masscan
-Mata Hari
-Mediatoolkitbot
-Microsoft URL Control
+Majestic12
+MarkWatch
+Mass Downloader
+Microsoft Data Access
 Moblie Safari
-Mozlila
+Morfeus Fucking Scanner
 Musobot
-Name Intelligence
+NICErsPRO
+Nessus
+Net Vampire
+NetAnts
+NetLyzer
+NetMechanic
 NetSpider
-NetZIP
-NextGenSearchBot
 Nikto
-NimbleCrawler
 Nimbostratus
-Ninja
-Nmap
-Nuclei
 Offline Explorer
 Offline Navigator
-Openfind
+OnCrawl
 OrangeBot
 OutclicksBot
 OutfoxBot
-POE-Component-Client-HTTP
+Page Analyzer
 PageAnalyzer
-PageGrabber
-PageScorer
-Pandalytics
-Panscient
-Pavuk
-Petalbot
-Pimonster
-Pixray
-Pockey
-Psbot
+PleaseCrawl
+ProWebWalker
 PyCurl
+Quick-Crawler
 RSSingBot
 RankActiveLinkBot
-RedesScrapy
+RebelMouse
+RepoMonkey
 Ripper
-Rogerbot
+RocketCrawler
+SEOkicks
+SEOkicks-Robot
+SEOprofiler
+SEOstats
 SISTRIX
-ScanAlert
+SMTBot
+SalesIntelligent
 Scanbot
-ScoutJet
-Scrapy
 SearchmetricsBot
-SeoSiteCheckup
+SemrushBot
 SeobilityBot
-Siphon
-SiteCheckerBotCrawler
-SiteSucker
-Sitebeam
-Siteimprove
-Sitevigil
-SmartDownload
-Snapbot
-SocialRankIOBot
-Sociscraper
+Site Sucker
+SiteLockSpider
+SlySearch
+Snake
 Sogou web spider
-Sottopop
-SpankBot
+SpaceBison
+Spammen
 Spbot
-SputnikBot
+Spinn3r
+Sqlmap
 Sqlworm
-Steeler
+Sqworm
+Stripper
 Sucuri
+SuperBot
 SuperHTTP
 SurveyBot
-Swiftbot
 Szukacz
-T0PHackTeam
+T8Abot
 TeleportPro
 Telesoft
 Telesphorep
-The Intraformant
-TheNomad
-Thumbor
-Toweyabot
-Trendiction
-True_Robot
-Turingos
-TwengaBot
-URLy Warning
+Tracemyfile
+Turnitin
+Typhoeus
+URLy.Warning
 UnisterBot
-Upflow
 V-BOT
 VB Project
-Vacuum
-VelenPublicWebCrawler
-Voltron
+VeriCiteCrawler
+VoidEYE
+Voil
 WBSearchBot
-WEBDAV
 WISENutbot
 WPScan
-WWWOFFLE
 Wallpapers/3.0
-WeSEE
 Web Auto
-Web Enhancer
+Web Fuck
 Web Pix
+Web Sauger
 Web Sucker
 WebAuto
-WebEnhancer
-WebImageCollector
-WebLeacher
-WebmasterWorldForumBot
-WebsiteQuester
-Webster
+WebBandit
+WebCollage
+WebGo IS
+WebReaper
+WebSucker
+WebWhacker
+Webshag
+Whatweb
 Who.is Bot
 WinHTTrack
+WiseGuys Robot
 Wonderbot
-Xaldon_WebSpider
+Woobot
+Wprecon
 Xenu
-YoudaoBot
+Zauba
+Zermelo
 Zeus
-ZumBot
+ZoominfoBot
 archive.org_bot
 clark-crawler
+coccocbot-web
+cognitiveseo
 com.plumanalytics
-crawler4j
-eCatch
-facebookscraper
+demandbase-bot
 gopher
 heritrix
-instabid
-internetVista monitor
-isitwp.com
+lwp-request
 lwp-trivial
+magpie-crawler
 meanpathbot
-mediawords
+muhstik-scan
 netEstate NE Crawler
+page scorer
 plumanalytics
-polaris version
-scan.lol
-sexsearcher
-siteripz
+ripz
+satoristudio.net
+scalaj-http
+seoscanners
+serpstatbot
+sogouspider
+spyfu
+tAkeOut
+ubermetrics-technologies.com
+webmeup-crawler
 webpros.com
+x09Mozilla
 x22Mozilla
 zauba.io
 zgrab