V4.2022.02.2908 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404enemy
-ALittle Client
-ASPSeek
-Abonti
+404checker
 Aboundexbot
 Acunetix
 AfD-Verbotsverfahren
 Aipbot
-Alexibot
-Alligator
-Anarchie
-Anthill
+AllSubmitter
+Apexoo
 Aspiegel
-Asterias
 AwarioRssBot
+AwarioSmartBot
 BBBike
-BDFetch
 BackDoorBot
+BackStreet
+BackWeb
 Backlink-Ceck
 BacklinkCrawler
+Badass
 BetaBot
-Black Hole
-BlowFish
-Boardreader
-Brandwatch
-BuiltWith
-CATExplorador
-CazoodleBot
-CheeseBot
-CherryPicker
+Blow
+Bolt
+BotALot
+CCBot
+CensysInspect
+CheTeam
 ChinaClaw
-Chlooe
+Citoid
 Claritybot
-Cocolyzebot
-Collector
-Copier
-Copyscape
-Cosmos
-Craftbot
-CrazyWebCrawler
+CopyRightCheck
+Crescent
 CrunchBot
+Custo
+CyotekWebCopy
+DBLBot
 DIIbot
-DTS Agent
-DataCha0s
-Deusu
-Devil
-Disco
-Dispatch
+Demon
+Digincore
+Discobot
+Discoverybot
 DomainAppender
-DomainStatsBot
+DomainSigmaCrawler
+Domains Project
 Dotbot
-Download Wonder
+Dragonfly
+ECCP/1.0
 EMail Wolf
-EirGrabber
-EroCrawler
+EasyDL
 Evil
-Express WebPictures
-ExtLinksBot
-Ezooms
-Fimap
-Flunky
-Foobot
+Exabot
+EyeNetIE
 Freeuploader
 FrontPage
+Fuzz
+FyberSpider
 Fyrebot
 G-i-g-a-b-o-t
-GalaxyBot
-Gigabot
-Go!Zilla
+GermCrawler
+Getintent
 Gotit
-Grabber
+GrabNet
 Grafula
-HEADMasterSEO
-HTTP::Lite
-HTTrack
+GrapeshotCrawler
+HMView
+HTMLparser
 Haansoft
+HaosouSpider
+Harvest
 Havij
-Iblog
+Heritrix
+HonoluluBot
+Humanlinks
+HybridBot
+Id-search
+Image Fetch
 IndeedBot
-Indy Library
-InfoTekies
-InterGET
 InternetSeer
-Iskanie
+JamesBOT
 Jbrofuzz
-JennyBot
-Jetty
+JetCar
 JikeSpider
 Jorgee
 JustView
-Kenjin Spider
+Jyxobot
 Keyword Density
-LNSpiderguy
-Leap
-LeechGet
-Libwhisker
+Kinza
+Kozmosbot
+LibWeb
 LieBaoFast
-Lightspeedsystems
-Likse
-LinkScan
-LinkWalker
+Linkbot
 Linkdexbot
 LinkextractorPro
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
+LinqiaScrapeBot
+Lipperhey
+Lmspider
 MFC_Tear_Sample
+MIDown tool
+MIIxpc
 MJ12bot
+MSFrontPage
+Mag-Net
 Magnet
-Mail.RU_Bot
-Majestic SEO
-Majestic-SEO
-MarkMonitor
 Mass Downloader
-Masscan
 Mata Hari
 MauiBot
 Mb2345Browser
 MeanPath Bot
-Metauri
-Minefield
-Moblie Safari
-Mojolicious
-Mr.4x3
-Msrabot
-NPbot
-Name Intelligence
-NearSite
-Needle
-Nessus
+MegaIndex.ru
+Mojeek
+Morfeus Fucking Scanner
+Mozlila
+Navroad
+NetLyzer
 NetMechanic
-NetZIP
+NetSpider
 Netcraft
-Nettrack
-Nibbler
-Nmap
-Not
+Netvibes
+NextGenSearchBot
 Nutch
-Octopus
 Offline Explorer
-OpenLinkProfiler
-Openfind
-OutfoxBot
+OrangeBot
+PECL::HTTP
+PHPCrawl
 POE-Component-Client-HTTP
 Page Analyzer
-Panscient
+PageThing.com
+PeoplePal
 Picsearch
-Piepmatz
-Probethenet
-PxBroker
+Pimonster
+Pixray
+Psbot
+Pump
 PyCurl
 QueryN Metasearch
-RSSingBot
 RankActiveLinkBot
-RankingBot2
-Re-re
-RebelMouse
-RedesScrapy
+RankFlex
+Reaper
 RepoMonkey
 Ripper
-Rogerbot
 SBIder
-SEOkicks-Robot
-SEOlyticsCrawler
-Scanbot
-Scrapy
-ScrepyBot
-SearchmetricsBot
-Seekport
-SemanticJuice
+SEOstats
+Searchestate
 Semrush
+SemrushBot
 Shodan
-Siphon
-Site Sucker
-SiteLockSpider
-Sitebeam
-Siteimprove
-Snapbot
+SiteCheckerBotCrawler
+SiteExplorer
+SiteSucker
+SlySearch
+SmartDownload
+Snoopy
+SocialRankIOBot
+Sociscraper
 Sogou web spider
-Spammen
+Sosospider
+Sottopop
 SpankBot
+Spanner
+SputnikBot
+Sqworm
+Steeler
 Sucker
+Sucuri
 SuperBot
+SuperHTTP
 SurveyBot
 Suzuran
-Szukacz
+Swiftbot
 T0PHackTeam
-Telesoft
-Telesphoreo
-Telesphorep
-The Intraformant
+Teleport
+Titan
 Toata
+Tracemyfile
+Trendiction
 Trendictionbot
+True_Robot
+TurnitinBot
 Typhoeus
 URLy Warning
-UnisterBot
-Upflow
-VCI
-Virusdie
-Voil
+V-BOT
+VB Project
+VidibleScraper
+VoidEYE
+Voltron
 WISENutbot
-WWW-Collector-E
-Wallpapers/3.0
-WallpapersHD
+WWWOFFLE
+Web Auto
 Web Fetch
-Web Sauger
-WebAuto
+Web Fuck
+Web Pix
+Web Sucker
+WebBandit
 WebCollage
 WebCopier
+WebFetch
+WebFuck
+WebImageCollector
 WebLeacher
+WebPix
+WebReaper
+WebSauger
+WebStripper
 WebSucker
+WebWhacker
 Webalta
-Webshag
-Website Quester
 WebsiteExtractor
-WebsiteQuester
-Who.is Bot
-Wonderbot
-Woobot
-Wotbox
+Whacker
+Whatweb
+WiseGuys Robot
+Wprecon
 Xaldon WebSpider
 YoudaoBot
 Zade
 Zeus
-ZoomBot
-adscanner
-archive.org_bot
+ZoominfoBot
+ZumBot
+ZyBorg
+arquivo.pt
 autoemailspider
-clark-crawler
-crawl.sogou.com
+check1.exe
 crawler.feedback
 crawler4j
+demandbase-bot
 eCatch
-instabid
-magpie-crawler
-mediawords
+facebookscraper
+gopher
+heritrix
+ips-agent
+isitwp.com
+lwp-request
+lwp-trivial
+meanpathbot
+netEstate NE Crawler
 oBot
+polaris version
 probe-image-size
-ripz
-s1z.ru
-satoristudio.net
+scalaj-http
+scan.lol
+seocompany.store
+seostar
 serpstatbot
-sexsearcher
-sogouspider
-sp_auditbot
-voyagerx.com
-webgains-bot
-webmeup-crawler
-x22Mozilla
-xpymep1.exe
+sitechecker.pro
+siteripz
+spyfu
+tAkeOut
+webpros.com
+webprosbot
+x09Mozilla
+zauba.io
+zgrab