V4.2021.08.2517 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+01h4x.com
 360Spider
-404enemy
+404checker
 ADmantX
-AIBOT
 ALittle Client
-ASPSeek
 Aboundexbot
-Acunetix
-AfD-Verbotsverfahren
+Aipbot
 AlphaBot
-Anarchy99
-Anthill
+Anarchy
+Apexoo
+Attach
 AwarioRssBot
-BBBike
+BDFetch
+BackDoorBot
+BackWeb
 Backlink-Ceck
-BacklinkCrawler
-BetaBot
-Bigfoot
-Blackboard
-Boardreader
-Bolt
+Badass
+Bandit
+Barkrowler
+Battleztar Bazinga
+Bitacle
+Black Hole
 Brandprotect
-Brandwatch
-Buddy
-BuiltBotTough
-Bullseye
-CCBot
+BunnySlippers
 CODE87
-CazoodleBot
-CheTeam
-CheeseBot
-CherryPicker
+Chlooe
 Claritybot
-Copyscape
-Cosmos
-Crawling at Home Project
-CrazyWebCrawler
+Cliqzbot
+Cocolyzebot
+Copier
 Crescent
-DIIbot
-DSearch
-DTS Agent
-DataCha0s
-Devil
+Custo
+DBLBot
+Demon
+Deusu
 Digincore
-DigitalPebble
-Dirbuster
-Disco
-Dispatch
-DnyzBot
-DomainCrawler
-DomainSigmaCrawler
+Discobot
+Discoverybot
+DittoSpyder
+DomCopBot
+Dotbot
 Download Wonder
+Dragonfly
 Drip
 EMail Wolf
-Ebingbong
-Ecxi
 EirGrabber
-Evil
-ExtLinksBot
-FHscan
+EroCrawler
+Exabot
+ExtractorPro
+FDM
+FemtosearchBot
 Fimap
 Firefox/7.0
 Flunky
+Foobot
+FrontPage
 Fuzz
+FyberSpider
 Fyrebot
 GT::WWW
-GetWeb
+GalaxyBot
+GermCrawler
+Getintent
 Gigabot
-Go!Zilla
+Go-Ahead-Got-It
 GoZilla
 Gotit
-Grabber
-GrapeshotCrawler
+Grafula
+GrapeFX
 GridBot
+HTTP::Lite
 HTTrack
 Harvest
-Humanlinks
+HybridBot
+IDBTE4M
+IDBot
 IRLbot
 Iblog
-IlseBot
-IndeedBot
-InterGET
-Internet Ninja
+Image Sucker
+Indy Library
 Iria
 Iskanie
 JOC Web Spider
-JamesBOT
+Jbrofuzz
+JennyBot
 JikeSpider
-Jyxobot
-Kinza
-Larbin
-Leap
-LeechFTP
-LexiBot
+Kenjin Spider
+Keyword Density
+LeechGet
 Lftp
-Libwhisker
-Lightspeedsystems
+LibWeb
 LinkScan
+Linkbot
 Linkdexbot
-LinqiaScrapeBot
+LinksManager
 Lipperhey
-Litemage_walker
 Lmspider
 Ltx71
 MFC_Tear_Sample
-MIDown tool
 MSFrontPage
-MSIECrawler
 MTRobot
 Mag-Net
-Majestic-SEO
-Majestic12
+Magnet
+Majestic SEO
 MarkMonitor
 MarkWatch
-Masscan
 MauiBot
-MeanPath Bot
-Meanpathbot
-Mediatoolkitbot
 Metauri
-Microsoft Data Access
-Mr.4x3
+MicroMessenger
+Microsoft URL Control
+Mojolicious
+NPbot
+Name Intelligence
+NearSite
 Needle
+Net Vampire
+NetLyzer
+NetMechanic
+Netcraft
 Netvibes
+Nibbler
+Niki-bot
 Nikto
-NimbleCrawler
 Nimbostratus
-Ninja
-Nutch
-Octopus
-Offline Navigator
-Openfind
+Offline Explorer
+OpenLinkProfiler
+OpenVAS
 OrangeSpider
-OutclicksBot
-OutfoxBot
-PHPCrawl
-POE-Component-Client-HTTP
-Page Analyzer
+PECL::HTTP
 PageAnalyzer
-PageScorer
-PageThing.com
-Papa Foto
-Picsearch
+PeoplePal
+Pi-Monster
+PictureFinder
 Piepmatz
-PleaseCrawl
+Pixray
+Pockey
 ProPowerBot
-Probethenet
+Pump
 PxBroker
+PyCurl
 QueryN Metasearch
-RankActive
+RankingBot
+RankurBot
+Recorder
 RedesScrapy
-RepoMonkey
 Ripper
-SEOlyticsCrawler
+SBIder
 SEOprofiler
-SalesIntelligent
-ScanAlert
 Scanbot
-Screaming
+ScoutJet
+Scrapy
+ScreenerBot
 ScrepyBot
 SearchmetricsBot
+SemrushBot
 SentiBot
+SeoSiteCheckup
 Seomoz
-Site Sucker
-SiteCheckerBotCrawler
-SiteLockSpider
+Siphon
 Sitebeam
+Sitevigil
 SlySearch
-Snapbot
+Snake
 Snoopy
-SocialRankIOBot
-Sogou web spider
-SpaceBison
-Spammen
-Spanner
-Spinn3r
-SputnikBot
-Sqlworm
-Steeler
-Sucker
+Sociscraper
+Sottopop
+Spbot
+Stripper
+Sucuri
+SuperBot
 SuperHTTP
-Suzuran
-T8Abot
-Telesphorep
-The Intraformant
-Thumbor
-TightTwatBot
+Surfbot
+SurveyBot
+Swiftbot
+Szukacz
+T0PHackTeam
+Teleport
+Telesoft
+Telesphoreo
+TheNomad
+Toata
+Trendictionbot
+True_Robot
 Turingos
-TurnitinBot
-Typhoeus
-URLy Warning
+Turnitin
 URLy.Warning
 UnisterBot
 V-BOT
+VCI
 Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
 Virusdie
-VoidEYE
-WBSearchBot
+Voltron
+WASALive-Bot
+WPScan
 WWW-Collector-E
+WWW::Mechanize
 WWWOFFLE
 Wallpapers
 WallpapersHD
-Web Pix
+WeSEE
+Web Fuck
 Web Sauger
+Web Sucker
+WebAuto
 WebBandit
-WebCollage
-WebCopier
 WebFuck
+WebGo IS
+WebLeacher
 WebReaper
-Webalta
-WebmasterWorldForumBot
+WebSauger
+WebStripper
+WebSucker
+Website Quester
+WebsiteExtractor
 Webster
 Whack
+Whacker
+Whatweb
 Who.is Bot
-WinHTTrack
-Wonderbot
-YoudaoBot
-Zade
-Zermelo
-Zitebot
+WiseGuys Robot
+Woobot
+Xaldon WebSpider
+Xaldon_WebSpider
+Xenu
+ZmEu
 ZoominfoBot
-ZyBorg
-archive.org_bot
 arquivo.pt
-autoemailspider
 backlink-check
-coccocbot-web
-cognitiveseo
 com.plumanalytics
-crawler4j
-facebookscraper
+crawl.sogou.com
+evc-batch
 gopher
-internetVista monitor
+isitwp.com
 lwp-request
-lwp-trivial
-magpie-crawler
 mediawords
-page scorer
-pcBrowser
-probe-image-size
-satoristudio.net
+muhstik-scan
+netEstate NE Crawler
+s1z.ru
+scalaj-http
+seoscanners
+seostar
 serpstatbot
 sexsearcher
 sitechecker.pro
-tAkeOut
-trendiction.de
-voyagerx.com
-x09Mozilla
+sp_auditbot
+spyfu
+trendiction.com
+ubermetrics-technologies.com
+webmeup-crawler
+zgrab