V4.2021.10.2629 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
+404checker
+80legs
+ADmantX
 ALittle Client
-Aboundex
-Acunetix
-AfD-Verbotsverfahren
+AiHitBot
 AllSubmitter
-Alligator
+AlphaBot
 Anarchie
+Anarchy
+Anarchy99
 Ankit
-Anthill
+Asterias
 AwarioRssBot
 AwarioSmartBot
 BDCbot
 BDFetch
 Backlink-Ceck
+Barkrowler
 Bigfoot
-Bitacle
-Black Hole
+Blackboard
 Blow
-BlowFish
-Brandwatch
-Buck
+Bolt
+Brandprotect
 Bullseye
+BunnySlippers
 BuzzSumo
-CCBot
+CATExplorador
+CODE87
+CazoodleBot
 Cegbfeieh
-CensysInspect
-CheeseBot
-Cloud mapping
-Cocolyzebot
-Collector
-Copyscape
-Cosmos
-CrunchBot
+Crescent
 Custo
 DIIbot
-DSearch
-Deusu
+Devil
 Digincore
-DigitalPebble
-Dirbuster
+Disco
 Discobot
-DnyzBot
+Dispatch
 DomCopBot
 DomainAppender
-DomainCrawler
 DomainSigmaCrawler
-Domains Project
-Dotbot
-Dragonfly
-Drip
+EMail Siphon
 EMail Wolf
-Evil
+Ecxi
+EroCrawler
+Exabot
+Extractor
 ExtractorPro
 Extreme Picture Finder
-EyeNetIE
 Ezooms
-Foobot
+FHscan
+FemtosearchBot
+Fimap
+FlashGet
 FrontPage
-Fuzz
-Go-Ahead-Got-It
+G-i-g-a-b-o-t
+GermCrawler
 Gotit
 GrabNet
+Grabber
 Grafula
 GrapeFX
 GrapeshotCrawler
-HEADMasterSEO
+HMView
 HTMLparser
 HTTP::Lite
-Haansoft
-HaosouSpider
+HTTrack
+Harvest
 Havij
 Hloader
-IDBTE4M
+Humanlinks
+IDBot
+IRLbot
 IlseBot
-IndeedBot
+Image Fetch
+Image Sucker
 InfoNaviRobot
+Intelliseek
 Internet Ninja
-Iria
-Iskanie
+InternetSeer
 IstellaBot
-JennyBot
-JetCar
-JikeSpider
+JOC Web Spider
+Joomla
 Jorgee
-Jyxobot
+Kenjin Spider
 Keyword Density
-Kinza
-LibWeb
+Kozmosbot
+LWP::Simple
+Lanshanbot
+LeechGet
+LexiBot
+Lftp
 Libwhisker
 LieBaoFast
+Likse
 LinkScan
-Lipperhey Spider
-MIIxpc
+LinkWalker
+Linkbot
+LinkextractorPro
+LinksManager
+LinqiaScrapeBot
+Lmspider
+Ltx71
+MFC_Tear_Sample
+MSFrontPage
 MSIECrawler
 MTRobot
 Mag-Net
-Magnet
-Majestic SEO
-Majestic-SEO
 MarkWatch
-Masscan
-MauiBot
-Mb2345Browser
 MeanPath Bot
 Meanpathbot
-Microsoft URL Control
+MegaIndex.ru
+Metauri
+MicroMessenger
 Mister PiX
-Mojeek
-Mojolicious
-Mozlila
+Moblie Safari
+Morfeus Fucking Scanner
+Mr.4x3
 Msrabot
-Name Intelligence
+Navroad
 Net Vampire
+NetAnts
 NetLyzer
+NetMechanic
 Netcraft
-Nettrack
-Netvibes
 NextGenSearchBot
-Niki-bot
-Nimbostratus
-Nmap
+Nikto
 Nuclei
+Nutch
 Octopus
-Offline Explorer
-OnCrawl
-OpenLinkProfiler
-OpenVAS
-OrangeSpider
+Offline Navigator
+OrangeBot
+OutclicksBot
 OutfoxBot
+PHPCrawl
 POE-Component-Client-HTTP
-PageAnalyzer
-PageGrabber
+Page Analyzer
 Pandalytics
-Papa Foto
-Pavuk
-Pi-Monster
-Picscout
+PeoplePal
+Piepmatz
+Pimonster
 PleaseCrawl
-Pockey
 Probethenet
-Pu_iN
-Pump
-QueryN Metasearch
-RankActive
-RankingBot2
-Rankivabot
-RankurBot
-Re-re
+Psbot
+PxBroker
+Quick-Crawler
+RSSingBot
+RankFlex
+Recorder
+RedesScrapy
+RepoMonkey
 Ripper
+RocketCrawler
 SBIder
 SEOkicks
-SISTRIX
-SalesIntelligent
 ScanAlert
-Scanbot
-ScoutJet
-Screaming
-ScrepyBot
 SearchmetricsBot
-SentiBot
-SeobilityBot
+Semrush
+Seomoz
 Site Sucker
 SiteSnagger
 SiteSucker
-Sitebeam
-Sitevigil
-SlySearch
+Siteimprove
+SmartDownload
 Snake
-Snoopy
-SpankBot
-Spanner
-SputnikBot
-Sqlworm
+Sociscraper
+Sogou web spider
+Sosospider
+Spbot
 Sqworm
 Steeler
-Sucuri
-Swiftbot
+Stripper
+SuperBot
+SuperHTTP
+Surfbot
+SurveyBot
 Szukacz
-T0PHackTeam
-Teleport
-Telesoft
-Telesphorep
-TightTwatBot
-Titan
-Toweyabot
-Tracemyfile
+T8Abot
+TeleportPro
+Telesphoreo
+The Intraformant
+TheNomad
 Trendiction
-Trendictionbot
-True_Robot
-Turnitin
+Turingos
 TurnitinBot
-Twice
-URLy.Warning
-Upflow
-VCI
+TwengaBot
+VB Project
 Vagabondo
 VelenPublicWebCrawler
-Virusdie
 VoidEYE
-WASALive-Bot
+Voltron
+WBSearchBot
 WEBDAV
-WISENutbot
-WPScan
-WWW-Collector-E
-WWW-Mechanize
+WWW::Mechanize
 WWWOFFLE
-Wallpapers
+Wallpapers/3.0
 Web Collage
-Web Enhancer
-Web Sauger
+Web Fuck
+Web Pix
 Web Sucker
 WebCollage
-WebCopier
-WebFetch
-WebPix
-WebSauger
-WebSucker
-Webalta
-WebmasterWorldForumBot
+WebGo IS
+WebImageCollector
+WebStripper
+WebZIP
 Website Quester
 WebsiteExtractor
-Webster
-Whack
-Widow
-Wotbox
+WebsiteQuester
+Whacker
+Whatweb
+Woobot
 Wprecon
-Zeus
+Zade
+Zauba
+Zitebot
+ZoomBot
+ZoominfoBot
+ZumBot
 ZyBorg
-arquivo.pt
+autoemailspider
 cah.io.community
 clark-crawler
-gopher
+coccocbot-web
+cognitiveseo
+com.plumanalytics
+crawl.sogou.com
+crawler.feedback
+dataforseo.com
+eCatch
+evc-batch
+heritrix
+ips-agent
+lwp-request
+lwp-trivial
+magpie-crawler
 muhstik-scan
 oBot
 pcBrowser
-polaris version
+plumanalytics
 s1z.ru
-scan.lol
-seoscanners
 seostar
-sitechecker.pro
 siteripz
 sogouspider
-sp_auditbot
+spyfu
+tAkeOut
 trendiction.com
-voyagerx.com
-webpros.com
-zauba.io
+trendiction.de
+webmeup-crawler