V4.2021.08.2558 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
 360Spider
-404checker
-ALittle Client
+ADmantX
 ASPSeek
-Abonti
 Aboundex
-Acunetix
-AhrefsBot
+AfD-Verbotsverfahren
+AiHitBot
+Alligator
 Anarchie
-Anarchy99
+Anthill
 Apexoo
-BBBike
-BLEXBot
-BackWeb
-Backlink-Ceck
+Aspiegel
+Asterias
+Attach
+BackStreet
 BacklinkCrawler
+Barkrowler
 BatchFTP
 Battleztar Bazinga
+BetaBot
 Bigfoot
-Blackboard
+Bitacle
 Blow
-Bolt
+Boardreader
 BotALot
 Brandprotect
-Brandwatch
+BuiltBotTough
+Bullseye
 BunnySlippers
-BuzzSumo
-CCBot
 CODE87
-CazoodleBot
 CheeseBot
 ChinaClaw
-Claritybot
 Cloud mapping
-Cocolyzebot
-Cogentbot
-Collector
+Copier
 CopyRightCheck
 Copyscape
 Cosmos
 Crawling at Home Project
-CrazyWebCrawler
-CrunchBot
+Custo
+CyotekWebCopy
+DBLBot
 DIIbot
-DSearch
+DataCha0s
+Demon
 Deusu
 Devil
-Digincore
-DigitalPebble
-DomainAppender
-Dotbot
-EMail Siphon
+Disco
+Discobot
+Dispatch
+DomCopBot
+Drip
 EMail Wolf
 EasyDL
 Evil
-Exabot
-Express WebPictures
+ExtractorPro
 Extreme Picture Finder
-Ezooms
+FHscan
 FlashGet
+Flunky
 Freeuploader
-FrontPage
-FyberSpider
-G-i-g-a-b-o-t
+GalaxyBot
+GermCrawler
+GetRight
 GetWeb
-Getintent
+Gigabot
 GoZilla
-GrabNet
 Grafula
-GrapeshotCrawler
-HMView
 HTMLparser
-Harvest
-Hloader
+Haansoft
+HaosouSpider
 Humanlinks
 IDBTE4M
-Iblog
+IDBot
 Id-search
+IlseBot
 Image Fetch
 Indy Library
 InfoTekies
-JOC Web Spider
+InterGET
+InternetSeer
+JennyBot
 JetCar
 Jetty
-Joomla
-JustView
+Jorgee
 Keyword Density
 LNSpiderguy
+Leap
 LexiBot
-Lftp
 Libwhisker
 Likse
-LinkScan
 Linkbot
+Linkdexbot
 LinkextractorPro
-LinqiaScrapeBot
+LinkpadBot
+LinksManager
+LinqiaRSSBot
+Lipperhey
+Ltx71
 MIDown tool
-MIIxpc
-MJ12bot
+MSFrontPage
 MSIECrawler
 MTRobot
-Magnet
-Majestic SEO
+Mag-Net
 MarkMonitor
-Mass Downloader
+Masscan
+Mb2345Browser
 MeanPath Bot
 Meanpathbot
-Mediatoolkitbot
-MegaIndex.ru
 Metauri
-MicroMessenger
 Microsoft Data Access
-Microsoft URL Control
 Mister PiX
 Moblie Safari
 Mojolicious
+Morfeus Fucking Scanner
 Mozlila
 Mr.4x3
-Musobot
-NICErsPRO
-NPbot
+Name Intelligence
+Navroad
 Needle
+Nessus
+NetAnts
 NetLyzer
-NetMechanic
-NetSpider
 NetZIP
-Niki-bot
-NimbleCrawler
-Ninja
+Netcraft
+Netvibes
+Nibbler
+Nikto
 Nmap
+Nuclei
 Nutch
-Octopus
-OpenLinkProfiler
+Offline Explorer
+OnCrawl
+OpenVAS
+Openfind
+Openvas
 OrangeBot
-PECL::HTTP
-POE-Component-Client-HTTP
-Panscient
-Papa Foto
+PageGrabber
+PageScorer
+PeoplePal
 Petalbot
 Pi-Monster
-Picscout
-PictureFinder
-Pimonster
-Pixray
-Pockey
-PyCurl
-QueryN Metasearch
-RSSingBot
-RankActive
+Picsearch
+Piepmatz
+PleaseCrawl
+Psbot
+Pu_iN
+PxBroker
+RankFlex
+RankingBot2
+Rankivabot
+RankurBot
+ReGet
 RealDownload
+Reaper
 RebelMouse
+RedesScrapy
 Ripper
-RocketCrawler
 Rogerbot
 SBIder
-SEOkicks-Robot
-SEOstats
-SISTRIX
-SMTBot
-ScoutJet
-Screaming
-ScrepyBot
+SemrushBot
 SentiBot
 Seomoz
-Siphon
-Site Sucker
-SiteSucker
-Sociscraper
-SpankBot
+Sitebeam
+Siteimprove
+SmartDownload
+Sosospider
+SpaceBison
+Spammen
 Spanner
-Spinn3r
-SputnikBot
+Sqworm
 Steeler
 Stripper
+Sucker
+Sucuri
 Surfbot
 Swiftbot
+Szukacz
 T8Abot
 Teleport
 TeleportPro
-Telesoft
-Telesphorep
+Telesphoreo
+The Intraformant
 TheNomad
+TightTwatBot
 Toata
-Turingos
+Trendiction
+True_Robot
+Turnitin
 TurnitinBot
 TwengaBot
-Twice
 Typhoeus
 URLy Warning
-VCI
-Vagabondo
-Virusdie
-WBSearchBot
-WEBDAV
+URLy.Warning
+UnisterBot
+Upflow
+V-BOT
+VB Project
+Vacuum
+Voltron
 WPScan
-Wallpapers/3.0
-WallpapersHD
-WeSEE
-Web Collage
-Web Fuck
-Web Sauger
-Web Sucker
+WWW-Mechanize
+Web Auto
+Web Fetch
+Web Pix
 WebAuto
-WebCollage
-WebFetch
-WebFuck
-WebImageCollector
-WebLeacher
-WebReaper
-WebWhacker
-Webalta
+WebBandit
+WebEnhancer
+WebPix
+WebStripper
 Webshag
+Website Quester
 WebsiteExtractor
 Webster
 Whack
-Widow
-WinHTTrack
-WiseGuys Robot
 Wonderbot
+Woobot
 Wotbox
 Wprecon
-Xaldon WebSpider
-Xenu
-Zade
-Zermelo
-Zitebot
+Xaldon_WebSpider
+YoudaoBot
+Zeus
 ZoomBot
 ZoominfoBot
-ZumBot
-archive.org_bot
+arquivo-web-crawler
+arquivo.pt
 autoemailspider
-crawl.sogou.com
+clark-crawler
+cognitiveseo
 crawler.feedback
+domainsproject.org
 gopher
-instabid
+ips-agent
+lwp-request
 lwp-trivial
-muhstik-scan
-ripz
+meanpathbot
+page scorer
+pcBrowser
+polaris version
 satoristudio.net
 seobility
 seoscanners
-serpstatbot
+sexsearcher
 sogouspider
-sysscan
+sp_auditbot
+spyfu
 tAkeOut
+trendiction.com
+ubermetrics-technologies.com
 webmeup-crawler
-webprosbot
 x22Mozilla
 zauba.io
-zgrab