V4.2021.10.2672 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
-80legs
-ADmantX
-AfD-Verbotsverfahren
-AhrefsBot
-Aipbot
-AllSubmitter
+360Spider
+AIBOT
+ASPSeek
+Aboundex
+Aboundexbot
+Alexibot
+Anarchie
 Anarchy
 Anarchy99
-Ankit
+Anthill
 Apexoo
-BDCbot
-BDFetch
-BLEXBot
+Aspiegel
+Attach
+AwarioRssBot
+AwarioSmartBot
+BackStreet
+Backlink-Ceck
 Badass
-Blackboard
+Bandit
+Barkrowler
+BetaBot
+Bitacle
+Black Hole
+Blow
 BlowFish
-Boardreader
-Brandprotect
-Buck
-Buddy
+Brandwatch
+BuiltBotTough
+BuiltWith
 Bullseye
-Calculon
+BuzzSumo
+CODE87
 CazoodleBot
 Chlooe
-Cliqzbot
-Cloud mapping
-Cocolyzebot
-Cosmos
-Custo
-CyotekWebCopy
-DBLBot
+Cogentbot
+Collector
+Copyscape
+Crawling at Home Project
+Crescent
 DSearch
-DataCha0s
+DTS Agent
 DatabaseDriverMysqli
-Demon
-Deusu
-Devil
-Dirbuster
-Discoverybot
+Digincore
+Disco
 Dispatch
-DittoSpyder
-DnBCrawler-Analytics
-DomainCrawler
-DomainStatsBot
-Download Wonder
-Dragonfly
-ECCP/1.0
-Ecxi
+DomCopBot
+EMail Siphon
+EMail Wolf
+EasyDL
+Ebingbong
+EroCrawler
 Evil
-Express WebPictures
-Extractor
-Ezooms
-Flunky
-Fuzz
-G-i-g-a-b-o-t
+Extreme Picture Finder
+EyeNetIE
+FHscan
 GalaxyBot
-GetRight
+GermCrawler
 GetWeb
-Getintent
-Go!Zilla
+Gigabot
 Go-Ahead-Got-It
 GoZilla
-Gotit
-GrabNet
+Grafula
 GrapeFX
 GrapeshotCrawler
-GridBot
-Haansoft
+HTMLparser
 HaosouSpider
+Harvest
 Havij
 HybridBot
+IDBTE4M
 IDBot
 IRLbot
-Id-search
-IlseBot
 Image Fetch
 Image Sucker
-Intelliseek
-Iskanie
+IndeedBot
 JOC Web Spider
-JennyBot
-JetCar
-Jetty
-Kenjin Spider
-Kinza
-LNSpiderguy
+JikeSpider
+Jorgee
+JustView
+Jyxobot
+Keyword Density
+Kozmosbot
 LWP::Simple
-Leap
-LibWeb
-LinkScan
-Linkbot
-Linkdexbot
-LinkpadBot
-LinksManager
-LinqiaRSSBot
-LinqiaScrapeBot
+LeechFTP
+Lftp
 Lipperhey
 Litemage_walker
 Lmspider
 Ltx71
-MFC_Tear_Sample
 MIDown tool
-MJ12bot
 MQQBrowser
-MSFrontPage
-Magnet
-Mail.RU_Bot
-Majestic-SEO
-MarkWatch
+MSIECrawler
+MTRobot
+Majestic12
+MarkMonitor
 Mass Downloader
-MauiBot
-Mb2345Browser
-Meanpathbot
+Masscan
 MegaIndex.ru
-Microsoft URL Control
+MicroMessenger
+Microsoft Data Access
 Mister PiX
-Mojeek
-Musobot
-NICErsPRO
-NPbot
+Moblie Safari
+Mozlila
+Mr.4x3
 Nameprotect
 Needle
-Nessus
+Net Vampire
+NetMechanic
 NetSpider
-Nibbler
-Niki-bot
-NimbleCrawler
+Nettrack
+Netvibes
 Nimbostratus
+Ninja
 Nmap
 Nuclei
-Nutch
 Octopus
-OnCrawl
-OpenLinkProfiler
-OpenVAS
+Offline Explorer
 Openfind
+Openvas
+OrangeBot
 OrangeSpider
-OutfoxBot
 PECL::HTTP
-PageGrabber
-Pandalytics
+POE-Component-Client-HTTP
+Page Analyzer
+PageAnalyzer
+Papa Foto
 Pavuk
-PeoplePal
-Pi-Monster
+Picscout
+Picsearch
 PictureFinder
-Pockey
-Psbot
+Pimonster
+Pixray
+Probethenet
 Pu_iN
-Pump
-PxBroker
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
 RankActiveLinkBot
+RankFlex
 RankingBot
-RankurBot
+Rankivabot
 ReGet
-RealDownload
-RebelMouse
-Recorder
+Reaper
 RepoMonkey
-SEOlyticsCrawler
+RocketCrawler
+SBIder
+SEOkicks
 SISTRIX
-SMTBot
 SalesIntelligent
-Scanbot
+ScanAlert
 ScoutJet
+Scrapy
+SearchmetricsBot
+SemrushBot
 SentiBot
-SeoSiteCheckup
-Shodan
+Seomoz
 Siphon
-Site Sucker
-SiteCheckerBotCrawler
-SiteExplorer
-Siteimprove
+SiteLockSpider
+SiteSnagger
+Sitebeam
+Sitevigil
 SmartDownload
+Snoopy
 SocialRankIOBot
-Sogou web spider
-Sosospider
-Sottopop
 SpaceBison
-SpankBot
+Spammen
+Spbot
+Spinn3r
+Sqlmap
 Sqworm
 Steeler
+Sucuri
+Surfbot
+SurveyBot
+Swiftbot
+Teleport
+TeleportPro
+Telesphoreo
 Telesphorep
+TightTwatBot
 Titan
 Toweyabot
-Tracemyfile
-Trendictionbot
-True_Robot
-Turingos
 TurnitinBot
-URLy Warning
+Twice
+Typhoeus
+UnisterBot
+Upflow
 V-BOT
+Vacuum
 VidibleScraper
 Virusdie
-WallpapersHD
-Web Enhancer
+VoidEYE
+Voil
+WASALive-Bot
+WISENutbot
+WWW-Mechanize
+WWW::Mechanize
+Wallpapers
+Web Auto
+Web Collage
 Web Fetch
-Web Fuck
-WebBandit
-WebGo IS
-WebPix
-WebReaper
-WebSauger
-WebStripper
+Web Sucker
+WebEnhancer
 WebSucker
-WebWhacker
-Website Quester
+WebZIP
+WebmasterWorldForumBot
+Webshag
+WebsiteExtractor
+WebsiteQuester
 Webster
-Whack
 Whacker
 Whatweb
 Who.is Bot
 WiseGuys Robot
 Wonderbot
-Xaldon WebSpider
+Woobot
+Wotbox
 Xenu
-Zade
-Zeus
-archive.org_bot
-arquivo-web-crawler
-clark-crawler
-coccocbot-web
-com.plumanalytics
-crawl.sogou.com
+YoudaoBot
+Zauba
+Zitebot
+ZmEu
+ZoominfoBot
+backlink-check
+cah.io.community
+check1.exe
+cognitiveseo
+crawler.feedback
+crawler4j
 demandbase-bot
 domainsproject.org
-eCatch
-evc-batch
 facebookscraper
-instabid
+gopher
+heritrix
 internetVista monitor
-ips-agent
 isitwp.com
+lwp-request
 lwp-trivial
-netEstate NE Crawler
 polaris version
-ripz
+probe-image-size
 s1z.ru
+satoristudio.net
 scalaj-http
-scan.lol
-seoscanners
-sitechecker.pro
-sogouspider
+seobility
+sp_auditbot
 spyfu
-sysscan
-tAkeOut
 trendiction.de
-voyagerx.com
 webpros.com
 x09Mozilla
-zauba.io
+x22Mozilla
+zgrab