V4.2021.09.2592 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
-404checker
+01h4x.com
 ADmantX
+AIBOT
 ASPSeek
 Abonti
-AfD-Verbotsverfahren
-Anarchy99
-Aspiegel
-Asterias
-AwarioRssBot
-BDFetch
+AhrefsBot
+Aipbot
+Alexibot
+AllSubmitter
+Alligator
+AlphaBot
 BLEXBot
+BackWeb
 Backlink-Ceck
-Bandit
-Barkrowler
-BatchFTP
-Battleztar Bazinga
 Bigfoot
-Bitacle
+BlackWidow
+Blow
 Boardreader
-Bolt
+BotALot
+Brandprotect
+Brandwatch
 Buck
 Buddy
 BuiltBotTough
-BunnySlippers
+BuiltWith
+Bullseye
+BuzzSumo
 CCBot
-CODE87
 Calculon
-CazoodleBot
-Cegbfeieh
-ChinaClaw
+CheeseBot
 Cliqzbot
 Cloud mapping
+Cocolyzebot
 Cogentbot
 Collector
 Copier
 Copyscape
 Cosmos
+Crawling at Home Project
 CrazyWebCrawler
-CrunchBot
-Curious
-DTS Agent
-DatabaseDriverMysqli
+Crescent
+Custo
+CyotekWebCopy
+DBLBot
+DIIbot
+DataCha0s
 Demon
 Devil
 Digincore
+DigitalPebble
 Dirbuster
 Disco
-Discobot
 Discoverybot
+DittoSpyder
 DomCopBot
-DomainAppender
 DomainSigmaCrawler
+DomainStatsBot
 Dotbot
-ECCP/1.0
-EMail Wolf
-Ebingbong
-EroCrawler
+Download Wonder
+Dragonfly
 Exabot
-ExtLinksBot
-Extractor
-ExtractorPro
+EyeNetIE
+FDM
 FHscan
-Flunky
-Freeuploader
+Fimap
 Fyrebot
-GT::WWW
+Genieo
 GermCrawler
-Getintent
 GoZilla
-GrabNet
+Gotit
+Grabber
 GrapeFX
-GridBot
-HEADMasterSEO
-HTMLparser
-Harvest
-Humanlinks
+HMView
+HTTP::Lite
+HybridBot
+IDBTE4M
 IDBot
-IRLbot
+Iblog
+IlseBot
+Image Fetch
 Indy Library
-InfoNaviRobot
 InfoTekies
-InterGET
-InternetSeer
+Iria
+Iskanie
 IstellaBot
+Jbrofuzz
 JennyBot
+JetCar
+Jetty
 Joomla
-Jyxobot
-Kenjin Spider
-Keyword Density
+Jorgee
+JustView
 Kozmosbot
+LNSpiderguy
+LWP::Simple
+Larbin
 Leap
-LinkScan
-LinqiaMetadataDownloaderBot
+Lftp
+LibWeb
+Libwhisker
+LieBaoFast
+Linkbot
+LinksManager
 LinqiaRSSBot
 LinqiaScrapeBot
 Lipperhey
-Litemage_walker
-Lmspider
 Ltx71
-MFC_Tear_Sample
-MJ12bot
-MSFrontPage
-Mail.RU_Bot
-MarkMonitor
+MIDown tool
+Mag-Net
+Magnet
+Majestic SEO
+Majestic-SEO
+Majestic12
 MarkWatch
-Masscan
-Metauri
+Mass Downloader
+Mb2345Browser
+MeanPath Bot
+Mediatoolkitbot
 Microsoft Data Access
+Microsoft URL Control
 Mister PiX
+Mojolicious
+Mr.4x3
 Musobot
-NICErsPRO
 Name Intelligence
-NearSite
+Nameprotect
 Nessus
-NetAnts
-NetLyzer
-NetMechanic
+NetSpider
 Netcraft
-NextGenSearchBot
+Nettrack
+Netvibes
 Nibbler
 Niki-bot
-Nikto
-Nimbostratus
-Nmap
-Octopus
+Offline Navigator
 OpenVAS
-PageAnalyzer
-PageGrabber
+OrangeBot
+OutfoxBot
+Page Analyzer
 PageScorer
-PageThing.com
-Pavuk
+Pandalytics
+Papa Foto
 Picscout
-Picsearch
 Pixray
+PleaseCrawl
 ProPowerBot
-Pu_iN
-Quick-Crawler
-RankActiveLinkBot
-RealDownload
+Probethenet
+PyCurl
+QueryN Metasearch
+RSSingBot
+RankFlex
+RankingBot
+ReGet
+Reaper
 RebelMouse
-RedesScrapy
-RepoMonkey
-Ripper
-SEOlyticsCrawler
-SEOprofiler
-SEOstats
-SalesIntelligent
-Scrapy
-Screaming
+Recorder
+SBIder
+SMTBot
+ScreenerBot
 ScrepyBot
+SearchmetricsBot
+Semrush
 SemrushBot
 SeoSiteCheckup
-Seomoz
-Shodan
 Siphon
-SiteLockSpider
-SiteSnagger
-Sitevigil
-Snake
-Sogou web spider
-SpaceBison
-Spanner
+Siteimprove
+SlySearch
+Snapbot
+Snoopy
+SocialRankIOBot
+SpankBot
 Spbot
-Spinn3r
-SputnikBot
-Sqlmap
 Sqlworm
-Steeler
+Stripper
 SuperBot
 Suzuran
 Swiftbot
 Teleport
 Telesoft
-Telesphorep
 The Intraformant
 Thumbor
-Tracemyfile
-Trendiction
-True_Robot
+Turnitin
 TurnitinBot
-Typhoeus
-URLy.Warning
+TwengaBot
+URLy Warning
 UnisterBot
-VCI
+V-BOT
+VB Project
+Vacuum
+Vagabondo
 VelenPublicWebCrawler
 VeriCiteCrawler
-VidibleScraper
+VoidEYE
 Voil
-WASALive-Bot
-WBSearchBot
 WEBDAV
-WWW::Mechanize
+WPScan
 WWWOFFLE
-Wallpapers/3.0
-WeSEE
-Web Fetch
-Web Pix
+Web Fuck
 Web Sauger
-Web Sucker
 WebAuto
-WebCopier
-WebFuck
-WebGo IS
-WebLeacher
+WebEnhancer
+WebImageCollector
+WebPix
 WebStripper
-WebSucker
-WebWhacker
+WebZIP
 WebsiteExtractor
 WebsiteQuester
-Whack
+Webster
+Whacker
 Whatweb
+WinHTTrack
 WiseGuys Robot
-Wprecon
-Xaldon WebSpider
-Xenu
-YoudaoBot
+Wonderbot
+Woobot
+Wotbox
+Zade
 Zauba
+ZoomBot
 ZumBot
+arquivo.pt
 autoemailspider
+backlink-check
 check1.exe
+clark-crawler
+coccocbot-web
 crawl.sogou.com
 crawler.feedback
-crawler4j
-demandbase-bot
 eCatch
 evc-batch
-facebookscraper
 heritrix
+internetVista monitor
 isitwp.com
-lwp-request
-lwp-trivial
-magpie-crawler
 meanpathbot
-mediawords
-netEstate NE Crawler
-plumanalytics
-polaris version
-s1z.ru
+oBot
+page scorer
+probe-image-size
 satoristudio.net
-seobility
+scan.lol
 sexsearcher
+sitechecker.pro
+siteripz
+sogouspider
+sp_auditbot
 spyfu
+sysscan
 tAkeOut
-trendiction.com
+trendiction.de
 ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
 webprosbot
-x09Mozilla
-zauba.io
+x22Mozilla
+zgrab