V4.2021.09.2614 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
-360Spider
-80legs
-AIBOT
-Abonti
-AhrefsBot
-AiHitBot
-Anarchy
+01h4x.com
+404checker
+ADmantX
+ALittle Client
+ASPSeek
+Aboundexbot
+Acunetix
+AfD-Verbotsverfahren
+Alexibot
+AllSubmitter
+AlphaBot
+Ankit
+Apexoo
 Aspiegel
-AwarioRssBot
-BDFetch
-BackStreet
-Backlink-Ceck
-Barkrowler
+BLEXBot
+BackWeb
+BacklinkCrawler
+Badass
+BatchFTP
 Battleztar Bazinga
-BetaBot
-Blow
 BlowFish
-BotALot
+Boardreader
 Brandprotect
 Brandwatch
-Buddy
+Buck
 BuiltBotTough
 BuiltWith
-BuzzSumo
-CCBot
-CSHttp
-Calculon
-CensysInspect
-CheeseBot
-CherryPicker
-Chlooe
-Cliqzbot
+ChinaClaw
 Cloud mapping
 Cocolyzebot
-Copier
-CrazyWebCrawler
+Cogentbot
+Collector
+CopyRightCheck
+Copyscape
 CrunchBot
+Custo
+CyotekWebCopy
 DBLBot
 DIIbot
-DatabaseDriverMysqli
+DTS Agent
 Demon
-Digincore
-DnyzBot
+Deusu
+DigitalPebble
+Discobot
+Dispatch
+DittoSpyder
 DomainCrawler
 Domains Project
 Download Wonder
-EasyDL
-Ebingbong
+EMail Wolf
+Ecxi
 EirGrabber
-EroCrawler
-Evil
-ExtractorPro
+Exabot
+ExtLinksBot
+Extreme Picture Finder
 Ezooms
-FDM
+FHscan
+Fimap
 FlashGet
-Foobot
-Freeuploader
+Flunky
 FrontPage
-Fuzz
-FyberSpider
-Fyrebot
+G-i-g-a-b-o-t
+GT::WWW
 GalaxyBot
 Genieo
-GetWeb
-Getintent
-Gigabot
+GermCrawler
+GetRight
 Go!Zilla
-Go-Ahead-Got-It
 GoZilla
-GrapeshotCrawler
-GridBot
+Gotit
+GrapeFX
 HTTP::Lite
 HTTrack
+Haansoft
+HaosouSpider
+Harvest
+Havij
 Hloader
-HybridBot
 IDBTE4M
+IDBot
 IRLbot
 Iblog
-InfoNaviRobot
+Image Sucker
+InfoTekies
 Intelliseek
-Internet Ninja
+InterGET
 InternetSeer
 Iria
-JennyBot
-JetCar
-Jetty
-JikeSpider
+Joomla
 JustView
 Kenjin Spider
 Kinza
-LNSpiderguy
 LWP::Simple
-Larbin
-LeechGet
-LexiBot
+LeechFTP
+LibWeb
 Libwhisker
-Lightspeedsystems
-LinkWalker
-LinksManager
-LinqiaMetadataDownloaderBot
+Linkbot
+LinkpadBot
+LinqiaRSSBot
 Lipperhey
 Lipperhey Spider
-MSFrontPage
-Mag-Net
-Magnet
-Majestic SEO
+MFC_Tear_Sample
+MIDown tool
+MIIxpc
+MJ12bot
+MQQBrowser
+MSIECrawler
+Mail.RU_Bot
+Majestic-SEO
+Majestic12
 MarkMonitor
-MarkWatch
-Mass Downloader
-MauiBot
 MeanPath Bot
 Meanpathbot
-MicroMessenger
+Mediatoolkitbot
 Microsoft Data Access
-Moblie Safari
-Mojeek
+Mozlila
 Mr.4x3
 Msrabot
-NPbot
 Nameprotect
+Navroad
 NearSite
-Nessus
-Net Vampire
-NetAnts
-NetZIP
+NetLyzer
 Nettrack
-NimbleCrawler
+Niki-bot
 Nimbostratus
-Offline Explorer
+Ninja
+Nuclei
+Nutch
+Offline Navigator
 OpenLinkProfiler
-Openvas
-OutclicksBot
-OutfoxBot
-Papa Foto
-Pavuk
-Petalbot
-Picsearch
-Pimonster
-Pixray
+POE-Component-Client-HTTP
+Page Analyzer
+Pandalytics
+PeoplePal
+Pi-Monster
+PleaseCrawl
+Pockey
+Probethenet
 Psbot
+Pu_iN
+Pump
+PxBroker
 RankActive
-RankActiveLinkBot
-RankingBot
 RankingBot2
 Rankivabot
-RealDownload
+RankurBot
+ReGet
 Reaper
-RebelMouse
 Recorder
+RedesScrapy
 RepoMonkey
-SBIder
+Ripper
+Rogerbot
+SEOkicks
 SEOkicks-Robot
-ScrepyBot
+SEOlyticsCrawler
+ScanAlert
+ScoutJet
+Scrapy
 SearchmetricsBot
-Semrush
+SemrushBot
+SentiBot
+SeoSiteCheckup
+SeobilityBot
 Seomoz
+Siphon
 Site Sucker
-SiteCheckerBotCrawler
 SiteLockSpider
-SiteSnagger
 SiteSucker
-Sitebeam
 Sitevigil
-Snapbot
-Spammen
+Snoopy
+SocialRankIOBot
+Sogou web spider
+Sosospider
+Sottopop
+SpankBot
+SputnikBot
+Sqlworm
 Steeler
-Sucker
 Sucuri
-Surfbot
-Suzuran
-T0PHackTeam
+SuperHTTP
 Teleport
 Telesoft
-Telesphoreo
-TheNomad
+The Intraformant
+Thumbor
 TightTwatBot
-Titan
+Toweyabot
 Tracemyfile
-Trendiction
 Trendictionbot
 Turingos
+TwengaBot
+Twice
+Typhoeus
+URLy Warning
 UnisterBot
-Upflow
 VB Project
-Vacuum
-Vagabondo
-VelenPublicWebCrawler
+VCI
+VeriCiteCrawler
+VidibleScraper
+VoidEYE
 WASALive-Bot
-WEBDAV
 WISENutbot
 WWW-Mechanize
+WWWOFFLE
+Wallpapers/3.0
 Web Auto
 Web Collage
 Web Fetch
-WebBandit
-WebCollage
+Web Pix
 WebCopier
-WebGo IS
-WebImageCollector
-WebLeacher
+WebFuck
 WebPix
-WebReaper
-WebWhacker
+WebSauger
 WebZIP
-Webalta
-Webster
-Whacker
-Whatweb
-Who.is Bot
-WiseGuys Robot
-Xaldon_WebSpider
-Xenu
+Website Quester
+WebsiteQuester
+Wonderbot
+Woobot
+Wprecon
+YoudaoBot
 Zade
-Zermelo
-ZmEu
-ZoominfoBot
+Zauba
+Zeus
+ZoomBot
 ZumBot
 arquivo.pt
 clark-crawler
+coccocbot-web
 cognitiveseo
 com.plumanalytics
-crawl.sogou.com
-crawler.feedback
-dataforseo.com
+crawler4j
 domainsproject.org
-gopher
+evc-batch
+facebookscraper
+heritrix
 internetVista monitor
-isitwp.com
 lwp-trivial
-meanpathbot
-mediawords
-muhstik-scan
-oBot
+netEstate NE Crawler
 page scorer
 pcBrowser
-polaris version
+plumanalytics
 probe-image-size
-s1z.ru
-satoristudio.net
+ripz
 scan.lol
+seoscanners
 seostar
-serpstatbot
+sexsearcher
 sogouspider
-sp_auditbot
-sysscan
 tAkeOut
-trendiction.com
-ubermetrics-technologies.com
-webprosbot
-x09Mozilla
+webmeup-crawler