V4.2021.11.2716 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
+404checker
 80legs
-ALittle Client
+ADmantX
+AIBOT
+ASPSeek
 Aboundexbot
-Acunetix
-AhrefsBot
-Alexibot
+AfD-Verbotsverfahren
+Alligator
 Anarchy99
-Anthill
-Apexoo
-AwarioRssBot
-BDCbot
+Aspiegel
+Asterias
+Attach
 BDFetch
 BLEXBot
-BackDoorBot
-BackStreet
-Backlink-Ceck
-Badass
-BetaBot
+Barkrowler
 Blackboard
 BlowFish
-BotALot
-Brandwatch
+Bolt
+Brandprotect
+Buck
+BuiltWith
 BunnySlippers
 CATExplorador
+CODE87
 Calculon
 CazoodleBot
-Cegbfeieh
-CheeseBot
-CherryPicker
-Claritybot
+CensysInspect
+CheTeam
+Chlooe
+Cliqzbot
 Cloud mapping
-Cocolyzebot
-Cogentbot
-Collector
 CopyRightCheck
-CrunchBot
+Copyscape
 Curious
+Custo
 DBLBot
-DIIbot
 DSearch
 DTS Agent
-DataCha0s
 Demon
-Deusu
 Digincore
-Discoverybot
-DittoSpyder
-DomCopBot
+Discobot
+DnBCrawler-Analytics
+DnyzBot
+DomainAppender
+DomainCrawler
 DomainSigmaCrawler
-Dotbot
-Dragonfly
+DomainStatsBot
+Download Wonder
+Drip
+EMail Siphon
 EasyDL
 Ebingbong
+Ecxi
+EroCrawler
 Exabot
-Extractor
-EyeNetIE
-Fimap
-Firefox/7.0
-Flunky
-Foobot
+Express WebPictures
+Extreme Picture Finder
+FemtosearchBot
 Freeuploader
-FyberSpider
-G-i-g-a-b-o-t
-Genieo
-GetWeb
-Go-Ahead-Got-It
+Fyrebot
+GermCrawler
+GetRight
+Gigabot
 Gotit
-Grabber
-GrapeshotCrawler
+GridBot
+HEADMasterSEO
+HTTP::Lite
+Harvest
 Heritrix
+Hloader
 HybridBot
-IRLbot
+IDBTE4M
+IDBot
+Iblog
+Id-search
 IlseBot
-Image Fetch
-InfoNaviRobot
+IndeedBot
+Indy Library
 InternetSeer
-Iskanie
-IstellaBot
-Jbrofuzz
-JetCar
+JennyBot
+Jetty
 JikeSpider
 Joomla
+Jorgee
+JustView
 Jyxobot
-Kenjin Spider
 Keyword Density
-Kinza
-Kozmosbot
-LNSpiderguy
 LWP::Simple
-Larbin
-Leap
+LeechFTP
 LexiBot
+LibWeb
 Libwhisker
-LieBaoFast
-Lightspeedsystems
-LinkScan
-LinkWalker
-Linkbot
-Linkdexbot
 LinkextractorPro
-LinkpadBot
-LinksManager
-LinqiaMetadataDownloaderBot
 LinqiaRSSBot
+LinqiaScrapeBot
 Lipperhey
-Litemage_walker
-Lmspider
 Ltx71
+MFC_Tear_Sample
 MIDown tool
-Mag-Net
-Majestic-SEO
+MTRobot
+Mail.RU_Bot
 Majestic12
-Mass Downloader
+Mata Hari
 MauiBot
 Mb2345Browser
 Meanpathbot
+Mediatoolkitbot
+MegaIndex.ru
 Metauri
+Microsoft Data Access
 Microsoft URL Control
+Moblie Safari
 Mojeek
 Morfeus Fucking Scanner
-Mr.4x3
-Musobot
-Name Intelligence
-Nameprotect
+Msrabot
+NPbot
+Navroad
 NearSite
+Needle
+Net Vampire
 NetLyzer
-NetSpider
-NetZIP
-Netvibes
+Nettrack
+Nibbler
 Niki-bot
-Nimbostratus
+Nikto
 Ninja
-Nutch
+OnCrawl
+OpenLinkProfiler
 Openfind
 Openvas
-OrangeSpider
-POE-Component-Client-HTTP
-Page Analyzer
-Pandalytics
-Papa Foto
-Picscout
-PictureFinder
-Pixray
+OrangeBot
+OutclicksBot
+OutfoxBot
+PageAnalyzer
+PageScorer
+Pi-Monster
+Pimonster
+PleaseCrawl
 Pockey
-ProPowerBot
-Psbot
-Pu_iN
+ProWebWalker
+Probethenet
+Pump
+QueryN Metasearch
 Quick-Crawler
+RSSingBot
 RankActive
-RankActiveLinkBot
-RankFlex
-RankingBot2
-ReGet
-RealDownload
-Reaper
-RebelMouse
+RankingBot
+Re-re
+Ripper
 Rogerbot
-SEOlyticsCrawler
+SEOkicks-Robot
+SEOprofiler
 SEOstats
+SMTBot
 Scanbot
-Scrapy
-ScreenerBot
 SearchmetricsBot
-SemrushBot
+SeobilityBot
+Seomoz
+Siphon
 Site Sucker
 SiteCheckerBotCrawler
 SiteExplorer
-SiteSnagger
-SiteSucker
-Sitebeam
 Siteimprove
-Sitevigil
 Snake
-Snapbot
-Spbot
-Sqlmap
-Sqlworm
-Sqworm
+SpankBot
+Spanner
+Stripper
 Sucker
-Sucuri
-SuperHTTP
-T0PHackTeam
-Teleport
+Surfbot
+SurveyBot
 Telesphoreo
-The Intraformant
-TheNomad
-Thumbor
-Titan
+TightTwatBot
+Toweyabot
+Tracemyfile
 Trendiction
 Trendictionbot
 Turingos
+Turnitin
 TurnitinBot
 Twice
-Typhoeus
-URLy Warning
 Upflow
-VB Project
 VeriCiteCrawler
-VidibleScraper
-WEBDAV
-WISENutbot
-WWWOFFLE
-WallpapersHD
-Web Collage
-Web Enhancer
-Web Fuck
-Web Sauger
+Voil
+Voltron
+WBSearchBot
+WWW-Mechanize
+WWW::Mechanize
+Wallpapers/3.0
+WeSEE
+Web Auto
+Web Pix
+Web Sucker
+WebAuto
+WebBandit
+WebCopier
 WebLeacher
+WebPix
 WebReaper
+WebSauger
+WebStripper
 WebSucker
+WebWhacker
+WebZIP
+Webalta
 Webshag
-Whatweb
+WebsiteExtractor
+Webster
 Who.is Bot
-Widow
-WiseGuys Robot
-Woobot
-Wprecon
-Xaldon_WebSpider
-Xenu
 YoudaoBot
 Zade
+Zeus
 Zitebot
-ZumBot
+ZoomBot
+adscanner
 arquivo-web-crawler
-autoemailspider
+arquivo.pt
+check1.exe
+clark-crawler
 cognitiveseo
 com.plumanalytics
-crawl.sogou.com
-crawler.feedback
-dataforseo.com
+crawler4j
 demandbase-bot
-evc-batch
-ips-agent
+eCatch
+facebookscraper
+heritrix
 isitwp.com
-meanpathbot
+magpie-crawler
 mediawords
-oBot
-page scorer
+netEstate NE Crawler
+pcBrowser
+plumanalytics
+probe-image-size
 s1z.ru
 satoristudio.net
+scalaj-http
 scan.lol
 seocompany.store
+seostar
+sitechecker.pro
+siteripz
 sogouspider
-sysscan
-voyagerx.com
-x09Mozilla
+spyfu
+trendiction.de
+ubermetrics-technologies.com
+webmeup-crawler
+webpros.com
+x22Mozilla
 xpymep1.exe
-zgrab