V4.2021.08.2549 [ci skip]

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
+ALittle Client
+ASPSeek
 Abonti
-Aboundex
 Aboundexbot
-AfD-Verbotsverfahren
-AiHitBot
-Alligator
-AlphaBot
-Anarchy
+Anarchie
 Ankit
-Anthill
-AwarioSmartBot
+Apexoo
+Aspiegel
+BDCbot
+BLEXBot
 BackDoorBot
 Backlink-Ceck
-BacklinkCrawler
-Bandit
-Barkrowler
+BatchFTP
+Battleztar Bazinga
 Bigfoot
 Bitacle
 Blackboard
-Blow
-Boardreader
-Bolt
-BuiltBotTough
+Brandprotect
+Brandwatch
+Buck
+BuiltWith
+Bullseye
+BunnySlippers
 BuzzSumo
+CCBot
 CSHttp
-Calculon
 Cegbfeieh
-CensysInspect
-CheTeam
-Claritybot
-Collector
-Copier
-Craftbot
+CherryPicker
+ChinaClaw
+Chlooe
+Cliqzbot
+CopyRightCheck
+Copyscape
+Cosmos
 Crawling at Home Project
-Curious
+Crescent
 Custo
+DBLBot
+DIIbot
 DTS Agent
-DatabaseDriverMysqli
 Devil
+Digincore
 DigitalPebble
-Discobot
-Discoverybot
-Dispatch
+Dirbuster
 DittoSpyder
-DnyzBot
-DomainAppender
 DomainCrawler
-DomainSigmaCrawler
-Domains Project
-Dragonfly
-ECCP/1.0
 EMail Wolf
-Ebingbong
 Ecxi
-EirGrabber
-ExtLinksBot
+Exabot
+Extractor
 ExtractorPro
-EyeNetIE
+Extreme Picture Finder
 FDM
-FHscan
-Fimap
+FlashGet
+Flunky
+Fuzz
 FyberSpider
-GermCrawler
-Go-Ahead-Got-It
-GoZilla
-Gotit
-Grafula
-GrapeshotCrawler
-GridBot
-HMView
-HTMLparser
+Fyrebot
+G-i-g-a-b-o-t
+GetRight
+Gigabot
+Grabber
+HTTP::Lite
+Haansoft
 HaosouSpider
-Harvest
-Hloader
+Heritrix
+Humanlinks
+HybridBot
 IDBot
-IRLbot
 Iblog
-Id-search
-IlseBot
-InfoTekies
+Image Fetch
+IndeedBot
+Indy Library
 Intelliseek
-Internet Ninja
+InterGET
+Iria
+IstellaBot
 JennyBot
 JetCar
+JikeSpider
+Joomla
 Jorgee
 JustView
-Jyxobot
-Kenjin Spider
+Keyword Density
+LWP::Simple
 Lanshanbot
-LeechFTP
+Leap
+LeechGet
+LexiBot
 Lftp
-Libwhisker
+LibWeb
 Likse
 LinkScan
+LinkWalker
 Linkdexbot
 LinkextractorPro
-LinqiaMetadataDownloaderBot
+LinkpadBot
+LinksManager
 LinqiaRSSBot
+LinqiaScrapeBot
 Lipperhey Spider
-MQQBrowser
+Litemage_walker
+Ltx71
+MIIxpc
+MJ12bot
 MTRobot
-Majestic SEO
-MarkWatch
-Mass Downloader
-Masscan
-Mb2345Browser
-Meanpathbot
-Mediatoolkitbot
-MegaIndex.ru
-Microsoft Data Access
-Microsoft URL Control
+Metauri
+Mister PiX
 Moblie Safari
+Morfeus Fucking Scanner
 Mozlila
-Mr.4x3
+Msrabot
 NICErsPRO
+NPbot
+Nameprotect
+NearSite
+Needle
 Net Vampire
-NextGenSearchBot
-Niki-bot
+NetSpider
+Netvibes
 Nikto
-Octopus
-Offline Explorer
-OnCrawl
-Openfind
+NimbleCrawler
+Nimbostratus
+Nmap
 Openvas
-OrangeBot
 OutclicksBot
+OutfoxBot
+PHPCrawl
 POE-Component-Client-HTTP
-PageAnalyzer
-PageGrabber
-PageScorer
-Panscient
-Pi-Monster
-Picsearch
+Page Analyzer
+PageThing.com
+Papa Foto
+Pavuk
+Picscout
 Piepmatz
 Pixray
-Pockey
-ProPowerBot
+PleaseCrawl
 ProWebWalker
-Probethenet
-Psbot
-PxBroker
+Pu_iN
+PyCurl
+QueryN Metasearch
+Quick-Crawler
+RankActive
 RankFlex
-RankingBot
-RankingBot2
-RankurBot
-ReGet
-Recorder
-RepoMonkey
-Ripper
+Rankivabot
+RealDownload
+RebelMouse
+RedesScrapy
 RocketCrawler
-SEOkicks-Robot
+SEOkicks
 SEOlyticsCrawler
-SMTBot
-Scanbot
+SEOprofiler
+SISTRIX
 Scrapy
-ScreenerBot
-SearchmetricsBot
 Semrush
-SentiBot
+SemrushBot
 SeoSiteCheckup
-Shodan
-Site Sucker
+Seomoz
+Siphon
+SiteCheckerBotCrawler
+SiteSucker
+Sitebeam
 SlySearch
-SmartDownload
 Snoopy
-Spammen
-Spinn3r
-Sqlworm
-Steeler
+Sociscraper
+Sogou web spider
+Sottopop
+Spanner
+SputnikBot
 Stripper
-Sucker
 SuperBot
-SuperHTTP
 Surfbot
-SurveyBot
-Suzuran
-Swiftbot
-T8Abot
-Teleport
+Szukacz
+T0PHackTeam
 TeleportPro
 Telesoft
+Telesphoreo
 Telesphorep
-The Intraformant
-TheNomad
-Thumbor
 TightTwatBot
+Toata
 Tracemyfile
-Trendictionbot
-True_Robot
+Trendiction
 Turingos
+Turnitin
 TurnitinBot
-TwengaBot
 Twice
-URLy Warning
-Upflow
-VCI
-Vagabondo
+Typhoeus
+UnisterBot
+V-BOT
+VB Project
 VelenPublicWebCrawler
-VeriCiteCrawler
-VidibleScraper
+Virusdie
+VoidEYE
+WASALive-Bot
+WBSearchBot
+WEBDAV
+WISENutbot
 WWW-Collector-E
+WWW-Mechanize
 WWW::Mechanize
-WWWOFFLE
-Web Auto
+Wallpapers/3.0
+WallpapersHD
+Web Collage
+Web Fetch
 Web Sucker
-WebAuto
-WebCollage
 WebCopier
 WebEnhancer
-WebFetch
-WebPix
-WebReaper
+WebImageCollector
 WebStripper
-WebSucker
 WebWhacker
-Webalta
+Webshag
 WebsiteExtractor
-Whacker
-Widow
-WiseGuys Robot
+WebsiteQuester
+Whack
+Whatweb
+WinHTTrack
+Woobot
 Wotbox
+Wprecon
+Xenu
+Zade
+Zauba
 Zermelo
 Zeus
-backlink-check
-cah.io.community
-clark-crawler
-crawl.sogou.com
-domainsproject.org
-eCatch
+archive.org_bot
+arquivo.pt
+autoemailspider
+instabid
 internetVista monitor
-ips-agent
+isitwp.com
 lwp-request
-lwp-trivial
+magpie-crawler
+meanpathbot
+muhstik-scan
 netEstate NE Crawler
-plumanalytics
 ripz
 s1z.ru
 satoristudio.net
-seobility
-sitechecker.pro
-sogouspider
-sp_auditbot
-sysscan
+scalaj-http
+seostar
+sexsearcher
+spyfu
 trendiction.com
 trendiction.de
-voyagerx.com
+ubermetrics-technologies.com
 webprosbot
+xpymep1.exe
 zauba.io