V4.2022.02.2916 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-80legs
+360Spider
+404checker
 AIBOT
-ALittle Client
+ASPSeek
+Abonti
+Aboundexbot
 AiHitBot
-Aipbot
-Alexibot
+AllSubmitter
 AlphaBot
-Anarchy
-Anarchy99
-Ankit
-Anthill
-Aspiegel
+Asterias
 Atomseobot
 Attach
 AwarioSmartBot
+BLEXBot
+BackDoorBot
 BackStreet
-Badass
-Barkrowler
+BacklinkCrawler
+Bandit
+Battleztar Bazinga
+Bigfoot
+Bitacle
+BlackWidow
 Blackboard
+Blow
 BlowFish
-Bolt
-BotALot
-Brandprotect
+Buck
 Bullseye
+CODE87
 Calculon
 CazoodleBot
 CheTeam
-ChinaClaw
+CherryPicker
+Citoid
 Cliqzbot
-Cloud mapping
+Cocolyzebot
+Cogentbot
 Copier
-Copyscape
-Cosmos
 Craftbot
+Crawling at Home Project
+CrazyWebCrawler
 Crescent
-CrunchBot
-CyotekWebCopy
+Curious
+Custo
+DBLBot
 DIIbot
-Digincore
-DigitalPebble
-Discobot
-Discoverybot
-Dispatch
-DomainCrawler
-Domains Project
+DSearch
+DnyzBot
+DomainAppender
 Dotbot
 Download Wonder
-Dragonfly
-EMail Siphon
-EasyDL
+Drip
+ECCP/1.0
 Ebingbong
-EroCrawler
-Express WebPictures
-ExtractorPro
-Extreme Picture Finder
+EirGrabber
+Extractor
 Ezooms
 FHscan
-FemtosearchBot
-Firefox/7.0
-FrontPage
+Flunky
+Freeuploader
+Fuzz
+FyberSpider
+G-i-g-a-b-o-t
+GT::WWW
+GalaxyBot
+Genieo
 GermCrawler
-GetWeb
-Getintent
+Gigabot
+Go!Zilla
 Go-Ahead-Got-It
+Gotit
+GrabNet
 Grabber
 Grafula
-GrapeFX
 HEADMasterSEO
 HTMLparser
 HTTrack
-Havij
+HaosouSpider
+Harvest
+Heritrix
 Hloader
+IDBot
 IRLbot
-Id-search
 Image Fetch
 IndeedBot
-Intelliseek
 Iskanie
-JOC Web Spider
-JamesBOT
-JikeSpider
+IstellaBot
+Jbrofuzz
 Jorgee
-Jyxobot
 Kinza
+Kozmosbot
 Lanshanbot
-LeechGet
+Leap
+LeechFTP
 LibWeb
-Lightspeedsystems
-Likse
+Libwhisker
+LieBaoFast
 LinkScan
 Linkbot
-LinqiaScrapeBot
-Litemage_walker
+LinksManager
+LinqiaMetadataDownloaderBot
+Lipperhey Spider
 Lmspider
-Ltx71
 MFC_Tear_Sample
-MIIxpc
-Magnet
-Mail.RU_Bot
-Majestic-SEO
-Majestic12
-MarkMonitor
+MQQBrowser
+MSFrontPage
+MSIECrawler
+Majestic SEO
+MarkWatch
 Mass Downloader
-MauiBot
-MegaIndex.ru
+Mb2345Browser
+Mediatoolkitbot
 Metauri
-Minefield
+Microsoft URL Control
 Mister PiX
+Mojeek
+Mojolicious
+MolokaiBot
+Morfeus Fucking Scanner
 Mozlila
-NPbot
+Mr.4x3
+Musobot
+NICErsPRO
 Name Intelligence
-Nameprotect
-Navroad
-NearSite
+Needle
 Net Vampire
 NetAnts
 NetLyzer
-NetZIP
 Netcraft
-Netvibes
+Nettrack
 NextGenSearchBot
-Nibbler
-NimbleCrawler
+Niki-bot
 Nimbostratus
-Ninja
 Nmap
 Offline Explorer
-Offline Navigator
+OnCrawl
+OpenLinkProfiler
 Openfind
-Page Analyzer
+OrangeBot
+OutclicksBot
+PECL::HTTP
+PageGrabber
 Panscient
 Papa Foto
-Petalbot
 Picsearch
-PictureFinder
-Pimonster
-Pixray
-ProWebWalker
-Psbot
+Piepmatz
+Pockey
 Pump
-PxBroker
-QueryN Metasearch
-RankActiveLinkBot
-Re-re
-ReGet
+RSSingBot
+RankActive
+RankingBot
+RealDownload
 RebelMouse
+Recorder
 Rogerbot
 SBIder
-SEOlyticsCrawler
-Scanbot
-Screaming
-ScreenerBot
+SEOstats
+SalesIntelligent
+Seekport
+SemanticJuice
+Semrush
 SentiBot
-SeoSiteCheckup
+SeobilityBot
 Site Sucker
 SiteExplorer
-SiteLockSpider
-SiteSnagger
-SiteSucker
 Sitebeam
+Sitevigil
 SlySearch
-Snoopy
-SocialRankIOBot
-Sottopop
-Spanner
+SmartDownload
+Snapbot
+SpankBot
 Spbot
+Spinn3r
 SputnikBot
-Sqlworm
-Sqworm
-SuperBot
+Steeler
+Stripper
+Sucuri
 SuperHTTP
-Szukacz
+Surfbot
+Suzuran
 T0PHackTeam
 T8Abot
-Teleport
 TeleportPro
+Telesphoreo
 The Intraformant
-TheNomad
 Thumbor
 Tracemyfile
 Trendiction
-True_Robot
 Turingos
-Turnitin
-Twice
 Typhoeus
+UnisterBot
+Upflow
+VB Project
 VCI
-Vacuum
+Vagabondo
 VeriCiteCrawler
-VidibleScraper
 Voltron
-WISENutbot
-WPScan
-WWW-Collector-E
+WBSearchBot
+WEBDAV
 Wallpapers/3.0
-Web Auto
 Web Collage
-Web Fuck
-Web Sucker
-WebAuto
 WebCollage
-WebEnhancer
 WebFetch
-WebFuck
-WebGo IS
 WebLeacher
-WebPix
-WebReaper
+WebStripper
+WebWhacker
 WebZIP
+WebmasterWorldForumBot
+Webshag
+Website Quester
+WebsiteExtractor
+WebsiteQuester
 Webster
-Whacker
-Widow
-WinHTTrack
+Whack
 WiseGuys Robot
-Woobot
-Zermelo
-Zeus
-Zitebot
+Wonderbot
+Wotbox
+Xaldon_WebSpider
+Xenu
+YoudaoBot
+Zade
 ZoomBot
 ZyBorg
-adscanner
-coccocbot-web
-com.plumanalytics
+arquivo-web-crawler
+cah.io.community
+check1.exe
+cognitiveseo
 crawl.sogou.com
-crawler.feedback
-demandbase-bot
-evc-batch
-instabid
+dataforseo.com
+gopher
 internetVista monitor
-lwp-trivial
-magpie-crawler
-meanpathbot
+ips-agent
+lwp-request
+muhstik-scan
 netEstate NE Crawler
-oBot
-page scorer
-plumanalytics
+probe-image-size
 ripz
-s1z.ru
-satoristudio.net
-scalaj-http
 scan.lol
+seobility
 seocompany.store
 seoscanners
 seostar
-siteripz
+sitechecker.pro
+sogouspider
 sysscan
-tAkeOut
+trendiction.com
 ubermetrics-technologies.com
-x22Mozilla
+voyagerx.com
+webgains-bot
+webprosbot
+zgrab