V4.2022.02.2917 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
-404checker
+ADmantX
 AIBOT
+ALittle Client
 ASPSeek
-Abonti
-Aboundexbot
+AhrefsBot
 AiHitBot
-AllSubmitter
-AlphaBot
-Asterias
-Atomseobot
-Attach
-AwarioSmartBot
-BLEXBot
-BackDoorBot
-BackStreet
-BacklinkCrawler
+Alexibot
+Anarchy
+Anarchy99
+Anthill
+Aspiegel
+BDCbot
+BackWeb
+Badass
 Bandit
-Battleztar Bazinga
+Barkrowler
 Bigfoot
 Bitacle
 BlackWidow
-Blackboard
-Blow
-BlowFish
+Brandprotect
+Brandwatch
 Buck
+Buddy
 Bullseye
 CODE87
-Calculon
-CazoodleBot
-CheTeam
-CherryPicker
+ChinaClaw
 Citoid
+Claritybot
 Cliqzbot
-Cocolyzebot
 Cogentbot
-Copier
-Craftbot
-Crawling at Home Project
-CrazyWebCrawler
-Crescent
-Curious
+CopyRightCheck
 Custo
 DBLBot
 DIIbot
-DSearch
+Demon
+Devil
+DigitalPebble
+Dispatch
+DittoSpyder
 DnyzBot
-DomainAppender
+DomCopBot
+DomainSigmaCrawler
+Domains Project
 Dotbot
 Download Wonder
+Dragonfly
 Drip
-ECCP/1.0
-Ebingbong
+EMail Siphon
+EasyDL
+Ecxi
 EirGrabber
-Extractor
+EroCrawler
+Evil
+Exabot
+ExtractorPro
+EyeNetIE
 Ezooms
-FHscan
-Flunky
-Freeuploader
-Fuzz
-FyberSpider
-G-i-g-a-b-o-t
+Firefox/7.0
 GT::WWW
-GalaxyBot
-Genieo
 GermCrawler
-Gigabot
-Go!Zilla
-Go-Ahead-Got-It
-Gotit
-GrabNet
-Grabber
+GetRight
 Grafula
 HEADMasterSEO
 HTMLparser
-HTTrack
-HaosouSpider
 Harvest
-Heritrix
-Hloader
-IDBot
+IDBTE4M
 IRLbot
-Image Fetch
-IndeedBot
+Id-search
+IlseBot
+Intelliseek
 Iskanie
 IstellaBot
-Jbrofuzz
-Jorgee
-Kinza
-Kozmosbot
-Lanshanbot
-Leap
-LeechFTP
-LibWeb
+JOC Web Spider
+JetCar
+Jyxobot
+Kenjin Spider
+Keyword Density
+LWP::Simple
+Larbin
+LexiBot
+Lftp
 Libwhisker
-LieBaoFast
-LinkScan
-Linkbot
+Likse
+LinkextractorPro
 LinksManager
 LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+LinqiaScrapeBot
+Lipperhey
 Lipperhey Spider
+Litemage_walker
 Lmspider
-MFC_Tear_Sample
+Ltx71
 MQQBrowser
 MSFrontPage
-MSIECrawler
-Majestic SEO
+Mag-Net
+Magnet
+Mail.RU_Bot
+Majestic12
 MarkWatch
 Mass Downloader
+Mata Hari
 Mb2345Browser
-Mediatoolkitbot
 Metauri
+Microsoft Data Access
 Microsoft URL Control
 Mister PiX
-Mojeek
+Moblie Safari
 Mojolicious
 MolokaiBot
-Morfeus Fucking Scanner
-Mozlila
-Mr.4x3
-Musobot
 NICErsPRO
-Name Intelligence
+Nameprotect
+Navroad
+NearSite
 Needle
+Nessus
 Net Vampire
-NetAnts
 NetLyzer
+NetMechanic
+NetSpider
 Netcraft
 Nettrack
+Netvibes
 NextGenSearchBot
-Niki-bot
 Nimbostratus
 Nmap
-Offline Explorer
+Octopus
+Offline Navigator
 OnCrawl
-OpenLinkProfiler
-Openfind
-OrangeBot
-OutclicksBot
-PECL::HTTP
-PageGrabber
-Panscient
+OpenVAS
+PageAnalyzer
 Papa Foto
+Pi-Monster
+Picscout
 Picsearch
 Piepmatz
-Pockey
+PleaseCrawl
+ProWebWalker
+Probethenet
 Pump
-RSSingBot
 RankActive
+RankFlex
 RankingBot
+RankingBot2
 RealDownload
+Reaper
 RebelMouse
-Recorder
+RedesScrapy
+RepoMonkey
+Ripper
 Rogerbot
 SBIder
+SEOkicks-Robot
+SEOlyticsCrawler
 SEOstats
-SalesIntelligent
-Seekport
-SemanticJuice
+SMTBot
+Scrapy
+Searchestate
+SearchmetricsBot
 Semrush
 SentiBot
+SeoSiteCheckup
 SeobilityBot
+Seomoz
+Shodan
+Siphon
 Site Sucker
-SiteExplorer
 Sitebeam
+Siteimprove
 Sitevigil
-SlySearch
-SmartDownload
-Snapbot
-SpankBot
+Sosospider
+SpaceBison
+Spanner
 Spbot
 Spinn3r
-SputnikBot
+Sqlworm
 Steeler
-Stripper
-Sucuri
-SuperHTTP
+Sucker
 Surfbot
 Suzuran
+Szukacz
 T0PHackTeam
-T8Abot
+Teleport
 TeleportPro
+Telesoft
 Telesphoreo
-The Intraformant
-Thumbor
+TheNomad
+TightTwatBot
+Titan
+Toweyabot
 Tracemyfile
 Trendiction
-Turingos
+Trendictionbot
 Typhoeus
-UnisterBot
-Upflow
+V-BOT
 VB Project
-VCI
-Vagabondo
+VelenPublicWebCrawler
 VeriCiteCrawler
-Voltron
-WBSearchBot
+VidibleScraper
 WEBDAV
-Wallpapers/3.0
+WWW-Mechanize
+WallpapersHD
+Web Auto
 Web Collage
-WebCollage
+Web Enhancer
+WebBandit
+WebCopier
 WebFetch
+WebImageCollector
 WebLeacher
-WebStripper
+WebPix
+WebSucker
 WebWhacker
 WebZIP
-WebmasterWorldForumBot
-Webshag
+Webalta
 Website Quester
-WebsiteExtractor
 WebsiteQuester
-Webster
-Whack
-WiseGuys Robot
+Widow
 Wonderbot
-Wotbox
-Xaldon_WebSpider
 Xenu
 YoudaoBot
 Zade
-ZoomBot
-ZyBorg
+ZmEu
+ZoominfoBot
 arquivo-web-crawler
-cah.io.community
-check1.exe
+arquivo.pt
+autoemailspider
 cognitiveseo
-crawl.sogou.com
-dataforseo.com
+com.plumanalytics
+crawler4j
+demandbase-bot
+domainsproject.org
+evc-batch
 gopher
 internetVista monitor
-ips-agent
-lwp-request
-muhstik-scan
+isitwp.com
+iubenda-radar
+lwp-trivial
+mediawords
 netEstate NE Crawler
-probe-image-size
-ripz
-scan.lol
-seobility
-seocompany.store
-seoscanners
-seostar
-sitechecker.pro
-sogouspider
+page scorer
+s1z.ru
+scalaj-http
+siteripz
 sysscan
+tAkeOut
 trendiction.com
-ubermetrics-technologies.com
-voyagerx.com
-webgains-bot
 webprosbot
-zgrab
+x22Mozilla