V4.2022.08.3323 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
-80legs
-ADmantX
-Acunetix
-AhrefsBot
-Anarchy
+01h4x.com
+AIBOT
+ALittle Client
+Abonti
+Alexibot
+AllSubmitter
+Anarchie
+Anarchy99
 Ankit
-Anthill
-Apexoo
+Aspiegel
 Asterias
-Attach
+Atomseobot
 AwarioRssBot
-AwarioSmartBot
-BBBike
+BLEXBot
 BackDoorBot
 BackStreet
+Backlink-Ceck
+Badass
 Bandit
 Barkrowler
 BatchFTP
+Battleztar Bazinga
 BetaBot
-Blackboard
-Boardreader
-Bolt
-BotALot
-BuiltBotTough
+Bitacle
+Black Hole
+Brandprotect
+Brandwatch
+Buck
 BuiltWith
-BunnySlippers
+Bullseye
 BuzzSumo
+CCBot
 Cegbfeieh
-CheeseBot
-CherryPicker
 Chlooe
-Cloud mapping
-Cocolyzebot
-Cogentbot
+Citoid
+Claritybot
+Copier
 CopyRightCheck
-Copyscape
-Cosmos
-Crawling at Home Project
 CrazyWebCrawler
-Crescent
-Curious
+Custo
 CyotekWebCopy
-DSearch
-DataCha0s
-Demon
-DigitalPebble
-Disco
-DomCopBot
-DomainAppender
-Dotbot
-Dragonfly
+Discobot
+Dispatch
+DittoSpyder
+DnBCrawler-Analytics
+DomainSigmaCrawler
+Domains Project
+ECCP/1.0
 EMail Siphon
-Ebingbong
+EMail Wolf
+EirGrabber
+Exabot
+Express WebPictures
 ExtLinksBot
+Extractor
+Ezooms
 FHscan
-Fimap
+FemtosearchBot
 Flunky
-Freeuploader
-G-i-g-a-b-o-t
-GermCrawler
+Foobot
+FrontPage
+FyberSpider
+GalaxyBot
+Genieo
+GetRight
 GetWeb
-Go-Ahead-Got-It
-GoZilla
-GrabNet
 Grafula
 GrapeshotCrawler
+HEADMasterSEO
+HMView
+HTMLparser
 Haansoft
-Havij
+HaosouSpider
+Heritrix
 HonoluluBot
+Humanlinks
 HybridBot
-IDBTE4M
-IDBot
-Id-search
+Iblog
 IlseBot
-Image Sucker
-Indy Library
-InfoTekies
-InterGET
-InternetSeer
-JennyBot
+IndeedBot
+Iria
+Iskanie
+JOC Web Spider
+JamesBOT
+Jbrofuzz
 JetCar
-JustView
+JikeSpider
 Kenjin Spider
-LWP::Simple
-Lanshanbot
-Lftp
-Likse
-LinkScan
-LinkWalker
+Keyword Density
+Kozmosbot
+Larbin
+Leap
+LeechGet
+LieBaoFast
 Linkbot
+LinkextractorPro
 LinkpadBot
-LinksManager
 Lipperhey
 Litemage_walker
+MFC_Tear_Sample
 MJ12bot
-MSIECrawler
-Mail.RU_Bot
+MQQBrowser
 Majestic SEO
-MarkMonitor
-MarkWatch
+Majestic12
 Mass Downloader
 Masscan
 Mata Hari
-MeanPath Bot
+MauiBot
 MegaIndex.ru
-MicroMessenger
-Microsoft Data Access
-Mister PiX
+Metauri
+Minefield
+Mojeek
+Mojolicious
 Morfeus Fucking Scanner
+Mr.4x3
+Msrabot
 Nameprotect
-Navroad
 NearSite
-Needle
-NetZIP
+Net Vampire
+NetLyzer
+NetMechanic
+NetSpider
 Netcraft
-NextGenSearchBot
-Niki-bot
-Ninja
+Nikto
+Nmap
+Nuclei
 Offline Explorer
-OnCrawl
-OpenLinkProfiler
 OpenVAS
-Openvas
 OrangeBot
-OrangeSpider
 OutclicksBot
+OutfoxBot
 PHPCrawl
-POE-Component-Client-HTTP
+Page Analyzer
 PageGrabber
-PageScorer
 PageThing.com
+Panscient
+Papa Foto
 PeoplePal
+Petalbot
 Picsearch
 PictureFinder
+Piepmatz
 Pimonster
-Pixray
 PleaseCrawl
+Pockey
+ProPowerBot
+ProWebWalker
 Probethenet
-Psbot
+Pump
 PyCurl
 QueryN Metasearch
 Quick-Crawler
-RankingBot
 RankingBot2
-Rankivabot
-ReGet
 RebelMouse
+Recorder
 RepoMonkey
-Ripper
-Rogerbot
-SEOlyticsCrawler
 SEOprofiler
-Scanbot
-ScoutJet
-Scrapy
+SEOstats
 ScreenerBot
-SentiBot
+SemanticJuice
+SeoSiteCheckup
 SeobilityBot
-Seomoz
-SiteCheckerBotCrawler
+Siphon
 SiteExplorer
+SiteLockSpider
 SiteSnagger
-SiteSucker
-Siteimprove
-Snapbot
-Sociscraper
+Sitebeam
+Sitevigil
+SlySearch
+SmartDownload
+Snake
+SocialRankIOBot
+Sosospider
 Sottopop
-Spanner
-Spbot
-Steeler
-Stripper
+SpaceBison
+SpankBot
+Sucker
 SuperHTTP
+Surfbot
+SurveyBot
+Suzuran
+T0PHackTeam
 T8Abot
 TeleportPro
 Telesoft
-Telesphorep
-TheNomad
-TightTwatBot
+Telesphoreo
+The Intraformant
 Titan
-Toata
 Toweyabot
-Trendiction
-Turingos
-TurnitinBot
+True_Robot
+Turnitin
+Twice
+URLy Warning
+UnisterBot
+V-BOT
 VB Project
-VeriCiteCrawler
 Virusdie
 VoidEYE
-Voil
 WEBDAV
-WPScan
+WISENutbot
 WWW-Collector-E
 WWWOFFLE
 Wallpapers
+WallpapersHD
+Web Auto
 Web Fetch
-Web Pix
-WebCopier
-WebFetch
-WebGo IS
-WebImageCollector
-WebReaper
-WebStripper
-Webalta
-WebsiteQuester
-WiseGuys Robot
-Woobot
+Web Sucker
+WebBandit
+WebPix
+WebSauger
+Webshag
+Website Quester
+WebsiteExtractor
+Whatweb
 Xaldon_WebSpider
+Xenu
 YoudaoBot
 Zade
-Zauba
-Zermelo
-Zeus
+Zitebot
 ZoomBot
 ZumBot
-arquivo-web-crawler
-backlink-check
-clark-crawler
-crawl.sogou.com
+adscanner
+arquivo.pt
+coccocbot
+com.plumanalytics
 crawler.feedback
 crawler4j
-dataforseo.com
-eCatch
-evc-batch
+demandbase-bot
+domainsproject.org
+gopher
 instabid
-internetVista monitor
+ips-agent
 isitwp.com
-linkdexbot
-lwp-trivial
-magpie-crawler
 meanpathbot
-page scorer
+oBot
 plumanalytics
-polaris version
 ripz
-satoristudio.net
-scalaj-http
+s1z.ru
 scan.lol
-seocompany.store
+seoscanners
+serpstatbot
 sexsearcher
-sysscan
+sitechecker.pro
+siteripz
+sogouspider
+spyfu
 tAkeOut
-trendiction.com
 voyagerx.com
-webgains-bot
-webmeup-crawler
-webpros.com
-webprosbot
-zauba.io
+x09Mozilla
 zgrab