V4.2022.10.3433 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
-ALittle Client
-Abonti
-AhrefsBot
-Aipbot
-Alexibot
+404enemy
+ADmantX
+AIBOT
+ASPSeek
+Aboundex
+AiHitBot
+Alligator
 AlphaBot
-Anarchy
-Atomseobot
-AwarioSmartBot
-BBBike
-BDFetch
+Asterias
+AwarioRssBot
 BLEXBot
 BackStreet
 BackWeb
+BacklinkCrawler
+Bandit
 Barkrowler
-BatchFTP
+Battleztar Bazinga
+Bigfoot
+Bitacle
 Black Hole
 BlackWidow
-Blow
+Blackboard
 BlowFish
-BotALot
-BuiltWith
+Bolt
+Buck
+Buddy
+Bullseye
+BunnySlippers
+BuzzSumo
 CATExplorador
-CCBot
-CazoodleBot
-Cegbfeieh
+CODE87
+Calculon
 CensysInspect
 CheTeam
+CheeseBot
 Citoid
-Claritybot
 Cloud mapping
-Collector
+Cocolyzebot
 Copier
-Copyscape
-Cosmos
-Craftbot
+CopyRightCheck
+Crawling at Home Project
+CrazyWebCrawler
+CrunchBot
 Curious
-Custo
 CyotekWebCopy
-DBLBot
+DIIbot
 DTS Agent
 DatabaseDriverMysqli
-Demon
-DigitalPebble
-Disco
+Dirbuster
+Discobot
+Discoverybot
 Dispatch
 DittoSpyder
-DnyzBot
-DomainSigmaCrawler
-Dotbot
-Dragonfly
+DnBCrawler-Analytics
+DomainStatsBot
+Domains Project
+Download Wonder
+Drip
 ECCP/1.0
 EMail Wolf
-Ebingbong
-Ecxi
-EirGrabber
-EroCrawler
-Evil
-Exabot
-Express WebPictures
-EyeNetIE
+ExtLinksBot
+Extractor
 Ezooms
-Fimap
+FHscan
+Freeuploader
 FrontPage
-Fuzz
-FyberSpider
-Fyrebot
-G-i-g-a-b-o-t
-GT::WWW
-Getintent
-Gigabot
+Genieo
+GetRight
 GoZilla
-GrabNet
 Grabber
-Grafula
-HEADMasterSEO
-HMView
-HTTrack
+GrapeFX
+GridBot
+HTMLparser
+HTTP::Lite
 HaosouSpider
-Harvest
-HonoluluBot
+Heritrix
+Humanlinks
 IDBTE4M
-IDBot
 IRLbot
+Id-search
 IlseBot
-Indy Library
-Intelliseek
+Image Sucker
+InfoNaviRobot
 Iria
-IstellaBot
-JOC Web Spider
-JennyBot
-JetCar
-Jetty
-Keyword Density
+Iskanie
+JamesBOT
+Jorgee
+Kenjin Spider
+Keybot Translation-Search-Machine
+Kozmosbot
 LNSpiderguy
+LWP::Simple
 Lanshanbot
-Larbin
-Leap
-Lftp
-LibWeb
 LieBaoFast
-LinqiaMetadataDownloaderBot
-Lipperhey Spider
-Litemage_walker
+Likse
+LinkextractorPro
+Lipperhey
+MFC_Tear_Sample
 MIIxpc
-MJ12bot
-MSFrontPage
-MSIECrawler
-MTRobot
+MQQBrowser
 Magnet
 Mail.RU_Bot
+Majestic-SEO
+Majestic12
+MarkWatch
 Mass Downloader
 Masscan
-MeanPath Bot
+Mata Hari
 Mediatoolkitbot
-Microsoft URL Control
-Minefield
-Mojeek
-Mojolicious
-Morfeus Fucking Scanner
-Mozlila
-Nessus
-NetMechanic
-NetZIP
-NextGenSearchBot
+MegaIndex.ru
+MicroMessenger
+Microsoft Data Access
+MolokaiBot
+Msrabot
+Nameprotect
+Needle
+Net Vampire
+NetAnts
+Netcraft
+Nettrack
 Nibbler
-Nikto
+Niki-bot
+NimbleCrawler
+Nimbostratus
+Ninja
 Nmap
 Not
-Nuclei
+Octopus
+Offline Navigator
+OnCrawl
 OpenVAS
-OrangeBot
 OutclicksBot
-PECL::HTTP
+PHPCrawl
+POE-Component-Client-HTTP
 Page Analyzer
-PageAnalyzer
-PageGrabber
+PageScorer
+Papa Foto
 PeoplePal
-Petalbot
-PleaseCrawl
-Probethenet
+Pi-Monster
+PictureFinder
+Pixray
 Pu_iN
-PxBroker
+Pump
 QueryN Metasearch
-Quick-Crawler
 RankActiveLinkBot
-RankFlex
-RankingBot
 RankingBot2
-Re-re
-ReGet
-RebelMouse
-Recorder
+RealDownload
+RedesScrapy
 RepoMonkey
-Ripper
-SBIder
+RocketCrawler
+Rogerbot
+SEOkicks-Robot
 SEOlyticsCrawler
+SEOstats
 SISTRIX
+SMTBot
+SalesIntelligent
 ScanAlert
-ScoutJet
-Searchestate
-SearchmetricsBot
+Scrapy
+Seekport
+Semrush
 SeoSiteCheckup
 SeobilityBot
 Siphon
-SiteExplorer
-SiteLockSpider
-Sitevigil
+Site Sucker
+Sitebeam
 SlySearch
-SpaceBison
-Spanner
-Spbot
+SmartDownload
+Snake
+Snoopy
+Sociscraper
+Sosospider
+Spinn3r
+Sucker
 Sucuri
 SuperHTTP
+SurveyBot
 Suzuran
 Swiftbot
 Szukacz
-T0PHackTeam
-Teleport
-Telesoft
+T8Abot
+TeleportPro
+Telesphorep
 TheNomad
-Tracemyfile
+Thumbor
 Trendiction
 True_Robot
-Turingos
-TwengaBot
 Twice
-Typhoeus
 URLy Warning
-URLy.Warning
-Upflow
-VCI
-Voltron
+VB Project
+Vacuum
+Vagabondo
+VelenPublicWebCrawler
+VeriCiteCrawler
+VidibleScraper
 WASALive-Bot
-WWW-Collector-E
-WeSEE
+WPScan
+WWW-Mechanize
 Web Auto
+Web Fetch
+Web Fuck
+Web Sucker
 WebCopier
-WebPix
-WebReaper
-WebStripper
-WebZIP
+WebLeacher
 Webalta
-Webshag
-WebsiteExtractor
 WebsiteQuester
+Webster
 Whacker
-Whatweb
-Who.is Bot
-Wonderbot
-Wprecon
-Xaldon WebSpider
+WinHTTrack
+WiseGuys Robot
 Xaldon_WebSpider
-Zauba
+YoudaoBot
+Zade
+Zermelo
 ZmEu
-ZoomBot
-ZoominfoBot
-ZyBorg
 adscanner
 archive.org_bot
-arquivo-web-crawler
-cah.io.community
-com.plumanalytics
-crawl.sogou.com
-crawler.feedback
-domainsproject.org
-ips-agent
-iubenda-radar
+arquivo.pt
+autoemailspider
+backlink-check
+check1.exe
+clark-crawler
+cognitiveseo
+crawler4j
+eCatch
+evc-batch
+instabid
+internetVista monitor
 linkdexbot
 lwp-request
+magpie-crawler
 muhstik-scan
 netEstate NE Crawler
 oBot
+page scorer
+pcBrowser
+probe-image-size
+ripz
 s1z.ru
+scalaj-http
 scan.lol
-seocompany.store
-seoscanners
 seostar
-sexsearcher
-sitechecker.pro
-sogouspider
-sp_auditbot
-tAkeOut
+trendiction.com
 ubermetrics-technologies.com
-webgains-bot
+webmeup-crawler
 webprosbot
-x09Mozilla
-xpymep1.exe
+zauba.io
+zgrab