V4.2022.03.2960 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
+404checker
 ALittle Client
-Acunetix
-AhrefsBot
-Aipbot
-AllSubmitter
+Abonti
+Aboundexbot
+AiHitBot
 Anarchie
-Anthill
-Asterias
-Attach
-BDFetch
+Anarchy
+Ankit
+Atomseobot
+AwarioRssBot
+BBBike
+BDCbot
 BLEXBot
 BackDoorBot
 BackStreet
-Barkrowler
+Backlink-Ceck
+Bandit
 BatchFTP
-Blackboard
-Boardreader
+Black Hole
+BotALot
 Buddy
-BuiltBotTough
-BuiltWith
+BunnySlippers
+CCBot
+CODE87
 CSHttp
-Calculon
-Cegbfeieh
+CazoodleBot
 CherryPicker
-ChinaClaw
-Claritybot
-Cloud mapping
-Cogentbot
+Chlooe
+Citoid
+Cocolyzebot
+Copier
 CopyRightCheck
 Cosmos
-Craftbot
 Crescent
+CrunchBot
 Custo
+CyotekWebCopy
 DBLBot
-DIIbot
-DTS Agent
-Demon
-Deusu
-Discobot
+DataCha0s
+Digincore
+DigitalPebble
+Dirbuster
+Disco
 DnBCrawler-Analytics
+DomCopBot
+DomainCrawler
 DomainStatsBot
-Ebingbong
-Ecxi
+Download Wonder
+Drip
+EMail Wolf
+EroCrawler
 Evil
-Extractor
+Exabot
+ExtLinksBot
+ExtractorPro
 Extreme Picture Finder
 EyeNetIE
-Ezooms
-FDM
-Fimap
+FemtosearchBot
+FlashGet
+Flunky
+FrontPage
 Fuzz
-Fyrebot
 G-i-g-a-b-o-t
-GermCrawler
 GetRight
-Getintent
-Go!Zilla
-GoZilla
+GrabNet
 Grabber
-GrapeshotCrawler
-GridBot
-HEADMasterSEO
-HMView
-HTTP::Lite
+Grafula
+GrapeFX
 HTTrack
+Harvest
+Havij
 Heritrix
-HonoluluBot
-HybridBot
+Humanlinks
 IDBTE4M
-IDBot
-Iblog
+IRLbot
 IlseBot
-Image Sucker
-Indy Library
-InfoNaviRobot
-Intelliseek
+IndeedBot
 Internet Ninja
-Iria
+InternetSeer
+Iskanie
 IstellaBot
 JamesBOT
 Jbrofuzz
+JennyBot
+JetCar
+Jetty
 Joomla
+Jorgee
+Jyxobot
 Kenjin Spider
-Kinza
-Kozmosbot
-LNSpiderguy
 LWP::Simple
-LeechGet
+Leap
 LexiBot
-Libwhisker
 LieBaoFast
-Likse
+Lightspeedsystems
+LinkScan
 LinkWalker
-Linkbot
 Linkdexbot
 LinkpadBot
-LinqiaScrapeBot
-Lipperhey Spider
-Ltx71
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+Lipperhey
 MFC_Tear_Sample
-MIIxpc
-MJ12bot
 MQQBrowser
 MSIECrawler
 Magnet
-Majestic SEO
-Majestic-SEO
 Majestic12
-MarkMonitor
 MauiBot
-MeanPath Bot
+Mb2345Browser
 Mediatoolkitbot
 Metauri
 Microsoft URL Control
 Minefield
+Moblie Safari
+Mojolicious
+MolokaiBot
+Morfeus Fucking Scanner
 Mr.4x3
-Msrabot
-NPbot
-Nameprotect
-NearSite
-Needle
+Musobot
+NICErsPRO
+Net Vampire
 NetAnts
+NetLyzer
 NetMechanic
 NetSpider
-Nettrack
 Netvibes
 NextGenSearchBot
-NimbleCrawler
-Ninja
-Nutch
-Offline Explorer
+Niki-bot
+Nimbostratus
+Not
 Offline Navigator
-OpenLinkProfiler
+Openfind
 Openvas
-OrangeBot
+OrangeSpider
+PECL::HTTP
+PHPCrawl
+POE-Component-Client-HTTP
 Page Analyzer
+PageAnalyzer
 PageScorer
+PageThing.com
 Papa Foto
 Pavuk
-PeoplePal
-PictureFinder
-Pixray
-Pockey
+Pi-Monster
+Picsearch
+ProWebWalker
 Probethenet
+Psbot
+Pump
 PyCurl
-RankActive
-RankingBot
+RankActiveLinkBot
 RankingBot2
-Rankivabot
-Recorder
-RedesScrapy
+Re-re
+RealDownload
+Rogerbot
 SEOkicks-Robot
-SEOlyticsCrawler
-SMTBot
-ScanAlert
-ScoutJet
-Scrapy
-Screaming
+SEOstats
+ScreenerBot
+ScrepyBot
 Searchestate
+SearchmetricsBot
+SemanticJuice
 Semrush
-Seomoz
-Siphon
+Shodan
 Site Sucker
-SiteExplorer
-SiteLockSpider
 SiteSnagger
 SiteSucker
-Siteimprove
-SmartDownload
-Snake
-Sosospider
+Snoopy
+Spammen
 SpankBot
-Spanner
-Sqlmap
-Sqworm
-Sucker
+Spbot
+Steeler
+Sucuri
+SuperHTTP
+Surfbot
+SurveyBot
 Suzuran
-Szukacz
 T0PHackTeam
 Teleport
-Telesoft
+TeleportPro
 Telesphorep
-Titan
-Trendiction
-TurnitinBot
-URLy Warning
-URLy.Warning
+TheNomad
+Thumbor
+TightTwatBot
+Tracemyfile
 V-BOT
-VelenPublicWebCrawler
-VidibleScraper
-VoidEYE
+VCI
 WBSearchBot
 WPScan
-WWW-Collector-E
-WWW::Mechanize
-Wallpapers
-Wallpapers/3.0
+WWWOFFLE
 WeSEE
+Web Auto
+Web Fetch
+Web Pix
+Web Sauger
 Web Sucker
+WebAuto
 WebBandit
-WebCollage
-WebCopier
 WebFetch
 WebImageCollector
+WebPix
 WebReaper
-WebSucker
-WebmasterWorldForumBot
+WebSauger
+WebWhacker
+WebZIP
+Webshag
+Website Quester
+WebsiteExtractor
+Webster
+Whack
+Whacker
 Whatweb
 Widow
-WinHTTrack
-Woobot
+WiseGuys Robot
+Wotbox
 Wprecon
 Xaldon WebSpider
+Xenu
 YoudaoBot
+Zade
 Zauba
-Zeus
+Zermelo
 Zitebot
-ZoomBot
+ZmEu
 ZoominfoBot
-adscanner
 arquivo-web-crawler
-arquivo.pt
 backlink-check
 cah.io.community
-check1.exe
-clark-crawler
-cognitiveseo
-com.plumanalytics
-dataforseo.com
-demandbase-bot
-eCatch
-facebookscraper
+coccocbot
+crawler.feedback
 gopher
 ips-agent
+lwp-request
+lwp-trivial
+magpie-crawler
 meanpathbot
 mediawords
-muhstik-scan
 oBot
-page scorer
+pcBrowser
 s1z.ru
+satoristudio.net
 scalaj-http
-seocompany.store
 seoscanners
-seostar
-ubermetrics-technologies.com
+serpstatbot
+sexsearcher
+sitechecker.pro
+sogouspider
 voyagerx.com
-webgains-bot
-webmeup-crawler
-x09Mozilla
+zauba.io