V4.2022.07.3206 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
-404checker
 404enemy
-80legs
-AIBOT
-ASPSeek
+ADmantX
+ALittle Client
 Abonti
-Aboundex
-Acunetix
+Aboundexbot
 AhrefsBot
-AiHitBot
+Aipbot
+Alligator
 AlphaBot
-Ankit
-Anthill
+Anarchy
 Aspiegel
 Atomseobot
-AwarioSmartBot
+Attach
 BBBike
-BDFetch
+BDCbot
 BackDoorBot
-Backlink-Ceck
-BacklinkCrawler
-Bandit
+BackWeb
 Barkrowler
-BatchFTP
-Blow
-Boardreader
-Buck
+BetaBot
+Bigfoot
+Black Hole
+BlackWidow
+Brandprotect
 Buddy
-BuiltWith
+Bullseye
 BuzzSumo
-CATExplorador
+CSHttp
 CazoodleBot
-CheeseBot
+CensysInspect
+ChinaClaw
 Chlooe
 Citoid
-Claritybot
-Cliqzbot
-Cogentbot
+Cloud mapping
+Collector
 Copier
-Cosmos
 Crawling at Home Project
-CrazyWebCrawler
-Crescent
 CrunchBot
-CyotekWebCopy
-DSearch
-DTS Agent
+DIIbot
 DataCha0s
+Demon
 Digincore
-Dirbuster
+DigitalPebble
 Disco
-Discoverybot
 DittoSpyder
 DnBCrawler-Analytics
-DomainSigmaCrawler
-ECCP/1.0
-EMail Wolf
-Ebingbong
+DomCopBot
+DomainStatsBot
+Domains Project
+Dragonfly
+Drip
+EMail Siphon
 Ecxi
-EirGrabber
-Exabot
-Extractor
+Evil
 Extreme Picture Finder
 EyeNetIE
-FDM
-FHscan
-Fimap
-Firefox/7.0
-Foobot
-FrontPage
+Flunky
+Freeuploader
+Fuzz
 Fyrebot
 G-i-g-a-b-o-t
 GalaxyBot
-GetRight
-Getintent
+Gigabot
+GoZilla
+Gotit
+GrapeshotCrawler
+HTMLparser
+Haansoft
 Havij
 Heritrix
+Hloader
 HonoluluBot
-IDBTE4M
+IDBot
+Iblog
 IlseBot
 Image Fetch
-Image Sucker
-IndeedBot
-Intelliseek
+InfoNaviRobot
+InfoTekies
 InternetSeer
-JOC Web Spider
+Iria
+Iskanie
 JamesBOT
 JennyBot
 JetCar
-Joomla
 JustView
-Jyxobot
 Keybot Translation-Search-Machine
+Keyword Density
 Kozmosbot
-LWP::Simple
-Lftp
+LexiBot
+LibWeb
+Lightspeedsystems
+LinkWalker
 LinksManager
 LinqiaRSSBot
-LinqiaScrapeBot
 Lipperhey
-Lmspider
-Ltx71
-MQQBrowser
-MSFrontPage
+Litemage_walker
+MFC_Tear_Sample
+MJ12bot
+MSIECrawler
 Magnet
+Mail.RU_Bot
 Majestic SEO
-MarkMonitor
-MarkWatch
+Majestic-SEO
+Mass Downloader
 Masscan
-MeanPath Bot
+Mata Hari
+Meanpathbot
 MegaIndex.ru
 Microsoft Data Access
-Mojeek
+Mister PiX
 MolokaiBot
-Mozlila
-Mr.4x3
-Musobot
-Nameprotect
+Morfeus Fucking Scanner
+Msrabot
+NPbot
+Name Intelligence
 NearSite
 Nessus
-Net Vampire
-NetAnts
-NetLyzer
-NetSpider
-Nettrack
+Nimbostratus
 Ninja
 Not
+Nuclei
+Nutch
 Octopus
-Offline Explorer
-PHPCrawl
-Page Analyzer
-PageAnalyzer
+Offline Navigator
+OnCrawl
+Openfind
 PageScorer
 PageThing.com
+Pandalytics
 Papa Foto
-Pavuk
-PeoplePal
 Picscout
-Picsearch
-Piepmatz
-PyCurl
-Quick-Crawler
+Pixray
+PleaseCrawl
+ProPowerBot
+ProWebWalker
+Psbot
+PxBroker
 RSSingBot
-RankActive
-RankingBot2
-Rankivabot
-RealDownload
-Reaper
-RedesScrapy
-Ripper
-Rogerbot
-SBIder
+RankActiveLinkBot
+RankFlex
+Re-re
+ReGet
+Recorder
 SEOkicks-Robot
+SEOlyticsCrawler
 SISTRIX
-Screaming
+SMTBot
+SalesIntelligent
+ScanAlert
+Scanbot
+ScoutJet
 ScreenerBot
 ScrepyBot
-Searchestate
 SearchmetricsBot
 Seekport
+Semrush
+SemrushBot
 SentiBot
-SeoSiteCheckup
 SeobilityBot
-Seomoz
 SiteCheckerBotCrawler
-Sitebeam
-Sitevigil
+SiteSucker
+Siteimprove
 SlySearch
 SmartDownload
 Sociscraper
+Sosospider
+SpaceBison
 Spammen
 Spanner
-Spinn3r
-SputnikBot
-Sqlworm
-Sqworm
 SuperBot
+SuperHTTP
 Surfbot
+SurveyBot
+Suzuran
 Swiftbot
 Szukacz
-T0PHackTeam
-T8Abot
+Teleport
 Telesphoreo
+The Intraformant
 TheNomad
 Thumbor
-TightTwatBot
 Tracemyfile
 Trendiction
-Trendictionbot
-True_Robot
-TurnitinBot
-URLy Warning
-URLy.Warning
+Turingos
+Turnitin
+Twice
+Typhoeus
 V-BOT
 Vagabondo
-Voltron
-WEBDAV
-WWW::Mechanize
-Web Auto
+VelenPublicWebCrawler
+VoidEYE
+WBSearchBot
+WWWOFFLE
+Wallpapers
+Wallpapers/3.0
+WeSEE
+Web Collage
 Web Fetch
+Web Pix
+Web Sauger
 Web Sucker
 WebAuto
 WebBandit
+WebCopier
 WebImageCollector
-WebPix
-WebStripper
+WebReaper
+WebSauger
 WebSucker
-WebWhacker
+WebZIP
+Webshag
 Website Quester
-Whatweb
+Webster
+Whacker
 WinHTTrack
-Wotbox
-Xaldon_WebSpider
-YoudaoBot
-Zauba
+Wonderbot
+Zade
+Zermelo
+Zitebot
+ZmEu
+ZoomBot
 ZoominfoBot
-adscanner
 archive.org_bot
 arquivo-web-crawler
-backlink-check
-cah.io.community
-check1.exe
+autoemailspider
 clark-crawler
-com.plumanalytics
-crawl.sogou.com
+cognitiveseo
 crawler.feedback
-dataforseo.com
 demandbase-bot
-eCatch
-facebookscraper
+evc-batch
 instabid
-internetVista monitor
-iubenda-radar
+ips-agent
+isitwp.com
 linkdexbot
-mediawords
-muhstik-scan
-oBot
-probe-image-size
+pcBrowser
+plumanalytics
+ripz
 s1z.ru
 satoristudio.net
 scalaj-http
-scan.lol
+seobility
+seostar
+sexsearcher
+sitechecker.pro
+siteripz
 spyfu
-trendiction.com
+trendiction.de
 voyagerx.com
-webgains-bot
-xpymep1.exe
+webmeup-crawler
 zauba.io
 zgrab