V4.2022.12.3503 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-AIBOT
+360Spider
+ADmantX
+ALittle Client
 Abonti
-Aboundexbot
+Aboundex
 Acunetix
+AhrefsBot
+AiHitBot
 Alexibot
 AllSubmitter
 Anarchy
-Anarchy99
-Apexoo
+Anthill
 Asterias
+Atomseobot
+Attach
 AwarioRssBot
-AwarioSmartBot
 BBBike
-BackStreet
-BackWeb
-BacklinkCrawler
-Bandit
-BatchFTP
-Battleztar Bazinga
+BetaBot
+Bigfoot
+Bitacle
 Black Hole
+BlackWidow
 Blackboard
 BlowFish
 Bolt
 Brandprotect
-Brandwatch
 Buck
 BuiltBotTough
-BuiltWith
-Bullseye
-BuzzSumo
 CATExplorador
 CODE87
-CensysInspect
+Calculon
+Cegbfeieh
 CheTeam
-CherryPicker
-Collector
-Copier
-Curious
-Custo
-DSearch
-DataCha0s
+Chlooe
+Citoid
+Cloud mapping
+Cogentbot
+CopyRightCheck
+Crawling at Home Project
+CrazyWebCrawler
+CrunchBot
+DTS Agent
 Demon
-Devil
-Digincore
-Discobot
+DigitalPebble
+Dirbuster
+Disco
 Dispatch
-DnyzBot
-DomCopBot
-DomainAppender
-DomainCrawler
+Dotbot
+Dragonfly
 Drip
-Ecxi
-EirGrabber
+ECCP/1.0
+EasyDL
+EroCrawler
 Evil
-Exabot
+Express WebPictures
+ExtLinksBot
 Extractor
-Extreme Picture Finder
-FDM
+FHscan
+FlashGet
 Flunky
-Foobot
-FrontPage
-FyberSpider
-GermCrawler
-GetRight
+Fyrebot
+G-i-g-a-b-o-t
+GalaxyBot
+Genieo
+GetWeb
 Go!Zilla
+Go-Ahead-Got-It
 Gotit
-Grafula
-GrapeshotCrawler
+GrabNet
+Grabber
 HEADMasterSEO
-HTMLparser
-HTTP::Lite
-Haansoft
-HaosouSpider
-Harvest
 Heritrix
 Hloader
 Humanlinks
 HybridBot
+IDBot
 IRLbot
 IndeedBot
-Indy Library
-Intelliseek
-Internet Ninja
+InfoTekies
+InterGET
+Iria
+IstellaBot
+JOC Web Spider
+Jbrofuzz
 JennyBot
-JetCar
 Jetty
-Joomla
-Jyxobot
-Kinza
-Kozmosbot
-Lanshanbot
-Larbin
-LexiBot
-Lftp
+JustView
+Kenjin Spider
+Keybot Translation-Search-Machine
+LeechGet
 LibWeb
-Libwhisker
-LieBaoFast
-Lightspeedsystems
-Likse
-LinkScan
-LinkWalker
 Linkbot
-LinkextractorPro
-LinkpadBot
 LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
 LinqiaScrapeBot
-Lipperhey
-MJ12bot
-MQQBrowser
+Lipperhey Spider
+Litemage_walker
+Ltx71
 MSFrontPage
 Mail.RU_Bot
-Majestic SEO
-MarkMonitor
+Majestic12
 Mass Downloader
 Mata Hari
+MauiBot
+Meanpathbot
 MegaIndex.ru
-Metauri
-Microsoft URL Control
-Minefield
-Moblie Safari
-Mozlila
-Navroad
-Nessus
-NetSpider
+MicroMessenger
+Mister PiX
+Mojeek
+MolokaiBot
+Morfeus Fucking Scanner
+Mr.4x3
+Name Intelligence
+Nameprotect
+NetAnts
 NetZIP
-Nettrack
-NextGenSearchBot
+Netvibes
 Nibbler
-Nikto
-NimbleCrawler
+Nmap
+Nuclei
 Nutch
-Offline Explorer
+Offline Navigator
+OpenLinkProfiler
+Openfind
+Openvas
 OrangeBot
-PageGrabber
-PageScorer
-Panscient
+OutclicksBot
+POE-Component-Client-HTTP
+Page Analyzer
+Pandalytics
+Papa Foto
+Pi-Monster
 Picsearch
 Pixray
 PleaseCrawl
-ProPowerBot
-ProWebWalker
 Probethenet
 Psbot
-Pump
-QueryN Metasearch
+RSSingBot
 RankActive
+RankFlex
+RankingBot
+RankingBot2
+Re-re
 ReGet
-RebelMouse
+RealDownload
 RedesScrapy
 RepoMonkey
 Ripper
-RocketCrawler
 Rogerbot
-SEOkicks-Robot
-SEOprofiler
+SBIder
+SEOkicks
+SEOlyticsCrawler
+SEOstats
 SMTBot
 SalesIntelligent
-Scrapy
-Screaming
-ScreenerBot
-Seekport
+ScanAlert
+ScoutJet
+ScrepyBot
+SearchmetricsBot
+Semrush
+SentiBot
 SeobilityBot
-Site Sucker
+Shodan
+Siphon
 SiteExplorer
-SiteSnagger
-SiteSucker
-Siteimprove
-Sitevigil
+SiteLockSpider
+Sitebeam
+SlySearch
+SmartDownload
+Snoopy
+SocialRankIOBot
+Sociscraper
+Sogou web spider
+Sosospider
 SpaceBison
-SpankBot
 Spanner
-Spbot
-Sqlmap
-Sqlworm
+Spinn3r
+SputnikBot
+Steeler
 Stripper
-T0PHackTeam
+Sucker
+Surfbot
+Suzuran
 T8Abot
-Teleport
 TeleportPro
-TheNomad
-Thumbor
-Toata
-Toweyabot
+Telesoft
+Telesphoreo
+Telesphorep
 Tracemyfile
+Trendiction
 Trendictionbot
-True_Robot
-Turnitin
 TwengaBot
-URLy.Warning
+Twice
+UnisterBot
 Upflow
-VCI
-Vagabondo
+VB Project
 VeriCiteCrawler
-Virusdie
+VidibleScraper
+VoidEYE
 WASALive-Bot
+WEBDAV
+WISENutbot
 WWW-Collector-E
-WWW-Mechanize
 WWWOFFLE
+Wallpapers/3.0
 WallpapersHD
-Web Auto
-Web Collage
-Web Pix
-Web Sucker
+WeSEE
+Web Enhancer
+WebBandit
+WebEnhancer
+WebFuck
 WebLeacher
-WebSauger
-WebSucker
-Webalta
-Webshag
 WebsiteExtractor
-Webster
-Widow
-WiseGuys Robot
-Wonderbot
+WebsiteQuester
+Whack
+Whacker
 Woobot
+Xaldon WebSpider
 Xenu
-Zitebot
+YoudaoBot
+Zade
+Zeus
+ZoomBot
 ZoominfoBot
-ZyBorg
-archive.org_bot
-autoemailspider
+arquivo-web-crawler
 clark-crawler
-coccocbot
-demandbase-bot
-eCatch
+crawl.sogou.com
+crawler4j
+dataforseo.com
 evc-batch
+facebookscraper
 gopher
+heritrix
 instabid
-iubenda-radar
+isitwp.com
 linkdexbot
-lwp-request
-magpie-crawler
-netEstate NE Crawler
 page scorer
-plumanalytics
-ripz
-s1z.ru
-satoristudio.net
+pcBrowser
+polaris version
 scalaj-http
-seostar
+seocompany.store
+serpstatbot
+siteripz
+spyfu
 sysscan
+trendiction.com
 trendiction.de
 ubermetrics-technologies.com
+voyagerx.com
 webmeup-crawler
-webprosbot
-x09Mozilla
 xpymep1.exe
-zgrab