V4.2023.02.3589 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
+404checker
+404enemy
 ADmantX
-AIBOT
+ASPSeek
 Abonti
-Aboundexbot
-Anarchie
+Aboundex
+AiHitBot
+Alligator
+AlphaBot
 Anarchy99
-Asterias
-AwarioRssBot
+Anthill
+Aspiegel
+AwarioSmartBot
 BBBike
-BackDoorBot
 Backlink-Ceck
 Badass
 Bandit
-Barkrowler
-Black Hole
-BlackWidow
+BatchFTP
+Blow
+BlowFish
 Boardreader
-Bolt
-BotALot
 Brandprotect
-Brandwatch
-Bullseye
+Buck
 BunnySlippers
 CSHttp
-Calculon
-CazoodleBot
-CensysInspect
-CheTeam
-Chlooe
+Cegbfeieh
+CheeseBot
+CherryPicker
+ChinaClaw
 Citoid
-Claritybot
-Cloud mapping
-Cocolyzebot
-Cogentbot
-Crawling at Home Project
-CrazyWebCrawler
+Copier
 Crescent
-CrunchBot
 Curious
-DSearch
+CyotekWebCopy
 DTS Agent
 Deusu
-DigitalPebble
-Disco
 Discobot
-Discoverybot
-DittoSpyder
-DomCopBot
 DomainCrawler
-Dragonfly
+DomainStatsBot
+Download Wonder
 ECCP/1.0
+EMail Siphon
 Ebingbong
-Ecxi
-EroCrawler
 Exabot
 ExtLinksBot
-ExtractorPro
 EyeNetIE
 Ezooms
-FHscan
 FemtosearchBot
-FlashGet
-Fyrebot
-G-i-g-a-b-o-t
+Fuzz
+FyberSpider
+Genieo
+GermCrawler
 GetRight
-Getintent
-Gigabot
-Go-Ahead-Got-It
-Grabber
+Go!Zilla
+Grafula
 GrapeFX
-HTMLparser
 HTTP::Lite
-Harvest
+HTTrack
+Havij
 Hloader
 HonoluluBot
-Id-search
-IstellaBot
+HybridBot
+IDBTE4M
+IDBot
+IlseBot
+Image Fetch
+Image Sucker
+Intelliseek
 JOC Web Spider
 JamesBOT
-Joomla
-Keybot Translation-Search-Machine
-LNSpiderguy
+Jbrofuzz
+JennyBot
+Jetty
+Jorgee
+JustView
+Jyxobot
+Keyword Density
+Kinza
 Lanshanbot
+Larbin
 Leap
 LeechFTP
-Libwhisker
-Likse
-LinkScan
+LeechGet
 Linkbot
+LinkpadBot
 LinksManager
-LinqiaRSSBot
+LinqiaMetadataDownloaderBot
 Lipperhey Spider
+Lmspider
+MFC_Tear_Sample
+MIDown tool
 MIIxpc
-MSIECrawler
-Magnet
-Mail.RU_Bot
-MauiBot
-MegaIndex.ru
+MSFrontPage
+MTRobot
+Majestic SEO
+Majestic12
+MarkWatch
+Mass Downloader
+Mata Hari
+Mb2345Browser
+Mediatoolkitbot
 Metauri
-MicroMessenger
 Microsoft Data Access
-Moblie Safari
+Microsoft URL Control
+Mister PiX
+Mojeek
+Morfeus Fucking Scanner
+Mr.4x3
+Msrabot
 Musobot
 NICErsPRO
 NPbot
 Nameprotect
 Nessus
+Net Vampire
+NetLyzer
 NetMechanic
-NetZIP
 Netvibes
 NextGenSearchBot
-Nibbler
-Niki-bot
-Nikto
 Nimbostratus
-Ninja
 Not
 Nutch
-Offline Explorer
-Offline Navigator
 OnCrawl
-OpenLinkProfiler
 OpenVAS
+Openvas
 OrangeSpider
-OutclicksBot
-OutfoxBot
 PECL::HTTP
-PHPCrawl
-POE-Component-Client-HTTP
-PageAnalyzer
 PageGrabber
 PageThing.com
-Pandalytics
 Panscient
 Pavuk
 PeoplePal
+Petalbot
 Pi-Monster
-PictureFinder
+Picsearch
+Pimonster
 PleaseCrawl
+Pockey
+ProWebWalker
 Probethenet
-Psbot
 Pump
-RankFlex
+QueryN Metasearch
+RankActive
+RankActiveLinkBot
+RankingBot2
+RankurBot
 Re-re
-ReGet
 RealDownload
-RepoMonkey
-RocketCrawler
+Recorder
+RedesScrapy
 Rogerbot
 SBIder
+SEOkicks-Robot
+SEOprofiler
 SEOstats
-Scanbot
+SISTRIX
+ScanAlert
+Scrapy
+Screaming
+Searchestate
 SearchmetricsBot
 Semrush
-SeoSiteCheckup
 SeobilityBot
-Seomoz
-SiteCheckerBotCrawler
+Site Sucker
+SiteLockSpider
+SiteSucker
 Sitebeam
-Sitevigil
+SlySearch
 Snake
-Snapbot
-Sociscraper
-Sosospider
+Sogou web spider
+SpaceBison
 SpankBot
 Spanner
+Spinn3r
 SputnikBot
-Stripper
-Sucuri
-SuperBot
+Sqworm
+Steeler
 Surfbot
-Suzuran
+SurveyBot
 Szukacz
-TeleportPro
 Telesoft
-Thumbor
-Toata
-Tracemyfile
-Trendictionbot
+True_Robot
+Turingos
 Turnitin
+TurnitinBot
+Typhoeus
 URLy Warning
 URLy.Warning
 UnisterBot
-Upflow
+V-BOT
 VB Project
 Vacuum
 Vagabondo
+VelenPublicWebCrawler
 VeriCiteCrawler
-WISENutbot
+Voltron
 WWW-Collector-E
-WWW::Mechanize
-WWWOFFLE
-WallpapersHD
+WeSEE
 Web Enhancer
-Web Fetch
-Web Fuck
-WebAuto
+Web Sauger
 WebEnhancer
-WebFetch
 WebFuck
 WebGo IS
-WebPix
-WebReaper
+WebSauger
+WebSucker
 WebZIP
+WebmasterWorldForumBot
 Webshag
 Website Quester
-WebsiteQuester
 Whack
 Whacker
+Whatweb
+Who.is Bot
 Widow
 WiseGuys Robot
-Wonderbot
-Xaldon WebSpider
-YoudaoBot
+Woobot
+Wotbox
+Wprecon
+Zade
+Zermelo
 Zeus
-ZoomBot
-ZoominfoBot
+Zitebot
+ZmEu
 adscanner
-autoemailspider
-backlink-check
-check1.exe
-clark-crawler
-coccocbot
-com.plumanalytics
-crawler4j
-facebookscraper
-gopher
+arquivo-web-crawler
+arquivo.pt
+dataforseo.com
+domainsproject.org
+eCatch
 iubenda-radar
-lwp-request
 lwp-trivial
+magpie-crawler
 mediawords
-page scorer
-pcBrowser
+muhstik-scan
+oBot
 plumanalytics
 polaris version
 ripz
 s1z.ru
 seobility
-seocompany.store
-seostar
+serpstatbot
 sexsearcher
+siteripz
 sogouspider
-sp_auditbot
 spyfu
-trendiction.com
+tAkeOut
 voyagerx.com
-webprosbot
-x09Mozilla
-xpymep1.exe
+webgains-bot
+x22Mozilla