V4.2021.07.2469 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-80legs
-ADmantX
-Aboundexbot
+360Spider
+404checker
+AIBOT
+ASPSeek
 Acunetix
-AhrefsBot
+Aipbot
 AllSubmitter
+Anarchie
 Anarchy
-Ankit
+Anthill
 Apexoo
 Aspiegel
+Asterias
+Attach
+AwarioRssBot
 BDFetch
-BLEXBot
 BackDoorBot
-BackStreet
-Bandit
-Battleztar Bazinga
+BackWeb
+Backlink-Ceck
+BacklinkCrawler
+Badass
+Barkrowler
+BetaBot
 Bigfoot
-BlackWidow
-Blow
-Brandwatch
-BuiltBotTough
+Blackboard
+Buck
+Buddy
 BuiltWith
-BuzzSumo
-CazoodleBot
-CensysInspect
+BunnySlippers
+CODE87
+CSHttp
+Calculon
 CheTeam
+ChinaClaw
+Chlooe
+Cliqzbot
 Cloud mapping
-Cocolyzebot
-Cogentbot
-Collector
-CopyRightCheck
-Cosmos
-CrazyWebCrawler
-Crescent
+CrunchBot
 Custo
-CyotekWebCopy
-DIIbot
-Demon
+DTS Agent
+DataCha0s
 Devil
 Digincore
 DigitalPebble
-Disco
 DittoSpyder
-DnyzBot
-DomCopBot
-DomainCrawler
 DomainStatsBot
-Domains Project
+Dragonfly
+EMail Wolf
 EasyDL
 Ebingbong
-Ecxi
-EirGrabber
 EroCrawler
-ExtLinksBot
 ExtractorPro
 Ezooms
-Fimap
-Flunky
+FHscan
+FlashGet
+Freeuploader
+FrontPage
 Fyrebot
+GT::WWW
 GalaxyBot
+Genieo
 GermCrawler
-GetRight
-Gigablast
+Gigabot
 Go-Ahead-Got-It
-GoZilla
-Gotit
-GrabNet
-Grabber
-Grafula
-GrapeshotCrawler
 GridBot
-HMView
-HTMLparser
-HTTrack
-Harvest
+HaosouSpider
 Humanlinks
+HybridBot
 IDBot
 Iblog
 Image Fetch
+Image Sucker
 IndeedBot
 Indy Library
-InfoNaviRobot
 InfoTekies
 Intelliseek
-InterGET
-Iria
-JOC Web Spider
-JamesBOT
-Jbrofuzz
+Internet Ninja
+InternetSeer
+Iskanie
+IstellaBot
+JennyBot
 JetCar
-JustView
+Kenjin Spider
+Kinza
+LNSpiderguy
 Lanshanbot
+Larbin
+LeechFTP
 LexiBot
-LibWeb
-LieBaoFast
-LinkextractorPro
+Libwhisker
+Likse
+LinkScan
+Linkdexbot
+Lipperhey
+Lipperhey Spider
 MFC_Tear_Sample
-MIIxpc
+MJ12bot
+MQQBrowser
+MSIECrawler
 Mag-Net
-Majestic SEO
-Majestic-SEO
 MarkMonitor
+MarkWatch
 Mass Downloader
-Mata Hari
+Masscan
+MauiBot
+Mb2345Browser
 MeanPath Bot
+Mediatoolkitbot
 MegaIndex.ru
-MicroMessenger
-Moblie Safari
-Mojeek
-Morfeus Fucking Scanner
+Microsoft Data Access
+Microsoft URL Control
+Mojolicious
 Mozlila
+Musobot
+NICErsPRO
+NPbot
 Name Intelligence
-Nameprotect
 Navroad
-Nessus
-Netcraft
-Nettrack
+NearSite
+Needle
+Net Vampire
+Netvibes
+NextGenSearchBot
+Nibbler
+Nikto
+Ninja
 Nmap
 Nuclei
+OPPO A33
 Offline Explorer
-OrangeBot
-PHPCrawl
-POE-Component-Client-HTTP
-Page Analyzer
+OnCrawl
+OpenLinkProfiler
 PageGrabber
+PageScorer
+Panscient
+Papa Foto
+PeoplePal
 Pi-Monster
 Picsearch
-Piepmatz
 Pimonster
+Pixray
 Pockey
 ProWebWalker
-Probethenet
-Pump
-PxBroker
 PyCurl
 QueryN Metasearch
+Quick-Crawler
+RankActiveLinkBot
 RankFlex
 RankingBot2
 RankurBot
 Re-re
-ReGet
-RebelMouse
-RedesScrapy
+RealDownload
+Ripper
 SEOkicks
-SEOkicks-Robot
 SEOlyticsCrawler
 SEOprofiler
-SISTRIX
-Scanbot
-ScreenerBot
+SEOstats
+SalesIntelligent
+ScanAlert
+Scrapy
+Screaming
 Searchestate
 SearchmetricsBot
+Semrush
 SemrushBot
+SentiBot
+SeoSiteCheckup
 Seomoz
-Shodan
-Siphon
 Site Sucker
-SiteCheckerBotCrawler
-SiteSnagger
+SiteLockSpider
 Sitebeam
-SmartDownload
-Snake
-Snapbot
-Sociscraper
-SpankBot
+SlySearch
+Sottopop
 Spanner
 Spbot
-Sqlworm
-Sucker
+Sqworm
+Steeler
 Sucuri
-SuperHTTP
-Surfbot
+SuperBot
 SurveyBot
-Szukacz
+T0PHackTeam
 Teleport
-Telesphorep
+TeleportPro
+Telesphoreo
 The Intraformant
-TheNomad
-TightTwatBot
-Toata
-Trendiction
-Trendictionbot
-True_Robot
-Turingos
+Toweyabot
+Tracemyfile
 Turnitin
 TurnitinBot
-Twice
+TwengaBot
 Typhoeus
-URLy Warning
 URLy.Warning
 Upflow
-VB Project
+V-BOT
+VCI
 Vacuum
-Vagabondo
-VidibleScraper
-Virusdie
-Voil
-Voltron
+VeriCiteCrawler
+VoidEYE
+WASALive-Bot
 WEBDAV
-WPScan
-Wallpapers/3.0
 WallpapersHD
-Web Collage
+Web Auto
+Web Enhancer
 Web Fetch
 Web Fuck
-WebBandit
-WebCopier
-WebFetch
+WebEnhancer
 WebFuck
 WebGo IS
+WebImageCollector
+WebLeacher
 WebPix
-WebSauger
+WebStripper
 WebWhacker
 WebZIP
+Webalta
+WebmasterWorldForumBot
+Webshag
+WebsiteExtractor
 WebsiteQuester
-Whack
-Who.is Bot
-WiseGuys Robot
+Webster
+Whatweb
+Widow
+WinHTTrack
 Wotbox
-Xaldon WebSpider
-Xenu
-Zade
 Zauba
 Zermelo
-Zitebot
+ZmEu
 ZoominfoBot
-archive.org_bot
-arquivo-web-crawler
-backlink-check
-check1.exe
 coccocbot-web
-com.plumanalytics
+crawl.sogou.com
+crawler.feedback
+crawler4j
+demandbase-bot
 domainsproject.org
-eCatch
-evc-batch
-instabid
 ips-agent
-lwp-request
-netEstate NE Crawler
+lwp-trivial
+mediawords
+muhstik-scan
+oBot
 pcBrowser
-plumanalytics
 polaris version
 satoristudio.net
-scan.lol
 seobility
-sexsearcher
-tAkeOut
-trendiction.de
-xpymep1.exe
+sitechecker.pro
+siteripz
+spyfu
+sysscan
+trendiction.com
+webmeup-crawler
+x09Mozilla