V4.2021.10.2631 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+360Spider
 80legs
-ADmantX
-ALittle Client
-Aboundex
+Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-Aipbot
-Alexibot
+AiHitBot
 AllSubmitter
 AlphaBot
 Anarchie
+Anarchy99
+Ankit
 Anthill
+Apexoo
+Aspiegel
+AwarioRssBot
 AwarioSmartBot
-BDFetch
+BDCbot
+BackStreet
+Backlink-Ceck
+BacklinkCrawler
 Bandit
-BatchFTP
-Battleztar Bazinga
-Blackboard
-Blow
+Bigfoot
+Bitacle
+Black Hole
 BlowFish
-Brandwatch
-BunnySlippers
-CCBot
-CODE87
-Calculon
-Cegbfeieh
+Bolt
+Brandprotect
+Buck
+BuiltBotTough
+Bullseye
+CATExplorador
 CheTeam
 ChinaClaw
-Claritybot
 Cliqzbot
 Cloud mapping
+Cocolyzebot
 Cogentbot
-CopyRightCheck
 Copyscape
-Cosmos
 Craftbot
+Crawling at Home Project
 CrazyWebCrawler
 CrunchBot
-Curious
+Custo
+CyotekWebCopy
+DataCha0s
+DomCopBot
 DomainAppender
-DomainSigmaCrawler
-DomainStatsBot
 Domains Project
-ECCP/1.0
-EMail Wolf
-EasyDL
-Ebingbong
-Express WebPictures
-ExtLinksBot
+Dotbot
+Dragonfly
+EMail Siphon
+EirGrabber
+Evil
 Extractor
-EyeNetIE
-FDM
+FHscan
 FemtosearchBot
 Fimap
-FlashGet
+Firefox/7.0
 Foobot
-Fuzz
+Freeuploader
+FyberSpider
 Fyrebot
+G-i-g-a-b-o-t
+GT::WWW
 Genieo
-GermCrawler
 Getintent
-GoZilla
+Go!Zilla
+Go-Ahead-Got-It
 Grabber
+Grafula
 GrapeFX
-GrapeshotCrawler
-HEADMasterSEO
-HTTP::Lite
+HMView
 Haansoft
 Harvest
-Havij
-IDBTE4M
+Id-search
 IlseBot
-Image Fetch
-Image Sucker
-InfoNaviRobot
-Intelliseek
-InterGET
-Internet Ninja
+IndeedBot
+InfoTekies
 InternetSeer
-Iria
-JamesBOT
-Jbrofuzz
+Iskanie
 JikeSpider
 Jorgee
-JustView
-Kenjin Spider
+Jyxobot
 Keyword Density
-LNSpiderguy
+Lanshanbot
 Larbin
-Leap
-LeechGet
-LexiBot
-LibWeb
-LieBaoFast
-LinkWalker
+LeechFTP
+Libwhisker
+Likse
+Linkdexbot
+LinkextractorPro
 LinkpadBot
 LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
+Lipperhey
 Lipperhey Spider
-Litemage_walker
+MFC_Tear_Sample
 MIDown tool
 MIIxpc
-MSFrontPage
-MSIECrawler
-MTRobot
-Mag-Net
-Magnet
-Majestic SEO
-Majestic12
+Majestic-SEO
 MarkWatch
+Mass Downloader
+Masscan
+Mata Hari
 Mb2345Browser
-Meanpathbot
+MeanPath Bot
 Mediatoolkitbot
 MegaIndex.ru
 MicroMessenger
-Microsoft Data Access
-Microsoft URL Control
-Mr.4x3
-Musobot
+Mojeek
+Msrabot
 NPbot
-Name Intelligence
-Navroad
-NearSite
-Nessus
+Needle
+Net Vampire
 NetAnts
 NetLyzer
-NetMechanic
-NetSpider
-NetZIP
-Nettrack
 Netvibes
 Nibbler
+Nikto
+Nimbostratus
+Ninja
+Nmap
 Nutch
 Octopus
-OnCrawl
 OpenLinkProfiler
-Openfind
+OpenVAS
 OutfoxBot
-PECL::HTTP
+POE-Component-Client-HTTP
 Page Analyzer
-PageGrabber
+PageAnalyzer
+PageScorer
+PageThing.com
 Pandalytics
-Picsearch
+Panscient
+Pavuk
+PeoplePal
+Picscout
+Piepmatz
 Pimonster
+Pockey
 Pu_iN
 PyCurl
 RSSingBot
 RankFlex
+RankingBot
 RankingBot2
 Rankivabot
 RankurBot
-Re-re
-ReGet
-RebelMouse
+RealDownload
+Reaper
 Recorder
 RedesScrapy
-Ripper
+RepoMonkey
+RocketCrawler
+Rogerbot
 SEOkicks-Robot
 SEOlyticsCrawler
-SEOprofiler
 SMTBot
-ScanAlert
-ScoutJet
-Scrapy
-Searchestate
-Seomoz
-Shodan
+SalesIntelligent
+Scanbot
+Screaming
+ScreenerBot
+ScrepyBot
+Semrush
 SiteCheckerBotCrawler
 SiteExplorer
-SlySearch
+SiteLockSpider
+SiteSucker
+Sitevigil
 SmartDownload
 Snoopy
 SocialRankIOBot
+Sogou web spider
+Sosospider
+Sottopop
 SpaceBison
 SpankBot
-Spbot
-SputnikBot
 Sqlmap
+Sqworm
+Steeler
 Stripper
 Sucuri
 SuperHTTP
-T0PHackTeam
+Surfbot
+SurveyBot
+Teleport
+TeleportPro
+Telesoft
+Telesphorep
 TheNomad
-Thumbor
-Toata
-Tracemyfile
-Trendiction
-Turingos
-Turnitin
-TurnitinBot
-URLy Warning
-VCI
-VelenPublicWebCrawler
+TightTwatBot
+Titan
+Toweyabot
+Twice
+Upflow
 VeriCiteCrawler
-VoidEYE
+Virusdie
 Voltron
-WASALive-Bot
-WBSearchBot
+WEBDAV
 WISENutbot
-Wallpapers
-Web Auto
-Web Fuck
-WebAuto
-WebBandit
-WebGo IS
+WWW::Mechanize
+Web Pix
+WebCopier
+WebEnhancer
 WebImageCollector
+WebLeacher
 WebReaper
 WebSauger
-WebStripper
 WebWhacker
-Webalta
+Webshag
 Webster
+Whatweb
 Widow
-WinHTTrack
-Wonderbot
-Woobot
-Xenu
+WiseGuys Robot
+Wotbox
+Wprecon
 YoudaoBot
-ZoominfoBot
-archive.org_bot
+Zauba
+Zermelo
+Zitebot
+ZoomBot
+ZumBot
+ZyBorg
+arquivo-web-crawler
 arquivo.pt
 backlink-check
-clark-crawler
+coccocbot-web
+cognitiveseo
 com.plumanalytics
-crawl.sogou.com
-crawler.feedback
-crawler4j
+dataforseo.com
 domainsproject.org
-evc-batch
 gopher
 isitwp.com
-magpie-crawler
 meanpathbot
-oBot
+mediawords
+netEstate NE Crawler
 page scorer
 pcBrowser
+plumanalytics
 polaris version
 probe-image-size
 scalaj-http
-seostar
-sitechecker.pro
+seobility
+siteripz
 spyfu
-sysscan
 tAkeOut
 trendiction.de
 ubermetrics-technologies.com
-voyagerx.com
 webmeup-crawler
-x22Mozilla
+webpros.com