V4.2021.07.2487 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
 404checker
 80legs
+AIBOT
+ALittle Client
 ASPSeek
+Abonti
+Aboundex
+Aboundexbot
 Acunetix
 AfD-Verbotsverfahren
+AhrefsBot
+AiHitBot
 Aipbot
-AlphaBot
-Ankit
-Aspiegel
+Alexibot
+Anarchie
+Anarchy
+Anarchy99
+Anthill
 Asterias
 Attach
+AwarioSmartBot
 BBBike
-BDCbot
+BackDoorBot
 BackStreet
-Badass
-Barkrowler
+BacklinkCrawler
+Bandit
+BatchFTP
+Bigfoot
 Bitacle
-Black Hole
-BlackWidow
-Boardreader
-Buddy
-BuiltWith
-BunnySlippers
+Blow
+Brandprotect
+Brandwatch
+Bullseye
 BuzzSumo
-CODE87
-CSHttp
+CazoodleBot
+CensysInspect
+CheTeam
 CheeseBot
-Cloud mapping
+ChinaClaw
+Chlooe
+Claritybot
 Cocolyzebot
-CopyRightCheck
+Cogentbot
+Copier
+Craftbot
 CrazyWebCrawler
-CrunchBot
 Curious
-DataCha0s
-Demon
-Digincore
-Disco
+Custo
+DBLBot
+DSearch
+Devil
+Discoverybot
 Dispatch
-DnyzBot
-DomainSigmaCrawler
+DomCopBot
+DomainAppender
+DomainCrawler
 DomainStatsBot
-Domains Project
-Drip
+ECCP/1.0
 EMail Siphon
-EasyDL
-Ebingbong
 Ecxi
-EirGrabber
-Evil
 Express WebPictures
-ExtLinksBot
 Extreme Picture Finder
 FDM
+FHscan
+FemtosearchBot
 Freeuploader
 FyberSpider
 Fyrebot
 Genieo
-Getintent
-GrabNet
-Grabber
+GetWeb
+Gotit
 Grafula
-GrapeshotCrawler
-HTTP::Lite
-Harvest
-Heritrix
+HEADMasterSEO
+Haansoft
+Havij
 Hloader
-Humanlinks
-HybridBot
-Iblog
+IDBTE4M
 Id-search
-Image Fetch
-Image Sucker
-IndeedBot
 InfoNaviRobot
 InfoTekies
 InterGET
-Internet Ninja
-IstellaBot
-JamesBOT
+InternetSeer
+Iskanie
+JOC Web Spider
+Jbrofuzz
+JennyBot
+JetCar
+Jetty
 JikeSpider
 Joomla
-JustView
-Kenjin Spider
+Keyword Density
 Kinza
+LWP::Simple
 Lanshanbot
 Larbin
-LeechFTP
-LexiBot
 Lftp
-LieBaoFast
-LinkScan
+Libwhisker
+LinkWalker
 Linkbot
-LinqiaMetadataDownloaderBot
+Linkdexbot
+LinkpadBot
+LinksManager
 LinqiaScrapeBot
-Lipperhey
-MQQBrowser
+Lipperhey Spider
+Lmspider
+Ltx71
+MFC_Tear_Sample
+MJ12bot
 MSIECrawler
-MTRobot
-Magnet
 Mail.RU_Bot
-Majestic SEO
-Majestic12
-MarkMonitor
+Majestic-SEO
 MarkWatch
-MauiBot
+Masscan
+Mata Hari
+MeanPath Bot
 Meanpathbot
 Metauri
 MicroMessenger
-Microsoft Data Access
-Microsoft URL Control
+Mister PiX
 Moblie Safari
-Mojolicious
-Morfeus Fucking Scanner
-Nameprotect
-NearSite
-Needle
-Nessus
+Mojeek
+Mr.4x3
+Msrabot
+NPbot
+Name Intelligence
 Net Vampire
-NetAnts
+NetLyzer
+Netcraft
+Nettrack
+Netvibes
+Nibbler
+Nikto
+NimbleCrawler
 Nimbostratus
 Nmap
-Nuclei
-Octopus
-Offline Explorer
+Nutch
 OnCrawl
-OrangeBot
-OutfoxBot
+Openfind
+Openvas
 PECL::HTTP
-Page Analyzer
-PageAnalyzer
-Pandalytics
 Papa Foto
-Petalbot
-Pi-Monster
-Picscout
+Pavuk
+PeoplePal
 PictureFinder
-Piepmatz
+Pimonster
+Pixray
+PleaseCrawl
 Pockey
-ProWebWalker
+Probethenet
 Pu_iN
-PyCurl
-RankActiveLinkBot
+PxBroker
+QueryN Metasearch
+RSSingBot
+RankActive
 RankingBot2
+ReGet
 RealDownload
-Recorder
-RedesScrapy
-Rogerbot
+RebelMouse
+RepoMonkey
 SEOkicks
-SEOstats
+SMTBot
+SalesIntelligent
 Screaming
-SearchmetricsBot
-Semrush
+SemrushBot
 SentiBot
-SeoSiteCheckup
-Seomoz
 Shodan
-Siphon
 Site Sucker
+SiteCheckerBotCrawler
 SiteExplorer
 SiteLockSpider
-SiteSnagger
-SiteSucker
+Sitebeam
 Siteimprove
-SmartDownload
-Sociscraper
+Sitevigil
+SlySearch
+Snoopy
+SocialRankIOBot
 Sottopop
+SpaceBison
+SpankBot
+Spinn3r
 SputnikBot
-Sqworm
+Sqlworm
 Steeler
-Stripper
-SuperBot
-SuperHTTP
-SurveyBot
+Sucuri
+Surfbot
 Suzuran
-Swiftbot
-T8Abot
-Teleport
+Szukacz
+TeleportPro
 Telesoft
 Telesphoreo
 Telesphorep
 The Intraformant
-Titan
-Toata
+TightTwatBot
 Tracemyfile
 Trendictionbot
-True_Robot
 Turnitin
 TurnitinBot
-UnisterBot
-VCI
+Upflow
+V-BOT
 Vacuum
+VeriCiteCrawler
 Voil
 Voltron
-WASALive-Bot
 WBSearchBot
-WEBDAV
-WISENutbot
-WPScan
-Web Auto
-Web Fetch
+WWW-Mechanize
+WWWOFFLE
+WallpapersHD
+Web Collage
+Web Enhancer
 Web Fuck
-Web Sucker
 WebBandit
-WebCollage
-WebReaper
-WebSauger
-WebStripper
-WebSucker
+WebEnhancer
+WebImageCollector
+WebPix
 WebWhacker
 WebZIP
 Webalta
 WebmasterWorldForumBot
-WebsiteQuester
-Whack
-Who.is Bot
+Webster
+Whacker
+Whatweb
 Widow
-WinHTTrack
-WiseGuys Robot
+Woobot
 Wotbox
-Wprecon
 Xaldon WebSpider
-YoudaoBot
-Zade
 Zauba
 Zeus
-ZmEu
 ZyBorg
 archive.org_bot
-arquivo.pt
 check1.exe
+coccocbot-web
 cognitiveseo
-crawl.sogou.com
-domainsproject.org
-heritrix
+com.plumanalytics
+crawler.feedback
+demandbase-bot
+evc-batch
 internetVista monitor
-isitwp.com
-lwp-request
-mediawords
-muhstik-scan
-page scorer
-plumanalytics
-probe-image-size
-scan.lol
+meanpathbot
+netEstate NE Crawler
+polaris version
+satoristudio.net
+scalaj-http
 seobility
-seostar
-sitechecker.pro
+sogouspider
 sp_auditbot
-tAkeOut
+voyagerx.com
+webmeup-crawler
+x22Mozilla
 xpymep1.exe
+zauba.io