V4.2021.10.2625 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-404checker
-ADmantX
+360Spider
+80legs
 ALittle Client
-Aboundexbot
-Acunetix
-Alexibot
-AlphaBot
+Abonti
+Aboundex
+AfD-Verbotsverfahren
 Anarchy
+Anarchy99
+Ankit
+Attach
 AwarioRssBot
-BDCbot
-BDFetch
+BBBike
 BackStreet
-BacklinkCrawler
+BackWeb
 Badass
-Barkrowler
-Battleztar Bazinga
-BetaBot
-Bigfoot
+Bandit
+BatchFTP
 Black Hole
-BlowFish
-Boardreader
-Bolt
-BotALot
-Brandwatch
-BuiltWith
+Blackboard
+BuiltBotTough
+BunnySlippers
+BuzzSumo
+CATExplorador
+CCBot
 CODE87
-CSHttp
-CazoodleBot
 Cegbfeieh
+CensysInspect
 CheTeam
-CherryPicker
-ChinaClaw
+CheeseBot
 Chlooe
+Claritybot
+Cliqzbot
 Cloud mapping
+Cocolyzebot
+Cogentbot
+Collector
 Copier
-Cosmos
-Crawling at Home Project
+Craftbot
+CrunchBot
 DIIbot
+DSearch
 DTS Agent
+DatabaseDriverMysqli
+Demon
 Digincore
 DigitalPebble
-Disco
-Dispatch
-DittoSpyder
-DomCopBot
+DomainAppender
+Domains Project
+Dotbot
 Download Wonder
 Dragonfly
-Drip
-EMail Wolf
-EasyDL
-Ecxi
-EirGrabber
-EroCrawler
-ExtLinksBot
 Ezooms
 FDM
+FemtosearchBot
+Fimap
 Firefox/7.0
 FlashGet
 Flunky
 Freeuploader
-Fuzz
-FyberSpider
-Fyrebot
 GT::WWW
 GalaxyBot
-Genieo
-GetWeb
+GetRight
+Getintent
 Go!Zilla
-GoZilla
-GridBot
-HMView
+Gotit
+GrabNet
+Grabber
+GrapeshotCrawler
+HEADMasterSEO
 HTMLparser
-HTTP::Lite
 Haansoft
-HaosouSpider
-Hloader
-IDBot
+Harvest
+Havij
 Id-search
-IlseBot
-Intelliseek
+Image Sucker
+InfoNaviRobot
+InfoTekies
 InterGET
-Internet Ninja
+Iria
 Iskanie
 IstellaBot
+JOC Web Spider
+JamesBOT
 JennyBot
 Jetty
-Jyxobot
-Keyword Density
+JikeSpider
 Kozmosbot
-LexiBot
-LieBaoFast
-LinkScan
-LinkWalker
-LinkextractorPro
-Lipperhey
+LNSpiderguy
+LWP::Simple
+Larbin
+LeechFTP
+Libwhisker
+Lightspeedsystems
+Linkbot
 Lipperhey Spider
 Litemage_walker
-Lmspider
-Ltx71
-MFC_Tear_Sample
-MIIxpc
-MSIECrawler
-Mag-Net
+MQQBrowser
 Magnet
-Majestic SEO
-Majestic-SEO
-MarkWatch
-Mass Downloader
-Masscan
-Meanpathbot
+Majestic12
+Mata Hari
+MauiBot
+MeanPath Bot
 Mediatoolkitbot
-Mr.4x3
-Musobot
-NICErsPRO
-NPbot
-Nameprotect
-Navroad
-NearSite
-NetZIP
-Netvibes
+MicroMessenger
+Microsoft URL Control
+Mister PiX
+Moblie Safari
+Mojolicious
+Morfeus Fucking Scanner
+Msrabot
+Name Intelligence
+Needle
+Netcraft
 NextGenSearchBot
-Niki-bot
-NimbleCrawler
 Ninja
-OpenLinkProfiler
+Offline Navigator
+OnCrawl
 OpenVAS
-Openvas
+Openfind
+OrangeBot
 OutfoxBot
 PHPCrawl
-Pandalytics
-Panscient
-Papa Foto
-Pavuk
+PageScorer
 Petalbot
-Picscout
+Picsearch
 PictureFinder
-Pixray
-Probethenet
+Pimonster
+Pockey
+ProPowerBot
 Pu_iN
-QueryN Metasearch
+PxBroker
 Quick-Crawler
 RSSingBot
+RankActiveLinkBot
+RankingBot
 RankingBot2
 Rankivabot
-ReGet
-Recorder
-RedesScrapy
+RankurBot
+Re-re
+RealDownload
+RebelMouse
 RepoMonkey
-Ripper
 Rogerbot
-SBIder
-SEOkicks
-SEOkicks-Robot
+SEOprofiler
 SISTRIX
-ScoutJet
-ScreenerBot
-ScrepyBot
-SearchmetricsBot
-SemrushBot
+ScanAlert
+Screaming
 SentiBot
 SeoSiteCheckup
-Seomoz
+SeobilityBot
+Siphon
 SiteCheckerBotCrawler
+SiteExplorer
 SiteLockSpider
-SiteSucker
-Sitebeam
 Siteimprove
-Snapbot
-Sosospider
-Sottopop
-Spammen
-Spbot
+Sitevigil
+Snake
+Snoopy
+SpaceBison
 SputnikBot
-Sqlmap
+Sqlworm
+Sqworm
+Steeler
+Sucker
 SuperBot
 SurveyBot
 Swiftbot
-Szukacz
-T8Abot
+T0PHackTeam
 Teleport
-TeleportPro
-Telesphoreo
-The Intraformant
+TheNomad
+Thumbor
 TightTwatBot
+Titan
 Toata
-True_Robot
+Trendictionbot
 Turingos
+Turnitin
 TwengaBot
 Twice
-Typhoeus
-UnisterBot
-Upflow
 VB Project
 VCI
-Vacuum
-VelenPublicWebCrawler
-VidibleScraper
-VoidEYE
-WASALive-Bot
+VeriCiteCrawler
+Voil
 WBSearchBot
+WEBDAV
 WPScan
 WWW-Collector-E
 WWW::Mechanize
+WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-Web Auto
-Web Collage
-Web Fetch
+WeSEE
+Web Enhancer
 Web Sauger
 Web Sucker
-WebAuto
-WebCollage
-WebEnhancer
-WebLeacher
+WebFuck
+WebGo IS
+WebReaper
 WebSucker
-Webalta
+WebZIP
 Webshag
 Website Quester
 WebsiteQuester
 Webster
 Whack
 Widow
-WiseGuys Robot
-Wonderbot
-Woobot
+WinHTTrack
+Wotbox
 Wprecon
-Xaldon_WebSpider
+Xaldon WebSpider
 Xenu
-autoemailspider
+Zade
+Zauba
+Zermelo
+Zeus
+ZmEu
+ZoomBot
+ZoominfoBot
+ZyBorg
+arquivo-web-crawler
+arquivo.pt
 backlink-check
 check1.exe
-crawler.feedback
+clark-crawler
+cognitiveseo
+com.plumanalytics
 dataforseo.com
-demandbase-bot
+domainsproject.org
 evc-batch
-facebookscraper
-heritrix
-instabid
 internetVista monitor
-lwp-trivial
-magpie-crawler
-mediawords
-ripz
-s1z.ru
+netEstate NE Crawler
+polaris version
+satoristudio.net
+scalaj-http
+scan.lol
 seoscanners
-sitechecker.pro
-ubermetrics-technologies.com
+seostar
+serpstatbot
+siteripz
+sp_auditbot
+spyfu
+sysscan
+tAkeOut
+trendiction.com
 voyagerx.com
+webmeup-crawler
+webpros.com
 webprosbot
+x22Mozilla
+zauba.io