V4.2021.08.2525 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 360Spider
-ADmantX
-ALittle Client
-Abonti
-Aboundexbot
+80legs
+AIBOT
 AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
-Aipbot
+Alexibot
+AllSubmitter
+Alligator
 AlphaBot
-Anarchie
-Ankit
-Anthill
-Aspiegel
+Anarchy
 Attach
-BackWeb
+AwarioSmartBot
+Backlink-Ceck
 BacklinkCrawler
-Badass
-Bandit
 Barkrowler
 BatchFTP
-Battleztar Bazinga
+BetaBot
 Bigfoot
 Bitacle
 Black Hole
-Blackboard
-Blow
-BotALot
+Bolt
 Brandprotect
-Brandwatch
 Buck
-BuiltBotTough
-BuiltWith
 Bullseye
-BunnySlippers
-BuzzSumo
+CATExplorador
 CCBot
 CODE87
-CSHttp
+CazoodleBot
+CheTeam
 CheeseBot
-Chlooe
-Crescent
+Claritybot
+Cliqzbot
+Cloud mapping
+Cocolyzebot
+Cogentbot
+Copier
+CopyRightCheck
+Craftbot
+CrunchBot
+Curious
 DBLBot
-DIIbot
 DTS Agent
-DataCha0s
-Demon
-Discobot
-Discoverybot
-DittoSpyder
-DnyzBot
+Dirbuster
 DomCopBot
-DomainCrawler
-DomainStatsBot
-Download Wonder
+DomainSigmaCrawler
+Dragonfly
+ECCP/1.0
+EMail Wolf
 EasyDL
-Ecxi
-Evil
-Extractor
-FHscan
+EirGrabber
+EroCrawler
+Ezooms
 FemtosearchBot
 Fimap
-Firefox/7.0
-Flunky
-Foobot
-Freeuploader
-GT::WWW
+FyberSpider
+Fyrebot
 Genieo
-Gigablast
-Go!Zilla
+GetWeb
+Getintent
 Go-Ahead-Got-It
 GoZilla
-GrabNet
-GridBot
-HEADMasterSEO
+GrapeFX
 HTMLparser
-Harvest
-Heritrix
-Hloader
 Humanlinks
-HybridBot
-IDBot
-Iblog
-Image Fetch
-IndeedBot
+IDBTE4M
+IRLbot
+Id-search
 Indy Library
-InfoNaviRobot
-InfoTekies
-Intelliseek
-InterGET
 Internet Ninja
+InternetSeer
+Iskanie
 IstellaBot
 JOC Web Spider
-JetCar
-Jetty
-JikeSpider
+Jbrofuzz
 Joomla
-Jorgee
+Keyword Density
+Kinza
 Kozmosbot
+LNSpiderguy
+LWP::Simple
 Lanshanbot
+Larbin
 LeechFTP
-LeechGet
+Lftp
 Libwhisker
-LinkScan
+LieBaoFast
+LinkWalker
 Linkbot
-LinkextractorPro
-LinkpadBot
 LinksManager
-LinqiaRSSBot
+LinqiaScrapeBot
 Lipperhey
-Litemage_walker
-Lmspider
+Lipperhey Spider
+Ltx71
 MJ12bot
-MQQBrowser
 MSFrontPage
+MSIECrawler
 MTRobot
-Mag-Net
 Majestic SEO
+Majestic-SEO
 MarkMonitor
 MarkWatch
 Mata Hari
-Mb2345Browser
 Meanpathbot
 MegaIndex.ru
+MicroMessenger
+Mojolicious
 Morfeus Fucking Scanner
-Mr.4x3
-Msrabot
 NICErsPRO
+Nameprotect
 Navroad
-NearSite
+NetAnts
+NetSpider
+NetZIP
 Netcraft
-NextGenSearchBot
-Nikto
-Nimbostratus
-Offline Explorer
+Nettrack
+Niki-bot
+NimbleCrawler
+Ninja
+Nutch
+Octopus
+Offline Navigator
 OnCrawl
-OpenLinkProfiler
 OpenVAS
-Openvas
+Openfind
+OrangeBot
+OrangeSpider
 OutfoxBot
+PECL::HTTP
+PHPCrawl
+POE-Component-Client-HTTP
+Page Analyzer
 Papa Foto
+Pavuk
+PeoplePal
+Petalbot
 Pi-Monster
-Picscout
-Piepmatz
-ProPowerBot
-ProWebWalker
+Picsearch
+Pimonster
+PleaseCrawl
+Probethenet
 Pu_iN
 Pump
-QueryN Metasearch
+PxBroker
+PyCurl
 Quick-Crawler
-RankActive
+RSSingBot
+RankActiveLinkBot
 RankurBot
-Reaper
-Ripper
+ReGet
+RealDownload
+RebelMouse
+RepoMonkey
 SBIder
 SEOkicks
-SEOprofiler
-SEOstats
-SMTBot
+SISTRIX
 SalesIntelligent
 ScoutJet
-Screaming
-ScreenerBot
+Scrapy
+ScrepyBot
+Searchestate
+SearchmetricsBot
+Semrush
 SeoSiteCheckup
-Shodan
-SiteExplorer
+SiteCheckerBotCrawler
 SiteSnagger
-SiteSucker
 Sitebeam
-SlySearch
-Snoopy
-SocialRankIOBot
+SmartDownload
+Snake
 Sosospider
+Spammen
 Spanner
 SputnikBot
 Sqworm
-Stripper
-Sucker
-Suzuran
-Szukacz
+Sucuri
+SuperHTTP
+SurveyBot
+Swiftbot
 T0PHackTeam
 Teleport
 The Intraformant
+Thumbor
 TightTwatBot
-Titan
-Toweyabot
 Trendictionbot
-Turnitin
-Typhoeus
+True_Robot
+Turingos
+Twice
 URLy Warning
-URLy.Warning
-Upflow
 VCI
+Vacuum
 VelenPublicWebCrawler
-VeriCiteCrawler
 Virusdie
-WBSearchBot
-WISENutbot
-WPScan
-WWW-Collector-E
-WWWOFFLE
+VoidEYE
+WASALive-Bot
+WWW::Mechanize
 Wallpapers
-WeSEE
-Web Collage
+WallpapersHD
 Web Fetch
-Web Sauger
-WebBandit
+Web Fuck
+Web Sucker
+WebAuto
+WebCollage
 WebEnhancer
 WebFetch
 WebImageCollector
+WebPix
 WebReaper
-WebStripper
+WebSauger
 WebSucker
-WebWhacker
-WebsiteExtractor
-WebsiteQuester
+WebZIP
+WebmasterWorldForumBot
+Webshag
+Webster
+Whacker
 Whatweb
-Who.is Bot
+Widow
 Wotbox
 Wprecon
-Xaldon_WebSpider
+Xaldon WebSpider
 Xenu
 YoudaoBot
 Zade
-Zauba
-ZoominfoBot
-ZyBorg
+Zeus
+Zitebot
+ZumBot
 arquivo.pt
+cah.io.community
+coccocbot-web
 com.plumanalytics
+crawl.sogou.com
+crawler.feedback
+crawler4j
 evc-batch
-instabid
-internetVista monitor
+heritrix
+ips-agent
 lwp-request
-lwp-trivial
-magpie-crawler
-mediawords
-netEstate NE Crawler
-oBot
+meanpathbot
 pcBrowser
+ripz
 s1z.ru
-satoristudio.net
-scalaj-http
 sexsearcher
-tAkeOut
+sitechecker.pro
+siteripz
+sp_auditbot
+spyfu
+sysscan
 trendiction.com
-webmeup-crawler
-webprosbot
-x09Mozilla
+voyagerx.com
 x22Mozilla
+xpymep1.exe
+zauba.io