V4.2022.02.2937 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-80legs
+404checker
 ADmantX
-AhrefsBot
-AiHitBot
+Abonti
+Aboundex
+Aboundexbot
+AfD-Verbotsverfahren
 AlphaBot
-Anarchy
+Anarchie
+Ankit
 Anthill
-Asterias
-Atomseobot
-BDCbot
+Aspiegel
+AwarioRssBot
+AwarioSmartBot
+BBBike
 BackDoorBot
 BackStreet
+BackWeb
 BacklinkCrawler
 Barkrowler
+BatchFTP
+Battleztar Bazinga
+BetaBot
+Bigfoot
+Bitacle
 Black Hole
-BlackWidow
-Blow
-BlowFish
 BotALot
-Brandwatch
+Brandprotect
 Buck
 Buddy
 BuiltBotTough
 BuiltWith
 Bullseye
-BunnySlippers
-CCBot
-CSHttp
-Cegbfeieh
-CensysInspect
-CheTeam
-CheeseBot
-CherryPicker
+CATExplorador
+Calculon
+Cliqzbot
+Cocolyzebot
+Cogentbot
 Collector
 Copier
-CrazyWebCrawler
-Crescent
-Curious
-DBLBot
+CopyRightCheck
+Cosmos
+CyotekWebCopy
+DIIbot
 DTS Agent
 DataCha0s
-Dirbuster
+Disco
 Discobot
 DnBCrawler-Analytics
 DnyzBot
 DomCopBot
-DomainAppender
+Domains Project
 Download Wonder
-Drip
-EasyDL
+Dragonfly
+ECCP/1.0
+EMail Wolf
+Ebingbong
+Ecxi
+EroCrawler
 Express WebPictures
-Extractor
-ExtractorPro
 FDM
 FHscan
 FemtosearchBot
-Foobot
+Firefox/7.0
+Flunky
 Freeuploader
 FrontPage
-Fuzz
-GT::WWW
+GalaxyBot
+Genieo
 GetRight
-GoZilla
-GrabNet
-Grabber
-Grafula
+Getintent
 GrapeFX
-GrapeshotCrawler
-GridBot
-HTMLparser
+HTTP::Lite
 HTTrack
-Humanlinks
+Haansoft
+Heritrix
 IDBTE4M
 Id-search
-IlseBot
-Image Fetch
-Image Sucker
 IndeedBot
-Indy Library
-InfoTekies
-Iria
+Intelliseek
+InterGET
+InternetSeer
 Iskanie
 IstellaBot
-JamesBOT
 JennyBot
-Jetty
-Jyxobot
+JetCar
+Kinza
 Larbin
-Leap
-Libwhisker
-LieBaoFast
-LinkWalker
-Linkdexbot
-LinqiaRSSBot
+LeechGet
+LexiBot
+Linkbot
+LinkextractorPro
+LinksManager
+LinqiaScrapeBot
+Lipperhey
+Lipperhey Spider
 Litemage_walker
+MFC_Tear_Sample
 MIDown tool
 MIIxpc
 MJ12bot
-MSFrontPage
 MSIECrawler
-MTRobot
-Majestic SEO
+Majestic-SEO
+Majestic12
 MarkWatch
-Masscan
-MauiBot
+Mass Downloader
+Mata Hari
 Mb2345Browser
-Mediatoolkitbot
-Minefield
+MicroMessenger
+Microsoft Data Access
+Mister PiX
+Moblie Safari
 Mojolicious
 Morfeus Fucking Scanner
-Mozlila
 Name Intelligence
+Nameprotect
 Navroad
-Needle
 Net Vampire
-Netcraft
+NetZIP
 Nettrack
 Netvibes
-Niki-bot
+Nibbler
 Nikto
+NimbleCrawler
 Ninja
-Nmap
-Not
 Nutch
+Octopus
 Offline Navigator
-OnCrawl
-OpenVAS
+OrangeBot
 OrangeSpider
+PECL::HTTP
+PHPCrawl
+Page Analyzer
 PageAnalyzer
+PageScorer
+PageThing.com
 Pandalytics
 Panscient
-Pavuk
+PeoplePal
 Petalbot
-Picscout
+Picsearch
+PictureFinder
+Pimonster
 Pockey
-Probethenet
-Psbot
-QueryN Metasearch
-Quick-Crawler
-RankActiveLinkBot
+PxBroker
+RankActive
 RankFlex
-RankingBot2
-RankurBot
-RebelMouse
+RankingBot
+ReGet
+RedesScrapy
 RepoMonkey
-RocketCrawler
-SEOkicks
-SEOkicks-Robot
+SEOprofiler
 SEOstats
-Scanbot
-Scrapy
+Screaming
+Searchestate
 SearchmetricsBot
 Seekport
-SeoSiteCheckup
-Site Sucker
-SiteExplorer
-SiteLockSpider
+SemanticJuice
+SentiBot
+SeobilityBot
+Seomoz
+Siphon
+SiteCheckerBotCrawler
+SiteSnagger
 SiteSucker
 Siteimprove
+SlySearch
 SmartDownload
 Snake
-Snapbot
+Snoopy
 SocialRankIOBot
-Sociscraper
-Sogou web spider
-Spammen
+SpaceBison
 SpankBot
-Spanner
-Steeler
-Sucuri
+Spinn3r
+SputnikBot
+Sqlworm
+Sqworm
 SuperBot
 SuperHTTP
-Surfbot
-SurveyBot
-Suzuran
+Swiftbot
 T8Abot
-Teleport
-TeleportPro
+Telesphoreo
 The Intraformant
 TheNomad
-Thumbor
-TightTwatBot
-True_Robot
+Titan
+Tracemyfile
 Turnitin
+TurnitinBot
 URLy Warning
-URLy.Warning
-V-BOT
-VCI
+UnisterBot
+Upflow
 Vacuum
-Vagabondo
-VidibleScraper
+VelenPublicWebCrawler
 Virusdie
 WBSearchBot
-WISENutbot
+WPScan
 WWW::Mechanize
 WWWOFFLE
-Wallpapers
+Wallpapers/3.0
+Web Collage
 Web Fetch
+Web Fuck
 Web Pix
-WebAuto
-WebFuck
+Web Sauger
+WebBandit
+WebCopier
+WebEnhancer
 WebGo IS
-WebImageCollector
-WebLeacher
-WebReaper
+WebPix
 WebSauger
-WebSucker
 Webalta
 Website Quester
 WebsiteExtractor
 WebsiteQuester
 Whatweb
-Widow
-Wonderbot
-Woobot
+Wotbox
 Wprecon
-Xaldon WebSpider
-Xenu
-YoudaoBot
-Zade
-ZoominfoBot
-ZumBot
-arquivo-web-crawler
-backlink-check
-cognitiveseo
-crawler4j
-dataforseo.com
-demandbase-bot
+Zitebot
+adscanner
+arquivo.pt
+autoemailspider
+cah.io.community
+clark-crawler
+com.plumanalytics
+crawler.feedback
 domainsproject.org
 gopher
-lwp-request
+ips-agent
+iubenda-radar
 lwp-trivial
-mediawords
-muhstik-scan
-netEstate NE Crawler
 page scorer
+pcBrowser
+plumanalytics
 polaris version
-satoristudio.net
+probe-image-size
+ripz
 scalaj-http
 scan.lol
+seobility
 seocompany.store
-serpstatbot
-sexsearcher
+sitechecker.pro
+siteripz
 spyfu
-trendiction.com
+tAkeOut
+trendiction.de
 ubermetrics-technologies.com
 voyagerx.com
 webprosbot
+xpymep1.exe
+zauba.io
 zgrab