V4.2021.12.2790 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
+360Spider
+404enemy
 ADmantX
 ASPSeek
 Aboundex
-AfD-Verbotsverfahren
+Acunetix
 AhrefsBot
-Aipbot
+Alexibot
+AllSubmitter
+Anarchie
+Anarchy
+BBBike
 BDCbot
 BLEXBot
+BackDoorBot
+BackStreet
 BackWeb
 BacklinkCrawler
+Badass
 BatchFTP
-BlackWidow
-Blackboard
-Blow
-Brandprotect
+Bitacle
+Boardreader
+Bolt
 Buck
+Buddy
 BuiltWith
-CATExplorador
-CODE87
+BunnySlippers
+BuzzSumo
+CCBot
+Calculon
+CazoodleBot
+Cegbfeieh
+CensysInspect
 CheTeam
-Claritybot
-Cogentbot
+CheeseBot
+CherryPicker
+Chlooe
+Citoid
+Copier
 CopyRightCheck
 Cosmos
-Crawling at Home Project
-CrazyWebCrawler
-Crescent
-Curious
-Custo
-DBLBot
 DIIbot
-DataCha0s
-Demon
-Digincore
+DatabaseDriverMysqli
 DigitalPebble
 Dirbuster
-Disco
 Discobot
-Discoverybot
 DittoSpyder
-DomainAppender
+DnBCrawler-Analytics
+DomCopBot
 DomainSigmaCrawler
-Domains Project
-Dragonfly
+DomainStatsBot
 EMail Wolf
 EasyDL
 Ebingbong
-Ecxi
+EroCrawler
 Evil
-Exabot
-Express WebPictures
-ExtLinksBot
-Extreme Picture Finder
-Firefox/7.0
+Fimap
+FlashGet
 Flunky
 Foobot
 FrontPage
+Fuzz
+FyberSpider
 Fyrebot
-GalaxyBot
 Genieo
+GermCrawler
 GetWeb
 Gigabot
-Go-Ahead-Got-It
-GoZilla
 Gotit
-Grafula
-GridBot
+Grabber
 HEADMasterSEO
 HMView
 HTTP::Lite
-HybridBot
-Image Fetch
-Image Sucker
+Heritrix
+IDBot
+IRLbot
+IlseBot
 IndeedBot
-InfoNaviRobot
 InfoTekies
-Intelliseek
-Internet Ninja
-InternetSeer
-IstellaBot
+InterGET
+Iskanie
 JOC Web Spider
-Jbrofuzz
-JetCar
+JamesBOT
+Jetty
 JikeSpider
-Keyword Density
+Joomla
+JustView
+Jyxobot
 Kozmosbot
 LNSpiderguy
-Larbin
 Leap
+LeechGet
 Lftp
+Likse
 LinkScan
-LinkWalker
 Linkbot
 Linkdexbot
-LinkextractorPro
+LinkpadBot
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
-Lipperhey Spider
-Litemage_walker
-Lmspider
+LinqiaRSSBot
+Lipperhey
 Ltx71
-MFC_Tear_Sample
+MIDown tool
 MIIxpc
 MQQBrowser
-MarkMonitor
-MarkWatch
+Majestic SEO
 Mass Downloader
-Masscan
-MauiBot
-Meanpathbot
+MeanPath Bot
 Mediatoolkitbot
 Metauri
-Microsoft URL Control
+Mister PiX
+Moblie Safari
 Mojeek
-Mojolicious
-Mr.4x3
+Mozlila
 Msrabot
-NICErsPRO
-Navroad
-Nessus
-NetAnts
+Musobot
+NPbot
+Nameprotect
+Net Vampire
+NetMechanic
 NetZIP
-Nettrack
 Netvibes
+NextGenSearchBot
 NimbleCrawler
 Ninja
 Not
-Offline Explorer
-OpenLinkProfiler
-Openfind
-OrangeBot
-OrangeSpider
-PECL::HTTP
-POE-Component-Client-HTTP
-Page Analyzer
+Nuclei
+Nutch
+Offline Navigator
+OnCrawl
+OutfoxBot
 PageAnalyzer
-PageThing.com
-Panscient
+PageScorer
+Pandalytics
+Pavuk
 PeoplePal
-Pi-Monster
+Petalbot
 Picsearch
 Pimonster
+Pixray
 Pockey
+ProPowerBot
 ProWebWalker
-Probethenet
-Psbot
-Pu_iN
+Pump
+QueryN Metasearch
 RSSingBot
 RankActive
+RankActiveLinkBot
 RankFlex
-RankingBot2
-Rankivabot
-Reaper
-RepoMonkey
+Re-re
+ReGet
+RealDownload
+RedesScrapy
+RocketCrawler
 Rogerbot
-SEOlyticsCrawler
-SEOstats
+SEOkicks
+SEOkicks-Robot
 SISTRIX
-ScanAlert
-Scanbot
-ScreenerBot
-SearchmetricsBot
-Semrush
-SeoSiteCheckup
-SiteCheckerBotCrawler
-SiteLockSpider
+SalesIntelligent
+ScoutJet
+Scrapy
+Searchestate
+SentiBot
+SeobilityBot
+Shodan
+SiteSnagger
+SiteSucker
 Sitebeam
-Siteimprove
-SmartDownload
-Snapbot
-Sottopop
-SpaceBison
+Snake
+Snoopy
+SocialRankIOBot
+Sociscraper
+Sosospider
+SpankBot
+Spanner
+Spbot
 Spinn3r
 Sqlmap
-Surfbot
-SurveyBot
+Sqworm
+SuperBot
 Suzuran
-Swiftbot
 T0PHackTeam
-T8Abot
-Teleport
-TheNomad
+Telesphoreo
+Telesphorep
 Titan
 Toata
-Tracemyfile
 Trendictionbot
-TwengaBot
+True_Robot
+Turnitin
 Typhoeus
-URLy.Warning
 UnisterBot
-VB Project
-VCI
-VidibleScraper
+Upflow
+Vacuum
 Voil
-WASALive-Bot
-WEBDAV
-WISENutbot
-WWW::Mechanize
-WWWOFFLE
+Voltron
+WPScan
 Web Auto
+Web Collage
 Web Enhancer
 Web Fetch
 Web Fuck
-Web Sauger
-Web Sucker
-WebFetch
-WebImageCollector
-WebReaper
+WebEnhancer
+WebGo IS
+WebLeacher
 WebSauger
-WebWhacker
-WebmasterWorldForumBot
+WebStripper
+WebZIP
 Webshag
 WebsiteExtractor
-Webster
+Whack
+Whacker
 Widow
 WiseGuys Robot
-Wprecon
+Xaldon WebSpider
 YoudaoBot
+Zauba
 Zitebot
-ZmEu
 ZoomBot
-adscanner
-archive.org_bot
-backlink-check
-cah.io.community
-check1.exe
-cognitiveseo
+ZumBot
+ZyBorg
+arquivo.pt
+coccocbot-web
+com.plumanalytics
 crawler.feedback
 crawler4j
-dataforseo.com
 facebookscraper
 gopher
-heritrix
-internetVista monitor
+instabid
 ips-agent
 isitwp.com
 lwp-request
-magpie-crawler
-mediawords
-muhstik-scan
-pcBrowser
-probe-image-size
-seocompany.store
-seostar
+lwp-trivial
+netEstate NE Crawler
+plumanalytics
+polaris version
+s1z.ru
+seobility
+seoscanners
+serpstatbot
+sitechecker.pro
+siteripz
+sogouspider
+trendiction.com
 ubermetrics-technologies.com
-webprosbot
-x09Mozilla
-xpymep1.exe
+voyagerx.com
+webmeup-crawler
+webpros.com
+x22Mozilla
+zauba.io