V4.2021.11.2720 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-80legs
+360Spider
 ADmantX
+ALittle Client
+Abonti
+Acunetix
 AfD-Verbotsverfahren
+AhrefsBot
 Alexibot
-Alligator
-Anarchy
-Aspiegel
-Asterias
-AwarioRssBot
+AllSubmitter
+AlphaBot
+Anarchie
+Anarchy99
+Anthill
+Apexoo
 AwarioSmartBot
 BLEXBot
+BackDoorBot
 BackStreet
 BackWeb
-Badass
+Backlink-Ceck
+BacklinkCrawler
 Barkrowler
 BatchFTP
 Battleztar Bazinga
 BetaBot
-Bigfoot
 Bitacle
-Black Hole
+Blackboard
 Blow
+Bolt
+BotALot
 Brandprotect
-Buck
-Bullseye
+Buddy
+BuiltBotTough
+BuiltWith
+BuzzSumo
 CATExplorador
-CCBot
 CODE87
-Calculon
-Cegbfeieh
-CensysInspect
-CherryPicker
+CSHttp
+ChinaClaw
 Chlooe
-Cliqzbot
-Cloud mapping
-Cogentbot
-Cosmos
+Claritybot
 Craftbot
 Crawling at Home Project
+CrazyWebCrawler
 Curious
+Custo
 CyotekWebCopy
-DIIbot
-DTS Agent
-Devil
+DBLBot
+DatabaseDriverMysqli
+Deusu
+Digincore
+Dirbuster
 Disco
 Discoverybot
+Dispatch
 DittoSpyder
+DnBCrawler-Analytics
+DnyzBot
 DomainCrawler
-DomainStatsBot
-Download Wonder
-EMail Siphon
-EMail Wolf
+Dotbot
+Dragonfly
 EasyDL
 EirGrabber
 EroCrawler
-Express WebPictures
-Extractor
-FDM
-FHscan
-Fimap
+Exabot
+Extreme Picture Finder
+EyeNetIE
+FemtosearchBot
 Firefox/7.0
 FlashGet
-Foobot
-Freeuploader
 FrontPage
+Fuzz
+Fyrebot
+GT::WWW
 GalaxyBot
-GetRight
-GetWeb
 Getintent
-Gigabot
 GoZilla
 Gotit
 Grabber
 Grafula
-GrapeshotCrawler
+GrapeFX
 GridBot
 HEADMasterSEO
-HMView
-HTTrack
-Haansoft
+HTMLparser
 HaosouSpider
-Havij
+Harvest
 Heritrix
-Iblog
-Image Fetch
-Iria
-IstellaBot
-JOC Web Spider
-JamesBOT
-JennyBot
-JetCar
-Jetty
-JikeSpider
-Joomla
+HybridBot
+IDBTE4M
+IRLbot
+IndeedBot
+Intelliseek
+InternetSeer
+Jorgee
 Jyxobot
 Kinza
-LNSpiderguy
-Leap
+Kozmosbot
 LeechFTP
-LeechGet
 Lftp
 LieBaoFast
-Lightspeedsystems
-Likse
 LinkScan
-LinkWalker
-Linkbot
-Linkdexbot
-LinqiaMetadataDownloaderBot
+LinkextractorPro
+LinkpadBot
 LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
-Lmspider
-Ltx71
 MFC_Tear_Sample
-MIDown tool
-MQQBrowser
 MSFrontPage
+Mag-Net
+Magnet
 Majestic SEO
-Mass Downloader
-Mb2345Browser
+Majestic-SEO
+Majestic12
+MarkWatch
+Masscan
+Mata Hari
+MauiBot
 MeanPath Bot
-Metauri
-Moblie Safari
-Mojeek
+Meanpathbot
+Mediatoolkitbot
+MicroMessenger
+Microsoft URL Control
+Mister PiX
+Mojolicious
 Morfeus Fucking Scanner
+Mozlila
+Msrabot
 NICErsPRO
 NPbot
 Name Intelligence
-NetAnts
-NetZIP
-Netcraft
+Needle
 Nettrack
-NextGenSearchBot
-NimbleCrawler
 Nimbostratus
 Ninja
-Nmap
-Nuclei
 Octopus
-OrangeBot
-OrangeSpider
-OutclicksBot
-PageGrabber
+Openfind
+OutfoxBot
+PHPCrawl
+PageAnalyzer
+PageScorer
 PageThing.com
-PeoplePal
-Picscout
-ProPowerBot
+Pandalytics
+Papa Foto
+Petalbot
+PictureFinder
+Pixray
 ProWebWalker
-Probethenet
-PyCurl
+PxBroker
 QueryN Metasearch
 RankActive
-RankingBot
-RankurBot
-Re-re
-Reaper
+RankActiveLinkBot
+RankFlex
+RankingBot2
 RedesScrapy
-Rogerbot
-SBIder
+RepoMonkey
+Ripper
+RocketCrawler
 SEOprofiler
-SEOstats
-SMTBot
+ScanAlert
+Scanbot
 ScoutJet
-Semrush
-SeobilityBot
-Seomoz
-Shodan
-Siphon
-SiteCheckerBotCrawler
-SiteSnagger
+Scrapy
+Screaming
+ScrepyBot
+SemanticJuice
+SemrushBot
+SiteLockSpider
+SiteSucker
 Siteimprove
-SmartDownload
-Snoopy
+Sitevigil
+SlySearch
+Snake
+SocialRankIOBot
 Sosospider
-SpankBot
-Sqworm
-SurveyBot
-Swiftbot
-T0PHackTeam
-T8Abot
-Teleport
+SpaceBison
+Spanner
+Spbot
+SputnikBot
+Sqlworm
+Stripper
+SuperBot
+SuperHTTP
 Telesoft
 Telesphoreo
-The Intraformant
+Thumbor
+TightTwatBot
 Titan
-Toata
 Toweyabot
-Trendiction
-Trendictionbot
-TwengaBot
-V-BOT
-VCI
-Vagabondo
+Tracemyfile
+UnisterBot
+Upflow
+VB Project
+Vacuum
 VelenPublicWebCrawler
 VeriCiteCrawler
-Virusdie
+VidibleScraper
+VoidEYE
+Voil
 Voltron
 WASALive-Bot
-WBSearchBot
+WEBDAV
 WISENutbot
-WWWOFFLE
-Wallpapers/3.0
-WallpapersHD
-Web Collage
-Web Pix
-WebCopier
-WebEnhancer
-WebFuck
+WPScan
+WWW-Collector-E
+WWW-Mechanize
+Wallpapers
+Web Fuck
+Web Sauger
+Web Sucker
+WebGo IS
 WebImageCollector
-WebReaper
+WebLeacher
 WebSauger
-Webshag
-Website Quester
-Whacker
-Who.is Bot
+WebStripper
+WebSucker
+WebWhacker
+Webalta
+WebsiteQuester
+Webster
+Widow
+WinHTTrack
 WiseGuys Robot
-Wotbox
+Woobot
 Wprecon
 Xaldon WebSpider
 Xenu
-Zeus
-ZoominfoBot
-check1.exe
-clark-crawler
-coccocbot-web
+Zade
+Zauba
+Zermelo
+Zitebot
+adscanner
+archive.org_bot
+arquivo-web-crawler
+autoemailspider
+cognitiveseo
+com.plumanalytics
 dataforseo.com
-demandbase-bot
-facebookscraper
-gopher
+domainsproject.org
+eCatch
+evc-batch
 instabid
-internetVista monitor
+isitwp.com
 lwp-request
 magpie-crawler
-meanpathbot
-mediawords
-pcBrowser
-plumanalytics
-ripz
-s1z.ru
+muhstik-scan
+polaris version
+probe-image-size
+satoristudio.net
 scalaj-http
-scan.lol
-seostar
-serpstatbot
-sitechecker.pro
-trendiction.com
-trendiction.de
-voyagerx.com
+seoscanners
 webpros.com
+x09Mozilla