V4.2022.12.3510 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
 404checker
-80legs
-ADmantX
-Aboundex
-AhrefsBot
-Alexibot
+ALittle Client
+Abonti
+Aboundexbot
+AfD-Verbotsverfahren
 AllSubmitter
+Alligator
 AlphaBot
 Anarchie
-Ankit
+Anarchy
+Anarchy99
 Aspiegel
+Asterias
 Atomseobot
 Attach
-AwarioRssBot
-AwarioSmartBot
-BBBike
-BDCbot
+BDFetch
 BackDoorBot
-Backlink-Ceck
 Badass
 Bandit
 Barkrowler
-BatchFTP
-BetaBot
-Bigfoot
-Bitacle
-Black Hole
-Blow
-BlowFish
-Boardreader
+Battleztar Bazinga
+Blackboard
 BotALot
-Brandprotect
-Buddy
-BuiltWith
+Brandwatch
+BuiltBotTough
 BunnySlippers
-CCBot
 CSHttp
-Cegbfeieh
-CensysInspect
-Cocolyzebot
+Calculon
+Chlooe
 Collector
-Copyscape
-Crawling at Home Project
+Cosmos
+CrazyWebCrawler
+Crescent
+CrunchBot
 Custo
 CyotekWebCopy
 DTS Agent
-DataCha0s
-DatabaseDriverMysqli
-Digincore
-Dirbuster
 Discobot
-Discoverybot
-DomainAppender
-DomainStatsBot
-Dotbot
-Download Wonder
-Drip
-EMail Siphon
+DomCopBot
+DomainCrawler
+DomainSigmaCrawler
 Ebingbong
-Ecxi
 EirGrabber
 EroCrawler
-Express WebPictures
-ExtractorPro
+Exabot
+ExtLinksBot
 Extreme Picture Finder
-FHscan
+EyeNetIE
+Ezooms
 FemtosearchBot
-Firefox/7.0
 FlashGet
-Flunky
-FyberSpider
-Genieo
+Freeuploader
+G-i-g-a-b-o-t
 GetRight
-Gigabot
+Go!Zilla
 GrabNet
-Grabber
-Grafula
-GrapeshotCrawler
-HMView
-Haansoft
+GrapeFX
+HEADMasterSEO
 HaosouSpider
 Havij
-Heritrix
+Hloader
+Humanlinks
 HybridBot
-IDBTE4M
-IRLbot
-Iblog
 IlseBot
 Image Sucker
-Indy Library
+InfoNaviRobot
+InfoTekies
 Intelliseek
+InterGET
+Internet Ninja
+Iskanie
+IstellaBot
 JOC Web Spider
-Joomla
+JennyBot
+JetCar
 Keybot Translation-Search-Machine
-Lanshanbot
+Kozmosbot
+Larbin
 LeechFTP
 Lftp
-Likse
-LinkScan
+LieBaoFast
 LinkWalker
+LinkpadBot
+LinqiaRSSBot
 LinqiaScrapeBot
+Lipperhey
 Litemage_walker
-MIIxpc
-MQQBrowser
+Lmspider
+Ltx71
+MIDown tool
+MJ12bot
+MSIECrawler
+MTRobot
+Mag-Net
 Magnet
-Mail.RU_Bot
 Majestic-SEO
-Majestic12
+MarkMonitor
 Mass Downloader
+MauiBot
 Mb2345Browser
-Meanpathbot
+MeanPath Bot
 Mediatoolkitbot
-MegaIndex.ru
-Metauri
-Microsoft Data Access
-Morfeus Fucking Scanner
+MicroMessenger
+Microsoft URL Control
+Mojeek
+Mojolicious
+MolokaiBot
 Mr.4x3
 NICErsPRO
+NPbot
+Name Intelligence
 Nameprotect
-Navroad
 Needle
-Nessus
-NetSpider
+NetMechanic
 Netcraft
 Nettrack
-NextGenSearchBot
+Nibbler
 Nikto
-NimbleCrawler
-Nutch
-OnCrawl
-OrangeSpider
+Nimbostratus
+Ninja
+Nuclei
+Octopus
+Offline Explorer
+Offline Navigator
+OpenVAS
+Openvas
+OutclicksBot
+PECL::HTTP
+PHPCrawl
 POE-Component-Client-HTTP
-Page Analyzer
-PageGrabber
+PageAnalyzer
 PageThing.com
-Papa Foto
+Pandalytics
+Panscient
 Pavuk
 Petalbot
-Pi-Monster
-PictureFinder
-Pimonster
 Pockey
-ProWebWalker
-Psbot
-Pu_iN
-Pump
+ProPowerBot
 PyCurl
-Quick-Crawler
-RankActive
-RankingBot2
-Rankivabot
+RankingBot
+RankurBot
+Re-re
+ReGet
+RealDownload
 Reaper
 RedesScrapy
-Ripper
-Rogerbot
+RepoMonkey
 SEOkicks
-SISTRIX
-SalesIntelligent
-Scanbot
-Scrapy
-Screaming
+SEOkicks-Robot
+SEOstats
+SMTBot
 ScreenerBot
 SemanticJuice
+Semrush
+SemrushBot
 SentiBot
-SeobilityBot
+SeoSiteCheckup
+Seomoz
+Siphon
+SiteLockSpider
+SiteSnagger
 Sitebeam
-Sitevigil
+SlySearch
 SmartDownload
-Sociscraper
+Snake
+Sogou web spider
 Spammen
+SpankBot
 Spinn3r
-Steeler
-SuperBot
-Suzuran
-T8Abot
+SputnikBot
+Sqlworm
+SuperHTTP
+Surfbot
+Swiftbot
+T0PHackTeam
+Teleport
 TeleportPro
-Telesoft
-TheNomad
-Thumbor
-Titan
-Toata
 Trendiction
+Turingos
+Turnitin
 TurnitinBot
-TwengaBot
-V-BOT
-VB Project
+URLy.Warning
+Upflow
 Vacuum
-VelenPublicWebCrawler
-Voil
+Vagabondo
+VeriCiteCrawler
+Virusdie
 Voltron
-WASALive-Bot
-WBSearchBot
-WPScan
+WWW-Collector-E
 WWW::Mechanize
 Wallpapers
-Wallpapers/3.0
 WallpapersHD
-Web Collage
-Web Fetch
-Web Fuck
+WeSEE
+Web Auto
 Web Pix
-WebBandit
-WebFetch
-WebGo IS
-WebLeacher
-WebPix
-WebSucker
-WebZIP
+Web Sauger
+Web Sucker
+WebAuto
+WebCopier
+WebFuck
+WebReaper
+WebStripper
+WebmasterWorldForumBot
+Website Quester
 WebsiteQuester
+Webster
 Whack
 Whacker
-Whatweb
-Who.is Bot
-Woobot
+WinHTTrack
+Wonderbot
 Wotbox
 Wprecon
-Xaldon_WebSpider
-Zade
 Zeus
 Zitebot
+ZmEu
 ZoominfoBot
-ZumBot
+adscanner
+archive.org_bot
 arquivo-web-crawler
 arquivo.pt
+autoemailspider
 backlink-check
 check1.exe
 clark-crawler
-coccocbot
 com.plumanalytics
+crawler4j
+facebookscraper
 heritrix
-instabid
-ips-agent
+internetVista monitor
 magpie-crawler
 meanpathbot
-mediawords
-muhstik-scan
-netEstate NE Crawler
 oBot
-ripz
+page scorer
+probe-image-size
 scalaj-http
 seobility
 seocompany.store
 seoscanners
 seostar
+sitechecker.pro
+sogouspider
+sp_auditbot
 tAkeOut
-trendiction.com
-webmeup-crawler
-xpymep1.exe
+trendiction.de
+webgains-bot
+webpros.com
+x22Mozilla
 zauba.io
+zgrab