mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-04-28 11:30:15 +00:00
539 lines
15 KiB
Go
539 lines
15 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
|
)
|
|
|
|
// fakeStorageClient provides minimal PVE responses needed by the optimized storage poller.
|
|
type fakeStorageClient struct {
|
|
allStorage []proxmox.Storage
|
|
storageByNode map[string][]proxmox.Storage
|
|
zfsPoolsByNode map[string][]proxmox.ZFSPoolInfo
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
|
|
if storages, ok := f.storageByNode[node]; ok {
|
|
return storages, nil
|
|
}
|
|
return nil, fmt.Errorf("unexpected node: %s", node)
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {
|
|
return f.allStorage, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetReplicationStatus(ctx context.Context) ([]proxmox.ReplicationJob, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
|
|
return nil, nil
|
|
}
|
|
func (f *fakeStorageClient) GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.ContainerInterface, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) IsClusterMember(ctx context.Context) (bool, error) {
|
|
return false, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
|
|
return "", nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetVMMemAvailableFromAgent(ctx context.Context, node string, vmid int) (uint64, error) {
|
|
return 0, fmt.Errorf("not implemented")
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) {
|
|
if pools, ok := f.zfsPoolsByNode[node]; ok {
|
|
return pools, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetCephDF(ctx context.Context) (*proxmox.CephDF, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStorageClient) GetNodePendingUpdates(ctx context.Context, node string) ([]proxmox.AptPackage, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func TestPollStorageWithNodesOptimizedRecordsMetricsAndAlerts(t *testing.T) {
|
|
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
|
|
|
monitor := &Monitor{
|
|
state: &models.State{},
|
|
metricsHistory: NewMetricsHistory(16, time.Hour),
|
|
alertManager: alerts.NewManager(),
|
|
}
|
|
t.Cleanup(func() {
|
|
monitor.alertManager.Stop()
|
|
})
|
|
|
|
// Ensure storage alerts trigger immediately for the test.
|
|
cfg := monitor.alertManager.GetConfig()
|
|
cfg.MinimumDelta = 0
|
|
if cfg.TimeThresholds == nil {
|
|
cfg.TimeThresholds = make(map[string]int)
|
|
}
|
|
cfg.TimeThresholds["storage"] = 0
|
|
monitor.alertManager.UpdateConfig(cfg)
|
|
|
|
storage := proxmox.Storage{
|
|
Storage: "local",
|
|
Type: "dir",
|
|
Content: "images",
|
|
Active: 1,
|
|
Enabled: 1,
|
|
Shared: 0,
|
|
Total: 1000,
|
|
Used: 900,
|
|
Available: 100,
|
|
}
|
|
|
|
client := &fakeStorageClient{
|
|
allStorage: []proxmox.Storage{storage},
|
|
storageByNode: map[string][]proxmox.Storage{
|
|
"node1": {storage},
|
|
},
|
|
}
|
|
|
|
nodes := []proxmox.Node{
|
|
{Node: "node1", Status: "online"},
|
|
}
|
|
|
|
monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)
|
|
|
|
metrics := monitor.metricsHistory.GetAllStorageMetrics("inst1-node1-local", time.Minute)
|
|
if len(metrics["usage"]) != 1 {
|
|
t.Fatalf("expected one usage metric entry, got %d", len(metrics["usage"]))
|
|
}
|
|
if len(metrics["used"]) != 1 {
|
|
t.Fatalf("expected one used metric entry, got %d", len(metrics["used"]))
|
|
}
|
|
if len(metrics["total"]) != 1 {
|
|
t.Fatalf("expected one total metric entry, got %d", len(metrics["total"]))
|
|
}
|
|
if len(metrics["avail"]) != 1 {
|
|
t.Fatalf("expected one avail metric entry, got %d", len(metrics["avail"]))
|
|
}
|
|
|
|
if diff := math.Abs(metrics["usage"][0].Value - 90); diff > 0.001 {
|
|
t.Fatalf("expected usage metric 90, diff %.4f", diff)
|
|
}
|
|
if diff := math.Abs(metrics["used"][0].Value - 900); diff > 0.001 {
|
|
t.Fatalf("expected used metric 900, diff %.4f", diff)
|
|
}
|
|
if diff := math.Abs(metrics["total"][0].Value - 1000); diff > 0.001 {
|
|
t.Fatalf("expected total metric 1000, diff %.4f", diff)
|
|
}
|
|
if diff := math.Abs(metrics["avail"][0].Value - 100); diff > 0.001 {
|
|
t.Fatalf("expected avail metric 100, diff %.4f", diff)
|
|
}
|
|
|
|
alerts := monitor.alertManager.GetActiveAlerts()
|
|
found := false
|
|
for _, alert := range alerts {
|
|
if alert.ID == "inst1-node1-local-usage" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("expected storage usage alert to be active")
|
|
}
|
|
}
|
|
|
|
func TestMatchZFSPoolForStorage(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
rpool := &models.ZFSPool{Name: "rpool"}
|
|
tank := &models.ZFSPool{Name: "tank"}
|
|
|
|
cases := []struct {
|
|
name string
|
|
storage models.Storage
|
|
pools map[string]*models.ZFSPool
|
|
want string
|
|
}{
|
|
{
|
|
name: "exact storage name match",
|
|
storage: models.Storage{Name: "tank"},
|
|
pools: map[string]*models.ZFSPool{"tank": tank},
|
|
want: "tank",
|
|
},
|
|
{
|
|
name: "matches explicit pool field before alias name",
|
|
storage: models.Storage{Name: "local-zfs", Pool: "rpool/data"},
|
|
pools: map[string]*models.ZFSPool{"rpool": rpool},
|
|
want: "rpool",
|
|
},
|
|
{
|
|
name: "matches pool from dataset path",
|
|
storage: models.Storage{Name: "local-zfs", Path: "/rpool/data"},
|
|
pools: map[string]*models.ZFSPool{"rpool": rpool},
|
|
want: "rpool",
|
|
},
|
|
{
|
|
name: "matches dir storage from dataset path",
|
|
storage: models.Storage{Name: "local", Type: "dir", Path: "/rpool/data"},
|
|
pools: map[string]*models.ZFSPool{"rpool": rpool},
|
|
want: "rpool",
|
|
},
|
|
{
|
|
name: "single pool fallback for local zfs",
|
|
storage: models.Storage{Name: "local-zfs", Type: "local-zfs"},
|
|
pools: map[string]*models.ZFSPool{"rpool": rpool},
|
|
want: "rpool",
|
|
},
|
|
{
|
|
name: "no ambiguous fallback across multiple pools",
|
|
storage: models.Storage{Name: "local-zfs"},
|
|
pools: map[string]*models.ZFSPool{"rpool": rpool, "tank": tank},
|
|
want: "",
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
tc := tc
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
matched := matchZFSPoolForStorage(tc.storage, tc.pools)
|
|
if tc.want == "" {
|
|
if matched != nil {
|
|
t.Fatalf("matched pool = %q, want nil", matched.Name)
|
|
}
|
|
return
|
|
}
|
|
if matched == nil {
|
|
t.Fatalf("expected pool %q, got nil", tc.want)
|
|
}
|
|
if matched.Name != tc.want {
|
|
t.Fatalf("matched pool = %q, want %q", matched.Name, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPollStorageWithNodesOptimizedAttachesZFSPoolFromDatasetPath(t *testing.T) {
|
|
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
|
|
|
monitor := &Monitor{
|
|
state: &models.State{},
|
|
}
|
|
|
|
storage := proxmox.Storage{
|
|
Storage: "local-zfs",
|
|
Type: "local-zfs",
|
|
Content: "images,rootdir",
|
|
Active: 1,
|
|
Enabled: 1,
|
|
Shared: 0,
|
|
Path: "/rpool/data",
|
|
Total: 1000,
|
|
Used: 400,
|
|
Available: 600,
|
|
}
|
|
|
|
client := &fakeStorageClient{
|
|
allStorage: []proxmox.Storage{storage},
|
|
storageByNode: map[string][]proxmox.Storage{
|
|
"pve1": {storage},
|
|
},
|
|
zfsPoolsByNode: map[string][]proxmox.ZFSPoolInfo{
|
|
"pve1": {
|
|
{Name: "rpool", State: "ONLINE", Health: "ONLINE"},
|
|
},
|
|
},
|
|
}
|
|
|
|
nodes := []proxmox.Node{
|
|
{Node: "pve1", Status: "online"},
|
|
}
|
|
|
|
monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)
|
|
|
|
if len(monitor.state.Storage) != 1 {
|
|
t.Fatalf("expected 1 storage entry, got %d", len(monitor.state.Storage))
|
|
}
|
|
if monitor.state.Storage[0].ZFSPool == nil {
|
|
t.Fatal("expected ZFS pool details to be attached")
|
|
}
|
|
if monitor.state.Storage[0].ZFSPool.Name != "rpool" {
|
|
t.Fatalf("ZFS pool name = %q, want rpool", monitor.state.Storage[0].ZFSPool.Name)
|
|
}
|
|
}
|
|
|
|
func TestPollStorageWithNodesOptimizedSynthesizesSharedClusterOnlyStorage(t *testing.T) {
|
|
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
|
|
|
monitor := &Monitor{
|
|
state: &models.State{},
|
|
}
|
|
|
|
localStorage := proxmox.Storage{
|
|
Storage: "local",
|
|
Type: "dir",
|
|
Content: "images",
|
|
Active: 1,
|
|
Enabled: 1,
|
|
Total: 1000,
|
|
Used: 250,
|
|
Available: 750,
|
|
}
|
|
|
|
cephStorage := proxmox.Storage{
|
|
Storage: "ceph-shared",
|
|
Type: "rbd",
|
|
Content: "images,rootdir",
|
|
Nodes: "pve1,pve2",
|
|
Pool: "ceph-pool",
|
|
}
|
|
|
|
client := &fakeStorageClient{
|
|
allStorage: []proxmox.Storage{localStorage, cephStorage},
|
|
storageByNode: map[string][]proxmox.Storage{
|
|
"pve1": {localStorage},
|
|
"pve2": {},
|
|
},
|
|
}
|
|
|
|
nodes := []proxmox.Node{
|
|
{Node: "pve1", Status: "online"},
|
|
{Node: "pve2", Status: "online"},
|
|
}
|
|
|
|
monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)
|
|
|
|
if len(monitor.state.Storage) != 2 {
|
|
t.Fatalf("expected 2 storage entries, got %d", len(monitor.state.Storage))
|
|
}
|
|
|
|
var shared *models.Storage
|
|
for i := range monitor.state.Storage {
|
|
if monitor.state.Storage[i].ID == "inst1-cluster-ceph-shared" {
|
|
shared = &monitor.state.Storage[i]
|
|
break
|
|
}
|
|
}
|
|
if shared == nil {
|
|
t.Fatal("expected synthesized shared Ceph storage entry")
|
|
}
|
|
if !shared.Shared {
|
|
t.Fatal("expected synthesized storage to be marked shared")
|
|
}
|
|
if shared.Node != "cluster" {
|
|
t.Fatalf("shared storage node = %q, want cluster", shared.Node)
|
|
}
|
|
if shared.Instance != "inst1" {
|
|
t.Fatalf("shared storage instance = %q, want inst1", shared.Instance)
|
|
}
|
|
if shared.Type != "rbd" {
|
|
t.Fatalf("shared storage type = %q, want rbd", shared.Type)
|
|
}
|
|
if shared.Pool != "ceph-pool" {
|
|
t.Fatalf("shared storage pool = %q, want ceph-pool", shared.Pool)
|
|
}
|
|
if shared.NodeCount != 2 {
|
|
t.Fatalf("shared storage node count = %d, want 2", shared.NodeCount)
|
|
}
|
|
if len(shared.Nodes) != 2 || shared.Nodes[0] != "pve1" || shared.Nodes[1] != "pve2" {
|
|
t.Fatalf("shared storage nodes = %#v, want [pve1 pve2]", shared.Nodes)
|
|
}
|
|
if len(shared.NodeIDs) != 2 || shared.NodeIDs[0] != "inst1-pve1" || shared.NodeIDs[1] != "inst1-pve2" {
|
|
t.Fatalf("shared storage node IDs = %#v, want [inst1-pve1 inst1-pve2]", shared.NodeIDs)
|
|
}
|
|
}
|
|
|
|
func TestPollStorageWithNodesOptimizedAttachesZFSPoolForDirStorageOnDatasetPath(t *testing.T) {
|
|
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
|
|
|
monitor := &Monitor{
|
|
state: &models.State{},
|
|
metricsHistory: NewMetricsHistory(16, time.Hour),
|
|
alertManager: alerts.NewManager(),
|
|
}
|
|
t.Cleanup(func() {
|
|
monitor.alertManager.Stop()
|
|
})
|
|
|
|
storage := proxmox.Storage{
|
|
Storage: "local",
|
|
Type: "dir",
|
|
Path: "/rpool/data",
|
|
Content: "images",
|
|
Active: 1,
|
|
Enabled: 1,
|
|
Shared: 0,
|
|
Total: 1000,
|
|
Used: 250,
|
|
Available: 750,
|
|
}
|
|
|
|
client := &fakeStorageClient{
|
|
allStorage: []proxmox.Storage{storage},
|
|
storageByNode: map[string][]proxmox.Storage{
|
|
"node1": {storage},
|
|
},
|
|
zfsPoolsByNode: map[string][]proxmox.ZFSPoolInfo{
|
|
"node1": {
|
|
{Name: "rpool", Size: 1000, Alloc: 250, Free: 750, Frag: 1, Dedup: 1.0, Health: "ONLINE"},
|
|
},
|
|
},
|
|
}
|
|
|
|
nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
|
|
monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)
|
|
|
|
if len(monitor.state.Storage) != 1 {
|
|
t.Fatalf("expected 1 storage entry, got %d", len(monitor.state.Storage))
|
|
}
|
|
if monitor.state.Storage[0].ZFSPool == nil {
|
|
t.Fatal("expected dir storage on ZFS dataset path to have ZFS pool attached")
|
|
}
|
|
if monitor.state.Storage[0].ZFSPool.Name != "rpool" {
|
|
t.Fatalf("ZFS pool name = %q, want rpool", monitor.state.Storage[0].ZFSPool.Name)
|
|
}
|
|
}
|
|
|
|
func TestPollStorageWithNodesOptimizedAttachesZFSPoolFromExplicitPoolField(t *testing.T) {
|
|
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
|
|
|
monitor := &Monitor{
|
|
state: &models.State{},
|
|
metricsHistory: NewMetricsHistory(16, time.Hour),
|
|
alertManager: alerts.NewManager(),
|
|
}
|
|
t.Cleanup(func() {
|
|
monitor.alertManager.Stop()
|
|
})
|
|
|
|
storage := proxmox.Storage{
|
|
Storage: "local-zfs",
|
|
Type: "zfspool",
|
|
Pool: "rpool/data",
|
|
Content: "images,rootdir",
|
|
Active: 1,
|
|
Enabled: 1,
|
|
Shared: 0,
|
|
Total: 1000,
|
|
Used: 250,
|
|
Available: 750,
|
|
}
|
|
|
|
client := &fakeStorageClient{
|
|
allStorage: []proxmox.Storage{storage},
|
|
storageByNode: map[string][]proxmox.Storage{
|
|
"node1": {storage},
|
|
},
|
|
zfsPoolsByNode: map[string][]proxmox.ZFSPoolInfo{
|
|
"node1": {
|
|
{Name: "rpool", Size: 1000, Alloc: 250, Free: 750, Frag: 1, Dedup: 1.0, Health: "ONLINE"},
|
|
},
|
|
},
|
|
}
|
|
|
|
nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
|
|
monitor.pollStorageWithNodes(context.Background(), "inst1", client, nodes)
|
|
|
|
if len(monitor.state.Storage) != 1 {
|
|
t.Fatalf("expected 1 storage entry, got %d", len(monitor.state.Storage))
|
|
}
|
|
if monitor.state.Storage[0].ZFSPool == nil {
|
|
t.Fatal("expected zfspool storage with explicit pool field to have ZFS pool attached")
|
|
}
|
|
if monitor.state.Storage[0].ZFSPool.Name != "rpool" {
|
|
t.Fatalf("ZFS pool name = %q, want rpool", monitor.state.Storage[0].ZFSPool.Name)
|
|
}
|
|
}
|