mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-05-07 08:57:12 +00:00
374 lines
12 KiB
Go
374 lines
12 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// NodeMemoryRaw captures the raw memory fields returned by Proxmox for a node.
|
|
type NodeMemoryRaw struct {
|
|
Total uint64 `json:"total"`
|
|
Used uint64 `json:"used"`
|
|
Free uint64 `json:"free"`
|
|
Available uint64 `json:"available"`
|
|
Avail uint64 `json:"avail"`
|
|
Buffers uint64 `json:"buffers"`
|
|
Cached uint64 `json:"cached"`
|
|
Shared uint64 `json:"shared"`
|
|
EffectiveAvailable uint64 `json:"effectiveAvailable"`
|
|
RRDAvailable uint64 `json:"rrdAvailable,omitempty"`
|
|
RRDUsed uint64 `json:"rrdUsed,omitempty"`
|
|
RRDTotal uint64 `json:"rrdTotal,omitempty"`
|
|
TotalMinusUsed uint64 `json:"totalMinusUsed,omitempty"`
|
|
FallbackTotal uint64 `json:"fallbackTotal,omitempty"`
|
|
FallbackUsed uint64 `json:"fallbackUsed,omitempty"`
|
|
FallbackFree uint64 `json:"fallbackFree,omitempty"`
|
|
FallbackCalculated bool `json:"fallbackCalculated,omitempty"`
|
|
ProxmoxMemorySource string `json:"proxmoxMemorySource,omitempty"`
|
|
}
|
|
|
|
// NodeMemorySnapshot records the final node memory calculation alongside the raw data.
|
|
type NodeMemorySnapshot struct {
|
|
Instance string `json:"instance"`
|
|
Node string `json:"node"`
|
|
RetrievedAt time.Time `json:"retrievedAt"`
|
|
MemorySource string `json:"memorySource"`
|
|
FallbackReason string `json:"fallbackReason,omitempty"`
|
|
Memory models.Memory `json:"memory"`
|
|
Raw NodeMemoryRaw `json:"raw"`
|
|
}
|
|
|
|
// VMMemoryRaw captures both the listing and detailed status memory fields for a VM/CT.
|
|
type VMMemoryRaw struct {
|
|
ListingMem uint64 `json:"listingMem"`
|
|
ListingMaxMem uint64 `json:"listingMaxmem"`
|
|
StatusMem uint64 `json:"statusMem,omitempty"`
|
|
StatusFreeMem uint64 `json:"statusFreemem,omitempty"`
|
|
StatusMaxMem uint64 `json:"statusMaxmem,omitempty"`
|
|
Balloon uint64 `json:"balloon,omitempty"`
|
|
BalloonMin uint64 `json:"balloonMin,omitempty"`
|
|
MemInfoUsed uint64 `json:"meminfoUsed,omitempty"`
|
|
MemInfoFree uint64 `json:"meminfoFree,omitempty"`
|
|
MemInfoTotal uint64 `json:"meminfoTotal,omitempty"`
|
|
MemInfoAvailable uint64 `json:"meminfoAvailable,omitempty"`
|
|
MemInfoBuffers uint64 `json:"meminfoBuffers,omitempty"`
|
|
MemInfoCached uint64 `json:"meminfoCached,omitempty"`
|
|
MemInfoShared uint64 `json:"meminfoShared,omitempty"`
|
|
MemInfoTotalMinusUsed uint64 `json:"meminfoTotalMinusUsed,omitempty"`
|
|
GuestAgentMemAvailable uint64 `json:"guestAgentMemAvailable,omitempty"`
|
|
HostAgentTotal uint64 `json:"hostAgentTotal,omitempty"`
|
|
HostAgentUsed uint64 `json:"hostAgentUsed,omitempty"`
|
|
Agent int `json:"agent,omitempty"`
|
|
DerivedFromBall bool `json:"derivedFromBalloon,omitempty"`
|
|
}
|
|
|
|
// GuestMemorySnapshot records the memory calculation for a guest (VM/LXC).
|
|
type GuestMemorySnapshot struct {
|
|
GuestType string `json:"guestType"`
|
|
Instance string `json:"instance"`
|
|
Node string `json:"node"`
|
|
Name string `json:"name"`
|
|
VMID int `json:"vmid"`
|
|
Status string `json:"status"`
|
|
RetrievedAt time.Time `json:"retrievedAt"`
|
|
MemorySource string `json:"memorySource"`
|
|
FallbackReason string `json:"fallbackReason,omitempty"`
|
|
Memory models.Memory `json:"memory"`
|
|
Raw VMMemoryRaw `json:"raw"`
|
|
Notes []string `json:"notes"`
|
|
}
|
|
|
|
// DiagnosticSnapshotSet aggregates the latest node and guest snapshots.
|
|
type DiagnosticSnapshotSet struct {
|
|
Nodes []NodeMemorySnapshot `json:"nodes"`
|
|
Guests []GuestMemorySnapshot `json:"guests"`
|
|
}
|
|
|
|
func makeNodeSnapshotKey(instance, node string) string {
|
|
return fmt.Sprintf("%s|%s", instance, node)
|
|
}
|
|
|
|
func makeGuestSnapshotKey(instance, guestType, node string, vmid int) string {
|
|
return fmt.Sprintf("%s|%s|%s|%d", instance, guestType, node, vmid)
|
|
}
|
|
|
|
func normalizeNodeMemorySnapshot(snapshot NodeMemorySnapshot) NodeMemorySnapshot {
|
|
snapshot.MemorySource = CanonicalMemorySource(snapshot.MemorySource)
|
|
if snapshot.FallbackReason == "" {
|
|
snapshot.FallbackReason = MemorySourceFallbackReason(snapshot.MemorySource)
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
func normalizeGuestMemorySnapshot(snapshot GuestMemorySnapshot) GuestMemorySnapshot {
|
|
snapshot.MemorySource = CanonicalMemorySource(snapshot.MemorySource)
|
|
if snapshot.FallbackReason == "" {
|
|
snapshot.FallbackReason = MemorySourceFallbackReason(snapshot.MemorySource)
|
|
}
|
|
if snapshot.Notes == nil {
|
|
snapshot.Notes = []string{}
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
func (m *Monitor) logNodeMemorySource(instance, node string, snapshot NodeMemorySnapshot) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
key := makeNodeSnapshotKey(instance, node)
|
|
|
|
source := snapshot.MemorySource
|
|
var prevSource string
|
|
m.diagMu.RLock()
|
|
if existing, ok := m.nodeSnapshots[key]; ok {
|
|
prevSource = existing.MemorySource
|
|
}
|
|
m.diagMu.RUnlock()
|
|
|
|
if prevSource == snapshot.MemorySource {
|
|
return
|
|
}
|
|
|
|
var evt *zerolog.Event
|
|
if MemorySourceIsFallback(source) {
|
|
evt = log.Warn()
|
|
} else {
|
|
evt = log.Debug()
|
|
}
|
|
|
|
evt = evt.
|
|
Str("instance", instance).
|
|
Str("node", node).
|
|
Str("memorySource", source).
|
|
Str("proxmoxSource", snapshot.Raw.ProxmoxMemorySource)
|
|
|
|
if snapshot.FallbackReason != "" {
|
|
evt = evt.Str("fallbackReason", snapshot.FallbackReason)
|
|
}
|
|
if snapshot.Raw.Available > 0 {
|
|
evt = evt.Uint64("rawAvailable", snapshot.Raw.Available)
|
|
}
|
|
if snapshot.Raw.Buffers > 0 {
|
|
evt = evt.Uint64("rawBuffers", snapshot.Raw.Buffers)
|
|
}
|
|
if snapshot.Raw.Cached > 0 {
|
|
evt = evt.Uint64("rawCached", snapshot.Raw.Cached)
|
|
}
|
|
if snapshot.Raw.TotalMinusUsed > 0 {
|
|
evt = evt.Uint64("rawTotalMinusUsed", snapshot.Raw.TotalMinusUsed)
|
|
}
|
|
if snapshot.Raw.RRDAvailable > 0 {
|
|
evt = evt.Uint64("rrdAvailable", snapshot.Raw.RRDAvailable)
|
|
}
|
|
if snapshot.Raw.RRDUsed > 0 {
|
|
evt = evt.Uint64("rrdUsed", snapshot.Raw.RRDUsed)
|
|
}
|
|
if snapshot.Raw.RRDTotal > 0 {
|
|
evt = evt.Uint64("rrdTotal", snapshot.Raw.RRDTotal)
|
|
}
|
|
if snapshot.Memory.Total > 0 {
|
|
evt = evt.Int64("total", snapshot.Memory.Total)
|
|
}
|
|
if snapshot.Memory.Used > 0 {
|
|
evt = evt.Int64("used", snapshot.Memory.Used)
|
|
}
|
|
if snapshot.Memory.Free > 0 {
|
|
evt = evt.Int64("free", snapshot.Memory.Free)
|
|
}
|
|
if snapshot.Memory.Usage > 0 {
|
|
evt = evt.Float64("usage", snapshot.Memory.Usage)
|
|
}
|
|
|
|
evt.Msg("node memory source updated")
|
|
}
|
|
|
|
func (m *Monitor) logGuestMemorySource(instance, guestType, node string, vmid int, snapshot GuestMemorySnapshot) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
key := makeGuestSnapshotKey(instance, guestType, node, vmid)
|
|
|
|
var prevSource string
|
|
m.diagMu.RLock()
|
|
if existing, ok := m.guestSnapshots[key]; ok {
|
|
prevSource = existing.MemorySource
|
|
}
|
|
m.diagMu.RUnlock()
|
|
|
|
if prevSource == snapshot.MemorySource {
|
|
return
|
|
}
|
|
|
|
source := snapshot.MemorySource
|
|
var evt *zerolog.Event
|
|
if MemorySourceIsFallback(source) {
|
|
evt = log.Warn()
|
|
} else {
|
|
evt = log.Debug()
|
|
}
|
|
|
|
evt = evt.
|
|
Str("instance", instance).
|
|
Str("guestType", guestType).
|
|
Str("node", node).
|
|
Int("vmid", vmid).
|
|
Str("name", snapshot.Name).
|
|
Str("status", snapshot.Status).
|
|
Str("memorySource", source)
|
|
|
|
if snapshot.FallbackReason != "" {
|
|
evt = evt.Str("fallbackReason", snapshot.FallbackReason)
|
|
}
|
|
if snapshot.Raw.ListingMem > 0 {
|
|
evt = evt.Uint64("listingMem", snapshot.Raw.ListingMem)
|
|
}
|
|
if snapshot.Raw.ListingMaxMem > 0 {
|
|
evt = evt.Uint64("listingMaxMem", snapshot.Raw.ListingMaxMem)
|
|
}
|
|
if snapshot.Raw.StatusMem > 0 {
|
|
evt = evt.Uint64("statusMem", snapshot.Raw.StatusMem)
|
|
}
|
|
if snapshot.Raw.StatusFreeMem > 0 {
|
|
evt = evt.Uint64("statusFreeMem", snapshot.Raw.StatusFreeMem)
|
|
}
|
|
if snapshot.Raw.StatusMaxMem > 0 {
|
|
evt = evt.Uint64("statusMaxMem", snapshot.Raw.StatusMaxMem)
|
|
}
|
|
if snapshot.Raw.MemInfoAvailable > 0 {
|
|
evt = evt.Uint64("memInfoAvailable", snapshot.Raw.MemInfoAvailable)
|
|
}
|
|
if snapshot.Raw.GuestAgentMemAvailable > 0 {
|
|
evt = evt.Uint64("guestAgentMemAvailable", snapshot.Raw.GuestAgentMemAvailable)
|
|
}
|
|
if snapshot.Raw.MemInfoBuffers > 0 {
|
|
evt = evt.Uint64("memInfoBuffers", snapshot.Raw.MemInfoBuffers)
|
|
}
|
|
if snapshot.Raw.MemInfoCached > 0 {
|
|
evt = evt.Uint64("memInfoCached", snapshot.Raw.MemInfoCached)
|
|
}
|
|
if snapshot.Raw.MemInfoTotalMinusUsed > 0 {
|
|
evt = evt.Uint64("memInfoTotalMinusUsed", snapshot.Raw.MemInfoTotalMinusUsed)
|
|
}
|
|
if snapshot.Raw.HostAgentTotal > 0 {
|
|
evt = evt.Uint64("hostAgentTotal", snapshot.Raw.HostAgentTotal)
|
|
}
|
|
if snapshot.Raw.HostAgentUsed > 0 {
|
|
evt = evt.Uint64("hostAgentUsed", snapshot.Raw.HostAgentUsed)
|
|
}
|
|
if snapshot.Memory.Total > 0 {
|
|
evt = evt.Int64("total", snapshot.Memory.Total)
|
|
}
|
|
if snapshot.Memory.Used > 0 {
|
|
evt = evt.Int64("used", snapshot.Memory.Used)
|
|
}
|
|
if snapshot.Memory.Free > 0 {
|
|
evt = evt.Int64("free", snapshot.Memory.Free)
|
|
}
|
|
if snapshot.Memory.Usage > 0 {
|
|
evt = evt.Float64("usage", snapshot.Memory.Usage)
|
|
}
|
|
|
|
evt.Msg("guest memory source updated")
|
|
}
|
|
|
|
func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemorySnapshot) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
snapshot = normalizeNodeMemorySnapshot(snapshot)
|
|
snapshot.Instance = instance
|
|
snapshot.Node = node
|
|
if snapshot.RetrievedAt.IsZero() {
|
|
snapshot.RetrievedAt = time.Now()
|
|
}
|
|
|
|
m.logNodeMemorySource(instance, node, snapshot)
|
|
|
|
m.diagMu.Lock()
|
|
defer m.diagMu.Unlock()
|
|
if m.nodeSnapshots == nil {
|
|
m.nodeSnapshots = make(map[string]NodeMemorySnapshot)
|
|
}
|
|
m.nodeSnapshots[makeNodeSnapshotKey(instance, node)] = snapshot
|
|
}
|
|
|
|
func (m *Monitor) recordGuestSnapshot(instance, guestType, node string, vmid int, snapshot GuestMemorySnapshot) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
snapshot = normalizeGuestMemorySnapshot(snapshot)
|
|
snapshot.Instance = instance
|
|
snapshot.GuestType = guestType
|
|
snapshot.Node = node
|
|
snapshot.VMID = vmid
|
|
if snapshot.RetrievedAt.IsZero() {
|
|
snapshot.RetrievedAt = time.Now()
|
|
}
|
|
|
|
m.logGuestMemorySource(instance, guestType, node, vmid, snapshot)
|
|
|
|
m.diagMu.Lock()
|
|
defer m.diagMu.Unlock()
|
|
if m.guestSnapshots == nil {
|
|
m.guestSnapshots = make(map[string]GuestMemorySnapshot)
|
|
}
|
|
m.guestSnapshots[makeGuestSnapshotKey(instance, guestType, node, vmid)] = snapshot
|
|
}
|
|
|
|
// GetDiagnosticSnapshots returns copies of the latest node and guest memory snapshots.
|
|
func (m *Monitor) GetDiagnosticSnapshots() DiagnosticSnapshotSet {
|
|
result := DiagnosticSnapshotSet{
|
|
Nodes: []NodeMemorySnapshot{},
|
|
Guests: []GuestMemorySnapshot{},
|
|
}
|
|
|
|
if m == nil {
|
|
return result
|
|
}
|
|
|
|
m.diagMu.RLock()
|
|
defer m.diagMu.RUnlock()
|
|
|
|
if len(m.nodeSnapshots) > 0 {
|
|
result.Nodes = make([]NodeMemorySnapshot, 0, len(m.nodeSnapshots))
|
|
for _, snapshot := range m.nodeSnapshots {
|
|
result.Nodes = append(result.Nodes, normalizeNodeMemorySnapshot(snapshot))
|
|
}
|
|
sort.Slice(result.Nodes, func(i, j int) bool {
|
|
if result.Nodes[i].Instance == result.Nodes[j].Instance {
|
|
return result.Nodes[i].Node < result.Nodes[j].Node
|
|
}
|
|
return result.Nodes[i].Instance < result.Nodes[j].Instance
|
|
})
|
|
}
|
|
|
|
if len(m.guestSnapshots) > 0 {
|
|
result.Guests = make([]GuestMemorySnapshot, 0, len(m.guestSnapshots))
|
|
for _, snapshot := range m.guestSnapshots {
|
|
result.Guests = append(result.Guests, normalizeGuestMemorySnapshot(snapshot))
|
|
}
|
|
sort.Slice(result.Guests, func(i, j int) bool {
|
|
if result.Guests[i].Instance == result.Guests[j].Instance {
|
|
if result.Guests[i].Node == result.Guests[j].Node {
|
|
if result.Guests[i].GuestType == result.Guests[j].GuestType {
|
|
return result.Guests[i].VMID < result.Guests[j].VMID
|
|
}
|
|
return result.Guests[i].GuestType < result.Guests[j].GuestType
|
|
}
|
|
return result.Guests[i].Node < result.Guests[j].Node
|
|
}
|
|
return result.Guests[i].Instance < result.Guests[j].Instance
|
|
})
|
|
}
|
|
|
|
return result
|
|
}
|