Pulse/internal/monitoring/mock_metrics_history.go
2026-04-01 01:04:06 +01:00

1883 lines
63 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package monitoring
import (
"context"
"fmt"
"hash/fnv"
"math"
"math/rand"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/mockmodel"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
"github.com/rcourtman/pulse-go-rewrite/internal/vmware"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
"github.com/rs/zerolog/log"
)
const (
defaultMockSeedDuration = 90 * 24 * time.Hour
defaultMockSampleInterval = 1 * time.Minute // 1m for detailed recent charts
)
type mockMetricsSamplerConfig struct {
SeedDuration time.Duration
SampleInterval time.Duration
}
func mockMetricsSamplerConfigFromEnv() mockMetricsSamplerConfig {
seedDuration := parseDurationEnv("PULSE_MOCK_TRENDS_SEED_DURATION", defaultMockSeedDuration)
sampleInterval := parseDurationEnv("PULSE_MOCK_TRENDS_SAMPLE_INTERVAL", defaultMockSampleInterval)
// Guardrails to keep memory and CPU bounded in demo mode.
if seedDuration < 5*time.Minute {
seedDuration = 5 * time.Minute
}
if seedDuration > 90*24*time.Hour {
seedDuration = 90 * 24 * time.Hour
}
if sampleInterval < 5*time.Second {
sampleInterval = 5 * time.Second
}
if sampleInterval > 5*time.Minute {
sampleInterval = 5 * time.Minute
}
// Ensure we can generate at least 2 points.
if seedDuration < sampleInterval {
seedDuration = sampleInterval
}
return mockMetricsSamplerConfig{
SeedDuration: seedDuration,
SampleInterval: sampleInterval,
}
}
// HashSeed produces a deterministic uint64 from the given parts.
func HashSeed(parts ...string) uint64 {
h := fnv.New64a()
for _, p := range parts {
_, _ = h.Write([]byte(p))
_, _ = h.Write([]byte{0})
}
return h.Sum64()
}
func kubernetesClusterMetricID(cluster models.KubernetesCluster) string {
if value := strings.TrimSpace(cluster.ID); value != "" {
return value
}
if value := strings.TrimSpace(cluster.Name); value != "" {
return value
}
if value := strings.TrimSpace(cluster.DisplayName); value != "" {
return value
}
return "k8s-cluster"
}
func kubernetesPodMetricID(cluster models.KubernetesCluster, pod models.KubernetesPod) string {
clusterKey := kubernetesClusterMetricID(cluster)
podKey := strings.TrimSpace(pod.UID)
if podKey == "" {
namespace := strings.TrimSpace(pod.Namespace)
name := strings.TrimSpace(pod.Name)
switch {
case namespace != "" && name != "":
podKey = namespace + "/" + name
case name != "":
podKey = name
default:
podKey = "pod"
}
}
if clusterKey == "" || podKey == "" {
return ""
}
return fmt.Sprintf("k8s:%s:pod:%s", clusterKey, podKey)
}
func kubernetesPodCurrentMetrics(cluster models.KubernetesCluster, pod models.KubernetesPod) map[string]float64 {
clusterKey := kubernetesClusterMetricID(cluster)
podKey := strings.TrimSpace(pod.UID)
if podKey == "" {
podKey = strings.TrimSpace(pod.Namespace) + "/" + strings.TrimSpace(pod.Name)
}
seed := HashSeed("k8s-pod-current", clusterKey, podKey)
rng := rand.New(rand.NewSource(int64(seed)))
phase := strings.ToLower(strings.TrimSpace(pod.Phase))
totalContainers := len(pod.Containers)
if totalContainers <= 0 {
totalContainers = 1
}
readyContainers := 0
for _, container := range pod.Containers {
if container.Ready {
readyContainers++
}
}
readiness := float64(readyContainers) / float64(totalContainers)
if readiness <= 0 && phase == "running" {
readiness = 0.35
}
restarts := float64(pod.Restarts)
if restarts < 0 {
restarts = 0
}
restartFactor := math.Min(restarts*1.6, 16)
cpu := clampFloat(pod.UsageCPUPercent, 0, 100)
memory := clampFloat(pod.UsageMemoryPercent, 0, 100)
disk := clampFloat(pod.DiskUsagePercent, 0, 100)
netIn := clampFloat(pod.NetInRate, 0, math.Max(pod.NetInRate, 0))
netOut := clampFloat(pod.NetOutRate, 0, math.Max(pod.NetOutRate, 0))
// When the live report has no usage sample yet, synthesize only the metrics
// that exist in real production Kubernetes collection paths.
if cpu <= 0 || memory <= 0 || disk <= 0 || netIn <= 0 || netOut <= 0 {
switch phase {
case "running":
if cpu <= 0 {
cpu = 7 + readiness*52 + rng.Float64()*14 - restartFactor*0.35
}
if memory <= 0 {
memory = 26 + readiness*46 + rng.Float64()*14 + restartFactor*0.25
}
if disk <= 0 {
disk = 17 + readiness*42 + rng.Float64()*16
}
if netIn <= 0 {
netIn = 14 + readiness*220 + rng.Float64()*70 + restarts*2
}
if netOut <= 0 {
netOut = 10 + readiness*180 + rng.Float64()*55 + restarts*1.6
}
case "pending":
if cpu <= 0 {
cpu = 2 + rng.Float64()*7
}
if memory <= 0 {
memory = 14 + rng.Float64()*16
}
if disk <= 0 {
disk = 8 + rng.Float64()*14
}
if netIn <= 0 {
netIn = 1 + rng.Float64()*10
}
if netOut <= 0 {
netOut = 1 + rng.Float64()*8
}
case "failed", "unknown":
if cpu <= 0 {
cpu = 1 + rng.Float64()*8
}
if memory <= 0 {
memory = 9 + rng.Float64()*15 + restartFactor*0.5
}
if disk <= 0 {
disk = 7 + rng.Float64()*16
}
if netIn <= 0 {
netIn = 1 + rng.Float64()*14 + restarts*1.4
}
if netOut <= 0 {
netOut = 1 + rng.Float64()*11 + restarts*1.2
}
default:
if cpu <= 0 {
cpu = 1 + rng.Float64()*6
}
if memory <= 0 {
memory = 8 + rng.Float64()*12
}
if disk <= 0 {
disk = 6 + rng.Float64()*12
}
if netIn <= 0 {
netIn = 1 + rng.Float64()*7
}
if netOut <= 0 {
netOut = 1 + rng.Float64()*6
}
}
if totalContainers > 1 {
multi := 1 + math.Min(float64(totalContainers-1)*0.08, 0.6)
if cpu > 0 {
cpu *= multi
}
if memory > 0 {
memory *= 1 + math.Min(float64(totalContainers-1)*0.1, 0.8)
}
if disk > 0 {
disk *= 1 + math.Min(float64(totalContainers-1)*0.07, 0.5)
}
if netIn > 0 {
netIn *= 1 + math.Min(float64(totalContainers-1)*0.09, 0.7)
}
if netOut > 0 {
netOut *= 1 + math.Min(float64(totalContainers-1)*0.08, 0.65)
}
}
}
return map[string]float64{
"cpu": clampFloat(cpu, 0, 100),
"memory": clampFloat(memory, 0, 100),
"disk": clampFloat(disk, 0, 100),
"netin": clampFloat(netIn, 0, math.Max(1800, netIn+50)),
"netout": clampFloat(netOut, 0, math.Max(1700, netOut+40)),
"diskread": 0,
"diskwrite": 0,
}
}
// SeriesStyle controls the visual character of generated metric series.
type SeriesStyle int
const (
// StyleSpiky: low baseline with sharp spike events (CPU, network/disk I/O).
StyleSpiky SeriesStyle = iota
// StylePlateau: stable level with slow drift between plateaus (memory).
StylePlateau
// StyleFlat: nearly constant with very gradual trend (disk usage, temperature).
StyleFlat
)
// Package-level aliases for internal callers that use the unexported names.
const (
styleSpiky = StyleSpiky
stylePlateau = StylePlateau
styleFlat = StyleFlat
)
// GenerateSeededSeries produces a deterministic metric series with the
// given style. Exported so the API chart layer can use the same generation.
func GenerateSeededSeries(current float64, points int, seed uint64, min, max float64, style SeriesStyle) []float64 {
current = clampFloat(current, min, max)
if points <= 1 {
return []float64{current}
}
rng := rand.New(rand.NewSource(int64(seed)))
span := math.Max(1, max-min)
var raw []float64
switch style {
case styleSpiky:
raw = generateSpikySeries(current, points, seed, min, max, span, rng)
case stylePlateau:
raw = generatePlateauSeries(current, points, min, max, span, rng)
default:
raw = generateFlatSeries(current, points, min, max, span, rng)
}
for i := range raw {
raw[i] = clampFloat(raw[i], min, max)
}
raw[points-1] = current
return raw
}
func GenerateSeededSeriesForTimestamps(
current float64,
timestamps []time.Time,
seed uint64,
min float64,
max float64,
style SeriesStyle,
) []float64 {
var mappedStyle mockmodel.SeriesStyle
switch style {
case stylePlateau:
mappedStyle = mockmodel.StylePlateau
case styleFlat:
mappedStyle = mockmodel.StyleFlat
default:
mappedStyle = mockmodel.StyleSpiky
}
return mockmodel.SeriesForTimestamps(current, timestamps, seed, min, max, mappedStyle)
}
func GenerateSeededMetricSeriesForTimestamps(
current float64,
timestamps []time.Time,
seed uint64,
min float64,
max float64,
metricType string,
style SeriesStyle,
) []float64 {
var mappedStyle mockmodel.SeriesStyle
switch style {
case stylePlateau:
mappedStyle = mockmodel.StylePlateau
case styleFlat:
mappedStyle = mockmodel.StyleFlat
default:
mappedStyle = mockmodel.StyleSpiky
}
return mockmodel.SeriesForMetricTimestamps(current, timestamps, seed, min, max, metricType, mappedStyle)
}
// generateSpikySeries produces a low baseline with occasional sharp spikes —
// matching how real CPU and I/O metrics behave (mostly idle, with bursts).
func generateSpikySeries(current float64, points int, seed uint64, min, max, span float64, rng *rand.Rand) []float64 {
// Baseline: most resources idle low. Seed determines personality tier.
personality := seed % 20
r := rng.Float64()
var baselineFraction float64
switch {
case personality == 0: // 5%: busy resource
baselineFraction = 0.30 + r*0.15
case personality <= 4: // 20%: moderate
baselineFraction = 0.16 + r*0.14
default: // 75%: idle
baselineFraction = 0.04 + r*0.12
}
baseline := min + span*baselineFraction
// Small drift over the series to avoid perfect flatness.
baselineDrift := span * 0.03 * (rng.Float64()*2 - 1)
raw := make([]float64, points)
for i := range raw {
progress := float64(i) / float64(points-1)
raw[i] = baseline + baselineDrift*progress + rng.NormFloat64()*span*0.004
}
// Generate spike events: fast rise, power-law decay.
spikeMinWidth := 3 + int(seed%6)
spikeProb := 0.015 + float64(seed%10)*0.003
for i := 0; i < points-spikeMinWidth; {
if rng.Float64() < spikeProb {
height := span * (0.10 + rng.Float64()*0.45)
width := spikeMinWidth + rng.Intn(10)
if i+width > points {
width = points - i
}
for j := 0; j < width; j++ {
progress := float64(j) / float64(width)
var envelope float64
if progress < 0.12 {
envelope = progress / 0.12
} else {
decay := (progress - 0.12) / 0.88
envelope = math.Pow(1-decay, 1.5)
}
raw[i+j] = math.Max(raw[i+j], baseline+height*envelope)
}
i += width + 2 + rng.Intn(5) // gap after spike
} else {
i++
}
}
// Taper-shift so the last point approaches current without distorting spikes.
diff := current - raw[points-1]
taperLen := points / 8
if taperLen < 5 {
taperLen = 5
}
if taperLen > points {
taperLen = points
}
for j := 0; j < taperLen; j++ {
idx := points - 1 - j
weight := float64(taperLen-j) / float64(taperLen)
raw[idx] += diff * weight
}
return raw
}
// generatePlateauSeries produces stable levels with slow transitions —
// matching how real memory usage behaves (applications hold allocations).
func generatePlateauSeries(current float64, points int, min, max, span float64, rng *rand.Rand) []float64 {
plateauCount := 3 + rng.Intn(4) // 3-6 plateaus
// Generate plateau levels near current.
levels := make([]float64, plateauCount)
for i := range levels {
offset := (rng.Float64()*2 - 1) * span * 0.12
levels[i] = clampFloat(current+offset, min, max)
}
levels[plateauCount-1] = current + rng.NormFloat64()*span*0.01
raw := make([]float64, points)
segmentLen := points / plateauCount
for i := 0; i < points; i++ {
seg := i / segmentLen
if seg >= plateauCount {
seg = plateauCount - 1
}
nextSeg := seg + 1
if nextSeg >= plateauCount {
nextSeg = plateauCount - 1
}
segEnd := segmentLen
if seg == plateauCount-1 {
segEnd = points - seg*segmentLen
}
if segEnd <= 0 {
segEnd = 1
}
posInSeg := i - seg*segmentLen
progress := float64(posInSeg) / float64(segEnd)
// Smooth transition in last 20% of each segment.
var value float64
if progress < 0.8 || seg == nextSeg {
value = levels[seg]
} else {
t := (progress - 0.8) / 0.2
t = t * t * (3 - 2*t) // smoothstep
value = levels[seg]*(1-t) + levels[nextSeg]*t
}
raw[i] = value + rng.NormFloat64()*span*0.002
}
return raw
}
// generateFlatSeries produces a nearly constant series with very gradual
// trend — matching disk usage or temperature that barely changes.
func generateFlatSeries(current float64, points int, min, max, span float64, rng *rand.Rand) []float64 {
raw := make([]float64, points)
trendDir := 1.0
if rng.Float64() < 0.33 {
trendDir = -1.0
}
totalDrift := span * 0.02 * trendDir
for i := range raw {
progress := float64(i) / float64(points-1)
raw[i] = current - totalDrift*(1-progress) + rng.NormFloat64()*span*0.001
}
return raw
}
// buildTieredTimestamps generates a sorted list of timestamps with denser
// intervals for recent data:
//
// Last 2h: 1min intervals (~120 points)
// 2h–24h: 2min intervals (~660 points)
// 24h–end: ~65min intervals (variable)
//
// This ensures short time ranges (1h, 4h) have enough data points without
// needing an API-level fallback layer.
func buildTieredTimestamps(now time.Time, totalDuration time.Duration) []time.Time {
// Each segment covers [now - startOffset, now - endOffset) and is walked
// chronologically from oldest to newest. Segments are defined oldest-first
// so the resulting slice is in chronological order.
type segment struct {
startOffset time.Duration // how far back from now this segment starts
endOffset time.Duration // how far back from now this segment ends
interval time.Duration
}
segments := []segment{
{totalDuration, 24 * time.Hour, 65 * time.Minute},
{24 * time.Hour, 2 * time.Hour, 2 * time.Minute},
{2 * time.Hour, 0, time.Minute},
}
var timestamps []time.Time
for _, seg := range segments {
startOff := seg.startOffset
if startOff > totalDuration {
startOff = totalDuration
}
endOff := seg.endOffset
if endOff > totalDuration {
endOff = totalDuration
}
if startOff <= endOff {
continue
}
segStart := now.Add(-startOff)
segEnd := now.Add(-endOff)
for ts := segStart; ts.Before(segEnd); ts = ts.Add(seg.interval) {
timestamps = append(timestamps, ts)
}
}
return timestamps
}
func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.FixtureGraph, now time.Time, seedDuration, interval time.Duration) {
if mh == nil {
return
}
state := graph.State
if seedDuration <= 0 || interval <= 0 {
return
}
// Build a tiered timestamp list so short time ranges (1h, 4h) have dense
// data without needing an API-level fallback layer.
// Last 2h: 1min intervals (~120 points)
// 2h–24h: 2min intervals (~660 points)
// 24h–90d: ~65min intervals (~1920 points)
// The current "now" point is appended explicitly by each recorder so the
// seed and live sampler share one canonical terminal timestamp.
seedTimestamps := buildTieredTimestamps(now, seedDuration)
const seedBatchSize = 5000
numPoints := len(seedTimestamps)
generateSeries := func(metricType string, current float64, seed uint64, min, max float64, style SeriesStyle) []float64 {
return GenerateSeededMetricSeriesForTimestamps(current, seedTimestamps, seed, min, max, metricType, style)
}
var seedBatch []metrics.WriteMetric
queueMetric := func(resourceType, resourceID, metricType string, value float64, ts time.Time) {
if ms == nil {
return
}
seedBatch = append(seedBatch,
metrics.WriteMetric{
ResourceType: resourceType,
ResourceID: resourceID,
MetricType: metricType,
Value: value,
Timestamp: ts,
Tier: metrics.TierHourly,
},
metrics.WriteMetric{
ResourceType: resourceType,
ResourceID: resourceID,
MetricType: metricType,
Value: value,
Timestamp: ts,
Tier: metrics.TierDaily,
},
)
if len(seedBatch) >= seedBatchSize {
ms.WriteBatchSync(seedBatch)
seedBatch = seedBatch[:0]
}
}
recordStorageTimeline := func(storageID string, currentUsed, currentTotal float64) {
if strings.TrimSpace(storageID) == "" || currentTotal <= 0 || numPoints == 0 {
return
}
series := mockmodel.StorageCapacitySeriesForTimestamps(
currentUsed,
currentTotal,
seedTimestamps,
mock.MetricSeed("storage", storageID, "usage"),
)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddStorageMetric(storageID, "usage", series.Usage[i], ts)
mh.AddStorageMetric(storageID, "used", series.Used[i], ts)
mh.AddStorageMetric(storageID, "avail", series.Avail[i], ts)
mh.AddStorageMetric(storageID, "total", series.Total[i], ts)
queueMetric("storage", storageID, "usage", series.Usage[i], ts)
queueMetric("storage", storageID, "used", series.Used[i], ts)
queueMetric("storage", storageID, "avail", series.Avail[i], ts)
queueMetric("storage", storageID, "total", series.Total[i], ts)
}
last := numPoints - 1
mh.AddStorageMetric(storageID, "usage", series.Usage[last], now)
mh.AddStorageMetric(storageID, "used", series.Used[last], now)
mh.AddStorageMetric(storageID, "avail", series.Avail[last], now)
mh.AddStorageMetric(storageID, "total", series.Total[last], now)
queueMetric("storage", storageID, "usage", series.Usage[last], now)
queueMetric("storage", storageID, "used", series.Used[last], now)
queueMetric("storage", storageID, "avail", series.Avail[last], now)
queueMetric("storage", storageID, "total", series.Total[last], now)
}
recordNode := func(node models.Node) {
if node.ID == "" {
return
}
cpuMin, cpuMax := mock.MetricBounds("node", "cpu")
memMin, memMax := mock.MetricBounds("node", "memory")
diskMin, diskMax := mock.MetricBounds("node", "disk")
cpuSeries := generateSeries("cpu", node.CPU*100, mock.MetricSeed("node", node.ID, "cpu"), cpuMin, cpuMax, styleSpiky)
memSeries := generateSeries("memory", node.Memory.Usage, mock.MetricSeed("node", node.ID, "memory"), memMin, memMax, stylePlateau)
diskSeries := generateSeries("disk", node.Disk.Usage, mock.MetricSeed("node", node.ID, "disk"), diskMin, diskMax, styleFlat)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddNodeMetric(node.ID, "cpu", cpuSeries[i], ts)
mh.AddNodeMetric(node.ID, "memory", memSeries[i], ts)
mh.AddNodeMetric(node.ID, "disk", diskSeries[i], ts)
queueMetric("node", node.ID, "cpu", cpuSeries[i], ts)
queueMetric("node", node.ID, "memory", memSeries[i], ts)
queueMetric("node", node.ID, "disk", diskSeries[i], ts)
}
// Ensure the latest point lands at "now" for full-range charts.
mh.AddNodeMetric(node.ID, "cpu", node.CPU*100, now)
mh.AddNodeMetric(node.ID, "memory", node.Memory.Usage, now)
mh.AddNodeMetric(node.ID, "disk", node.Disk.Usage, now)
queueMetric("node", node.ID, "cpu", node.CPU*100, now)
queueMetric("node", node.ID, "memory", node.Memory.Usage, now)
queueMetric("node", node.ID, "disk", node.Disk.Usage, now)
}
recordGuest := func(
metricIDs []string, storeType, storeID string,
cpuPercent, memPercent, diskPercent, diskRead, diskWrite, netIn, netOut float64,
includeDisk bool,
includeDiskIO bool,
includeNetwork bool,
) {
if len(metricIDs) == 0 || storeID == "" {
return
}
uniqueMetricIDs := make([]string, 0, len(metricIDs))
seenMetricIDs := make(map[string]struct{}, len(metricIDs))
for _, rawID := range metricIDs {
metricID := strings.TrimSpace(rawID)
if metricID == "" {
continue
}
if _, exists := seenMetricIDs[metricID]; exists {
continue
}
seenMetricIDs[metricID] = struct{}{}
uniqueMetricIDs = append(uniqueMetricIDs, metricID)
}
if len(uniqueMetricIDs) == 0 {
return
}
cpuMin, cpuMax := mock.MetricBounds(storeType, "cpu")
memMin, memMax := mock.MetricBounds(storeType, "memory")
diskMin, diskMax := mock.MetricBounds(storeType, "disk")
readMin, readMax := mock.MetricBounds(storeType, "diskread")
writeMin, writeMax := mock.MetricBounds(storeType, "diskwrite")
netInMin, netInMax := mock.MetricBounds(storeType, "netin")
netOutMin, netOutMax := mock.MetricBounds(storeType, "netout")
cpuSeries := generateSeries("cpu", cpuPercent, mock.MetricSeed(storeType, storeID, "cpu"), cpuMin, cpuMax, styleSpiky)
memSeries := generateSeries("memory", memPercent, mock.MetricSeed(storeType, storeID, "memory"), memMin, memMax, stylePlateau)
var diskSeries []float64
if includeDisk {
diskSeries = generateSeries("disk", diskPercent, mock.MetricSeed(storeType, storeID, "disk"), diskMin, diskMax, styleFlat)
}
var diskReadSeries, diskWriteSeries, netInSeries, netOutSeries []float64
if includeDiskIO {
diskReadSeries = generateSeries("diskread", diskRead, mock.MetricSeed(storeType, storeID, "diskread"), readMin, readMax, styleSpiky)
diskWriteSeries = generateSeries("diskwrite", diskWrite, mock.MetricSeed(storeType, storeID, "diskwrite"), writeMin, writeMax, styleSpiky)
}
if includeNetwork {
netInSeries = generateSeries("netin", netIn, mock.MetricSeed(storeType, storeID, "netin"), netInMin, netInMax, styleSpiky)
netOutSeries = generateSeries("netout", netOut, mock.MetricSeed(storeType, storeID, "netout"), netOutMin, netOutMax, styleSpiky)
}
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "cpu", cpuSeries[i], ts)
mh.AddGuestMetric(metricID, "memory", memSeries[i], ts)
}
queueMetric(storeType, storeID, "cpu", cpuSeries[i], ts)
queueMetric(storeType, storeID, "memory", memSeries[i], ts)
if includeDisk {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "disk", diskSeries[i], ts)
}
queueMetric(storeType, storeID, "disk", diskSeries[i], ts)
}
if includeDiskIO {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "diskread", diskReadSeries[i], ts)
mh.AddGuestMetric(metricID, "diskwrite", diskWriteSeries[i], ts)
}
queueMetric(storeType, storeID, "diskread", diskReadSeries[i], ts)
queueMetric(storeType, storeID, "diskwrite", diskWriteSeries[i], ts)
}
if includeNetwork {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "netin", netInSeries[i], ts)
mh.AddGuestMetric(metricID, "netout", netOutSeries[i], ts)
}
queueMetric(storeType, storeID, "netin", netInSeries[i], ts)
queueMetric(storeType, storeID, "netout", netOutSeries[i], ts)
}
}
// Ensure the latest point lands at "now" for full-range charts.
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "cpu", cpuPercent, now)
mh.AddGuestMetric(metricID, "memory", memPercent, now)
}
queueMetric(storeType, storeID, "cpu", cpuPercent, now)
queueMetric(storeType, storeID, "memory", memPercent, now)
if includeDisk {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "disk", diskPercent, now)
}
queueMetric(storeType, storeID, "disk", diskPercent, now)
}
if includeDiskIO {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "diskread", diskRead, now)
mh.AddGuestMetric(metricID, "diskwrite", diskWrite, now)
}
queueMetric(storeType, storeID, "diskread", diskRead, now)
queueMetric(storeType, storeID, "diskwrite", diskWrite, now)
}
if includeNetwork {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "netin", netIn, now)
mh.AddGuestMetric(metricID, "netout", netOut, now)
}
queueMetric(storeType, storeID, "netin", netIn, now)
queueMetric(storeType, storeID, "netout", netOut, now)
}
}
log.Debug().Int("count", len(state.Nodes)).Msg("mock seeding: processing nodes")
for _, node := range state.Nodes {
recordNode(node)
time.Sleep(50 * time.Millisecond) // Reduced from 200ms for faster startup
}
runningVMs := 0
for _, vm := range state.VMs {
if vm.Status == "running" {
runningVMs++
}
}
log.Debug().Int("total", len(state.VMs)).Int("running", runningVMs).Msg("mock seeding: processing VMs (including stopped)")
for _, vm := range state.VMs {
cpuPct := vm.CPU * 100
memPct := vm.Memory.Usage
diskPct := vm.Disk.Usage
diskR := float64(vm.DiskRead)
diskW := float64(vm.DiskWrite)
netI := float64(vm.NetworkIn)
netO := float64(vm.NetworkOut)
// Stopped VMs have 0 metrics; synthesize "was running" baselines so
// their charts show historical activity before the resource stopped.
if vm.Status != "running" {
h := HashSeed("vm", vm.ID, "baseline")
cpuPct = 10 + float64(h%30) // 10-39 %
memPct = 30 + float64((h>>8)%40) // 30-69 %
diskPct = 15 + float64((h>>16)%35) // 15-49 %
diskR = float64(1000 + h%5000)
diskW = float64(500 + (h>>8)%3000)
netI = float64(2000 + (h>>16)%8000)
netO = float64(1000 + (h>>24)%4000)
}
recordGuest([]string{vm.ID}, "vm", vm.ID, cpuPct, memPct, diskPct, diskR, diskW, netI, netO, true, true, true)
time.Sleep(50 * time.Millisecond) // Reduced from 200ms for faster startup
}
runningContainers := 0
for _, ct := range state.Containers {
if ct.Status == "running" {
runningContainers++
}
}
log.Debug().Int("total", len(state.Containers)).Int("running", runningContainers).Msg("mock seeding: processing containers (including stopped)")
for _, ct := range state.Containers {
cpuPct := ct.CPU * 100
memPct := ct.Memory.Usage
diskPct := ct.Disk.Usage
diskR := float64(ct.DiskRead)
diskW := float64(ct.DiskWrite)
netI := float64(ct.NetworkIn)
netO := float64(ct.NetworkOut)
if ct.Status != "running" {
h := HashSeed("container", ct.ID, "baseline")
cpuPct = 10 + float64(h%30)
memPct = 30 + float64((h>>8)%40)
diskPct = 15 + float64((h>>16)%35)
diskR = float64(1000 + h%5000)
diskW = float64(500 + (h>>8)%3000)
netI = float64(2000 + (h>>16)%8000)
netO = float64(1000 + (h>>24)%4000)
}
recordGuest([]string{ct.ID}, "container", ct.ID, cpuPct, memPct, diskPct, diskR, diskW, netI, netO, true, true, true)
time.Sleep(50 * time.Millisecond) // Reduced from 200ms for faster startup
}
k8sPodCount := 0
for _, cluster := range state.KubernetesClusters {
if cluster.Hidden {
continue
}
k8sPodCount += len(cluster.Pods)
}
log.Debug().Int("clusters", len(state.KubernetesClusters)).Int("pods", k8sPodCount).Msg("mock seeding: processing kubernetes pods")
for _, cluster := range state.KubernetesClusters {
if cluster.Hidden {
continue
}
for _, pod := range cluster.Pods {
metricID := kubernetesPodMetricID(cluster, pod)
if metricID == "" {
continue
}
current := kubernetesPodCurrentMetrics(cluster, pod)
recordGuest(
[]string{metricID},
"k8s",
metricID,
current["cpu"],
current["memory"],
current["disk"],
current["diskread"],
current["diskwrite"],
current["netin"],
current["netout"],
true,
false,
true,
)
}
time.Sleep(30 * time.Millisecond)
}
log.Debug().Int("count", len(state.Storage)).Msg("mock seeding: processing storage")
for _, storage := range state.Storage {
if storage.ID == "" {
continue
}
_, used, total, _ := normalizedStorageCapacityMetrics(storage.Total, storage.Used, storage.Free, storage.Usage)
recordStorageTimeline(storage.ID, used, total)
time.Sleep(50 * time.Millisecond) // Reduced from 200ms for faster startup
}
log.Debug().Int("count", len(state.PhysicalDisks)).Msg("mock seeding: processing physical disks")
for _, disk := range state.PhysicalDisks {
if disk.Temperature <= 0 {
continue
}
resourceID := diskMetricsResourceID(disk)
if resourceID == "" {
continue
}
tempMin, tempMax := mock.MetricBounds("disk", "smart_temp")
tempSeries := generateSeries("smart_temp", float64(disk.Temperature), mock.MetricSeed("disk", resourceID, "smart_temp"), tempMin, tempMax, styleFlat)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddDiskMetric(resourceID, "smart_temp", tempSeries[i], ts)
queueMetric("disk", resourceID, "smart_temp", tempSeries[i], ts)
}
// Ensure the latest point lands at "now" for full-range charts.
mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now)
queueMetric("disk", resourceID, "smart_temp", float64(disk.Temperature), now)
}
log.Debug().Int("count", len(state.CephClusters)).Msg("mock seeding: processing ceph clusters")
for _, cluster := range state.CephClusters {
if cluster.TotalBytes <= 0 {
continue
}
cephID := cluster.FSID
if cephID == "" {
cephID = cluster.ID
}
if cephID == "" {
continue
}
usageSeries := generateSeries("usage", cluster.UsagePercent, mock.MetricSeed("ceph", cephID, "usage"), 0, 100, styleFlat)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
queueMetric("ceph", cephID, "usage", usageSeries[i], ts)
}
// Ensure the latest point lands at "now" for full-range charts.
queueMetric("ceph", cephID, "usage", cluster.UsagePercent, now)
}
log.Debug().Int("count", len(state.DockerHosts)).Msg("mock seeding: processing docker hosts")
for _, host := range state.DockerHosts {
if host.ID == "" {
continue
}
var diskPercent float64
var usedTotal, totalTotal int64
for _, d := range host.Disks {
if d.Total > 0 {
usedTotal += d.Used
totalTotal += d.Total
}
}
if totalTotal > 0 {
diskPercent = float64(usedTotal) / float64(totalTotal) * 100
}
recordGuest([]string{"dockerHost:" + host.ID}, "dockerHost", host.ID, host.CPUUsage, host.Memory.Usage, diskPercent, 0, 0, 0, 0, true, false, false)
for _, container := range host.Containers {
if container.ID == "" {
continue
}
cpuPct := container.CPUPercent
memPct := container.MemoryPercent
var diskPct float64
if container.RootFilesystemBytes > 0 && container.WritableLayerBytes > 0 {
diskPct = float64(container.WritableLayerBytes) / float64(container.RootFilesystemBytes) * 100
diskPct = clampFloat(diskPct, 0, 100)
}
if container.State != "running" {
h := HashSeed("dockerContainer", container.ID, "baseline")
cpuPct = 5 + float64(h%25) // 5-29 %
memPct = 10 + float64((h>>8)%40) // 10-49 %
diskPct = 5 + float64((h>>16)%30)
}
recordGuest([]string{"docker:" + container.ID}, "dockerContainer", container.ID, cpuPct, memPct, diskPct, 0, 0, 0, 0, true, false, false)
}
time.Sleep(50 * time.Millisecond) // Add delay for docker hosts
}
log.Debug().Int("count", len(state.Hosts)).Msg("mock seeding: processing host agents")
for _, host := range state.Hosts {
if host.ID == "" || host.Status != "online" {
continue
}
var diskPercent float64
if len(host.Disks) > 0 {
diskPercent = host.Disks[0].Usage
}
recordGuest([]string{"agent:" + host.ID}, "agent", host.ID, host.CPUUsage, host.Memory.Usage, diskPercent, host.DiskReadRate, host.DiskWriteRate, host.NetInRate, host.NetOutRate, true, true, true)
time.Sleep(50 * time.Millisecond)
}
platformFixtures := graph.PlatformFixtures
trueNASFixtures := platformFixtures.TrueNAS
log.Debug().Int("pools", len(trueNASFixtures.Pools)).Int("datasets", len(trueNASFixtures.Datasets)).Msg("mock seeding: processing TrueNAS fixtures")
// System host: aggregated disk usage across all pools
totalCap, totalUsed := int64(0), int64(0)
for _, pool := range trueNASFixtures.Pools {
totalCap += pool.TotalBytes
totalUsed += pool.UsedBytes
}
systemDisk := float64(0)
if totalCap > 0 {
systemDisk = float64(totalUsed) / float64(totalCap) * 100
}
systemMemoryPercent := float64(0)
if trueNASFixtures.System.MemoryTotalBytes > 0 {
usedMemory := trueNASFixtures.System.MemoryTotalBytes - trueNASFixtures.System.MemoryAvailableBytes
if usedMemory < 0 {
usedMemory = 0
}
systemMemoryPercent = (float64(usedMemory) / float64(trueNASFixtures.System.MemoryTotalBytes)) * 100
}
systemMetricIDs := []string{
"system:" + trueNASFixtures.System.Hostname,
"agent:" + trueNASFixtures.System.Hostname,
}
recordGuest(
systemMetricIDs,
"agent",
trueNASFixtures.System.Hostname,
trueNASFixtures.System.CPUPercent,
systemMemoryPercent,
systemDisk,
trueNASFixtures.System.DiskReadRate,
trueNASFixtures.System.DiskWriteRate,
trueNASFixtures.System.NetInRate,
trueNASFixtures.System.NetOutRate,
true,
true,
true,
)
for _, pool := range trueNASFixtures.Pools {
poolKey := "pool:" + pool.Name
diskPercent := float64(0)
if pool.TotalBytes > 0 {
diskPercent = float64(pool.UsedBytes) / float64(pool.TotalBytes) * 100
}
diskSeries := generateSeries("usage", diskPercent, mock.MetricSeed("pool", pool.Name, "disk"), 0, 100, styleFlat)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddGuestMetric(poolKey, "disk", diskSeries[i], ts)
queueMetric("pool", pool.Name, "disk", diskSeries[i], ts)
}
mh.AddGuestMetric(poolKey, "disk", diskPercent, now)
queueMetric("pool", pool.Name, "disk", diskPercent, now)
recordStorageTimeline(poolKey, float64(pool.UsedBytes), float64(pool.TotalBytes))
}
for _, dataset := range trueNASFixtures.Datasets {
dsKey := "dataset:" + dataset.Name
totalBytes := dataset.UsedBytes + dataset.AvailBytes
diskPercent := float64(0)
if totalBytes > 0 {
diskPercent = float64(dataset.UsedBytes) / float64(totalBytes) * 100
}
diskSeries := generateSeries("usage", diskPercent, mock.MetricSeed("dataset", dataset.Name, "disk"), 0, 100, styleFlat)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddGuestMetric(dsKey, "disk", diskSeries[i], ts)
queueMetric("dataset", dataset.Name, "disk", diskSeries[i], ts)
}
mh.AddGuestMetric(dsKey, "disk", diskPercent, now)
queueMetric("dataset", dataset.Name, "disk", diskPercent, now)
recordStorageTimeline(dsKey, float64(dataset.UsedBytes), float64(totalBytes))
}
for _, disk := range trueNASFixtures.Disks {
if disk.Temperature <= 0 {
continue
}
resourceID := trueNASDiskMetricsResourceID(disk)
if resourceID == "" {
continue
}
tempSeries := generateSeries("smart_temp", float64(disk.Temperature), mock.MetricSeed("disk", resourceID, "smart_temp"), 25, 95, styleFlat)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddDiskMetric(resourceID, "smart_temp", tempSeries[i], ts)
queueMetric("disk", resourceID, "smart_temp", tempSeries[i], ts)
}
mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now)
queueMetric("disk", resourceID, "smart_temp", float64(disk.Temperature), now)
}
for _, app := range trueNASFixtures.Apps {
if app.Stats == nil {
continue
}
appID := strings.TrimSpace(app.ID)
if appID == "" {
appID = strings.TrimSpace(app.Name)
}
if appID == "" {
continue
}
memPercent := float64(0)
if trueNASFixtures.System.MemoryTotalBytes > 0 {
memPercent = (float64(app.Stats.MemoryBytes) / float64(trueNASFixtures.System.MemoryTotalBytes)) * 100
}
diskPercent := 8 + float64(HashSeed("truenas-app", appID, "disk")%35)
if strings.EqualFold(strings.TrimSpace(app.State), "stopped") {
diskPercent = clampFloat(diskPercent*0.6, 0, 100)
}
recordGuest(
[]string{"docker:" + appID},
"dockerContainer",
appID,
app.Stats.CPUPercent,
memPercent,
diskPercent,
app.Stats.DiskReadRate,
app.Stats.DiskWriteRate,
app.Stats.NetInRate,
app.Stats.NetOutRate,
true,
true,
true,
)
}
vmwareFixtures := platformFixtures.VMware
vmwareDatastoreUsage := vmwareDatastoreUsageByID(vmwareFixtures.Datastores)
log.Debug().
Int("hosts", len(vmwareFixtures.Hosts)).
Int("vms", len(vmwareFixtures.VMs)).
Int("datastores", len(vmwareFixtures.Datastores)).
Msg("mock seeding: processing VMware fixtures")
for _, host := range vmwareFixtures.Hosts {
sourceID := vmware.SourceID(vmwareFixtures.ConnectionID, "host", host.Host)
if sourceID == "" {
continue
}
recordGuest(
[]string{"agent:" + sourceID},
"agent",
sourceID,
vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.CPUPercent }),
vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.MemoryPercent }),
vmwareAverageDatastoreUsage(vmwareDatastoreUsage, host.DatastoreIDs),
vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskReadBytesPerSecond }),
vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskWriteBytesPerSecond }),
vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetInBytesPerSecond }),
vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetOutBytesPerSecond }),
true,
true,
true,
)
}
for _, guest := range vmwareFixtures.VMs {
sourceID := vmware.SourceID(vmwareFixtures.ConnectionID, "vm", guest.VM)
if sourceID == "" {
continue
}
recordGuest(
[]string{sourceID},
"vm",
sourceID,
vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.CPUPercent }),
vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.MemoryPercent }),
vmwareAverageDatastoreUsage(vmwareDatastoreUsage, guest.DatastoreIDs),
vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskReadBytesPerSecond }),
vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskWriteBytesPerSecond }),
vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetInBytesPerSecond }),
vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetOutBytesPerSecond }),
true,
true,
true,
)
}
for _, datastore := range vmwareFixtures.Datastores {
usage := vmwareDatastoreUsagePercent(datastore)
total := datastore.Capacity
avail := datastore.FreeSpace
used := total - avail
if used < 0 {
used = 0
}
if usage <= 0 && total <= 0 {
continue
}
sourceID := vmware.SourceID(vmwareFixtures.ConnectionID, "datastore", datastore.Datastore)
if sourceID == "" {
continue
}
recordStorageTimeline(sourceID, float64(used), float64(total))
}
if ms != nil && len(seedBatch) > 0 {
ms.WriteBatchSync(seedBatch)
}
log.Debug().Msg("mock seeding: completed")
}
// recordTrueNASFixturesMetrics records live fixture ticks for TrueNAS host,
// storage, and app-container metrics.
func recordTrueNASFixturesMetrics(mh *MetricsHistory, ms *metrics.Store, fixtures mock.PlatformFixtures, ts time.Time) {
snapshot := fixtures.TrueNAS
totalCap, totalUsed := int64(0), int64(0)
for _, pool := range snapshot.Pools {
totalCap += pool.TotalBytes
totalUsed += pool.UsedBytes
}
if totalCap > 0 {
systemDisk := float64(totalUsed) / float64(totalCap) * 100
systemMetricIDs := []string{
"system:" + snapshot.System.Hostname,
"agent:" + snapshot.System.Hostname,
}
for _, metricID := range systemMetricIDs {
mh.AddGuestMetric(metricID, "cpu", snapshot.System.CPUPercent, ts)
if snapshot.System.MemoryTotalBytes > 0 {
usedMemory := snapshot.System.MemoryTotalBytes - snapshot.System.MemoryAvailableBytes
if usedMemory < 0 {
usedMemory = 0
}
mh.AddGuestMetric(metricID, "memory", (float64(usedMemory)/float64(snapshot.System.MemoryTotalBytes))*100, ts)
}
mh.AddGuestMetric(metricID, "disk", systemDisk, ts)
mh.AddGuestMetric(metricID, "diskread", snapshot.System.DiskReadRate, ts)
mh.AddGuestMetric(metricID, "diskwrite", snapshot.System.DiskWriteRate, ts)
mh.AddGuestMetric(metricID, "netin", snapshot.System.NetInRate, ts)
mh.AddGuestMetric(metricID, "netout", snapshot.System.NetOutRate, ts)
}
if ms != nil {
var systemMemoryPercent float64
if snapshot.System.MemoryTotalBytes > 0 {
usedMemory := snapshot.System.MemoryTotalBytes - snapshot.System.MemoryAvailableBytes
if usedMemory < 0 {
usedMemory = 0
}
systemMemoryPercent = (float64(usedMemory) / float64(snapshot.System.MemoryTotalBytes)) * 100
}
ms.Write("agent", snapshot.System.Hostname, "cpu", snapshot.System.CPUPercent, ts)
ms.Write("agent", snapshot.System.Hostname, "memory", systemMemoryPercent, ts)
ms.Write("agent", snapshot.System.Hostname, "disk", systemDisk, ts)
ms.Write("agent", snapshot.System.Hostname, "diskread", snapshot.System.DiskReadRate, ts)
ms.Write("agent", snapshot.System.Hostname, "diskwrite", snapshot.System.DiskWriteRate, ts)
ms.Write("agent", snapshot.System.Hostname, "netin", snapshot.System.NetInRate, ts)
ms.Write("agent", snapshot.System.Hostname, "netout", snapshot.System.NetOutRate, ts)
}
}
for _, pool := range snapshot.Pools {
if pool.TotalBytes > 0 {
poolKey := "pool:" + pool.Name
diskPct := float64(pool.UsedBytes) / float64(pool.TotalBytes) * 100
mh.AddGuestMetric(poolKey, "disk", diskPct, ts)
mh.AddStorageMetric(poolKey, "usage", diskPct, ts)
mh.AddStorageMetric(poolKey, "used", float64(pool.UsedBytes), ts)
mh.AddStorageMetric(poolKey, "avail", float64(pool.FreeBytes), ts)
mh.AddStorageMetric(poolKey, "total", float64(pool.TotalBytes), ts)
if ms != nil {
ms.Write("pool", pool.Name, "disk", diskPct, ts)
ms.Write("storage", poolKey, "usage", diskPct, ts)
ms.Write("storage", poolKey, "used", float64(pool.UsedBytes), ts)
ms.Write("storage", poolKey, "avail", float64(pool.FreeBytes), ts)
ms.Write("storage", poolKey, "total", float64(pool.TotalBytes), ts)
}
}
}
for _, dataset := range snapshot.Datasets {
totalBytes := dataset.UsedBytes + dataset.AvailBytes
if totalBytes > 0 {
dsKey := "dataset:" + dataset.Name
diskPct := float64(dataset.UsedBytes) / float64(totalBytes) * 100
mh.AddGuestMetric(dsKey, "disk", diskPct, ts)
mh.AddStorageMetric(dsKey, "usage", diskPct, ts)
mh.AddStorageMetric(dsKey, "used", float64(dataset.UsedBytes), ts)
mh.AddStorageMetric(dsKey, "avail", float64(dataset.AvailBytes), ts)
mh.AddStorageMetric(dsKey, "total", float64(totalBytes), ts)
if ms != nil {
ms.Write("dataset", dataset.Name, "disk", diskPct, ts)
ms.Write("storage", dsKey, "usage", diskPct, ts)
ms.Write("storage", dsKey, "used", float64(dataset.UsedBytes), ts)
ms.Write("storage", dsKey, "avail", float64(dataset.AvailBytes), ts)
ms.Write("storage", dsKey, "total", float64(totalBytes), ts)
}
}
}
for _, disk := range snapshot.Disks {
if disk.Temperature <= 0 {
continue
}
resourceID := trueNASDiskMetricsResourceID(disk)
if resourceID == "" {
continue
}
mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), ts)
if ms != nil {
ms.Write("disk", resourceID, "smart_temp", float64(disk.Temperature), ts)
}
}
for _, app := range snapshot.Apps {
if app.Stats == nil {
continue
}
appID := strings.TrimSpace(app.ID)
if appID == "" {
appID = strings.TrimSpace(app.Name)
}
if appID == "" {
continue
}
memPercent := float64(0)
if snapshot.System.MemoryTotalBytes > 0 {
memPercent = (float64(app.Stats.MemoryBytes) / float64(snapshot.System.MemoryTotalBytes)) * 100
}
diskPercent := 8 + float64(HashSeed("truenas-app", appID, "disk")%35)
if strings.EqualFold(strings.TrimSpace(app.State), "stopped") {
diskPercent = clampFloat(diskPercent*0.6, 0, 100)
}
metricKey := "docker:" + appID
mh.AddGuestMetric(metricKey, "cpu", app.Stats.CPUPercent, ts)
mh.AddGuestMetric(metricKey, "memory", memPercent, ts)
mh.AddGuestMetric(metricKey, "disk", diskPercent, ts)
mh.AddGuestMetric(metricKey, "diskread", app.Stats.DiskReadRate, ts)
mh.AddGuestMetric(metricKey, "diskwrite", app.Stats.DiskWriteRate, ts)
mh.AddGuestMetric(metricKey, "netin", app.Stats.NetInRate, ts)
mh.AddGuestMetric(metricKey, "netout", app.Stats.NetOutRate, ts)
if ms != nil {
ms.Write("dockerContainer", appID, "cpu", app.Stats.CPUPercent, ts)
ms.Write("dockerContainer", appID, "memory", memPercent, ts)
ms.Write("dockerContainer", appID, "disk", diskPercent, ts)
ms.Write("dockerContainer", appID, "diskread", app.Stats.DiskReadRate, ts)
ms.Write("dockerContainer", appID, "diskwrite", app.Stats.DiskWriteRate, ts)
ms.Write("dockerContainer", appID, "netin", app.Stats.NetInRate, ts)
ms.Write("dockerContainer", appID, "netout", app.Stats.NetOutRate, ts)
}
}
}
func recordVMwareFixturesMetrics(mh *MetricsHistory, ms *metrics.Store, fixtures mock.PlatformFixtures, ts time.Time) {
snapshot := fixtures.VMware
datastoreUsage := vmwareDatastoreUsageByID(snapshot.Datastores)
for _, host := range snapshot.Hosts {
sourceID := vmware.SourceID(snapshot.ConnectionID, "host", host.Host)
if sourceID == "" {
continue
}
mh.AddGuestMetric("agent:"+sourceID, "cpu", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.CPUPercent }), ts)
mh.AddGuestMetric("agent:"+sourceID, "memory", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.MemoryPercent }), ts)
mh.AddGuestMetric("agent:"+sourceID, "disk", vmwareAverageDatastoreUsage(datastoreUsage, host.DatastoreIDs), ts)
mh.AddGuestMetric("agent:"+sourceID, "diskread", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskReadBytesPerSecond }), ts)
mh.AddGuestMetric("agent:"+sourceID, "diskwrite", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskWriteBytesPerSecond }), ts)
mh.AddGuestMetric("agent:"+sourceID, "netin", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetInBytesPerSecond }), ts)
mh.AddGuestMetric("agent:"+sourceID, "netout", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetOutBytesPerSecond }), ts)
if ms != nil {
ms.Write("agent", sourceID, "cpu", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.CPUPercent }), ts)
ms.Write("agent", sourceID, "memory", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.MemoryPercent }), ts)
ms.Write("agent", sourceID, "disk", vmwareAverageDatastoreUsage(datastoreUsage, host.DatastoreIDs), ts)
ms.Write("agent", sourceID, "diskread", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskReadBytesPerSecond }), ts)
ms.Write("agent", sourceID, "diskwrite", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskWriteBytesPerSecond }), ts)
ms.Write("agent", sourceID, "netin", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetInBytesPerSecond }), ts)
ms.Write("agent", sourceID, "netout", vmwareFloat64Metric(host.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetOutBytesPerSecond }), ts)
}
}
for _, guest := range snapshot.VMs {
sourceID := vmware.SourceID(snapshot.ConnectionID, "vm", guest.VM)
if sourceID == "" {
continue
}
mh.AddGuestMetric(sourceID, "cpu", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.CPUPercent }), ts)
mh.AddGuestMetric(sourceID, "memory", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.MemoryPercent }), ts)
mh.AddGuestMetric(sourceID, "disk", vmwareAverageDatastoreUsage(datastoreUsage, guest.DatastoreIDs), ts)
mh.AddGuestMetric(sourceID, "diskread", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskReadBytesPerSecond }), ts)
mh.AddGuestMetric(sourceID, "diskwrite", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskWriteBytesPerSecond }), ts)
mh.AddGuestMetric(sourceID, "netin", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetInBytesPerSecond }), ts)
mh.AddGuestMetric(sourceID, "netout", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetOutBytesPerSecond }), ts)
if ms != nil {
ms.Write("vm", sourceID, "cpu", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.CPUPercent }), ts)
ms.Write("vm", sourceID, "memory", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.MemoryPercent }), ts)
ms.Write("vm", sourceID, "disk", vmwareAverageDatastoreUsage(datastoreUsage, guest.DatastoreIDs), ts)
ms.Write("vm", sourceID, "diskread", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskReadBytesPerSecond }), ts)
ms.Write("vm", sourceID, "diskwrite", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.DiskWriteBytesPerSecond }), ts)
ms.Write("vm", sourceID, "netin", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetInBytesPerSecond }), ts)
ms.Write("vm", sourceID, "netout", vmwareFloat64Metric(guest.Metrics, func(m *vmware.InventoryMetrics) *float64 { return m.NetOutBytesPerSecond }), ts)
}
}
for _, datastore := range snapshot.Datastores {
sourceID := vmware.SourceID(snapshot.ConnectionID, "datastore", datastore.Datastore)
if sourceID == "" {
continue
}
usage := vmwareDatastoreUsagePercent(datastore)
total := datastore.Capacity
avail := datastore.FreeSpace
used := total - avail
if used < 0 {
used = 0
}
mh.AddStorageMetric(sourceID, "usage", usage, ts)
mh.AddStorageMetric(sourceID, "used", float64(used), ts)
mh.AddStorageMetric(sourceID, "avail", float64(avail), ts)
mh.AddStorageMetric(sourceID, "total", float64(total), ts)
if ms != nil {
ms.Write("storage", sourceID, "usage", usage, ts)
ms.Write("storage", sourceID, "used", float64(used), ts)
ms.Write("storage", sourceID, "avail", float64(avail), ts)
ms.Write("storage", sourceID, "total", float64(total), ts)
}
}
}
func vmwareFloat64Metric(metrics *vmware.InventoryMetrics, pick func(*vmware.InventoryMetrics) *float64) float64 {
if metrics == nil || pick == nil {
return 0
}
value := pick(metrics)
if value == nil {
return 0
}
return *value
}
func vmwareDatastoreUsageByID(datastores []vmware.InventoryDatastore) map[string]float64 {
out := make(map[string]float64, len(datastores))
for _, datastore := range datastores {
id := strings.TrimSpace(datastore.Datastore)
if id == "" {
continue
}
out[id] = vmwareDatastoreUsagePercent(datastore)
}
return out
}
func vmwareAverageDatastoreUsage(byID map[string]float64, ids []string) float64 {
if len(ids) == 0 || len(byID) == 0 {
return 0
}
var total float64
var count int
for _, id := range ids {
usage, ok := byID[strings.TrimSpace(id)]
if !ok {
continue
}
total += usage
count++
}
if count == 0 {
return 0
}
return total / float64(count)
}
func vmwareDatastoreUsagePercent(datastore vmware.InventoryDatastore) float64 {
if datastore.Capacity <= 0 {
return 0
}
used := datastore.Capacity - datastore.FreeSpace
if used < 0 {
used = 0
}
return clampFloat((float64(used)/float64(datastore.Capacity))*100, 0, 100)
}
type guestMetricSource interface {
GetID() string
GetStatus() string
GetCPU() float64
GetMemory() models.Memory
GetDisk() models.Disk
GetDiskRead() int64
GetDiskWrite() int64
GetNetworkIn() int64
GetNetworkOut() int64
}
type vmAdapter struct{ *models.VM }
func (v vmAdapter) GetID() string { return v.VM.ID }
func (v vmAdapter) GetStatus() string { return v.VM.Status }
func (v vmAdapter) GetCPU() float64 { return v.VM.CPU }
func (v vmAdapter) GetMemory() models.Memory { return v.VM.Memory }
func (v vmAdapter) GetDisk() models.Disk { return v.VM.Disk }
func (v vmAdapter) GetDiskRead() int64 { return v.VM.DiskRead }
func (v vmAdapter) GetDiskWrite() int64 { return v.VM.DiskWrite }
func (v vmAdapter) GetNetworkIn() int64 { return v.VM.NetworkIn }
func (v vmAdapter) GetNetworkOut() int64 { return v.VM.NetworkOut }
type containerAdapter struct{ *models.Container }
func (c containerAdapter) GetID() string { return c.Container.ID }
func (c containerAdapter) GetStatus() string { return c.Container.Status }
func (c containerAdapter) GetCPU() float64 { return c.Container.CPU }
func (c containerAdapter) GetMemory() models.Memory { return c.Container.Memory }
func (c containerAdapter) GetDisk() models.Disk { return c.Container.Disk }
func (c containerAdapter) GetDiskRead() int64 { return c.Container.DiskRead }
func (c containerAdapter) GetDiskWrite() int64 { return c.Container.DiskWrite }
func (c containerAdapter) GetNetworkIn() int64 { return c.Container.NetworkIn }
func (c containerAdapter) GetNetworkOut() int64 { return c.Container.NetworkOut }
func recordGuestMetrics[T guestMetricSource](mh *MetricsHistory, ms *metrics.Store, guests []T, prefix string, ts time.Time) {
for _, guest := range guests {
if guest.GetID() == "" || guest.GetStatus() != "running" {
continue
}
id := guest.GetID()
cpu := guest.GetCPU() * 100
memory := guest.GetMemory().Usage
disk := guest.GetDisk().Usage
diskread := float64(guest.GetDiskRead())
diskwrite := float64(guest.GetDiskWrite())
netin := float64(guest.GetNetworkIn())
netout := float64(guest.GetNetworkOut())
mh.AddGuestMetric(id, "cpu", cpu, ts)
mh.AddGuestMetric(id, "memory", memory, ts)
mh.AddGuestMetric(id, "disk", disk, ts)
mh.AddGuestMetric(id, "diskread", diskread, ts)
mh.AddGuestMetric(id, "diskwrite", diskwrite, ts)
mh.AddGuestMetric(id, "netin", netin, ts)
mh.AddGuestMetric(id, "netout", netout, ts)
if ms != nil {
ms.Write(prefix, id, "cpu", cpu, ts)
ms.Write(prefix, id, "memory", memory, ts)
ms.Write(prefix, id, "disk", disk, ts)
ms.Write(prefix, id, "diskread", diskread, ts)
ms.Write(prefix, id, "diskwrite", diskwrite, ts)
ms.Write(prefix, id, "netin", netin, ts)
ms.Write(prefix, id, "netout", netout, ts)
}
}
}
func adaptVMs(vms []models.VM) []vmAdapter {
result := make([]vmAdapter, len(vms))
for i := range vms {
result[i] = vmAdapter{&vms[i]}
}
return result
}
func adaptContainers(cts []models.Container) []containerAdapter {
result := make([]containerAdapter, len(cts))
for i := range cts {
result[i] = containerAdapter{&cts[i]}
}
return result
}
func recordMockStateToMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.FixtureGraph, ts time.Time) {
if mh == nil {
return
}
state := graph.State
for _, node := range state.Nodes {
if node.ID == "" || node.Status != "online" {
continue
}
mh.AddNodeMetric(node.ID, "cpu", node.CPU*100, ts)
mh.AddNodeMetric(node.ID, "memory", node.Memory.Usage, ts)
mh.AddNodeMetric(node.ID, "disk", node.Disk.Usage, ts)
if ms != nil {
ms.Write("node", node.ID, "cpu", node.CPU*100, ts)
ms.Write("node", node.ID, "memory", node.Memory.Usage, ts)
ms.Write("node", node.ID, "disk", node.Disk.Usage, ts)
}
}
recordGuestMetrics(mh, ms, adaptVMs(state.VMs), "vm", ts)
recordGuestMetrics(mh, ms, adaptContainers(state.Containers), "container", ts)
for _, cluster := range state.KubernetesClusters {
if cluster.Hidden {
continue
}
for _, pod := range cluster.Pods {
metricID := kubernetesPodMetricID(cluster, pod)
if metricID == "" {
continue
}
current := kubernetesPodCurrentMetrics(cluster, pod)
mh.AddGuestMetric(metricID, "cpu", current["cpu"], ts)
mh.AddGuestMetric(metricID, "memory", current["memory"], ts)
if current["disk"] > 0 {
mh.AddGuestMetric(metricID, "disk", current["disk"], ts)
}
if current["diskread"] > 0 {
mh.AddGuestMetric(metricID, "diskread", current["diskread"], ts)
}
if current["diskwrite"] > 0 {
mh.AddGuestMetric(metricID, "diskwrite", current["diskwrite"], ts)
}
if current["netin"] > 0 {
mh.AddGuestMetric(metricID, "netin", current["netin"], ts)
}
if current["netout"] > 0 {
mh.AddGuestMetric(metricID, "netout", current["netout"], ts)
}
if ms != nil {
ms.Write("k8s", metricID, "cpu", current["cpu"], ts)
ms.Write("k8s", metricID, "memory", current["memory"], ts)
if current["disk"] > 0 {
ms.Write("k8s", metricID, "disk", current["disk"], ts)
}
if current["diskread"] > 0 {
ms.Write("k8s", metricID, "diskread", current["diskread"], ts)
}
if current["diskwrite"] > 0 {
ms.Write("k8s", metricID, "diskwrite", current["diskwrite"], ts)
}
if current["netin"] > 0 {
ms.Write("k8s", metricID, "netin", current["netin"], ts)
}
if current["netout"] > 0 {
ms.Write("k8s", metricID, "netout", current["netout"], ts)
}
}
}
}
for _, storage := range state.Storage {
if storage.ID == "" || storage.Status != "available" {
continue
}
usage, used, total, avail := normalizedStorageCapacityMetrics(storage.Total, storage.Used, storage.Free, storage.Usage)
mh.AddStorageMetric(storage.ID, "usage", usage, ts)
mh.AddStorageMetric(storage.ID, "used", used, ts)
mh.AddStorageMetric(storage.ID, "total", total, ts)
mh.AddStorageMetric(storage.ID, "avail", avail, ts)
if ms != nil {
ms.Write("storage", storage.ID, "usage", usage, ts)
ms.Write("storage", storage.ID, "used", used, ts)
ms.Write("storage", storage.ID, "total", total, ts)
ms.Write("storage", storage.ID, "avail", avail, ts)
}
}
for _, disk := range state.PhysicalDisks {
if disk.Temperature <= 0 {
continue
}
resourceID := diskMetricsResourceID(disk)
if resourceID == "" {
continue
}
mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), ts)
if ms != nil {
ms.Write("disk", resourceID, "smart_temp", float64(disk.Temperature), ts)
}
}
for _, cluster := range state.CephClusters {
if cluster.TotalBytes <= 0 {
continue
}
cephID := cluster.FSID
if cephID == "" {
cephID = cluster.ID
}
if cephID == "" {
continue
}
if ms != nil {
ms.Write("ceph", cephID, "usage", cluster.UsagePercent, ts)
}
}
for _, host := range state.DockerHosts {
if host.ID == "" || host.Status != "online" {
continue
}
var diskPercent float64
var usedTotal, totalTotal int64
for _, d := range host.Disks {
if d.Total > 0 {
usedTotal += d.Used
totalTotal += d.Total
}
}
if totalTotal > 0 {
diskPercent = float64(usedTotal) / float64(totalTotal) * 100
}
hostKey := "dockerHost:" + host.ID
mh.AddGuestMetric(hostKey, "cpu", host.CPUUsage, ts)
mh.AddGuestMetric(hostKey, "memory", host.Memory.Usage, ts)
mh.AddGuestMetric(hostKey, "disk", diskPercent, ts)
if ms != nil {
ms.Write("dockerHost", host.ID, "cpu", host.CPUUsage, ts)
ms.Write("dockerHost", host.ID, "memory", host.Memory.Usage, ts)
ms.Write("dockerHost", host.ID, "disk", diskPercent, ts)
}
for _, container := range host.Containers {
if container.ID == "" || container.State != "running" {
continue
}
var containerDisk float64
if container.RootFilesystemBytes > 0 && container.WritableLayerBytes > 0 {
containerDisk = float64(container.WritableLayerBytes) / float64(container.RootFilesystemBytes) * 100
containerDisk = clampFloat(containerDisk, 0, 100)
}
metricKey := "docker:" + container.ID
mh.AddGuestMetric(metricKey, "cpu", container.CPUPercent, ts)
mh.AddGuestMetric(metricKey, "memory", container.MemoryPercent, ts)
mh.AddGuestMetric(metricKey, "disk", containerDisk, ts)
if ms != nil {
ms.Write("dockerContainer", container.ID, "cpu", container.CPUPercent, ts)
ms.Write("dockerContainer", container.ID, "memory", container.MemoryPercent, ts)
ms.Write("dockerContainer", container.ID, "disk", containerDisk, ts)
}
}
}
for _, host := range state.Hosts {
if host.ID == "" || host.Status != "online" {
continue
}
var diskPercent float64
if len(host.Disks) > 0 {
diskPercent = host.Disks[0].Usage
}
hostKey := "agent:" + host.ID
mh.AddGuestMetric(hostKey, "cpu", host.CPUUsage, ts)
mh.AddGuestMetric(hostKey, "memory", host.Memory.Usage, ts)
mh.AddGuestMetric(hostKey, "disk", diskPercent, ts)
mh.AddGuestMetric(hostKey, "diskread", host.DiskReadRate, ts)
mh.AddGuestMetric(hostKey, "diskwrite", host.DiskWriteRate, ts)
mh.AddGuestMetric(hostKey, "netin", host.NetInRate, ts)
mh.AddGuestMetric(hostKey, "netout", host.NetOutRate, ts)
if ms != nil {
ms.Write("agent", host.ID, "cpu", host.CPUUsage, ts)
ms.Write("agent", host.ID, "memory", host.Memory.Usage, ts)
ms.Write("agent", host.ID, "disk", diskPercent, ts)
ms.Write("agent", host.ID, "diskread", host.DiskReadRate, ts)
ms.Write("agent", host.ID, "diskwrite", host.DiskWriteRate, ts)
ms.Write("agent", host.ID, "netin", host.NetInRate, ts)
ms.Write("agent", host.ID, "netout", host.NetOutRate, ts)
}
}
// Record TrueNAS pool/dataset disk-usage live ticks
recordTrueNASFixturesMetrics(mh, ms, graph.PlatformFixtures, ts)
recordVMwareFixturesMetrics(mh, ms, graph.PlatformFixtures, ts)
}
func diskMetricsResourceID(disk models.PhysicalDisk) string {
resourceID := strings.TrimSpace(disk.Serial)
if resourceID == "" {
resourceID = strings.TrimSpace(disk.WWN)
}
if resourceID == "" {
resourceID = strings.TrimSpace(disk.ID)
}
if resourceID == "" {
resourceID = fmt.Sprintf("%s-%s-%s", disk.Instance, disk.Node, strings.ReplaceAll(disk.DevPath, "/", "-"))
}
return resourceID
}
func normalizedStorageCapacityMetrics(total, used, free int64, usage float64) (float64, float64, float64, float64) {
totalValue := float64(total)
usedValue := float64(used)
freeValue := float64(free)
if totalValue > 0 {
if free >= 0 {
derivedUsed := float64(total - free)
if derivedUsed >= 0 && derivedUsed <= totalValue {
usedValue = derivedUsed
freeValue = totalValue - usedValue
}
} else {
if usedValue < 0 {
usedValue = 0
}
if usedValue > totalValue {
usedValue = totalValue
}
freeValue = totalValue - usedValue
}
usage = (usedValue / totalValue) * 100
}
return clampFloat(usage, 0, 100), usedValue, totalValue, math.Max(0, freeValue)
}
func trueNASDiskMetricsResourceID(disk truenas.Disk) string {
resourceID := strings.TrimSpace(disk.Serial)
if resourceID == "" {
resourceID = strings.TrimSpace(disk.ID)
}
if resourceID == "" {
resourceID = strings.TrimSpace(disk.Name)
}
return resourceID
}
func (m *Monitor) startMockMetricsSampler(ctx context.Context) {
if ctx == nil || m == nil {
log.Debug().Msg("mock metrics sampler: nil context or monitor")
return
}
if !mock.IsMockEnabled() {
log.Debug().Msg("mock metrics sampler: mock mode not enabled")
return
}
log.Info().Msg("mock metrics sampler: starting initialization")
cfg := mockMetricsSamplerConfigFromEnv()
seedDuration := cfg.SeedDuration
// Reduced minimum from 7 days to 1 hour for faster startup on resource-constrained systems
if seedDuration < time.Hour {
seedDuration = time.Hour
}
// Tiered seeding produces ~2820 points per metric; allow headroom for
// live updates on top of that.
maxPoints := 3500
m.mu.Lock()
if m.mockMetricsCancel != nil {
m.mu.Unlock()
log.Debug().Msg("mock metrics sampler: already running")
return
}
samplerCtx, cancel := context.WithCancel(ctx)
m.mockMetricsCancel = cancel
m.metricsHistory = NewMetricsHistory(maxPoints, seedDuration)
m.mu.Unlock()
graph := mock.CurrentFixtureGraph()
state := graph.State
log.Info().
Int("nodes", len(state.Nodes)).
Int("vms", len(state.VMs)).
Int("containers", len(state.Containers)).
Dur("seedDuration", seedDuration).
Dur("sampleInterval", cfg.SampleInterval).
Msg("Mock metrics sampler: seeding historical data")
// Keep mock trend generation in-memory only so production history in the
// persistent metrics store remains untouched while mock mode is active.
seedMockMetricsHistory(m.metricsHistory, nil, graph, time.Now(), seedDuration, cfg.SampleInterval)
m.mockMetricsWg.Add(1)
go func() {
defer m.mockMetricsWg.Done()
ticker := time.NewTicker(cfg.SampleInterval)
defer ticker.Stop()
for {
select {
case <-samplerCtx.Done():
return
case <-ticker.C:
if !mock.IsMockEnabled() {
continue
}
recordMockStateToMetricsHistory(m.metricsHistory, nil, mock.CurrentFixtureGraph(), time.Now())
}
}
}()
log.Info().
Dur("seedDuration", seedDuration).
Dur("sampleInterval", cfg.SampleInterval).
Msg("Mock metrics history sampler started")
}
func (m *Monitor) stopMockMetricsSampler() {
if m == nil {
return
}
m.mu.Lock()
cancel := m.mockMetricsCancel
m.mockMetricsCancel = nil
m.mu.Unlock()
if cancel != nil {
cancel()
m.mockMetricsWg.Wait()
log.Info().Msg("mock metrics history sampler stopped")
}
}