From 7eb9fe8ade53e450962e71548129d776c8fa7e6d Mon Sep 17 00:00:00 2001 From: Arsolitt Date: Sat, 18 Apr 2026 06:41:45 +0300 Subject: [PATCH] refactor(monitoring): drop unused GPU recording rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - namespace:gpu_count:sum — never consumed by any tracked dashboard; the billable view is already covered by namespace:gpu_count:allocated, and the admin view by cluster:gpu_count:allocated. - namespace:energy_joules:sum — no panel integrates joules; kWh readings on the tenant dashboard compute their own integrations from namespace:power_watts:sum. - pod:tensor_to_nvml_ratio:avg5m — interesting tenant signal in theory, but not wired into any panel and carrying it just burns cardinality on large fleets. Signed-off-by: Arsolitt --- .../alerts/gpu-recording.rules.yaml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml index 3a31f5a2..28d6a45e 100644 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -25,11 +25,6 @@ spec: - name: gpu.recording.namespace.1m interval: 1m rules: - # DCGM-visible GPU count per namespace — counts GPUs that are actually - # running a tenant pod right now (driver loaded, scheduler placed it). - # Differs from :allocated when pods are Pending or stuck. - - record: namespace:gpu_count:sum - expr: count by (namespace) (DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}) # Kube-requested GPU count per namespace — billable view, includes # Pending pods. Use this for GPU-hour reporting via # sum_over_time(...[1h:1m])/60. @@ -62,8 +57,6 @@ spec: expr: sum by (namespace) (DCGM_FI_DEV_FB_USED{namespace!="", namespace!~"cozy-.*|kube-.*"}) * 1048576 - record: namespace:power_watts:sum expr: sum by (namespace) (DCGM_FI_DEV_POWER_USAGE{namespace!="", namespace!~"cozy-.*|kube-.*"}) - - record: namespace:energy_joules:sum - expr: sum by (namespace) (DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION{namespace!="", namespace!~"cozy-.*|kube-.*"}) / 1000 - name: gpu.recording.efficiency.1m interval: 1m @@ -76,19 +69,6 @@ spec: avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) ) * 100 - # NVML vs tensor gap — ratio <10% means NVML lies: user thinks GPU - # is busy but specialized hardware is idle (cheap tenant signal). - - record: pod:tensor_to_nvml_ratio:avg5m - expr: | - ( - avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]) * 100 - ) - / - clamp_min( - avg_over_time(DCGM_FI_DEV_GPU_UTIL{namespace!="", namespace!~"cozy-.*|kube-.*"}[5m]), - 1 - ) - # Power efficiency — utilization per watt, reveals unoptimized clients. - record: pod:util_per_watt:avg5m expr: |