fix(monitoring): close DCGM coverage gap for gpu-fleet TDP panel
gpu-fleet.json references DCGM_FI_DEV_POWER_MGMT_LIMIT for its "TDP vs draw" panel, but the custom DCGM Exporter CSV did not declare it, so the panel silently rendered "No data" on clusters using that config. Declare the counter, fix the dashboards table in the gpu-operator examples README, and add a bats test that cross-checks every DCGM_FI_* reference in tracked dashboards and recording rules against the union of the upstream default set (snapshotted under hack/) and the project's custom CSV. Signed-off-by: Arsolitt <arsolitt@gmail.com>
This commit is contained in:
Arsolitt
parent
2fa4b3e31c
commit
ccfec2ef62
4 changed files with 220 additions and 27 deletions
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bats
|
||||
# -----------------------------------------------------------------------------
|
||||
# Cross-validation between GPU recording rules and the dashboards that consume
|
||||
# them. Catches:
|
||||
# Cross-validation between GPU recording rules, the dashboards that consume
|
||||
# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches:
|
||||
#
|
||||
# 1. dangling references — a dashboard query mentions a recording rule name
|
||||
# that doesn't exist in gpu-recording.rules.yaml. This is the bug the
|
||||
|
|
@ -12,6 +12,13 @@
|
|||
# 2. typos in rule names — same bug class, manifested as a single-character
|
||||
# difference between rule and reference.
|
||||
#
|
||||
# 3. undeclared DCGM metrics — a dashboard query or recording rule mentions
|
||||
# a DCGM_FI_* metric that is neither in the upstream default CSV nor in
|
||||
# the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM
|
||||
# Exporter would never emit it and the panel silently shows "No data".
|
||||
# Example regression: gpu-fleet.json shipped a TDP panel referencing
|
||||
# DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it.
|
||||
#
|
||||
# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list
|
||||
# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in
|
||||
# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one
|
||||
|
|
@ -33,6 +40,8 @@ REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
|
|||
RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml"
|
||||
DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list"
|
||||
DASHBOARDS_DIR="$REPO_ROOT/dashboards"
|
||||
DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv"
|
||||
DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml"
|
||||
|
||||
# Extract the set of "- record: NAME" entries from the rules YAML.
|
||||
# Outputs one rule name per line, sorted and deduplicated.
|
||||
|
|
@ -61,6 +70,37 @@ list_tracked_gpu_dashboards() {
|
|||
awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST"
|
||||
}
|
||||
|
||||
# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles
|
||||
# both the upstream-style default CSV (unindented) and the ConfigMap-style
|
||||
# custom CSV (YAML-indented). A declaration line starts — after any leading
|
||||
# whitespace — with "DCGM_FI_<NAME>," ; comment lines begin with "#" and are
|
||||
# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions
|
||||
# are required.
|
||||
extract_csv_metrics() {
|
||||
file=$1
|
||||
awk '
|
||||
{
|
||||
line = $0
|
||||
sub(/^[[:space:]]+/, "", line)
|
||||
if (line ~ /^#/) next
|
||||
if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) {
|
||||
print substr(line, RSTART, RLENGTH)
|
||||
}
|
||||
}
|
||||
' "$file" | sort -u
|
||||
}
|
||||
|
||||
# Extract the set of DCGM_FI_* metric references from a text file (dashboard
|
||||
# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited
|
||||
# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_
|
||||
# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps
|
||||
# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that
|
||||
# appear in comments.
|
||||
extract_dcgm_refs() {
|
||||
file=$1
|
||||
grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u
|
||||
}
|
||||
|
||||
@test "every recording rule reference in tracked GPU dashboards has a matching record" {
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
|
@ -94,6 +134,54 @@ list_tracked_gpu_dashboards() {
|
|||
[ "$failed" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "every DCGM metric referenced in tracked dashboards and rules is declared" {
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
[ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; }
|
||||
[ -f "$DCGM_CUSTOM_CSV" ] || { echo "missing $DCGM_CUSTOM_CSV" >&2; exit 1; }
|
||||
|
||||
{
|
||||
extract_csv_metrics "$DCGM_DEFAULT_CSV"
|
||||
extract_csv_metrics "$DCGM_CUSTOM_CSV"
|
||||
} | sort -u > "$TMP/declared.txt"
|
||||
[ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; }
|
||||
|
||||
list_tracked_gpu_dashboards > "$TMP/dashboards.txt"
|
||||
[ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; }
|
||||
|
||||
failed=0
|
||||
|
||||
# Dashboard coverage — every dashboard's DCGM references must resolve.
|
||||
while IFS= read -r dashboard_rel; do
|
||||
dashboard="$DASHBOARDS_DIR/$dashboard_rel"
|
||||
[ -f "$dashboard" ] || continue # handled by the existence test
|
||||
extract_dcgm_refs "$dashboard" > "$TMP/refs.txt"
|
||||
[ -s "$TMP/refs.txt" ] || continue # dashboard relies entirely on recording rules
|
||||
comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt"
|
||||
if [ -s "$TMP/missing.txt" ]; then
|
||||
echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2
|
||||
sed 's/^/ - /' "$TMP/missing.txt" >&2
|
||||
failed=1
|
||||
fi
|
||||
done < "$TMP/dashboards.txt"
|
||||
|
||||
# Rules coverage — recording rules consume DCGM directly, so their set
|
||||
# must be declared too, otherwise derived series on every dashboard
|
||||
# collapse to empty.
|
||||
extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt"
|
||||
if [ -s "$TMP/rule-refs.txt" ]; then
|
||||
comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt"
|
||||
if [ -s "$TMP/rule-missing.txt" ]; then
|
||||
echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2
|
||||
sed 's/^/ - /' "$TMP/rule-missing.txt" >&2
|
||||
failed=1
|
||||
fi
|
||||
fi
|
||||
|
||||
[ "$failed" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" {
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
|
|
|||
104
hack/dcgm-default-counters.csv
Normal file
104
hack/dcgm-default-counters.csv
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# Snapshot of the upstream DCGM Exporter default-counters.csv used as a
|
||||
# fixture by hack/check-gpu-recording-rules.bats. The test verifies that
|
||||
# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either
|
||||
# declared here (upstream defaults) or in
|
||||
# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml
|
||||
# (the project's custom CSV).
|
||||
#
|
||||
# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv
|
||||
# Pinned to the DCGM Exporter image tag shipped by the gpu-operator
|
||||
# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml
|
||||
# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is
|
||||
# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter
|
||||
# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`.
|
||||
|
||||
# Format
|
||||
# If line starts with a '#' it is considered a comment
|
||||
# DCGM FIELD, Prometheus metric type, help message
|
||||
|
||||
# Clocks
|
||||
DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz).
|
||||
DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
|
||||
|
||||
# Temperature
|
||||
DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
|
||||
DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C).
|
||||
|
||||
# Power
|
||||
DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W).
|
||||
DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
|
||||
|
||||
# PCIE
|
||||
# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML.
|
||||
# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML.
|
||||
DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.
|
||||
|
||||
# Utilization (the sample period varies depending on the product)
|
||||
DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).
|
||||
DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
|
||||
DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %).
|
||||
DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %).
|
||||
|
||||
# Errors and violations
|
||||
DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.
|
||||
# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us).
|
||||
# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us).
|
||||
# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us).
|
||||
# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
|
||||
# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us).
|
||||
# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
|
||||
|
||||
# Memory usage
|
||||
DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
|
||||
DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).
|
||||
|
||||
# ECC
|
||||
# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
|
||||
# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
|
||||
# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
|
||||
# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.
|
||||
|
||||
# Retired pages
|
||||
# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors.
|
||||
# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors.
|
||||
# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.
|
||||
|
||||
# NVLink
|
||||
# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
|
||||
# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
|
||||
# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries.
|
||||
# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
|
||||
DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes.
|
||||
# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload.
|
||||
|
||||
# VGPU License status
|
||||
DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status
|
||||
|
||||
# Remapped rows
|
||||
DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
|
||||
DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors
|
||||
DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed
|
||||
|
||||
# Static configuration information. These appear as labels on the other metrics
|
||||
DCGM_FI_DRIVER_VERSION, label, Driver Version
|
||||
# DCGM_FI_NVML_VERSION, label, NVML Version
|
||||
# DCGM_FI_DEV_BRAND, label, Device Brand
|
||||
# DCGM_FI_DEV_SERIAL, label, Device Serial Number
|
||||
# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version
|
||||
# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version
|
||||
# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version
|
||||
# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version
|
||||
# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device
|
||||
|
||||
# Datacenter Profiling (DCP) metrics
|
||||
# NOTE: supported on Nvidia datacenter Volta GPUs and newer
|
||||
DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active.
|
||||
# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned.
|
||||
# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM.
|
||||
DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active.
|
||||
DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data.
|
||||
# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active.
|
||||
# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active.
|
||||
# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active.
|
||||
DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
|
||||
DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
|
||||
|
Can't render this file because it has a wrong number of fields in line 12.
|
|
|
@ -68,34 +68,34 @@ scraped, or when optional counters are missing.
|
|||
What each dashboard needs on top of the upstream DCGM Exporter
|
||||
[`default-counters.csv`][default-csv]:
|
||||
|
||||
| Dashboard | Scope | Needs beyond defaults |
|
||||
| ----------------- | ---------------------------------- | -------------------------------------------------------------------------------------------- |
|
||||
| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE`, `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` |
|
||||
| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` |
|
||||
| `gpu-fleet` | Cluster-wide admin inventory | nothing (works on default counters) |
|
||||
| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) |
|
||||
| `gpu-tenants` | Per-namespace tenant view | `DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` for the tensor-saturation panel; other panels work on defaults |
|
||||
| Dashboard | Scope | Needs beyond defaults |
|
||||
| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- |
|
||||
| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` |
|
||||
| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) |
|
||||
| `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) |
|
||||
| `gpu-quotas` | Kube-quota vs live usage | nothing (kube-state-metrics + default counters) |
|
||||
| `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) |
|
||||
|
||||
The throttling counters (`DCGM_FI_DEV_POWER_VIOLATION`,
|
||||
`DCGM_FI_DEV_THERMAL_VIOLATION`) are only required by `gpu-performance`.
|
||||
The profiling counters (`DCGM_FI_PROF_*`) are required by
|
||||
`gpu-performance` and `gpu-efficiency`, and unlock the tensor panel in
|
||||
`gpu-tenants`. Everything else the recording rules consume —
|
||||
utilization, FB used/free, power, temperature, energy — is already in
|
||||
the default counter set.
|
||||
`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE`
|
||||
are already in the upstream default set for the pinned DCGM Exporter
|
||||
version, so the tensor-saturation and engine-active panels work without
|
||||
any CSV override. The three counters listed in the table — throttling
|
||||
violations and the power management limit — are the only extras the
|
||||
tracked dashboards need. The recording rules in
|
||||
`gpu-recording.rules.yaml` consume utilization, FB used, power,
|
||||
temperature and the tensor-active profiling counter, all of which are
|
||||
in the default set.
|
||||
|
||||
## Verification status
|
||||
|
||||
> **Pending verification on an updated GPU Operator release.**
|
||||
>
|
||||
> The minimum-CSV claims above are derived by cross-referencing
|
||||
> `gpu-recording.rules.yaml` and each dashboard against the DCGM
|
||||
> Exporter [`default-counters.csv`][default-csv] for the version pinned
|
||||
> in the currently shipped `gpu-operator` package. The package in this
|
||||
> branch is **not** the latest GPU Operator release; once we move to a
|
||||
> newer version, the claims must be re-checked because the upstream
|
||||
> default set occasionally adds or removes counters between releases.
|
||||
> Until then, treat the CSV in `dcgm-custom-metrics.yaml` as a
|
||||
> known-good superset rather than a minimal config.
|
||||
The minimum-CSV claims above are verified by
|
||||
`hack/check-gpu-recording-rules.bats`, which cross-checks every
|
||||
`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules
|
||||
against the union of the upstream default set (snapshotted at
|
||||
`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version)
|
||||
and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM
|
||||
Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml`
|
||||
is bumped, refresh the snapshot from the matching tag of the
|
||||
[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test.
|
||||
|
||||
[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ data:
|
|||
|
||||
# Power
|
||||
DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W).
|
||||
DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Current power management limit (in W).
|
||||
DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).
|
||||
|
||||
# PCIE
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue