From f7161c2af806faa414ec53d47e99f303eed79a67 Mon Sep 17 00:00:00 2001 From: sasha-sup Date: Wed, 25 Mar 2026 15:42:07 +0300 Subject: [PATCH 1/3] [piraeus-operator] Fix LINSTOR satellite alert labels and scrape flapping false positives Signed-off-by: sasha-sup (cherry picked from commit a562d9cb80b28ab75afa95c636fb8fff8bae0595) --- .../piraeus-operator/alerts/piraeus-datastore.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml index 77512847..bbc1c81a 100644 --- a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml +++ b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml @@ -17,9 +17,12 @@ spec: - alert: linstorSatelliteErrorRate annotations: description: | - LINSTOR Satellite "{{ $labels.name }}" reports {{ $value }} errors in the last 15 minutes. - Use "linstor error-reports list --nodes {{ $labels.name }} --since 15minutes" to see them. - expr: increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0 + LINSTOR Satellite "{{ $labels.hostname }}" reports {{ $value }} errors in the last 15 minutes. + Use "linstor error-reports list --nodes {{ $labels.hostname }}" to inspect the reports. + expr: | + increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0 + and on(instance, job) + min_over_time(up{job="linstor-controller"}[15m]) == 1 labels: severity: warning - alert: linstorControllerErrorRate @@ -33,7 +36,7 @@ spec: - alert: linstorSatelliteNotOnline annotations: description: | - LINSTOR Satellite "{{ $labels.name }}" is not ONLINE. + LINSTOR Satellite "{{ $labels.hostname }}" is not ONLINE. Check that the Satellite is running and reachable from the LINSTOR Controller. expr: linstor_node_state{nodetype="SATELLITE"} != 2 labels: From 53d603f50ca18490ad12ec78ea3833ada58e5e2b Mon Sep 17 00:00:00 2001 From: sasha-sup Date: Wed, 25 Mar 2026 16:02:59 +0300 Subject: [PATCH 2/3] [piraeus-operator] Add hold time to LINSTOR controller offline alert Signed-off-by: sasha-sup (cherry picked from commit 4a92d7753ceffad979a83793703e99e68e0ab126) --- packages/system/piraeus-operator/alerts/piraeus-datastore.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml index bbc1c81a..f63c7d10 100644 --- a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml +++ b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml @@ -12,6 +12,7 @@ spec: description: | LINSTOR Controller is not reachable. expr: up{job="linstor-controller"} == 0 + for: 3m labels: severity: critical - alert: linstorSatelliteErrorRate From f961438c9d76ace094a96480fcb00f3ae4c51af9 Mon Sep 17 00:00:00 2001 From: sasha-sup Date: Wed, 25 Mar 2026 16:11:14 +0300 Subject: [PATCH 3/3] [piraeus-operator] Split LINSTOR controller availability and metrics alerts Signed-off-by: sasha-sup (cherry picked from commit a052a650b0308538fb0e85511c899a77a60e6b3e) --- .../piraeus-operator/alerts/piraeus-datastore.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml index f63c7d10..0cc39fe8 100644 --- a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml +++ b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml @@ -7,14 +7,22 @@ spec: groups: - name: linstor.rules rules: - - alert: linstorControllerOffline + - alert: linstorControllerUnavailable annotations: description: | - LINSTOR Controller is not reachable. - expr: up{job="linstor-controller"} == 0 + LINSTOR Controller deployment has no available replicas. + expr: kube_deployment_status_replicas_available{namespace="cozy-linstor",deployment="linstor-controller"} < 1 for: 3m labels: severity: critical + - alert: linstorControllerMetricsScrapeFailing + annotations: + description: | + LINSTOR Controller metrics endpoint is not being scraped successfully. + expr: up{job="linstor-controller"} == 0 + for: 10m + labels: + severity: warning - alert: linstorSatelliteErrorRate annotations: description: |