vrr/cozystack
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(controller): resolve Prometheus URL from namespace labels

Browse source 
Replace static --prometheus-url flag with dynamic resolution from
namespace.cozystack.io/monitoring label. Each tenant namespace knows
which tenant hosts its monitoring stack, so the controller constructs
the vmselect URL automatically. This correctly handles multi-tenant
setups where different tenants may use different monitoring instances.

- Remove PrometheusURL field from WorkloadMonitorReconciler struct
- Remove --prometheus-url flag and prometheusUrl chart value
- Add resolvePrometheusURL() that reads namespace label
- queryPrometheusMetric() now accepts prometheusBaseURL as parameter
- Add tests for resolvePrometheusURL with and without label

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy 2026-04-13 18:42:50 +03:00
parent 0e6b8f7ba1
commit 7f7f4b218d
5 changed files with 108 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

9
cmd/cozystack-controller/main.go
View file

@ -70,7 +70,6 @@ func main() {
var disableTelemetry bool
var telemetryEndpoint string
var telemetryInterval string
var prometheusURL string
var tlsOpts []func(*tls.Config)
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@ -88,9 +87,6 @@ func main() {
"Endpoint for sending telemetry data")
flag.StringVar(&telemetryInterval, "telemetry-interval", "15m",
"Interval between telemetry data collection (e.g. 15m, 1h)")
flag.StringVar(&prometheusURL, "prometheus-url", "",
"Prometheus-compatible API URL for querying SeaweedFS bucket metrics (e.g. http://vmselect:8481). "+
"If empty, S3 bucket size metrics are not collected.")
opts := zap.Options{
Development: false,
}
@ -184,9 +180,8 @@ func main() {
}
if err = (&controller.WorkloadMonitorReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
PrometheusURL: prometheusURL,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "WorkloadMonitor")
os.Exit(1)

47
internal/controller/workloadmonitor_controller.go
View file

@ -30,13 +30,21 @@ import (
cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1"
)
const (
// namespaceMonitoringLabel is the namespace label that indicates which tenant
// namespace hosts the monitoring stack (VictoriaMetrics/Prometheus).
namespaceMonitoringLabel = "namespace.cozystack.io/monitoring"
// vmSelectService is the well-known service name for VictoriaMetrics vmselect
// within a monitoring namespace. Port 8481, path /select/0/prometheus.
vmSelectService = "vmselect-shortterm"
vmSelectPort = "8481"
vmSelectPath = "/select/0/prometheus"
)
// WorkloadMonitorReconciler reconciles a WorkloadMonitor object
type WorkloadMonitorReconciler struct {
client.Client
Scheme *runtime.Scheme
// PrometheusURL is the base URL of a Prometheus-compatible API for querying
// SeaweedFS bucket metrics. If empty, bucket size metrics are not collected.
PrometheusURL string
}
// +kubebuilder:rbac:groups=cozystack.io,resources=workloadmonitors,verbs=get;list;watch;create;update;patch;delete
@ -116,15 +124,30 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) {
obj.SetOwnerReferences(owners)
}
// resolvePrometheusURL returns the Prometheus-compatible API base URL for the given namespace.
// It reads the namespace.cozystack.io/monitoring label to find the monitoring namespace,
// then constructs the vmselect URL. Returns empty string if monitoring is not configured.
func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, namespace string) string {
ns := &corev1.Namespace{}
if err := r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil {
return ""
}
monitoringNS := ns.Labels[namespaceMonitoringLabel]
if monitoringNS == "" {
return ""
}
return fmt.Sprintf("http://%s.%s.svc:%s%s", vmSelectService, monitoringNS, vmSelectPort, vmSelectPath)
}
// queryPrometheusMetric queries a Prometheus-compatible API for a single instant value.
// Returns 0 if the metric is not available or PrometheusURL is not configured.
func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, promQL string) int64 {
if r.PrometheusURL == "" {
// Returns 0 if prometheusBaseURL is empty or the metric is not available.
func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, prometheusBaseURL, promQL string) int64 {
if prometheusBaseURL == "" {
return 0
}
logger := log.FromContext(ctx)
u, err := url.Parse(strings.TrimRight(r.PrometheusURL, "/") + "/api/v1/query")
u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query")
if err != nil {
logger.Error(err, "Failed to parse Prometheus URL")
return 0
@ -209,13 +232,17 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor(
resources["s3-buckets"] = resource.MustParse("1")
// Query actual bucket sizes from SeaweedFS metrics via Prometheus.
// The monitoring endpoint is resolved from the namespace label
// namespace.cozystack.io/monitoring, which points to the tenant
// namespace hosting VictoriaMetrics.
// bc.Status.BucketName is the COSI Bucket name, which the COSI driver
// uses directly as the SeaweedFS bucket name.
if bn := bc.Status.BucketName; bn != "" {
if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 {
promURL := r.resolvePrometheusURL(ctx, bc.Namespace)
if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 {
resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI)
}
if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 {
if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 {
resources["s3-physical-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI)
}
}
@ -564,7 +591,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
// Requeue periodically if there are BucketClaims to keep sizes up to date.
// Bucket sizes come from Prometheus metrics that update every 60s.
if len(bucketClaimList.Items) > 0 && r.PrometheusURL != "" {
if len(bucketClaimList.Items) > 0 {
return ctrl.Result{RequeueAfter: 60 * time.Second}, nil
}

111
internal/controller/workloadmonitor_controller_test.go
View file

@ -5,7 +5,6 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
@ -393,8 +392,8 @@ func TestQueryPrometheusMetric(t *testing.T) {
}))
defer srv.Close()
reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL}
size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`)
reconciler := &WorkloadMonitorReconciler{}
size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`)
if size != 5368709120 {
t.Errorf("expected 5368709120, got %d", size)
}
@ -406,8 +405,8 @@ func TestQueryPrometheusMetricEmpty(t *testing.T) {
}))
defer srv.Close()
reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL}
size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`)
reconciler := &WorkloadMonitorReconciler{}
size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`)
if size != 0 {
t.Errorf("expected 0 for empty result, got %d", size)
}
@ -419,35 +418,78 @@ func TestQueryPrometheusMetricServerError(t *testing.T) {
}))
defer srv.Close()
reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL}
size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`)
reconciler := &WorkloadMonitorReconciler{}
size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`)
if size != 0 {
t.Errorf("expected 0 for server error, got %d", size)
}
}
func TestQueryPrometheusMetricNoURL(t *testing.T) {
reconciler := &WorkloadMonitorReconciler{PrometheusURL: ""}
size := reconciler.queryPrometheusMetric(context.TODO(), `anything`)
reconciler := &WorkloadMonitorReconciler{}
size := reconciler.queryPrometheusMetric(context.TODO(), "", `anything`)
if size != 0 {
t.Errorf("expected 0 when PrometheusURL is empty, got %d", size)
}
}
func TestReconcileBucketClaimWithPrometheus(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query().Get("query")
switch {
case strings.HasPrefix(query, "SeaweedFS_s3_bucket_physical"):
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"2147483648"]}]}}`)
default:
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`)
}
}))
defer srv.Close()
func TestResolvePrometheusURL(t *testing.T) {
s := newTestScheme()
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-demo",
Labels: map[string]string{
"namespace.cozystack.io/monitoring": "tenant-root",
},
},
}
fakeClient := fake.NewClientBuilder().
WithScheme(s).
WithObjects(ns).
Build()
reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s}
url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo")
expected := "http://vmselect-shortterm.tenant-root.svc:8481/select/0/prometheus"
if url != expected {
t.Errorf("expected %q, got %q", expected, url)
}
}
func TestResolvePrometheusURLNoLabel(t *testing.T) {
s := newTestScheme()
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-demo",
},
}
fakeClient := fake.NewClientBuilder().
WithScheme(s).
WithObjects(ns).
Build()
reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s}
url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo")
if url != "" {
t.Errorf("expected empty URL when no monitoring label, got %q", url)
}
}
func TestReconcileBucketClaimRequeuesWhenBucketsExist(t *testing.T) {
s := newTestScheme()
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-demo",
},
}
monitor := &cozyv1alpha1.WorkloadMonitor{
ObjectMeta: metav1.ObjectMeta{
Name: "my-bucket",
@ -482,15 +524,11 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) {
fakeClient := fake.NewClientBuilder().
WithScheme(s).
WithObjects(monitor, bc).
WithObjects(ns, monitor, bc).
WithStatusSubresource(monitor).
Build()
reconciler := &WorkloadMonitorReconciler{
Client: fakeClient,
Scheme: s,
PrometheusURL: srv.URL,
}
reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s}
req := reconcile.Request{NamespacedName: types.NamespacedName{
Name: "my-bucket",
Namespace: "tenant-demo",
@ -502,7 +540,7 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) {
}
if result.RequeueAfter == 0 {
t.Error("expected RequeueAfter > 0 when PrometheusURL is set and buckets exist")
t.Error("expected RequeueAfter > 0 when buckets exist")
}
workload := &cozyv1alpha1.Workload{}
@ -514,12 +552,9 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) {
t.Fatalf("expected Workload to be created, got error: %v", err)
}
sizeQty, ok := workload.Status.Resources["s3-storage-bytes"]
if !ok {
t.Fatal("expected s3-storage-bytes resource to be set")
}
if sizeQty.Value() != 1073741824 {
t.Errorf("expected s3-storage-bytes=1073741824 (1 GiB), got %d", sizeQty.Value())
// Without monitoring label on namespace, only s3-buckets should be set (no size metrics)
if _, ok := workload.Status.Resources["s3-storage-bytes"]; ok {
t.Error("expected no s3-storage-bytes when monitoring is not configured")
}
bucketsQty, ok := workload.Status.Resources["s3-buckets"]
@ -529,12 +564,4 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) {
if bucketsQty.Cmp(resource.MustParse("1")) != 0 {
t.Errorf("expected s3-buckets=1, got %s", bucketsQty.String())
}
physQty, ok := workload.Status.Resources["s3-physical-storage-bytes"]
if !ok {
t.Fatal("expected s3-physical-storage-bytes resource to be set")
}
if physQty.Value() != 2147483648 {
t.Errorf("expected s3-physical-storage-bytes=2147483648 (2 GiB), got %d", physQty.Value())
}
}

3
packages/system/cozystack-controller/templates/deployment.yaml
View file

@ -27,6 +27,3 @@ spec:
{{- if .Values.cozystackController.disableTelemetry }}
- --disable-telemetry
{{- end }}
{{- if .Values.cozystackController.prometheusUrl }}
- --prometheus-url={{ .Values.cozystackController.prometheusUrl }}
{{- end }}

1
packages/system/cozystack-controller/values.yaml
View file

@ -2,4 +2,3 @@ cozystackController:
image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0-rc.1@sha256:5ab50893e9d0237d26f366c9d647da6337ca9b97bae764430571d4fb080f6200
debug: false
disableTelemetry: false
prometheusUrl: ""