diff --git a/cmd/cozystack-controller/main.go b/cmd/cozystack-controller/main.go index 1175e2b4..bcaca9c9 100644 --- a/cmd/cozystack-controller/main.go +++ b/cmd/cozystack-controller/main.go @@ -70,7 +70,6 @@ func main() { var disableTelemetry bool var telemetryEndpoint string var telemetryInterval string - var prometheusURL string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -88,9 +87,6 @@ func main() { "Endpoint for sending telemetry data") flag.StringVar(&telemetryInterval, "telemetry-interval", "15m", "Interval between telemetry data collection (e.g. 15m, 1h)") - flag.StringVar(&prometheusURL, "prometheus-url", "", - "Prometheus-compatible API URL for querying SeaweedFS bucket metrics (e.g. http://vmselect:8481). "+ - "If empty, S3 bucket size metrics are not collected.") opts := zap.Options{ Development: false, } @@ -184,9 +180,8 @@ func main() { } if err = (&controller.WorkloadMonitorReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - PrometheusURL: prometheusURL, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkloadMonitor") os.Exit(1) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 1e011313..e86e5a75 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -30,13 +30,21 @@ import ( cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" ) +const ( + // namespaceMonitoringLabel is the namespace label that indicates which tenant + // namespace hosts the monitoring stack (VictoriaMetrics/Prometheus). + namespaceMonitoringLabel = "namespace.cozystack.io/monitoring" + // vmSelectService is the well-known service name for VictoriaMetrics vmselect + // within a monitoring namespace. Port 8481, path /select/0/prometheus. + vmSelectService = "vmselect-shortterm" + vmSelectPort = "8481" + vmSelectPath = "/select/0/prometheus" +) + // WorkloadMonitorReconciler reconciles a WorkloadMonitor object type WorkloadMonitorReconciler struct { client.Client Scheme *runtime.Scheme - // PrometheusURL is the base URL of a Prometheus-compatible API for querying - // SeaweedFS bucket metrics. If empty, bucket size metrics are not collected. - PrometheusURL string } // +kubebuilder:rbac:groups=cozystack.io,resources=workloadmonitors,verbs=get;list;watch;create;update;patch;delete @@ -116,15 +124,30 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { obj.SetOwnerReferences(owners) } +// resolvePrometheusURL returns the Prometheus-compatible API base URL for the given namespace. +// It reads the namespace.cozystack.io/monitoring label to find the monitoring namespace, +// then constructs the vmselect URL. Returns empty string if monitoring is not configured. +func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, namespace string) string { + ns := &corev1.Namespace{} + if err := r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { + return "" + } + monitoringNS := ns.Labels[namespaceMonitoringLabel] + if monitoringNS == "" { + return "" + } + return fmt.Sprintf("http://%s.%s.svc:%s%s", vmSelectService, monitoringNS, vmSelectPort, vmSelectPath) +} + // queryPrometheusMetric queries a Prometheus-compatible API for a single instant value. -// Returns 0 if the metric is not available or PrometheusURL is not configured. -func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, promQL string) int64 { - if r.PrometheusURL == "" { +// Returns 0 if prometheusBaseURL is empty or the metric is not available. +func (r *WorkloadMonitorReconciler) queryPrometheusMetric(ctx context.Context, prometheusBaseURL, promQL string) int64 { + if prometheusBaseURL == "" { return 0 } logger := log.FromContext(ctx) - u, err := url.Parse(strings.TrimRight(r.PrometheusURL, "/") + "/api/v1/query") + u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query") if err != nil { logger.Error(err, "Failed to parse Prometheus URL") return 0 @@ -209,13 +232,17 @@ func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( resources["s3-buckets"] = resource.MustParse("1") // Query actual bucket sizes from SeaweedFS metrics via Prometheus. + // The monitoring endpoint is resolved from the namespace label + // namespace.cozystack.io/monitoring, which points to the tenant + // namespace hosting VictoriaMetrics. // bc.Status.BucketName is the COSI Bucket name, which the COSI driver // uses directly as the SeaweedFS bucket name. if bn := bc.Status.BucketName; bn != "" { - if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 { + promURL := r.resolvePrometheusURL(ctx, bc.Namespace) + if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_size_bytes{bucket="%s"}`, bn)); v > 0 { resources["s3-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } - if v := r.queryPrometheusMetric(ctx, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 { + if v := r.queryPrometheusMetric(ctx, promURL, fmt.Sprintf(`SeaweedFS_s3_bucket_physical_size_bytes{bucket="%s"}`, bn)); v > 0 { resources["s3-physical-storage-bytes"] = *resource.NewQuantity(v, resource.BinarySI) } } @@ -564,7 +591,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Requeue periodically if there are BucketClaims to keep sizes up to date. // Bucket sizes come from Prometheus metrics that update every 60s. - if len(bucketClaimList.Items) > 0 && r.PrometheusURL != "" { + if len(bucketClaimList.Items) > 0 { return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index e8feb0b0..c78efd18 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "strings" "testing" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" @@ -393,8 +392,8 @@ func TestQueryPrometheusMetric(t *testing.T) { })) defer srv.Close() - reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="cosi-abc123"}`) if size != 5368709120 { t.Errorf("expected 5368709120, got %d", size) } @@ -406,8 +405,8 @@ func TestQueryPrometheusMetricEmpty(t *testing.T) { })) defer srv.Close() - reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="nonexistent"}`) if size != 0 { t.Errorf("expected 0 for empty result, got %d", size) } @@ -419,35 +418,78 @@ func TestQueryPrometheusMetricServerError(t *testing.T) { })) defer srv.Close() - reconciler := &WorkloadMonitorReconciler{PrometheusURL: srv.URL} - size := reconciler.queryPrometheusMetric(context.TODO(), `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), srv.URL, `SeaweedFS_s3_bucket_size_bytes{bucket="test"}`) if size != 0 { t.Errorf("expected 0 for server error, got %d", size) } } func TestQueryPrometheusMetricNoURL(t *testing.T) { - reconciler := &WorkloadMonitorReconciler{PrometheusURL: ""} - size := reconciler.queryPrometheusMetric(context.TODO(), `anything`) + reconciler := &WorkloadMonitorReconciler{} + size := reconciler.queryPrometheusMetric(context.TODO(), "", `anything`) if size != 0 { t.Errorf("expected 0 when PrometheusURL is empty, got %d", size) } } -func TestReconcileBucketClaimWithPrometheus(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query().Get("query") - switch { - case strings.HasPrefix(query, "SeaweedFS_s3_bucket_physical"): - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"2147483648"]}]}}`) - default: - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"bucket":"cosi-abc123"},"value":[1713000000,"1073741824"]}]}}`) - } - })) - defer srv.Close() - +func TestResolvePrometheusURL(t *testing.T) { s := newTestScheme() + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-demo", + Labels: map[string]string{ + "namespace.cozystack.io/monitoring": "tenant-root", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(ns). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo") + + expected := "http://vmselect-shortterm.tenant-root.svc:8481/select/0/prometheus" + if url != expected { + t.Errorf("expected %q, got %q", expected, url) + } +} + +func TestResolvePrometheusURLNoLabel(t *testing.T) { + s := newTestScheme() + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-demo", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(ns). + Build() + + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} + url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo") + + if url != "" { + t.Errorf("expected empty URL when no monitoring label, got %q", url) + } +} + +func TestReconcileBucketClaimRequeuesWhenBucketsExist(t *testing.T) { + s := newTestScheme() + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-demo", + }, + } + monitor := &cozyv1alpha1.WorkloadMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: "my-bucket", @@ -482,15 +524,11 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(s). - WithObjects(monitor, bc). + WithObjects(ns, monitor, bc). WithStatusSubresource(monitor). Build() - reconciler := &WorkloadMonitorReconciler{ - Client: fakeClient, - Scheme: s, - PrometheusURL: srv.URL, - } + reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} req := reconcile.Request{NamespacedName: types.NamespacedName{ Name: "my-bucket", Namespace: "tenant-demo", @@ -502,7 +540,7 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { } if result.RequeueAfter == 0 { - t.Error("expected RequeueAfter > 0 when PrometheusURL is set and buckets exist") + t.Error("expected RequeueAfter > 0 when buckets exist") } workload := &cozyv1alpha1.Workload{} @@ -514,12 +552,9 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { t.Fatalf("expected Workload to be created, got error: %v", err) } - sizeQty, ok := workload.Status.Resources["s3-storage-bytes"] - if !ok { - t.Fatal("expected s3-storage-bytes resource to be set") - } - if sizeQty.Value() != 1073741824 { - t.Errorf("expected s3-storage-bytes=1073741824 (1 GiB), got %d", sizeQty.Value()) + // Without monitoring label on namespace, only s3-buckets should be set (no size metrics) + if _, ok := workload.Status.Resources["s3-storage-bytes"]; ok { + t.Error("expected no s3-storage-bytes when monitoring is not configured") } bucketsQty, ok := workload.Status.Resources["s3-buckets"] @@ -529,12 +564,4 @@ func TestReconcileBucketClaimWithPrometheus(t *testing.T) { if bucketsQty.Cmp(resource.MustParse("1")) != 0 { t.Errorf("expected s3-buckets=1, got %s", bucketsQty.String()) } - - physQty, ok := workload.Status.Resources["s3-physical-storage-bytes"] - if !ok { - t.Fatal("expected s3-physical-storage-bytes resource to be set") - } - if physQty.Value() != 2147483648 { - t.Errorf("expected s3-physical-storage-bytes=2147483648 (2 GiB), got %d", physQty.Value()) - } } diff --git a/packages/system/cozystack-controller/templates/deployment.yaml b/packages/system/cozystack-controller/templates/deployment.yaml index 94567881..aab7bbe1 100644 --- a/packages/system/cozystack-controller/templates/deployment.yaml +++ b/packages/system/cozystack-controller/templates/deployment.yaml @@ -27,6 +27,3 @@ spec: {{- if .Values.cozystackController.disableTelemetry }} - --disable-telemetry {{- end }} - {{- if .Values.cozystackController.prometheusUrl }} - - --prometheus-url={{ .Values.cozystackController.prometheusUrl }} - {{- end }} diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 2999730b..6addae2b 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -2,4 +2,3 @@ cozystackController: image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0-rc.1@sha256:5ab50893e9d0237d26f366c9d647da6337ca9b97bae764430571d4fb080f6200 debug: false disableTelemetry: false - prometheusUrl: ""