From 15d319bfa121074efcec8bd0b88edd3594da9821 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 18 Mar 2026 12:01:28 +0400 Subject: [PATCH] feat(backup): no manual actions required to perform restorejob for VMInstance Signed-off-by: Andrey Kolkov --- api/backups/v1alpha1/backup_types.go | 32 + api/backups/v1alpha1/zz_generated.deepcopy.go | 56 +- cmd/backupstrategy-controller/main.go | 9 + examples/desired-backup.yaml | 20 - .../backupcontroller/backup_controller.go | 136 ++++ .../backupclass_resolver_test.go | 4 - .../backupcontroller/restorejob_controller.go | 59 +- .../velerostrategy_controller.go | 716 ++++++++++++++++-- .../backups.cozystack.io_backups.yaml | 23 + .../templates/rbac-bind.yaml | 14 + .../templates/rbac.yaml | 62 +- 11 files changed, 1026 insertions(+), 105 deletions(-) delete mode 100644 examples/desired-backup.yaml create mode 100644 internal/backupcontroller/backup_controller.go diff --git a/api/backups/v1alpha1/backup_types.go b/api/backups/v1alpha1/backup_types.go index 9371c27f..5859d592 100644 --- a/api/backups/v1alpha1/backup_types.go +++ b/api/backups/v1alpha1/backup_types.go @@ -72,6 +72,33 @@ type BackupSpec struct { DriverMetadata map[string]string `json:"driverMetadata,omitempty"` } +// DataVolumeResource describes a dataVolume associated with the backed-up application. +type DataVolumeResource struct { + // DataVolumeName is the name of the dataVolume/PVC (e.g., "vm-disk-ubuntu-source"). + DataVolumeName string `json:"dataVolumeName"` + + // ApplicationName is the cozystack application name for this disk (e.g., "ubuntu-source"). + ApplicationName string `json:"applicationName"` +} + +// UnderlyingResources contains information about resources associated with the +// backed-up application that were discovered at backup time (e.g., VM disks, +// network configuration). This data is used during restore to correctly +// reconstruct the application's environment. +type UnderlyingResources struct { + // DataVolumes lists the dataVolume resources used by the application. + // +optional + DataVolumes []DataVolumeResource `json:"dataVolumes,omitempty"` + + // IP is the OVN IP address assigned to the application at backup time. + // +optional + IP string `json:"ip,omitempty"` + + // MAC is the OVN MAC address assigned to the application at backup time. + // +optional + MAC string `json:"mac,omitempty"` +} + // BackupStatus represents the observed state of a Backup. type BackupStatus struct { // Phase is a simple, high-level summary of the backup's state. @@ -83,6 +110,11 @@ type BackupStatus struct { // +optional Artifact *BackupArtifact `json:"artifact,omitempty"` + // UnderlyingResources contains information about underlying resources + // discovered during backup (e.g., VM disks, IP/MAC addresses). + // +optional + UnderlyingResources *UnderlyingResources `json:"underlyingResources,omitempty"` + // Conditions represents the latest available observations of a Backup's state. // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index b2e4a680..7b5ab25b 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -1,21 +1,5 @@ //go:build !ignore_autogenerated -/* -Copyright 2025 The Cozystack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Code generated by controller-gen. DO NOT EDIT. package v1alpha1 @@ -400,6 +384,11 @@ func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = new(BackupArtifact) **out = **in } + if in.UnderlyingResources != nil { + in, out := &in.UnderlyingResources, &out.UnderlyingResources + *out = new(UnderlyingResources) + (*in).DeepCopyInto(*out) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) @@ -419,6 +408,21 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataVolumeResource) DeepCopyInto(out *DataVolumeResource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataVolumeResource. +func (in *DataVolumeResource) DeepCopy() *DataVolumeResource { + if in == nil { + return nil + } + out := new(DataVolumeResource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Plan) DeepCopyInto(out *Plan) { *out = *in @@ -641,3 +645,23 @@ func (in *RestoreJobStatus) DeepCopy() *RestoreJobStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UnderlyingResources) DeepCopyInto(out *UnderlyingResources) { + *out = *in + if in.DataVolumes != nil { + in, out := &in.DataVolumes, &out.DataVolumes + *out = make([]DataVolumeResource, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnderlyingResources. +func (in *UnderlyingResources) DeepCopy() *UnderlyingResources { + if in == nil { + return nil + } + out := new(UnderlyingResources) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/backupstrategy-controller/main.go b/cmd/backupstrategy-controller/main.go index 6c0ee2e6..03396979 100644 --- a/cmd/backupstrategy-controller/main.go +++ b/cmd/backupstrategy-controller/main.go @@ -176,6 +176,15 @@ func main() { os.Exit(1) } + if err = (&backupcontroller.BackupReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("backup-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Backup") + os.Exit(1) + } + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/examples/desired-backup.yaml b/examples/desired-backup.yaml deleted file mode 100644 index c06b8f82..00000000 --- a/examples/desired-backup.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: backups.cozystack.io/v1alpha1 -kind: BackupJob -metadata: - name: desired-backup - namespace: tenant-root - labels: - backups.cozystack.io/triggered-by: manual -spec: - applicationRef: - apiGroup: apps.cozystack.io - kind: VirtualMachine - name: vm1 - storageRef: - apiGroup: apps.cozystack.io - kind: Bucket - name: test-bucket - strategyRef: - apiGroup: strategy.backups.cozystack.io - kind: Velero - name: velero-strategy-default \ No newline at end of file diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go new file mode 100644 index 00000000..486a24ae --- /dev/null +++ b/internal/backupcontroller/backup_controller.go @@ -0,0 +1,136 @@ +package backupcontroller + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +const backupFinalizer = "backups.cozystack.io/cleanup-velero" + +// BackupReconciler reconciles Backup objects. +// It manages a finalizer that ensures the underlying Velero backup is deleted +// when the cozystack Backup resource is deleted. +type BackupReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + logger.V(1).Info("reconciling Backup", "namespace", req.Namespace, "name", req.Name) + + backup := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, backup); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Handle deletion: clean up Velero backup + if !backup.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(backup, backupFinalizer) { + if err := r.cleanupVeleroBackup(ctx, backup); err != nil { + logger.Error(err, "failed to clean up Velero backup") + return ctrl.Result{}, err + } + + controllerutil.RemoveFinalizer(backup, backupFinalizer) + if err := r.Update(ctx, backup); err != nil { + return ctrl.Result{}, err + } + logger.V(1).Info("removed finalizer and cleaned up Velero backup", "backup", backup.Name) + } + return ctrl.Result{}, nil + } + + // Ensure finalizer is present + if !controllerutil.ContainsFinalizer(backup, backupFinalizer) { + controllerutil.AddFinalizer(backup, backupFinalizer) + if err := r.Update(ctx, backup); err != nil { + return ctrl.Result{}, err + } + logger.V(1).Info("added finalizer to Backup", "backup", backup.Name) + } + + return ctrl.Result{}, nil +} + +// cleanupVeleroBackup deletes the Velero backup and its data from storage +// by creating a Velero DeleteBackupRequest. A direct Delete of the +// backup.velero.io resource only removes the Kubernetes object; Velero's +// BSL sync will recreate it from the object store. The DeleteBackupRequest +// tells Velero to also purge the data, preventing resurrection. +func (r *BackupReconciler) cleanupVeleroBackup(ctx context.Context, backup *backupsv1alpha1.Backup) error { + logger := log.FromContext(ctx) + + veleroBackupName, ok := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if !ok || veleroBackupName == "" { + logger.V(1).Info("no Velero backup name in driverMetadata, nothing to clean up") + return nil + } + + veleroBackupNamespace := backup.Spec.DriverMetadata[veleroBackupNamespaceMetadataKey] + if veleroBackupNamespace == "" { + veleroBackupNamespace = veleroNamespace + } + + // Check if the Velero Backup still exists + veleroBackup := &velerov1.Backup{} + err := r.Get(ctx, types.NamespacedName{ + Namespace: veleroBackupNamespace, + Name: veleroBackupName, + }, veleroBackup) + if err != nil { + if apierrors.IsNotFound(err) { + logger.V(1).Info("Velero backup already deleted", "name", veleroBackupName) + return nil + } + return fmt.Errorf("failed to get Velero backup %s/%s: %w", veleroBackupNamespace, veleroBackupName, err) + } + + // Create a DeleteBackupRequest so Velero removes backup data from storage. + // Without this, BSL sync will recreate the backup.velero.io resource. + dbr := &velerov1.DeleteBackupRequest{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: veleroBackupName + "-", + Namespace: veleroBackupNamespace, + }, + Spec: velerov1.DeleteBackupRequestSpec{ + BackupName: veleroBackupName, + }, + } + if err := r.Create(ctx, dbr); err != nil { + if apierrors.IsAlreadyExists(err) { + logger.V(1).Info("DeleteBackupRequest already exists", "backup", veleroBackupName) + return nil + } + return fmt.Errorf("failed to create DeleteBackupRequest for %s/%s: %w", veleroBackupNamespace, veleroBackupName, err) + } + + logger.Info("created DeleteBackupRequest for Velero backup", + "name", veleroBackupName, "namespace", veleroBackupNamespace, + "deleteRequest", dbr.Name) + return nil +} + +// SetupWithManager registers the BackupReconciler with the Manager. +func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&backupsv1alpha1.Backup{}). + Complete(r) +} diff --git a/internal/backupcontroller/backupclass_resolver_test.go b/internal/backupcontroller/backupclass_resolver_test.go index 5e29cd02..cd65f22a 100644 --- a/internal/backupcontroller/backupclass_resolver_test.go +++ b/internal/backupcontroller/backupclass_resolver_test.go @@ -369,7 +369,3 @@ func TestResolveBackupClass(t *testing.T) { }) } } - -func stringPtr(s string) *string { - return &s -} diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 71307ba3..d73b0c24 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -17,12 +17,16 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) +const restoreJobFinalizer = "backups.cozystack.io/cleanup-velero-restore" + // RestoreJobReconciler reconciles RestoreJob objects. // It routes RestoreJobs to strategy-specific handlers based on the strategy // referenced in the Backup that the RestoreJob is restoring from. @@ -49,6 +53,27 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } + // Handle deletion: clean up Velero Restore + if !restoreJob.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(restoreJob, restoreJobFinalizer) { + r.cleanupVeleroRestore(ctx, restoreJob) + controllerutil.RemoveFinalizer(restoreJob, restoreJobFinalizer) + if err := r.Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + logger.V(1).Info("removed finalizer and cleaned up Velero Restore", "restoreJob", restoreJob.Name) + } + return ctrl.Result{}, nil + } + + // Ensure finalizer is present + if !controllerutil.ContainsFinalizer(restoreJob, restoreJobFinalizer) { + controllerutil.AddFinalizer(restoreJob, restoreJobFinalizer) + if err := r.Update(ctx, restoreJob); err != nil { + return ctrl.Result{}, err + } + } + // If already completed, no need to reconcile if restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || restoreJob.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { @@ -103,17 +128,6 @@ func (r *RestoreJobReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -// getTargetApplicationRef determines the effective target application reference. -// According to DESIGN.md, if spec.targetApplicationRef is omitted, drivers SHOULD -// restore into backup.spec.applicationRef. -// The returned reference is normalized to ensure APIGroup has a default value. -func (r *RestoreJobReconciler) getTargetApplicationRef(restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) corev1.TypedLocalObjectReference { - if restoreJob.Spec.TargetApplicationRef != nil { - return backupsv1alpha1.NormalizeApplicationRef(*restoreJob.Spec.TargetApplicationRef) - } - return backup.Spec.ApplicationRef -} - // markRestoreJobFailed updates the RestoreJob status to Failed with the given message. func (r *RestoreJobReconciler) markRestoreJobFailed(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) { logger := getLogger(ctx) @@ -138,3 +152,26 @@ func (r *RestoreJobReconciler) markRestoreJobFailed(ctx context.Context, restore logger.Debug("RestoreJob failed", "message", message) return ctrl.Result{}, nil } + +// cleanupVeleroRestore deletes all Velero Restores and resourceModifier +// ConfigMaps owned by this RestoreJob (identified by labels). +func (r *RestoreJobReconciler) cleanupVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) { + logger := log.FromContext(ctx) + opts := []client.DeleteAllOfOption{ + client.InNamespace(veleroNamespace), + client.MatchingLabels{ + backupsv1alpha1.OwningJobNameLabel: restoreJob.Name, + backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace, + }, + } + + if err := r.DeleteAllOf(ctx, &velerov1.Restore{}, opts...); err != nil { + logger.Error(err, "failed to delete Velero Restore(s)") + r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "CleanupFailed", + fmt.Sprintf("Failed to delete Velero Restore: %v", err)) + } + + if err := r.DeleteAllOf(ctx, &corev1.ConfigMap{}, opts...); err != nil { + logger.Error(err, "failed to delete resourceModifiers ConfigMap(s)") + } +} diff --git a/internal/backupcontroller/velerostrategy_controller.go b/internal/backupcontroller/velerostrategy_controller.go index 6809bd4f..5ff3348a 100644 --- a/internal/backupcontroller/velerostrategy_controller.go +++ b/internal/backupcontroller/velerostrategy_controller.go @@ -2,13 +2,20 @@ package backupcontroller import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" + "strings" "time" + "sigs.k8s.io/yaml" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,15 +44,6 @@ func (l loggerWithDebug) Debug(msg string, keysAndValues ...interface{}) { l.Logger.V(1).Info(msg, keysAndValues...) } -// S3Credentials holds the discovered S3 credentials from a Bucket storageRef -type S3Credentials struct { - BucketName string - Endpoint string - Region string - AccessKeyID string - AccessSecretKey string -} - const ( defaultRequeueAfter = 5 * time.Second defaultActiveJobPollingInterval = defaultRequeueAfter @@ -55,10 +53,25 @@ const ( veleroNamespace = "cozy-velero" veleroBackupNameMetadataKey = "velero.io/backup-name" veleroBackupNamespaceMetadataKey = "velero.io/backup-namespace" + + // Annotation key for persisting underlying resources on the Velero Backup object + underlyingResourcesAnnotation = "backups.cozystack.io/underlying-resources" + + // VM-specific constants + vmInstanceKind = "VMInstance" + vmDiskAppKind = "VMDisk" + vmNamePrefix = "vm-instance-" + vmDiskNamePrefix = "vm-disk-" + appKindLabel = "apps.cozystack.io/application.kind" + appNameLabel = "apps.cozystack.io/application.name" + vmPodNameLabel = "vm.kubevirt.io/name" + ovnIPAnnotation = "ovn.kubernetes.io/ip_address" + ovnMACAnnotation = "ovn.kubernetes.io/mac_address" + cdiAllowClaimAdoption = "cdi.kubevirt.io/allowClaimAdoption" ) -func boolPtr(b bool) *bool { - return &b +func stringPtr(s string) *string { + return &s } func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) { @@ -197,10 +210,7 @@ func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1a // Step 5: On failure if phase == "Failed" || phase == "PartiallyFailed" { - message := fmt.Sprintf("Velero Backup failed with phase: %s", phase) - if len(veleroBackup.Status.ValidationErrors) > 0 { - message = fmt.Sprintf("%s: %v", message, veleroBackup.Status.ValidationErrors) - } + message := formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, veleroBackup) return r.markBackupJobFailed(ctx, j, message) } @@ -208,6 +218,76 @@ func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1a return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } +// collectUnderlyingResources discovers resources associated with a VM application +// (dataVolumes, IP/MAC addresses) that need to be backed up and restored. +// Returns nil if the application is not a VM type or has no underlying resources. +func (r *BackupJobReconciler) collectUnderlyingResources(ctx context.Context, app *unstructured.Unstructured, appKind, ns string) (*backupsv1alpha1.UnderlyingResources, error) { + logger := getLogger(ctx) + + if appKind != vmInstanceKind { + logger.Debug("application is not a VMInstance, skipping underlying resource collection", "kind", appKind) + return nil, nil + } + + appName := app.GetName() + + // Extract disk names from VMInstance spec.disks[].name + disks, found, err := unstructured.NestedSlice(app.Object, "spec", "disks") + if err != nil { + return nil, fmt.Errorf("failed to read spec.disks from application: %w", err) + } + + var dataVolumes []backupsv1alpha1.DataVolumeResource + if found { + for _, d := range disks { + disk, ok := d.(map[string]interface{}) + if !ok { + continue + } + name, ok := disk["name"].(string) + if !ok || name == "" { + continue + } + dataVolumes = append(dataVolumes, backupsv1alpha1.DataVolumeResource{ + DataVolumeName: vmDiskNamePrefix + name, + ApplicationName: name, + }) + } + } + logger.Debug("collected dataVolumes from VMInstance", "count", len(dataVolumes), "appName", appName) + + // Find VM Pod to extract OVN IP/MAC addresses + vmName := vmNamePrefix + appName + podList := &corev1.PodList{} + if err := r.List(ctx, podList, + client.InNamespace(ns), + client.MatchingLabels{vmPodNameLabel: vmName}, + ); err != nil { + logger.Error(err, "failed to list VM pods for IP/MAC collection", "vmName", vmName) + // Non-fatal: we can still proceed without IP/MAC + } + + var ip, mac string + if len(podList.Items) > 0 { + pod := podList.Items[0] + ip = pod.Annotations[ovnIPAnnotation] + mac = pod.Annotations[ovnMACAnnotation] + logger.Debug("collected OVN network info from VM pod", "ip", ip, "mac", mac, "pod", pod.Name) + } else { + logger.Debug("no VM pod found for OVN info", "vmName", vmName) + } + + if len(dataVolumes) == 0 && ip == "" && mac == "" { + return nil, nil + } + + return &backupsv1alpha1.UnderlyingResources{ + DataVolumes: dataVolumes, + IP: ip, + MAC: mac, + }, nil +} + func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Velero, resolved *ResolvedBackupConfig) error { logger := getLogger(ctx) logger.Debug("createVeleroBackup called", "strategy", strategy.Name) @@ -225,6 +305,13 @@ func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob return err } + // Collect underlying resources (VM disks, IP/MAC) + underlyingResources, err := r.collectUnderlyingResources(ctx, app, backupJob.Spec.ApplicationRef.Kind, backupJob.Namespace) + if err != nil { + logger.Error(err, "failed to collect underlying resources, proceeding without them") + // Non-fatal: proceed with backup even if collection fails + } + templateContext := map[string]interface{}{ "Application": app.Object, "Parameters": resolved.Parameters, @@ -234,6 +321,33 @@ func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob if err != nil { return err } + + // Add label selectors for underlying VMDisk HelmReleases + if underlyingResources != nil { + for _, dv := range underlyingResources.DataVolumes { + veleroBackupSpec.OrLabelSelectors = append(veleroBackupSpec.OrLabelSelectors, &metav1.LabelSelector{ + MatchLabels: map[string]string{ + appKindLabel: vmDiskAppKind, + appNameLabel: dv.ApplicationName, + }, + }) + } + if len(underlyingResources.DataVolumes) > 0 { + logger.Debug("added VMDisk label selectors to Velero backup", "count", len(underlyingResources.DataVolumes)) + } + } + + // Serialize underlying resources as annotation to persist across reconcile cycles + annotations := map[string]string{} + if underlyingResources != nil { + urJSON, err := json.Marshal(underlyingResources) + if err != nil { + logger.Error(err, "failed to marshal underlying resources annotation") + } else { + annotations[underlyingResourcesAnnotation] = string(urJSON) + } + } + veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("%s.%s-", backupJob.Namespace, backupJob.Name), @@ -242,6 +356,7 @@ func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob backupsv1alpha1.OwningJobNameLabel: backupJob.Name, backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace, }, + Annotations: annotations, }, Spec: *veleroBackupSpec, } @@ -284,19 +399,22 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name), } + // Read underlying resources from Velero Backup annotation + var underlyingResources *backupsv1alpha1.UnderlyingResources + if urJSON, ok := veleroBackup.Annotations[underlyingResourcesAnnotation]; ok && urJSON != "" { + underlyingResources = &backupsv1alpha1.UnderlyingResources{} + if err := json.Unmarshal([]byte(urJSON), underlyingResources); err != nil { + logger.Error(err, "failed to unmarshal underlying resources from Velero Backup annotation") + underlyingResources = nil + } + } + + // Note: No OwnerReferences set on Backup. The Backup must survive BackupJob deletion + // so users don't lose their backup artifacts when cleaning up completed jobs. backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: backupJob.Name, Namespace: backupJob.Namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: backupJob.APIVersion, - Kind: backupJob.Kind, - Name: backupJob.Name, - UID: backupJob.UID, - Controller: boolPtr(true), - }, - }, }, Spec: backupsv1alpha1.BackupSpec{ ApplicationRef: backupJob.Spec.ApplicationRef, @@ -305,8 +423,9 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo DriverMetadata: driverMetadata, }, Status: backupsv1alpha1.BackupStatus{ - Phase: backupsv1alpha1.BackupPhaseReady, - Artifact: artifact, + Phase: backupsv1alpha1.BackupPhaseReady, + Artifact: artifact, + UnderlyingResources: underlyingResources, }, } @@ -319,7 +438,8 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo return nil, err } - logger.Debug("created Backup resource", "name", backup.Name) + logger.Debug("created Backup resource", "name", backup.Name, + "hasUnderlyingResources", underlyingResources != nil) return backup, nil } @@ -378,6 +498,16 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto } if len(veleroRestoreList.Items) == 0 { + // Pre-restore: graceful shutdown, suspend HRs, rename PVCs + ready, result, err := r.prepareForRestore(ctx, restoreJob, backup) + if err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("pre-restore preparation failed: %v", err)) + } + if !ready { + logger.Debug("pre-restore preparation in progress, requeuing") + return result, nil + } + // Create Velero Restore logger.Debug("Velero Restore not found, creating new one") if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName); err != nil { @@ -430,31 +560,436 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto return ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil } +// Velero resource modifier types (local mirrors of the internal Velero types). + +type resourceModifiers struct { + Version string `yaml:"version"` + ResourceModifierRules []resourceModifierRule `yaml:"resourceModifierRules"` +} + +type resourceModifierRule struct { + Conditions resourceModifierConditions `yaml:"conditions"` + MergePatches []mergePatch `yaml:"mergePatches,omitempty"` +} + +type resourceModifierConditions struct { + GroupResource string `yaml:"groupResource"` + ResourceNameRegex string `yaml:"resourceNameRegex,omitempty"` + Namespaces []string `yaml:"namespaces,omitempty"` +} + +type mergePatch struct { + PatchData string `yaml:"patchData"` +} + +// marshalPatchData marshals an arbitrary object to YAML for use as +// mergePatch.PatchData in Velero resource modifiers. +func marshalPatchData(v interface{}) (string, error) { + b, err := yaml.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal patch data: %w", err) + } + return string(b), nil +} + +// createResourceModifiersConfigMap creates a Velero resource modifiers ConfigMap +// that patches VM resources during restore: +// - PVC adoption: always adds cdi.kubevirt.io/allowClaimAdoption=true to all +// restored PVCs so CDI can adopt them when a HelmRelease of VMDisk recreates a DV. +// - OVN IP/MAC: sets OVN annotations on the VirtualMachine for correct ssh access to restored VM. +func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (*corev1.ConfigMap, error) { + logger := getLogger(ctx) + + ur := backup.Status.UnderlyingResources + targetNS := backup.Namespace + + var rules []resourceModifierRule + + // PVC adoption: allow CDI to adopt restored PVCs when the HelmRelease recreates a DV. + pvcPatch, err := marshalPatchData(map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]string{ + cdiAllowClaimAdoption: "true", + }, + }, + }) + if err != nil { + return nil, err + } + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{ + GroupResource: "persistentvolumeclaims", + ResourceNameRegex: ".*", + Namespaces: []string{targetNS}, + }, + MergePatches: []mergePatch{{PatchData: pvcPatch}}, + }) + + // OVN IP/MAC annotations on VirtualMachine for correct network identity after restore. + if ur != nil && (ur.IP != "" || ur.MAC != "") { + ovnAnnotations := map[string]string{} + if ur.IP != "" { + ovnAnnotations[ovnIPAnnotation] = ur.IP + } + if ur.MAC != "" { + ovnAnnotations[ovnMACAnnotation] = ur.MAC + } + vmPatch, err := marshalPatchData(map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": ovnAnnotations, + }, + }, + }, + }) + if err != nil { + return nil, err + } + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{ + GroupResource: "virtualmachines.kubevirt.io", + ResourceNameRegex: ".*", + Namespaces: []string{targetNS}, + }, + MergePatches: []mergePatch{{PatchData: vmPatch}}, + }) + } + + rulesYAML, err := yaml.Marshal(resourceModifiers{ + Version: "v1", + ResourceModifierRules: rules, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal resource modifier rules: %w", err) + } + + cmName := fmt.Sprintf("restore-modifiers-%s-%s", restoreJob.Namespace, restoreJob.Name) + // Truncate name to fit Kubernetes 253-char limit + if len(cmName) > 253 { + cmName = cmName[:253] + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: veleroNamespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: restoreJob.Name, + backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace, + }, + }, + Data: map[string]string{ + "resource-modifier-rules.yaml": string(rulesYAML), + }, + } + + if err := r.Create(ctx, cm); err != nil { + if errors.IsAlreadyExists(err) { + // ConfigMap already exists (e.g. RestoreJob recreated with same name). + // Update its data to reflect the current backup's underlying resources. + existing := &corev1.ConfigMap{} + if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: cmName}, existing); err != nil { + return nil, fmt.Errorf("failed to get existing resourceModifiers ConfigMap: %w", err) + } + existing.Data = cm.Data + if err := r.Update(ctx, existing); err != nil { + return nil, fmt.Errorf("failed to update existing resourceModifiers ConfigMap: %w", err) + } + logger.Debug("updated existing resourceModifiers ConfigMap", "name", cmName) + return existing, nil + } + return nil, fmt.Errorf("failed to create resourceModifiers ConfigMap: %w", err) + } + + logger.Debug("created resourceModifiers ConfigMap", "name", cm.Name, "namespace", cm.Namespace) + return cm, nil +} + +// resolveUnderlyingResourcesForRestore returns disk (and network) metadata for symmetric +// restore label selectors. Velero Backup annotation is used when Backup.status was empty +// (e.g. CRD without underlyingResources in schema). +func (r *RestoreJobReconciler) resolveUnderlyingResourcesForRestore(ctx context.Context, backup *backupsv1alpha1.Backup, veleroBackupName string) *backupsv1alpha1.UnderlyingResources { + if backup.Status.UnderlyingResources != nil && len(backup.Status.UnderlyingResources.DataVolumes) > 0 { + return backup.Status.UnderlyingResources + } + vb := &velerov1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}, vb); err != nil { + return backup.Status.UnderlyingResources + } + if urJSON, ok := vb.Annotations[underlyingResourcesAnnotation]; ok && urJSON != "" { + ur := &backupsv1alpha1.UnderlyingResources{} + if err := json.Unmarshal([]byte(urJSON), ur); err != nil { + return backup.Status.UnderlyingResources + } + return ur + } + return backup.Status.UnderlyingResources +} + +// GVRs used during pre-restore preparation. +var ( + helmReleaseGVR = schema.GroupVersionResource{Group: "helm.toolkit.fluxcd.io", Version: "v2", Resource: "helmreleases"} + virtualMachineGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + vmiGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachineinstances"} + dataVolumeGVR = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"} +) + +// shortHash returns the first 4 hex characters of sha256(input). +func shortHash(input string) string { + h := sha256.Sum256([]byte(input)) + return hex.EncodeToString(h[:])[:4] +} + +// prepareForRestore performs graceful pre-restore cleanup: +// 1. Suspends HelmReleases that belong to the backup scope. +// 2. Halts the VirtualMachine (sets spec.runStrategy=Halted). +// 3. Waits for the VMI to disappear (graceful shutdown complete). +// 4. Deletes DataVolumes so CDI doesn't recreate PVCs after rename. +// 5. Renames existing PVCs to -orig- so Velero can create fresh +// ones via Data Movement. +// +// Failures in individual steps are non-fatal: missing resources are expected +// (e.g. restore requested when app was already deleted). Each action emits +// a Kubernetes Event on the RestoreJob for observability. +// +// Returns true when preparation is complete and the Velero Restore can be created. +// Returns false (with a requeue) when still waiting for VM shutdown. +func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ready bool, result ctrl.Result, err error) { + ns := restoreJob.Namespace + appName := backup.Spec.ApplicationRef.Name + appKind := backup.Spec.ApplicationRef.Kind + origSuffix := "-orig-" + shortHash(restoreJob.Name) + + // --- Step 1: Suspend HelmReleases --- + hrNames := []string{} + if appKind == vmInstanceKind { + hrNames = append(hrNames, vmNamePrefix+appName) + } + if backup.Status.UnderlyingResources != nil { + for _, dv := range backup.Status.UnderlyingResources.DataVolumes { + hrNames = append(hrNames, dv.DataVolumeName) + } + } + for _, hrName := range hrNames { + if err := r.suspendHelmRelease(ctx, ns, hrName); err != nil { + r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore", + fmt.Sprintf("Failed to suspend HelmRelease %s: %v", hrName, err)) + } else { + r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", + fmt.Sprintf("Suspended HelmRelease %s", hrName)) + } + } + + // --- Step 2: Halt VM and wait for shutdown --- + if appKind == vmInstanceKind { + vmName := vmNamePrefix + appName + halted, err := r.haltVirtualMachine(ctx, ns, vmName) + if err != nil { + r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore", + fmt.Sprintf("Failed to halt VM %s: %v", vmName, err)) + // Non-fatal: proceed even if halting fails (VM might not exist) + } else if !halted { + r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", + fmt.Sprintf("Waiting for VM %s to shut down", vmName)) + return false, ctrl.Result{RequeueAfter: defaultRestoreRequeueAfter}, nil + } else { + r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", + fmt.Sprintf("VM %s is halted", vmName)) + } + } + + // --- Step 3: Rename PVCs to -orig- --- + // Must happen BEFORE deleting DVs: the PVC has an ownerReference to the DV, + // so deleting the DV first would cascade-delete the PVC via garbage collection. + if backup.Status.UnderlyingResources != nil { + for _, dv := range backup.Status.UnderlyingResources.DataVolumes { + if err := r.renamePVC(ctx, restoreJob, ns, dv.DataVolumeName, dv.DataVolumeName+origSuffix); err != nil { + r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore", + fmt.Sprintf("Failed to keep old PVC %s: %v", dv.DataVolumeName, err)) + } + } + } + + // --- Step 4: Delete DataVolumes so CDI doesn't recreate PVCs --- + if backup.Status.UnderlyingResources != nil { + for _, dv := range backup.Status.UnderlyingResources.DataVolumes { + if err := r.deleteDataVolume(ctx, ns, dv.DataVolumeName); err != nil { + r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore", + fmt.Sprintf("Failed to delete DataVolume %s: %v", dv.DataVolumeName, err)) + } else { + r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", + fmt.Sprintf("Deleted DataVolume %s", dv.DataVolumeName)) + } + } + } + + r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", "Pre-restore preparation complete") + return true, ctrl.Result{}, nil +} + +// suspendHelmRelease sets spec.suspend=true on a HelmRelease. +func (r *RestoreJobReconciler) suspendHelmRelease(ctx context.Context, ns, name string) error { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend") + if suspended { + return nil + } + if err := unstructured.SetNestedField(hr.Object, true, "spec", "suspend"); err != nil { + return err + } + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Update(ctx, hr, metav1.UpdateOptions{}); err != nil { + return err + } + return nil +} + +// haltVirtualMachine sets runStrategy=Halted and returns true when the VMI is gone. +func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmName string) (bool, error) { + vm, err := r.Resource(virtualMachineGVR).Namespace(ns).Get(ctx, vmName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return true, nil + } + return false, err + } + + currentStrategy, _, _ := unstructured.NestedString(vm.Object, "spec", "runStrategy") + if currentStrategy != "Halted" { + if err := unstructured.SetNestedField(vm.Object, "Halted", "spec", "runStrategy"); err != nil { + return false, err + } + if _, err := r.Resource(virtualMachineGVR).Namespace(ns).Update(ctx, vm, metav1.UpdateOptions{}); err != nil { + return false, err + } + } + + // VMI gone = shutdown complete + _, err = r.Resource(vmiGVR).Namespace(ns).Get(ctx, vmName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil +} + +// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after rename. +func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) error { + err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return err + } + return nil +} + +// renamePVC preserves an existing PVC by rebinding it under a new name. +// The original PVC is deleted and a new one pointing to the same PV is created. +// Missing resources are silently skipped (non-fatal). +func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, ns, oldName, newName string) error { + logger := getLogger(ctx) + + // Check if already renamed + existingNew := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: newName}, existingNew); err == nil { + return nil + } + + // Get the original PVC + oldPVC := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: oldName}, oldPVC); err != nil { + if errors.IsNotFound(err) { + return nil // nothing to rename + } + return err + } + + pvName := oldPVC.Spec.VolumeName + if pvName == "" { + logger.Debug("PVC not bound, deleting", "name", oldName) + return r.Delete(ctx, oldPVC) + } + + // Patch PV reclaim policy to Retain so it survives PVC deletion + pv := &corev1.PersistentVolume{} + if err := r.Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil { + return fmt.Errorf("failed to get PV %s: %w", pvName, err) + } + if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + if err := r.Update(ctx, pv); err != nil { + return fmt.Errorf("failed to set Retain policy on PV %s: %w", pvName, err) + } + } + + // Create the new -orig PVC first (unbound, just the object) + newPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: newName, + Namespace: ns, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: oldPVC.Spec.AccessModes, + Resources: oldPVC.Spec.Resources, + StorageClassName: oldPVC.Spec.StorageClassName, + VolumeMode: oldPVC.Spec.VolumeMode, + VolumeName: pvName, + }, + } + if err := r.Create(ctx, newPVC); err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create -orig PVC %s: %w", newName, err) + } + // Re-read to get UID for the claimRef + if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: newName}, newPVC); err != nil { + return fmt.Errorf("failed to get -orig PVC %s: %w", newName, err) + } + + // Delete the original PVC + if err := r.Delete(ctx, oldPVC); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete PVC %s: %w", oldName, err) + } + + // Point the PV's claimRef directly to the new -orig PVC. + // This is atomic — no window where the PV is Available for other PVCs to grab. + if err := r.Get(ctx, client.ObjectKey{Name: pvName}, pv); err != nil { + return fmt.Errorf("failed to re-fetch PV %s: %w", pvName, err) + } + pv.Spec.ClaimRef = &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: ns, + Name: newName, + UID: newPVC.UID, + } + if err := r.Update(ctx, pv); err != nil { + return fmt.Errorf("failed to rebind PV %s to %s: %w", pvName, newName, err) + } + + r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", + fmt.Sprintf("Keep old PVC %s as %s (PV: %s)", oldName, newName, pvName)) + return nil +} + // createVeleroRestore creates a Velero Restore resource. func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Velero, veleroBackupName string) error { logger := getLogger(ctx) logger.Debug("createVeleroRestore called", "strategy", strategy.Name, "veleroBackupName", veleroBackupName) - // Determine target application reference - targetAppRef := r.getTargetApplicationRef(restoreJob, backup) - - // Get the target application object for templating - mapping, err := r.RESTMapping(schema.GroupKind{Group: *targetAppRef.APIGroup, Kind: targetAppRef.Kind}) - if err != nil { - return fmt.Errorf("failed to get REST mapping for target application: %w", err) - } - ns := restoreJob.Namespace - if mapping.Scope.Name() != meta.RESTScopeNameNamespace { - ns = "" - } - app, err := r.Resource(mapping.Resource).Namespace(ns).Get(ctx, targetAppRef.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get target application: %w", err) - } - - // Build template context templateContext := map[string]interface{}{ - "Application": app.Object, + "Application": map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": backup.Spec.ApplicationRef.Name, + "namespace": backup.Namespace, + }, + "kind": backup.Spec.ApplicationRef.Kind, + }, // TODO: Parameters are not currently stored on Backup, so they're unavailable during restore. // This is a design limitation that should be addressed by persisting Parameters on the Backup object. "Parameters": map[string]string{}, @@ -473,6 +1008,38 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ // Set the backupName in the spec (required by Velero) veleroRestoreSpec.BackupName = veleroBackupName + // Match backup: add OR selectors for each underlying VMDisk so restore applies the same + // scope as the intended backup (see createVeleroBackup). Prefer Backup status; fall back + // to Velero Backup annotation when status was pruned by an older CRD. + ur := r.resolveUnderlyingResourcesForRestore(ctx, backup, veleroBackupName) + if ur != nil { + for _, dv := range ur.DataVolumes { + veleroRestoreSpec.OrLabelSelectors = append(veleroRestoreSpec.OrLabelSelectors, &metav1.LabelSelector{ + MatchLabels: map[string]string{ + appKindLabel: vmDiskAppKind, + appNameLabel: dv.ApplicationName, + }, + }) + } + if len(ur.DataVolumes) > 0 { + logger.Debug("added VMDisk label selectors to Velero restore", "count", len(ur.DataVolumes)) + } + } + + // Create resourceModifiers ConfigMap + resourceModifierCM, err := r.createResourceModifiersConfigMap(ctx, restoreJob, backup) + if err != nil { + return fmt.Errorf("failed to create resourceModifiers ConfigMap: %w", err) + } + if resourceModifierCM != nil { + veleroRestoreSpec.ResourceModifier = &corev1.TypedLocalObjectReference{ + APIGroup: stringPtr(""), + Kind: "ConfigMap", + Name: resourceModifierCM.Name, + } + logger.Debug("set resourceModifier on Velero Restore", "configMap", resourceModifierCM.Name) + } + generateName := fmt.Sprintf("%s.%s-", restoreJob.Namespace, restoreJob.Name) veleroRestore := &velerov1.Restore{ ObjectMeta: metav1.ObjectMeta{ @@ -497,3 +1064,60 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ fmt.Sprintf("Created Velero Restore %s/%s", veleroNamespace, veleroRestore.Name)) return nil } + +// dataUploadListGVK is the API version shipped with Velero data mover CRDs (see velero datauploads CRD). +var dataUploadListGVK = schema.GroupVersionKind{Group: "velero.io", Version: "v2alpha1", Kind: "DataUploadList"} + +// formatVeleroBackupFailureMessageForBackupJob builds a BackupJob status message from Velero Backup +// status plus failed DataUpload resources (CSI data mover), similar to `velero backup describe`. +func formatVeleroBackupFailureMessageForBackupJob(ctx context.Context, c client.Client, veleroBackup *velerov1.Backup) string { + var b strings.Builder + fmt.Fprintf(&b, "Velero Backup failed with phase %s", veleroBackup.Status.Phase) + if fr := strings.TrimSpace(veleroBackup.Status.FailureReason); fr != "" { + fmt.Fprintf(&b, ": %s", fr) + } + if len(veleroBackup.Status.ValidationErrors) > 0 { + fmt.Fprintf(&b, "; validation: %v", veleroBackup.Status.ValidationErrors) + } + if h := veleroBackup.Status.HookStatus; h != nil && h.HooksFailed > 0 { + fmt.Fprintf(&b, "; hooks failed %d/%d", h.HooksFailed, h.HooksAttempted) + } + if veleroBackup.Status.BackupItemOperationsFailed > 0 { + fmt.Fprintf(&b, "; async item operations failed %d (completed %d, attempted %d)", + veleroBackup.Status.BackupItemOperationsFailed, + veleroBackup.Status.BackupItemOperationsCompleted, + veleroBackup.Status.BackupItemOperationsAttempted) + } + b.WriteString(appendFailedDataUploadMessages(ctx, c, veleroBackup.Name)) + return b.String() +} + +func appendFailedDataUploadMessages(ctx context.Context, c client.Client, veleroBackupName string) string { + ul := unstructured.UnstructuredList{} + ul.SetGroupVersionKind(dataUploadListGVK) + if err := c.List(ctx, &ul, client.InNamespace(veleroNamespace)); err != nil { + return "" + } + prefix := veleroBackupName + "-" + var b strings.Builder + for _, item := range ul.Items { + if !strings.HasPrefix(item.GetName(), prefix) { + continue + } + phase, _, _ := unstructured.NestedString(item.Object, "status", "phase") + if phase != "Failed" { + continue + } + msg, _, _ := unstructured.NestedString(item.Object, "status", "message") + if strings.TrimSpace(msg) == "" { + msg = "(empty status.message)" + } + srcPVC, _, _ := unstructured.NestedString(item.Object, "spec", "sourcePVC") + if srcPVC != "" { + fmt.Fprintf(&b, "; DataUpload %s failed for PVC %s: %s", item.GetName(), srcPVC, msg) + } else { + fmt.Fprintf(&b, "; DataUpload %s failed: %s", item.GetName(), msg) + } + } + return b.String() +} diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index 6d55cb84..76d8a477 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -142,6 +142,29 @@ spec: required: - uri type: object + underlyingResources: + description: |- + UnderlyingResources contains information about underlying resources + discovered during backup (e.g., VM disks, IP/MAC addresses). + properties: + dataVolumes: + items: + properties: + applicationName: + description: Cozystack application name for this disk (e.g. ubuntu-source). + type: string + dataVolumeName: + description: Name of the DataVolume/PVC (e.g. vm-disk-ubuntu-source). + type: string + type: object + type: array + ip: + description: OVN IP address assigned to the application at backup time. + type: string + mac: + description: OVN MAC address assigned to the application at backup time. + type: string + type: object conditions: description: Conditions represents the latest available observations of a Backup's state. diff --git a/packages/system/backupstrategy-controller/templates/rbac-bind.yaml b/packages/system/backupstrategy-controller/templates/rbac-bind.yaml index 03578cc2..4ecacce0 100644 --- a/packages/system/backupstrategy-controller/templates/rbac-bind.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac-bind.yaml @@ -10,3 +10,17 @@ subjects: - kind: ServiceAccount name: backupstrategy-controller namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: backups.cozystack.io:strategy-controller:velero-configmaps + namespace: cozy-velero +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: backups.cozystack.io:strategy-controller:velero-configmaps +subjects: +- kind: ServiceAccount + name: backupstrategy-controller + namespace: {{ .Release.Namespace }} diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 7c31d408..2216577e 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -11,25 +11,60 @@ rules: - apiGroups: ["backups.cozystack.io"] resources: ["backupclasses"] verbs: ["get", "list", "watch"] -# BackupJob / RestoreJob: reconcile and update status +# BackupJob / RestoreJob: reconcile; update for RestoreJob finalizers - apiGroups: ["backups.cozystack.io"] resources: ["backupjobs", "restorejobs"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["backups.cozystack.io"] resources: ["backupjobs/status", "restorejobs/status"] verbs: ["get", "update", "patch"] -# Backup: create after Velero backup completes +# Backup: create after Velero job completes; update/patch for BackupReconciler finalizers - apiGroups: ["backups.cozystack.io"] resources: ["backups"] - verbs: ["create", "get", "list", "watch"] -# Application refs (e.g. VMInstance, VirtualMachine) for backup/restore scope + verbs: ["create", "get", "list", "watch", "update", "patch"] +# Pods: BackupJob lists virt-launcher pods by label (manager cache uses cluster-scoped list/watch) +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +# ConfigMaps: controller-runtime cache requires cluster-scoped list/watch; +# create/update/delete is scoped to cozy-velero via the Role below. +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +# Application refs (e.g. VMInstance) for backup/restore scope - apiGroups: ["apps.cozystack.io"] resources: ["*"] verbs: ["get", "list", "watch"] -# Velero Backup/Restore in cozy-velero namespace +# KubeVirt: pre-restore halts VMs and checks VMI shutdown +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["get", "update"] +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["get"] +# CDI DataVolumes: pre-restore deletes DVs so CDI doesn't recreate PVCs after rename +- apiGroups: ["cdi.kubevirt.io"] + resources: ["datavolumes"] + verbs: ["delete"] +# HelmReleases: pre-restore suspends HRs to prevent Flux interference +- apiGroups: ["helm.toolkit.fluxcd.io"] + resources: ["helmreleases"] + verbs: ["get", "update"] +# PVCs and PVs: pre-restore renames PVCs (delete old, create new) and patches PV reclaim policy +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] +# Velero Backup/Restore/DataUpload in cozy-velero namespace (DataUpload: enrich BackupJob failure messages) - apiGroups: ["velero.io"] - resources: ["backups", "restores"] - verbs: ["create", "get", "list", "watch", "update", "patch"] + resources: ["backups", "restores", "datauploads"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete", "deletecollection"] +# Velero DeleteBackupRequest: used by BackupReconciler to delete Velero backups with their storage data +- apiGroups: ["velero.io"] + resources: ["deletebackuprequests"] + verbs: ["create", "get", "list", "watch"] # Events from Recorder.Event() calls - apiGroups: [""] resources: ["events"] @@ -38,3 +73,14 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +# To create ResourceModifiers in ConfigMaps for Restore in Velero install namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: backups.cozystack.io:strategy-controller:velero-configmaps + namespace: cozy-velero +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "delete", "deletecollection", "patch", "update"]