feat(backups): restore vmi to copy in another namespace

Signed-off-by: Andrey Kolkov <androndo@gmail.com>
This commit is contained in:
Andrey Kolkov 2026-04-03 11:53:59 +04:00
parent d27b01c61d
commit 2234824feb
19 changed files with 2360 additions and 30 deletions

View file

@ -42,6 +42,12 @@ type RestoreJobSpec struct {
// application as referenced by backup.spec.applicationRef.
// +optional
TargetApplicationRef *corev1.TypedLocalObjectReference `json:"targetApplicationRef,omitempty"`
// Options is a driver-specific blob of restore options, typed based on
// targetApplicationRef and the current controller implementation.
// +optional
// +kubebuilder:pruning:PreserveUnknownFields
Options *runtime.RawExtension `json:"options,omitempty"`
}
// RestoreJobStatus represents the observed state of a RestoreJob.

View file

@ -0,0 +1,112 @@
#!/bin/bash
# Helper functions and variables for the VMInstance backup/restore demo
# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh"
# ANSI color codes
export RED='\033[0;31m'
export GREEN='\033[0;32m'
export YELLOW='\033[1;33m'
export BLUE='\033[0;34m'
export MAGENTA='\033[0;35m'
export CYAN='\033[0;36m'
export WHITE='\033[1;37m'
export NC='\033[0m' # No Color
export BOLD='\033[1m'
# Default settings
export NAMESPACE="${NAMESPACE:-tenant-root}"
export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-default}"
# Logging functions (output to stderr to avoid polluting captured output)
log_info() {
echo -e "${BLUE}${NC} $*" >&2
}
log_success() {
echo -e "${GREEN}${NC} $*" >&2
}
log_warning() {
echo -e "${YELLOW}${NC} $*" >&2
}
log_error() {
echo -e "${RED}${NC} $*" >&2
}
log_step() {
echo -e "\n${MAGENTA}${BOLD}$*${NC}" >&2
}
log_substep() {
echo -e "${CYAN}$*${NC}" >&2
}
log_command() {
echo -e "${WHITE} \$ $*${NC}" >&2
}
# Wait for user to press Enter
wait_for_enter() {
echo -e "\n${CYAN}Press Enter to continue...${NC}" >&2
read -r
}
# Check if a Kubernetes resource exists
resource_exists() {
local resource_type="$1"
local resource_name="$2"
local namespace="${3:-}"
if [[ -n "$namespace" ]]; then
kubectl get "$resource_type" "$resource_name" -n "$namespace" &>/dev/null
else
kubectl get "$resource_type" "$resource_name" &>/dev/null
fi
}
# Wait for a resource field to reach a desired value
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=""
[[ -n "$namespace" ]] && ns_flag="-n $namespace"
while true; do
local current
# shellcheck disable=SC2086
current=$(kubectl get "$resource_type" "$resource_name" $ns_flag -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
echo -n "." >&2
done
}
# Print a separator line
separator() {
echo -e "\n${CYAN}────────────────────────────────────────────────────────────${NC}\n" >&2
}
# Print script header
print_header() {
local title="$1"
echo -e "\n${MAGENTA}${BOLD}╔════════════════════════════════════════════════════════════╗${NC}" >&2
echo -e "${MAGENTA}${BOLD}${NC} ${WHITE}${BOLD}$title${NC}" >&2
echo -e "${MAGENTA}${BOLD}╚════════════════════════════════════════════════════════════╝${NC}\n" >&2
}

View file

@ -0,0 +1,145 @@
#!/bin/bash
# Step 01: Create Velero backup/restore strategies for VMInstance and VMDisk
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 1: Create Velero Strategies"
log_step "Creating VMInstance strategy..."
log_command "kubectl apply -f - (Velero strategy: vminstance-strategy)"
kubectl apply -f - <<'EOF'
apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Velero
metadata:
name: vminstance-strategy
spec:
template:
# Symmetric restore filters: kubevirt-velero-plugin requires launcher pods in the backup,
# but restore orLabelSelectors limit what is applied from the tarball (e.g. skip another
# VM's pod that Velero added via PVC item action). VMDisk OR branches are appended by
# the controller from backup.status.underlyingResources or the Velero backup annotation.
restoreSpec:
existingResourcePolicy: update
includedNamespaces:
- '{{ .Application.metadata.namespace }}'
orLabelSelectors:
- matchLabels:
app.kubernetes.io/instance: 'vm-instance-{{ .Application.metadata.name }}'
- matchLabels:
apps.cozystack.io/application.kind: '{{ .Application.kind }}'
apps.cozystack.io/application.name: '{{ .Application.metadata.name }}'
includedResources:
- helmreleases.helm.toolkit.fluxcd.io
- virtualmachines.kubevirt.io
- virtualmachineinstances.kubevirt.io
- pods
- persistentvolumeclaims
- configmaps
- secrets
- controllerrevisions.apps
includeClusterResources: false
excludedResources:
# Required to avoid conflict with restored DV from HR VMDisk
- datavolumes.cdi.kubevirt.io
spec: # see https://velero.io/docs/v1.17/api-types/backup/
includedNamespaces:
- '{{ .Application.metadata.namespace }}'
orLabelSelectors:
# VM resources (VirtualMachine, DataVolume, PVC, etc.)
- matchLabels:
app.kubernetes.io/instance: 'vm-instance-{{ .Application.metadata.name }}'
# HelmRelease (the Cozystack app object)
- matchLabels:
apps.cozystack.io/application.kind: '{{ .Application.kind }}'
apps.cozystack.io/application.name: '{{ .Application.metadata.name }}'
includedResources:
- helmreleases.helm.toolkit.fluxcd.io
- virtualmachines.kubevirt.io
- virtualmachineinstances.kubevirt.io
# Required by kubevirt-velero-plugin for running VMs ("launcher pod must be in backup").
- pods
# Required by kubevirt-velero-plugin requires DV to be in backup of VM, but it excludes in restores
- datavolumes.cdi.kubevirt.io
- persistentvolumeclaims
- configmaps
- secrets
- controllerrevisions.apps
includeClusterResources: false
storageLocation: '{{ .Parameters.backupStorageLocationName }}'
volumeSnapshotLocations:
- '{{ .Parameters.backupStorageLocationName }}'
snapshotVolumes: true
snapshotMoveData: true
ttl: 720h0m0s
itemOperationTimeout: 24h0m0s
EOF
log_success "VMInstance strategy created"
separator
log_step "Creating VMDisk strategy..."
log_command "kubectl apply -f - (Velero strategy: vmdisk-strategy)"
kubectl apply -f - <<'EOF'
apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Velero
metadata:
name: vmdisk-strategy
spec:
template:
restoreSpec:
existingResourcePolicy: update
includedNamespaces:
- '{{ .Application.metadata.namespace }}'
orLabelSelectors:
- matchLabels:
app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}'
- matchLabels:
apps.cozystack.io/application.kind: '{{ .Application.kind }}'
apps.cozystack.io/application.name: '{{ .Application.metadata.name }}'
includedResources:
- helmreleases.helm.toolkit.fluxcd.io
- persistentvolumeclaims
- configmaps
includeClusterResources: false
spec:
includedNamespaces:
- '{{ .Application.metadata.namespace }}'
orLabelSelectors:
- matchLabels:
app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}'
- matchLabels:
apps.cozystack.io/application.kind: '{{ .Application.kind }}'
apps.cozystack.io/application.name: '{{ .Application.metadata.name }}'
includedResources:
- helmreleases.helm.toolkit.fluxcd.io
- persistentvolumeclaims
- configmaps
includeClusterResources: false
storageLocation: '{{ .Parameters.backupStorageLocationName }}'
volumeSnapshotLocations:
- '{{ .Parameters.backupStorageLocationName }}'
snapshotVolumes: true
snapshotMoveData: true
ttl: 720h0m0s
itemOperationTimeout: 24h0m0s
EOF
log_success "VMDisk strategy created"
separator
log_step "Verifying strategies..."
log_command "kubectl get velero.strategy.backups.cozystack.io"
kubectl get velero.strategy.backups.cozystack.io
separator
log_success "Velero strategies are ready"
echo -e "\n${GREEN}${BOLD}Next step:${NC} ./02-create-backupclass.sh"

View file

@ -0,0 +1,52 @@
#!/bin/bash
# Step 02: Create BackupClass that binds strategies to application types
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 2: Create BackupClass"
log_step "Creating BackupClass 'velero'..."
log_info "BackupClass maps application kinds (VMInstance, VMDisk) to their Velero strategies"
log_command "kubectl apply -f - (BackupClass: velero)"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: BackupClass
metadata:
name: velero
spec:
strategies:
- strategyRef:
apiGroup: strategy.backups.cozystack.io
kind: Velero
name: vminstance-strategy
application:
kind: VMInstance
apiGroup: apps.cozystack.io
parameters:
backupStorageLocationName: ${BACKUP_STORAGE_LOCATION}
- strategyRef:
apiGroup: strategy.backups.cozystack.io
kind: Velero
name: vmdisk-strategy
application:
kind: VMDisk
apiGroup: apps.cozystack.io
parameters:
backupStorageLocationName: ${BACKUP_STORAGE_LOCATION}
EOF
log_success "BackupClass created"
separator
log_step "Verifying BackupClass..."
log_command "kubectl get backupclass velero -o yaml"
kubectl get backupclass velero -o yaml
separator
log_success "BackupClass is ready"
echo -e "\n${GREEN}${BOLD}Next step:${NC} ./03-create-vmdisk.sh"

View file

@ -0,0 +1,40 @@
#!/bin/bash
# Step 03: Create a VMDisk with Ubuntu cloud image
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 3: Create VMDisk"
log_step "Creating VMDisk 'ubuntu-source' in namespace $NAMESPACE..."
log_info "This will download the Ubuntu Noble cloud image (~700MB)"
log_command "kubectl apply -f - (VMDisk: ubuntu-source)"
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: VMDisk
metadata:
name: ubuntu-source
namespace: ${NAMESPACE}
spec:
optical: false
source:
http:
url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
storage: 20Gi
storageClass: replicated
EOF
log_success "VMDisk created"
separator
log_step "Verifying VMDisk..."
log_command "kubectl get vmdisk ubuntu-source -n $NAMESPACE"
kubectl get vmdisk ubuntu-source -n "$NAMESPACE"
separator
log_success "VMDisk is ready (image download may still be in progress)"
echo -e "\n${GREEN}${BOLD}Next step:${NC} ./04-create-vminstance.sh"

View file

@ -0,0 +1,44 @@
#!/bin/bash
# Step 04: Create a VMInstance using the previously created VMDisk
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 4: Create VMInstance"
log_step "Creating VMInstance 'test' in namespace $NAMESPACE..."
log_command "kubectl apply -f - (VMInstance: test)"
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: VMInstance
metadata:
name: test
namespace: ${NAMESPACE}
spec:
disks:
- name: ubuntu-source
instanceProfile: ubuntu
instanceType: "u1.medium"
running: true
sshKeys:
#- <paste your ssh public key here>
external: false
externalMethod: PortList
externalPorts:
- 22
EOF
log_success "VMInstance created"
separator
log_step "Verifying VMInstance..."
log_command "kubectl get vminstance test -n $NAMESPACE"
kubectl get vminstance test -n "$NAMESPACE"
separator
log_success "VMInstance is ready"
echo -e "\n${GREEN}${BOLD}Next step:${NC} ./05-create-backupjob.sh"

View file

@ -0,0 +1,50 @@
#!/bin/bash
# Step 05: Create a BackupJob to back up the VMInstance
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 5: Create BackupJob"
log_step "Creating BackupJob 'test-backup' in namespace $NAMESPACE..."
log_info "This triggers a Velero backup of VMInstance 'test' and all its disks"
log_command "kubectl apply -f - (BackupJob: test-backup)"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: BackupJob
metadata:
name: test-backup
namespace: ${NAMESPACE}
spec:
applicationRef:
apiGroup: apps.cozystack.io
kind: VMInstance
name: test
backupClassName: velero
EOF
log_success "BackupJob created"
separator
log_step "Waiting for BackupJob to complete..."
wait_for_field backupjob test-backup '{.status.phase}' Succeeded "$NAMESPACE" 600
separator
log_step "Verifying BackupJob result..."
log_command "kubectl get backupjob test-backup -n $NAMESPACE -o yaml"
kubectl get backupjob test-backup -n "$NAMESPACE" -o wide
separator
log_step "Checking created Backup..."
log_command "kubectl get backups -n $NAMESPACE"
kubectl get backups -n "$NAMESPACE"
separator
log_success "BackupJob completed successfully"
echo -e "\n${GREEN}${BOLD}Next step:${NC} ./06-restore-in-place.sh"

View file

@ -0,0 +1,49 @@
#!/bin/bash
# Step 06: Restore the VMInstance in-place (same namespace, same application)
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "Step 6: Restore VMInstance In-Place"
log_step "Creating RestoreJob 'restore-in-place-test' in namespace $NAMESPACE..."
log_info "In-place restore: the VM will be halted, PVCs renamed, and data restored from backup"
log_command "kubectl apply -f - (RestoreJob: restore-in-place-test)"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
metadata:
name: restore-in-place-test
namespace: ${NAMESPACE}
spec:
backupRef:
name: test-backup
targetApplicationRef:
apiGroup: apps.cozystack.io
kind: VMInstance
name: test
options:
failIfTargetExists: true # if true, restore will fail when the target resource already exists
keepOriginalPVC: true # renames original VMI PVC before restore to `<name>-orig-<hash>`, only for in-place restore
keepOriginalIpAndMac: true # restores original IP and MAC address of VMI via OVN annotations
EOF
log_success "RestoreJob created"
separator
log_step "Waiting for RestoreJob to complete..."
wait_for_field restorejob restore-in-place-test '{.status.phase}' Succeeded "$NAMESPACE" 600
separator
log_step "Verifying RestoreJob result..."
log_command "kubectl get restorejob restore-in-place-test -n $NAMESPACE -o yaml"
kubectl get restorejob restore-in-place-test -n "$NAMESPACE" -o wide
separator
log_success "In-place restore completed successfully"
echo -e "\n${GREEN}${BOLD}Next step:${NC} ./07-restore-to-copy.sh"

View file

@ -0,0 +1,68 @@
#!/bin/bash
# Step 07: Restore the VMInstance to a copy in a different namespace
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
TARGET_NAMESPACE="${TARGET_NAMESPACE:-tenant-root-copy}"
print_header "Step 7: Restore VMInstance to Copy (Cross-Namespace)"
log_info "Restoring to the same namespace with a different app name is not supported"
log_info "due to Velero DataUpload limitations. Cross-namespace restore uses Velero's namespaceMapping."
separator
log_step "Ensuring target namespace '$TARGET_NAMESPACE' exists..."
kubectl create namespace "$TARGET_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
log_success "Namespace '$TARGET_NAMESPACE' is ready"
separator
log_step "Creating RestoreJob 'restore-to-copy-test' in namespace $NAMESPACE..."
log_info "The backup will be restored into namespace '$TARGET_NAMESPACE' using Velero namespaceMapping"
log_command "kubectl apply -f - (RestoreJob: restore-to-copy-test)"
kubectl apply -f - <<EOF
apiVersion: backups.cozystack.io/v1alpha1
kind: RestoreJob
metadata:
name: restore-to-copy-test
namespace: ${NAMESPACE}
spec:
backupRef:
name: test-backup
targetApplicationRef:
apiGroup: apps.cozystack.io
kind: VMInstance
name: test
options: # runtime.RawExtension, typed based on targetApplicationRef and current controller implementation (for additional restore options)
targetNamespace: ${TARGET_NAMESPACE} # when set to a different namespace, triggers cross-namespace restore via Velero namespaceMapping
failIfTargetExists: true # if true, restore will fail when the target resource already exists
keepOriginalPVC: false # renames original VMI PVC before restore to `<name>-orig-<hash>`, only for in-place restore
keepOriginalIpAndMac: false # restores original IP and MAC address of VMI via OVN annotations
EOF
log_success "RestoreJob created"
separator
log_step "Waiting for RestoreJob to complete..."
wait_for_field restorejob restore-to-copy-test '{.status.phase}' Succeeded "$NAMESPACE" 600
separator
log_step "Verifying RestoreJob result..."
log_command "kubectl get restorejob restore-to-copy-test -n $NAMESPACE -o yaml"
kubectl get restorejob restore-to-copy-test -n "$NAMESPACE" -o wide
separator
log_step "Checking resources in target namespace..."
log_command "kubectl get all -n $TARGET_NAMESPACE"
kubectl get all -n "$TARGET_NAMESPACE" 2>/dev/null || log_warning "No resources found in $TARGET_NAMESPACE"
separator
log_success "Cross-namespace restore completed successfully"

70
examples/backups/vmi/cleanup.sh Executable file
View file

@ -0,0 +1,70 @@
#!/bin/bash
# Clean up all resources created by the demo
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
TARGET_NAMESPACE="${TARGET_NAMESPACE:-tenant-root-copy}"
print_header "Cleanup Demo Resources"
log_warning "This script will delete all resources created during the demo"
echo -e "\n${YELLOW}The following will be deleted:${NC}"
echo " - RestoreJobs (restore-in-place-test, restore-to-copy-test)"
echo " - BackupJob (test-backup) and associated Backup"
echo " - VMInstance (test)"
echo " - VMDisk (ubuntu-source)"
echo " - BackupClass (velero)"
echo " - Velero strategies (vminstance-strategy, vmdisk-strategy)"
echo " - Target namespace ($TARGET_NAMESPACE)"
echo ""
read -p "Continue? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
log_info "Cancelled"
exit 0
fi
separator
log_step "Deleting RestoreJobs..."
kubectl delete restorejob restore-to-copy-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-to-copy-test not found"
kubectl delete restorejob restore-in-place-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-in-place-test not found"
separator
log_step "Deleting BackupJob and Backup..."
kubectl delete backupjob test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "BackupJob test-backup not found"
kubectl delete backup test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "Backup test-backup not found"
separator
log_step "Deleting VMInstance..."
kubectl delete vminstance test -n "$NAMESPACE" 2>/dev/null || log_warning "VMInstance test not found"
log_step "Deleting VMDisk..."
kubectl delete vmdisk ubuntu-source -n "$NAMESPACE" 2>/dev/null || log_warning "VMDisk ubuntu-source not found"
separator
log_step "Deleting BackupClass..."
kubectl delete backupclass velero 2>/dev/null || log_warning "BackupClass velero not found"
separator
log_step "Deleting Velero strategies..."
kubectl delete velero.strategy.backups.cozystack.io vminstance-strategy 2>/dev/null || log_warning "Strategy vminstance-strategy not found"
kubectl delete velero.strategy.backups.cozystack.io vmdisk-strategy 2>/dev/null || log_warning "Strategy vmdisk-strategy not found"
separator
log_step "Deleting target namespace..."
kubectl delete namespace "$TARGET_NAMESPACE" 2>/dev/null || log_warning "Namespace $TARGET_NAMESPACE not found"
separator
log_success "Cleanup complete"
log_info "All demo resources have been deleted"
echo -e "\n${GREEN}${BOLD}To re-run the demo:${NC} ./01-create-strategies.sh"

62
examples/backups/vmi/run-all.sh Executable file
View file

@ -0,0 +1,62 @@
#!/bin/bash
# Run the full VMInstance backup/restore demo sequentially
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
print_header "VMInstance Backup & Restore Demo"
log_info "This script will run all demo steps sequentially"
log_info "There will be a pause between steps for observation"
echo ""
SCRIPTS=(
"01-create-strategies.sh"
"02-create-backupclass.sh"
"03-create-vmdisk.sh"
"04-create-vminstance.sh"
"05-create-backupjob.sh"
"06-restore-in-place.sh"
"07-restore-to-copy.sh"
)
echo -e "${CYAN}Demo steps:${NC}"
for i in "${!SCRIPTS[@]}"; do
echo " $((i+1)). ${SCRIPTS[$i]}"
done
echo ""
read -p "Run demo? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
log_info "Cancelled"
exit 0
fi
separator
for script in "${SCRIPTS[@]}"; do
if [[ -x "$SCRIPT_DIR/$script" ]]; then
"$SCRIPT_DIR/$script"
separator
wait_for_enter
else
log_error "Script not found or not executable: $script"
exit 1
fi
done
print_header "Demo Complete"
log_success "All steps completed successfully!"
echo ""
log_info "What was demonstrated:"
echo " 1. Created Velero backup strategies for VMInstance and VMDisk"
echo " 2. Created BackupClass binding strategies to application types"
echo " 3. Provisioned a VMDisk and VMInstance"
echo " 4. Created a backup of the VMInstance via BackupJob"
echo " 5. Restored the VMInstance in-place"
echo " 6. Restored the VMInstance to a copy in a different namespace"
echo ""
log_info "To clean up resources: ./cleanup.sh"

View file

@ -0,0 +1,73 @@
# Scenario: Cluster Administrator Configures Backups
This scenario walks through the cluster-level setup required before users can
back up and restore their virtual machines. A cluster administrator creates
Velero strategies and a BackupClass that binds those strategies to Cozystack
application types.
## Prerequisites
- A running Cozystack cluster with the backup-controller installed.
- Velero deployed in the `cozy-velero` namespace with a configured
BackupStorageLocation (default: `default`).
- `kubectl` access with cluster-admin privileges.
## Steps
### 1. Create Velero Strategies
```bash
./01-create-strategies.sh
```
This creates two cluster-scoped `Velero` strategy objects:
| Strategy | Purpose |
|---|---|
| `vminstance-strategy` | Defines backup and restore templates for `VMInstance` applications. Includes VirtualMachine, PVCs, HelmReleases, pods (required by kubevirt-velero-plugin), and supporting resources. Uses `snapshotMoveData: true` for portable volume snapshots. |
| `vmdisk-strategy` | Defines backup and restore templates for standalone `VMDisk` applications. Covers HelmReleases, PVCs, and ConfigMaps. |
Both strategies use Go templates with `{{ .Application.metadata.name }}`,
`{{ .Application.metadata.namespace }}`, and `{{ .Parameters.backupStorageLocationName }}`
so they can be reused across any application instance.
**Key design details:**
- `orLabelSelectors` scope backups to only the resources belonging to a
specific application, preventing cross-contamination between VMs in the
same namespace.
- The restore spec excludes `datavolumes.cdi.kubevirt.io` to avoid conflicts
when the HelmRelease of VMDisk recreates DataVolumes after restore.
- `existingResourcePolicy: update` allows restoring over existing resources
during in-place restore.
### 2. Create BackupClass
```bash
./02-create-backupclass.sh
```
Creates a `BackupClass` named `velero` that maps application types to their
strategies:
| Application Kind | Strategy | Parameters |
|---|---|---|
| `VMInstance` (`apps.cozystack.io`) | `vminstance-strategy` | `backupStorageLocationName: default` |
| `VMDisk` (`apps.cozystack.io`) | `vmdisk-strategy` | `backupStorageLocationName: default` |
The `backupStorageLocationName` parameter can be overridden via the
`BACKUP_STORAGE_LOCATION` environment variable if your Velero installation
uses a non-default storage location.
## Configuration
| Variable | Default | Description |
|---|---|---|
| `BACKUP_STORAGE_LOCATION` | `default` | Velero BackupStorageLocation name |
## Result
After completing these steps the cluster is ready for users to create backups
of their `VMInstance` and `VMDisk` applications by referencing the `velero`
BackupClass. No further admin action is required for individual backup or
restore operations.

View file

@ -0,0 +1,91 @@
# Scenario: User Creates a VM Backup
This scenario demonstrates how a tenant user provisions a virtual machine and
creates a backup of it. The backup captures the VM definition, its disks, and
network identity so it can be restored later.
## Prerequisites
- Cluster administrator has completed the setup from
[scenario-admin.md](scenario-admin.md) (strategies and BackupClass exist).
- A tenant namespace (default: `tenant-root`).
- `kubectl` access scoped to the tenant namespace.
## Steps
### 1. Create a VMDisk
```bash
./03-create-vmdisk.sh
```
Creates a `VMDisk` named `ubuntu-source` that downloads the Ubuntu Noble cloud
image and provisions a 20Gi replicated PVC. The disk serves as the boot volume
for the virtual machine.
Wait for the image download to complete before proceeding. You can monitor
progress with:
```bash
kubectl get vmdisk ubuntu-source -n tenant-root -w
```
### 2. Create a VMInstance
```bash
./04-create-vminstance.sh
```
Creates a `VMInstance` named `test` that references the `ubuntu-source` disk.
The VM boots with the `ubuntu` instance profile on a `u1.medium` instance type.
Verify the VM is running:
```bash
kubectl get vmi -n tenant-root
```
### 3. Create a BackupJob
```bash
./05-create-backupjob.sh
```
Creates a `BackupJob` named `test-backup` that triggers a full backup of the
`test` VMInstance. The backup controller:
1. Resolves the `velero` BackupClass to find the matching `vminstance-strategy`.
2. Discovers underlying resources (DataVolumes, OVN IP/MAC from the VM pod).
3. Creates a Velero Backup in the `cozy-velero` namespace with label selectors
scoped to this specific VM and its disks.
4. Velero snapshots and moves the volume data to the configured storage
location.
The script waits up to 10 minutes for the BackupJob to reach the `Succeeded`
phase. On completion, a `Backup` object is created in the same namespace
containing:
- `spec.applicationRef` — reference to the backed-up VMInstance.
- `spec.strategyRef` — reference to the Velero strategy used.
- `spec.driverMetadata` — Velero backup name for later restore.
- `status.underlyingResources` — captured DataVolume names and OVN IP/MAC
addresses.
You can inspect the resulting Backup:
```bash
kubectl get backups -n tenant-root
kubectl get backup test-backup -n tenant-root -o yaml
```
## Configuration
| Variable | Default | Description |
|---|---|---|
| `NAMESPACE` | `tenant-root` | Tenant namespace for all resources |
## Result
After completing these steps you have a `Backup` artifact that can be used to
restore the VM either in-place or to a different namespace. See
[scenario-user-restore.md](scenario-user-restore.md) for restore options.

View file

@ -0,0 +1,101 @@
# Scenario: User Restores a VM
This scenario demonstrates two restore methods: in-place restore (rollback the
VM to a previous state) and cross-namespace restore (create a copy of the VM in
a different namespace).
## Prerequisites
- A completed backup from [scenario-user-backup.md](scenario-user-backup.md)
(the `test-backup` Backup object exists in the tenant namespace).
- `kubectl` access scoped to the tenant namespace.
## Method 1: In-Place Restore
```bash
./06-restore-in-place.sh
```
Creates a `RestoreJob` that restores the `test` VMInstance back to the state
captured in the `test-backup` Backup. The `targetApplicationRef` points to the
same application as the original backup.
The backup controller performs the following steps automatically:
1. **Suspends HelmReleases** — prevents Flux from reconciling the VM and its
disks during restore.
2. **Halts the VirtualMachine** — sets `runStrategy: Halted` and waits for the
VirtualMachineInstance (launcher pod) to terminate.
3. **Renames existing PVCs** — moves each PVC to `<name>-orig-<hash>` to
preserve the current disk data as a safety net.
4. **Deletes DataVolumes** — removes DVs so CDI does not recreate PVCs before
Velero restores them.
5. **Creates Velero Restore** — restores resources from the backup with:
- `existingResourcePolicy: update` to overwrite existing objects.
- Resource modifier rules that add `cdi.kubevirt.io/allowClaimAdoption=true`
to restored PVCs so CDI can adopt them.
- OVN IP/MAC annotations to preserve the VM's network identity.
After the Velero Restore completes, the HelmReleases resume and the VM boots
with the restored disk data while retaining its original IP and MAC addresses.
## Method 2: Cross-Namespace Restore (Copy)
```bash
./07-restore-to-copy.sh
```
Creates a `RestoreJob` with `spec.options.targetNamespace` set to a different namespace
(default: `tenant-root-copy`). This creates an independent copy of the VM
without affecting the original.
Key differences from in-place restore:
| Aspect | In-Place | Cross-Namespace |
|---|---|---|
| Source VM affected | Yes (halted, PVCs renamed) | No (untouched) |
| Velero namespaceMapping | Not used | Maps source NS to target NS |
| OVN IP/MAC | Preserved from backup | Skipped (new IP/MAC assigned) |
| Pre-restore preparation | Full (suspend, halt, rename, delete) | Skipped |
The script ensures the target namespace exists before creating the RestoreJob.
Velero's `namespaceMapping` redirects all resources from the source namespace to
the target namespace during restore.
### Important limitation
Restoring to the **same namespace** with a **different application name** is not
supported. Velero's DataUpload always writes volume data to PVCs with the
original name regardless of resource modifiers, which causes conflicts. If you
attempt this, the RestoreJob will fail with an error message suggesting
cross-namespace restore instead.
## Configuration
| Variable | Default | Description |
|---|---|---|
| `NAMESPACE` | `tenant-root` | Source namespace (where the Backup lives) |
| `TARGET_NAMESPACE` | `tenant-root-copy` | Target namespace for cross-namespace restore |
## Verify
After either restore method, verify the VM is running:
```bash
# In-place
kubectl get vmi -n tenant-root
# Cross-namespace copy
kubectl get vmi -n tenant-root-copy
```
## Cleanup
To remove all resources created by the demo:
```bash
./cleanup.sh
```
This deletes RestoreJobs, BackupJobs, Backups, VMInstance, VMDisk, BackupClass,
strategies, and the target namespace.

View file

@ -153,6 +153,23 @@ func (r *RestoreJobReconciler) markRestoreJobFailed(ctx context.Context, restore
return ctrl.Result{}, nil
}
// cleanupResourceModifierConfigMaps deletes resource modifier ConfigMaps owned
// by this RestoreJob. Called on completion (success or failure) to avoid leaking
// ConfigMaps in cozy-velero when RestoreJobs are not immediately deleted.
func (r *RestoreJobReconciler) cleanupResourceModifierConfigMaps(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) {
logger := log.FromContext(ctx)
opts := []client.DeleteAllOfOption{
client.InNamespace(veleroNamespace),
client.MatchingLabels{
backupsv1alpha1.OwningJobNameLabel: restoreJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace,
},
}
if err := r.DeleteAllOf(ctx, &corev1.ConfigMap{}, opts...); err != nil {
logger.Error(err, "failed to clean up resourceModifiers ConfigMap(s)")
}
}
// cleanupVeleroRestore deletes all Velero Restores and resourceModifier
// ConfigMaps owned by this RestoreJob (identified by labels).
func (r *RestoreJobReconciler) cleanupVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) {

View file

@ -75,6 +75,107 @@ func stringPtr(s string) *string {
return &s
}
// boolPtr returns a pointer to a bool value.
func boolPtr(b bool) *bool {
return &b
}
// boolDefault returns the value of a *bool pointer, or the given default if nil.
func boolDefault(p *bool, def bool) bool {
if p != nil {
return *p
}
return def
}
// CommonRestoreOptions contains driver-agnostic restore options shared across
// all application kinds.
type CommonRestoreOptions struct {
// TargetNamespace is the namespace to restore into. When set (and differs
// from the backup namespace), a cross-namespace restore (copy) is performed
// using Velero's namespaceMapping.
TargetNamespace string `json:"targetNamespace,omitempty"`
// FailIfTargetExists makes the restore fail if the target resource already
// exists. Defaults to true when omitted.
FailIfTargetExists *bool `json:"failIfTargetExists,omitempty"`
}
// RestoreOptions is the typed representation of RestoreJob.Spec.Options for the
// Velero driver. The struct is deserialized from runtime.RawExtension and used
// for all application kinds. VMInstance-specific fields (KeepOriginalPVC,
// KeepOriginalIpAndMac) are only effective when the application kind is VMInstance.
type RestoreOptions struct {
CommonRestoreOptions `json:",inline"`
// KeepOriginalPVC renames the original PVC to <name>-orig-<hash> before restore.
// Only effective for in-place VMInstance restore (no targetNamespace). Defaults to true when omitted.
KeepOriginalPVC *bool `json:"keepOriginalPVC,omitempty"`
// KeepOriginalIpAndMac preserves the original IP and MAC address via OVN
// annotations. Only effective for VMInstance restores. Defaults to true when omitted.
KeepOriginalIpAndMac *bool `json:"keepOriginalIpAndMac,omitempty"`
}
// GetFailIfTargetExists returns the effective value (default: true).
func (o *CommonRestoreOptions) GetFailIfTargetExists() bool {
return boolDefault(o.FailIfTargetExists, true)
}
// GetKeepOriginalPVC returns the effective value (default: true).
func (o *RestoreOptions) GetKeepOriginalPVC() bool {
return boolDefault(o.KeepOriginalPVC, true)
}
// GetKeepOriginalIpAndMac returns the effective value (default: true).
func (o *RestoreOptions) GetKeepOriginalIpAndMac() bool {
return boolDefault(o.KeepOriginalIpAndMac, true)
}
// parseRestoreOptions deserializes RestoreJob.Spec.Options into RestoreOptions.
// Returns zero-value RestoreOptions if options is nil.
func parseRestoreOptions(opts *runtime.RawExtension) (RestoreOptions, error) {
var ro RestoreOptions
if opts == nil || len(opts.Raw) == 0 {
return ro, nil
}
if err := json.Unmarshal(opts.Raw, &ro); err != nil {
return ro, fmt.Errorf("failed to parse restore options: %w", err)
}
return ro, nil
}
// restoreTarget holds the resolved target namespace and app identity for a restore operation.
type restoreTarget struct {
Namespace string
AppName string
AppKind string
IsCopy bool // true when targetNamespace differs from backup namespace
IsRenamed bool // true when target app name differs from source app name
}
// resolveRestoreTarget computes the effective restore target from RestoreJob, Backup, and options.
func resolveRestoreTarget(restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, opts RestoreOptions) restoreTarget {
targetNS := backup.Namespace
isCopy := false
if opts.TargetNamespace != "" && opts.TargetNamespace != backup.Namespace {
targetNS = opts.TargetNamespace
isCopy = true
}
targetAppName := backup.Spec.ApplicationRef.Name
if restoreJob.Spec.TargetApplicationRef != nil && restoreJob.Spec.TargetApplicationRef.Name != "" {
targetAppName = restoreJob.Spec.TargetApplicationRef.Name
}
targetAppKind := backup.Spec.ApplicationRef.Kind
if restoreJob.Spec.TargetApplicationRef != nil && restoreJob.Spec.TargetApplicationRef.Kind != "" {
targetAppKind = restoreJob.Spec.TargetApplicationRef.Kind
}
return restoreTarget{
Namespace: targetNS,
AppName: targetAppName,
AppKind: targetAppKind,
IsCopy: isCopy,
IsRenamed: targetAppName != backup.Spec.ApplicationRef.Name,
}
}
// vmInstanceResources contains VM-specific underlying resources discovered during backup.
type vmInstanceResources struct {
DataVolumes []backupsv1alpha1.DataVolumeResource `json:"dataVolumes,omitempty"`
@ -475,6 +576,37 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
logger := getLogger(ctx)
logger.Debug("reconciling Velero strategy restore", "restorejob", restoreJob.Name, "backup", backup.Name)
// Parse restore options from the opaque blob
restoreOpts, err := parseRestoreOptions(restoreJob.Spec.Options)
if err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("invalid restore options: %v", err))
}
target := resolveRestoreTarget(restoreJob, backup, restoreOpts)
logger.Debug("resolved restore target", "targetNS", target.Namespace, "targetApp", target.AppName, "isCopy", target.IsCopy)
// Validate: target namespace must exist for cross-namespace copies
if target.IsCopy {
targetNS := &corev1.Namespace{}
if err := r.Get(ctx, client.ObjectKey{Name: target.Namespace}, targetNS); err != nil {
if errors.IsNotFound(err) {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
"target namespace %q does not exist; create it before requesting a cross-namespace restore",
target.Namespace))
}
return ctrl.Result{}, fmt.Errorf("failed to check target namespace %q: %w", target.Namespace, err)
}
}
// Validate: same-namespace restore with a different app name is not supported
// due to Velero DataUpload always writing to PVCs with the original name.
if !target.IsCopy && target.AppName != backup.Spec.ApplicationRef.Name {
return r.markRestoreJobFailed(ctx, restoreJob,
"restoring to the same namespace with a different application name is not supported "+
"due to Velero DataUpload limitations: data is always uploaded to PVCs with the original name. "+
"Use options.targetNamespace to restore into a different namespace")
}
// Step 1: On first reconcile, set startedAt and phase = Running
if restoreJob.Status.StartedAt == nil {
logger.Debug("setting RestoreJob StartedAt and phase to Running")
@ -525,11 +657,29 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
}
if len(veleroRestoreList.Items) == 0 {
// For copy restores, enforce failIfTargetExists before touching anything.
// In-place restores are excluded: the source application is expected to exist
// and will be halted/overwritten deliberately.
if target.IsCopy && restoreOpts.GetFailIfTargetExists() {
targetHRName := helmReleaseNameForApp(target.AppKind, target.AppName)
exists, err := r.targetHelmReleaseExists(ctx, target.AppKind, targetHRName, target.Namespace)
if err != nil {
logger.Error(err, "failed to check whether target HelmRelease exists")
return ctrl.Result{}, err
}
if exists {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
"target application %q already exists in namespace %q; "+
"set options.failIfTargetExists=false to overwrite",
target.AppName, target.Namespace))
}
}
// Resolve underlying resources once; prefer Backup status, fall back to Velero annotation.
ur := r.resolveUnderlyingResourcesForRestore(ctx, backup, veleroBackupName)
// Pre-restore: graceful shutdown, suspend HRs, rename PVCs
ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur)
// Pre-restore: graceful shutdown, suspend HRs, rename PVCs (skipped for copy)
ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur, target, restoreOpts)
if err != nil {
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("pre-restore preparation failed: %v", err))
}
@ -540,7 +690,7 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
// Create Velero Restore
logger.Debug("Velero Restore not found, creating new one")
if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName, ur); err != nil {
if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName, ur, target, restoreOpts); err != nil {
logger.Error(err, "failed to create Velero Restore")
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to create Velero Restore: %v", err))
}
@ -566,6 +716,18 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
// Step 4: On success
if phase == "Completed" {
// Post-restore: rename resources if target app name differs from source.
// Velero resource modifiers cannot change metadata.name, so we do it after restore.
if target.IsRenamed {
if err := r.postRestoreRename(ctx, restoreJob, backup, target); err != nil {
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("post-restore rename failed: %v", err))
}
}
// Clean up resource modifier ConfigMaps now that the restore is complete.
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
now := metav1.Now()
restoreJob.Status.CompletedAt = &now
restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
@ -579,6 +741,7 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto
// Step 5: On failure
if phase == "Failed" || phase == "PartiallyFailed" {
r.cleanupResourceModifierConfigMaps(ctx, restoreJob)
message := fmt.Sprintf("Velero Restore failed with phase: %s", phase)
if veleroRestore.Status.FailureReason != "" {
message = fmt.Sprintf("%s: %s", message, veleroRestore.Status.FailureReason)
@ -600,6 +763,7 @@ type resourceModifiers struct {
type resourceModifierRule struct {
Conditions resourceModifierConditions `yaml:"conditions"`
MergePatches []mergePatch `yaml:"mergePatches,omitempty"`
Patches []jsonPatch `yaml:"patches,omitempty"`
}
type resourceModifierConditions struct {
@ -612,6 +776,12 @@ type mergePatch struct {
PatchData string `yaml:"patchData"`
}
type jsonPatch struct {
Operation string `yaml:"operation"`
Path string `yaml:"path"`
Value string `yaml:"value,omitempty"`
}
// marshalPatchData marshals an arbitrary object to YAML for use as
// mergePatch.PatchData in Velero resource modifiers.
func marshalPatchData(v interface{}) (string, error) {
@ -627,10 +797,10 @@ func marshalPatchData(v interface{}) (string, error) {
// - PVC adoption: always adds cdi.kubevirt.io/allowClaimAdoption=true to all
// restored PVCs so CDI can adopt them when a HelmRelease of VMDisk recreates a DV.
// - OVN IP/MAC: sets OVN annotations on the VirtualMachine for correct ssh access to restored VM.
func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension) (*corev1.ConfigMap, error) {
func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) (*corev1.ConfigMap, error) {
logger := getLogger(ctx)
targetNS := backup.Namespace
targetNS := target.Namespace
var rules []resourceModifierRule
@ -654,22 +824,17 @@ func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Cont
MergePatches: []mergePatch{{PatchData: pvcPatch}},
})
// OVN IP/MAC annotations on VirtualMachine for correct network identity after restore.
if vmRes := getVMInstanceResources(ur); vmRes != nil && (vmRes.IP != "" || vmRes.MAC != "") {
ovnAnnotations := map[string]string{}
if vmRes.IP != "" {
ovnAnnotations[ovnIPAnnotation] = vmRes.IP
}
if vmRes.MAC != "" {
ovnAnnotations[ovnMACAnnotation] = vmRes.MAC
}
vmPatch, err := marshalPatchData(map[string]interface{}{
// For cross-namespace restore: strip Velero's dynamic PV restore selector and
// volumeName from PVCs so the storage provisioner can dynamically provision new PVs.
// Without this, Velero's CSI PVCAction adds spec.selector with a velero.io/dynamic-pv-restore
// label that prevents dynamic provisioning when PVs are not included in the restore.
// Uses merge patch (null values) instead of JSON Patch remove to avoid RFC 6902
// failures when the fields don't exist on the PVC (e.g. statically provisioned PVCs).
if target.IsCopy {
pvcStripPatch, err := marshalPatchData(map[string]interface{}{
"spec": map[string]interface{}{
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"annotations": ovnAnnotations,
},
},
"selector": nil,
"volumeName": nil,
},
})
if err != nil {
@ -677,14 +842,49 @@ func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Cont
}
rules = append(rules, resourceModifierRule{
Conditions: resourceModifierConditions{
GroupResource: "virtualmachines.kubevirt.io",
GroupResource: "persistentvolumeclaims",
ResourceNameRegex: ".*",
Namespaces: []string{targetNS},
},
MergePatches: []mergePatch{{PatchData: vmPatch}},
MergePatches: []mergePatch{{PatchData: pvcStripPatch}},
})
}
// OVN IP/MAC annotations on VirtualMachine for correct network identity after restore.
// Only applied when keepOriginalIpAndMac is true; for restore-to-copy the copy
// should get new IP/MAC from the network to avoid conflicts.
if opts.GetKeepOriginalIpAndMac() {
if vmRes := getVMInstanceResources(ur); vmRes != nil && (vmRes.IP != "" || vmRes.MAC != "") {
ovnAnnotations := map[string]string{}
if vmRes.IP != "" {
ovnAnnotations[ovnIPAnnotation] = vmRes.IP
}
if vmRes.MAC != "" {
ovnAnnotations[ovnMACAnnotation] = vmRes.MAC
}
vmPatch, err := marshalPatchData(map[string]interface{}{
"spec": map[string]interface{}{
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"annotations": ovnAnnotations,
},
},
},
})
if err != nil {
return nil, err
}
rules = append(rules, resourceModifierRule{
Conditions: resourceModifierConditions{
GroupResource: "virtualmachines.kubevirt.io",
ResourceNameRegex: ".*",
Namespaces: []string{targetNS},
},
MergePatches: []mergePatch{{PatchData: vmPatch}},
})
}
}
rulesYAML, err := yaml.Marshal(resourceModifiers{
Version: "v1",
ResourceModifierRules: rules,
@ -760,6 +960,36 @@ var (
dataVolumeGVR = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"}
)
// helmReleaseNameForApp returns the HelmRelease name for the given application
// kind and name. Returns empty string for unsupported kinds.
func helmReleaseNameForApp(appKind, appName string) string {
switch appKind {
case vmInstanceKind:
return vmNamePrefix + appName
case vmDiskAppKind:
return vmDiskNamePrefix + appName
default:
return ""
}
}
// targetHelmReleaseExists returns true when a HelmRelease with the given name
// already exists in namespace. Returns false for unsupported application kinds
// (those where helmReleaseNameForApp returns "").
func (r *RestoreJobReconciler) targetHelmReleaseExists(ctx context.Context, appKind, hrName, namespace string) (bool, error) {
if hrName == "" {
return false, nil
}
_, err := r.Resource(helmReleaseGVR).Namespace(namespace).Get(ctx, hrName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
return true, nil
}
// shortHash returns the first 4 hex characters of sha256(input).
func shortHash(input string) string {
h := sha256.Sum256([]byte(input))
@ -778,9 +1008,90 @@ func shortHash(input string) string {
// (e.g. restore requested when app was already deleted). Each action emits
// a Kubernetes Event on the RestoreJob for observability.
//
// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes.
// Velero resource modifiers cannot change metadata.name, so this step creates
// a new HelmRelease with the target name and deletes the old one.
// Flux will reconcile the renamed HelmRelease and recreate downstream resources
// (VM, VMI) with the new name.
func (r *RestoreJobReconciler) postRestoreRename(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, target restoreTarget) error {
logger := getLogger(ctx)
sourceAppName := backup.Spec.ApplicationRef.Name
sourceHRName := vmNamePrefix + sourceAppName
targetHRName := vmNamePrefix + target.AppName
logger.Debug("post-restore rename", "from", sourceHRName, "to", targetHRName, "namespace", target.Namespace)
// Get the restored HelmRelease with the original name
hrClient := r.Resource(helmReleaseGVR).Namespace(target.Namespace)
oldHR, err := hrClient.Get(ctx, sourceHRName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
logger.Debug("source HelmRelease not found, skipping rename", "name", sourceHRName)
return nil
}
return fmt.Errorf("failed to get HelmRelease %s: %w", sourceHRName, err)
}
// Create new HelmRelease with the target name
newHR := oldHR.DeepCopy()
newHR.SetName(targetHRName)
newHR.SetResourceVersion("")
newHR.SetUID("")
newHR.SetCreationTimestamp(metav1.Time{})
newHR.SetManagedFields(nil)
newHR.SetGeneration(0)
// Update labels
labels := newHR.GetLabels()
if labels == nil {
labels = map[string]string{}
}
labels[appNameLabel] = target.AppName
labels["app.kubernetes.io/instance"] = targetHRName
labels["helm.toolkit.fluxcd.io/name"] = targetHRName
newHR.SetLabels(labels)
// Remove Velero restore annotations/labels that tie it to the old restore
annotations := newHR.GetAnnotations()
delete(annotations, "velero.io/restore-name")
newHR.SetAnnotations(annotations)
// Clear status so Flux reconciles fresh
unstructured.RemoveNestedField(newHR.Object, "status")
if _, err := hrClient.Create(ctx, newHR, metav1.CreateOptions{}); err != nil {
if errors.IsAlreadyExists(err) {
logger.Debug("target HelmRelease already exists, skipping create", "name", targetHRName)
} else {
return fmt.Errorf("failed to create renamed HelmRelease %s: %w", targetHRName, err)
}
} else {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PostRestoreRename",
fmt.Sprintf("Created renamed HelmRelease %s (from %s)", targetHRName, sourceHRName))
}
// Delete the old HelmRelease
if err := hrClient.Delete(ctx, sourceHRName, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("failed to delete old HelmRelease %s: %w", sourceHRName, err)
}
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PostRestoreRename",
fmt.Sprintf("Deleted old HelmRelease %s", sourceHRName))
logger.Debug("post-restore rename complete", "from", sourceHRName, "to", targetHRName)
return nil
}
// Returns true when preparation is complete and the Velero Restore can be created.
// Returns false (with a requeue) when still waiting for VM shutdown.
func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension) (ready bool, result ctrl.Result, err error) {
func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) (ready bool, result ctrl.Result, err error) {
// For restore-to-copy, skip all source-app preparation.
// The source application remains untouched; we're restoring a copy into another namespace.
if target.IsCopy {
r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore",
"Restore to copy: skipping source application preparation")
return true, ctrl.Result{}, nil
}
ns := restoreJob.Namespace
appName := backup.Spec.ApplicationRef.Name
appKind := backup.Spec.ApplicationRef.Kind
@ -828,7 +1139,8 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob
// --- Step 3: Rename PVCs to <name>-orig-<hash> ---
// Must happen BEFORE deleting DVs: the PVC has an ownerReference to the DV,
// so deleting the DV first would cascade-delete the PVC via garbage collection.
if vmRes != nil {
// Only when keepOriginalPVC is true (default for in-place restore).
if opts.GetKeepOriginalPVC() && vmRes != nil {
for _, dv := range vmRes.DataVolumes {
if err := r.renamePVC(ctx, restoreJob, ns, dv.DataVolumeName, dv.DataVolumeName+origSuffix); err != nil {
r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore",
@ -908,8 +1220,14 @@ func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmNam
}
// deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after rename.
// Uses Orphan propagation to avoid cascade-deleting the PVC that the DV owns via
// ownerReference. Without this, keepOriginalPVC=false would silently destroy the
// original PVC through garbage collection instead of leaving it for Velero to overwrite.
func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) error {
err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{})
orphan := metav1.DeletePropagationOrphan
err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{
PropagationPolicy: &orphan,
})
if err != nil && !errors.IsNotFound(err) {
return err
}
@ -1004,10 +1322,16 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup
}
// createVeleroRestore creates a Velero Restore resource.
func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Velero, veleroBackupName string, ur *runtime.RawExtension) error {
func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Velero, veleroBackupName string, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) error {
logger := getLogger(ctx)
logger.Debug("createVeleroRestore called", "strategy", strategy.Name, "veleroBackupName", veleroBackupName)
logger.Debug("createVeleroRestore called", "strategy", strategy.Name, "veleroBackupName", veleroBackupName, "targetNS", target.Namespace, "isCopy", target.IsCopy)
// For restore template context, always use the source (backup) namespace and app name.
// The strategy template uses includedNamespaces and orLabelSelectors to select
// resources from the backup tarball, which are stored under the source namespace
// and labeled with the source app name.
// Velero's namespaceMapping handles redirecting to the target namespace;
// resource modifiers handle renaming when the target app name differs.
templateContext := map[string]interface{}{
"Application": map[string]interface{}{
"metadata": map[string]interface{}{
@ -1034,6 +1358,16 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ
// Set the backupName in the spec (required by Velero)
veleroRestoreSpec.BackupName = veleroBackupName
// For restore-to-copy, set Velero namespaceMapping to redirect resources
// from the source namespace to the target namespace.
if target.IsCopy {
if veleroRestoreSpec.NamespaceMapping == nil {
veleroRestoreSpec.NamespaceMapping = make(map[string]string)
}
veleroRestoreSpec.NamespaceMapping[backup.Namespace] = target.Namespace
logger.Debug("set namespaceMapping on Velero Restore", "from", backup.Namespace, "to", target.Namespace)
}
// Match backup: add OR selectors for each underlying VMDisk so restore applies the same
// scope as the intended backup (see createVeleroBackup).
if vmRes := getVMInstanceResources(ur); vmRes != nil {
@ -1051,7 +1385,7 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ
}
// Create resourceModifiers ConfigMap
resourceModifierCM, err := r.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur)
resourceModifierCM, err := r.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts)
if err != nil {
return fmt.Errorf("failed to create resourceModifiers ConfigMap: %w", err)
}

File diff suppressed because it is too large Load diff

View file

@ -59,6 +59,12 @@ spec:
type: string
type: object
x-kubernetes-map-type: atomic
options:
description: |-
Options is a driver-specific blob of restore options, typed based on
targetApplicationRef and the current controller implementation.
type: object
x-kubernetes-preserve-unknown-fields: true
targetApplicationRef:
description: |-
TargetApplicationRef refers to the application into which the backup

View file

@ -46,10 +46,10 @@ rules:
- apiGroups: ["cdi.kubevirt.io"]
resources: ["datavolumes"]
verbs: ["delete"]
# HelmReleases: pre-restore suspends HRs to prevent Flux interference
# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR and deletes old
- apiGroups: ["helm.toolkit.fluxcd.io"]
resources: ["helmreleases"]
verbs: ["get", "update"]
verbs: ["get", "create", "update", "delete"]
# PVCs and PVs: pre-restore renames PVCs (delete old, create new) and patches PV reclaim policy
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
@ -65,6 +65,11 @@ rules:
- apiGroups: ["velero.io"]
resources: ["deletebackuprequests"]
verbs: ["create", "get", "list", "watch"]
# Namespaces: validate target namespace exists for cross-namespace restore
# controller-runtime cache requires list/watch for any resource accessed via r.Get()
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list", "watch"]
# Events from Recorder.Event() calls
- apiGroups: [""]
resources: ["events"]