[backups] Add templating of velero backups

## What this PR does

This patch narrows the scope of the Velero backup strategy controller to
simply template Velero Backups according to the application being backed
up and the template in the strategy. Creating storage locations is now
out of scope.

### Release note

```release-note
[backups] Implement templating for Velero backups and remove creation of
backup storage locations and volume snapshot locations.
```

Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>
This commit is contained in:
Timofei Larkin 2026-01-04 12:33:08 +03:00
parent bfafcaa3ab
commit f4228ffc20
10 changed files with 268 additions and 171 deletions

View file

@ -6,6 +6,7 @@
package v1alpha1
import (
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
@ -47,7 +48,15 @@ type VeleroList struct {
}
// VeleroSpec specifies the desired strategy for backing up with Velero.
type VeleroSpec struct{}
type VeleroSpec struct {
Template VeleroTemplate `json:"template"`
}
// VeleroTemplate describes the data a backup.velero.io should have when
// templated from a Velero backup strategy.
type VeleroTemplate struct {
Spec velerov1.BackupSpec `json:"spec"`
}
type VeleroStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`

View file

@ -127,7 +127,7 @@ func (in *Velero) DeepCopyInto(out *Velero) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
@ -184,6 +184,7 @@ func (in *VeleroList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *VeleroSpec) DeepCopyInto(out *VeleroSpec) {
*out = *in
in.Template.DeepCopyInto(&out.Template)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VeleroSpec.
@ -217,3 +218,19 @@ func (in *VeleroStatus) DeepCopy() *VeleroStatus {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *VeleroTemplate) DeepCopyInto(out *VeleroTemplate) {
*out = *in
in.Spec.DeepCopyInto(&out.Spec)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VeleroTemplate.
func (in *VeleroTemplate) DeepCopy() *VeleroTemplate {
if in == nil {
return nil
}
out := new(VeleroTemplate)
in.DeepCopyInto(out)
return out
}

View file

@ -21,6 +21,11 @@ func init() {
})
}
const (
OwningJobNameLabel = thisGroup + "/owned-by.BackupJobName"
OwningJobNamespaceLabel = thisGroup + "/owned-by.BackupJobNamespace"
)
// BackupJobPhase represents the lifecycle phase of a BackupJob.
type BackupJobPhase string

View file

@ -25,8 +25,13 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
)
const (
thisGroup = "backups.cozystack.io"
thisVersion = "v1alpha1"
)
var (
GroupVersion = schema.GroupVersion{Group: "backups.cozystack.io", Version: "v1alpha1"}
GroupVersion = schema.GroupVersion{Group: thisGroup, Version: thisVersion}
SchemeBuilder = runtime.NewSchemeBuilder(addGroupVersion)
AddToScheme = SchemeBuilder.AddToScheme
)

View file

@ -25,10 +25,6 @@ EOF
SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' bucket-test-credentials.json)
BUCKET_NAME=$(jq -r '.spec.bucketName' bucket-test-credentials.json)
# Tmp for test via s3 endpoint from bucket credentials
S3_ENDPOINT=$(jq -r '.spec.secretS3.endpoint' bucket-test-credentials.json)
echo "# S3 endopint = $S3_ENDPOINT" >&3
# Start port-forwarding
bash -c 'timeout 100s kubectl port-forward service/seaweedfs-s3 -n tenant-root 8333:8333 > /dev/null 2>&1 &'
@ -36,7 +32,7 @@ EOF
timeout 30 sh -ec 'until nc -z localhost 8333; do sleep 1; done'
# Set up MinIO alias with error handling
mc alias set local $S3_ENDPOINT $ACCESS_KEY $SECRET_KEY --insecure
mc alias set local https://localhost:8333 $ACCESS_KEY $SECRET_KEY --insecure
# Upload file to bucket
mc cp bucket-test-credentials.json $BUCKET_NAME/bucket-test-credentials.json

View file

@ -2,13 +2,18 @@ package backupcontroller
import (
"context"
"net/http"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/log"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
@ -19,6 +24,8 @@ import (
// Velero.strategy.backups.cozystack.io objects.
type BackupJobReconciler struct {
client.Client
dynamic.Interface
meta.RESTMapper
Scheme *runtime.Scheme
Recorder record.EventRecorder
}
@ -68,6 +75,18 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// SetupWithManager registers our controller with the Manager and sets up watches.
func (r *BackupJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
cfg := mgr.GetConfig()
var err error
if r.Interface, err = dynamic.NewForConfig(cfg); err != nil {
return err
}
var h *http.Client
if h, err = rest.HTTPClientFor(cfg); err != nil {
return err
}
if r.RESTMapper, err = apiutil.NewDynamicRESTMapper(cfg, h); err != nil {
return err
}
return ctrl.NewControllerManagedBy(mgr).
For(&backupsv1alpha1.BackupJob{}).
Complete(r)

View file

@ -9,6 +9,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
@ -18,6 +19,7 @@ import (
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/template"
"github.com/go-logr/logr"
velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@ -87,31 +89,6 @@ func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1a
return ctrl.Result{}, nil
}
// For now implemented backup logic for apps.cozystack.io VirtualMachine only
logger.Debug("validating BackupJob spec",
"applicationRef", fmt.Sprintf("%s/%s", j.Spec.ApplicationRef.APIGroup, j.Spec.ApplicationRef.Kind),
"storageRef", fmt.Sprintf("%s/%s", j.Spec.StorageRef.APIGroup, j.Spec.StorageRef.Kind))
if j.Spec.ApplicationRef.Kind != "VirtualMachine" {
logger.Error(nil, "Unsupported application type", "kind", j.Spec.ApplicationRef.Kind)
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Unsupported application type: %s", j.Spec.ApplicationRef.Kind))
}
if j.Spec.ApplicationRef.APIGroup == nil || *j.Spec.ApplicationRef.APIGroup != "apps.cozystack.io" {
logger.Error(nil, "Unsupported application APIGroup", "apiGroup", j.Spec.ApplicationRef.APIGroup, "expected", "apps.cozystack.io")
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Unsupported application APIGroup: %v, expected apps.cozystack.io", j.Spec.ApplicationRef.APIGroup))
}
if j.Spec.StorageRef.Kind != "Bucket" {
logger.Error(nil, "Unsupported storage type", "kind", j.Spec.StorageRef.Kind)
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Unsupported storage type: %s", j.Spec.StorageRef.Kind))
}
if j.Spec.StorageRef.APIGroup == nil || *j.Spec.StorageRef.APIGroup != "apps.cozystack.io" {
logger.Error(nil, "Unsupported storage APIGroup", "apiGroup", j.Spec.StorageRef.APIGroup, "expected", "apps.cozystack.io")
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Unsupported storage APIGroup: %v, expected apps.cozystack.io", j.Spec.StorageRef.APIGroup))
}
logger.Debug("BackupJob spec validation passed")
// Step 1: On first reconcile, set startedAt (but not phase yet - phase will be set after backup creation)
logger.Debug("checking BackupJob status", "startedAt", j.Status.StartedAt, "phase", j.Status.Phase)
if j.Status.StartedAt == nil {
@ -148,37 +125,53 @@ func (r *BackupJobReconciler) reconcileVelero(ctx context.Context, j *backupsv1a
logger.Error(nil, "StartedAt is nil after status update, this should not happen")
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
timestamp := j.Status.StartedAt.Time.Format("2006-01-02-15-04-05")
veleroBackupName := fmt.Sprintf("%s-%s-%s", j.Namespace, j.Name, timestamp)
logger.Debug("checking for existing Velero Backup", "veleroBackupName", veleroBackupName, "namespace", veleroNamespace)
veleroBackup := &velerov1.Backup{}
veleroBackupKey := client.ObjectKey{Namespace: veleroNamespace, Name: veleroBackupName}
logger.Debug("checking for existing Velero Backup", "namespace", veleroNamespace)
veleroBackupList := &velerov1.BackupList{}
opts := []client.ListOption{
client.InNamespace(veleroNamespace),
client.MatchingLabels{
backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace,
backupsv1alpha1.OwningJobNameLabel: j.Name,
},
}
if err := r.Get(ctx, veleroBackupKey, veleroBackup); err != nil {
if errors.IsNotFound(err) {
// Create Velero Backup
logger.Debug("Velero Backup not found, creating new one", "veleroBackupName", veleroBackupName)
if err := r.createVeleroBackup(ctx, j, veleroBackupName); err != nil {
logger.Error(err, "failed to create Velero Backup")
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Velero Backup: %v", err))
}
// After successful Velero backup creation, set phase to Running
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
logger.Debug("setting BackupJob phase to Running after successful Velero backup creation")
j.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob phase to Running")
return ctrl.Result{}, err
}
}
logger.Debug("created Velero Backup, requeuing", "veleroBackupName", veleroBackupName)
// Requeue to check status
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
if err := r.List(ctx, veleroBackupList, opts...); err != nil {
logger.Error(err, "failed to get Velero Backup")
return ctrl.Result{}, err
}
logger.Debug("found existing Velero Backup", "veleroBackupName", veleroBackupName, "phase", veleroBackup.Status.Phase)
if len(veleroBackupList.Items) == 0 {
// Create Velero Backup
logger.Debug("Velero Backup not found, creating new one")
if err := r.createVeleroBackup(ctx, j, veleroStrategy); err != nil {
logger.Error(err, "failed to create Velero Backup")
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Velero Backup: %v", err))
}
// After successful Velero backup creation, set phase to Running
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseRunning {
logger.Debug("setting BackupJob phase to Running after successful Velero backup creation")
j.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob phase to Running")
return ctrl.Result{}, err
}
}
logger.Debug("created Velero Backup, requeuing")
// Requeue to check status
return ctrl.Result{RequeueAfter: defaultRequeueAfter}, nil
}
if len(veleroBackupList.Items) > 1 {
logger.Error(fmt.Errorf("too many Velero backups for BackupJob"), "found more than one Velero Backup referencing a single BackupJob as owner")
j.Status.Phase = backupsv1alpha1.BackupJobPhaseFailed
if err := r.Status().Update(ctx, j); err != nil {
logger.Error(err, "failed to update BackupJob status")
}
return ctrl.Result{}, nil
}
veleroBackup := veleroBackupList.Items[0].DeepCopy()
logger.Debug("found existing Velero Backup", "phase", veleroBackup.Status.Phase)
// If Velero backup exists but phase is not Running, set it to Running
// This handles the case where the backup was created but phase wasn't set yet
@ -519,126 +512,43 @@ func (r *BackupJobReconciler) markBackupJobFailed(ctx context.Context, backupJob
return ctrl.Result{}, nil
}
func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, name string) error {
func (r *BackupJobReconciler) createVeleroBackup(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Velero) error {
logger := getLogger(ctx)
logger.Debug("createVeleroBackup called", "backupJob", backupJob.Name, "veleroBackupName", name)
logger.Debug("createVeleroBackup called", "strategy", strategy.Name)
// Resolve StorageRef to get S3 credentials if it's a Bucket
// Prefix with namespace to avoid conflicts in cozy-velero namespace
var locationName string = fmt.Sprintf("%s-%s", backupJob.Namespace, backupJob.Name)
if backupJob.Spec.StorageRef.Kind == "Bucket" {
logger.Debug("resolving Bucket storageRef", "storageRef", backupJob.Spec.StorageRef.Name)
creds, err := r.resolveBucketStorageRef(ctx, backupJob.Spec.StorageRef, backupJob.Namespace)
if err != nil {
logger.Error(err, "failed to resolve Bucket storageRef")
return fmt.Errorf("failed to resolve Bucket storageRef: %w", err)
}
logger.Debug("discovered S3 credentials from Bucket storageRef",
"bucketName", creds.BucketName,
"endpoint", creds.Endpoint,
"region", creds.Region)
if err := r.createS3CredsForVelero(ctx, backupJob, creds); err != nil {
return fmt.Errorf("failed to create or update Velero credentials secret: %w", err)
}
// Dynamically create a Velero BackupStorageLocation and VolumeSnapshotLocation using discovered credentials.
// BackupStorageLocation manifest
// Note: Cannot set owner reference for cross-namespace resources
bsl := &velerov1.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Name: locationName,
Namespace: veleroNamespace,
},
Spec: velerov1.BackupStorageLocationSpec{
Provider: "aws",
StorageType: velerov1.StorageType{
ObjectStorage: &velerov1.ObjectStorageLocation{
Bucket: creds.BucketName,
},
},
Config: map[string]string{
"checksumAlgorithm": "",
"profile": "default",
"s3ForcePathStyle": "true",
"s3Url": creds.Endpoint,
},
Credential: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: storageS3SecretName(backupJob.Namespace, backupJob.Name),
},
Key: "cloud",
},
},
}
// Create or update the BackupStorageLocation
if err := r.createBackupStorageLocation(ctx, bsl); err != nil {
logger.Error(err, "failed to create or update BackupStorageLocation for Velero")
r.Recorder.Event(backupJob, corev1.EventTypeWarning, "BackupStorageLocationCreationFailed",
fmt.Sprintf("Failed to create or update BackupStorageLocation %s/%s: %v", veleroNamespace, locationName, err))
return fmt.Errorf("failed to create or update Velero BackupStorageLocation: %w", err)
}
r.Recorder.Event(backupJob, corev1.EventTypeNormal, "BackupStorageLocationCreated",
fmt.Sprintf("Created or updated BackupStorageLocation %s/%s", veleroNamespace, locationName))
// VolumeSnapshotLocation manifest
// Note: Cannot set owner reference for cross-namespace resources
vsl := &velerov1.VolumeSnapshotLocation{
ObjectMeta: metav1.ObjectMeta{
Name: locationName,
Namespace: veleroNamespace,
},
Spec: velerov1.VolumeSnapshotLocationSpec{
Provider: "aws",
Credential: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{
Name: storageS3SecretName(backupJob.Namespace, backupJob.Name),
},
Key: "cloud",
},
Config: map[string]string{
"region": creds.Region,
"profile": "default",
},
},
}
// Create or update the VolumeSnapshotLocation
if err := r.createVolumeSnapshotLocation(ctx, vsl); err != nil {
logger.Error(err, "failed to create or update VolumeSnapshotLocation for Velero")
r.Recorder.Event(backupJob, corev1.EventTypeWarning, "VolumeSnapshotLocationCreationFailed",
fmt.Sprintf("Failed to create or update VolumeSnapshotLocation %s/%s: %v", veleroNamespace, locationName, err))
return fmt.Errorf("failed to create or update Velero VolumeSnapshotLocation: %w", err)
}
r.Recorder.Event(backupJob, corev1.EventTypeNormal, "VolumeSnapshotLocationCreated",
fmt.Sprintf("Created or updated VolumeSnapshotLocation %s/%s", veleroNamespace, locationName))
mapping, err := r.RESTMapping(schema.GroupKind{Group: *backupJob.Spec.ApplicationRef.APIGroup, Kind: backupJob.Spec.ApplicationRef.Kind})
if err != nil {
return err
}
ns := backupJob.Namespace
if mapping.Scope.Name() != meta.RESTScopeNameNamespace {
ns = ""
}
app, err := r.Resource(mapping.Resource).Namespace(ns).Get(ctx, backupJob.Spec.ApplicationRef.Name, metav1.GetOptions{})
if err != nil {
return err
}
// Create a Velero Backup (velero.io/v1) using typed object
// Now implemented only for backup of VirtualMachine resources
// Note: Cannot set owner reference for cross-namespace resources
veleroBackupSpec, err := template.Template(&strategy.Spec.Template.Spec, app.Object)
if err != nil {
return err
}
veleroBackup := &velerov1.Backup{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: veleroNamespace,
},
Spec: velerov1.BackupSpec{
IncludedNamespaces: []string{backupJob.Namespace},
IncludedResources: []string{"virtualmachines.kubevirt.io"},
SnapshotVolumes: boolPtr(true),
StorageLocation: locationName,
VolumeSnapshotLocations: []string{locationName},
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/instance": virtualMachinePrefix + backupJob.Spec.ApplicationRef.Name,
},
GenerateName: fmt.Sprintf("%s.%s-", backupJob.Namespace, backupJob.Name),
Namespace: veleroNamespace,
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: backupJob.Name,
backupsv1alpha1.OwningJobNamespaceLabel: backupJob.Namespace,
},
},
Spec: *veleroBackupSpec,
}
name := veleroBackup.GenerateName
if err := r.Create(ctx, veleroBackup); err != nil {
if veleroBackup.Name != "" {
name = veleroBackup.Name
}
logger.Error(err, "failed to create Velero Backup", "name", veleroBackup.Name)
r.Recorder.Event(backupJob, corev1.EventTypeWarning, "VeleroBackupCreationFailed",
fmt.Sprintf("Failed to create Velero Backup %s/%s: %v", veleroNamespace, name, err))
@ -675,7 +585,7 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-backup", backupJob.Name),
Name: fmt.Sprintf("%s", backupJob.Name),
Namespace: backupJob.Namespace,
OwnerReferences: []metav1.OwnerReference{
{

View file

@ -0,0 +1,68 @@
package template
import (
"bytes"
"encoding/json"
tmpl "text/template"
)
func Template[T any](obj *T, templateContext map[string]any) (*T, error) {
b, err := json.Marshal(obj)
if err != nil {
return nil, err
}
var unstructured any
err = json.Unmarshal(b, &unstructured)
if err != nil {
return nil, err
}
templateFunc := func(in string) string {
out, err := template(in, templateContext)
if err != nil {
return in
}
return out
}
unstructured = mapAtStrings(unstructured, templateFunc)
b, err = json.Marshal(unstructured)
if err != nil {
return nil, err
}
var out T
err = json.Unmarshal(b, &out)
if err != nil {
return nil, err
}
return &out, nil
}
func mapAtStrings(v any, f func(string) string) any {
switch x := v.(type) {
case map[string]any:
for k, val := range x {
x[k] = mapAtStrings(val, f)
}
return x
case []any:
for i, val := range x {
x[i] = mapAtStrings(val, f)
}
return x
case string:
return f(x)
default:
return v
}
}
func template(in string, templateContext map[string]any) (string, error) {
tpl, err := tmpl.New("this").Parse(in)
if err != nil {
return "", err
}
var buf bytes.Buffer
if err := tpl.Execute(&buf, templateContext); err != nil {
return "", err
}
return buf.String(), nil
}

View file

@ -0,0 +1,68 @@
package template
import (
"encoding/json"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestTemplate_PodTemplateSpec(t *testing.T) {
original := corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: "my-pod",
Labels: map[string]string{
"app": "demo",
},
Annotations: map[string]string{
"note": "hello",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "{{ .Release.Name }}",
Image: "nginx:1.21",
Args: []string{"--flag={{ .Values.value }}"},
Env: []corev1.EnvVar{
{
Name: "FOO",
Value: "{{ .Release.Namespace }}",
},
},
},
},
},
}
templateContext := map[string]any{
"Release": map[string]any{
"Name": "foo",
"Namespace": "notdefault",
},
"Values": map[string]any{
"value": 3,
},
}
reference := *original.DeepCopy()
reference.Spec.Containers[0].Name = "foo"
reference.Spec.Containers[0].Args[0] = "--flag=3"
reference.Spec.Containers[0].Env[0].Value = "notdefault"
got, err := Template(&original, templateContext)
if err != nil {
t.Fatalf("Template returned error: %v", err)
}
b1, err := json.Marshal(reference)
t.Logf("reference:\n%s", string(b1))
if err != nil {
t.Fatalf("failed to marshal reference value: %v", err)
}
b2, err := json.Marshal(got)
t.Logf("got:\n%s", string(b2))
if err != nil {
t.Fatalf("failed to marshal transformed value: %v", err)
}
if string(b1) != string(b2) {
t.Fatalf("transformed value not equal to reference value, expected: %s, got: %s", string(b1), string(b2))
}
}