Compare commits
No commits in common. "feat/kubernetes-persistent-kubelet-storage" and "main" have entirely different histories.
feat/kuber
...
main
12 changed files with 65 additions and 599 deletions
|
|
@ -22,7 +22,7 @@ type ConfigSpec struct {
|
|||
// +kubebuilder:default:="replicated"
|
||||
StorageClass string `json:"storageClass"`
|
||||
// Worker nodes configuration map.
|
||||
// +kubebuilder:default:={"md0":{"diskSize":"20Gi","gpus":{},"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":{"ingress-nginx"},"storageClass":""}}
|
||||
// +kubebuilder:default:={"md0":{"ephemeralStorage":"20Gi","gpus":{},"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":{"ingress-nginx"}}}
|
||||
NodeGroups map[string]NodeGroup `json:"nodeGroups,omitempty"`
|
||||
// Kubernetes major.minor version to deploy
|
||||
// +kubebuilder:default:="v1.35"
|
||||
|
|
@ -218,9 +218,9 @@ type MonitoringAgentsAddon struct {
|
|||
}
|
||||
|
||||
type NodeGroup struct {
|
||||
// Persistent disk size for kubelet and containerd data.
|
||||
// Ephemeral storage size.
|
||||
// +kubebuilder:default:="20Gi"
|
||||
DiskSize resource.Quantity `json:"diskSize"`
|
||||
EphemeralStorage resource.Quantity `json:"ephemeralStorage"`
|
||||
// List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
|
||||
Gpus []GPU `json:"gpus,omitempty"`
|
||||
// Virtual machine instance type.
|
||||
|
|
@ -236,8 +236,6 @@ type NodeGroup struct {
|
|||
Resources Resources `json:"resources"`
|
||||
// List of node roles.
|
||||
Roles []string `json:"roles,omitempty"`
|
||||
// StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true).
|
||||
StorageClass string `json:"storageClass,omitempty"`
|
||||
}
|
||||
|
||||
type Resources struct {
|
||||
|
|
|
|||
|
|
@ -365,7 +365,7 @@ func (in *MonitoringAgentsAddon) DeepCopy() *MonitoringAgentsAddon {
|
|||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NodeGroup) DeepCopyInto(out *NodeGroup) {
|
||||
*out = *in
|
||||
out.DiskSize = in.DiskSize.DeepCopy()
|
||||
out.EphemeralStorage = in.EphemeralStorage.DeepCopy()
|
||||
if in.Gpus != nil {
|
||||
in, out := &in.Gpus, &out.Gpus
|
||||
*out = make([]GPU, len(*in))
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -309,8 +309,6 @@ k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA=
|
|||
k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0=
|
||||
k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY=
|
||||
k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8=
|
||||
k8s.io/code-generator v0.34.1 h1:WpphT26E+j7tEgIUfFr5WfbJrktCGzB3JoJH9149xYc=
|
||||
k8s.io/code-generator v0.34.1/go.mod h1:DeWjekbDnJWRwpw3s0Jat87c+e0TgkxoR4ar608yqvg=
|
||||
k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A=
|
||||
k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0=
|
||||
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ spec:
|
|||
host: ""
|
||||
nodeGroups:
|
||||
md0:
|
||||
diskSize: 20Gi
|
||||
ephemeralStorage: 20Gi
|
||||
gpus: []
|
||||
instanceType: u1.medium
|
||||
maxReplicas: 10
|
||||
|
|
|
|||
|
|
@ -63,9 +63,6 @@ Deploying it involves the following components:
|
|||
|
||||
- **Worker Nodes**: Virtual Machines are provisioned to serve as worker nodes using KubeVirt.
|
||||
These nodes are configured to join the tenant Kubernetes cluster, enabling the deployment and management of workloads.
|
||||
Worker node disks automatically detect and match the underlying volume's block size
|
||||
(`blockSize.matchVolume`) to ensure compatibility with storage backends that use
|
||||
non-512-byte sectors, such as LINSTOR DRBD with 4Kn drives.
|
||||
|
||||
- **Cluster API**: Cozystack is using the [Kubernetes Cluster API](https://cluster-api.sigs.k8s.io/) to provision the components of a cluster.
|
||||
|
||||
|
|
@ -83,10 +80,6 @@ See the reference for components utilized in this service:
|
|||
- [github.com/kubernetes-sigs/cluster-api-provider-kubevirt](https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt)
|
||||
- [github.com/kubevirt/csi-driver](https://github.com/kubevirt/csi-driver)
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
- **`ephemeralStorage` renamed to `diskSize`**: The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize` to better reflect its purpose (persistent disk for kubelet and containerd data). There is no backward-compatibility fallback; users MUST update their configurations to use `diskSize` instead of `ephemeralStorage`. If `ephemeralStorage` is still present in values, Helm template rendering will fail with an error directing you to use `diskSize`. When upgrading the CRD directly (bypassing Helm), the unrecognized field is silently dropped and kubelet storage reverts to the default 20Gi. Existing VMs will be automatically rolling-updated via CAPI when the new values are applied. State persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh PVC.
|
||||
|
||||
## Parameters
|
||||
|
||||
### Common Parameters
|
||||
|
|
@ -98,22 +91,21 @@ See the reference for components utilized in this service:
|
|||
|
||||
### Application-specific Parameters
|
||||
|
||||
| Name | Description | Type | Value |
|
||||
| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------- |
|
||||
| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` |
|
||||
| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` |
|
||||
| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` |
|
||||
| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` |
|
||||
| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` |
|
||||
| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). | `string` | `""` |
|
||||
| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` |
|
||||
| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` |
|
||||
| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` |
|
||||
| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` |
|
||||
| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` |
|
||||
| `host` | External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty. | `string` | `""` |
|
||||
| Name | Description | Type | Value |
|
||||
| ----------------------------------- | ---------------------------------------------------------------------------------------------- | ------------------- | ----------- |
|
||||
| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` |
|
||||
| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` |
|
||||
| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` |
|
||||
| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` |
|
||||
| `nodeGroups[name].ephemeralStorage` | Ephemeral storage size. | `quantity` | `20Gi` |
|
||||
| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` |
|
||||
| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` |
|
||||
| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` |
|
||||
| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` |
|
||||
| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` |
|
||||
| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` |
|
||||
| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` |
|
||||
| `host` | External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty. | `string` | `""` |
|
||||
|
||||
|
||||
### Cluster Addons
|
||||
|
|
|
|||
|
|
@ -31,25 +31,6 @@ spec:
|
|||
name: {{ . }}
|
||||
{{- end }}
|
||||
runStrategy: Always
|
||||
dataVolumeTemplates:
|
||||
- metadata:
|
||||
name: disk-kubelet
|
||||
annotations:
|
||||
{{/* Populator creates intermediate PVC + PV rebind that causes ClaimMisbound with LINSTOR CSI */}}
|
||||
cdi.kubevirt.io/storage.usePopulator: "false"
|
||||
spec:
|
||||
{{- if hasKey .group "ephemeralStorage" }}
|
||||
{{- fail (printf "nodeGroup %q: ephemeralStorage is no longer supported. Rename it to diskSize. See README.md for migration instructions." .groupName) }}
|
||||
{{- end }}
|
||||
source:
|
||||
blank: {}
|
||||
storage:
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .group.diskSize | default "20Gi" | quote }}
|
||||
{{- with .group.storageClass }}
|
||||
storageClassName: {{ . | quote }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
|
|
@ -89,13 +70,10 @@ spec:
|
|||
disk:
|
||||
bus: virtio
|
||||
pciAddress: 0000:07:00.0
|
||||
- name: disk-kubelet
|
||||
- name: ephemeral
|
||||
disk:
|
||||
bus: virtio
|
||||
pciAddress: 0000:08:00.0
|
||||
blockSize:
|
||||
matchVolume:
|
||||
enabled: true
|
||||
interfaces:
|
||||
- name: default
|
||||
bridge: {}
|
||||
|
|
@ -104,16 +82,15 @@ spec:
|
|||
guest: {{ .group.resources.memory }}
|
||||
{{- end }}
|
||||
resources:
|
||||
{{/* 2Gi is enough: persistent DataVolumes are not counted as pod ephemeral storage */}}
|
||||
{{- include "cozy-lib.resources.sanitize" (list (dict "ephemeral-storage" "2Gi") $) | nindent 14 }}
|
||||
{{- include "cozy-lib.resources.sanitize" (list (dict "ephemeral-storage" .group.ephemeralStorage) $) | nindent 14 }}
|
||||
evictionStrategy: External
|
||||
volumes:
|
||||
- name: system
|
||||
containerDisk:
|
||||
image: "{{ $.Files.Get (printf "images/ubuntu-container-disk-%s.tag" $.Values.version) | trim }}"
|
||||
- name: disk-kubelet
|
||||
dataVolume:
|
||||
name: disk-kubelet
|
||||
- name: ephemeral
|
||||
emptyDisk:
|
||||
capacity: {{ .group.ephemeralStorage | default "20Gi" }}
|
||||
networks:
|
||||
- name: default
|
||||
pod: {}
|
||||
|
|
@ -271,36 +248,35 @@ spec:
|
|||
chmod 0755 /root/kubeadm
|
||||
if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi
|
||||
if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi
|
||||
{{- $sec := lookup "v1" "Secret" $.Release.Namespace (printf "%s-patch-containerd" $.Release.Name) }}
|
||||
{{- if $sec }}
|
||||
{{- range $key, $_ := $sec.data }}
|
||||
- path: /etc/containerd/certs.d/{{ trimSuffix ".toml" $key }}/hosts.toml
|
||||
contentFrom:
|
||||
secret:
|
||||
name: {{ $.Release.Name }}-patch-containerd
|
||||
key: {{ $key }}
|
||||
permissions: "0400"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
diskSetup:
|
||||
filesystems:
|
||||
- device: /dev/vdb
|
||||
filesystem: xfs
|
||||
label: persistent
|
||||
label: ephemeral
|
||||
partition: "none"
|
||||
{{/* overwrite must stay false: XFS on /dev/vdb is created on first boot; subsequent reboots of the same VM must reuse the existing filesystem via LABEL=persistent */}}
|
||||
overwrite: false
|
||||
mounts:
|
||||
- ["LABEL=persistent", "/persistent"]
|
||||
- ["/persistent/kubelet", "/var/lib/kubelet", "none", "bind,nofail"]
|
||||
- ["/persistent/containerd", "/var/lib/containerd", "none", "bind,nofail"]
|
||||
- ["LABEL=ephemeral", "/ephemeral"]
|
||||
- ["/ephemeral/kubelet", "/var/lib/kubelet", "none", "bind,nofail"]
|
||||
- ["/ephemeral/containerd", "/var/lib/containerd", "none", "bind,nofail"]
|
||||
{{- $sec := lookup "v1" "Secret" $.Release.Namespace (printf "%s-patch-containerd" $.Release.Name) }}
|
||||
{{- if $sec }}
|
||||
files:
|
||||
{{- range $key, $_ := $sec.data }}
|
||||
- path: /etc/containerd/certs.d/{{ trimSuffix ".toml" $key }}/hosts.toml
|
||||
contentFrom:
|
||||
secret:
|
||||
name: {{ $.Release.Name }}-patch-containerd
|
||||
key: {{ $key }}
|
||||
permissions: "0400"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
preKubeadmCommands:
|
||||
- KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true
|
||||
- sed -i 's|root:x:|root::|' /etc/passwd
|
||||
- systemctl stop containerd.service
|
||||
- mkdir -p /persistent/kubelet /persistent/containerd /var/lib/kubelet /var/lib/containerd
|
||||
- mountpoint -q /var/lib/kubelet || mount -o bind /persistent/kubelet /var/lib/kubelet
|
||||
- mountpoint -q /var/lib/containerd || mount -o bind /persistent/containerd /var/lib/containerd
|
||||
- mkdir -p /ephemeral/kubelet /ephemeral/containerd
|
||||
- mount -o bind /ephemeral/kubelet /var/lib/kubelet
|
||||
- mount -o bind /ephemeral/containerd /var/lib/containerd
|
||||
- sudo sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/,/^\[/ s|^\(\s*config_path\s*=\s*\).*|\1"/etc/containerd/certs.d"|' /etc/containerd/config.toml
|
||||
- systemctl start containerd.service
|
||||
joinConfiguration:
|
||||
|
|
|
|||
|
|
@ -1,446 +0,0 @@
|
|||
suite: cluster.yaml template tests
|
||||
|
||||
templates:
|
||||
- templates/cluster.yaml
|
||||
|
||||
values:
|
||||
- values/common.yaml
|
||||
|
||||
# The KubevirtMachineTemplate is document index 5 when rendering with one
|
||||
# nodeGroup (md0). Documents before it: Cluster(0), KamajiControlPlane(1),
|
||||
# WorkloadMonitor-cp(2), KubevirtCluster(3), KubeadmConfigTemplate-md0(4).
|
||||
# The named template "kubevirtmachinetemplate" is inlined inside document 5
|
||||
# under spec.template.spec.
|
||||
#
|
||||
# Note: instanceType must be set to "" to bypass the lookup() call that
|
||||
# validates VirtualMachineClusterInstancetype existence against a live cluster.
|
||||
|
||||
tests:
|
||||
###############################################
|
||||
# dataVolumeTemplates — default disk size #
|
||||
###############################################
|
||||
|
||||
- it: falls back to 20Gi when diskSize is not set
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- isKind:
|
||||
of: KubevirtMachineTemplate
|
||||
documentIndex: 5
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.resources.requests.storage
|
||||
value: "20Gi"
|
||||
documentIndex: 5
|
||||
|
||||
- it: renders dataVolumeTemplate with default 20Gi storage
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.diskSize: 20Gi
|
||||
asserts:
|
||||
- isKind:
|
||||
of: KubevirtMachineTemplate
|
||||
documentIndex: 5
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].metadata.name
|
||||
value: disk-kubelet
|
||||
documentIndex: 5
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.blank
|
||||
value: {}
|
||||
documentIndex: 5
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.resources.requests.storage
|
||||
value: 20Gi
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# dataVolumeTemplates — custom disk size #
|
||||
###############################################
|
||||
|
||||
- it: renders dataVolumeTemplate with custom 50Gi storage
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.diskSize: 50Gi
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.resources.requests.storage
|
||||
value: 50Gi
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# storageClassName omitted when empty #
|
||||
###############################################
|
||||
|
||||
- it: omits storageClassName when storageClass is empty
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.storageClass: ""
|
||||
asserts:
|
||||
- notExists:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# storageClassName present when set #
|
||||
###############################################
|
||||
|
||||
- it: sets storageClassName when storageClass is provided
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.storageClass: "my-storage"
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName
|
||||
value: my-storage
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# ephemeral-storage is always 2Gi #
|
||||
###############################################
|
||||
|
||||
- it: hardcodes ephemeral-storage to 2Gi with default diskSize
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.diskSize: 20Gi
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.resources.limits.ephemeral-storage
|
||||
value: 2Gi
|
||||
documentIndex: 5
|
||||
|
||||
- it: hardcodes ephemeral-storage to 2Gi even with large diskSize
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.diskSize: 100Gi
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.resources.limits.ephemeral-storage
|
||||
value: 2Gi
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# disk-kubelet name consistency #
|
||||
###############################################
|
||||
|
||||
- it: uses disk-kubelet name consistently across dataVolumeTemplates, volumes, and disks
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
# dataVolumeTemplates name
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].metadata.name
|
||||
value: disk-kubelet
|
||||
documentIndex: 5
|
||||
# disks name
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.devices.disks[1].name
|
||||
value: disk-kubelet
|
||||
documentIndex: 5
|
||||
# volumes name
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.template.spec.volumes[1].name
|
||||
value: disk-kubelet
|
||||
documentIndex: 5
|
||||
# volumes dataVolume reference
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.template.spec.volumes[1].dataVolume.name
|
||||
value: disk-kubelet
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# CDI populator annotation #
|
||||
###############################################
|
||||
|
||||
- it: includes CDI populator annotation on dataVolumeTemplate
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].metadata.annotations["cdi.kubevirt.io/storage.usePopulator"]
|
||||
value: "false"
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# multiple node groups — independent storage #
|
||||
###############################################
|
||||
|
||||
# With two nodeGroups (md0, md1) the range loop emits per-group documents
|
||||
# alphabetically. Documents before the loop: Cluster(0),
|
||||
# KamajiControlPlane(1), WorkloadMonitor-cp(2), KubevirtCluster(3).
|
||||
# Each group adds: KubeadmConfigTemplate, KubevirtMachineTemplate,
|
||||
# MachineDeployment, MachineHealthCheck, WorkloadMonitor.
|
||||
# md0 KubevirtMachineTemplate → index 5, md1 → index 10.
|
||||
|
||||
- it: renders independent dataVolumeTemplates per node group
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.minReplicas: 0
|
||||
nodeGroups.md0.maxReplicas: 10
|
||||
nodeGroups.md0.instanceType: ""
|
||||
nodeGroups.md0.roles[0]: worker
|
||||
nodeGroups.md0.resources.cpu: 2
|
||||
nodeGroups.md0.resources.memory: 4Gi
|
||||
nodeGroups.md0.diskSize: 20Gi
|
||||
nodeGroups.md1.minReplicas: 0
|
||||
nodeGroups.md1.maxReplicas: 5
|
||||
nodeGroups.md1.instanceType: ""
|
||||
nodeGroups.md1.roles[0]: worker
|
||||
nodeGroups.md1.resources.cpu: 4
|
||||
nodeGroups.md1.resources.memory: 8Gi
|
||||
nodeGroups.md1.diskSize: 50Gi
|
||||
nodeGroups.md1.storageClass: my-storage-class
|
||||
asserts:
|
||||
# --- md0: KubevirtMachineTemplate at document index 5 ---
|
||||
- isKind:
|
||||
of: KubevirtMachineTemplate
|
||||
documentIndex: 5
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.resources.requests.storage
|
||||
value: 20Gi
|
||||
documentIndex: 5
|
||||
- notExists:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName
|
||||
documentIndex: 5
|
||||
# --- md1: KubevirtMachineTemplate at document index 10 ---
|
||||
- isKind:
|
||||
of: KubevirtMachineTemplate
|
||||
documentIndex: 10
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.resources.requests.storage
|
||||
value: 50Gi
|
||||
documentIndex: 10
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.storage.storageClassName
|
||||
value: my-storage-class
|
||||
documentIndex: 10
|
||||
|
||||
###############################################
|
||||
# KubeadmConfigTemplate — persistent storage #
|
||||
###############################################
|
||||
|
||||
# KubeadmConfigTemplate is document index 4 when rendering with one
|
||||
# nodeGroup (md0).
|
||||
|
||||
- it: configures persistent filesystem as xfs with label persistent
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- isKind:
|
||||
of: KubeadmConfigTemplate
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.diskSetup.filesystems[0].label
|
||||
value: persistent
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.diskSetup.filesystems[0].overwrite
|
||||
value: false
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.diskSetup.filesystems[0].filesystem
|
||||
value: xfs
|
||||
documentIndex: 4
|
||||
|
||||
- it: configures persistent filesystem device and partition
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.diskSetup.filesystems[0].device
|
||||
value: /dev/vdb
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.diskSetup.filesystems[0].partition
|
||||
value: "none"
|
||||
documentIndex: 4
|
||||
|
||||
- it: mounts persistent volume and bind-mounts kubelet and containerd
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.mounts[0]
|
||||
value:
|
||||
- LABEL=persistent
|
||||
- /persistent
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.mounts[1]
|
||||
value:
|
||||
- /persistent/kubelet
|
||||
- /var/lib/kubelet
|
||||
- none
|
||||
- bind,nofail
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.mounts[2]
|
||||
value:
|
||||
- /persistent/containerd
|
||||
- /var/lib/containerd
|
||||
- none
|
||||
- bind,nofail
|
||||
documentIndex: 4
|
||||
|
||||
- it: includes mountpoint guards in preKubeadmCommands
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.preKubeadmCommands
|
||||
content: "mountpoint -q /var/lib/kubelet || mount -o bind /persistent/kubelet /var/lib/kubelet"
|
||||
documentIndex: 4
|
||||
- contains:
|
||||
path: spec.template.spec.preKubeadmCommands
|
||||
content: "mountpoint -q /var/lib/containerd || mount -o bind /persistent/containerd /var/lib/containerd"
|
||||
documentIndex: 4
|
||||
|
||||
- it: creates persistent directories in preKubeadmCommands
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.preKubeadmCommands
|
||||
content: "mkdir -p /persistent/kubelet /persistent/containerd /var/lib/kubelet /var/lib/containerd"
|
||||
documentIndex: 4
|
||||
|
||||
###############################################
|
||||
# disk-kubelet matchVolume blockSize #
|
||||
###############################################
|
||||
|
||||
- it: sets matchVolume blockSize on disk-kubelet
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.devices.disks[1].blockSize.matchVolume.enabled
|
||||
value: true
|
||||
documentIndex: 5
|
||||
|
||||
###############################################
|
||||
# ephemeralStorage migration guard #
|
||||
###############################################
|
||||
|
||||
- it: fails when ephemeralStorage is set instead of diskSize
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.ephemeralStorage: 50Gi
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'nodeGroup "md0": ephemeralStorage is no longer supported. Rename it to diskSize. See README.md for migration instructions.'
|
||||
|
||||
- it: fails when ephemeralStorage is set to empty string
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.ephemeralStorage: ""
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'nodeGroup "md0": ephemeralStorage is no longer supported. Rename it to diskSize. See README.md for migration instructions.'
|
||||
|
||||
- it: fails when both ephemeralStorage and diskSize are set
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
set:
|
||||
nodeGroups.md0.ephemeralStorage: 50Gi
|
||||
nodeGroups.md0.diskSize: 50Gi
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'nodeGroup "md0": ephemeralStorage is no longer supported. Rename it to diskSize. See README.md for migration instructions.'
|
||||
|
||||
###############################################
|
||||
# KubeadmConfigTemplate — files block #
|
||||
###############################################
|
||||
|
||||
# When the containerd patch Secret does not exist (lookup returns nil),
|
||||
# only the update-k8s.sh entry must appear in the files list.
|
||||
|
||||
- it: includes update-k8s.sh in KubeadmConfigTemplate files
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- isKind:
|
||||
of: KubeadmConfigTemplate
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.files[0].path
|
||||
value: /usr/bin/update-k8s.sh
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.files[0].permissions
|
||||
value: "0755"
|
||||
documentIndex: 4
|
||||
- matchRegex:
|
||||
path: spec.template.spec.files[0].content
|
||||
pattern: "KUBELET_VERSION"
|
||||
documentIndex: 4
|
||||
- matchRegex:
|
||||
path: spec.template.spec.files[0].content
|
||||
pattern: "curl -fsSL"
|
||||
documentIndex: 4
|
||||
|
||||
- it: has exactly one file entry when no containerd patch Secret exists
|
||||
release:
|
||||
name: test-k8s
|
||||
namespace: tenant-test
|
||||
asserts:
|
||||
- isKind:
|
||||
of: KubeadmConfigTemplate
|
||||
documentIndex: 4
|
||||
- equal:
|
||||
path: spec.template.spec.files
|
||||
value:
|
||||
- path: /usr/bin/update-k8s.sh
|
||||
owner: root:root
|
||||
permissions: "0755"
|
||||
content: |
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Expected to be passed in via preKubeadmCommands
|
||||
: "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}"
|
||||
|
||||
ARCH="$(uname -m)"
|
||||
case "${ARCH}" in
|
||||
x86_64) ARCH=amd64 ;;
|
||||
aarch64) ARCH=arm64 ;;
|
||||
esac
|
||||
|
||||
# Use your internal mirror here for real-world use.
|
||||
BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}"
|
||||
|
||||
echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..."
|
||||
curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet
|
||||
curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm
|
||||
chmod 0755 /root/kubelet
|
||||
chmod 0755 /root/kubeadm
|
||||
if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi
|
||||
if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi
|
||||
documentIndex: 4
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
_cluster: {}
|
||||
_namespace:
|
||||
etcd: cozy-etcd
|
||||
ingress: nginx
|
||||
host: example.com
|
||||
version: "v1.32"
|
||||
controlPlane:
|
||||
replicas: 2
|
||||
apiServer:
|
||||
resources: {}
|
||||
resourcesPreset: "large"
|
||||
controllerManager:
|
||||
resources: {}
|
||||
resourcesPreset: "micro"
|
||||
scheduler:
|
||||
resources: {}
|
||||
resourcesPreset: "micro"
|
||||
konnectivity:
|
||||
server:
|
||||
resources: {}
|
||||
resourcesPreset: "micro"
|
||||
# diskSize and storageClass are intentionally omitted from md0 to verify template
|
||||
# fallback defaults: diskSize defaults to "20Gi", storageClass falls back to cluster default.
|
||||
nodeGroups:
|
||||
md0:
|
||||
minReplicas: 0
|
||||
maxReplicas: 10
|
||||
instanceType: ""
|
||||
roles:
|
||||
- worker
|
||||
resources:
|
||||
cpu: 2
|
||||
memory: 4Gi
|
||||
gpus: []
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
"type": "object",
|
||||
"default": {
|
||||
"md0": {
|
||||
"diskSize": "20Gi",
|
||||
"ephemeralStorage": "20Gi",
|
||||
"gpus": [],
|
||||
"instanceType": "u1.medium",
|
||||
"maxReplicas": 10,
|
||||
|
|
@ -20,22 +20,21 @@
|
|||
"resources": {},
|
||||
"roles": [
|
||||
"ingress-nginx"
|
||||
],
|
||||
"storageClass": ""
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"diskSize",
|
||||
"ephemeralStorage",
|
||||
"instanceType",
|
||||
"maxReplicas",
|
||||
"minReplicas",
|
||||
"resources"
|
||||
],
|
||||
"properties": {
|
||||
"diskSize": {
|
||||
"description": "Persistent disk size for kubelet and containerd data.",
|
||||
"ephemeralStorage": {
|
||||
"description": "Ephemeral storage size.",
|
||||
"default": "20Gi",
|
||||
"pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
|
||||
"anyOf": [
|
||||
|
|
@ -117,10 +116,6 @@
|
|||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"storageClass": {
|
||||
"description": "StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true).",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ storageClass: replicated
|
|||
## @field {int} minReplicas=0 - Minimum number of replicas.
|
||||
## @field {int} maxReplicas=10 - Maximum number of replicas.
|
||||
## @field {string} instanceType="u1.medium" - Virtual machine instance type.
|
||||
## @field {quantity} diskSize="20Gi" - Persistent disk size for kubelet and containerd data.
|
||||
## @field {string} [storageClass] - StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true).
|
||||
## @field {quantity} ephemeralStorage="20Gi" - Ephemeral storage size.
|
||||
## @field {[]string} roles - List of node roles.
|
||||
## @field {Resources} resources - CPU and memory resources for each worker node.
|
||||
## @field {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
|
||||
|
|
@ -41,8 +40,7 @@ nodeGroups:
|
|||
minReplicas: 0
|
||||
maxReplicas: 10
|
||||
instanceType: "u1.medium"
|
||||
diskSize: 20Gi
|
||||
storageClass: ""
|
||||
ephemeralStorage: 20Gi
|
||||
roles:
|
||||
- ingress-nginx
|
||||
resources: {}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -36,10 +36,9 @@ var _ = Describe("defaultLikeKubernetes", func() {
|
|||
|
||||
Expect(ng).To(HaveKeyWithValue("minReplicas", BeNumerically("==", 3)))
|
||||
Expect(ng).To(HaveKeyWithValue("instanceType", "u1.medium"))
|
||||
Expect(ng).To(HaveKeyWithValue("storageClass", ""))
|
||||
Expect(ng["roles"]).To(ConsistOf("ingress-nginx"))
|
||||
|
||||
Expect(ng).NotTo(HaveKey("diskSize"))
|
||||
Expect(ng).NotTo(HaveKey("ephemeralStorage"))
|
||||
Expect(ng).NotTo(HaveKey("maxReplicas"))
|
||||
Expect(ng).NotTo(HaveKey("resources"))
|
||||
})
|
||||
|
|
@ -57,8 +56,6 @@ var _ = Describe("defaultLikeKubernetes", func() {
|
|||
})
|
||||
})
|
||||
|
||||
// buildTestSchema returns a minimal schema that exercises defaultLikeKubernetes.
|
||||
// Defaults intentionally differ from values.schema.json — do not use this as a reference for production defaults.
|
||||
func buildTestSchema() *apischema.Structural {
|
||||
instanceType := apischema.Structural{
|
||||
Generic: apischema.Generic{
|
||||
|
|
@ -78,15 +75,9 @@ func buildTestSchema() *apischema.Structural {
|
|||
minReplicas := apischema.Structural{
|
||||
Generic: apischema.Generic{Type: "integer"},
|
||||
}
|
||||
diskSize := apischema.Structural{
|
||||
ephemeralStorage := apischema.Structural{
|
||||
Generic: apischema.Generic{Type: "string"},
|
||||
}
|
||||
storageClass := apischema.Structural{
|
||||
Generic: apischema.Generic{
|
||||
Type: "string",
|
||||
Default: apischema.JSON{Object: ""},
|
||||
},
|
||||
}
|
||||
maxReplicas := apischema.Structural{
|
||||
Generic: apischema.Generic{Type: "integer"},
|
||||
}
|
||||
|
|
@ -98,13 +89,12 @@ func buildTestSchema() *apischema.Structural {
|
|||
nodeGroupsValue := &apischema.Structural{
|
||||
Generic: apischema.Generic{Type: "object"},
|
||||
Properties: map[string]apischema.Structural{
|
||||
"instanceType": instanceType,
|
||||
"roles": roles,
|
||||
"minReplicas": minReplicas,
|
||||
"diskSize": diskSize,
|
||||
"storageClass": storageClass,
|
||||
"maxReplicas": maxReplicas,
|
||||
"resources": resources,
|
||||
"instanceType": instanceType,
|
||||
"roles": roles,
|
||||
"minReplicas": minReplicas,
|
||||
"ephemeralStorage": ephemeralStorage,
|
||||
"maxReplicas": maxReplicas,
|
||||
"resources": resources,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -113,11 +103,10 @@ func buildTestSchema() *apischema.Structural {
|
|||
Type: "object",
|
||||
Default: apischema.JSON{Object: map[string]any{
|
||||
"md0": map[string]any{
|
||||
"diskSize": "20Gi",
|
||||
"storageClass": "",
|
||||
"maxReplicas": 10,
|
||||
"minReplicas": 0,
|
||||
"resources": map[string]any{},
|
||||
"ephemeralStorage": "20Gi",
|
||||
"maxReplicas": 10,
|
||||
"minReplicas": 0,
|
||||
"resources": map[string]any{},
|
||||
},
|
||||
}},
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue