diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 1a119259..fefdab82 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -labels: 'bug' +labels: 'kind/bug' assignees: '' --- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b9475846..044dd6a0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,10 @@ + +# v1.3.1 (2026-04-28) + +Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero. + +## Security + +* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524) + +## Features + +* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505) + +## Fixes + +* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467) + +* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511) + +* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491) + +## Build + +* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518) + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats new file mode 100644 index 00000000..a4b98b00 --- /dev/null +++ b/hack/admin-kubeconfig-invariant.bats @@ -0,0 +1,145 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Chart-wide invariant for packages/apps/kubernetes: +# +# Every Deployment in this chart that mounts -admin-kubeconfig as a +# Secret volume MUST: +# - declare that volume optional: true (so kubelet does not FailedMount +# while Kamaji is still provisioning the Secret), AND +# - include the wait-for-kubeconfig init container (so the pod becomes +# Ready only after Kamaji publishes the Secret). +# +# The per-template unittests in packages/apps/kubernetes/tests/ lock in +# today's three Deployments (cluster-autoscaler, kccm, csi controller) by +# name. This invariant is stricter: any future Deployment added to this +# chart that mounts the same Secret but forgets the guard will fail here. +# +# Requires: helm, yq (mikefarah v4+), jq. All three are available on the +# project's CI runners and on the maintainer workstation. +# ----------------------------------------------------------------------------- + +@test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" { + values_file="packages/apps/kubernetes/tests/values-ci.yaml" + [ -f "$values_file" ] + + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values "$values_file" \ + 2>/dev/null > "$tmp/rendered.yaml" + + # yq streams one JSON object per input document. jq -s slurps the stream + # into an array so we can treat all Deployments as a single collection. + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s --raw-output ' + map(select(.kind == "Deployment")) | + map({ + name: .metadata.name, + volumes: (.spec.template.spec.volumes // []), + initNames: ((.spec.template.spec.initContainers // []) | map(.name)), + }) | + map( + .name as $n | + .initNames as $ins | + (.volumes[] | select(.secret.secretName | test("-admin-kubeconfig$")?)) + | { + name: $n, + optional: (.secret.optional == true), + hasInit: ($ins | index("wait-for-kubeconfig") != null), + } + ) + ' > "$tmp/summary.json" + + # At least one Deployment must match; if a refactor removes every + # admin-kubeconfig volume from this chart, the test must be updated + # deliberately rather than silently passing. + matched=$(jq 'length' "$tmp/summary.json") + [ "$matched" -ge 1 ] + + offenders=$(jq --raw-output '.[] | select(.optional != true or .hasInit != true) | .name' "$tmp/summary.json") + + if [ -n "$offenders" ]; then + echo "Deployments mounting *-admin-kubeconfig without optional:true + wait-for-kubeconfig init:" >&2 + echo "$offenders" >&2 + echo "Full summary:" >&2 + cat "$tmp/summary.json" >&2 + exit 1 + fi + + echo "Invariant holds for $matched Deployment(s)" +} + +@test "chart emits zero admin-kubeconfig Deployments when tenant has no etcd DataStore" { + # Without a DataStore (parent Tenant has not populated _namespace.etcd yet) + # the control-plane-side Deployments must NOT render at all. If they did, + # the wait-for-kubeconfig init would CrashLoopBackOff indefinitely - there + # would be no KamajiControlPlane to provision the Secret - consuming the + # HelmRelease wait budget and triggering exactly the remediation cycle the + # rest of this chart tries to avoid. This test renders the whole chart + # with etcd empty and asserts no Deployment references the admin-kubeconfig + # Secret. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ + 2>/dev/null > "$tmp/rendered.yaml" + + matched=$( + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s ' + map(select(.kind == "Deployment")) | + map(select( + (.spec.template.spec.volumes // []) + | any(.secret.secretName | test("-admin-kubeconfig$")?) + )) | + length + ' + ) + + if [ "$matched" -ne 0 ]; then + echo "Expected zero Deployments mounting *-admin-kubeconfig when etcd is empty, got $matched:" >&2 + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s 'map(select(.kind == "Deployment") | .metadata.name)' >&2 + exit 1 + fi + + echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)" +} + +@test "chart emits zero admin-kubeconfig HelmReleases when tenant has no etcd DataStore" { + # Same principle as the Deployment variant above, extended to every child + # HelmRelease (cilium, coredns, csi, cert-manager, ...). They reference + # *-admin-kubeconfig via kubeConfig.secretRef and would otherwise sit in + # NotReady forever on an etcd-less tenant, polluting the HelmRelease list + # the operator sees and contradicting the "awaiting-etcd beacon only" + # contract of the soft-skip path. + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + helm template invariant packages/apps/kubernetes \ + --namespace tenant-root \ + --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ + 2>/dev/null > "$tmp/rendered.yaml" + + matched=$( + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s ' + map(select(.kind == "HelmRelease")) | + map(select(.spec.kubeConfig.secretRef.name | test("-admin-kubeconfig$")?)) | + length + ' + ) + + if [ "$matched" -ne 0 ]; then + echo "Expected zero HelmReleases referencing *-admin-kubeconfig when etcd is empty, got $matched:" >&2 + yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ + | jq -s 'map(select(.kind == "HelmRelease") | .metadata.name)' >&2 + exit 1 + fi + + echo "No admin-kubeconfig HelmReleases rendered for empty etcd (as expected)" +} diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats new file mode 100644 index 00000000..bae6785e --- /dev/null +++ b/hack/check-gpu-recording-rules.bats @@ -0,0 +1,206 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Cross-validation between GPU recording rules, the dashboards that consume +# them, and the DCGM Exporter metric set the cluster actually scrapes. Catches: +# +# 1. dangling references — a dashboard query mentions a recording rule name +# that doesn't exist in gpu-recording.rules.yaml. This is the bug the +# pre-merge review caught: gpu-efficiency.json shipped panels keyed on +# pod:tensor_saturation:avg5m without the rule being defined, so the +# panel showed "No data" everywhere. +# +# 2. typos in rule names — same bug class, manifested as a single-character +# difference between rule and reference. +# +# 3. undeclared DCGM metrics — a dashboard query or recording rule mentions +# a DCGM_FI_* metric that is neither in the upstream default CSV nor in +# the project's custom CSV (dcgm-custom-metrics.yaml), meaning DCGM +# Exporter would never emit it and the panel silently shows "No data". +# Example regression: gpu-fleet.json shipped a TDP panel referencing +# DCGM_FI_DEV_POWER_MGMT_LIMIT before the custom CSV declared it. +# +# Scope: only dashboards listed in packages/system/monitoring/dashboards-infra.list +# under the "gpu/" prefix (i.e. shipped to production), not every JSON file in +# dashboards/gpu/. Untracked drafts stay out of scope on purpose — adding one +# to dashboards-infra.list is what brings it under the test. +# +# Reverse direction (rule defined but never consumed) is intentionally NOT +# enforced: some rules exist for ad-hoc PromQL or upcoming dashboards. Treat +# unused rules as an editorial concern, not a regression. +# +# Title syntax constraints from cozytest.sh's awk parser: +# - Titles delimited by ASCII double quotes; embedded quotes truncate. +# - Only [A-Za-z0-9] from the title survives into the function name; titles +# differing only in punctuation collapse to the same function. +# +# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats +# ----------------------------------------------------------------------------- + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" +DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" +DASHBOARDS_DIR="$REPO_ROOT/dashboards" +DCGM_DEFAULT_CSV="$REPO_ROOT/hack/dcgm-default-counters.csv" +DCGM_CUSTOM_CSV="$REPO_ROOT/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml" + +# Extract the set of "- record: NAME" entries from the rules YAML. +# Outputs one rule name per line, sorted and deduplicated. +extract_rules() { + awk '/^[[:space:]]*-[[:space:]]*record:[[:space:]]/ { + sub(/^[[:space:]]*-[[:space:]]*record:[[:space:]]*/, "") + sub(/[[:space:]]*$/, "") + print + }' "$RULES_FILE" | sort -u +} + +# Extract the set of recording-rule references from a dashboard JSON. +# A recording-rule reference is matched by the pattern +# :(:)+ +# where each is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*), +# kube-state-metrics (kube_*) and similar uppercase / single-word metric +# names do not match because the leading segment must be lowercase and the +# whole expression must contain at least two ':' characters. +extract_refs() { + json_file=$1 + # Prometheus convention allows 2-segment rule names (level:metric); this + # regex is tuned to the 3+ segment convention used in this repo + # (level:metric:op — e.g. cluster:gpu_count:total). Update if future + # rules use 2 segments, otherwise they will be silently skipped. + grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u +} + +# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json" +list_tracked_gpu_dashboards() { + awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" +} + +# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles +# both the upstream-style default CSV (unindented) and the ConfigMap-style +# custom CSV (YAML-indented). A declaration line starts — after any leading +# whitespace — with "DCGM_FI_," ; comment lines begin with "#" and are +# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions +# are required. +extract_csv_metrics() { + file=$1 + awk ' + { + line = $0 + sub(/^[[:space:]]+/, "", line) + if (line ~ /^#/) next + if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) { + print substr(line, RSTART, RLENGTH) + } + } + ' "$file" | sort -u +} + +# Extract the set of DCGM_FI_* metric references from a text file (dashboard +# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited +# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_ +# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps +# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that +# appear in comments. +extract_dcgm_refs() { + file=$1 + grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u +} + +@test "every recording rule reference in tracked GPU dashboards has a matching record" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + extract_rules > "$TMP/rules.txt" + [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; } + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + if [ ! -f "$dashboard" ]; then + echo "ERROR: dashboard listed but file missing: $dashboard" >&2 + failed=1 + continue + fi + + extract_refs "$dashboard" > "$TMP/refs.txt" + # comm -23: lines unique to refs.txt (referenced but not defined) + # Both inputs must be sorted; extract_* helpers already sort. + comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt" + if [ -s "$TMP/missing.txt" ]; then + echo "ERROR: $dashboard_rel references undefined recording rules:" >&2 + sed 's/^/ - /' "$TMP/missing.txt" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + [ "$failed" -eq 0 ] +} + +@test "every DCGM metric referenced in tracked dashboards and rules is declared" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; } + [ -f "$DCGM_CUSTOM_CSV" ] || { echo "missing $DCGM_CUSTOM_CSV" >&2; exit 1; } + + { + extract_csv_metrics "$DCGM_DEFAULT_CSV" + extract_csv_metrics "$DCGM_CUSTOM_CSV" + } | sort -u > "$TMP/declared.txt" + [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; } + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + + # Dashboard coverage — every dashboard's DCGM references must resolve. + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + [ -f "$dashboard" ] || continue # handled by the existence test + extract_dcgm_refs "$dashboard" > "$TMP/refs.txt" + [ -s "$TMP/refs.txt" ] || continue # dashboard relies entirely on recording rules + comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt" + if [ -s "$TMP/missing.txt" ]; then + echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2 + sed 's/^/ - /' "$TMP/missing.txt" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + # Rules coverage — recording rules consume DCGM directly, so their set + # must be declared too, otherwise derived series on every dashboard + # collapse to empty. + extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt" + if [ -s "$TMP/rule-refs.txt" ]; then + comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt" + if [ -s "$TMP/rule-missing.txt" ]; then + echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2 + sed 's/^/ - /' "$TMP/rule-missing.txt" >&2 + failed=1 + fi + fi + + [ "$failed" -eq 0 ] +} + +@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + + list_tracked_gpu_dashboards > "$TMP/dashboards.txt" + [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } + + failed=0 + while IFS= read -r dashboard_rel; do + dashboard="$DASHBOARDS_DIR/$dashboard_rel" + if [ ! -f "$dashboard" ]; then + echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2 + failed=1 + fi + done < "$TMP/dashboards.txt" + + [ "$failed" -eq 0 ] +} diff --git a/hack/dcgm-default-counters.csv b/hack/dcgm-default-counters.csv new file mode 100644 index 00000000..b5e94540 --- /dev/null +++ b/hack/dcgm-default-counters.csv @@ -0,0 +1,104 @@ +# Snapshot of the upstream DCGM Exporter default-counters.csv used as a +# fixture by hack/check-gpu-recording-rules.bats. The test verifies that +# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either +# declared here (upstream defaults) or in +# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +# (the project's custom CSV). +# +# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv +# Pinned to the DCGM Exporter image tag shipped by the gpu-operator +# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml +# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is +# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter +# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`. + +# Format +# If line starts with a '#' it is considered a comment +# DCGM FIELD, Prometheus metric type, help message + +# Clocks +DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). +DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). + +# Temperature +DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). +DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). + +# Power +DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). +DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). + +# PCIE +# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. +# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. +DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + +# Utilization (the sample period varies depending on the product) +DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). +DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). +DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). +DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). + +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). + +# Memory usage +DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). +DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + +# ECC +# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. +# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. +# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. +# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + +# Retired pages +# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. +# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. +# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + +# NVLink +# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. +# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. +# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. +# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. +DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. +# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. + +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status + +# Remapped rows +DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors +DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors +DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed + +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +# DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version +# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device + +# Datacenter Profiling (DCP) metrics +# NOTE: supported on Nvidia datacenter Volta GPUs and newer +DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. +# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. +# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. +DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. +# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. +# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. +# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. +DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. +DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/hack/e2e-apps/gateway.bats b/hack/e2e-apps/gateway.bats deleted file mode 100644 index c97fc80a..00000000 --- a/hack/e2e-apps/gateway.bats +++ /dev/null @@ -1,356 +0,0 @@ -#!/usr/bin/env bats - -@test "Gateway API CRDs are installed and the cilium GatewayClass is Accepted" { - # Gateway API CRDs must exist — installed by packages/system/gateway-api-crds - kubectl wait crd/gatewayclasses.gateway.networking.k8s.io --for=condition=Established --timeout=60s - kubectl wait crd/gateways.gateway.networking.k8s.io --for=condition=Established --timeout=60s - kubectl wait crd/httproutes.gateway.networking.k8s.io --for=condition=Established --timeout=60s - - # Cilium must have registered its built-in GatewayClass once gatewayAPI.enabled - # is true in the cilium values. This verifies the flip in - # packages/system/cilium/values.yaml propagated end-to-end. - timeout 120 sh -ec 'until kubectl get gatewayclass cilium >/dev/null 2>&1; do sleep 2; done' - kubectl wait gatewayclass/cilium --for=condition=Accepted --timeout=3m -} - -@test "Cilium Gateway API controller reconciles a minimal Gateway to Programmed" { - # Use the pre-existing tenant-test namespace created by e2e-install-cozystack.bats. - kubectl apply -f - <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: gateway-e2e-probe - namespace: tenant-test -spec: - gatewayClassName: cilium - listeners: - - name: http - protocol: HTTP - port: 80 - allowedRoutes: - namespaces: - from: Same -EOF - - # The controller must accept and program the Gateway. - kubectl -n tenant-test wait gateway/gateway-e2e-probe --for=condition=Accepted --timeout=2m - kubectl -n tenant-test wait gateway/gateway-e2e-probe --for=condition=Programmed --timeout=3m - - # Cilium materialises a LoadBalancer Service named cilium-gateway- - # for each programmed Gateway. Its existence is the observable proof that the - # full data-plane wiring kicked in. - kubectl -n tenant-test get svc cilium-gateway-gateway-e2e-probe - - # Cleanup - kubectl -n tenant-test delete gateway/gateway-e2e-probe --ignore-not-found --timeout=1m -} - -@test "cozystack-gateway-hostname-policy VAP and binding are installed" { - # Diagnose the hostname-policy enforcement path from the ground up. - # If these resources are missing, admission cannot reject anything. - kubectl get validatingadmissionpolicy cozystack-gateway-hostname-policy -o yaml - kubectl get validatingadmissionpolicybinding cozystack-gateway-hostname-policy -o yaml - # Binding MUST have validationActions: [Deny] — [Audit] / [] would let requests through silently. - actions=$(kubectl get validatingadmissionpolicybinding cozystack-gateway-hostname-policy -o jsonpath='{.spec.validationActions}') - echo "binding.validationActions=$actions" - case "$actions" in - *Deny*) ;; - *) echo "SETUP FAILURE: binding.validationActions lacks Deny (got '$actions')" >&2; return 1 ;; - esac -} - -@test "tenant-test namespace carries namespace.cozystack.io/host label from tenant chart" { - # Diagnostic: the whole hostname-policy VAP keys off this label. If it is - # missing or empty, the VAP's matchCondition returns false, VAP skips, and - # EVERY Gateway in the namespace is admitted regardless of listener hostname. - # Make that bug loud instead of letting it fall through as a silent pass. - host_label=$(kubectl get namespace tenant-test -o jsonpath='{.metadata.labels.namespace\.cozystack\.io/host}') - if [ -z "$host_label" ]; then - echo "SETUP FAILURE: tenant-test namespace lacks namespace.cozystack.io/host label" >&2 - kubectl get namespace tenant-test -o yaml >&2 - return 1 - fi - if [ "$host_label" != "test.example.org" ]; then - echo "SETUP FAILURE: tenant-test host label is '$host_label', expected 'test.example.org'" >&2 - kubectl get namespace tenant-test -o yaml >&2 - return 1 - fi -} - -@test "ValidatingAdmissionPolicy rejects Gateway with foreign hostname" { - # tenant-test namespace should only be allowed to publish its own - # domain suffix ('.test.example.org'); a listener hostname from the - # root tenant's apex must be denied by cozystack-gateway-hostname-policy. - # hack/cozytest.sh is /bin/sh (dash) with set -e — a failing command - # substitution propagates its exit status through variable assignment and - # kills the test. We specifically EXPECT admission to reject the apply, so - # use `if !` — set -e is disabled inside the if-condition, the exit status - # is captured, and $output is filled either way for the follow-up greps. - if ! output=$(kubectl apply -f - 2>&1 <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: hostname-hijack-probe - namespace: tenant-test -spec: - gatewayClassName: cilium - listeners: - - name: https - protocol: HTTPS - port: 443 - hostname: "dashboard.example.org" - tls: - mode: Terminate - certificateRefs: - - name: noop - allowedRoutes: - namespaces: - from: Same -EOF -); then - # Happy path: admission rejected the Gateway. Verify the rejection came - # from our VAP and names the expected tenant host. - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "must equal test.example.org" - else - echo "BUG: admission accepted cross-tenant hostname — Gateway 'hostname-hijack-probe' was created in tenant-test" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "cozystack-gateway-attached-namespaces-policy rejects Packages with tenant-* entries" { - # The platform Package default name is cozystack.cozystack-platform, managed by - # cozystack-api. Creating a dummy Package with tenant-alice in gateway.attachedNamespaces - # must fail at admission time. - if ! output=$(kubectl apply -f - 2>&1 <<'EOF' -apiVersion: cozystack.io/v1alpha1 -kind: Package -metadata: - name: vap-reject-probe -spec: - variant: isp-full - components: - platform: - values: - gateway: - attachedNamespaces: - - tenant-alice -EOF -); then - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "must not contain any tenant-" - else - echo "BUG: admission accepted tenant-* in attachedNamespaces — Package 'vap-reject-probe' was created" >&2 - echo "$output" >&2 - kubectl delete package vap-reject-probe --ignore-not-found - return 1 - fi -} - -@test "cozystack-tenant-host-policy blocks non-trusted callers from setting tenant.spec.host" { - # Impersonate a tenant-scoped ServiceAccount that is NOT in the trustedCaller - # group list. First grant RBAC to create Tenants — authorization runs BEFORE - # admission, so without this grant the apiserver returns a plain RBAC - # Forbidden and the test would fail grep-ing for 'ValidatingAdmissionPolicy' - # even though the VAP itself is fine. - kubectl apply -f - <<'EOF' -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: vap-probe-tenant-create - namespace: tenant-test -rules: -- apiGroups: ["apps.cozystack.io"] - resources: ["tenants"] - verbs: ["create","get","list","watch","update","patch","delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: vap-probe-tenant-create - namespace: tenant-test -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: vap-probe-tenant-create -subjects: -- kind: ServiceAccount - name: default - namespace: tenant-test -EOF - if ! output=$(kubectl --as=system:serviceaccount:tenant-test:default \ - --as-group=system:serviceaccounts \ - --as-group=system:serviceaccounts:tenant-test \ - apply -f - 2>&1 <<'EOF' -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -metadata: - name: vaphostprobe - namespace: tenant-test -spec: - host: foreign.example.org -EOF -); then - kubectl -n tenant-test delete rolebinding vap-probe-tenant-create --ignore-not-found - kubectl -n tenant-test delete role vap-probe-tenant-create --ignore-not-found - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "spec.host can only be set" - else - kubectl -n tenant-test delete tenants.apps.cozystack.io vaphostprobe --ignore-not-found - kubectl -n tenant-test delete rolebinding vap-probe-tenant-create --ignore-not-found - kubectl -n tenant-test delete role vap-probe-tenant-create --ignore-not-found - echo "BUG: admission accepted tenant.spec.host from untrusted SA — Tenant 'vaphostprobe' was created" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "cozystack-namespace-host-label-policy blocks non-trusted callers from changing the host label" { - # tenant-test namespace already has namespace.cozystack.io/host set by the - # cozystack tenant chart. Grant patch on namespaces cluster-wide to the - # impersonated SA — namespaces is a cluster-scoped resource so this needs a - # ClusterRole. Authorization runs before admission, so without this grant - # the test would fail with plain RBAC Forbidden rather than a VAP rejection. - kubectl apply -f - <<'EOF' -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: vap-probe-namespace-patch -rules: -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get","list","watch","update","patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: vap-probe-namespace-patch -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: vap-probe-namespace-patch -subjects: -- kind: ServiceAccount - name: default - namespace: tenant-test -EOF - if ! output=$(kubectl --as=system:serviceaccount:tenant-test:default \ - --as-group=system:serviceaccounts \ - --as-group=system:serviceaccounts:tenant-test \ - label namespace tenant-test \ - namespace.cozystack.io/host=foreign.example.org --overwrite 2>&1); then - kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "immutable" - else - # Revert label if apiserver somehow accepted the overwrite. - kubectl label namespace tenant-test namespace.cozystack.io/host=test.example.org --overwrite - kubectl delete clusterrolebinding vap-probe-namespace-patch --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-patch --ignore-not-found - echo "BUG: admission accepted host label change from untrusted SA" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "cozystack-namespace-host-label-policy blocks non-trusted callers from setting the host label at CREATE" { - # Defense-in-depth: a non-trusted caller must not be able to stamp - # namespace.cozystack.io/host=X on a brand-new namespace either — only - # cozystack/Flux SAs may write the label. Authorization runs before - # admission, so grant cluster-wide namespace create to the impersonated SA - # first, otherwise the test would fail with plain RBAC Forbidden. - kubectl apply -f - <<'EOF' -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: vap-probe-namespace-create -rules: -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["create","get","list","delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: vap-probe-namespace-create -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: vap-probe-namespace-create -subjects: -- kind: ServiceAccount - name: default - namespace: tenant-test -EOF - if ! output=$(kubectl --as=system:serviceaccount:tenant-test:default \ - --as-group=system:serviceaccounts \ - --as-group=system:serviceaccounts:tenant-test \ - apply -f - 2>&1 <<'EOF' -apiVersion: v1 -kind: Namespace -metadata: - name: vap-host-label-probe - labels: - namespace.cozystack.io/host: foreign.example.org -EOF -); then - kubectl delete clusterrolebinding vap-probe-namespace-create --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-create --ignore-not-found - echo "$output" | grep -qi "ValidatingAdmissionPolicy" - echo "$output" | grep -q "immutable" - else - kubectl delete namespace vap-host-label-probe --ignore-not-found --wait=false - kubectl delete clusterrolebinding vap-probe-namespace-create --ignore-not-found - kubectl delete clusterrole vap-probe-namespace-create --ignore-not-found - echo "BUG: admission accepted first-time host label write from untrusted SA at CREATE — Namespace 'vap-host-label-probe' was created" >&2 - echo "$output" >&2 - return 1 - fi -} - -@test "HTTPRoute with a matching parentRef reaches Accepted status" { - # Put a Gateway and a route in the same namespace so allowedRoutes: Same accepts them. - kubectl apply -f - <<'EOF' -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: gateway-route-probe - namespace: tenant-test -spec: - gatewayClassName: cilium - listeners: - - name: http - protocol: HTTP - port: 80 - allowedRoutes: - namespaces: - from: Same ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: httproute-probe - namespace: tenant-test -spec: - parentRefs: - - name: gateway-route-probe - sectionName: http - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - name: kubernetes - namespace: default - port: 443 -EOF - - kubectl -n tenant-test wait gateway/gateway-route-probe --for=condition=Programmed --timeout=3m - timeout 120 sh -ec 'until kubectl -n tenant-test get httproute httproute-probe -o jsonpath="{.status.parents[0].conditions[?(@.type==\"Accepted\")].status}" 2>/dev/null | grep -q True; do sleep 2; done' - - kubectl -n tenant-test delete httproute/httproute-probe --ignore-not-found - kubectl -n tenant-test delete gateway/gateway-route-probe --ignore-not-found -} diff --git a/hack/e2e-apps/remediation-guard.sh b/hack/e2e-apps/remediation-guard.sh new file mode 100644 index 00000000..b45561b9 --- /dev/null +++ b/hack/e2e-apps/remediation-guard.sh @@ -0,0 +1,39 @@ +# Helpers for asserting that a Flux HelmRelease did not fall into an +# install/upgrade remediation cycle during an e2e run. +# +# Background: Flux helm-controller's ClearFailures() zeroes +# .status.installFailures / .status.upgradeFailures on every successful +# reconciliation (see the upstream ClearFailures method on +# HelmReleaseStatus). That makes those counters useless for a guard that +# runs after the HelmRelease has reached Ready - the values are always 0. +# +# What survives a successful reconciliation is .status.history, a bounded +# list of release Snapshots. Each Snapshot carries a status field that +# tracks the Helm release state: deployed, superseded, failed, uninstalled, +# and so on. A remediation cycle leaves the footprint behind: a snapshot +# with status "uninstalled" (from install/upgrade remediation) or "failed" +# (Helm release failure that remediation then uninstalled). Those stay in +# history even after a subsequent successful reinstall. +# +# helmrelease_has_remediation_cycle takes a newline-delimited list of +# snapshot statuses (whatever the caller extracted via kubectl -o jsonpath +# or equivalent) and returns 0 (detected) when any entry is "failed" or +# "uninstalled", 1 otherwise. Empty input is treated as "no history yet, +# no cycle observed". + +helmrelease_has_remediation_cycle() { + statuses="$1" + if [ -z "${statuses}" ]; then + return 1 + fi + # printf + grep over the pipe, rather than a heredoc plus while read. + # printf %s treats the status string as a literal payload, so any stray + # $ in a future caller's input does not trigger shell expansion. grep + # returns 0 iff at least one line matches the allowlist, which is + # exactly the contract the caller wants, so we can return its exit + # status directly. + if printf '%s\n' "${statuses}" | grep --extended-regexp --quiet '^(failed|uninstalled)$'; then + return 0 + fi + return 1 +} diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index be6dcd6f..761ee57d 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -1,3 +1,5 @@ +. hack/e2e-apps/remediation-guard.sh + run_kubernetes_test() { local version_expr="$1" local test_name="$2" @@ -320,6 +322,35 @@ EOF done kubectl wait hr kubernetes-${test_name}-ingress-nginx -n tenant-test --timeout=5m --for=condition=ready + # Guard: parent HelmRelease must not have entered an install/upgrade remediation cycle. + # A non-zero installFailures/upgradeFailures indicates the helm-wait budget expired while + # admin-kubeconfig was still being provisioned, which would trigger uninstall remediation + # and churn the Cluster CR. + # Flux helm-controller v2 retains per-revision release Snapshots in + # .status.history; each Snapshot's .status reflects the Helm release + # state (deployed/superseded/failed/uninstalled). A remediation cycle + # leaves a "failed" or "uninstalled" entry behind that survives a later + # successful reinstall, unlike the installFailures/upgradeFailures + # counters (which ClearFailures zeroes on every successful reconcile). + # The shape is pinned by hack/remediation-guard.bats; the upstream + # types are github.com/fluxcd/helm-controller/api v2 Snapshot. + history_statuses=$(kubectl get hr -n tenant-test "kubernetes-${test_name}" \ + -ojsonpath='{range .status.history[*]}{.status}{"\n"}{end}') + # Always emit the raw value so a silent future-Flux field rename shows + # up as "empty history on a Ready HR" in CI logs rather than vanishing. + echo "Parent HelmRelease history statuses:" + printf '%s\n' "${history_statuses:-}" + if [ -z "${history_statuses}" ]; then + echo "Unexpected empty .status.history on a Ready HelmRelease - Flux API shape may have changed." >&2 + kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 + exit 1 + fi + if helmrelease_has_remediation_cycle "${history_statuses}"; then + echo "Parent HelmRelease entered remediation cycle." >&2 + kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 + exit 1 + fi + # Clean up pkill -f "port-forward.*${port}:" 2>/dev/null || true rm -f "tenantkubeconfig-${test_name}" diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats new file mode 100644 index 00000000..092fe06d --- /dev/null +++ b/hack/remediation-guard.bats @@ -0,0 +1,127 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for hack/e2e-apps/remediation-guard.sh +# +# helmrelease_has_remediation_cycle takes a newline-delimited list of +# HelmRelease history snapshot status values (deployed/superseded/failed/ +# uninstalled/...) and returns 0 when any entry is "failed" or "uninstalled" +# (meaning flux helm-controller performed install/upgrade remediation). +# +# This is used by the e2e script after the HelmRelease reaches Ready. The +# failure/upgrade counters (.status.installFailures / .status.upgradeFailures) +# are useless there because flux's ClearFailures zeroes them on successful +# reconciliation; .status.history retains the snapshot trail. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on +# its own line; there is no bats `run` or `$status`. Assertions are +# expressed as direct shell tests that exit non-zero on failure. +# +# Run with: hack/cozytest.sh hack/remediation-guard.bats +# ----------------------------------------------------------------------------- + +@test "empty history returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + if helmrelease_has_remediation_cycle ""; then + echo "expected not-detected for empty history" >&2 + exit 1 + fi +} + +@test "single deployed snapshot returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + if helmrelease_has_remediation_cycle "deployed"; then + echo "expected not-detected for deployed-only history" >&2 + exit 1 + fi +} + +@test "deployed then superseded returns not-detected" { + . hack/e2e-apps/remediation-guard.sh + statuses=$(printf 'deployed\nsuperseded\n') + if helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected not-detected for deployed+superseded history" >&2 + exit 1 + fi +} + +@test "single failed snapshot returns detected" { + . hack/e2e-apps/remediation-guard.sh + if ! helmrelease_has_remediation_cycle "failed"; then + echo "expected detected when history contains failed snapshot" >&2 + exit 1 + fi +} + +@test "single uninstalled snapshot returns detected" { + # The exact signature of the install-remediation race: the first install + # exceeded flux's wait budget, remediation uninstalled, the next retry + # eventually succeeded. History still carries the uninstalled snapshot. + . hack/e2e-apps/remediation-guard.sh + if ! helmrelease_has_remediation_cycle "uninstalled"; then + echo "expected detected when history contains uninstalled snapshot" >&2 + exit 1 + fi +} + +@test "uninstalled then deployed still returns detected" { + . hack/e2e-apps/remediation-guard.sh + statuses=$(printf 'uninstalled\ndeployed\n') + if ! helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected detected despite later successful deploy" >&2 + exit 1 + fi +} + +@test "deployed then failed still returns detected" { + . hack/e2e-apps/remediation-guard.sh + statuses=$(printf 'deployed\nfailed\n') + if ! helmrelease_has_remediation_cycle "${statuses}"; then + echo "expected detected when any entry is failed" >&2 + exit 1 + fi +} + +@test "status.history extraction pins HR v2 status.history shape" { + # Pins the Flux HelmRelease v2 .status.history[].status shape that + # run-kubernetes.sh relies on. If a future flux release renames the + # field, the jsonpath returns nothing, the guard reports no cycle, + # and real remediation loops slip past the e2e assertion. This test + # uses yq to read the exact path used in the e2e script; the upstream + # Snapshot type lives at + # github.com/fluxcd/helm-controller/api/v2.Snapshot (via go.mod). + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + + cat > "$tmp/hr.yaml" <<'YAML' +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kubernetes-test + namespace: tenant-test +status: + history: + - name: kubernetes-test + namespace: tenant-test + version: 1 + status: uninstalled + - name: kubernetes-test + namespace: tenant-test + version: 2 + status: deployed +YAML + + # Default yq output is yaml scalar format, which for string values emits + # bare unquoted tokens - matching what kubectl -o jsonpath produces in + # e2e. Do not switch to JSON output here; that would quote the values + # and break the loop in helmrelease_has_remediation_cycle. + statuses=$(yq '.status.history[].status' "$tmp/hr.yaml") + + [ -n "$statuses" ] + echo "$statuses" | grep --quiet '^uninstalled$' + + . hack/e2e-apps/remediation-guard.sh + if ! helmrelease_has_remediation_cycle "$statuses"; then + echo "expected detected for pinned HR snippet with uninstalled + deployed history" >&2 + exit 1 + fi +} diff --git a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml index 0acfcdd2..1d0037df 100644 --- a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml +++ b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml @@ -118,6 +118,19 @@ spec: ReleaseName is the name of the HelmRelease resource that will be created If not specified, defaults to the component Name field type: string + upgradeCRDs: + description: |- + UpgradeCRDs controls how CRDs from the chart's crds/ directory are + handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs. + Empty string (default) preserves the helm-controller default (Skip). + Use "CreateReplace" for operators that evolve their CRD set between + versions. Warning: CreateReplace overwrites CRDs and may cause data + loss if upstream drops fields from a CRD with live objects. + enum: + - Skip + - Create + - CreateReplace + type: string type: object libraries: description: |- diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index 32e667e4..0e724e49 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -45,6 +45,16 @@ const ( SecretCozystackValues = "cozystack-values" ) +// parseCRDPolicy maps ComponentInstall.UpgradeCRDs to a helmv2.CRDsPolicy. +// Empty / nil preserves the helm-controller default (Skip on upgrade); +// the CRD enum marker restricts the string to Skip/Create/CreateReplace. +func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { + if install == nil || install.UpgradeCRDs == "" { + return "" + } + return helmv2.CRDsPolicy(install.UpgradeCRDs) +} + // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client @@ -221,6 +231,7 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Remediation: &helmv2.UpgradeRemediation{ Retries: -1, }, + CRDs: parseCRDPolicy(component.Install), }, }, } diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go new file mode 100644 index 00000000..f0ee2c19 --- /dev/null +++ b/internal/operator/package_reconciler_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + helmv2 "github.com/fluxcd/helm-controller/api/v2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/yaml" +) + +func TestParseCRDPolicy(t *testing.T) { + tests := []struct { + name string + install *cozyv1alpha1.ComponentInstall + want helmv2.CRDsPolicy + }{ + { + name: "nil install leaves flux default", + install: nil, + want: "", + }, + { + name: "empty upgradeCRDs leaves flux default", + install: &cozyv1alpha1.ComponentInstall{}, + want: "", + }, + { + name: "Skip is passed through", + install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Skip"}, + want: helmv2.Skip, + }, + { + name: "Create is passed through", + install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Create"}, + want: helmv2.Create, + }, + { + name: "CreateReplace is passed through", + install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "CreateReplace"}, + want: helmv2.CreateReplace, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parseCRDPolicy(tc.install) + if got != tc.want { + t.Errorf("parseCRDPolicy() = %q, want %q", got, tc.want) + } + }) + } +} + +// TestPackageSourceCRDHasUpgradeCRDsEnum guards the generated CRD schema: the +// invalid-value case from the spec is enforced at the API server via a +// kubebuilder enum marker, not in the reconciler. If someone drops the marker +// and forgets to regenerate, this test catches it. +func TestPackageSourceCRDHasUpgradeCRDsEnum(t *testing.T) { + path := filepath.Join("..", "crdinstall", "manifests", "cozystack.io_packagesources.yaml") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + t.Fatalf("unmarshal CRD: %v", err) + } + + var field *apiextensionsv1.JSONSchemaProps + for i := range crd.Spec.Versions { + v := &crd.Spec.Versions[i] + if v.Schema == nil || v.Schema.OpenAPIV3Schema == nil { + continue + } + spec, ok := v.Schema.OpenAPIV3Schema.Properties["spec"] + if !ok { + continue + } + variants, ok := spec.Properties["variants"] + if !ok || variants.Items == nil || variants.Items.Schema == nil { + continue + } + components, ok := variants.Items.Schema.Properties["components"] + if !ok || components.Items == nil || components.Items.Schema == nil { + continue + } + install, ok := components.Items.Schema.Properties["install"] + if !ok { + continue + } + f, ok := install.Properties["upgradeCRDs"] + if !ok { + continue + } + field = &f + break + } + + if field == nil { + t.Fatal("upgradeCRDs field not found in PackageSource CRD schema") + } + + got := map[string]bool{} + for _, e := range field.Enum { + var s string + if err := json.Unmarshal(e.Raw, &s); err != nil { + t.Fatalf("unmarshal enum value %q: %v", e.Raw, err) + } + got[s] = true + } + + for _, want := range []string{"Skip", "Create", "CreateReplace"} { + if !got[want] { + t.Errorf("enum value %q missing from upgradeCRDs; got %v", want, got) + } + } +} diff --git a/packages/apps/harbor/templates/httproute.yaml b/packages/apps/harbor/templates/httproute.yaml deleted file mode 100644 index d7e2a632..00000000 --- a/packages/apps/harbor/templates/httproute.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- $gateway := .Values._namespace.gateway | default "" }} -{{- $host := .Values._namespace.host }} -{{- $harborHost := .Values.host | default (printf "%s.%s" .Release.Name $host) }} -{{- if $gateway }} -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: {{ .Release.Name }} -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $gateway }} - sectionName: https - hostnames: - - {{ $harborHost | quote }} - rules: - - backendRefs: - - name: {{ .Release.Name }} - port: 80 -{{- end }} diff --git a/packages/apps/harbor/templates/ingress.yaml b/packages/apps/harbor/templates/ingress.yaml index a8ac6463..70933f5f 100644 --- a/packages/apps/harbor/templates/ingress.yaml +++ b/packages/apps/harbor/templates/ingress.yaml @@ -1,10 +1,8 @@ {{- $ingress := .Values._namespace.ingress }} -{{- $gateway := .Values._namespace.gateway | default "" }} {{- $host := .Values._namespace.host }} {{- $harborHost := .Values.host | default (printf "%s.%s" .Release.Name $host) }} {{- $solver := (index .Values._cluster "solver") | default "http01" }} {{- $clusterIssuer := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} -{{- if not $gateway }} --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -37,4 +35,3 @@ spec: name: {{ .Release.Name }} port: number: 80 -{{- end }} diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 01cf736d..4fea42ef 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -4,6 +4,9 @@ KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml) include ../../../hack/common-envs.mk include ../../../hack/package.mk +test: + helm unittest . + generate: cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go ../../../hack/update-crd.sh @@ -67,3 +70,4 @@ image-cluster-autoscaler: echo "$(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \ > images/cluster-autoscaler.tag rm -f images/cluster-autoscaler.json + diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index d62d3a39..bed1117a 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,6 +128,9 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | +| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | +| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | +| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | @@ -145,31 +148,33 @@ See the reference for components utilized in this service: ### Kubernetes Control Plane Configuration -| Name | Description | Type | Value | -| --------------------------------------------------- | ------------------------------------------------ | ---------- | ------- | -| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | -| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | -| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | -| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | -| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `large` | -| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | -| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | -| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | -| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | -| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| Name | Description | Type | Value | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ------- | +| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | +| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | +| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | +| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | +| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `large` | +| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | +| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | +| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | +| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | +| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` | +| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag new file mode 100644 index 00000000..39de220a --- /dev/null +++ b/packages/apps/kubernetes/images/busybox.tag @@ -0,0 +1 @@ +docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 36c06b64..40a8ae7a 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -49,3 +49,52 @@ Selector labels app.kubernetes.io/name: {{ include "kubernetes.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + +{{/* +wait-for-kubeconfig init container shared by the control-plane-side +Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the +*-admin-kubeconfig Secret provisioned asynchronously by Kamaji. The +Secret volume is declared optional so kubelet does not FailedMount while +Kamaji is still bootstrapping; this container polls the mounted path and +exits only when super-admin.svc appears, which happens after kubelet's +optional-Secret refresh cycle. + +The 10m deadline stays strictly below the 15m HelmRelease +Install.Timeout set by cozystack-api for the Kubernetes kind (via the +release.cozystack.io/helm-install-timeout annotation) so the +CrashLoopBackOff surfaces before flux remediation fires and uninstalls +the Cluster CR. + +The default image lives in images/busybox.tag and points directly at +docker.io by digest (not mirrored to ghcr.io like the other .tag files +here): the payload is a one-shot sh loop and the digest pin makes the +pull immutable. Operators in air-gapped or rate-limited environments +can override it via .Values.images.waitForKubeconfig (any registry +reference kubelet can pull). When the value is empty the chart falls +back to the bundled digest pin, preserving the prior default. + +Call site owns the surrounding volumes block; the kubeconfig volume +must exist on the pod and mount at /etc/kubernetes/kubeconfig. +*/}} +{{- define "kubernetes.waitForAdminKubeconfig" -}} +- name: wait-for-kubeconfig + image: "{{ default (.Files.Get "images/busybox.tag" | trim) .Values.images.waitForKubeconfig }}" + command: + - sh + - -c + - | + set -eu + deadline=$(( $(date +%s) + 600 )) + until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "admin kubeconfig was not provisioned within 10m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 + exit 1 + fi + echo "waiting for admin kubeconfig (provisioned by Kamaji, visible after kubelet Secret refresh)..." + sleep 5 + done + volumeMounts: + - name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true +{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index a00e0155..298d86db 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -1,3 +1,14 @@ +{{- /* + Gate the control-plane-side workloads on the parent tenant having an etcd + DataStore. Without it no KamajiControlPlane is ever created, Kamaji never + provisions -admin-kubeconfig, and rendering these Deployments would cause + the wait-for-kubeconfig init to CrashLoopBackOff indefinitely, consuming + the parent HelmRelease install timeout and triggering the very uninstall + remediation cycle this chart is supposed to avoid. Rendering them only + when $etcd is set keeps the HelmRelease Ready while flux retries on its + interval and picks up the DataStore as soon as the Tenant chart finishes. +*/}} +{{- if .Values._namespace.etcd }} --- apiVersion: apps/v1 kind: Deployment @@ -23,6 +34,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}" name: cluster-autoscaler @@ -56,6 +69,7 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-cluster-autoscaler terminationGracePeriodSeconds: 10 @@ -105,3 +119,4 @@ rules: - list - update - watch +{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 10d6fd80..e1494e64 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,4 +1,15 @@ {{- $etcd := .Values._namespace.etcd }} +{{- /* + When $etcd is empty, the parent Tenant application has not populated + _namespace.etcd in cozystack-values yet - either the operator forgot to + set etcd: true on an ancestor Tenant, or the Tenant HelmRelease is still + reconciling. Either way, rendering a KamajiControlPlane with an empty + dataStoreName would be rejected by Kamaji's admission webhook and the + HelmRelease would fail to install, triggering remediation. Instead, emit + a single ConfigMap as a user-visible status beacon and skip the rest so + flux marks the HelmRelease Ready and retries its 5m reconcile loop until + the Tenant chart catches up. +*/}} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} {{- $kubevirtmachinetemplateNames := list }} @@ -84,6 +95,26 @@ spec: - name: default pod: {} {{- end }} +{{- if not $etcd }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-awaiting-etcd + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/name: kubernetes + app.kubernetes.io/instance: {{ .Release.Name }} +data: + status: "awaiting-etcd" + message: | + No DataStore is available for this tenant Kubernetes cluster. The parent + Tenant application has not populated _namespace.etcd. Set spec.etcd: true + on an ancestor Tenant (usually tenant-root) and wait for its HelmRelease + to reconcile - this HelmRelease will pick up the DataStore on its next + 5m reconcile loop and provision the cluster. +{{- else }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -404,3 +435,4 @@ metadata: spec: {{- .spec | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index 938b6d67..de62104c 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} kind: Deployment apiVersion: apps/v1 metadata: @@ -24,6 +25,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: csi-driver imagePullPolicy: Always @@ -234,4 +237,6 @@ spec: emptyDir: {} - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml index be07a8b9..fd9dac7e 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.certManager.enabled }} +{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml index 6857581a..700b666e 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml @@ -8,7 +8,7 @@ cert-manager: {{- end }} {{- end }} -{{- if .Values.addons.certManager.enabled }} +{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml index d032f5b6..d8c90cbf 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml @@ -14,6 +14,7 @@ cilium: {{- end }} {{- end }} +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -55,3 +56,4 @@ spec: - name: {{ .Release.Name }}-gateway-api-crds namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml index bdb6c682..0711a51d 100644 --- a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml @@ -4,6 +4,7 @@ coredns: clusterIP: "10.95.0.10" {{- end }} +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -42,3 +43,4 @@ spec: {{- end }} - name: {{ .Release.Name }}-cilium namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/csi.yaml b/packages/apps/kubernetes/templates/helmreleases/csi.yaml index dd2c69a6..109d78e1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/csi.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/csi.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -39,3 +40,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml index 76499dfe..25fff01c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.fluxcd.enabled }} +{{- if and .Values.addons.fluxcd.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml index 2bcc8d4d..b4172ed1 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml @@ -1,4 +1,4 @@ -{{- if $.Values.addons.gatewayAPI.enabled }} +{{- if and $.Values.addons.gatewayAPI.enabled $.Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 5ef48912..655dc868 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,4 +1,12 @@ -{{- if .Values.addons.gpuOperator.enabled }} +{{- define "cozystack.defaultGpuOperatorValues" -}} +{{- if .Values.addons.hami.enabled }} +gpu-operator: + devicePlugin: + enabled: false +{{- end }} +{{- end }} + +{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -29,9 +37,12 @@ spec: force: true remediation: retries: -1 - {{- with .Values.addons.gpuOperator.valuesOverride }} + {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} + {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} + {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} + {{- if $merged }} values: - {{- toYaml . | nindent 4 }} + {{- toYaml $merged | nindent 4 }} {{- end }} dependsOn: diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml new file mode 100644 index 00000000..f1538c7c --- /dev/null +++ b/packages/apps/kubernetes/templates/helmreleases/hami.yaml @@ -0,0 +1,49 @@ +{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} +{{- if not .Values.addons.gpuOperator.enabled }} +{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} +{{- end }} +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: {{ .Release.Name }}-hami + labels: + cozystack.io/repository: system + cozystack.io/target-cluster-name: {{ .Release.Name }} + sharding.fluxcd.io/key: tenants +spec: + releaseName: hami + chartRef: + kind: ExternalArtifact + name: cozystack-kubernetes-application-kubevirt-kubernetes-hami + namespace: cozy-system + kubeConfig: + secretRef: + name: {{ .Release.Name }}-admin-kubeconfig + key: super-admin.svc + targetNamespace: cozy-hami + storageNamespace: cozy-hami + interval: 5m + timeout: 10m + install: + createNamespace: true + remediation: + retries: -1 + upgrade: + force: true + remediation: + retries: -1 + {{- with .Values.addons.hami.valuesOverride }} + values: + {{- toYaml . | nindent 4 }} + {{- end }} + + dependsOn: + {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} + - name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + {{- end }} + - name: {{ .Release.Name }}-cilium + namespace: {{ .Release.Namespace }} + - name: {{ .Release.Name }}-gpu-operator + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml index 5cafff90..6e2183d3 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml @@ -20,7 +20,7 @@ ingress-nginx: node-role.kubernetes.io/ingress-nginx: "" {{- end }} -{{- if .Values.addons.ingressNginx.enabled }} +{{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml index 3cc81a14..3e6f9660 100644 --- a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -36,3 +37,4 @@ spec: namespace: {{ .Release.Namespace }} - name: {{ .Release.Name }}-prometheus-operator-crds namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml index ea84dec0..a811f7dd 100644 --- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml @@ -1,6 +1,6 @@ {{- $targetTenant := .Values._namespace.monitoring }} {{- $clusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml index 600a7994..3038a058 100644 --- a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -31,3 +32,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/velero.yaml b/packages/apps/kubernetes/templates/helmreleases/velero.yaml index ad236d53..781b9c49 100644 --- a/packages/apps/kubernetes/templates/helmreleases/velero.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/velero.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.velero.enabled }} +{{- if and .Values.addons.velero.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml index a3b7a9b4..55a5faac 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml index 178df3e3..74fb5a39 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml @@ -24,7 +24,7 @@ vertical-pod-autoscaler: memory: 1600Mi {{- end }} -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml index 99744277..7302f8f5 100644 --- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml @@ -1,4 +1,4 @@ -{{- if .Values.addons.monitoringAgents.enabled }} +{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml index 025f01b7..d50fd93c 100644 --- a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -33,3 +34,4 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index 81426d4e..bd9e2798 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -1,3 +1,4 @@ +{{- if .Values._namespace.etcd }} apiVersion: apps/v1 kind: Deployment metadata: @@ -22,6 +23,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" + initContainers: + {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: kubevirt-cloud-controller-manager args: @@ -55,5 +58,7 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig + optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-kccm +{{- end }} diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml new file mode 100644 index 00000000..507e00af --- /dev/null +++ b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml @@ -0,0 +1,155 @@ +suite: admin-kubeconfig wait guards + +release: + name: test + namespace: tenant-root + +values: + - values-ci.yaml + +tests: + - it: cluster-autoscaler mounts admin-kubeconfig as optional + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: cluster-autoscaler waits for admin-kubeconfig via initContainer + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: kccm mounts admin-kubeconfig as optional + template: templates/kccm/manager.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: kccm waits for admin-kubeconfig via initContainer + template: templates/kccm/manager.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: csi controller mounts admin-kubeconfig as optional + template: templates/csi/deploy.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName + value: test-admin-kubeconfig + - equal: + path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional + value: true + + - it: csi controller waits for admin-kubeconfig via initContainer + template: templates/csi/deploy.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-kubeconfig + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: kubeconfig + mountPath: /etc/kubernetes/kubeconfig + readOnly: true + + - it: wait-for-kubeconfig defaults to bundled busybox digest pin + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + asserts: + - matchRegex: + path: spec.template.spec.initContainers[0].image + pattern: '^docker\.io/library/busybox:[^@]+@sha256:[0-9a-f]{64}$' + + - it: wait-for-kubeconfig honours images.waitForKubeconfig override + template: templates/cluster-autoscaler/deployment.yaml + documentSelector: + path: kind + value: Deployment + set: + images: + waitForKubeconfig: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + asserts: + - equal: + path: spec.template.spec.initContainers[0].image + value: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" + + - it: cluster.yaml renders and wires dataStoreName when tenant has etcd + template: templates/cluster.yaml + documentSelector: + path: kind + value: KamajiControlPlane + asserts: + - equal: + path: spec.dataStoreName + value: tenant-root + + - it: cluster.yaml skips Cluster resources when tenant has no etcd DataStore + # Must NOT fail rendering - the parent Tenant chart populates + # _namespace.etcd asynchronously, so rendering failures here would cause + # flux install remediation on every cold bootstrap. Instead, emit only a + # ConfigMap status beacon so the HelmRelease reports Ready while flux + # retries on its interval until the DataStore appears. + template: templates/cluster.yaml + set: + _namespace: + etcd: "" + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: test-awaiting-etcd + - equal: + path: data.status + value: awaiting-etcd diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml new file mode 100644 index 00000000..44528470 --- /dev/null +++ b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml @@ -0,0 +1,99 @@ +suite: GPU Operator HelmRelease HAMi integration tests +templates: + - templates/helmreleases/gpu-operator.yaml +values: + - values-ci.yaml +tests: + - it: should disable devicePlugin when hami is enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should not have values when hami is disabled and no overrides + set: + addons: + gpuOperator: + enabled: true + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - notExists: + path: spec.values + + - it: should apply hami defaults when valuesOverride key is omitted + set: + addons: + gpuOperator: + enabled: true + hami: + enabled: true + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + + - it: should allow user overrides to merge with hami defaults + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: false + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should let user explicitly override devicePlugin.enabled to true with hami enabled + set: + addons: + gpuOperator: + enabled: true + valuesOverride: + gpu-operator: + devicePlugin: + enabled: true + driver: + enabled: false + hami: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.gpu-operator.devicePlugin.enabled + value: true + - equal: + path: spec.values.gpu-operator.driver.enabled + value: false + + - it: should not render when gpuOperator is disabled + set: + addons: + gpuOperator: + enabled: false + valuesOverride: {} + hami: + enabled: false + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml new file mode 100644 index 00000000..27f7c75b --- /dev/null +++ b/packages/apps/kubernetes/tests/hami_test.yaml @@ -0,0 +1,153 @@ +suite: HAMi HelmRelease tests +templates: + - templates/helmreleases/hami.yaml +values: + - values-ci.yaml +tests: + - it: should not render when hami is disabled + set: + addons: + hami: + enabled: false + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 0 + + - it: should render HelmRelease when hami is enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + + - it: should fail when gpuOperator is not enabled + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: false + valuesOverride: {} + asserts: + - failedTemplate: + errorMessage: "addons.hami requires addons.gpuOperator to be enabled" + + - it: should have correct metadata labels + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: metadata.labels["cozystack.io/repository"] + value: system + - equal: + path: metadata.labels["sharding.fluxcd.io/key"] + value: tenants + + - it: should use ExternalArtifact chartRef + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.chartRef.kind + value: ExternalArtifact + - equal: + path: spec.chartRef.namespace + value: cozy-system + + - it: should target cozy-hami namespace + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.targetNamespace + value: cozy-hami + - equal: + path: spec.storageNamespace + value: cozy-hami + + - it: should depend on gpu-operator and cilium + release: + name: test + namespace: test-ns + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - contains: + path: spec.dependsOn + content: + name: test-cilium + namespace: test-ns + - contains: + path: spec.dependsOn + content: + name: test-gpu-operator + namespace: test-ns + + - it: should not render spec.values when valuesOverride is empty + set: + addons: + hami: + enabled: true + valuesOverride: {} + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - hasDocuments: + count: 1 + - notExists: + path: spec.values + + - it: should pass through valuesOverride + set: + addons: + hami: + enabled: true + valuesOverride: + hami: + devicePlugin: + deviceSplitCount: 5 + gpuOperator: + enabled: true + valuesOverride: {} + asserts: + - equal: + path: spec.values.hami.devicePlugin.deviceSplitCount + value: 5 diff --git a/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml new file mode 100644 index 00000000..c7c8196f --- /dev/null +++ b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml @@ -0,0 +1,9 @@ +_namespace: + etcd: "" + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" +_cluster: + cluster-domain: cozy.local +nodeGroups: null diff --git a/packages/apps/kubernetes/tests/values-ci.yaml b/packages/apps/kubernetes/tests/values-ci.yaml new file mode 100644 index 00000000..13365e8c --- /dev/null +++ b/packages/apps/kubernetes/tests/values-ci.yaml @@ -0,0 +1,9 @@ +_namespace: + etcd: tenant-root + monitoring: "" + ingress: "" + seaweedfs: "" + host: "" +_cluster: + cluster-domain: cozy.local +nodeGroups: null diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 1beeff9c..a43deaec 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,6 +149,7 @@ "fluxcd", "gatewayAPI", "gpuOperator", + "hami", "ingressNginx", "monitoringAgents", "velero", @@ -268,6 +269,28 @@ } } }, + "hami": { + "description": "HAMi GPU virtualization middleware.", + "type": "object", + "default": {}, + "required": [ + "enabled", + "valuesOverride" + ], + "properties": { + "enabled": { + "description": "Enable HAMi (requires GPU Operator).", + "type": "boolean", + "default": false + }, + "valuesOverride": { + "description": "Custom Helm values overrides.", + "type": "object", + "default": {}, + "x-kubernetes-preserve-unknown-fields": true + } + } + }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", @@ -630,6 +653,18 @@ } } } + }, + "images": { + "description": "Optional image overrides for air-gapped or rate-limited registries.", + "type": "object", + "default": {}, + "properties": { + "waitForKubeconfig": { + "description": "Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.", + "type": "string", + "default": "" + } + } } } } diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index a67b5d69..d609476c 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,6 +94,10 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. +## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. +## @field {bool} enabled - Enable HAMi (requires GPU Operator). +## @field {object} valuesOverride - Custom Helm values overrides. + ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -114,6 +118,7 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. +## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -137,6 +142,9 @@ addons: gpuOperator: enabled: false valuesOverride: {} + hami: + enabled: false + valuesOverride: {} fluxcd: enabled: false valuesOverride: {} @@ -197,3 +205,10 @@ controlPlane: server: resources: {} resourcesPreset: "micro" + +## @typedef {struct} Images - Optional image overrides for chart-internal helpers. +## @field {string} [waitForKubeconfig] - Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. + +## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries. +images: + waitForKubeconfig: "" diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 5a550f7a..4cda7284 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,12 +133,13 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | ------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | +| `bootstrap.serverName` | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 5c40b2c7..2cdaec1d 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -32,6 +32,9 @@ spec: - name: {{ .Values.bootstrap.oldName }} barmanObjectStore: destinationPath: {{ .Values.backup.destinationPath }} + {{- if .Values.bootstrap.serverName }} + serverName: {{ .Values.bootstrap.serverName }} + {{- end }} endpointURL: {{ .Values.backup.endpointURL }} s3Credentials: accessKeyId: diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index b2a4aeba..98e29822 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -254,6 +254,11 @@ "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.", "type": "string", "default": "" + }, + "serverName": { + "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.", + "type": "string", + "default": "" } } } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index b8f07f63..2ceaa9cf 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -154,6 +154,7 @@ backup: ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. ## @field {string} oldName - Previous cluster name before deletion. +## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: @@ -161,3 +162,4 @@ bootstrap: # example: 2020-11-26 15:22:00.00000+00 recoveryTime: "" oldName: "" + serverName: "" diff --git a/packages/apps/tenant/README.md b/packages/apps/tenant/README.md index 7bad6fa5..96cf5c6e 100644 --- a/packages/apps/tenant/README.md +++ b/packages/apps/tenant/README.md @@ -80,7 +80,6 @@ tenant-u1 | `etcd` | Deploy own Etcd cluster. | `bool` | `false` | | `monitoring` | Deploy own Monitoring Stack. | `bool` | `false` | | `ingress` | Deploy own Ingress Controller. | `bool` | `false` | -| `gateway` | Deploy own Gateway API Gateway (backed by Cilium Gateway API controller). | `bool` | `false` | | `seaweedfs` | Deploy own SeaweedFS. | `bool` | `false` | | `schedulingClass` | The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads. | `string` | `""` | | `resourceQuotas` | Define resource quotas for the tenant. | `map[string]quantity` | `{}` | diff --git a/packages/apps/tenant/templates/cilium-lb-pool.yaml b/packages/apps/tenant/templates/cilium-lb-pool.yaml index 28fc7a7a..63dbcaca 100644 --- a/packages/apps/tenant/templates/cilium-lb-pool.yaml +++ b/packages/apps/tenant/templates/cilium-lb-pool.yaml @@ -12,11 +12,7 @@ (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) .Values.ingress }} -{{- $hasOwnGateway := .Values.gateway | default false }} -{{- if and $hasOwnGateway (not $ipsList) }} -{{- fail "publishing.externalIPs is empty — a tenant Gateway has no CiliumLoadBalancerIPPool to draw from. The Gateway Service would stay in with no address. Set publishing.externalIPs on the platform (or publishing.exposure=loadBalancer with a BGP / L2-announce pool you manage separately) before enabling tenant.spec.gateway=true." }} -{{- end }} -{{- if and (or $isPublishingIngressLB $hasOwnGateway) $ipsList }} +{{- if and $isPublishingIngressLB $ipsList }} apiVersion: cilium.io/v2 kind: CiliumLoadBalancerIPPool metadata: diff --git a/packages/apps/tenant/templates/gateway.yaml b/packages/apps/tenant/templates/gateway.yaml deleted file mode 100644 index 7dc9b733..00000000 --- a/packages/apps/tenant/templates/gateway.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.gateway }} -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: gateway - namespace: {{ include "tenant.name" . }} - labels: - sharding.fluxcd.io/key: tenants - internal.cozystack.io/tenantmodule: "true" - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - apps.cozystack.io/application.kind: Gateway - apps.cozystack.io/application.group: apps.cozystack.io - apps.cozystack.io/application.name: gateway -spec: - chartRef: - kind: ExternalArtifact - name: cozystack-gateway-application-default-gateway - namespace: cozy-system - interval: 5m - timeout: 10m - install: - remediation: - retries: -1 - upgrade: - force: true - remediation: - retries: -1 - valuesFrom: - - kind: Secret - name: cozystack-values -{{- end }} diff --git a/packages/apps/tenant/templates/namespace.yaml b/packages/apps/tenant/templates/namespace.yaml index 34a1d39c..dfb83730 100644 --- a/packages/apps/tenant/templates/namespace.yaml +++ b/packages/apps/tenant/templates/namespace.yaml @@ -29,23 +29,6 @@ {{- $ingress = $tenantName }} {{- end }} -{{/* - Gateway reference is NOT inherited from parent. Unlike ingress (where nginx - serves from the publishing namespace and any tenant can attach via the shared - apex), a Gateway's allowedRoutes.namespaces selector is restrictive: a child - tenant namespace is not on the parent Gateway's whitelist, so an HTTPRoute - from the child would be NotAllowedByListeners. The parent Gateway's cert also - only covers and *. (one level), and a child apex is - . — two levels deep — so even if admission let the route - through, TLS would break. Require explicit tenant.spec.gateway=true on each - tenant that wants Gateway-attached apps; otherwise apps fall back to the - inherited Ingress. -*/}} -{{- $gateway := "" }} -{{- if .Values.gateway }} -{{- $gateway = $tenantName }} -{{- end }} - {{- $monitoring := $parentNamespace.monitoring | default "" }} {{- if .Values.monitoring }} {{- $monitoring = $tenantName }} @@ -78,7 +61,6 @@ metadata: {{/* Labels for network policies */}} namespace.cozystack.io/etcd: {{ $etcd | quote }} namespace.cozystack.io/ingress: {{ $ingress | quote }} - namespace.cozystack.io/gateway: {{ $gateway | quote }} namespace.cozystack.io/monitoring: {{ $monitoring | quote }} namespace.cozystack.io/seaweedfs: {{ $seaweedfs | quote }} namespace.cozystack.io/host: {{ $computedHost | quote }} @@ -110,7 +92,6 @@ stringData: _namespace: etcd: {{ $etcd | quote }} ingress: {{ $ingress | quote }} - gateway: {{ $gateway | quote }} monitoring: {{ $monitoring | quote }} seaweedfs: {{ $seaweedfs | quote }} host: {{ $computedHost | quote }} diff --git a/packages/apps/tenant/tests/exposure_test.yaml b/packages/apps/tenant/tests/exposure_test.yaml index 89e0e0f5..e86fab5b 100644 --- a/packages/apps/tenant/tests/exposure_test.yaml +++ b/packages/apps/tenant/tests/exposure_test.yaml @@ -139,78 +139,3 @@ tests: asserts: - hasDocuments: count: 0 - - - it: gateway=true in any tenant renders pool with namespace-only selector - set: - gateway: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - asserts: - - hasDocuments: - count: 1 - - equal: - path: metadata.name - value: root-exposure - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - equal: - path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] - value: tenant-root - - - it: gateway=true in a non-publishing tenant renders its own pool (not tied to publishing tenant) - set: - gateway: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - release: - name: tenant-u1 - namespace: tenant-u1 - asserts: - - hasDocuments: - count: 1 - - equal: - path: metadata.name - value: u1-exposure - - equal: - path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] - value: tenant-u1 - - - it: gateway=true + ingress=true + loadBalancer mode renders a single shared pool (no overlapping CIDR conflict) - set: - gateway: true - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 1 - - equal: - path: metadata.name - value: root-exposure - - - it: gateway=true with ingress=false still renders pool (gateway alone is enough) - set: - gateway: true - ingress: false - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - asserts: - - hasDocuments: - count: 1 - - - it: gateway=true with empty externalIPs fails chart render with explicit message - set: - gateway: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "" - asserts: - - failedTemplate: - errorMessage: "publishing.externalIPs is empty — a tenant Gateway has no CiliumLoadBalancerIPPool to draw from. The Gateway Service would stay in with no address. Set publishing.externalIPs on the platform (or publishing.exposure=loadBalancer with a BGP / L2-announce pool you manage separately) before enabling tenant.spec.gateway=true." diff --git a/packages/apps/tenant/values.schema.json b/packages/apps/tenant/values.schema.json index 7a1f3df0..28d9ac1e 100644 --- a/packages/apps/tenant/values.schema.json +++ b/packages/apps/tenant/values.schema.json @@ -22,11 +22,6 @@ "type": "boolean", "default": false }, - "gateway": { - "description": "Deploy own Gateway API Gateway (backed by Cilium Gateway API controller).", - "type": "boolean", - "default": false - }, "seaweedfs": { "description": "Deploy own SeaweedFS.", "type": "boolean", diff --git a/packages/apps/tenant/values.yaml b/packages/apps/tenant/values.yaml index d55ddcf9..f9f8a999 100644 --- a/packages/apps/tenant/values.yaml +++ b/packages/apps/tenant/values.yaml @@ -14,9 +14,6 @@ monitoring: false ## @param {bool} ingress - Deploy own Ingress Controller. ingress: false -## @param {bool} gateway - Deploy own Gateway API Gateway (backed by Cilium Gateway API controller). -gateway: false - ## @param {bool} seaweedfs - Deploy own SeaweedFS. seaweedfs: false diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index a2b6603e..9d6a52b2 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -36,31 +36,32 @@ virtctl ssh @ ### Common parameters -| Name | Description | Type | Value | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `external` | Enable external access from outside the cluster. | `bool` | `false` | -| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | -| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | -| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | -| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | -| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | -| `disks` | List of disks to attach. | `[]object` | `[]` | -| `disks[i].name` | Disk name. | `string` | `""` | -| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | -| `networks` | Networks to attach the VM to. | `[]object` | `[]` | -| `networks[i].name` | Network attachment name. | `string` | `""` | -| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | -| `subnets[i].name` | Network attachment name. | `string` | `""` | -| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | -| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | -| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | -| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | -| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | -| `cloudInit` | Cloud-init user data. | `string` | `""` | -| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `external` | Enable external access from outside the cluster. | `bool` | `false` | +| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | +| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | +| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool` | `true` | +| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | +| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | +| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | +| `disks` | List of disks to attach. | `[]object` | `[]` | +| `disks[i].name` | Disk name. | `string` | `""` | +| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `networks` | Networks to attach the VM to. | `[]object` | `[]` | +| `networks[i].name` | Network attachment name. | `string` | `""` | +| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | +| `subnets[i].name` | Network attachment name. | `string` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | +| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | +| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | +| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | +| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | +| `cloudInit` | Cloud-init user data. | `string` | `""` | +| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | ## U Series diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index b7e3a420..cfeffa81 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -9,7 +9,10 @@ metadata: {{- if .Values.external }} service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: - networking.cozystack.io/wholeIP: "true" + networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} + {{- if eq .Values.externalMethod "PortList" }} + networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }} + {{- end }} {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 34f7f634..01bd30a9 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -26,6 +26,11 @@ "type": "integer" } }, + "externalAllowICMP": { + "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.", + "type": "boolean", + "default": true + }, "runStrategy": { "description": "Requested running state of the VirtualMachineInstance", "type": "string", diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index f07b75d3..92e399c2 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -31,6 +31,9 @@ externalMethod: PortList externalPorts: - 22 +## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. +externalAllowICMP: true + ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance ## @value Always - VMI should always be running ## @value Halted - VMI should never be running diff --git a/packages/core/platform/sources/cert-manager.yaml b/packages/core/platform/sources/cert-manager.yaml index 857e9a08..0907021e 100644 --- a/packages/core/platform/sources/cert-manager.yaml +++ b/packages/core/platform/sources/cert-manager.yaml @@ -13,7 +13,6 @@ spec: - name: default dependsOn: - cozystack.networking - - cozystack.gateway-api-crds components: - name: cert-manager-crds path: system/cert-manager-crds diff --git a/packages/core/platform/sources/gateway-application.yaml b/packages/core/platform/sources/gateway-application.yaml deleted file mode 100644 index a75c37b1..00000000 --- a/packages/core/platform/sources/gateway-application.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: PackageSource -metadata: - name: cozystack.gateway-application -spec: - sourceRef: - kind: OCIRepository - name: cozystack-packages - namespace: cozy-system - path: / - variants: - - name: default - dependsOn: - - cozystack.gateway-api-crds - - cozystack.cert-manager - libraries: - - name: cozy-lib - path: library/cozy-lib - components: - - name: gateway - path: extra/gateway - libraries: ["cozy-lib"] diff --git a/packages/core/platform/sources/gateway-api-crds.yaml b/packages/core/platform/sources/hami.yaml similarity index 51% rename from packages/core/platform/sources/gateway-api-crds.yaml rename to packages/core/platform/sources/hami.yaml index f6bcff9f..0184e405 100644 --- a/packages/core/platform/sources/gateway-api-crds.yaml +++ b/packages/core/platform/sources/hami.yaml @@ -2,7 +2,7 @@ apiVersion: cozystack.io/v1alpha1 kind: PackageSource metadata: - name: cozystack.gateway-api-crds + name: cozystack.hami spec: sourceRef: kind: OCIRepository @@ -11,10 +11,14 @@ spec: path: / variants: - name: default - dependsOn: [] + dependsOn: + - cozystack.gpu-operator components: - - name: gateway-api-crds - path: system/gateway-api-crds + - name: hami + path: system/hami + valuesFiles: + - values.yaml install: - namespace: cozy-gateway-api - releaseName: gateway-api-crds + privileged: true + namespace: cozy-hami + releaseName: hami diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 9787cb19..088383ad 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,6 +52,8 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator + - name: kubernetes-hami + path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds diff --git a/packages/core/platform/sources/networking.yaml b/packages/core/platform/sources/networking.yaml index 3a568f59..bece76f8 100644 --- a/packages/core/platform/sources/networking.yaml +++ b/packages/core/platform/sources/networking.yaml @@ -13,8 +13,7 @@ spec: - name: noop components: [] - name: cilium - dependsOn: - - cozystack.gateway-api-crds + dependsOn: [] components: - name: cilium path: system/cilium @@ -35,8 +34,7 @@ spec: dependsOn: - cilium - name: cilium-kilo - dependsOn: - - cozystack.gateway-api-crds + dependsOn: [] components: - name: cilium path: system/cilium @@ -62,8 +60,7 @@ spec: - cilium # Generic Cilium variant for non-Talos clusters (kubeadm, k3s, RKE2, etc.) - name: cilium-generic - dependsOn: - - cozystack.gateway-api-crds + dependsOn: [] components: - name: cilium path: system/cilium @@ -84,8 +81,7 @@ spec: dependsOn: - cilium - name: kubeovn-cilium - dependsOn: - - cozystack.gateway-api-crds + dependsOn: [] components: - name: cilium path: system/cilium @@ -116,8 +112,7 @@ spec: - cilium # Generic KubeOVN+Cilium variant for non-Talos clusters (kubeadm, k3s, RKE2, etc.) - name: kubeovn-cilium-generic - dependsOn: - - cozystack.gateway-api-crds + dependsOn: [] components: - name: cilium path: system/cilium diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index fa4db7bc..6e1ae25a 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -31,8 +31,6 @@ stringData: expose-ingress: {{ .Values.publishing.ingressName | quote }} expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }} expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }} - gateway-enabled: {{ .Values.gateway.enabled | default false | quote }} - gateway-attached-namespaces: {{ .Values.gateway.attachedNamespaces | default (list) | join "," | quote }} cluster-domain: {{ .Values.networking.clusterDomain | quote }} api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }} {{- with .Values.branding }} diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index ced0a322..66ec98b2 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -10,6 +10,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} +{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index e901b09d..6c587c82 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -105,9 +105,6 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} -{{- /* lineage-controller-webhook DaemonSet */ -}} -{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} -{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} @@ -118,7 +115,6 @@ {{- end }} # Common Packages -{{include "cozystack.platform.package.default" (list "cozystack.gateway-api-crds" $) }} {{include "cozystack.platform.package.default" (list "cozystack.cert-manager" $) }} {{include "cozystack.platform.package.default" (list "cozystack.flux-plunger" $) }} {{include "cozystack.platform.package.default" (list "cozystack.victoria-metrics-operator" $) }} @@ -127,7 +123,6 @@ {{- $_ := set $tenantComponents "tenant" (dict "values" $tenantClusterValues) }} {{include "cozystack.platform.package" (list "cozystack.tenant-application" "default" $ $tenantComponents) }} {{include "cozystack.platform.package.default" (list "cozystack.ingress-application" $) }} -{{include "cozystack.platform.package.default" (list "cozystack.gateway-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.seaweedfs-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.info-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.monitoring-application" $) }} diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 2f1b5002..605e23bc 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -83,15 +83,6 @@ publishing: certificates: solver: http01 # "http01" or "dns01" issuerName: letsencrypt-prod - # Rate-limit note: every tenant with gateway=true issues one - # Certificate for its wildcard + apex hostnames. If many tenants share - # the same apex domain with issuerName=letsencrypt-prod, the cluster - # can hit the Let's Encrypt limits (50 certs / registered domain / - # week, 5 duplicate certs / week, 300 new orders / account / 3h). - # Use letsencrypt-stage for non-production clusters or gate the - # number of certificates per tenant via tenant.spec.resourceQuotas - # with count/certificates.cert-manager.io. See - # https://letsencrypt.org/docs/rate-limits/. # Authentication configuration authentication: oidc: @@ -103,42 +94,6 @@ authentication: # userinfo, logout) through this URL while keeping browser redirects on the external URL. # Example: http://keycloak-http.cozy-keycloak.svc:8080/realms/cozy keycloakInternalUrl: "" -# Gateway API configuration -gateway: - # Enable Gateway API support across the platform. When true: - # - cert-manager ClusterIssuers use an http01.gatewayHTTPRoute solver - # that attaches to the root tenant's Gateway, so Certificate issuance - # works for tenants whose ingress is via Gateway API rather than - # ingress-nginx. - # - Exposed services (dashboard, keycloak, harbor, bucket, grafana, - # cozystack-api, vm-exportproxy, cdi-uploadproxy) render an - # HTTPRoute attached to the tenant's Gateway. - # Per-tenant opt-in is still governed by tenant.spec.gateway=true, which - # materialises the Gateway itself. This flag only controls platform-wide - # Gateway API integration (cert-manager solver + per-service HTTPRoutes). - enabled: false - # Namespaces that are allowed to attach HTTPRoute or TLSRoute to a tenant - # Gateway. The publishing tenant namespace is always included implicitly. - # Additional entries must be the exact namespace name (matched on the - # built-in kubernetes.io/metadata.name label). Tenants not on this list - # cannot attach a route to somebody else's Gateway, which prevents - # hostname hijacking across tenants. - # - # The `default` namespace is included because the Kubernetes API's - # TLSRoute (shipped by the cozystack-api package) has to live next to - # the `kubernetes` Service it points at, which is always in `default`. - attachedNamespaces: - - cozy-cert-manager - - cozy-dashboard - - cozy-keycloak - - cozy-system - - cozy-harbor - - cozy-bucket - - cozy-kubevirt - - cozy-kubevirt-cdi - - cozy-monitoring - - cozy-linstor-gui - - default # Pod scheduling configuration scheduling: globalAppTopologySpreadConstraints: "" diff --git a/packages/extra/etcd/Makefile b/packages/extra/etcd/Makefile index f37d6e1e..2b3ed61e 100644 --- a/packages/extra/etcd/Makefile +++ b/packages/extra/etcd/Makefile @@ -5,3 +5,6 @@ include ../../../hack/package.mk generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh + +test: + helm unittest . diff --git a/packages/extra/etcd/templates/hook/job.yaml b/packages/extra/etcd/templates/hook/job.yaml deleted file mode 100644 index 3bf2f84b..00000000 --- a/packages/extra/etcd/templates/hook/job.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- $shouldUpdateCerts := true }} -{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }} -{{- if $configMap }} - {{- $deployedVersion := index $configMap "data" "version" }} - {{- if $deployedVersion | semverCompare ">= 2.6.1" }} - {{- $shouldUpdateCerts = false }} - {{- end }} -{{- end }} - -{{- if $shouldUpdateCerts }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - template: - metadata: - labels: - policy.cozystack.io/allow-to-apiserver: "true" - spec: - serviceAccountName: etcd-hook - containers: - - name: kubectl - image: docker.io/alpine/k8s:1.33.4 - command: - - sh - args: - - -exc - - |- - kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls - sleep 10 - kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls - kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd - restartPolicy: Never -{{- end }} diff --git a/packages/extra/etcd/templates/hook/role.yaml b/packages/extra/etcd/templates/hook/role.yaml deleted file mode 100644 index 327eeadb..00000000 --- a/packages/extra/etcd/templates/hook/role.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - name: etcd-hook -rules: -- apiGroups: - - "" - resources: - - secrets - - pods - verbs: - - get - - list - - watch - - delete -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch diff --git a/packages/extra/etcd/templates/hook/rolebinding.yaml b/packages/extra/etcd/templates/hook/rolebinding.yaml deleted file mode 100644 index 0ee0ffd1..00000000 --- a/packages/extra/etcd/templates/hook/rolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: etcd-hook -subjects: - - kind: ServiceAccount - name: etcd-hook - namespace: {{ .Release.Namespace | quote }} diff --git a/packages/extra/etcd/templates/hook/serviceaccount.yaml b/packages/extra/etcd/templates/hook/serviceaccount.yaml deleted file mode 100644 index 552fb5fc..00000000 --- a/packages/extra/etcd/templates/hook/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: etcd-hook - annotations: - helm.sh/hook: post-upgrade - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/packages/extra/etcd/templates/version.yaml b/packages/extra/etcd/templates/version.yaml deleted file mode 100644 index cc9375bb..00000000 --- a/packages/extra/etcd/templates/version.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: etcd-deployed-version -data: - version: {{ .Chart.Version }} diff --git a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml new file mode 100644 index 00000000..0e4f5aa3 --- /dev/null +++ b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml @@ -0,0 +1,34 @@ +suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook + +release: + name: etcd + namespace: tenant-root + +templates: + - templates/check-release-name.yaml + - templates/dashboard-resourcemap.yaml + - templates/datastore.yaml + - templates/etcd-defrag.yaml + - templates/hook/job.yaml + - templates/podscrape.yaml + - templates/prometheus-rules.yaml + - templates/version.yaml + +tests: + - it: renders no Job named etcd-hook + documentSelector: + path: metadata.name + value: etcd-hook + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 + + - it: renders no ConfigMap named etcd-deployed-version + documentSelector: + path: metadata.name + value: etcd-deployed-version + skipEmptyTemplates: true + asserts: + - hasDocuments: + count: 0 diff --git a/packages/extra/gateway/.helmignore b/packages/extra/gateway/.helmignore deleted file mode 100644 index ab636230..00000000 --- a/packages/extra/gateway/.helmignore +++ /dev/null @@ -1,2 +0,0 @@ -.helmignore -/Makefile diff --git a/packages/extra/gateway/Makefile b/packages/extra/gateway/Makefile deleted file mode 100644 index f5d77613..00000000 --- a/packages/extra/gateway/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -NAME=gateway - -include ../../../hack/package.mk - -generate: - cozyvalues-gen -v values.yaml -s values.schema.json -r README.md - -test: - helm unittest . diff --git a/packages/extra/gateway/README.md b/packages/extra/gateway/README.md deleted file mode 100644 index e25d181e..00000000 --- a/packages/extra/gateway/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Cozystack Tenant Gateway - -Per-tenant Gateway API Gateway backed by Cilium. Installed automatically when `tenant.spec.gateway=true` on the publishing tenant. - -## Parameters - -### Common parameters - -| Name | Description | Type | Value | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------- | ---------------------------------------- | -| `gatewayClassName` | GatewayClass to attach the tenant Gateway to. Must exist cluster-wide. Default matches the Cilium-managed class. | `string` | `cilium` | -| `tlsPassthroughServices` | Names (from publishing.exposedServices) whose traffic is TLS-passthrough rather than TLS-terminate. For each such service a dedicated HTTPS listener with tls.mode=Passthrough is rendered on the Gateway, and the service is expected to attach a TLSRoute instead of an HTTPRoute. | `[]string` | `[api, vm-exportproxy, cdi-uploadproxy]` | - - -## Security model - -Six independent layers protect cross-tenant isolation. Compromising one of them does not bypass the others; admission-time checks (layers 2–5) fail closed (`failurePolicy: Fail`, `validationActions: [Deny]`). - -1. **Namespace whitelist on listeners.** The Gateway's `allowedRoutes.namespaces.from: Selector` matches the built-in `kubernetes.io/metadata.name` label (written by kube-apiserver, unspoofable). It accepts routes from the publishing tenant's namespace plus `publishing.gateway.attachedNamespaces` in the platform chart (default includes the `cozy-*` namespaces for platform services and `default` for the Kubernetes API TLSRoute). A namespace outside the list literally cannot attach any `HTTPRoute` or `TLSRoute` to this Gateway. -2. **`cozystack-gateway-hostname-policy`** — `ValidatingAdmissionPolicy` on `gateway.networking.k8s.io/v1 Gateway` CREATE/UPDATE. Reads `namespaceObject.metadata.labels["namespace.cozystack.io/host"]` and rejects any listener hostname that is not equal to that value or a subdomain of it. `matchConditions` gate the VAP to cozystack-managed namespaces only — Gateways in unrelated namespaces (e.g. `kube-system`) are not touched. -3. **`cozystack-gateway-attached-namespaces-policy`** — VAP on `cozystack.io/v1alpha1 Package` CREATE/UPDATE. Rejects any `tenant-*` entry in `spec.components.platform.values.gateway.attachedNamespaces`. Catches direct `kubectl edit packages.cozystack.io` that would bypass the helm render-time guard in layer 6. -4. **`cozystack-tenant-host-policy`** — VAP on `apps.cozystack.io/v1alpha1 Tenant` CREATE/UPDATE. Rejects setting or changing `spec.host` unless the caller's groups contain `system:masters`, `system:serviceaccounts:cozy-system`, `system:serviceaccounts:cozy-cert-manager`, `system:serviceaccounts:cozy-fluxcd` or `system:serviceaccounts:kube-system`. Closes the path where a tenant user sets `spec.host=dashboard.example.org` on their own tenant to have the tenant chart write a hijacked label into the namespace. -5. **`cozystack-namespace-host-label-policy`** — VAP on core `v1 Namespace` CREATE/UPDATE. Rejects any set or change of the `namespace.cozystack.io/host` label, except by the same trusted-caller whitelist as layer 4. This closes both first-time label writes on CREATE and first-time adds on UPDATE — only cozystack/Flux service accounts (which apply the tenant chart) can stamp the label. -6. **Render-time `fail` in cozystack-basics.** The cozystack-basics chart fails the helm render if `_cluster.gateway-attached-namespaces` contains any `tenant-*` entry. Triggers on the helm-install path before the cluster ever sees the values — complements layer 3 which triggers at `kubectl apply` time. - -For `tenant-root` the allowed host suffix is `publishing.host`; for any `tenant-` that inherits from its parent the suffix is `.`. A child tenant with an independent apex (`customer1.io` instead of a subdomain) is handled correctly because the VAP reads the per-namespace label rather than assuming a subdomain hierarchy. - -## Rate limits - -cert-manager issues one `Certificate` per Gateway release. With `issuerName: letsencrypt-prod` (the default), the `Certificate` for a tenant Gateway counts against the [Let's Encrypt rate limits](https://letsencrypt.org/docs/rate-limits/): - -- 50 new certificates per registered domain per week. -- 5 duplicate certificates per week for the same set of hostnames. -- 300 new orders per account per 3 hours. - -A cluster where many tenants share the same apex domain can exhaust these quickly. Mitigations: - -- Use `publishing.certificates.issuerName: letsencrypt-stage` for non-production clusters (staging does not count against prod quotas). -- Limit the number of simultaneous tenant Gateways per cluster via the platform's package quota, or cap it via `tenant.spec.resourceQuotas` with `count/certificates.cert-manager.io` to limit how many `Certificate` objects a tenant may create. -- For bare-metal or air-gapped deployments consider an internal ACME server or the self-signed `ClusterIssuer` (`selfsigned-cluster-issuer`) that ships alongside the Let's Encrypt issuers. - -Recommended tenant-level quota to contain a misbehaving tenant: - -```yaml -apiVersion: apps.cozystack.io/v1alpha1 -kind: Tenant -spec: - gateway: true - resourceQuotas: - count/certificates.cert-manager.io: "10" -``` - -The default for a fresh tenant is unlimited; operators running shared-apex multi-tenant clusters should set this explicitly (or stage it via the tenant-application default values) before opening `gateway: true` to non-trusted tenants. - -## Known limitations - -- **Upstream application gaps** — some chart-level features (harbor ACL integrations, bucket upstream limitations) remain on ingress-nginx workflows in upstream docs; cozystack tracks those separately as upstream PRs. -- **Supported ACME issuers** — `publishing.certificates.issuerName` for Gateway-based tenants must be `letsencrypt-prod` or `letsencrypt-stage` (the chart maps those names to concrete ACME server URLs). To support another ACME provider, extend `templates/issuer.yaml` with an additional branch. - diff --git a/packages/extra/gateway/charts/cozy-lib b/packages/extra/gateway/charts/cozy-lib deleted file mode 120000 index e1813509..00000000 --- a/packages/extra/gateway/charts/cozy-lib +++ /dev/null @@ -1 +0,0 @@ -../../../library/cozy-lib \ No newline at end of file diff --git a/packages/extra/gateway/config.json b/packages/extra/gateway/config.json deleted file mode 100644 index b2f956f3..00000000 --- a/packages/extra/gateway/config.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "comments": { - "format": "##" - }, - "tags": { - "param": "@param", - "section": "@section", - "descriptionStart": "@descriptionStart", - "descriptionEnd": "@descriptionEnd", - "skip": "@skip", - "extra": "@extra" - }, - "modifiers": { - "array": "array", - "object": "object", - "string": "string", - "nullable": "nullable", - "default": "default" - }, - "regexp": { - "paramsSectionTitle": "Parameters" - } -} diff --git a/packages/extra/gateway/templates/gateway.yaml b/packages/extra/gateway/templates/gateway.yaml deleted file mode 100644 index 955ae427..00000000 --- a/packages/extra/gateway/templates/gateway.yaml +++ /dev/null @@ -1,108 +0,0 @@ -{{- $host := (index .Values._namespace "host") | default "" }} -{{- if not $host }} -{{- fail "_namespace.host is empty — the tenant has no apex domain. Either set tenant.spec.host on this tenant or inherit it from a parent that has host set." }} -{{- end }} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ trimPrefix "tenant-" .Release.Namespace }}-gateway-tls -spec: - secretName: {{ trimPrefix "tenant-" .Release.Namespace }}-gateway-tls - issuerRef: - name: gateway - kind: Issuer - dnsNames: - - {{ $host | quote }} - - {{ printf "*.%s" $host | quote }} ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: cozystack - labels: - cozystack.io/gateway: {{ .Release.Namespace }} -spec: - gatewayClassName: {{ .Values.gatewayClassName | quote }} - listeners: - {{- $extraNs := list }} - {{- range splitList "," ((index .Values._cluster "gateway-attached-namespaces") | default "") }} - {{- $ns := . | trim }} - {{- if $ns }} - {{- $extraNs = append $extraNs $ns }} - {{- end }} - {{- end }} - {{- $allowedNamespaces := concat (list .Release.Namespace) $extraNs | uniq }} - {{- $allowedNamespacesSelector := dict "matchExpressions" (list (dict - "key" "kubernetes.io/metadata.name" - "operator" "In" - "values" $allowedNamespaces)) }} - - name: http - protocol: HTTP - port: 80 - allowedRoutes: - namespaces: - from: Selector - selector: - {{- toYaml $allowedNamespacesSelector | nindent 10 }} - - name: https - protocol: HTTPS - port: 443 - hostname: {{ printf "*.%s" $host | quote }} - tls: - mode: Terminate - certificateRefs: - - name: {{ trimPrefix "tenant-" .Release.Namespace }}-gateway-tls - allowedRoutes: - namespaces: - from: Selector - selector: - {{- toYaml $allowedNamespacesSelector | nindent 10 }} - - name: https-apex - protocol: HTTPS - port: 443 - hostname: {{ $host | quote }} - tls: - mode: Terminate - certificateRefs: - - name: {{ trimPrefix "tenant-" .Release.Namespace }}-gateway-tls - allowedRoutes: - namespaces: - from: Selector - selector: - {{- toYaml $allowedNamespacesSelector | nindent 10 }} - {{- $exposedServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} - {{- range .Values.tlsPassthroughServices }} - {{- $svc := . }} - {{- if has $svc $exposedServices }} - - name: {{ printf "tls-%s" $svc | quote }} - protocol: TLS - port: 443 - hostname: {{ printf "%s.%s" $svc $host | quote }} - tls: - mode: Passthrough - allowedRoutes: - kinds: - - group: gateway.networking.k8s.io - kind: TLSRoute - namespaces: - from: Selector - selector: - {{- toYaml $allowedNamespacesSelector | nindent 10 }} - {{- end }} - {{- end }} ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: http-to-https-redirect -spec: - parentRefs: - - name: cozystack - sectionName: http - rules: - - filters: - - type: RequestRedirect - requestRedirect: - scheme: https - statusCode: 301 diff --git a/packages/extra/gateway/templates/issuer.yaml b/packages/extra/gateway/templates/issuer.yaml deleted file mode 100644 index 44ce7bda..00000000 --- a/packages/extra/gateway/templates/issuer.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{- $issuerName := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} -{{- $solver := (index .Values._cluster "solver") | default "http01" }} -{{- $acmeServer := "" }} -{{- if eq $issuerName "letsencrypt-prod" }} - {{- $acmeServer = "https://acme-v02.api.letsencrypt.org/directory" }} -{{- else if eq $issuerName "letsencrypt-stage" }} - {{- $acmeServer = "https://acme-staging-v02.api.letsencrypt.org/directory" }} -{{- else }} - {{- fail (printf "packages/extra/gateway currently supports publishing.certificates.issuerName=letsencrypt-prod or letsencrypt-stage (got %q). For other issuers, extend templates/issuer.yaml to map the name to the corresponding ACME server URL." $issuerName) }} -{{- end }} -{{- /* - Let's Encrypt refuses to issue wildcard certificates via HTTP-01 (see - cert-manager/cert-manager-crds/.../crd-cert-manager.io_clusterissuers.yaml - at the http01 field description, and LE policy - https://letsencrypt.org/docs/challenge-types/#http-01-challenge). The - tenant Gateway's Certificate requests apex + *.apex, so HTTP-01 would - loop the Order forever and the HTTPS listeners would never get a cert. - Require dns01 (or selfsigned) for Gateway API tenants — fail fast so - the operator sees the mismatch at install time instead of during a - stuck cert-manager challenge. -*/}} -{{- if eq $solver "http01" }} - {{- fail "publishing.certificates.solver=http01 cannot issue the wildcard certificate the tenant Gateway needs (Let's Encrypt refuses HTTP-01 for wildcard identifiers). Set publishing.certificates.solver=dns01 (with a Cloudflare API token Secret in cozy-cert-manager) before enabling tenant.spec.gateway=true, or use the self-signed ClusterIssuer for air-gapped clusters." }} -{{- else if ne $solver "dns01" }} - {{- fail (printf "packages/extra/gateway currently supports publishing.certificates.solver=dns01 only (got %q). Extend templates/issuer.yaml to add additional solver types." $solver) }} -{{- end }} ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: gateway -spec: - acme: - server: {{ $acmeServer }} - privateKeySecretRef: - name: gateway-acme-account - solvers: - - dns01: - cloudflare: - apiTokenSecretRef: - name: cloudflare-api-token-secret - key: api-token diff --git a/packages/extra/gateway/tests/gateway_test.yaml b/packages/extra/gateway/tests/gateway_test.yaml deleted file mode 100644 index 52c70ba5..00000000 --- a/packages/extra/gateway/tests/gateway_test.yaml +++ /dev/null @@ -1,284 +0,0 @@ -suite: tenant gateway -templates: - - templates/gateway.yaml - - templates/issuer.yaml - -release: - name: gateway - namespace: tenant-root - -tests: - - it: renders Certificate, Gateway and redirect HTTPRoute when IPs and host are set - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - issuer-name: letsencrypt-prod - _namespace: - host: example.org - asserts: - - template: templates/gateway.yaml - hasDocuments: - count: 3 - - template: templates/gateway.yaml - documentIndex: 0 - equal: - path: kind - value: Certificate - - template: templates/gateway.yaml - documentIndex: 0 - equal: - path: spec.dnsNames - value: - - example.org - - "*.example.org" - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: kind - value: Gateway - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.gatewayClassName - value: cilium - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].name - value: http - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[1].hostname - value: "*.example.org" - - template: templates/gateway.yaml - documentIndex: 2 - equal: - path: kind - value: HTTPRoute - - template: templates/gateway.yaml - documentIndex: 2 - equal: - path: spec.rules[0].filters[0].requestRedirect.scheme - value: https - - - it: child tenant renders its own Gateway with the child's per-tenant apex - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.11" - _namespace: - host: alice.example.org - release: - namespace: tenant-alice - asserts: - - template: templates/gateway.yaml - hasDocuments: - count: 3 - - template: templates/gateway.yaml - documentIndex: 0 - equal: - path: spec.dnsNames - value: - - alice.example.org - - "*.alice.example.org" - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[1].hostname - value: "*.alice.example.org" - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[2].hostname - value: "alice.example.org" - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].allowedRoutes.namespaces.selector.matchExpressions[0].values - value: - - tenant-alice - - - it: empty _namespace.host fails with explicit error - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - _namespace: - host: "" - asserts: - - template: templates/gateway.yaml - failedTemplate: - errorMessage: "_namespace.host is empty — the tenant has no apex domain. Either set tenant.spec.host on this tenant or inherit it from a parent that has host set." - - - it: allowedRoutes whitelist defaults to publishing namespace only when no extra namespaces are set - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - _namespace: - host: example.org - asserts: - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].allowedRoutes.namespaces.from - value: Selector - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].allowedRoutes.namespaces.selector.matchExpressions[0].key - value: kubernetes.io/metadata.name - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].allowedRoutes.namespaces.selector.matchExpressions[0].operator - value: In - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].allowedRoutes.namespaces.selector.matchExpressions[0].values - value: - - tenant-root - - - it: allowedRoutes whitelist prepends publishing namespace to the gateway-attached-namespaces list and filters empty entries - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - gateway-attached-namespaces: "cozy-cert-manager,,cozy-dashboard, cozy-keycloak," - _namespace: - host: example.org - asserts: - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[0].allowedRoutes.namespaces.selector.matchExpressions[0].values - value: - - tenant-root - - cozy-cert-manager - - cozy-dashboard - - cozy-keycloak - - - it: Certificate references the per-tenant Issuer (not a shared ClusterIssuer) so each tenant gets isolated ACME state - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - issuer-name: letsencrypt-prod - _namespace: - host: example.org - asserts: - - template: templates/gateway.yaml - documentIndex: 0 - equal: - path: kind - value: Certificate - - template: templates/gateway.yaml - documentIndex: 0 - equal: - path: spec.issuerRef.kind - value: Issuer - - template: templates/gateway.yaml - documentIndex: 0 - equal: - path: spec.issuerRef.name - value: gateway - - template: templates/issuer.yaml - hasDocuments: - count: 1 - - template: templates/issuer.yaml - equal: - path: kind - value: Issuer - - template: templates/issuer.yaml - equal: - path: metadata.name - value: gateway - - template: templates/issuer.yaml - equal: - path: spec.acme.server - value: https://acme-v02.api.letsencrypt.org/directory - - template: templates/issuer.yaml - equal: - path: spec.acme.solvers[0].dns01.cloudflare.apiTokenSecretRef.name - value: cloudflare-api-token-secret - - template: templates/issuer.yaml - equal: - path: spec.acme.solvers[0].dns01.cloudflare.apiTokenSecretRef.key - value: api-token - - - it: chart render fails when publishing.certificates.solver=http01 because Let's Encrypt refuses HTTP-01 for wildcard certs - set: - _cluster: - solver: http01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - issuer-name: letsencrypt-prod - _namespace: - host: example.org - asserts: - - template: templates/issuer.yaml - failedTemplate: - errorMessage: "publishing.certificates.solver=http01 cannot issue the wildcard certificate the tenant Gateway needs (Let's Encrypt refuses HTTP-01 for wildcard identifiers). Set publishing.certificates.solver=dns01 (with a Cloudflare API token Secret in cozy-cert-manager) before enabling tenant.spec.gateway=true, or use the self-signed ClusterIssuer for air-gapped clusters." - - - it: Issuer picks the staging ACME server when publishing.certificates.issuerName=letsencrypt-stage - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - issuer-name: letsencrypt-stage - _namespace: - host: example.org - asserts: - - template: templates/issuer.yaml - equal: - path: spec.acme.server - value: https://acme-staging-v02.api.letsencrypt.org/directory - - - it: unknown issuer-name fails chart render with an explicit error - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - issuer-name: my-custom-ca - _namespace: - host: example.org - asserts: - - template: templates/issuer.yaml - failedTemplate: - errorMessage: 'packages/extra/gateway currently supports publishing.certificates.issuerName=letsencrypt-prod or letsencrypt-stage (got "my-custom-ca"). For other issuers, extend templates/issuer.yaml to map the name to the corresponding ACME server URL.' - - - it: HTTPS and HTTPS-apex listeners share the same allowedRoutes whitelist - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - gateway-attached-namespaces: "cozy-dashboard" - _namespace: - host: example.org - asserts: - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[1].allowedRoutes.namespaces.selector.matchExpressions[0].values - value: - - tenant-root - - cozy-dashboard - - template: templates/gateway.yaml - documentIndex: 1 - equal: - path: spec.listeners[2].allowedRoutes.namespaces.selector.matchExpressions[0].values - value: - - tenant-root - - cozy-dashboard diff --git a/packages/extra/gateway/values.schema.json b/packages/extra/gateway/values.schema.json deleted file mode 100644 index 6a093ebb..00000000 --- a/packages/extra/gateway/values.schema.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "title": "Chart Values", - "type": "object", - "properties": { - "gatewayClassName": { - "description": "GatewayClass to attach the tenant Gateway to. Must exist cluster-wide. Default matches the Cilium-managed class.", - "type": "string", - "default": "cilium" - }, - "tlsPassthroughServices": { - "description": "Names (from publishing.exposedServices) whose traffic is TLS-passthrough rather than TLS-terminate. For each such service a dedicated HTTPS listener with tls.mode=Passthrough is rendered on the Gateway, and the service is expected to attach a TLSRoute instead of an HTTPRoute.", - "type": "array", - "default": [ - "api", - "vm-exportproxy", - "cdi-uploadproxy" - ], - "items": { - "type": "string" - } - } - } -} diff --git a/packages/extra/gateway/values.yaml b/packages/extra/gateway/values.yaml deleted file mode 100644 index 74d82248..00000000 --- a/packages/extra/gateway/values.yaml +++ /dev/null @@ -1,12 +0,0 @@ -## -## @section Common parameters -## - -## @param {string} gatewayClassName - GatewayClass to attach the tenant Gateway to. Must exist cluster-wide. Default matches the Cilium-managed class. -gatewayClassName: cilium - -## @param {[]string} tlsPassthroughServices - Names (from publishing.exposedServices) whose traffic is TLS-passthrough rather than TLS-terminate. For each such service a dedicated HTTPS listener with tls.mode=Passthrough is rendered on the Gateway, and the service is expected to attach a TLSRoute instead of an HTTPRoute. -tlsPassthroughServices: - - api - - vm-exportproxy - - cdi-uploadproxy diff --git a/packages/system/bucket/templates/httproute.yaml b/packages/system/bucket/templates/httproute.yaml deleted file mode 100644 index cadc88d8..00000000 --- a/packages/system/bucket/templates/httproute.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- $host := .Values._namespace.host }} -{{- $gateway := .Values._namespace.gateway | default "" }} -{{- if $gateway }} -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: {{ .Values.bucketName }}-ui -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $gateway }} - sectionName: https - hostnames: - - {{ printf "%s.%s" .Values.bucketName $host | quote }} - rules: - - backendRefs: - - name: {{ .Values.bucketName }}-ui - port: 8080 -{{- end }} diff --git a/packages/system/bucket/templates/ingress.yaml b/packages/system/bucket/templates/ingress.yaml index c2d80647..b7ffaf8b 100644 --- a/packages/system/bucket/templates/ingress.yaml +++ b/packages/system/bucket/templates/ingress.yaml @@ -1,9 +1,7 @@ {{- $host := .Values._namespace.host }} {{- $ingress := .Values._namespace.ingress }} -{{- $gateway := .Values._namespace.gateway | default "" }} {{- $solver := (index .Values._cluster "solver") | default "http01" }} {{- $clusterIssuer := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} -{{- if not $gateway }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -34,4 +32,3 @@ spec: name: {{ .Values.bucketName }}-ui port: number: 8080 -{{- end }} diff --git a/packages/system/cert-manager-issuers/Makefile b/packages/system/cert-manager-issuers/Makefile index 8d8245f4..8808dbfd 100644 --- a/packages/system/cert-manager-issuers/Makefile +++ b/packages/system/cert-manager-issuers/Makefile @@ -2,6 +2,3 @@ export NAME=cert-manager-issuers export NAMESPACE=cozy-cert-manager include ../../../hack/package.mk - -test: - helm unittest . diff --git a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml index 3414dbbc..3b582082 100644 --- a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml +++ b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml @@ -1,73 +1,57 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} - -{{- define "cert-manager-issuers.httpSolver" -}} -{{- $gatewayEnabled := index . "gatewayEnabled" -}} -{{- $exposeIngress := index . "exposeIngress" -}} -{{- if eq $gatewayEnabled "true" }} -http01: - gatewayHTTPRoute: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $exposeIngress }} - sectionName: http -{{- else }} -http01: - ingress: - ingressClassName: {{ $exposeIngress }} -{{- end }} -{{- end }} apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-prod -spec: - acme: - privateKeySecretRef: - name: letsencrypt-prod - server: https://acme-v02.api.letsencrypt.org/directory - solvers: - - {{- if eq $solver "dns01" }} - dns01: - cloudflare: - apiTokenSecretRef: - name: cloudflare-api-token-secret - key: api-token - {{- else }} - {{- include "cert-manager-issuers.httpSolver" (dict "gatewayEnabled" $gatewayEnabled "exposeIngress" $exposeIngress) | nindent 8 }} - {{- end }} +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + privateKeySecretRef: + name: letsencrypt-prod + server: https://acme-v02.api.letsencrypt.org/directory + solvers: + - {{- if eq $solver "dns01" }} + dns01: + cloudflare: + apiTokenSecretRef: + name: cloudflare-api-token-secret + key: api-token + {{- else }} + http01: + ingress: + ingressClassName: {{ $exposeIngress }} + {{- end }} ---- +--- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: letsencrypt-stage -spec: - acme: - privateKeySecretRef: +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-stage +spec: + acme: + privateKeySecretRef: name: letsencrypt-stage - server: https://acme-staging-v02.api.letsencrypt.org/directory - solvers: - - {{- if eq $solver "dns01" }} - dns01: - cloudflare: - apiTokenSecretRef: - name: cloudflare-api-token-secret - key: api-token - {{- else }} - {{- include "cert-manager-issuers.httpSolver" (dict "gatewayEnabled" $gatewayEnabled "exposeIngress" $exposeIngress) | nindent 8 }} - {{- end }} + server: https://acme-staging-v02.api.letsencrypt.org/directory + solvers: + - {{- if eq $solver "dns01" }} + dns01: + cloudflare: + apiTokenSecretRef: + name: cloudflare-api-token-secret + key: api-token + {{- else }} + http01: + ingress: + ingressClassName: {{ $exposeIngress }} + {{- end }} ---- +--- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: selfsigned-cluster-issuer -spec: +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-cluster-issuer +spec: selfSigned: {} diff --git a/packages/system/cert-manager-issuers/tests/solver_test.yaml b/packages/system/cert-manager-issuers/tests/solver_test.yaml deleted file mode 100644 index 67f76b80..00000000 --- a/packages/system/cert-manager-issuers/tests/solver_test.yaml +++ /dev/null @@ -1,103 +0,0 @@ -suite: cluster-issuer solver selection -templates: - - templates/cluster-issuers.yaml - -release: - name: cert-manager-issuers - namespace: cozy-cert-manager - -tests: - - it: default renders ingress-nginx HTTP-01 solver on both ACME issuers - set: - _cluster: - solver: http01 - expose-ingress: tenant-root - asserts: - - documentIndex: 0 - equal: - path: metadata.name - value: letsencrypt-prod - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.ingress.ingressClassName - value: tenant-root - - documentIndex: 0 - notExists: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute - - documentIndex: 1 - equal: - path: metadata.name - value: letsencrypt-stage - - documentIndex: 1 - equal: - path: spec.acme.solvers[0].http01.ingress.ingressClassName - value: tenant-root - - - it: gateway-enabled=true renders gatewayHTTPRoute solver on both ACME issuers - set: - _cluster: - solver: http01 - expose-ingress: tenant-root - gateway-enabled: "true" - asserts: - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].group - value: gateway.networking.k8s.io - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].kind - value: Gateway - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].name - value: cozystack - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].namespace - value: tenant-root - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName - value: http - - documentIndex: 0 - notExists: - path: spec.acme.solvers[0].http01.ingress - - documentIndex: 1 - equal: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].name - value: cozystack - - - it: solver=dns01 is unaffected by gateway-enabled flag - set: - _cluster: - solver: dns01 - expose-ingress: tenant-root - gateway-enabled: "true" - asserts: - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].dns01.cloudflare.apiTokenSecretRef.name - value: cloudflare-api-token-secret - - documentIndex: 0 - notExists: - path: spec.acme.solvers[0].http01 - - documentIndex: 1 - equal: - path: spec.acme.solvers[0].dns01.cloudflare.apiTokenSecretRef.name - value: cloudflare-api-token-secret - - - it: gateway-enabled defaults to ingress solver when unset - set: - _cluster: - solver: http01 - expose-ingress: tenant-root - gateway-enabled: "" - asserts: - - documentIndex: 0 - equal: - path: spec.acme.solvers[0].http01.ingress.ingressClassName - value: tenant-root - - documentIndex: 0 - notExists: - path: spec.acme.solvers[0].http01.gatewayHTTPRoute diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index af03c972..9aba7cb9 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -18,20 +18,7 @@ cilium: tag: 1.19.3 digest: "sha256:700f06f4803a838a8e830be5ace4650e3ad82bdefabfb2f4d110368d307a5efb" envoy: - enabled: true - # Upstream default is {} — no requests/limits. The embedded envoy DaemonSet - # idles at ~100 MB RAM/node and grows under L7 traffic; without requests - # the DaemonSet schedules onto any node with no reserved headroom, and - # without a memory limit a runaway listener could OOM the node. - # The baseline matches the commented-out example in the upstream Helm values. - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 1Gi - gatewayAPI: - enabled: true + enabled: false rollOutCiliumPods: true operator: rollOutPods: true diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml index 13352680..58f166f6 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cozy-proxy description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend type: application -version: 0.2.0 -appVersion: 0.2.0 +version: 0.3.0 +appVersion: 0.3.0 diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml index 8cde5bed..e143e926 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/cozystack/cozystack/cozy-proxy - tag: v0.2.0 + tag: v0.3.0 pullPolicy: IfNotPresent daemonset: diff --git a/packages/system/cozystack-api/templates/api-ingress.yaml b/packages/system/cozystack-api/templates/api-ingress.yaml index 454f2367..9226d887 100644 --- a/packages/system/cozystack-api/templates/api-ingress.yaml +++ b/packages/system/cozystack-api/templates/api-ingress.yaml @@ -1,9 +1,8 @@ {{- $host := index .Values._cluster "root-host" }} {{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "api" $exposeServices) (ne $gatewayEnabled "true") }} +{{- if and (has "api" $exposeServices) }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/packages/system/cozystack-api/templates/api-tlsroute.yaml b/packages/system/cozystack-api/templates/api-tlsroute.yaml deleted file mode 100644 index 10ce9e16..00000000 --- a/packages/system/cozystack-api/templates/api-tlsroute.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} -{{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "api" $exposeServices) (eq $gatewayEnabled "true") }} -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: TLSRoute -metadata: - name: kubernetes-api - namespace: default -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $exposeIngress }} - sectionName: tls-api - hostnames: - - {{ printf "api.%s" $host | quote }} - rules: - - backendRefs: - - name: kubernetes - port: 443 -{{- end }} diff --git a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml deleted file mode 100644 index 99be904c..00000000 --- a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml +++ /dev/null @@ -1,203 +0,0 @@ -{{- $attachedNamespaces := (index .Values._cluster "gateway-attached-namespaces") | default "" }} -{{- range splitList "," $attachedNamespaces }} - {{- $ns := . | trim }} - {{- if and $ns (hasPrefix "tenant-" $ns) }} - {{- fail (printf "publishing.gateway.attachedNamespaces must not contain tenant-* namespaces (got %q). Adding a tenant namespace here lets that tenant attach HTTPRoutes to the publishing tenant's Gateway and hijack hostnames. Child tenants that need their own Gateway must opt in via tenant.spec.gateway=true instead." $ns) }} - {{- end }} -{{- end }} ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicy -metadata: - name: cozystack-gateway-hostname-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - failurePolicy: Fail - matchConstraints: - resourceRules: - - apiGroups: ["gateway.networking.k8s.io"] - apiVersions: ["v1", "v1beta1"] - operations: ["CREATE", "UPDATE"] - resources: ["gateways"] - matchConditions: - - name: tenant-namespace - expression: object.metadata.namespace.startsWith("tenant-") - variables: - - name: tenantHost - expression: >- - (namespaceObject != null && - has(namespaceObject.metadata.labels) && - "namespace.cozystack.io/host" in namespaceObject.metadata.labels) - ? namespaceObject.metadata.labels["namespace.cozystack.io/host"] - : "" - validations: - - expression: >- - variables.tenantHost == "" || - !has(object.spec.listeners) || - object.spec.listeners.all(l, - !has(l.hostname) || - l.hostname == variables.tenantHost || - l.hostname.endsWith("." + variables.tenantHost) - ) - messageExpression: >- - "Gateway listener hostname must equal " + variables.tenantHost + - " or end with ." + variables.tenantHost + - " (namespace " + object.metadata.namespace + " carries label namespace.cozystack.io/host=" + variables.tenantHost + ")" - reason: Forbidden ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: cozystack-gateway-hostname-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - policyName: cozystack-gateway-hostname-policy - validationActions: [Deny] ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicy -metadata: - name: cozystack-gateway-attached-namespaces-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - failurePolicy: Fail - matchConstraints: - resourceRules: - - apiGroups: ["cozystack.io"] - apiVersions: ["v1alpha1"] - operations: ["CREATE", "UPDATE"] - resources: ["packages"] - matchConditions: - - name: has-gateway-attached-namespaces - expression: >- - has(object.spec) && - has(object.spec.components) && - "platform" in object.spec.components && - has(object.spec.components.platform.values) && - "gateway" in object.spec.components.platform.values && - "attachedNamespaces" in object.spec.components.platform.values.gateway - validations: - - expression: >- - object.spec.components.platform.values.gateway.attachedNamespaces.all(ns, !ns.startsWith("tenant-")) - messageExpression: >- - "spec.components.platform.values.gateway.attachedNamespaces must not contain any tenant-* namespace; use tenant.spec.gateway=true instead of adding a tenant namespace to the publishing Gateway's whitelist" - reason: Forbidden ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: cozystack-gateway-attached-namespaces-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - policyName: cozystack-gateway-attached-namespaces-policy - validationActions: [Deny] ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicy -metadata: - name: cozystack-tenant-host-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - failurePolicy: Fail - matchConstraints: - resourceRules: - - apiGroups: ["apps.cozystack.io"] - apiVersions: ["v1alpha1"] - operations: ["CREATE", "UPDATE"] - resources: ["tenants"] - matchConditions: - - name: has-host-field - expression: >- - (has(object.spec) && "host" in object.spec && object.spec.host != "") || - (oldObject != null && has(oldObject.spec) && "host" in oldObject.spec && oldObject.spec.host != "") - variables: - - name: oldHost - expression: >- - (oldObject != null && has(oldObject.spec) && "host" in oldObject.spec) ? oldObject.spec.host : "" - - name: newHost - expression: >- - (has(object.spec) && "host" in object.spec) ? object.spec.host : "" - - name: hostChanged - expression: variables.oldHost != variables.newHost - - name: trustedCaller - expression: >- - has(request.userInfo.groups) && request.userInfo.groups.exists(g, - g == "system:masters" || - g == "system:serviceaccounts:cozy-system" || - g == "system:serviceaccounts:cozy-cert-manager" || - g == "system:serviceaccounts:cozy-fluxcd" || - g == "system:serviceaccounts:kube-system") - validations: - - expression: "!variables.hostChanged || variables.trustedCaller" - messageExpression: >- - "tenant.spec.host can only be set or changed by cluster-admins (system:masters) or by cozystack/Flux service accounts; request user is " + request.userInfo.username + ". A tenant with empty spec.host inherits the apex from its parent tenant." - reason: Forbidden ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: cozystack-tenant-host-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - policyName: cozystack-tenant-host-policy - validationActions: [Deny] ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicy -metadata: - name: cozystack-namespace-host-label-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - failurePolicy: Fail - matchConstraints: - resourceRules: - - apiGroups: [""] - apiVersions: ["v1"] - operations: ["CREATE", "UPDATE"] - resources: ["namespaces"] - matchConditions: - - name: touches-host-label - expression: >- - (has(object.metadata.labels) && "namespace.cozystack.io/host" in object.metadata.labels) || - (oldObject != null && has(oldObject.metadata.labels) && "namespace.cozystack.io/host" in oldObject.metadata.labels) - variables: - - name: oldHost - expression: >- - (oldObject != null && has(oldObject.metadata.labels) && "namespace.cozystack.io/host" in oldObject.metadata.labels) - ? oldObject.metadata.labels["namespace.cozystack.io/host"] - : "" - - name: newHost - expression: >- - has(object.metadata.labels) && "namespace.cozystack.io/host" in object.metadata.labels - ? object.metadata.labels["namespace.cozystack.io/host"] - : "" - - name: trustedCaller - expression: >- - has(request.userInfo.groups) && request.userInfo.groups.exists(g, - g == "system:masters" || - g == "system:serviceaccounts:cozy-system" || - g == "system:serviceaccounts:cozy-cert-manager" || - g == "system:serviceaccounts:cozy-fluxcd" || - g == "system:serviceaccounts:kube-system") - validations: - - expression: "variables.newHost == variables.oldHost || variables.trustedCaller" - messageExpression: >- - "namespace label namespace.cozystack.io/host is immutable once set (was " + variables.oldHost + ", requested " + variables.newHost + "); only cluster-admins and cozystack/Flux service accounts may change it. Request user: " + request.userInfo.username - reason: Forbidden ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: cozystack-namespace-host-label-policy - labels: - internal.cozystack.io/managed-by-cozystack: "" -spec: - policyName: cozystack-namespace-host-label-policy - validationActions: [Deny] diff --git a/packages/system/cozystack-basics/templates/tenant-root.yaml b/packages/system/cozystack-basics/templates/tenant-root.yaml index 4569139a..93f5129e 100644 --- a/packages/system/cozystack-basics/templates/tenant-root.yaml +++ b/packages/system/cozystack-basics/templates/tenant-root.yaml @@ -3,8 +3,6 @@ apiVersion: v1 kind: Namespace metadata: name: tenant-root - labels: - namespace.cozystack.io/host: {{ index .Values._cluster "root-host" | quote }} --- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease diff --git a/packages/system/dashboard/templates/httproute.yaml b/packages/system/dashboard/templates/httproute.yaml deleted file mode 100644 index 40c72ac0..00000000 --- a/packages/system/dashboard/templates/httproute.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} -{{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "dashboard" $exposeServices) (eq $gatewayEnabled "true") }} -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: dashboard -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $exposeIngress }} - sectionName: https - hostnames: - - {{ printf "dashboard.%s" $host | quote }} - rules: - - backendRefs: - - name: incloud-web-gatekeeper - port: 8000 -{{- end }} diff --git a/packages/system/dashboard/templates/ingress.yaml b/packages/system/dashboard/templates/ingress.yaml index f69e7e4e..6cf01490 100644 --- a/packages/system/dashboard/templates/ingress.yaml +++ b/packages/system/dashboard/templates/ingress.yaml @@ -3,9 +3,8 @@ {{- $host := index .Values._cluster "root-host" }} {{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "dashboard" $exposeServices) (ne $gatewayEnabled "true") }} +{{- if and (has "dashboard" $exposeServices) }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/packages/system/gateway-api-crds/Makefile b/packages/system/gateway-api-crds/Makefile index 0d005d8b..ad51d591 100644 --- a/packages/system/gateway-api-crds/Makefile +++ b/packages/system/gateway-api-crds/Makefile @@ -6,4 +6,4 @@ include ../../../hack/package.mk update: rm -rf templates mkdir templates - kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd/experimental?ref=v1.5.1" > templates/crds-experimental.yaml + kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd/experimental?ref=v1.2.0" > templates/crds-experimental.yaml diff --git a/packages/system/gateway-api-crds/templates/crds-experimental.yaml b/packages/system/gateway-api-crds/templates/crds-experimental.yaml index cf0011eb..88b3060e 100644 --- a/packages/system/gateway-api-crds/templates/crds-experimental.yaml +++ b/packages/system/gateway-api-crds/templates/crds-experimental.yaml @@ -2,35 +2,36 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null labels: gateway.networking.k8s.io/policy: Direct - name: backendtlspolicies.gateway.networking.k8s.io + name: backendlbpolicies.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io names: categories: - gateway-api - kind: BackendTLSPolicy - listKind: BackendTLSPolicyList - plural: backendtlspolicies + kind: BackendLBPolicy + listKind: BackendLBPolicyList + plural: backendlbpolicies shortNames: - - btlspolicy - singular: backendtlspolicy + - blbpolicy + singular: backendlbpolicy scope: Namespaced versions: - additionalPrinterColumns: - jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1 + name: v1alpha2 schema: openAPIV3Schema: description: |- - BackendTLSPolicy provides a way to configure how a Gateway - connects to a Backend via TLS. + BackendLBPolicy provides a way to define load balancing rules + for a backend. properties: apiVersion: description: |- @@ -50,101 +51,108 @@ spec: metadata: type: object spec: - description: Spec defines the desired state of BackendTLSPolicy. + description: Spec defines the desired state of BackendLBPolicy. properties: - options: - additionalProperties: - description: |- - AnnotationValue is the value of an annotation in Gateway API. This is used - for validation of maps such as TLS options. This roughly matches Kubernetes - annotation validation, although the length validation in that case is based - on the entire size of the annotations struct. - maxLength: 4096 - minLength: 0 - type: string + sessionPersistence: description: |- - Options are a list of key/value pairs to enable extended TLS - configuration for each implementation. For example, configuring the - minimum TLS version or supported cipher suites. + SessionPersistence defines and configures session persistence + for the backend. - A set of common keys MAY be defined by the API in the future. To avoid - any ambiguity, implementation-specific definitions MUST use - domain-prefixed names, such as `example.com/my-custom-option`. - Un-prefixed names are reserved for key names defined by Gateway API. + Support: Extended + properties: + absoluteTimeout: + description: |- + AbsoluteTimeout defines the absolute timeout of the persistent + session. Once the AbsoluteTimeout duration has elapsed, the + session becomes invalid. - Support: Implementation-specific - maxProperties: 16 + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + cookieConfig: + description: |- + CookieConfig provides configuration settings that are specific + to cookie-based session persistence. + + Support: Core + properties: + lifetimeType: + default: Session + description: |- + LifetimeType specifies whether the cookie has a permanent or + session-based lifetime. A permanent cookie persists until its + specified expiry time, defined by the Expires or Max-Age cookie + attributes, while a session cookie is deleted when the current + session ends. + + When set to "Permanent", AbsoluteTimeout indicates the + cookie's lifetime via the Expires or Max-Age cookie attributes + and is required. + + When set to "Session", AbsoluteTimeout indicates the + absolute lifetime of the cookie tracked by the gateway and + is optional. + + Support: Core for "Session" type + + Support: Extended for "Permanent" type + enum: + - Permanent + - Session + type: string + type: object + idleTimeout: + description: |- + IdleTimeout defines the idle timeout of the persistent session. + Once the session has been idle for more than the specified + IdleTimeout duration, the session becomes invalid. + + Support: Extended + pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ + type: string + sessionName: + description: |- + SessionName defines the name of the persistent session token + which may be reflected in the cookie or the header. Users + should avoid reusing session names to prevent unintended + consequences, such as rejection or unpredictable behavior. + + Support: Implementation-specific + maxLength: 128 + type: string + type: + default: Cookie + description: |- + Type defines the type of session persistence such as through + the use a header or cookie. Defaults to cookie based session + persistence. + + Support: Core for "Cookie" type + + Support: Extended for "Header" type + enum: + - Cookie + - Header + type: string type: object + x-kubernetes-validations: + - message: AbsoluteTimeout must be specified when cookie lifetimeType + is Permanent + rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) + || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' targetRefs: description: |- - TargetRefs identifies an API object to apply the policy to. - Note that this config applies to the entire referenced resource - by default, but this default may change in the future to provide - a more granular application of the policy. - - TargetRefs must be _distinct_. This means either that: - - * They select different targets. If this is the case, then targetRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, and `name` must - be unique across all targetRef entries in the BackendTLSPolicy. - * They select different sectionNames in the same target. - - When more than one BackendTLSPolicy selects the same target and - sectionName, implementations MUST determine precedence using the - following criteria, continuing on ties: - - * The older policy by creation timestamp takes precedence. For - example, a policy with a creation timestamp of "2021-07-15 - 01:02:03" MUST be given precedence over a policy with a - creation timestamp of "2021-07-15 01:02:04". - * The policy appearing first in alphabetical order by {namespace}/{name}. - For example, a policy named `foo/bar` is given precedence over a - policy named `foo/baz`. - - For any BackendTLSPolicy that does not take precedence, the - implementation MUST ensure the `Accepted` Condition is set to - `status: False`, with Reason `Conflicted`. - - Implementations SHOULD NOT support more than one targetRef at this - time. Although the API technically allows for this, the current guidance - for conflict resolution and status handling is lacking. Until that can be - clarified in a future release, the safest approach is to support a single - targetRef. - - Support Levels: - - * Extended: Kubernetes Service referenced by HTTPRoute backendRefs. - - * Implementation-Specific: Services not connected via HTTPRoute, and any - other kind of backend. Implementations MAY use BackendTLSPolicy for: - - Services not referenced by any Route (e.g., infrastructure services) - - Gateway feature backends (e.g., ExternalAuth, rate-limiting services) - - Service mesh workload-to-service communication - - Other resource types beyond Service - - Implementations SHOULD aim to ensure that BackendTLSPolicy behavior is consistent, - even outside of the extended HTTPRoute -(backendRef) -> Service path. - They SHOULD clearly document how BackendTLSPolicy is interpreted in these - scenarios, including: - - Which resources beyond Service are supported - - How the policy is discovered and applied - - Any implementation-specific semantics or restrictions - - Note that this config applies to the entire referenced resource - by default, but this default may change in the future to provide - a more granular application of the policy. + TargetRef identifies an API object to apply policy to. + Currently, Backends (i.e. Service, ServiceImport, or any + implementation-specific backendRef) are the only valid API + target references. items: description: |- - LocalPolicyTargetReferenceWithSectionName identifies an API object to apply a - direct policy to. This should be used as part of Policy resources that can - target single resources. For more information on how this policy attachment - mode works, and a sample Policy resource, refer to the policy attachment - documentation for Gateway API. - - Note: This should only be used for direct policy attachment when references - to SectionName are actually needed. In all other cases, - LocalPolicyTargetReference should be used. + LocalPolicyTargetReference identifies an API object to apply a direct or + inherited policy to. This should be used as part of Policy resources + that can target Gateway API resources. For more information on how this + policy attachment model works, and a sample Policy resource, refer to + the policy attachment documentation for Gateway API. properties: group: description: Group is the group of the target resource. @@ -162,23 +170,6 @@ spec: maxLength: 253 minLength: 1 type: string - sectionName: - description: |- - SectionName is the name of a section within the target resource. When - unspecified, this targetRef targets the entire resource. In the following - resources, SectionName is interpreted as the following: - - * Gateway: Listener name - * HTTPRoute: HTTPRouteRule name - * Service: Port name - - If a SectionName is specified, but does not exist on the targeted object, - the Policy must fail to attach, and the policy implementation should record - a `ResolvedRefs` or similar Condition in the Policy's status. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string required: - group - kind @@ -187,232 +178,16 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: sectionName must be specified when targetRefs includes - 2 or more references to the same target - rule: 'self.all(p1, self.all(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name ? ((!has(p1.sectionName) || p1.sectionName - == '''') == (!has(p2.sectionName) || p2.sectionName == '''')) - : true))' - - message: sectionName must be unique when targetRefs includes 2 or - more references to the same target - rule: self.all(p1, self.exists_one(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name && (((!has(p1.sectionName) || - p1.sectionName == '') && (!has(p2.sectionName) || p2.sectionName - == '')) || (has(p1.sectionName) && has(p2.sectionName) && p1.sectionName - == p2.sectionName)))) - validation: - description: Validation contains backend TLS validation configuration. - properties: - caCertificateRefs: - description: |- - CACertificateRefs contains one or more references to Kubernetes objects that - contain a PEM-encoded TLS CA certificate bundle, which is used to - validate a TLS handshake between the Gateway and backend Pod. - - If CACertificateRefs is empty or unspecified, then WellKnownCACertificates must be - specified. Only one of CACertificateRefs or WellKnownCACertificates may be specified, - not both. If CACertificateRefs is empty or unspecified, the configuration for - WellKnownCACertificates MUST be honored instead if supported by the implementation. - - A CACertificateRef is invalid if: - - * It refers to a resource that cannot be resolved (e.g., the referenced resource - does not exist) or is misconfigured (e.g., a ConfigMap does not contain a key - named `ca.crt`). In this case, the Reason must be set to `InvalidCACertificateRef` - and the Message of the Condition must indicate which reference is invalid and why. - - * It refers to an unknown or unsupported kind of resource. In this case, the Reason - must be set to `InvalidKind` and the Message of the Condition must explain which - kind of resource is unknown or unsupported. - - * It refers to a resource in another namespace. This may change in future - spec updates. - - Implementations MAY choose to perform further validation of the certificate - content (e.g., checking expiry or enforcing specific formats). In such cases, - an implementation-specific Reason and Message must be set for the invalid reference. - - In all cases, the implementation MUST ensure the `ResolvedRefs` Condition on - the BackendTLSPolicy is set to `status: False`, with a Reason and Message - that indicate the cause of the error. Connections using an invalid - CACertificateRef MUST fail, and the client MUST receive an HTTP 5xx error - response. If ALL CACertificateRefs are invalid, the implementation MUST also - ensure the `Accepted` Condition on the BackendTLSPolicy is set to - `status: False`, with a Reason `NoValidCACertificate`. - - A single CACertificateRef to a Kubernetes ConfigMap kind has "Core" support. - Implementations MAY choose to support attaching multiple certificates to - a backend, but this behavior is implementation-specific. - - Support: Core - An optional single reference to a Kubernetes ConfigMap, - with the CA certificate in a key named `ca.crt`. - - Support: Implementation-specific - More than one reference, other kinds - of resources, or a single reference that includes multiple certificates. - items: - description: |- - LocalObjectReference identifies an API object within the namespace of the - referrer. - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - - References to objects with invalid Group and Kind are not valid, and must - be rejected by the implementation, with appropriate Conditions set - on the containing object. - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For example "HTTPRoute" - or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - hostname: - description: |- - Hostname is used for two purposes in the connection between Gateways and - backends: - - 1. Hostname MUST be used as the SNI to connect to the backend (RFC 6066). - 2. Hostname MUST be used for authentication and MUST match the certificate - served by the matching backend, unless SubjectAltNames is specified. - 3. If SubjectAltNames are specified, Hostname can be used for certificate selection - but MUST NOT be used for authentication. If you want to use the value - of the Hostname field for authentication, you MUST add it to the SubjectAltNames list. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - subjectAltNames: - description: |- - SubjectAltNames contains one or more Subject Alternative Names. - When specified the certificate served from the backend MUST - have at least one Subject Alternate Name matching one of the specified SubjectAltNames. - - Support: Extended - items: - description: SubjectAltName represents Subject Alternative Name. - properties: - hostname: - description: |- - Hostname contains Subject Alternative Name specified in DNS name format. - Required when Type is set to Hostname, ignored otherwise. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - type: - description: |- - Type determines the format of the Subject Alternative Name. Always required. - - Support: Core - enum: - - Hostname - - URI - type: string - uri: - description: |- - URI contains Subject Alternative Name specified in a full URI format. - It MUST include both a scheme (e.g., "http" or "ftp") and a scheme-specific-part. - Common values include SPIFFE IDs like "spiffe://mycluster.example.com/ns/myns/sa/svc1sa". - Required when Type is set to URI, ignored otherwise. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^(([^:/?#]+):)(//([^/?#]*))([^?#]*)(\?([^#]*))?(#(.*))? - type: string - required: - - type - type: object - x-kubernetes-validations: - - message: SubjectAltName element must contain Hostname, if - Type is set to Hostname - rule: '!(self.type == "Hostname" && (!has(self.hostname) || - self.hostname == ""))' - - message: SubjectAltName element must not contain Hostname, - if Type is not set to Hostname - rule: '!(self.type != "Hostname" && has(self.hostname) && - self.hostname != "")' - - message: SubjectAltName element must contain URI, if Type - is set to URI - rule: '!(self.type == "URI" && (!has(self.uri) || self.uri - == ""))' - - message: SubjectAltName element must not contain URI, if Type - is not set to URI - rule: '!(self.type != "URI" && has(self.uri) && self.uri != - "")' - maxItems: 5 - type: array - x-kubernetes-list-type: atomic - wellKnownCACertificates: - description: |- - WellKnownCACertificates specifies whether a well-known set of CA certificates - may be used in the TLS handshake between the gateway and backend pod. - - If WellKnownCACertificates is unspecified or empty (""), then CACertificateRefs - must be specified with at least one entry for a valid configuration. Only one of - CACertificateRefs or WellKnownCACertificates may be specified, not both. - If an implementation does not support the WellKnownCACertificates field, or - the supplied value is not recognized, the implementation MUST ensure the - `Accepted` Condition on the BackendTLSPolicy is set to `status: False`, with - a Reason `Invalid`. - - Valid values include: - * "System" - indicates that well-known system CA certificates should be used. - - Implementations MAY define their own sets of CA certificates. Such definitions - MUST use an implementation-specific, prefixed name, such as - `mycompany.com/my-custom-ca-certificates`. - - Support: Implementation-specific - maxLength: 253 - minLength: 1 - pattern: ^(System|([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9]))$ - type: string - required: - - hostname - type: object - x-kubernetes-validations: - - message: must not contain both CACertificateRefs and WellKnownCACertificates - rule: '!(has(self.caCertificateRefs) && size(self.caCertificateRefs) - > 0 && has(self.wellKnownCACertificates) && self.wellKnownCACertificates - != "")' - - message: must specify either CACertificateRefs or WellKnownCACertificates - rule: (has(self.caCertificateRefs) && size(self.caCertificateRefs) - > 0 || has(self.wellKnownCACertificates) && self.wellKnownCACertificates - != "") + x-kubernetes-list-map-keys: + - group + - kind + - name + x-kubernetes-list-type: map required: - targetRefs - - validation type: object status: - description: Status defines the current state of BackendTLSPolicy. + description: Status defines the current state of BackendLBPolicy. properties: ancestors: description: |- @@ -700,12 +475,10 @@ spec: type: string required: - ancestorRef - - conditions - controllerName type: object maxItems: 16 type: array - x-kubernetes-list-type: atomic required: - ancestors type: object @@ -716,9 +489,41 @@ spec: storage: true subresources: status: {} - - deprecated: true - deprecationWarning: The v1alpha3 version of BackendTLSPolicy has been deprecated - and will be removed in a future release of the API. Please upgrade to v1. +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 + gateway.networking.k8s.io/channel: experimental + creationTimestamp: null + labels: + gateway.networking.k8s.io/policy: Direct + name: backendtlspolicies.gateway.networking.k8s.io +spec: + group: gateway.networking.k8s.io + names: + categories: + - gateway-api + kind: BackendTLSPolicy + listKind: BackendTLSPolicyList + plural: backendtlspolicies + shortNames: + - btlspolicy + singular: backendtlspolicy + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date name: v1alpha3 schema: openAPIV3Schema: @@ -772,62 +577,15 @@ spec: targetRefs: description: |- TargetRefs identifies an API object to apply the policy to. + Only Services have Extended support. Implementations MAY support + additional objects, with Implementation Specific support. Note that this config applies to the entire referenced resource by default, but this default may change in the future to provide a more granular application of the policy. - TargetRefs must be _distinct_. This means either that: + Support: Extended for Kubernetes Service - * They select different targets. If this is the case, then targetRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, and `name` must - be unique across all targetRef entries in the BackendTLSPolicy. - * They select different sectionNames in the same target. - - When more than one BackendTLSPolicy selects the same target and - sectionName, implementations MUST determine precedence using the - following criteria, continuing on ties: - - * The older policy by creation timestamp takes precedence. For - example, a policy with a creation timestamp of "2021-07-15 - 01:02:03" MUST be given precedence over a policy with a - creation timestamp of "2021-07-15 01:02:04". - * The policy appearing first in alphabetical order by {namespace}/{name}. - For example, a policy named `foo/bar` is given precedence over a - policy named `foo/baz`. - - For any BackendTLSPolicy that does not take precedence, the - implementation MUST ensure the `Accepted` Condition is set to - `status: False`, with Reason `Conflicted`. - - Implementations SHOULD NOT support more than one targetRef at this - time. Although the API technically allows for this, the current guidance - for conflict resolution and status handling is lacking. Until that can be - clarified in a future release, the safest approach is to support a single - targetRef. - - Support Levels: - - * Extended: Kubernetes Service referenced by HTTPRoute backendRefs. - - * Implementation-Specific: Services not connected via HTTPRoute, and any - other kind of backend. Implementations MAY use BackendTLSPolicy for: - - Services not referenced by any Route (e.g., infrastructure services) - - Gateway feature backends (e.g., ExternalAuth, rate-limiting services) - - Service mesh workload-to-service communication - - Other resource types beyond Service - - Implementations SHOULD aim to ensure that BackendTLSPolicy behavior is consistent, - even outside of the extended HTTPRoute -(backendRef) -> Service path. - They SHOULD clearly document how BackendTLSPolicy is interpreted in these - scenarios, including: - - Which resources beyond Service are supported - - How the policy is discovered and applied - - Any implementation-specific semantics or restrictions - - Note that this config applies to the entire referenced resource - by default, but this default may change in the future to provide - a more granular application of the policy. + Support: Implementation-specific for any other resource items: description: |- LocalPolicyTargetReferenceWithSectionName identifies an API object to apply a @@ -881,21 +639,6 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: sectionName must be specified when targetRefs includes - 2 or more references to the same target - rule: 'self.all(p1, self.all(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name ? ((!has(p1.sectionName) || p1.sectionName - == '''') == (!has(p2.sectionName) || p2.sectionName == '''')) - : true))' - - message: sectionName must be unique when targetRefs includes 2 or - more references to the same target - rule: self.all(p1, self.exists_one(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name && (((!has(p1.sectionName) || - p1.sectionName == '') && (!has(p2.sectionName) || p2.sectionName - == '')) || (has(p1.sectionName) && has(p2.sectionName) && p1.sectionName - == p2.sectionName)))) validation: description: Validation contains backend TLS validation configuration. properties: @@ -907,34 +650,11 @@ spec: If CACertificateRefs is empty or unspecified, then WellKnownCACertificates must be specified. Only one of CACertificateRefs or WellKnownCACertificates may be specified, - not both. If CACertificateRefs is empty or unspecified, the configuration for + not both. If CACertifcateRefs is empty or unspecified, the configuration for WellKnownCACertificates MUST be honored instead if supported by the implementation. - A CACertificateRef is invalid if: - - * It refers to a resource that cannot be resolved (e.g., the referenced resource - does not exist) or is misconfigured (e.g., a ConfigMap does not contain a key - named `ca.crt`). In this case, the Reason must be set to `InvalidCACertificateRef` - and the Message of the Condition must indicate which reference is invalid and why. - - * It refers to an unknown or unsupported kind of resource. In this case, the Reason - must be set to `InvalidKind` and the Message of the Condition must explain which - kind of resource is unknown or unsupported. - - * It refers to a resource in another namespace. This may change in future - spec updates. - - Implementations MAY choose to perform further validation of the certificate - content (e.g., checking expiry or enforcing specific formats). In such cases, - an implementation-specific Reason and Message must be set for the invalid reference. - - In all cases, the implementation MUST ensure the `ResolvedRefs` Condition on - the BackendTLSPolicy is set to `status: False`, with a Reason and Message - that indicate the cause of the error. Connections using an invalid - CACertificateRef MUST fail, and the client MUST receive an HTTP 5xx error - response. If ALL CACertificateRefs are invalid, the implementation MUST also - ensure the `Accepted` Condition on the BackendTLSPolicy is set to - `status: False`, with a Reason `NoValidCACertificate`. + References to a resource in a different namespace are invalid for the + moment, although we will revisit this in the future. A single CACertificateRef to a Kubernetes ConfigMap kind has "Core" support. Implementations MAY choose to support attaching multiple certificates to @@ -943,8 +663,8 @@ spec: Support: Core - An optional single reference to a Kubernetes ConfigMap, with the CA certificate in a key named `ca.crt`. - Support: Implementation-specific - More than one reference, other kinds - of resources, or a single reference that includes multiple certificates. + Support: Implementation-specific (More than one reference, or other kinds + of resources). items: description: |- LocalObjectReference identifies an API object within the namespace of the @@ -982,18 +702,15 @@ spec: type: object maxItems: 8 type: array - x-kubernetes-list-type: atomic hostname: description: |- Hostname is used for two purposes in the connection between Gateways and backends: 1. Hostname MUST be used as the SNI to connect to the backend (RFC 6066). - 2. Hostname MUST be used for authentication and MUST match the certificate - served by the matching backend, unless SubjectAltNames is specified. - 3. If SubjectAltNames are specified, Hostname can be used for certificate selection - but MUST NOT be used for authentication. If you want to use the value - of the Hostname field for authentication, you MUST add it to the SubjectAltNames list. + 2. If SubjectAltNames is not specified, Hostname MUST be used for + authentication and MUST match the certificate served by the matching + backend. Support: Core maxLength: 253 @@ -1003,10 +720,10 @@ spec: subjectAltNames: description: |- SubjectAltNames contains one or more Subject Alternative Names. - When specified the certificate served from the backend MUST - have at least one Subject Alternate Name matching one of the specified SubjectAltNames. + When specified, the certificate served from the backend MUST have at least one + Subject Alternate Name matching one of the specified SubjectAltNames. - Support: Extended + Support: Core items: description: SubjectAltName represents Subject Alternative Name. properties: @@ -1063,31 +780,21 @@ spec: "")' maxItems: 5 type: array - x-kubernetes-list-type: atomic wellKnownCACertificates: description: |- - WellKnownCACertificates specifies whether a well-known set of CA certificates - may be used in the TLS handshake between the gateway and backend pod. + WellKnownCACertificates specifies whether system CA certificates may be used in + the TLS handshake between the gateway and backend pod. If WellKnownCACertificates is unspecified or empty (""), then CACertificateRefs must be specified with at least one entry for a valid configuration. Only one of - CACertificateRefs or WellKnownCACertificates may be specified, not both. - If an implementation does not support the WellKnownCACertificates field, or - the supplied value is not recognized, the implementation MUST ensure the - `Accepted` Condition on the BackendTLSPolicy is set to `status: False`, with - a Reason `Invalid`. - - Valid values include: - * "System" - indicates that well-known system CA certificates should be used. - - Implementations MAY define their own sets of CA certificates. Such definitions - MUST use an implementation-specific, prefixed name, such as - `mycompany.com/my-custom-ca-certificates`. + CACertificateRefs or WellKnownCACertificates may be specified, not both. If an + implementation does not support the WellKnownCACertificates field or the value + supplied is not supported, the Status Conditions on the Policy MUST be + updated to include an Accepted: False Condition with Reason: Invalid. Support: Implementation-specific - maxLength: 253 - minLength: 1 - pattern: ^(System|([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9]))$ + enum: + - System type: string required: - hostname @@ -1394,12 +1101,10 @@ spec: type: string required: - ancestorRef - - conditions - controllerName type: object maxItems: 16 type: array - x-kubernetes-list-type: atomic required: - ancestors type: object @@ -1407,7 +1112,7 @@ spec: - spec type: object served: true - storage: false + storage: true subresources: status: {} status: @@ -1421,9 +1126,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: gatewayclasses.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -1656,7 +1362,7 @@ spec: - type x-kubernetes-list-type: map supportedFeatures: - description: |- + description: | SupportedFeatures is the set of features the GatewayClass support. It MUST be sorted in ascending alphabetical order by the Name key. items: @@ -1900,7 +1606,7 @@ spec: - type x-kubernetes-list-type: map supportedFeatures: - description: |- + description: | SupportedFeatures is the set of features the GatewayClass support. It MUST be sorted in ascending alphabetical order by the Name key. items: @@ -1937,9 +1643,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: gateways.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -1995,11 +1702,11 @@ spec: description: Spec defines the desired state of Gateway. properties: addresses: - description: |- + description: |+ Addresses requested for this Gateway. This is optional and behavior can depend on the implementation. If a value is set in the spec and the requested address is invalid or unavailable, the implementation MUST - indicate this in an associated entry in GatewayStatus.Conditions. + indicate this in the associated entry in GatewayStatus.Addresses. The Addresses field represents a request for the address(es) on the "outside of the Gateway", that traffic bound for this Gateway will use. @@ -2016,9 +1723,10 @@ spec: GatewayStatus.Addresses. Support: Extended + items: - description: GatewaySpecAddress describes an address that can be - bound to a Gateway. + description: GatewayAddress describes an address that can be bound + to a Gateway. oneOf: - properties: type: @@ -2043,137 +1751,96 @@ spec: type: string value: description: |- - When a value is unspecified, an implementation SHOULD automatically - assign an address matching the requested type if possible. - - If an implementation does not support an empty value, they MUST set the - "Programmed" condition in status to False with a reason of "AddressNotAssigned". + Value of the address. The validity of the values will depend + on the type and support by the controller. Examples: `1.2.3.4`, `128::1`, `my-ip-address`. maxLength: 253 + minLength: 1 type: string + required: + - value type: object x-kubernetes-validations: - - message: Hostname value must be empty or contain only valid characters - (matching ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) - rule: 'self.type == ''Hostname'' ? (!has(self.value) || self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$""")): + - message: Hostname value must only contain valid characters (matching + ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) + rule: 'self.type == ''Hostname'' ? self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"""): true' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: IPAddress values must be unique - rule: 'self.all(a1, a1.type == ''IPAddress'' && has(a1.value) ? - self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value - == a1.value) : true )' + rule: 'self.all(a1, a1.type == ''IPAddress'' ? self.exists_one(a2, + a2.type == a1.type && a2.value == a1.value) : true )' - message: Hostname values must be unique - rule: 'self.all(a1, a1.type == ''Hostname'' && has(a1.value) ? - self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value - == a1.value) : true )' - allowedListeners: - description: |- - AllowedListeners defines which ListenerSets can be attached to this Gateway. - The default value is to allow no ListenerSets. + rule: 'self.all(a1, a1.type == ''Hostname'' ? self.exists_one(a2, + a2.type == a1.type && a2.value == a1.value) : true )' + backendTLS: + description: |+ + BackendTLS configures TLS settings for when this Gateway is connecting to + backends with TLS. + + Support: Core + properties: - namespaces: - default: - from: None - description: |- - Namespaces defines which namespaces ListenerSets can be attached to this Gateway. - The default value is to allow no ListenerSets. + clientCertificateRef: + description: |+ + ClientCertificateRef is a reference to an object that contains a Client + Certificate and the associated private key. + + References to a resource in different namespace are invalid UNLESS there + is a ReferenceGrant in the target namespace that allows the certificate + to be attached. If a ReferenceGrant does not allow this reference, the + "ResolvedRefs" condition MUST be set to False for this listener with the + "RefNotPermitted" reason. + + ClientCertificateRef can reference to standard Kubernetes resources, i.e. + Secret, or implementation-specific custom resources. + + This setting can be overridden on the service level by use of BackendTLSPolicy. + + Support: Core + properties: - from: - default: None + group: + default: "" description: |- - From indicates where ListenerSets can attach to this Gateway. Possible - values are: - - * Same: Only ListenerSets in the same namespace may be attached to this Gateway. - * Selector: ListenerSets in namespaces selected by the selector may be attached to this Gateway. - * All: ListenerSets in all namespaces may be attached to this Gateway. - * None: Only listeners defined in the Gateway's spec are allowed - - The default value None - enum: - - All - - Selector - - Same - - None + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - selector: + kind: + default: Secret + description: Kind is kind of the referent. For example "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: description: |- - Selector must be specified when From is set to "Selector". In that case, - only ListenerSets in Namespaces matching this Selector will be selected by this - Gateway. This field is ignored for other values of "From". - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name type: object type: object - defaultScope: - description: |- - DefaultScope, when set, configures the Gateway as a default Gateway, - meaning it will dynamically and implicitly have Routes (e.g. HTTPRoute) - attached to it, according to the scope configured here. - - If unset (the default) or set to None, the Gateway will not act as a - default Gateway; if set, the Gateway will claim any Route with a - matching scope set in its UseDefaultGateway field, subject to the usual - rules about which routes the Gateway can attach to. - - Think carefully before using this functionality! While the normal rules - about which Route can apply are still enforced, it is simply easier for - the wrong Route to be accidentally attached to this Gateway in this - configuration. If the Gateway operator is not also the operator in - control of the scope (e.g. namespace) with tight controls and checks on - what kind of workloads and Routes get added in that scope, we strongly - recommend not using this just because it seems convenient, and instead - stick to direct Route attachment. - enum: - - All - - None - type: string gatewayClassName: description: |- GatewayClassName used for this Gateway. This is the name of a @@ -2268,11 +1935,6 @@ spec: the merging behavior is implementation specific. It is generally recommended that GatewayClass provides defaults that can be overridden by a Gateway. - If the referent cannot be found, refers to an unsupported kind, or when - the data within that resource is malformed, the Gateway SHOULD be - rejected with the "Accepted" status condition set to "False" and an - "InvalidParameters" reason. - Support: Implementation-specific properties: group: @@ -2303,8 +1965,6 @@ spec: logical endpoints that are bound on this Gateway's addresses. At least one Listener MUST be specified. - ## Distinct Listeners - Each Listener in a set of Listeners (for example, in a single Gateway) MUST be _distinct_, in that a traffic flow MUST be able to be assigned to exactly one listener. (This section uses "set of Listeners" rather than @@ -2316,76 +1976,55 @@ spec: combination of Port, Protocol, and, if supported by the protocol, Hostname. Some combinations of port, protocol, and TLS settings are considered - Core support and MUST be supported by implementations based on the objects - they support: + Core support and MUST be supported by implementations based on their + targeted conformance profile: - HTTPRoute + HTTP Profile 1. HTTPRoute, Port: 80, Protocol: HTTP 2. HTTPRoute, Port: 443, Protocol: HTTPS, TLS Mode: Terminate, TLS keypair provided - TLSRoute + TLS Profile 1. TLSRoute, Port: 443, Protocol: TLS, TLS Mode: Passthrough "Distinct" Listeners have the following property: - **The implementation can match inbound requests to a single distinct - Listener**. - - When multiple Listeners share values for fields (for + The implementation can match inbound requests to a single distinct + Listener. When multiple Listeners share values for fields (for example, two Listeners with the same Port value), the implementation can match requests to only one of the Listeners using other Listener fields. - When multiple listeners have the same value for the Protocol field, then - each of the Listeners with matching Protocol values MUST have different - values for other fields. + For example, the following Listener scenarios are distinct: - The set of fields that MUST be different for a Listener differs per protocol. - The following rules define the rules for what fields MUST be considered for - Listeners to be distinct with each protocol currently defined in the - Gateway API spec. + 1. Multiple Listeners with the same Port that all use the "HTTP" + Protocol that all have unique Hostname values. + 2. Multiple Listeners with the same Port that use either the "HTTPS" or + "TLS" Protocol that all have unique Hostname values. + 3. A mixture of "TCP" and "UDP" Protocol Listeners, where no Listener + with the same Protocol has the same Port value. - The set of listeners that all share a protocol value MUST have _different_ - values for _at least one_ of these fields to be distinct: + Some fields in the Listener struct have possible values that affect + whether the Listener is distinct. Hostname is particularly relevant + for HTTP or HTTPS protocols. - * **HTTP, HTTPS, TLS**: Port, Hostname - * **TCP, UDP**: Port + When using the Hostname value to select between same-Port, same-Protocol + Listeners, the Hostname value must be different on each Listener for the + Listener to be distinct. - One **very** important rule to call out involves what happens when an - implementation: - - * Supports TCP protocol Listeners, as well as HTTP, HTTPS, or TLS protocol - Listeners, and - * sees HTTP, HTTPS, or TLS protocols with the same `port` as one with TCP - Protocol. - - In this case all the Listeners that share a port with the - TCP Listener are not distinct and so MUST NOT be accepted. - - If an implementation does not support TCP Protocol Listeners, then the - previous rule does not apply, and the TCP Listeners SHOULD NOT be - accepted. - - Note that the `tls` field is not used for determining if a listener is distinct, because - Listeners that _only_ differ on TLS config will still conflict in all cases. - - ### Listeners that are distinct only by Hostname - - When the Listeners are distinct based only on Hostname, inbound request + When the Listeners are distinct based on Hostname, inbound request hostnames MUST match from the most specific to least specific Hostname values to choose the correct Listener and its associated set of Routes. - Exact matches MUST be processed before wildcard matches, and wildcard - matches MUST be processed before fallback (empty Hostname value) + Exact matches must be processed before wildcard matches, and wildcard + matches must be processed before fallback (empty Hostname value) matches. For example, `"foo.example.com"` takes precedence over `"*.example.com"`, and `"*.example.com"` takes precedence over `""`. Additionally, if there are multiple wildcard entries, more specific wildcard entries must be processed before less specific wildcard entries. For example, `"*.foo.example.com"` takes precedence over `"*.example.com"`. - The precise definition here is that the higher the number of dots in the hostname to the right of the wildcard character, the higher the precedence. @@ -2393,26 +2032,18 @@ spec: the left, however, so `"*.example.com"` will match both `"foo.bar.example.com"` _and_ `"bar.example.com"`. - ## Handling indistinct Listeners - If a set of Listeners contains Listeners that are not distinct, then those - Listeners are _Conflicted_, and the implementation MUST set the "Conflicted" + Listeners are Conflicted, and the implementation MUST set the "Conflicted" condition in the Listener Status to "True". - The words "indistinct" and "conflicted" are considered equivalent for the - purpose of this documentation. - Implementations MAY choose to accept a Gateway with some Conflicted Listeners only if they only accept the partial Listener set that contains - no Conflicted Listeners. - - Specifically, an implementation MAY accept a partial Listener set subject to - the following rules: - - * The implementation MUST NOT pick one conflicting Listener as the winner. - ALL indistinct Listeners must not be accepted for processing. - * At least one distinct Listener MUST be present, or else the Gateway effectively - contains _no_ Listeners, and must be rejected from processing as a whole. + no Conflicted Listeners. To put this another way, implementations may + accept a partial Listener set only if they throw out *all* the conflicting + Listeners. No picking one of the conflicting listeners as the winner. + This also means that the Gateway must have at least one non-conflicting + Listener in this case, otherwise it violates the requirement that at + least one Listener must be present. The implementation MUST set a "ListenersNotValid" condition on the Gateway Status when the Gateway contains Conflicted Listeners whether or @@ -2421,25 +2052,7 @@ spec: Accepted. Additionally, the Listener status for those listeners SHOULD indicate which Listeners are conflicted and not Accepted. - ## General Listener behavior - - Note that, for all distinct Listeners, requests SHOULD match at most one Listener. - For example, if Listeners are defined for "foo.example.com" and "*.example.com", a - request to "foo.example.com" SHOULD only be routed using routes attached - to the "foo.example.com" Listener (and not the "*.example.com" Listener). - - This concept is known as "Listener Isolation", and it is an Extended feature - of Gateway API. Implementations that do not support Listener Isolation MUST - clearly document this, and MUST NOT claim support for the - `GatewayHTTPListenerIsolation` feature. - - Implementations that _do_ support Listener Isolation SHOULD claim support - for the Extended `GatewayHTTPListenerIsolation` feature and pass the associated - conformance tests. - - ## Compatible Listeners - - A Gateway's Listeners are considered _compatible_ if: + A Gateway's Listeners are considered "compatible" if: 1. They are distinct. 2. The implementation can serve them in compliance with the Addresses @@ -2454,11 +2067,16 @@ spec: on the same address, or cannot mix HTTPS and generic TLS listens on the same port would not consider those cases compatible, even though they are distinct. + Note that requests SHOULD match at most one Listener. For example, if + Listeners are defined for "foo.example.com" and "*.example.com", a + request to "foo.example.com" SHOULD only be routed using routes attached + to the "foo.example.com" Listener (and not the "*.example.com" Listener). + This concept is known as "Listener Isolation". Implementations that do + not support Listener Isolation MUST clearly document this. + Implementations MAY merge separate Gateways onto a single set of Addresses if all Listeners across all Gateways are compatible. - In a future release the MinItems=1 requirement MAY be dropped. - Support: Core items: description: |- @@ -2529,7 +2147,6 @@ spec: type: object maxItems: 8 type: array - x-kubernetes-list-type: atomic namespaces: default: from: Same @@ -2621,31 +2238,10 @@ spec: * TLS: The Listener Hostname MUST match the SNI. * HTTP: The Listener Hostname MUST match the Host header of the request. - * HTTPS: The Listener Hostname SHOULD match both the SNI and Host header. - Note that this does not require the SNI and Host header to be the same. - The semantics of this are described in more detail below. - - To ensure security, Section 11.1 of RFC-6066 emphasizes that server - implementations that rely on SNI hostname matching MUST also verify - hostnames within the application protocol. - - Section 9.1.2 of RFC-7540 provides a mechanism for servers to reject the - reuse of a connection by responding with the HTTP 421 Misdirected Request - status code. This indicates that the origin server has rejected the - request because it appears to have been misdirected. - - To detect misdirected requests, Gateways SHOULD match the authority of - the requests with all the SNI hostname(s) configured across all the - Gateway Listeners on the same port and protocol: - - * If another Listener has an exact match or more specific wildcard entry, - the Gateway SHOULD return a 421. - * If the current Listener (selected by SNI matching during ClientHello) - does not match the Host: - * If another Listener does match the Host, the Gateway SHOULD return a - 421. - * If no other Listener matches the Host, the Gateway MUST return a - 404. + * HTTPS: The Listener Hostname SHOULD match at both the TLS and HTTP + protocol layers as described above. If an implementation does not + ensure that both the SNI and Host header match the Listener hostname, + it MUST clearly document that. For HTTPRoute and TLSRoute resources, there is an interaction with the `spec.hostnames` array. When both listener and route specify hostnames, @@ -2697,7 +2293,7 @@ spec: the Protocol field is "HTTPS" or "TLS". It is invalid to set this field if the Protocol field is "HTTP", "TCP", or "UDP". - The association of SNIs to Certificate defined in ListenerTLSConfig is + The association of SNIs to Certificate defined in GatewayTLSConfig is defined based on the Hostname field for this listener. The GatewayClass MUST use the longest matching SNI out of all @@ -2784,7 +2380,94 @@ spec: type: object maxItems: 64 type: array - x-kubernetes-list-type: atomic + frontendValidation: + description: |+ + FrontendValidation holds configuration information for validating the frontend (client). + Setting this field will require clients to send a client certificate + required for validation during the TLS handshake. In browsers this may result in a dialog appearing + that requests a user to specify the client certificate. + The maximum depth of a certificate chain accepted in verification is Implementation specific. + + Support: Extended + + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to + Kubernetes objects that contain TLS certificates of + the Certificate Authorities that can be used + as a trust anchor to validate the certificates presented by the client. + + A single CA certificate reference to a Kubernetes ConfigMap + has "Core" support. + Implementations MAY choose to support attaching multiple CA certificates to + a Listener, but this behavior is implementation-specific. + + Support: Core - A single reference to a Kubernetes ConfigMap + with the CA certificate in a key named `ca.crt`. + + Support: Implementation-specific (More than one reference, or other kinds + of resources). + + References to a resource in a different namespace are invalid UNLESS there + is a ReferenceGrant in the target namespace that allows the certificate + to be attached. If a ReferenceGrant does not allow this reference, the + "ResolvedRefs" condition MUST be set to False for this listener with the + "RefNotPermitted" reason. + items: + description: |- + ObjectReference identifies an API object including its namespace. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For + example "ConfigMap" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + maxItems: 8 + minItems: 1 + type: array + type: object mode: default: Terminate description: |- @@ -2853,9 +2536,6 @@ spec: - message: tls mode must be Terminate for protocol HTTPS rule: 'self.all(l, (l.protocol == ''HTTPS'' && has(l.tls)) ? (l.tls.mode == '''' || l.tls.mode == ''Terminate'') : true)' - - message: tls mode must be set for protocol TLS - rule: 'self.all(l, (l.protocol == ''TLS'' ? has(l.tls) && has(l.tls.mode) - && l.tls.mode != '''' : true))' - message: hostname must not be specified for protocols ['TCP', 'UDP'] rule: 'self.all(l, l.protocol in [''TCP'', ''UDP''] ? (!has(l.hostname) || l.hostname == '''') : true)' @@ -2866,421 +2546,6 @@ spec: rule: 'self.all(l1, self.exists_one(l2, l1.port == l2.port && l1.protocol == l2.protocol && (has(l1.hostname) && has(l2.hostname) ? l1.hostname == l2.hostname : !has(l1.hostname) && !has(l2.hostname))))' - tls: - description: |- - TLS specifies frontend and backend tls configuration for entire gateway. - - Support: Extended - properties: - backend: - description: |- - Backend describes TLS configuration for gateway when connecting - to backends. - - Note that this contains only details for the Gateway as a TLS client, - and does _not_ imply behavior about how to choose which backend should - get a TLS connection. That is determined by the presence of a BackendTLSPolicy. - - Support: Core - properties: - clientCertificateRef: - description: |- - ClientCertificateRef references an object that contains a client certificate - and its associated private key. It can reference standard Kubernetes resources, - i.e., Secret, or implementation-specific custom resources. - - A ClientCertificateRef is considered invalid if: - - * It refers to a resource that cannot be resolved (e.g., the referenced resource - does not exist) or is misconfigured (e.g., a Secret does not contain the keys - named `tls.crt` and `tls.key`). In this case, the `ResolvedRefs` condition - on the Gateway MUST be set to False with the Reason `InvalidClientCertificateRef` - and the Message of the Condition MUST indicate why the reference is invalid. - - * It refers to a resource in another namespace UNLESS there is a ReferenceGrant - in the target namespace that allows the certificate to be attached. - If a ReferenceGrant does not allow this reference, the `ResolvedRefs` condition - on the Gateway MUST be set to False with the Reason `RefNotPermitted`. - - Implementations MAY choose to perform further validation of the certificate - content (e.g., checking expiry or enforcing specific formats). In such cases, - an implementation-specific Reason and Message MUST be set. - - Support: Core - Reference to a Kubernetes TLS Secret (with the type `kubernetes.io/tls`). - Support: Implementation-specific - Other resource kinds or Secrets with a - different type (e.g., `Opaque`). - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Secret - description: Kind is kind of the referent. For example - "Secret". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - type: object - frontend: - description: |- - Frontend describes TLS config when client connects to Gateway. - Support: Core - properties: - default: - description: |- - Default specifies the default client certificate validation configuration - for all Listeners handling HTTPS traffic, unless a per-port configuration - is defined. - - support: Core - properties: - validation: - description: |- - Validation holds configuration information for validating the frontend (client). - Setting this field will result in mutual authentication when connecting to the gateway. - In browsers this may result in a dialog appearing - that requests a user to specify the client certificate. - The maximum depth of a certificate chain accepted in verification is Implementation specific. - - Support: Core - properties: - caCertificateRefs: - description: |- - CACertificateRefs contains one or more references to Kubernetes - objects that contain a PEM-encoded TLS CA certificate bundle, which - is used as a trust anchor to validate the certificates presented by - the client. - - A CACertificateRef is invalid if: - - * It refers to a resource that cannot be resolved (e.g., the - referenced resource does not exist) or is misconfigured (e.g., a - ConfigMap does not contain a key named `ca.crt`). In this case, the - Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` - and the Message of the Condition must indicate which reference is invalid and why. - - * It refers to an unknown or unsupported kind of resource. In this - case, the Reason on all matching HTTPS listeners must be set to - `InvalidCACertificateKind` and the Message of the Condition must explain - which kind of resource is unknown or unsupported. - - * It refers to a resource in another namespace UNLESS there is a - ReferenceGrant in the target namespace that allows the CA - certificate to be attached. If a ReferenceGrant does not allow this - reference, the `ResolvedRefs` on all matching HTTPS listeners condition - MUST be set with the Reason `RefNotPermitted`. - - Implementations MAY choose to perform further validation of the - certificate content (e.g., checking expiry or enforcing specific formats). - In such cases, an implementation-specific Reason and Message MUST be set. - - In all cases, the implementation MUST ensure that the `ResolvedRefs` - condition is set to `status: False` on all targeted listeners (i.e., - listeners serving HTTPS on a matching port). The condition MUST - include a Reason and Message that indicate the cause of the error. If - ALL CACertificateRefs are invalid, the implementation MUST also ensure - the `Accepted` condition on the listener is set to `status: False`, with - the Reason `NoValidCACertificate`. - Implementations MAY choose to support attaching multiple CA certificates - to a listener, but this behavior is implementation-specific. - - Support: Core - A single reference to a Kubernetes ConfigMap, with the - CA certificate in a key named `ca.crt`. - - Support: Implementation-specific - More than one reference, other kinds - of resources, or a single reference that includes multiple certificates. - items: - description: |- - ObjectReference identifies an API object including its namespace. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - - References to objects with invalid Group and Kind are not valid, and must - be rejected by the implementation, with appropriate Conditions set - on the containing object. - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When set to the empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For - example "ConfigMap" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - name - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - mode: - default: AllowValidOnly - description: |- - FrontendValidationMode defines the mode for validating the client certificate. - There are two possible modes: - - - AllowValidOnly: In this mode, the gateway will accept connections only if - the client presents a valid certificate. This certificate must successfully - pass validation against the CA certificates specified in `CACertificateRefs`. - - AllowInsecureFallback: In this mode, the gateway will accept connections - even if the client certificate is not presented or fails verification. - - This approach delegates client authorization to the backend and introduce - a significant security risk. It should be used in testing environments or - on a temporary basis in non-testing environments. - - Defaults to AllowValidOnly. - - Support: Core - enum: - - AllowValidOnly - - AllowInsecureFallback - type: string - required: - - caCertificateRefs - type: object - type: object - perPort: - description: |- - PerPort specifies tls configuration assigned per port. - Per port configuration is optional. Once set this configuration overrides - the default configuration for all Listeners handling HTTPS traffic - that match this port. - Each override port requires a unique TLS configuration. - - support: Core - items: - properties: - port: - description: |- - The Port indicates the Port Number to which the TLS configuration will be - applied. This configuration will be applied to all Listeners handling HTTPS - traffic that match this port. - - Support: Core - format: int32 - maximum: 65535 - minimum: 1 - type: integer - tls: - description: |- - TLS store the configuration that will be applied to all Listeners handling - HTTPS traffic and matching given port. - - Support: Core - properties: - validation: - description: |- - Validation holds configuration information for validating the frontend (client). - Setting this field will result in mutual authentication when connecting to the gateway. - In browsers this may result in a dialog appearing - that requests a user to specify the client certificate. - The maximum depth of a certificate chain accepted in verification is Implementation specific. - - Support: Core - properties: - caCertificateRefs: - description: |- - CACertificateRefs contains one or more references to Kubernetes - objects that contain a PEM-encoded TLS CA certificate bundle, which - is used as a trust anchor to validate the certificates presented by - the client. - - A CACertificateRef is invalid if: - - * It refers to a resource that cannot be resolved (e.g., the - referenced resource does not exist) or is misconfigured (e.g., a - ConfigMap does not contain a key named `ca.crt`). In this case, the - Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` - and the Message of the Condition must indicate which reference is invalid and why. - - * It refers to an unknown or unsupported kind of resource. In this - case, the Reason on all matching HTTPS listeners must be set to - `InvalidCACertificateKind` and the Message of the Condition must explain - which kind of resource is unknown or unsupported. - - * It refers to a resource in another namespace UNLESS there is a - ReferenceGrant in the target namespace that allows the CA - certificate to be attached. If a ReferenceGrant does not allow this - reference, the `ResolvedRefs` on all matching HTTPS listeners condition - MUST be set with the Reason `RefNotPermitted`. - - Implementations MAY choose to perform further validation of the - certificate content (e.g., checking expiry or enforcing specific formats). - In such cases, an implementation-specific Reason and Message MUST be set. - - In all cases, the implementation MUST ensure that the `ResolvedRefs` - condition is set to `status: False` on all targeted listeners (i.e., - listeners serving HTTPS on a matching port). The condition MUST - include a Reason and Message that indicate the cause of the error. If - ALL CACertificateRefs are invalid, the implementation MUST also ensure - the `Accepted` condition on the listener is set to `status: False`, with - the Reason `NoValidCACertificate`. - Implementations MAY choose to support attaching multiple CA certificates - to a listener, but this behavior is implementation-specific. - - Support: Core - A single reference to a Kubernetes ConfigMap, with the - CA certificate in a key named `ca.crt`. - - Support: Implementation-specific - More than one reference, other kinds - of resources, or a single reference that includes multiple certificates. - items: - description: |- - ObjectReference identifies an API object including its namespace. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - - References to objects with invalid Group and Kind are not valid, and must - be rejected by the implementation, with appropriate Conditions set - on the containing object. - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When set to the empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. - For example "ConfigMap" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - name - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - mode: - default: AllowValidOnly - description: |- - FrontendValidationMode defines the mode for validating the client certificate. - There are two possible modes: - - - AllowValidOnly: In this mode, the gateway will accept connections only if - the client presents a valid certificate. This certificate must successfully - pass validation against the CA certificates specified in `CACertificateRefs`. - - AllowInsecureFallback: In this mode, the gateway will accept connections - even if the client certificate is not presented or fails verification. - - This approach delegates client authorization to the backend and introduce - a significant security risk. It should be used in testing environments or - on a temporary basis in non-testing environments. - - Defaults to AllowValidOnly. - - Support: Core - enum: - - AllowValidOnly - - AllowInsecureFallback - type: string - required: - - caCertificateRefs - type: object - type: object - required: - - port - - tls - type: object - maxItems: 64 - type: array - x-kubernetes-list-map-keys: - - port - x-kubernetes-list-type: map - x-kubernetes-validations: - - message: Port for TLS configuration must be unique within - the Gateway - rule: self.all(t1, self.exists_one(t2, t1.port == t2.port)) - required: - - default - type: object - type: object required: - gatewayClassName - listeners @@ -3301,7 +2566,7 @@ spec: description: Status defines the current state of Gateway. properties: addresses: - description: |- + description: |+ Addresses lists the network addresses that have been bound to the Gateway. @@ -3311,6 +2576,7 @@ spec: * no addresses are specified, all addresses are dynamically assigned * a combination of specified and dynamic addresses are assigned * a specified address was unusable (e.g. already in use) + items: description: GatewayStatusAddress describes a network address that is bound to a Gateway. @@ -3355,21 +2621,6 @@ spec: true' maxItems: 16 type: array - x-kubernetes-list-type: atomic - attachedListenerSets: - description: |- - AttachedListenerSets represents the total number of ListenerSets that have been - successfully attached to this Gateway. - - A ListenerSet is successfully attached to a Gateway when all the following conditions are met: - - The ListenerSet is selected by the Gateway's AllowedListeners field - - The ListenerSet has a valid ParentRef selecting the Gateway - - The ListenerSet's status has the condition "Accepted: true" - - Uses for this field include troubleshooting AttachedListenerSets attachment and - measuring blast radius/impact of changes to a Gateway. - format: int32 - type: integer conditions: default: - lastTransitionTime: "1970-01-01T00:00:00Z" @@ -3474,11 +2725,8 @@ spec: attachment semantics can be found in the documentation on the various Route kinds ParentRefs fields). Listener or Route status does not impact successful attachment, i.e. the AttachedRoutes field count MUST be set - for Listeners, even if the Accepted condition of an individual Listener is set - to "False". The AttachedRoutes number represents the number of Routes with - the Accepted condition set to "True" that have been attached to this Listener. - Routes with any other value for the Accepted condition MUST NOT be included - in this count. + for Listeners with condition Accepted: false and MUST count successfully + attached Routes that may themselves have Accepted: false conditions. Uses for this field include troubleshooting Route attachment and measuring blast radius/impact of changes to a Listener. @@ -3557,7 +2805,7 @@ spec: supportedKinds: description: |- SupportedKinds is the list indicating the Kinds supported by this - listener. This MUST represent the kinds supported by an implementation for + listener. This MUST represent the kinds an implementation supports for that Listener configuration. If kinds are specified in Spec that are not supported, they MUST NOT @@ -3586,11 +2834,11 @@ spec: type: object maxItems: 8 type: array - x-kubernetes-list-type: atomic required: - attachedRoutes - conditions - name + - supportedKinds type: object maxItems: 64 type: array @@ -3646,11 +2894,11 @@ spec: description: Spec defines the desired state of Gateway. properties: addresses: - description: |- + description: |+ Addresses requested for this Gateway. This is optional and behavior can depend on the implementation. If a value is set in the spec and the requested address is invalid or unavailable, the implementation MUST - indicate this in an associated entry in GatewayStatus.Conditions. + indicate this in the associated entry in GatewayStatus.Addresses. The Addresses field represents a request for the address(es) on the "outside of the Gateway", that traffic bound for this Gateway will use. @@ -3667,9 +2915,10 @@ spec: GatewayStatus.Addresses. Support: Extended + items: - description: GatewaySpecAddress describes an address that can be - bound to a Gateway. + description: GatewayAddress describes an address that can be bound + to a Gateway. oneOf: - properties: type: @@ -3694,137 +2943,96 @@ spec: type: string value: description: |- - When a value is unspecified, an implementation SHOULD automatically - assign an address matching the requested type if possible. - - If an implementation does not support an empty value, they MUST set the - "Programmed" condition in status to False with a reason of "AddressNotAssigned". + Value of the address. The validity of the values will depend + on the type and support by the controller. Examples: `1.2.3.4`, `128::1`, `my-ip-address`. maxLength: 253 + minLength: 1 type: string + required: + - value type: object x-kubernetes-validations: - - message: Hostname value must be empty or contain only valid characters - (matching ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) - rule: 'self.type == ''Hostname'' ? (!has(self.value) || self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$""")): + - message: Hostname value must only contain valid characters (matching + ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$) + rule: 'self.type == ''Hostname'' ? self.value.matches(r"""^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"""): true' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: IPAddress values must be unique - rule: 'self.all(a1, a1.type == ''IPAddress'' && has(a1.value) ? - self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value - == a1.value) : true )' + rule: 'self.all(a1, a1.type == ''IPAddress'' ? self.exists_one(a2, + a2.type == a1.type && a2.value == a1.value) : true )' - message: Hostname values must be unique - rule: 'self.all(a1, a1.type == ''Hostname'' && has(a1.value) ? - self.exists_one(a2, a2.type == a1.type && has(a2.value) && a2.value - == a1.value) : true )' - allowedListeners: - description: |- - AllowedListeners defines which ListenerSets can be attached to this Gateway. - The default value is to allow no ListenerSets. + rule: 'self.all(a1, a1.type == ''Hostname'' ? self.exists_one(a2, + a2.type == a1.type && a2.value == a1.value) : true )' + backendTLS: + description: |+ + BackendTLS configures TLS settings for when this Gateway is connecting to + backends with TLS. + + Support: Core + properties: - namespaces: - default: - from: None - description: |- - Namespaces defines which namespaces ListenerSets can be attached to this Gateway. - The default value is to allow no ListenerSets. + clientCertificateRef: + description: |+ + ClientCertificateRef is a reference to an object that contains a Client + Certificate and the associated private key. + + References to a resource in different namespace are invalid UNLESS there + is a ReferenceGrant in the target namespace that allows the certificate + to be attached. If a ReferenceGrant does not allow this reference, the + "ResolvedRefs" condition MUST be set to False for this listener with the + "RefNotPermitted" reason. + + ClientCertificateRef can reference to standard Kubernetes resources, i.e. + Secret, or implementation-specific custom resources. + + This setting can be overridden on the service level by use of BackendTLSPolicy. + + Support: Core + properties: - from: - default: None + group: + default: "" description: |- - From indicates where ListenerSets can attach to this Gateway. Possible - values are: - - * Same: Only ListenerSets in the same namespace may be attached to this Gateway. - * Selector: ListenerSets in namespaces selected by the selector may be attached to this Gateway. - * All: ListenerSets in all namespaces may be attached to this Gateway. - * None: Only listeners defined in the Gateway's spec are allowed - - The default value None - enum: - - All - - Selector - - Same - - None + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - selector: + kind: + default: Secret + description: Kind is kind of the referent. For example "Secret". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: description: |- - Selector must be specified when From is set to "Selector". In that case, - only ListenerSets in Namespaces matching this Selector will be selected by this - Gateway. This field is ignored for other values of "From". - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name type: object type: object - defaultScope: - description: |- - DefaultScope, when set, configures the Gateway as a default Gateway, - meaning it will dynamically and implicitly have Routes (e.g. HTTPRoute) - attached to it, according to the scope configured here. - - If unset (the default) or set to None, the Gateway will not act as a - default Gateway; if set, the Gateway will claim any Route with a - matching scope set in its UseDefaultGateway field, subject to the usual - rules about which routes the Gateway can attach to. - - Think carefully before using this functionality! While the normal rules - about which Route can apply are still enforced, it is simply easier for - the wrong Route to be accidentally attached to this Gateway in this - configuration. If the Gateway operator is not also the operator in - control of the scope (e.g. namespace) with tight controls and checks on - what kind of workloads and Routes get added in that scope, we strongly - recommend not using this just because it seems convenient, and instead - stick to direct Route attachment. - enum: - - All - - None - type: string gatewayClassName: description: |- GatewayClassName used for this Gateway. This is the name of a @@ -3919,11 +3127,6 @@ spec: the merging behavior is implementation specific. It is generally recommended that GatewayClass provides defaults that can be overridden by a Gateway. - If the referent cannot be found, refers to an unsupported kind, or when - the data within that resource is malformed, the Gateway SHOULD be - rejected with the "Accepted" status condition set to "False" and an - "InvalidParameters" reason. - Support: Implementation-specific properties: group: @@ -3954,8 +3157,6 @@ spec: logical endpoints that are bound on this Gateway's addresses. At least one Listener MUST be specified. - ## Distinct Listeners - Each Listener in a set of Listeners (for example, in a single Gateway) MUST be _distinct_, in that a traffic flow MUST be able to be assigned to exactly one listener. (This section uses "set of Listeners" rather than @@ -3967,76 +3168,55 @@ spec: combination of Port, Protocol, and, if supported by the protocol, Hostname. Some combinations of port, protocol, and TLS settings are considered - Core support and MUST be supported by implementations based on the objects - they support: + Core support and MUST be supported by implementations based on their + targeted conformance profile: - HTTPRoute + HTTP Profile 1. HTTPRoute, Port: 80, Protocol: HTTP 2. HTTPRoute, Port: 443, Protocol: HTTPS, TLS Mode: Terminate, TLS keypair provided - TLSRoute + TLS Profile 1. TLSRoute, Port: 443, Protocol: TLS, TLS Mode: Passthrough "Distinct" Listeners have the following property: - **The implementation can match inbound requests to a single distinct - Listener**. - - When multiple Listeners share values for fields (for + The implementation can match inbound requests to a single distinct + Listener. When multiple Listeners share values for fields (for example, two Listeners with the same Port value), the implementation can match requests to only one of the Listeners using other Listener fields. - When multiple listeners have the same value for the Protocol field, then - each of the Listeners with matching Protocol values MUST have different - values for other fields. + For example, the following Listener scenarios are distinct: - The set of fields that MUST be different for a Listener differs per protocol. - The following rules define the rules for what fields MUST be considered for - Listeners to be distinct with each protocol currently defined in the - Gateway API spec. + 1. Multiple Listeners with the same Port that all use the "HTTP" + Protocol that all have unique Hostname values. + 2. Multiple Listeners with the same Port that use either the "HTTPS" or + "TLS" Protocol that all have unique Hostname values. + 3. A mixture of "TCP" and "UDP" Protocol Listeners, where no Listener + with the same Protocol has the same Port value. - The set of listeners that all share a protocol value MUST have _different_ - values for _at least one_ of these fields to be distinct: + Some fields in the Listener struct have possible values that affect + whether the Listener is distinct. Hostname is particularly relevant + for HTTP or HTTPS protocols. - * **HTTP, HTTPS, TLS**: Port, Hostname - * **TCP, UDP**: Port + When using the Hostname value to select between same-Port, same-Protocol + Listeners, the Hostname value must be different on each Listener for the + Listener to be distinct. - One **very** important rule to call out involves what happens when an - implementation: - - * Supports TCP protocol Listeners, as well as HTTP, HTTPS, or TLS protocol - Listeners, and - * sees HTTP, HTTPS, or TLS protocols with the same `port` as one with TCP - Protocol. - - In this case all the Listeners that share a port with the - TCP Listener are not distinct and so MUST NOT be accepted. - - If an implementation does not support TCP Protocol Listeners, then the - previous rule does not apply, and the TCP Listeners SHOULD NOT be - accepted. - - Note that the `tls` field is not used for determining if a listener is distinct, because - Listeners that _only_ differ on TLS config will still conflict in all cases. - - ### Listeners that are distinct only by Hostname - - When the Listeners are distinct based only on Hostname, inbound request + When the Listeners are distinct based on Hostname, inbound request hostnames MUST match from the most specific to least specific Hostname values to choose the correct Listener and its associated set of Routes. - Exact matches MUST be processed before wildcard matches, and wildcard - matches MUST be processed before fallback (empty Hostname value) + Exact matches must be processed before wildcard matches, and wildcard + matches must be processed before fallback (empty Hostname value) matches. For example, `"foo.example.com"` takes precedence over `"*.example.com"`, and `"*.example.com"` takes precedence over `""`. Additionally, if there are multiple wildcard entries, more specific wildcard entries must be processed before less specific wildcard entries. For example, `"*.foo.example.com"` takes precedence over `"*.example.com"`. - The precise definition here is that the higher the number of dots in the hostname to the right of the wildcard character, the higher the precedence. @@ -4044,26 +3224,18 @@ spec: the left, however, so `"*.example.com"` will match both `"foo.bar.example.com"` _and_ `"bar.example.com"`. - ## Handling indistinct Listeners - If a set of Listeners contains Listeners that are not distinct, then those - Listeners are _Conflicted_, and the implementation MUST set the "Conflicted" + Listeners are Conflicted, and the implementation MUST set the "Conflicted" condition in the Listener Status to "True". - The words "indistinct" and "conflicted" are considered equivalent for the - purpose of this documentation. - Implementations MAY choose to accept a Gateway with some Conflicted Listeners only if they only accept the partial Listener set that contains - no Conflicted Listeners. - - Specifically, an implementation MAY accept a partial Listener set subject to - the following rules: - - * The implementation MUST NOT pick one conflicting Listener as the winner. - ALL indistinct Listeners must not be accepted for processing. - * At least one distinct Listener MUST be present, or else the Gateway effectively - contains _no_ Listeners, and must be rejected from processing as a whole. + no Conflicted Listeners. To put this another way, implementations may + accept a partial Listener set only if they throw out *all* the conflicting + Listeners. No picking one of the conflicting listeners as the winner. + This also means that the Gateway must have at least one non-conflicting + Listener in this case, otherwise it violates the requirement that at + least one Listener must be present. The implementation MUST set a "ListenersNotValid" condition on the Gateway Status when the Gateway contains Conflicted Listeners whether or @@ -4072,25 +3244,7 @@ spec: Accepted. Additionally, the Listener status for those listeners SHOULD indicate which Listeners are conflicted and not Accepted. - ## General Listener behavior - - Note that, for all distinct Listeners, requests SHOULD match at most one Listener. - For example, if Listeners are defined for "foo.example.com" and "*.example.com", a - request to "foo.example.com" SHOULD only be routed using routes attached - to the "foo.example.com" Listener (and not the "*.example.com" Listener). - - This concept is known as "Listener Isolation", and it is an Extended feature - of Gateway API. Implementations that do not support Listener Isolation MUST - clearly document this, and MUST NOT claim support for the - `GatewayHTTPListenerIsolation` feature. - - Implementations that _do_ support Listener Isolation SHOULD claim support - for the Extended `GatewayHTTPListenerIsolation` feature and pass the associated - conformance tests. - - ## Compatible Listeners - - A Gateway's Listeners are considered _compatible_ if: + A Gateway's Listeners are considered "compatible" if: 1. They are distinct. 2. The implementation can serve them in compliance with the Addresses @@ -4105,11 +3259,16 @@ spec: on the same address, or cannot mix HTTPS and generic TLS listens on the same port would not consider those cases compatible, even though they are distinct. + Note that requests SHOULD match at most one Listener. For example, if + Listeners are defined for "foo.example.com" and "*.example.com", a + request to "foo.example.com" SHOULD only be routed using routes attached + to the "foo.example.com" Listener (and not the "*.example.com" Listener). + This concept is known as "Listener Isolation". Implementations that do + not support Listener Isolation MUST clearly document this. + Implementations MAY merge separate Gateways onto a single set of Addresses if all Listeners across all Gateways are compatible. - In a future release the MinItems=1 requirement MAY be dropped. - Support: Core items: description: |- @@ -4180,7 +3339,6 @@ spec: type: object maxItems: 8 type: array - x-kubernetes-list-type: atomic namespaces: default: from: Same @@ -4272,31 +3430,10 @@ spec: * TLS: The Listener Hostname MUST match the SNI. * HTTP: The Listener Hostname MUST match the Host header of the request. - * HTTPS: The Listener Hostname SHOULD match both the SNI and Host header. - Note that this does not require the SNI and Host header to be the same. - The semantics of this are described in more detail below. - - To ensure security, Section 11.1 of RFC-6066 emphasizes that server - implementations that rely on SNI hostname matching MUST also verify - hostnames within the application protocol. - - Section 9.1.2 of RFC-7540 provides a mechanism for servers to reject the - reuse of a connection by responding with the HTTP 421 Misdirected Request - status code. This indicates that the origin server has rejected the - request because it appears to have been misdirected. - - To detect misdirected requests, Gateways SHOULD match the authority of - the requests with all the SNI hostname(s) configured across all the - Gateway Listeners on the same port and protocol: - - * If another Listener has an exact match or more specific wildcard entry, - the Gateway SHOULD return a 421. - * If the current Listener (selected by SNI matching during ClientHello) - does not match the Host: - * If another Listener does match the Host, the Gateway SHOULD return a - 421. - * If no other Listener matches the Host, the Gateway MUST return a - 404. + * HTTPS: The Listener Hostname SHOULD match at both the TLS and HTTP + protocol layers as described above. If an implementation does not + ensure that both the SNI and Host header match the Listener hostname, + it MUST clearly document that. For HTTPRoute and TLSRoute resources, there is an interaction with the `spec.hostnames` array. When both listener and route specify hostnames, @@ -4348,7 +3485,7 @@ spec: the Protocol field is "HTTPS" or "TLS". It is invalid to set this field if the Protocol field is "HTTP", "TCP", or "UDP". - The association of SNIs to Certificate defined in ListenerTLSConfig is + The association of SNIs to Certificate defined in GatewayTLSConfig is defined based on the Hostname field for this listener. The GatewayClass MUST use the longest matching SNI out of all @@ -4435,7 +3572,94 @@ spec: type: object maxItems: 64 type: array - x-kubernetes-list-type: atomic + frontendValidation: + description: |+ + FrontendValidation holds configuration information for validating the frontend (client). + Setting this field will require clients to send a client certificate + required for validation during the TLS handshake. In browsers this may result in a dialog appearing + that requests a user to specify the client certificate. + The maximum depth of a certificate chain accepted in verification is Implementation specific. + + Support: Extended + + properties: + caCertificateRefs: + description: |- + CACertificateRefs contains one or more references to + Kubernetes objects that contain TLS certificates of + the Certificate Authorities that can be used + as a trust anchor to validate the certificates presented by the client. + + A single CA certificate reference to a Kubernetes ConfigMap + has "Core" support. + Implementations MAY choose to support attaching multiple CA certificates to + a Listener, but this behavior is implementation-specific. + + Support: Core - A single reference to a Kubernetes ConfigMap + with the CA certificate in a key named `ca.crt`. + + Support: Implementation-specific (More than one reference, or other kinds + of resources). + + References to a resource in a different namespace are invalid UNLESS there + is a ReferenceGrant in the target namespace that allows the certificate + to be attached. If a ReferenceGrant does not allow this reference, the + "ResolvedRefs" condition MUST be set to False for this listener with the + "RefNotPermitted" reason. + items: + description: |- + ObjectReference identifies an API object including its namespace. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + + References to objects with invalid Group and Kind are not valid, and must + be rejected by the implementation, with appropriate Conditions set + on the containing object. + properties: + group: + description: |- + Group is the group of the referent. For example, "gateway.networking.k8s.io". + When unspecified or empty string, core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. For + example "ConfigMap" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referenced object. When unspecified, the local + namespace is inferred. + + Note that when a namespace different than the local namespace is specified, + a ReferenceGrant object is required in the referent namespace to allow that + namespace's owner to accept the reference. See the ReferenceGrant + documentation for details. + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + maxItems: 8 + minItems: 1 + type: array + type: object mode: default: Terminate description: |- @@ -4504,9 +3728,6 @@ spec: - message: tls mode must be Terminate for protocol HTTPS rule: 'self.all(l, (l.protocol == ''HTTPS'' && has(l.tls)) ? (l.tls.mode == '''' || l.tls.mode == ''Terminate'') : true)' - - message: tls mode must be set for protocol TLS - rule: 'self.all(l, (l.protocol == ''TLS'' ? has(l.tls) && has(l.tls.mode) - && l.tls.mode != '''' : true))' - message: hostname must not be specified for protocols ['TCP', 'UDP'] rule: 'self.all(l, l.protocol in [''TCP'', ''UDP''] ? (!has(l.hostname) || l.hostname == '''') : true)' @@ -4517,421 +3738,6 @@ spec: rule: 'self.all(l1, self.exists_one(l2, l1.port == l2.port && l1.protocol == l2.protocol && (has(l1.hostname) && has(l2.hostname) ? l1.hostname == l2.hostname : !has(l1.hostname) && !has(l2.hostname))))' - tls: - description: |- - TLS specifies frontend and backend tls configuration for entire gateway. - - Support: Extended - properties: - backend: - description: |- - Backend describes TLS configuration for gateway when connecting - to backends. - - Note that this contains only details for the Gateway as a TLS client, - and does _not_ imply behavior about how to choose which backend should - get a TLS connection. That is determined by the presence of a BackendTLSPolicy. - - Support: Core - properties: - clientCertificateRef: - description: |- - ClientCertificateRef references an object that contains a client certificate - and its associated private key. It can reference standard Kubernetes resources, - i.e., Secret, or implementation-specific custom resources. - - A ClientCertificateRef is considered invalid if: - - * It refers to a resource that cannot be resolved (e.g., the referenced resource - does not exist) or is misconfigured (e.g., a Secret does not contain the keys - named `tls.crt` and `tls.key`). In this case, the `ResolvedRefs` condition - on the Gateway MUST be set to False with the Reason `InvalidClientCertificateRef` - and the Message of the Condition MUST indicate why the reference is invalid. - - * It refers to a resource in another namespace UNLESS there is a ReferenceGrant - in the target namespace that allows the certificate to be attached. - If a ReferenceGrant does not allow this reference, the `ResolvedRefs` condition - on the Gateway MUST be set to False with the Reason `RefNotPermitted`. - - Implementations MAY choose to perform further validation of the certificate - content (e.g., checking expiry or enforcing specific formats). In such cases, - an implementation-specific Reason and Message MUST be set. - - Support: Core - Reference to a Kubernetes TLS Secret (with the type `kubernetes.io/tls`). - Support: Implementation-specific - Other resource kinds or Secrets with a - different type (e.g., `Opaque`). - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Secret - description: Kind is kind of the referent. For example - "Secret". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - type: object - frontend: - description: |- - Frontend describes TLS config when client connects to Gateway. - Support: Core - properties: - default: - description: |- - Default specifies the default client certificate validation configuration - for all Listeners handling HTTPS traffic, unless a per-port configuration - is defined. - - support: Core - properties: - validation: - description: |- - Validation holds configuration information for validating the frontend (client). - Setting this field will result in mutual authentication when connecting to the gateway. - In browsers this may result in a dialog appearing - that requests a user to specify the client certificate. - The maximum depth of a certificate chain accepted in verification is Implementation specific. - - Support: Core - properties: - caCertificateRefs: - description: |- - CACertificateRefs contains one or more references to Kubernetes - objects that contain a PEM-encoded TLS CA certificate bundle, which - is used as a trust anchor to validate the certificates presented by - the client. - - A CACertificateRef is invalid if: - - * It refers to a resource that cannot be resolved (e.g., the - referenced resource does not exist) or is misconfigured (e.g., a - ConfigMap does not contain a key named `ca.crt`). In this case, the - Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` - and the Message of the Condition must indicate which reference is invalid and why. - - * It refers to an unknown or unsupported kind of resource. In this - case, the Reason on all matching HTTPS listeners must be set to - `InvalidCACertificateKind` and the Message of the Condition must explain - which kind of resource is unknown or unsupported. - - * It refers to a resource in another namespace UNLESS there is a - ReferenceGrant in the target namespace that allows the CA - certificate to be attached. If a ReferenceGrant does not allow this - reference, the `ResolvedRefs` on all matching HTTPS listeners condition - MUST be set with the Reason `RefNotPermitted`. - - Implementations MAY choose to perform further validation of the - certificate content (e.g., checking expiry or enforcing specific formats). - In such cases, an implementation-specific Reason and Message MUST be set. - - In all cases, the implementation MUST ensure that the `ResolvedRefs` - condition is set to `status: False` on all targeted listeners (i.e., - listeners serving HTTPS on a matching port). The condition MUST - include a Reason and Message that indicate the cause of the error. If - ALL CACertificateRefs are invalid, the implementation MUST also ensure - the `Accepted` condition on the listener is set to `status: False`, with - the Reason `NoValidCACertificate`. - Implementations MAY choose to support attaching multiple CA certificates - to a listener, but this behavior is implementation-specific. - - Support: Core - A single reference to a Kubernetes ConfigMap, with the - CA certificate in a key named `ca.crt`. - - Support: Implementation-specific - More than one reference, other kinds - of resources, or a single reference that includes multiple certificates. - items: - description: |- - ObjectReference identifies an API object including its namespace. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - - References to objects with invalid Group and Kind are not valid, and must - be rejected by the implementation, with appropriate Conditions set - on the containing object. - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When set to the empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. For - example "ConfigMap" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - name - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - mode: - default: AllowValidOnly - description: |- - FrontendValidationMode defines the mode for validating the client certificate. - There are two possible modes: - - - AllowValidOnly: In this mode, the gateway will accept connections only if - the client presents a valid certificate. This certificate must successfully - pass validation against the CA certificates specified in `CACertificateRefs`. - - AllowInsecureFallback: In this mode, the gateway will accept connections - even if the client certificate is not presented or fails verification. - - This approach delegates client authorization to the backend and introduce - a significant security risk. It should be used in testing environments or - on a temporary basis in non-testing environments. - - Defaults to AllowValidOnly. - - Support: Core - enum: - - AllowValidOnly - - AllowInsecureFallback - type: string - required: - - caCertificateRefs - type: object - type: object - perPort: - description: |- - PerPort specifies tls configuration assigned per port. - Per port configuration is optional. Once set this configuration overrides - the default configuration for all Listeners handling HTTPS traffic - that match this port. - Each override port requires a unique TLS configuration. - - support: Core - items: - properties: - port: - description: |- - The Port indicates the Port Number to which the TLS configuration will be - applied. This configuration will be applied to all Listeners handling HTTPS - traffic that match this port. - - Support: Core - format: int32 - maximum: 65535 - minimum: 1 - type: integer - tls: - description: |- - TLS store the configuration that will be applied to all Listeners handling - HTTPS traffic and matching given port. - - Support: Core - properties: - validation: - description: |- - Validation holds configuration information for validating the frontend (client). - Setting this field will result in mutual authentication when connecting to the gateway. - In browsers this may result in a dialog appearing - that requests a user to specify the client certificate. - The maximum depth of a certificate chain accepted in verification is Implementation specific. - - Support: Core - properties: - caCertificateRefs: - description: |- - CACertificateRefs contains one or more references to Kubernetes - objects that contain a PEM-encoded TLS CA certificate bundle, which - is used as a trust anchor to validate the certificates presented by - the client. - - A CACertificateRef is invalid if: - - * It refers to a resource that cannot be resolved (e.g., the - referenced resource does not exist) or is misconfigured (e.g., a - ConfigMap does not contain a key named `ca.crt`). In this case, the - Reason on all matching HTTPS listeners must be set to `InvalidCACertificateRef` - and the Message of the Condition must indicate which reference is invalid and why. - - * It refers to an unknown or unsupported kind of resource. In this - case, the Reason on all matching HTTPS listeners must be set to - `InvalidCACertificateKind` and the Message of the Condition must explain - which kind of resource is unknown or unsupported. - - * It refers to a resource in another namespace UNLESS there is a - ReferenceGrant in the target namespace that allows the CA - certificate to be attached. If a ReferenceGrant does not allow this - reference, the `ResolvedRefs` on all matching HTTPS listeners condition - MUST be set with the Reason `RefNotPermitted`. - - Implementations MAY choose to perform further validation of the - certificate content (e.g., checking expiry or enforcing specific formats). - In such cases, an implementation-specific Reason and Message MUST be set. - - In all cases, the implementation MUST ensure that the `ResolvedRefs` - condition is set to `status: False` on all targeted listeners (i.e., - listeners serving HTTPS on a matching port). The condition MUST - include a Reason and Message that indicate the cause of the error. If - ALL CACertificateRefs are invalid, the implementation MUST also ensure - the `Accepted` condition on the listener is set to `status: False`, with - the Reason `NoValidCACertificate`. - Implementations MAY choose to support attaching multiple CA certificates - to a listener, but this behavior is implementation-specific. - - Support: Core - A single reference to a Kubernetes ConfigMap, with the - CA certificate in a key named `ca.crt`. - - Support: Implementation-specific - More than one reference, other kinds - of resources, or a single reference that includes multiple certificates. - items: - description: |- - ObjectReference identifies an API object including its namespace. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - - References to objects with invalid Group and Kind are not valid, and must - be rejected by the implementation, with appropriate Conditions set - on the containing object. - properties: - group: - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When set to the empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. - For example "ConfigMap" or "Service". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - name - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - mode: - default: AllowValidOnly - description: |- - FrontendValidationMode defines the mode for validating the client certificate. - There are two possible modes: - - - AllowValidOnly: In this mode, the gateway will accept connections only if - the client presents a valid certificate. This certificate must successfully - pass validation against the CA certificates specified in `CACertificateRefs`. - - AllowInsecureFallback: In this mode, the gateway will accept connections - even if the client certificate is not presented or fails verification. - - This approach delegates client authorization to the backend and introduce - a significant security risk. It should be used in testing environments or - on a temporary basis in non-testing environments. - - Defaults to AllowValidOnly. - - Support: Core - enum: - - AllowValidOnly - - AllowInsecureFallback - type: string - required: - - caCertificateRefs - type: object - type: object - required: - - port - - tls - type: object - maxItems: 64 - type: array - x-kubernetes-list-map-keys: - - port - x-kubernetes-list-type: map - x-kubernetes-validations: - - message: Port for TLS configuration must be unique within - the Gateway - rule: self.all(t1, self.exists_one(t2, t1.port == t2.port)) - required: - - default - type: object - type: object required: - gatewayClassName - listeners @@ -4952,7 +3758,7 @@ spec: description: Status defines the current state of Gateway. properties: addresses: - description: |- + description: |+ Addresses lists the network addresses that have been bound to the Gateway. @@ -4962,6 +3768,7 @@ spec: * no addresses are specified, all addresses are dynamically assigned * a combination of specified and dynamic addresses are assigned * a specified address was unusable (e.g. already in use) + items: description: GatewayStatusAddress describes a network address that is bound to a Gateway. @@ -5006,21 +3813,6 @@ spec: true' maxItems: 16 type: array - x-kubernetes-list-type: atomic - attachedListenerSets: - description: |- - AttachedListenerSets represents the total number of ListenerSets that have been - successfully attached to this Gateway. - - A ListenerSet is successfully attached to a Gateway when all the following conditions are met: - - The ListenerSet is selected by the Gateway's AllowedListeners field - - The ListenerSet has a valid ParentRef selecting the Gateway - - The ListenerSet's status has the condition "Accepted: true" - - Uses for this field include troubleshooting AttachedListenerSets attachment and - measuring blast radius/impact of changes to a Gateway. - format: int32 - type: integer conditions: default: - lastTransitionTime: "1970-01-01T00:00:00Z" @@ -5125,11 +3917,8 @@ spec: attachment semantics can be found in the documentation on the various Route kinds ParentRefs fields). Listener or Route status does not impact successful attachment, i.e. the AttachedRoutes field count MUST be set - for Listeners, even if the Accepted condition of an individual Listener is set - to "False". The AttachedRoutes number represents the number of Routes with - the Accepted condition set to "True" that have been attached to this Listener. - Routes with any other value for the Accepted condition MUST NOT be included - in this count. + for Listeners with condition Accepted: false and MUST count successfully + attached Routes that may themselves have Accepted: false conditions. Uses for this field include troubleshooting Route attachment and measuring blast radius/impact of changes to a Listener. @@ -5208,7 +3997,7 @@ spec: supportedKinds: description: |- SupportedKinds is the list indicating the Kinds supported by this - listener. This MUST represent the kinds supported by an implementation for + listener. This MUST represent the kinds an implementation supports for that Listener configuration. If kinds are specified in Spec that are not supported, they MUST NOT @@ -5237,11 +4026,11 @@ spec: type: object maxItems: 8 type: array - x-kubernetes-list-type: atomic required: - attachedRoutes - conditions - name + - supportedKinds type: object maxItems: 64 type: array @@ -5267,9 +4056,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: grpcroutes.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -5415,9 +4205,8 @@ spec: type: string maxItems: 16 type: array - x-kubernetes-list-type: atomic parentRefs: - description: |- + description: |+ ParentRefs references the resources (usually Gateways) that a Route wants to be attached to. Note that the referenced parent resource needs to allow this for the attachment to be complete. For Gateways, that means @@ -5479,6 +4268,11 @@ spec: connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. + + + + + items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered @@ -5628,7 +4422,6 @@ spec: type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: sectionName or port must be specified when parentRefs includes 2 or more references to the same parent @@ -5653,7 +4446,9 @@ spec: || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port == p2.port)))) rules: - description: Rules are a list of GRPC matchers, filters and actions. + description: |+ + Rules are a list of GRPC matchers, filters and actions. + items: description: |- GRPCRouteRule defines the semantics for matching a gRPC request based on @@ -5699,6 +4494,7 @@ spec: namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + When the BackendRef points to a Kubernetes Service, implementations SHOULD honor the appProtocol field if it is set for the target Service Port. @@ -5713,6 +4509,8 @@ spec: If a Route is not able to send traffic to the backend using the specified protocol then the backend is considered invalid. Implementations MUST set the "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + properties: filters: description: |- @@ -5798,7 +4596,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -5810,14 +4608,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -5877,7 +4671,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -5889,14 +4683,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -5909,7 +4699,7 @@ spec: x-kubernetes-list-type: map type: object requestMirror: - description: |- + description: |+ RequestMirror defines a schema for a filter that mirrors requests. Requests are sent to the specified destination, but responses from that destination are ignored. @@ -5919,6 +4709,7 @@ spec: backends. Support: Extended + properties: backendRef: description: |- @@ -6014,12 +4805,13 @@ spec: rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' fraction: - description: |- + description: |+ Fraction represents the fraction of requests that should be mirrored to BackendRef. Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + properties: denominator: default: 100 @@ -6038,13 +4830,14 @@ spec: to denominator rule: self.numerator <= self.denominator percent: - description: |- + description: |+ Percent represents the percentage of requests that should be mirrored to BackendRef. Its minimum value is 0 (indicating 0% of requests) and its maximum value is 100 (indicating 100% of requests). Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + format: int32 maximum: 100 minimum: 0 @@ -6089,7 +4882,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -6101,14 +4894,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -6168,7 +4957,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -6180,14 +4969,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -6200,7 +4985,7 @@ spec: x-kubernetes-list-type: map type: object type: - description: |- + description: |+ Type identifies the type of filter to apply. As with other API fields, types are classified into three conformance levels: @@ -6225,6 +5010,7 @@ spec: If a reference to a custom filter type cannot be resolved, the filter MUST NOT be skipped. Instead, requests that would have been processed by that filter MUST receive a HTTP error response. + enum: - ResponseHeaderModifier - RequestHeaderModifier @@ -6266,7 +5052,6 @@ spec: rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: RequestHeaderModifier filter cannot be repeated rule: self.filter(f, f.type == 'RequestHeaderModifier').size() @@ -6363,7 +5148,6 @@ spec: ? has(self.port) : true' maxItems: 16 type: array - x-kubernetes-list-type: atomic filters: description: |- Filters define the filters that are applied to requests that match @@ -6383,7 +5167,7 @@ spec: Specifying the same filter multiple times is not supported unless explicitly indicated in the filter. - If an implementation cannot support a combination of filters, it must clearly + If an implementation can not support a combination of filters, it must clearly document that limitation. In cases where incompatible or unsupported filters are specified and cause the `Accepted` condition to be set to status `False`, implementations may use the `IncompatibleFilters` reason to specify @@ -6466,7 +5250,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -6478,14 +5262,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -6544,7 +5324,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -6556,14 +5336,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -6576,7 +5352,7 @@ spec: x-kubernetes-list-type: map type: object requestMirror: - description: |- + description: |+ RequestMirror defines a schema for a filter that mirrors requests. Requests are sent to the specified destination, but responses from that destination are ignored. @@ -6586,6 +5362,7 @@ spec: backends. Support: Extended + properties: backendRef: description: |- @@ -6681,12 +5458,13 @@ spec: rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' fraction: - description: |- + description: |+ Fraction represents the fraction of requests that should be mirrored to BackendRef. Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + properties: denominator: default: 100 @@ -6705,13 +5483,14 @@ spec: denominator rule: self.numerator <= self.denominator percent: - description: |- + description: |+ Percent represents the percentage of requests that should be mirrored to BackendRef. Its minimum value is 0 (indicating 0% of requests) and its maximum value is 100 (indicating 100% of requests). Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + format: int32 maximum: 100 minimum: 0 @@ -6755,7 +5534,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -6767,14 +5546,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -6833,7 +5608,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -6845,14 +5620,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -6865,7 +5636,7 @@ spec: x-kubernetes-list-type: map type: object type: - description: |- + description: |+ Type identifies the type of filter to apply. As with other API fields, types are classified into three conformance levels: @@ -6890,6 +5661,7 @@ spec: If a reference to a custom filter type cannot be resolved, the filter MUST NOT be skipped. Instead, requests that would have been processed by that filter MUST receive a HTTP error response. + enum: - ResponseHeaderModifier - RequestHeaderModifier @@ -6930,7 +5702,6 @@ spec: rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: RequestHeaderModifier filter cannot be repeated rule: self.filter(f, f.type == 'RequestHeaderModifier').size() @@ -7003,8 +5774,8 @@ spec: - method: type: Exact service: "foo" - - headers: - name: "version" + headers: + - name: "version" value "v1" ``` @@ -7106,11 +5877,10 @@ spec: has(self.method) ? self.method.matches(r"""^[A-Za-z_][A-Za-z_0-9]*$"""): true' type: object - maxItems: 64 + maxItems: 8 type: array - x-kubernetes-list-type: atomic name: - description: |- + description: | Name is the name of the route rule. This name MUST be unique within a Route if it is set. Support: Extended @@ -7119,11 +5889,12 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string sessionPersistence: - description: |- + description: |+ SessionPersistence defines and configures session persistence for the route rule. Support: Extended + properties: absoluteTimeout: description: |- @@ -7158,8 +5929,6 @@ spec: absolute lifetime of the cookie tracked by the gateway and is optional. - Defaults to "Session". - Support: Core for "Session" type Support: Extended for "Permanent" type @@ -7191,7 +5960,7 @@ spec: default: Cookie description: |- Type defines the type of session persistence such as through - the use of a header or cookie. Defaults to cookie based session + the use a header or cookie. Defaults to cookie based session persistence. Support: Core for "Cookie" type @@ -7207,12 +5976,9 @@ spec: is Permanent rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' - - message: cookieConfig can only be set with type Cookie - rule: '!has(self.cookieConfig) || self.type == ''Cookie''' type: object maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less @@ -7237,24 +6003,6 @@ spec: - message: Rule name must be unique within the route rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) && l1.name == l2.name)) - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string type: object status: description: Status defines the current state of GRPCRoute. @@ -7298,9 +6046,9 @@ spec: There are a number of cases where the "Accepted" condition may not be set due to lack of controller visibility, that includes when: - * The Route refers to a nonexistent parent. + * The Route refers to a non-existent parent. * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. + * The Route is in a namespace the controller does not have access to. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -7519,18 +6267,14 @@ spec: - name type: object required: - - conditions - controllerName - parentRef type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic required: - parents type: object - required: - - spec type: object served: true storage: true @@ -7547,9 +6291,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: httproutes.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -7675,9 +6420,8 @@ spec: type: string maxItems: 16 type: array - x-kubernetes-list-type: atomic parentRefs: - description: |- + description: |+ ParentRefs references the resources (usually Gateways) that a Route wants to be attached to. Note that the referenced parent resource needs to allow this for the attachment to be complete. For Gateways, that means @@ -7739,6 +6483,11 @@ spec: connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. + + + + + items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered @@ -7888,7 +6637,6 @@ spec: type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: sectionName or port must be specified when parentRefs includes 2 or more references to the same parent @@ -7918,7 +6666,9 @@ spec: - path: type: PathPrefix value: / - description: Rules are a list of HTTP matchers, filters and actions. + description: |+ + Rules are a list of HTTP matchers, filters and actions. + items: description: |- HTTPRouteRule defines semantics for matching an HTTP request based on @@ -7971,6 +6721,7 @@ spec: namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + When the BackendRef points to a Kubernetes Service, implementations SHOULD honor the appProtocol field if it is set for the target Service Port. @@ -7985,6 +6736,8 @@ spec: If a Route is not able to send traffic to the backend using the specified protocol then the backend is considered invalid. Implementations MUST set the "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + properties: filters: description: |- @@ -8002,316 +6755,6 @@ spec: authentication strategies, rate-limiting, and traffic shaping. API guarantee/conformance is defined based on the type of the filter. properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - When set to true, the gateway will include the `Access-Control-Allow-Credentials` - response header with value true (case-sensitive). - - When set to false or omitted the gateway will omit the header - `Access-Control-Allow-Credentials` entirely (this is the standard CORS - behavior). - - Support: Extended - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - If config contains the wildcard "*" in allowHeaders and the request is - not credentialed, the `Access-Control-Allow-Headers` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Headers from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Headers` response header. When - also the `AllowCredentials` field is true and `AllowHeaders` field - is specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowHeaders cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case-sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - If config contains the wildcard "*" in allowMethods and the request is - not credentialed, the `Access-Control-Allow-Methods` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Method from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Methods` response header. When - also the `AllowCredentials` field is true and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - Conversely, if the request `Origin` matches one of the configured - allowed origins, the gateway sets the response header - `Access-Control-Allow-Origin` to the same value as the `Origin` - header provided by the client. - - When config has the wildcard ("*") in allowOrigins, and the request - is not credentialed (e.g., it is a preflight request), the - `Access-Control-Allow-Origin` response header either contains the - wildcard as well or the Origin from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Origin` response header. When - also the `AllowCredentials` field is true and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The CORSOrigin MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The CORSOrigin MUST include both a - scheme ("http" or "https") and a scheme-specific-part, or it should be a single '*' character. - URIs that include an authority MUST include a fully qualified domain name or - IP address as the host. - maxLength: 253 - minLength: 1 - pattern: (^\*$)|(^(http(s)?):\/\/(((\*\.)?([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9-]+|\*)(:([0-9]{1,5}))?)$) - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowOrigins cannot contain '*' alongside - other origins - rule: '!(''*'' in self && self.size() > 1)' - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the request is not credentialed. - - When the `exposeHeaders` config field contains the "*" wildcard and - the request is credentialed, the gateway cannot use the `*` wildcard in - the `Access-Control-Expose-Headers` response header. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - - When the `MaxAge` field is unspecified, the gateway sets the response - header "Access-Control-Max-Age: 5" by default. - format: int32 - minimum: 1 - type: integer - type: object extensionRef: description: |- ExtensionRef is an optional, implementation-specific extension to the @@ -8347,256 +6790,6 @@ spec: - kind - name type: object - externalAuth: - description: |- - ExternalAuth configures settings related to sending request details - to an external auth service. The external service MUST authenticate - the request, and MAY authorize the request as well. - - If there is any problem communicating with the external service, - this filter MUST fail closed. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef is a reference to a backend to send authorization - requests to. - - The backend must speak the selected protocol (GRPC or HTTP) on the - referenced port. - - If the backend service requires TLS, use BackendTLSPolicy to tell the - implementation to supply the TLS details to be used to connect to that - backend. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind - == ''Service'') ? has(self.port) : true' - forwardBody: - description: |- - ForwardBody controls if requests to the authorization server should include - the body of the client request; and if so, how big that body is allowed - to be. - - It is expected that implementations will buffer the request body up to - `forwardBody.maxSize` bytes. Bodies over that size must be rejected with a - 4xx series error (413 or 403 are common examples), and fail processing - of the filter. - - If unset, or `forwardBody.maxSize` is set to `0`, then the body will not - be forwarded. - - Feature Name: HTTPRouteExternalAuthForwardBody - properties: - maxSize: - description: |- - MaxSize specifies how large in bytes the largest body that will be buffered - and sent to the authorization server. If the body size is larger than - `maxSize`, then the body sent to the authorization server must be - truncated to `maxSize` bytes. - - Experimental note: This behavior needs to be checked against - various dataplanes; it may need to be changed. - See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 - for more. - - If 0, the body will not be sent to the authorization server. - type: integer - type: object - grpc: - description: |- - GRPCAuthConfig contains configuration for communication with ext_authz - protocol-speaking backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what headers from the client request - will be sent to the authorization server. - - If this list is empty, then all headers must be sent. - - If the list has entries, only those entries must be sent. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - type: object - http: - description: |- - HTTPAuthConfig contains configuration for communication with HTTP-speaking - backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what additional headers from the client request - will be sent to the authorization server. - - The following headers must always be sent to the authorization server, - regardless of this setting: - - * `Host` - * `Method` - * `Path` - * `Content-Length` - * `Authorization` - - If this list is empty, then only those headers must be sent. - - Note that `Content-Length` has a special behavior, in that the length - sent must be correct for the actual request to the external authorization - server - that is, it must reflect the actual number of bytes sent in the - body of the request to the authorization server. - - So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set - to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set - to anything other than `0`, then the `Content-Length` of the authorization - request must be set to the actual number of bytes forwarded. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowedResponseHeaders: - description: |- - AllowedResponseHeaders specifies what headers from the authorization response - will be copied into the request to the backend. - - If this list is empty, then all headers from the authorization server - except Authority or Host must be copied. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - path: - description: |- - Path sets the prefix that paths from the client request will have added - when forwarded to the authorization server. - - When empty or unspecified, no prefix is added. - - Valid values are the same as the "value" regex for path values in the `match` - stanza, and the validation regex will screen out invalid paths in the same way. - Even with the validation, implementations MUST sanitize this input before using it - directly. - maxLength: 1024 - pattern: ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$ - type: string - type: object - protocol: - description: |- - ExternalAuthProtocol describes which protocol to use when communicating with an - ext_authz authorization server. - - When this is set to GRPC, each backend must use the Envoy ext_authz protocol - on the port specified in `backendRefs`. Requests and responses are defined - in the protobufs explained at: - https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto - - When this is set to HTTP, each backend must respond with a `200` status - code in on a successful authorization. Any other code is considered - an authorization failure. - - Feature Names: - GRPC Support - HTTPRouteExternalAuthGRPC - HTTP Support - HTTPRouteExternalAuthHTTP - enum: - - HTTP - - GRPC - type: string - required: - - backendRef - - protocol - type: object - x-kubernetes-validations: - - message: grpc must be specified when protocol - is set to 'GRPC' - rule: 'self.protocol == ''GRPC'' ? has(self.grpc) - : true' - - message: protocol must be 'GRPC' when grpc is - set - rule: 'has(self.grpc) ? self.protocol == ''GRPC'' - : true' - - message: http must be specified when protocol - is set to 'HTTP' - rule: 'self.protocol == ''HTTP'' ? has(self.http) - : true' - - message: protocol must be 'HTTP' when http is - set - rule: 'has(self.http) ? self.protocol == ''HTTP'' - : true' requestHeaderModifier: description: |- RequestHeaderModifier defines a schema for a filter that modifies request @@ -8630,7 +6823,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -8642,14 +6835,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -8709,7 +6898,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -8721,14 +6910,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -8741,7 +6926,7 @@ spec: x-kubernetes-list-type: map type: object requestMirror: - description: |- + description: |+ RequestMirror defines a schema for a filter that mirrors requests. Requests are sent to the specified destination, but responses from that destination are ignored. @@ -8751,6 +6936,7 @@ spec: backends. Support: Extended + properties: backendRef: description: |- @@ -8846,12 +7032,13 @@ spec: rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' fraction: - description: |- + description: |+ Fraction represents the fraction of requests that should be mirrored to BackendRef. Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + properties: denominator: default: 100 @@ -8870,13 +7057,14 @@ spec: to denominator rule: self.numerator <= self.denominator percent: - description: |- + description: |+ Percent represents the percentage of requests that should be mirrored to BackendRef. Its minimum value is 0 (indicating 0% of requests) and its maximum value is 100 (indicating 100% of requests). Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + format: int32 maximum: 100 minimum: 0 @@ -9039,9 +7227,6 @@ spec: enum: - 301 - 302 - - 303 - - 307 - - 308 type: integer type: object responseHeaderModifier: @@ -9077,7 +7262,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -9089,14 +7274,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -9156,7 +7337,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -9168,14 +7349,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -9228,8 +7405,6 @@ spec: - RequestRedirect - URLRewrite - ExtensionRef - - CORS - - ExternalAuth type: string urlRewrite: description: |- @@ -9319,11 +7494,6 @@ spec: - type type: object x-kubernetes-validations: - - message: filter.cors must be nil if the filter.type - is not CORS - rule: '!(has(self.cors) && self.type != ''CORS'')' - - message: filter.cors must be specified for CORS filter.type - rule: '!(!has(self.cors) && self.type == ''CORS'')' - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier rule: '!(has(self.requestHeaderModifier) && self.type @@ -9367,22 +7537,17 @@ spec: - message: filter.extensionRef must be specified for ExtensionRef filter.type rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - - message: filter.externalAuth must be nil if the filter.type - is not ExternalAuth - rule: '!(has(self.externalAuth) && self.type != ''ExternalAuth'')' - - message: filter.externalAuth must be specified for - ExternalAuth filter.type - rule: '!(!has(self.externalAuth) && self.type == ''ExternalAuth'')' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: CORS filter cannot be repeated - rule: self.filter(f, f.type == 'CORS').size() <= 1 + - message: May specify either httpRouteFilterRequestRedirect + or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') + && self.exists(f, f.type == ''URLRewrite''))' - message: RequestHeaderModifier filter cannot be repeated rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 @@ -9484,7 +7649,6 @@ spec: ? has(self.port) : true' maxItems: 16 type: array - x-kubernetes-list-type: atomic filters: description: |- Filters define the filters that are applied to requests that match @@ -9494,7 +7658,7 @@ spec: they are specified. Implementations MAY choose to implement this ordering strictly, rejecting - any combination or order of filters that cannot be supported. If implementations + any combination or order of filters that can not be supported. If implementations choose a strict interpretation of filter ordering, they MUST clearly document that behavior. @@ -9516,7 +7680,7 @@ spec: All filters are expected to be compatible with each other except for the URLRewrite and RequestRedirect filters, which may not be combined. If an - implementation cannot support other combinations of filters, they must clearly + implementation can not support other combinations of filters, they must clearly document that limitation. In cases where incompatible or unsupported filters are specified and cause the `Accepted` condition to be set to status `False`, implementations may use the `IncompatibleFilters` reason to specify @@ -9532,316 +7696,6 @@ spec: authentication strategies, rate-limiting, and traffic shaping. API guarantee/conformance is defined based on the type of the filter. properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - When set to true, the gateway will include the `Access-Control-Allow-Credentials` - response header with value true (case-sensitive). - - When set to false or omitted the gateway will omit the header - `Access-Control-Allow-Credentials` entirely (this is the standard CORS - behavior). - - Support: Extended - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - If config contains the wildcard "*" in allowHeaders and the request is - not credentialed, the `Access-Control-Allow-Headers` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Headers from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Headers` response header. When - also the `AllowCredentials` field is true and `AllowHeaders` field - is specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowHeaders cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case-sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - If config contains the wildcard "*" in allowMethods and the request is - not credentialed, the `Access-Control-Allow-Methods` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Method from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Methods` response header. When - also the `AllowCredentials` field is true and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - Conversely, if the request `Origin` matches one of the configured - allowed origins, the gateway sets the response header - `Access-Control-Allow-Origin` to the same value as the `Origin` - header provided by the client. - - When config has the wildcard ("*") in allowOrigins, and the request - is not credentialed (e.g., it is a preflight request), the - `Access-Control-Allow-Origin` response header either contains the - wildcard as well or the Origin from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Origin` response header. When - also the `AllowCredentials` field is true and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The CORSOrigin MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The CORSOrigin MUST include both a - scheme ("http" or "https") and a scheme-specific-part, or it should be a single '*' character. - URIs that include an authority MUST include a fully qualified domain name or - IP address as the host. - maxLength: 253 - minLength: 1 - pattern: (^\*$)|(^(http(s)?):\/\/(((\*\.)?([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9-]+|\*)(:([0-9]{1,5}))?)$) - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowOrigins cannot contain '*' alongside - other origins - rule: '!(''*'' in self && self.size() > 1)' - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the request is not credentialed. - - When the `exposeHeaders` config field contains the "*" wildcard and - the request is credentialed, the gateway cannot use the `*` wildcard in - the `Access-Control-Expose-Headers` response header. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - - When the `MaxAge` field is unspecified, the gateway sets the response - header "Access-Control-Max-Age: 5" by default. - format: int32 - minimum: 1 - type: integer - type: object extensionRef: description: |- ExtensionRef is an optional, implementation-specific extension to the @@ -9877,254 +7731,6 @@ spec: - kind - name type: object - externalAuth: - description: |- - ExternalAuth configures settings related to sending request details - to an external auth service. The external service MUST authenticate - the request, and MAY authorize the request as well. - - If there is any problem communicating with the external service, - this filter MUST fail closed. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef is a reference to a backend to send authorization - requests to. - - The backend must speak the selected protocol (GRPC or HTTP) on the - referenced port. - - If the backend service requires TLS, use BackendTLSPolicy to tell the - implementation to supply the TLS details to be used to connect to that - backend. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') - ? has(self.port) : true' - forwardBody: - description: |- - ForwardBody controls if requests to the authorization server should include - the body of the client request; and if so, how big that body is allowed - to be. - - It is expected that implementations will buffer the request body up to - `forwardBody.maxSize` bytes. Bodies over that size must be rejected with a - 4xx series error (413 or 403 are common examples), and fail processing - of the filter. - - If unset, or `forwardBody.maxSize` is set to `0`, then the body will not - be forwarded. - - Feature Name: HTTPRouteExternalAuthForwardBody - properties: - maxSize: - description: |- - MaxSize specifies how large in bytes the largest body that will be buffered - and sent to the authorization server. If the body size is larger than - `maxSize`, then the body sent to the authorization server must be - truncated to `maxSize` bytes. - - Experimental note: This behavior needs to be checked against - various dataplanes; it may need to be changed. - See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 - for more. - - If 0, the body will not be sent to the authorization server. - type: integer - type: object - grpc: - description: |- - GRPCAuthConfig contains configuration for communication with ext_authz - protocol-speaking backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what headers from the client request - will be sent to the authorization server. - - If this list is empty, then all headers must be sent. - - If the list has entries, only those entries must be sent. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - type: object - http: - description: |- - HTTPAuthConfig contains configuration for communication with HTTP-speaking - backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what additional headers from the client request - will be sent to the authorization server. - - The following headers must always be sent to the authorization server, - regardless of this setting: - - * `Host` - * `Method` - * `Path` - * `Content-Length` - * `Authorization` - - If this list is empty, then only those headers must be sent. - - Note that `Content-Length` has a special behavior, in that the length - sent must be correct for the actual request to the external authorization - server - that is, it must reflect the actual number of bytes sent in the - body of the request to the authorization server. - - So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set - to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set - to anything other than `0`, then the `Content-Length` of the authorization - request must be set to the actual number of bytes forwarded. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowedResponseHeaders: - description: |- - AllowedResponseHeaders specifies what headers from the authorization response - will be copied into the request to the backend. - - If this list is empty, then all headers from the authorization server - except Authority or Host must be copied. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - path: - description: |- - Path sets the prefix that paths from the client request will have added - when forwarded to the authorization server. - - When empty or unspecified, no prefix is added. - - Valid values are the same as the "value" regex for path values in the `match` - stanza, and the validation regex will screen out invalid paths in the same way. - Even with the validation, implementations MUST sanitize this input before using it - directly. - maxLength: 1024 - pattern: ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$ - type: string - type: object - protocol: - description: |- - ExternalAuthProtocol describes which protocol to use when communicating with an - ext_authz authorization server. - - When this is set to GRPC, each backend must use the Envoy ext_authz protocol - on the port specified in `backendRefs`. Requests and responses are defined - in the protobufs explained at: - https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto - - When this is set to HTTP, each backend must respond with a `200` status - code in on a successful authorization. Any other code is considered - an authorization failure. - - Feature Names: - GRPC Support - HTTPRouteExternalAuthGRPC - HTTP Support - HTTPRouteExternalAuthHTTP - enum: - - HTTP - - GRPC - type: string - required: - - backendRef - - protocol - type: object - x-kubernetes-validations: - - message: grpc must be specified when protocol is set - to 'GRPC' - rule: 'self.protocol == ''GRPC'' ? has(self.grpc) : - true' - - message: protocol must be 'GRPC' when grpc is set - rule: 'has(self.grpc) ? self.protocol == ''GRPC'' : - true' - - message: http must be specified when protocol is set - to 'HTTP' - rule: 'self.protocol == ''HTTP'' ? has(self.http) : - true' - - message: protocol must be 'HTTP' when http is set - rule: 'has(self.http) ? self.protocol == ''HTTP'' : - true' requestHeaderModifier: description: |- RequestHeaderModifier defines a schema for a filter that modifies request @@ -10157,7 +7763,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -10169,14 +7775,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -10235,7 +7837,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -10247,14 +7849,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -10267,7 +7865,7 @@ spec: x-kubernetes-list-type: map type: object requestMirror: - description: |- + description: |+ RequestMirror defines a schema for a filter that mirrors requests. Requests are sent to the specified destination, but responses from that destination are ignored. @@ -10277,6 +7875,7 @@ spec: backends. Support: Extended + properties: backendRef: description: |- @@ -10372,12 +7971,13 @@ spec: rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' fraction: - description: |- + description: |+ Fraction represents the fraction of requests that should be mirrored to BackendRef. Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + properties: denominator: default: 100 @@ -10396,13 +7996,14 @@ spec: denominator rule: self.numerator <= self.denominator percent: - description: |- + description: |+ Percent represents the percentage of requests that should be mirrored to BackendRef. Its minimum value is 0 (indicating 0% of requests) and its maximum value is 100 (indicating 100% of requests). Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + format: int32 maximum: 100 minimum: 0 @@ -10565,9 +8166,6 @@ spec: enum: - 301 - 302 - - 303 - - 307 - - 308 type: integer type: object responseHeaderModifier: @@ -10602,7 +8200,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -10614,14 +8212,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -10680,7 +8274,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -10692,14 +8286,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -10752,8 +8342,6 @@ spec: - RequestRedirect - URLRewrite - ExtensionRef - - CORS - - ExternalAuth type: string urlRewrite: description: |- @@ -10843,11 +8431,6 @@ spec: - type type: object x-kubernetes-validations: - - message: filter.cors must be nil if the filter.type is not - CORS - rule: '!(has(self.cors) && self.type != ''CORS'')' - - message: filter.cors must be specified for CORS filter.type - rule: '!(!has(self.cors) && self.type == ''CORS'')' - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier rule: '!(has(self.requestHeaderModifier) && self.type != @@ -10888,22 +8471,13 @@ spec: - message: filter.extensionRef must be specified for ExtensionRef filter.type rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - - message: filter.externalAuth must be nil if the filter.type - is not ExternalAuth - rule: '!(has(self.externalAuth) && self.type != ''ExternalAuth'')' - - message: filter.externalAuth must be specified for ExternalAuth - filter.type - rule: '!(!has(self.externalAuth) && self.type == ''ExternalAuth'')' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: CORS filter cannot be repeated - rule: self.filter(f, f.type == 'CORS').size() <= 1 - message: RequestHeaderModifier filter cannot be repeated rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 @@ -11000,7 +8574,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, only the first entry with an equivalent name MUST be considered for a match. Subsequent @@ -11035,14 +8609,10 @@ spec: - RegularExpression type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header to + be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -11214,9 +8784,8 @@ spec: type: object maxItems: 64 type: array - x-kubernetes-list-type: atomic name: - description: |- + description: | Name is the name of the route rule. This name MUST be unique within a Route if it is set. Support: Extended @@ -11225,14 +8794,15 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string retry: - description: |- + description: |+ Retry defines the configuration for when to retry an HTTP request. Support: Extended + properties: attempts: description: |- - Attempts specifies the maximum number of times an individual request + Attempts specifies the maxmimum number of times an individual request from the gateway to a backend should be retried. If the maximum number of retries has been attempted without a successful @@ -11251,7 +8821,7 @@ spec: For example, setting the `rules[].retry.backoff` field to the value `100ms` will cause a backend request to first be retried approximately 100 milliseconds after timing out or receiving a response code configured - to be retriable. + to be retryable. An implementation MAY use an exponential or alternative backoff strategy for subsequent retry attempts, MAY cap the maximum backoff duration to @@ -11294,7 +8864,7 @@ spec: HTTPRouteRetryStatusCode defines an HTTP response status code for which a backend request should be retried. - Implementations MUST support the following status codes as retriable: + Implementations MUST support the following status codes as retryable: * 500 * 502 @@ -11306,18 +8876,20 @@ spec: Implementations MAY support specifying discrete values in the 400-499 range, which are often inadvisable to retry. + + maximum: 599 minimum: 400 type: integer type: array - x-kubernetes-list-type: atomic type: object sessionPersistence: - description: |- + description: |+ SessionPersistence defines and configures session persistence for the route rule. Support: Extended + properties: absoluteTimeout: description: |- @@ -11352,8 +8924,6 @@ spec: absolute lifetime of the cookie tracked by the gateway and is optional. - Defaults to "Session". - Support: Core for "Session" type Support: Extended for "Permanent" type @@ -11385,7 +8955,7 @@ spec: default: Cookie description: |- Type defines the type of session persistence such as through - the use of a header or cookie. Defaults to cookie based session + the use a header or cookie. Defaults to cookie based session persistence. Support: Core for "Cookie" type @@ -11401,8 +8971,6 @@ spec: is Permanent rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' - - message: cookieConfig can only be set with type Cookie - rule: '!has(self.cookieConfig) || self.type == ''Cookie''' timeouts: description: |- Timeouts defines the timeouts that can be configured for an HTTP request. @@ -11507,9 +9075,7 @@ spec: != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' maxItems: 16 - minItems: 1 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less @@ -11528,24 +9094,6 @@ spec: - message: Rule name must be unique within the route rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) && l1.name == l2.name)) - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string type: object status: description: Status defines the current state of HTTPRoute. @@ -11589,9 +9137,9 @@ spec: There are a number of cases where the "Accepted" condition may not be set due to lack of controller visibility, that includes when: - * The Route refers to a nonexistent parent. + * The Route refers to a non-existent parent. * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. + * The Route is in a namespace the controller does not have access to. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -11810,13 +9358,11 @@ spec: - name type: object required: - - conditions - controllerName - parentRef type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic required: - parents type: object @@ -11940,9 +9486,8 @@ spec: type: string maxItems: 16 type: array - x-kubernetes-list-type: atomic parentRefs: - description: |- + description: |+ ParentRefs references the resources (usually Gateways) that a Route wants to be attached to. Note that the referenced parent resource needs to allow this for the attachment to be complete. For Gateways, that means @@ -12004,6 +9549,11 @@ spec: connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. + + + + + items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered @@ -12153,7 +9703,6 @@ spec: type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: sectionName or port must be specified when parentRefs includes 2 or more references to the same parent @@ -12183,7 +9732,9 @@ spec: - path: type: PathPrefix value: / - description: Rules are a list of HTTP matchers, filters and actions. + description: |+ + Rules are a list of HTTP matchers, filters and actions. + items: description: |- HTTPRouteRule defines semantics for matching an HTTP request based on @@ -12236,6 +9787,7 @@ spec: namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + When the BackendRef points to a Kubernetes Service, implementations SHOULD honor the appProtocol field if it is set for the target Service Port. @@ -12250,6 +9802,8 @@ spec: If a Route is not able to send traffic to the backend using the specified protocol then the backend is considered invalid. Implementations MUST set the "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + + properties: filters: description: |- @@ -12267,316 +9821,6 @@ spec: authentication strategies, rate-limiting, and traffic shaping. API guarantee/conformance is defined based on the type of the filter. properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - When set to true, the gateway will include the `Access-Control-Allow-Credentials` - response header with value true (case-sensitive). - - When set to false or omitted the gateway will omit the header - `Access-Control-Allow-Credentials` entirely (this is the standard CORS - behavior). - - Support: Extended - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - If config contains the wildcard "*" in allowHeaders and the request is - not credentialed, the `Access-Control-Allow-Headers` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Headers from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Headers` response header. When - also the `AllowCredentials` field is true and `AllowHeaders` field - is specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowHeaders cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case-sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - If config contains the wildcard "*" in allowMethods and the request is - not credentialed, the `Access-Control-Allow-Methods` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Method from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Methods` response header. When - also the `AllowCredentials` field is true and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - Conversely, if the request `Origin` matches one of the configured - allowed origins, the gateway sets the response header - `Access-Control-Allow-Origin` to the same value as the `Origin` - header provided by the client. - - When config has the wildcard ("*") in allowOrigins, and the request - is not credentialed (e.g., it is a preflight request), the - `Access-Control-Allow-Origin` response header either contains the - wildcard as well or the Origin from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Origin` response header. When - also the `AllowCredentials` field is true and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The CORSOrigin MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The CORSOrigin MUST include both a - scheme ("http" or "https") and a scheme-specific-part, or it should be a single '*' character. - URIs that include an authority MUST include a fully qualified domain name or - IP address as the host. - maxLength: 253 - minLength: 1 - pattern: (^\*$)|(^(http(s)?):\/\/(((\*\.)?([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9-]+|\*)(:([0-9]{1,5}))?)$) - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowOrigins cannot contain '*' alongside - other origins - rule: '!(''*'' in self && self.size() > 1)' - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the request is not credentialed. - - When the `exposeHeaders` config field contains the "*" wildcard and - the request is credentialed, the gateway cannot use the `*` wildcard in - the `Access-Control-Expose-Headers` response header. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - - When the `MaxAge` field is unspecified, the gateway sets the response - header "Access-Control-Max-Age: 5" by default. - format: int32 - minimum: 1 - type: integer - type: object extensionRef: description: |- ExtensionRef is an optional, implementation-specific extension to the @@ -12612,256 +9856,6 @@ spec: - kind - name type: object - externalAuth: - description: |- - ExternalAuth configures settings related to sending request details - to an external auth service. The external service MUST authenticate - the request, and MAY authorize the request as well. - - If there is any problem communicating with the external service, - this filter MUST fail closed. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef is a reference to a backend to send authorization - requests to. - - The backend must speak the selected protocol (GRPC or HTTP) on the - referenced port. - - If the backend service requires TLS, use BackendTLSPolicy to tell the - implementation to supply the TLS details to be used to connect to that - backend. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind - == ''Service'') ? has(self.port) : true' - forwardBody: - description: |- - ForwardBody controls if requests to the authorization server should include - the body of the client request; and if so, how big that body is allowed - to be. - - It is expected that implementations will buffer the request body up to - `forwardBody.maxSize` bytes. Bodies over that size must be rejected with a - 4xx series error (413 or 403 are common examples), and fail processing - of the filter. - - If unset, or `forwardBody.maxSize` is set to `0`, then the body will not - be forwarded. - - Feature Name: HTTPRouteExternalAuthForwardBody - properties: - maxSize: - description: |- - MaxSize specifies how large in bytes the largest body that will be buffered - and sent to the authorization server. If the body size is larger than - `maxSize`, then the body sent to the authorization server must be - truncated to `maxSize` bytes. - - Experimental note: This behavior needs to be checked against - various dataplanes; it may need to be changed. - See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 - for more. - - If 0, the body will not be sent to the authorization server. - type: integer - type: object - grpc: - description: |- - GRPCAuthConfig contains configuration for communication with ext_authz - protocol-speaking backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what headers from the client request - will be sent to the authorization server. - - If this list is empty, then all headers must be sent. - - If the list has entries, only those entries must be sent. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - type: object - http: - description: |- - HTTPAuthConfig contains configuration for communication with HTTP-speaking - backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what additional headers from the client request - will be sent to the authorization server. - - The following headers must always be sent to the authorization server, - regardless of this setting: - - * `Host` - * `Method` - * `Path` - * `Content-Length` - * `Authorization` - - If this list is empty, then only those headers must be sent. - - Note that `Content-Length` has a special behavior, in that the length - sent must be correct for the actual request to the external authorization - server - that is, it must reflect the actual number of bytes sent in the - body of the request to the authorization server. - - So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set - to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set - to anything other than `0`, then the `Content-Length` of the authorization - request must be set to the actual number of bytes forwarded. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowedResponseHeaders: - description: |- - AllowedResponseHeaders specifies what headers from the authorization response - will be copied into the request to the backend. - - If this list is empty, then all headers from the authorization server - except Authority or Host must be copied. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - path: - description: |- - Path sets the prefix that paths from the client request will have added - when forwarded to the authorization server. - - When empty or unspecified, no prefix is added. - - Valid values are the same as the "value" regex for path values in the `match` - stanza, and the validation regex will screen out invalid paths in the same way. - Even with the validation, implementations MUST sanitize this input before using it - directly. - maxLength: 1024 - pattern: ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$ - type: string - type: object - protocol: - description: |- - ExternalAuthProtocol describes which protocol to use when communicating with an - ext_authz authorization server. - - When this is set to GRPC, each backend must use the Envoy ext_authz protocol - on the port specified in `backendRefs`. Requests and responses are defined - in the protobufs explained at: - https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto - - When this is set to HTTP, each backend must respond with a `200` status - code in on a successful authorization. Any other code is considered - an authorization failure. - - Feature Names: - GRPC Support - HTTPRouteExternalAuthGRPC - HTTP Support - HTTPRouteExternalAuthHTTP - enum: - - HTTP - - GRPC - type: string - required: - - backendRef - - protocol - type: object - x-kubernetes-validations: - - message: grpc must be specified when protocol - is set to 'GRPC' - rule: 'self.protocol == ''GRPC'' ? has(self.grpc) - : true' - - message: protocol must be 'GRPC' when grpc is - set - rule: 'has(self.grpc) ? self.protocol == ''GRPC'' - : true' - - message: http must be specified when protocol - is set to 'HTTP' - rule: 'self.protocol == ''HTTP'' ? has(self.http) - : true' - - message: protocol must be 'HTTP' when http is - set - rule: 'has(self.http) ? self.protocol == ''HTTP'' - : true' requestHeaderModifier: description: |- RequestHeaderModifier defines a schema for a filter that modifies request @@ -12895,7 +9889,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -12907,14 +9901,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -12974,7 +9964,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -12986,14 +9976,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -13006,7 +9992,7 @@ spec: x-kubernetes-list-type: map type: object requestMirror: - description: |- + description: |+ RequestMirror defines a schema for a filter that mirrors requests. Requests are sent to the specified destination, but responses from that destination are ignored. @@ -13016,6 +10002,7 @@ spec: backends. Support: Extended + properties: backendRef: description: |- @@ -13111,12 +10098,13 @@ spec: rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' fraction: - description: |- + description: |+ Fraction represents the fraction of requests that should be mirrored to BackendRef. Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + properties: denominator: default: 100 @@ -13135,13 +10123,14 @@ spec: to denominator rule: self.numerator <= self.denominator percent: - description: |- + description: |+ Percent represents the percentage of requests that should be mirrored to BackendRef. Its minimum value is 0 (indicating 0% of requests) and its maximum value is 100 (indicating 100% of requests). Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + format: int32 maximum: 100 minimum: 0 @@ -13304,9 +10293,6 @@ spec: enum: - 301 - 302 - - 303 - - 307 - - 308 type: integer type: object responseHeaderModifier: @@ -13342,7 +10328,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -13354,14 +10340,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -13421,7 +10403,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -13433,14 +10415,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP + Header to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -13493,8 +10471,6 @@ spec: - RequestRedirect - URLRewrite - ExtensionRef - - CORS - - ExternalAuth type: string urlRewrite: description: |- @@ -13584,11 +10560,6 @@ spec: - type type: object x-kubernetes-validations: - - message: filter.cors must be nil if the filter.type - is not CORS - rule: '!(has(self.cors) && self.type != ''CORS'')' - - message: filter.cors must be specified for CORS filter.type - rule: '!(!has(self.cors) && self.type == ''CORS'')' - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier rule: '!(has(self.requestHeaderModifier) && self.type @@ -13632,22 +10603,17 @@ spec: - message: filter.extensionRef must be specified for ExtensionRef filter.type rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - - message: filter.externalAuth must be nil if the filter.type - is not ExternalAuth - rule: '!(has(self.externalAuth) && self.type != ''ExternalAuth'')' - - message: filter.externalAuth must be specified for - ExternalAuth filter.type - rule: '!(!has(self.externalAuth) && self.type == ''ExternalAuth'')' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: CORS filter cannot be repeated - rule: self.filter(f, f.type == 'CORS').size() <= 1 + - message: May specify either httpRouteFilterRequestRedirect + or httpRouteFilterRequestRewrite, but not both + rule: '!(self.exists(f, f.type == ''RequestRedirect'') + && self.exists(f, f.type == ''URLRewrite''))' - message: RequestHeaderModifier filter cannot be repeated rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 @@ -13749,7 +10715,6 @@ spec: ? has(self.port) : true' maxItems: 16 type: array - x-kubernetes-list-type: atomic filters: description: |- Filters define the filters that are applied to requests that match @@ -13759,7 +10724,7 @@ spec: they are specified. Implementations MAY choose to implement this ordering strictly, rejecting - any combination or order of filters that cannot be supported. If implementations + any combination or order of filters that can not be supported. If implementations choose a strict interpretation of filter ordering, they MUST clearly document that behavior. @@ -13781,7 +10746,7 @@ spec: All filters are expected to be compatible with each other except for the URLRewrite and RequestRedirect filters, which may not be combined. If an - implementation cannot support other combinations of filters, they must clearly + implementation can not support other combinations of filters, they must clearly document that limitation. In cases where incompatible or unsupported filters are specified and cause the `Accepted` condition to be set to status `False`, implementations may use the `IncompatibleFilters` reason to specify @@ -13797,316 +10762,6 @@ spec: authentication strategies, rate-limiting, and traffic shaping. API guarantee/conformance is defined based on the type of the filter. properties: - cors: - description: |- - CORS defines a schema for a filter that responds to the - cross-origin request based on HTTP response header. - - Support: Extended - properties: - allowCredentials: - description: |- - AllowCredentials indicates whether the actual cross-origin request allows - to include credentials. - - When set to true, the gateway will include the `Access-Control-Allow-Credentials` - response header with value true (case-sensitive). - - When set to false or omitted the gateway will omit the header - `Access-Control-Allow-Credentials` entirely (this is the standard CORS - behavior). - - Support: Extended - type: boolean - allowHeaders: - description: |- - AllowHeaders indicates which HTTP request headers are supported for - accessing the requested resource. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Allow-Headers` - response header are separated by a comma (","). - - When the `AllowHeaders` field is configured with one or more headers, the - gateway must return the `Access-Control-Allow-Headers` response header - which value is present in the `AllowHeaders` field. - - If any header name in the `Access-Control-Request-Headers` request header - is not included in the list of header names specified by the response - header `Access-Control-Allow-Headers`, it will present an error on the - client side. - - If any header name in the `Access-Control-Allow-Headers` response header - does not recognize by the client, it will also occur an error on the - client side. - - A wildcard indicates that the requests with all HTTP headers are allowed. - If config contains the wildcard "*" in allowHeaders and the request is - not credentialed, the `Access-Control-Allow-Headers` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Headers from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Headers` response header. When - also the `AllowCredentials` field is true and `AllowHeaders` field - is specified with the `*` wildcard, the gateway must specify one or more - HTTP headers in the value of the `Access-Control-Allow-Headers` response - header. The value of the header `Access-Control-Allow-Headers` is same as - the `Access-Control-Request-Headers` header provided by the client. If - the header `Access-Control-Request-Headers` is not included in the - request, the gateway will omit the `Access-Control-Allow-Headers` - response header, instead of specifying the `*` wildcard. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowHeaders cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowMethods: - description: |- - AllowMethods indicates which HTTP methods are supported for accessing the - requested resource. - - Valid values are any method defined by RFC9110, along with the special - value `*`, which represents all HTTP methods are allowed. - - Method names are case-sensitive, so these values are also case-sensitive. - (See https://www.rfc-editor.org/rfc/rfc2616#section-5.1.1) - - Multiple method names in the value of the `Access-Control-Allow-Methods` - response header are separated by a comma (","). - - A CORS-safelisted method is a method that is `GET`, `HEAD`, or `POST`. - (See https://fetch.spec.whatwg.org/#cors-safelisted-method) The - CORS-safelisted methods are always allowed, regardless of whether they - are specified in the `AllowMethods` field. - - When the `AllowMethods` field is configured with one or more methods, the - gateway must return the `Access-Control-Allow-Methods` response header - which value is present in the `AllowMethods` field. - - If the HTTP method of the `Access-Control-Request-Method` request header - is not included in the list of methods specified by the response header - `Access-Control-Allow-Methods`, it will present an error on the client - side. - - If config contains the wildcard "*" in allowMethods and the request is - not credentialed, the `Access-Control-Allow-Methods` response header - can either use the `*` wildcard or the value of - Access-Control-Request-Method from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Methods` response header. When - also the `AllowCredentials` field is true and `AllowMethods` field - specified with the `*` wildcard, the gateway must specify one HTTP method - in the value of the Access-Control-Allow-Methods response header. The - value of the header `Access-Control-Allow-Methods` is same as the - `Access-Control-Request-Method` header provided by the client. If the - header `Access-Control-Request-Method` is not included in the request, - the gateway will omit the `Access-Control-Allow-Methods` response header, - instead of specifying the `*` wildcard. - - Support: Extended - items: - enum: - - GET - - HEAD - - POST - - PUT - - DELETE - - CONNECT - - OPTIONS - - TRACE - - PATCH - - '*' - type: string - maxItems: 9 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowMethods cannot contain '*' alongside - other methods - rule: '!(''*'' in self && self.size() > 1)' - allowOrigins: - description: |- - AllowOrigins indicates whether the response can be shared with requested - resource from the given `Origin`. - - The `Origin` consists of a scheme and a host, with an optional port, and - takes the form `://(:)`. - - Valid values for scheme are: `http` and `https`. - - Valid values for port are any integer between 1 and 65535 (the list of - available TCP/UDP ports). Note that, if not included, port `80` is - assumed for `http` scheme origins, and port `443` is assumed for `https` - origins. This may affect origin matching. - - The host part of the origin may contain the wildcard character `*`. These - wildcard characters behave as follows: - - * `*` is a greedy match to the _left_, including any number of - DNS labels to the left of its position. This also means that - `*` will include any number of period `.` characters to the - left of its position. - * A wildcard by itself matches all hosts. - - An origin value that includes _only_ the `*` character indicates requests - from all `Origin`s are allowed. - - When the `AllowOrigins` field is configured with multiple origins, it - means the server supports clients from multiple origins. If the request - `Origin` matches the configured allowed origins, the gateway must return - the given `Origin` and sets value of the header - `Access-Control-Allow-Origin` same as the `Origin` header provided by the - client. - - The status code of a successful response to a "preflight" request is - always an OK status (i.e., 204 or 200). - - If the request `Origin` does not match the configured allowed origins, - the gateway returns 204/200 response but doesn't set the relevant - cross-origin response headers. Alternatively, the gateway responds with - 403 status to the "preflight" request is denied, coupled with omitting - the CORS headers. The cross-origin request fails on the client side. - Therefore, the client doesn't attempt the actual cross-origin request. - - Conversely, if the request `Origin` matches one of the configured - allowed origins, the gateway sets the response header - `Access-Control-Allow-Origin` to the same value as the `Origin` - header provided by the client. - - When config has the wildcard ("*") in allowOrigins, and the request - is not credentialed (e.g., it is a preflight request), the - `Access-Control-Allow-Origin` response header either contains the - wildcard as well or the Origin from the request. - - When the request is credentialed, the gateway must not specify the `*` - wildcard in the `Access-Control-Allow-Origin` response header. When - also the `AllowCredentials` field is true and `AllowOrigins` field - specified with the `*` wildcard, the gateway must return a single origin - in the value of the `Access-Control-Allow-Origin` response header, - instead of specifying the `*` wildcard. The value of the header - `Access-Control-Allow-Origin` is same as the `Origin` header provided by - the client. - - Support: Extended - items: - description: |- - The CORSOrigin MUST NOT be a relative URI, and it MUST follow the URI syntax and - encoding rules specified in RFC3986. The CORSOrigin MUST include both a - scheme ("http" or "https") and a scheme-specific-part, or it should be a single '*' character. - URIs that include an authority MUST include a fully qualified domain name or - IP address as the host. - maxLength: 253 - minLength: 1 - pattern: (^\*$)|(^(http(s)?):\/\/(((\*\.)?([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9-]+|\*)(:([0-9]{1,5}))?)$) - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - x-kubernetes-validations: - - message: AllowOrigins cannot contain '*' alongside - other origins - rule: '!(''*'' in self && self.size() > 1)' - exposeHeaders: - description: |- - ExposeHeaders indicates which HTTP response headers can be exposed - to client-side scripts in response to a cross-origin request. - - A CORS-safelisted response header is an HTTP header in a CORS response - that it is considered safe to expose to the client scripts. - The CORS-safelisted response headers include the following headers: - `Cache-Control` - `Content-Language` - `Content-Length` - `Content-Type` - `Expires` - `Last-Modified` - `Pragma` - (See https://fetch.spec.whatwg.org/#cors-safelisted-response-header-name) - The CORS-safelisted response headers are exposed to client by default. - - When an HTTP header name is specified using the `ExposeHeaders` field, - this additional header will be exposed as part of the response to the - client. - - Header names are not case-sensitive. - - Multiple header names in the value of the `Access-Control-Expose-Headers` - response header are separated by a comma (","). - - A wildcard indicates that the responses with all HTTP headers are exposed - to clients. The `Access-Control-Expose-Headers` response header can only - use `*` wildcard as value when the request is not credentialed. - - When the `exposeHeaders` config field contains the "*" wildcard and - the request is credentialed, the gateway cannot use the `*` wildcard in - the `Access-Control-Expose-Headers` response header. - - Support: Extended - items: - description: |- - HTTPHeaderName is the name of an HTTP header. - - Valid values include: - - * "Authorization" - * "Set-Cookie" - - Invalid values include: - - - ":method" - ":" is an invalid character. This means that HTTP/2 pseudo - headers are not currently supported by this type. - - "/invalid" - "/ " is an invalid character - maxLength: 256 - minLength: 1 - pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - maxAge: - default: 5 - description: |- - MaxAge indicates the duration (in seconds) for the client to cache the - results of a "preflight" request. - - The information provided by the `Access-Control-Allow-Methods` and - `Access-Control-Allow-Headers` response headers can be cached by the - client until the time specified by `Access-Control-Max-Age` elapses. - - The default value of `Access-Control-Max-Age` response header is 5 - (seconds). - - When the `MaxAge` field is unspecified, the gateway sets the response - header "Access-Control-Max-Age: 5" by default. - format: int32 - minimum: 1 - type: integer - type: object extensionRef: description: |- ExtensionRef is an optional, implementation-specific extension to the @@ -14142,254 +10797,6 @@ spec: - kind - name type: object - externalAuth: - description: |- - ExternalAuth configures settings related to sending request details - to an external auth service. The external service MUST authenticate - the request, and MAY authorize the request as well. - - If there is any problem communicating with the external service, - this filter MUST fail closed. - - Support: Extended - properties: - backendRef: - description: |- - BackendRef is a reference to a backend to send authorization - requests to. - - The backend must speak the selected protocol (GRPC or HTTP) on the - referenced port. - - If the backend service requires TLS, use BackendTLSPolicy to tell the - implementation to supply the TLS details to be used to connect to that - backend. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') - ? has(self.port) : true' - forwardBody: - description: |- - ForwardBody controls if requests to the authorization server should include - the body of the client request; and if so, how big that body is allowed - to be. - - It is expected that implementations will buffer the request body up to - `forwardBody.maxSize` bytes. Bodies over that size must be rejected with a - 4xx series error (413 or 403 are common examples), and fail processing - of the filter. - - If unset, or `forwardBody.maxSize` is set to `0`, then the body will not - be forwarded. - - Feature Name: HTTPRouteExternalAuthForwardBody - properties: - maxSize: - description: |- - MaxSize specifies how large in bytes the largest body that will be buffered - and sent to the authorization server. If the body size is larger than - `maxSize`, then the body sent to the authorization server must be - truncated to `maxSize` bytes. - - Experimental note: This behavior needs to be checked against - various dataplanes; it may need to be changed. - See https://github.com/kubernetes-sigs/gateway-api/pull/4001#discussion_r2291405746 - for more. - - If 0, the body will not be sent to the authorization server. - type: integer - type: object - grpc: - description: |- - GRPCAuthConfig contains configuration for communication with ext_authz - protocol-speaking backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what headers from the client request - will be sent to the authorization server. - - If this list is empty, then all headers must be sent. - - If the list has entries, only those entries must be sent. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - type: object - http: - description: |- - HTTPAuthConfig contains configuration for communication with HTTP-speaking - backends. - - If unset, implementations must assume the default behavior for each - included field is intended. - properties: - allowedHeaders: - description: |- - AllowedRequestHeaders specifies what additional headers from the client request - will be sent to the authorization server. - - The following headers must always be sent to the authorization server, - regardless of this setting: - - * `Host` - * `Method` - * `Path` - * `Content-Length` - * `Authorization` - - If this list is empty, then only those headers must be sent. - - Note that `Content-Length` has a special behavior, in that the length - sent must be correct for the actual request to the external authorization - server - that is, it must reflect the actual number of bytes sent in the - body of the request to the authorization server. - - So if the `forwardBody` stanza is unset, or `forwardBody.maxSize` is set - to `0`, then `Content-Length` must be `0`. If `forwardBody.maxSize` is set - to anything other than `0`, then the `Content-Length` of the authorization - request must be set to the actual number of bytes forwarded. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - allowedResponseHeaders: - description: |- - AllowedResponseHeaders specifies what headers from the authorization response - will be copied into the request to the backend. - - If this list is empty, then all headers from the authorization server - except Authority or Host must be copied. - items: - type: string - maxItems: 64 - type: array - x-kubernetes-list-type: set - path: - description: |- - Path sets the prefix that paths from the client request will have added - when forwarded to the authorization server. - - When empty or unspecified, no prefix is added. - - Valid values are the same as the "value" regex for path values in the `match` - stanza, and the validation regex will screen out invalid paths in the same way. - Even with the validation, implementations MUST sanitize this input before using it - directly. - maxLength: 1024 - pattern: ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$ - type: string - type: object - protocol: - description: |- - ExternalAuthProtocol describes which protocol to use when communicating with an - ext_authz authorization server. - - When this is set to GRPC, each backend must use the Envoy ext_authz protocol - on the port specified in `backendRefs`. Requests and responses are defined - in the protobufs explained at: - https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/auth/v3/external_auth.proto - - When this is set to HTTP, each backend must respond with a `200` status - code in on a successful authorization. Any other code is considered - an authorization failure. - - Feature Names: - GRPC Support - HTTPRouteExternalAuthGRPC - HTTP Support - HTTPRouteExternalAuthHTTP - enum: - - HTTP - - GRPC - type: string - required: - - backendRef - - protocol - type: object - x-kubernetes-validations: - - message: grpc must be specified when protocol is set - to 'GRPC' - rule: 'self.protocol == ''GRPC'' ? has(self.grpc) : - true' - - message: protocol must be 'GRPC' when grpc is set - rule: 'has(self.grpc) ? self.protocol == ''GRPC'' : - true' - - message: http must be specified when protocol is set - to 'HTTP' - rule: 'self.protocol == ''HTTP'' ? has(self.http) : - true' - - message: protocol must be 'HTTP' when http is set - rule: 'has(self.http) ? self.protocol == ''HTTP'' : - true' requestHeaderModifier: description: |- RequestHeaderModifier defines a schema for a filter that modifies request @@ -14422,7 +10829,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -14434,14 +10841,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -14500,7 +10903,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -14512,14 +10915,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -14532,7 +10931,7 @@ spec: x-kubernetes-list-type: map type: object requestMirror: - description: |- + description: |+ RequestMirror defines a schema for a filter that mirrors requests. Requests are sent to the specified destination, but responses from that destination are ignored. @@ -14542,6 +10941,7 @@ spec: backends. Support: Extended + properties: backendRef: description: |- @@ -14637,12 +11037,13 @@ spec: rule: '(size(self.group) == 0 && self.kind == ''Service'') ? has(self.port) : true' fraction: - description: |- + description: |+ Fraction represents the fraction of requests that should be mirrored to BackendRef. Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + properties: denominator: default: 100 @@ -14661,13 +11062,14 @@ spec: denominator rule: self.numerator <= self.denominator percent: - description: |- + description: |+ Percent represents the percentage of requests that should be mirrored to BackendRef. Its minimum value is 0 (indicating 0% of requests) and its maximum value is 100 (indicating 100% of requests). Only one of Fraction or Percent may be specified. If neither field is specified, 100% of requests will be mirrored. + format: int32 maximum: 100 minimum: 0 @@ -14830,9 +11232,6 @@ spec: enum: - 301 - 302 - - 303 - - 307 - - 308 type: integer type: object responseHeaderModifier: @@ -14867,7 +11266,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -14879,14 +11278,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -14945,7 +11340,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, the first entry with an equivalent name MUST be considered for a match. Subsequent entries @@ -14957,14 +11352,10 @@ spec: pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header + to be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -15017,8 +11408,6 @@ spec: - RequestRedirect - URLRewrite - ExtensionRef - - CORS - - ExternalAuth type: string urlRewrite: description: |- @@ -15108,11 +11497,6 @@ spec: - type type: object x-kubernetes-validations: - - message: filter.cors must be nil if the filter.type is not - CORS - rule: '!(has(self.cors) && self.type != ''CORS'')' - - message: filter.cors must be specified for CORS filter.type - rule: '!(!has(self.cors) && self.type == ''CORS'')' - message: filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier rule: '!(has(self.requestHeaderModifier) && self.type != @@ -15153,22 +11537,13 @@ spec: - message: filter.extensionRef must be specified for ExtensionRef filter.type rule: '!(!has(self.extensionRef) && self.type == ''ExtensionRef'')' - - message: filter.externalAuth must be nil if the filter.type - is not ExternalAuth - rule: '!(has(self.externalAuth) && self.type != ''ExternalAuth'')' - - message: filter.externalAuth must be specified for ExternalAuth - filter.type - rule: '!(!has(self.externalAuth) && self.type == ''ExternalAuth'')' maxItems: 16 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both rule: '!(self.exists(f, f.type == ''RequestRedirect'') && self.exists(f, f.type == ''URLRewrite''))' - - message: CORS filter cannot be repeated - rule: self.filter(f, f.type == 'CORS').size() <= 1 - message: RequestHeaderModifier filter cannot be repeated rule: self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1 @@ -15265,7 +11640,7 @@ spec: name: description: |- Name is the name of the HTTP Header to be matched. Name matching MUST be - case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). If multiple entries specify equivalent header names, only the first entry with an equivalent name MUST be considered for a match. Subsequent @@ -15300,14 +11675,10 @@ spec: - RegularExpression type: string value: - description: |- - Value is the value of HTTP Header to be matched. - - Must consist of printable US-ASCII characters, optionally separated - by single tabs or spaces. See: https://tools.ietf.org/html/rfc7230#section-3.2 + description: Value is the value of HTTP Header to + be matched. maxLength: 4096 minLength: 1 - pattern: ^[!-~]+([\t ]?[!-~]+)*$ type: string required: - name @@ -15479,9 +11850,8 @@ spec: type: object maxItems: 64 type: array - x-kubernetes-list-type: atomic name: - description: |- + description: | Name is the name of the route rule. This name MUST be unique within a Route if it is set. Support: Extended @@ -15490,14 +11860,15 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string retry: - description: |- + description: |+ Retry defines the configuration for when to retry an HTTP request. Support: Extended + properties: attempts: description: |- - Attempts specifies the maximum number of times an individual request + Attempts specifies the maxmimum number of times an individual request from the gateway to a backend should be retried. If the maximum number of retries has been attempted without a successful @@ -15516,7 +11887,7 @@ spec: For example, setting the `rules[].retry.backoff` field to the value `100ms` will cause a backend request to first be retried approximately 100 milliseconds after timing out or receiving a response code configured - to be retriable. + to be retryable. An implementation MAY use an exponential or alternative backoff strategy for subsequent retry attempts, MAY cap the maximum backoff duration to @@ -15559,7 +11930,7 @@ spec: HTTPRouteRetryStatusCode defines an HTTP response status code for which a backend request should be retried. - Implementations MUST support the following status codes as retriable: + Implementations MUST support the following status codes as retryable: * 500 * 502 @@ -15571,18 +11942,20 @@ spec: Implementations MAY support specifying discrete values in the 400-499 range, which are often inadvisable to retry. + + maximum: 599 minimum: 400 type: integer type: array - x-kubernetes-list-type: atomic type: object sessionPersistence: - description: |- + description: |+ SessionPersistence defines and configures session persistence for the route rule. Support: Extended + properties: absoluteTimeout: description: |- @@ -15617,8 +11990,6 @@ spec: absolute lifetime of the cookie tracked by the gateway and is optional. - Defaults to "Session". - Support: Core for "Session" type Support: Extended for "Permanent" type @@ -15650,7 +12021,7 @@ spec: default: Cookie description: |- Type defines the type of session persistence such as through - the use of a header or cookie. Defaults to cookie based session + the use a header or cookie. Defaults to cookie based session persistence. Support: Core for "Cookie" type @@ -15666,8 +12037,6 @@ spec: is Permanent rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' - - message: cookieConfig can only be set with type Cookie - rule: '!has(self.cookieConfig) || self.type == ''Cookie''' timeouts: description: |- Timeouts defines the timeouts that can be configured for an HTTP request. @@ -15772,9 +12141,7 @@ spec: != 1 || !has(self.matches[0].path) || self.matches[0].path.type != ''PathPrefix'') ? false : true) : true' maxItems: 16 - minItems: 1 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: While 16 rules and 64 matches per rule are allowed, the total number of matches across all rules in a route must be less @@ -15793,24 +12160,6 @@ spec: - message: Rule name must be unique within the route rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) && l1.name == l2.name)) - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string type: object status: description: Status defines the current state of HTTPRoute. @@ -15854,9 +12203,9 @@ spec: There are a number of cases where the "Accepted" condition may not be set due to lack of controller visibility, that includes when: - * The Route refers to a nonexistent parent. + * The Route refers to a non-existent parent. * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. + * The Route is in a namespace the controller does not have access to. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -16075,13 +12424,11 @@ spec: - name type: object required: - - conditions - controllerName - parentRef type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic required: - parents type: object @@ -16103,790 +12450,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 - gateway.networking.k8s.io/channel: experimental - name: listenersets.gateway.networking.k8s.io -spec: - group: gateway.networking.k8s.io - names: - categories: - - gateway-api - kind: ListenerSet - listKind: ListenerSetList - plural: listenersets - shortNames: - - lset - singular: listenerset - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.conditions[?(@.type=="Accepted")].status - name: Accepted - type: string - - jsonPath: .status.conditions[?(@.type=="Programmed")].status - name: Programmed - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: |- - ListenerSet defines a set of additional listeners to attach to an existing Gateway. - This resource provides a mechanism to merge multiple listeners into a single Gateway. - - The parent Gateway must explicitly allow ListenerSet attachment through its - AllowedListeners configuration. By default, Gateways do not allow ListenerSet - attachment. - - Routes can attach to a ListenerSet by specifying it as a parentRef, and can - optionally target specific listeners using the sectionName field. - - Policy Attachment: - - Policies that attach to a ListenerSet apply to all listeners defined in that resource - - Policies do not impact listeners in the parent Gateway - - Different ListenerSets attached to the same Gateway can have different policies - - If an implementation cannot apply a policy to specific listeners, it should reject the policy - - ReferenceGrant Semantics: - - ReferenceGrants applied to a Gateway are not inherited by child ListenerSets - - ReferenceGrants applied to a ListenerSet do not grant permission to the parent Gateway's listeners - - A ListenerSet can reference secrets/backends in its own namespace without a ReferenceGrant - - Gateway Integration: - - The parent Gateway's status will include "AttachedListenerSets" - which is the count of ListenerSets that have successfully attached to a Gateway - A ListenerSet is successfully attached to a Gateway when all the following conditions are met: - - The ListenerSet is selected by the Gateway's AllowedListeners field - - The ListenerSet has a valid ParentRef selecting the Gateway - - The ListenerSet's status has the condition "Accepted: true" - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of ListenerSet. - properties: - listeners: - description: |- - Listeners associated with this ListenerSet. Listeners define - logical endpoints that are bound on this referenced parent Gateway's addresses. - - Listeners in a `Gateway` and their attached `ListenerSets` are concatenated - as a list when programming the underlying infrastructure. Each listener - name does not need to be unique across the Gateway and ListenerSets. - See ListenerEntry.Name for more details. - - Implementations MUST treat the parent Gateway as having the merged - list of all listeners from itself and attached ListenerSets using - the following precedence: - - 1. "parent" Gateway - 2. ListenerSet ordered by creation time (oldest first) - 3. ListenerSet ordered alphabetically by "{namespace}/{name}". - - An implementation MAY reject listeners by setting the ListenerEntryStatus - `Accepted` condition to False with the Reason `TooManyListeners` - - If a listener has a conflict, this will be reported in the - Status.ListenerEntryStatus setting the `Conflicted` condition to True. - - Implementations SHOULD be cautious about what information from the - parent or siblings are reported to avoid accidentally leaking - sensitive information that the child would not otherwise have access - to. This can include contents of secrets etc. - items: - properties: - allowedRoutes: - default: - namespaces: - from: Same - description: |- - AllowedRoutes defines the types of routes that MAY be attached to a - Listener and the trusted namespaces where those Route resources MAY be - present. - - Although a client request may match multiple route rules, only one rule - may ultimately receive the request. Matching precedence MUST be - determined in order of the following criteria: - - * The most specific match as defined by the Route type. - * The oldest Route based on creation timestamp. For example, a Route with - a creation timestamp of "2020-09-08 01:02:03" is given precedence over - a Route with a creation timestamp of "2020-09-08 01:02:04". - * If everything else is equivalent, the Route appearing first in - alphabetical order (namespace/name) should be given precedence. For - example, foo/bar is given precedence over foo/baz. - - All valid rules within a Route attached to this Listener should be - implemented. Invalid Route rules can be ignored (sometimes that will mean - the full Route). If a Route rule transitions from valid to invalid, - support for that Route rule should be dropped to ensure consistency. For - example, even if a filter specified by a Route rule is invalid, the rest - of the rules within that Route should still be supported. - properties: - kinds: - description: |- - Kinds specifies the groups and kinds of Routes that are allowed to bind - to this Gateway Listener. When unspecified or empty, the kinds of Routes - selected are determined using the Listener protocol. - - A RouteGroupKind MUST correspond to kinds of Routes that are compatible - with the application protocol specified in the Listener's Protocol field. - If an implementation does not support or recognize this resource type, it - MUST set the "ResolvedRefs" condition to False for this Listener with the - "InvalidRouteKinds" reason. - - Support: Core - items: - description: RouteGroupKind indicates the group and kind - of a Route resource. - properties: - group: - default: gateway.networking.k8s.io - description: Group is the group of the Route. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is the kind of the Route. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - required: - - kind - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - namespaces: - default: - from: Same - description: |- - Namespaces indicates namespaces from which Routes may be attached to this - Listener. This is restricted to the namespace of this Gateway by default. - - Support: Core - properties: - from: - default: Same - description: |- - From indicates where Routes will be selected for this Gateway. Possible - values are: - - * All: Routes in all namespaces may be used by this Gateway. - * Selector: Routes in namespaces selected by the selector may be used by - this Gateway. - * Same: Only Routes in the same namespace may be used by this Gateway. - - Support: Core - enum: - - All - - Selector - - Same - type: string - selector: - description: |- - Selector must be specified when From is set to "Selector". In that case, - only Routes in Namespaces matching this Selector will be selected by this - Gateway. This field is ignored for other values of "From". - - Support: Core - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - type: object - type: object - hostname: - description: |- - Hostname specifies the virtual hostname to match for protocol types that - define this concept. When unspecified, all hostnames are matched. This - field is ignored for protocols that don't require hostname based - matching. - - Implementations MUST apply Hostname matching appropriately for each of - the following protocols: - - * TLS: The Listener Hostname MUST match the SNI. - * HTTP: The Listener Hostname MUST match the Host header of the request. - * HTTPS: The Listener Hostname SHOULD match at both the TLS and HTTP - protocol layers as described above. If an implementation does not - ensure that both the SNI and Host header match the Listener hostname, - it MUST clearly document that. - - For HTTPRoute and TLSRoute resources, there is an interaction with the - `spec.hostnames` array. When both listener and route specify hostnames, - there MUST be an intersection between the values for a Route to be - accepted. For more information, refer to the Route specific Hostnames - documentation. - - Hostnames that are prefixed with a wildcard label (`*.`) are interpreted - as a suffix match. That means that a match for `*.example.com` would match - both `test.example.com`, and `foo.test.example.com`, but not `example.com`. - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - name: - description: |- - Name is the name of the Listener. This name MUST be unique within a - ListenerSet. - - Name is not required to be unique across a Gateway and ListenerSets. - Routes can attach to a Listener by having a ListenerSet as a parentRef - and setting the SectionName - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - port: - description: |- - Port is the network port. Multiple listeners may use the - same port, subject to the Listener compatibility rules. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - protocol: - description: Protocol specifies the network protocol this listener - expects to receive. - maxLength: 255 - minLength: 1 - pattern: ^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])?$|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9]+$ - type: string - tls: - description: |- - TLS is the TLS configuration for the Listener. This field is required if - the Protocol field is "HTTPS" or "TLS". It is invalid to set this field - if the Protocol field is "HTTP", "TCP", or "UDP". - - The association of SNIs to Certificate defined in ListenerTLSConfig is - defined based on the Hostname field for this listener. - - The GatewayClass MUST use the longest matching SNI out of all - available certificates for any TLS handshake. - properties: - certificateRefs: - description: |- - CertificateRefs contains a series of references to Kubernetes objects that - contains TLS certificates and private keys. These certificates are used to - establish a TLS handshake for requests that match the hostname of the - associated listener. - - A single CertificateRef to a Kubernetes Secret has "Core" support. - Implementations MAY choose to support attaching multiple certificates to - a Listener, but this behavior is implementation-specific. - - References to a resource in different namespace are invalid UNLESS there - is a ReferenceGrant in the target namespace that allows the certificate - to be attached. If a ReferenceGrant does not allow this reference, the - "ResolvedRefs" condition MUST be set to False for this listener with the - "RefNotPermitted" reason. - - This field is required to have at least one element when the mode is set - to "Terminate" (default) and is optional otherwise. - - CertificateRefs can reference to standard Kubernetes resources, i.e. - Secret, or implementation-specific custom resources. - - Support: Core - A single reference to a Kubernetes Secret of type kubernetes.io/tls - - Support: Implementation-specific (More than one reference or other resource types) - items: - description: |- - SecretObjectReference identifies an API object including its namespace, - defaulting to Secret. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - - References to objects with invalid Group and Kind are not valid, and must - be rejected by the implementation, with appropriate Conditions set - on the containing object. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Secret - description: Kind is kind of the referent. For example - "Secret". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referenced object. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - maxItems: 64 - type: array - x-kubernetes-list-type: atomic - mode: - default: Terminate - description: |- - Mode defines the TLS behavior for the TLS session initiated by the client. - There are two possible modes: - - - Terminate: The TLS session between the downstream client and the - Gateway is terminated at the Gateway. This mode requires certificates - to be specified in some way, such as populating the certificateRefs - field. - - Passthrough: The TLS session is NOT terminated by the Gateway. This - implies that the Gateway can't decipher the TLS stream except for - the ClientHello message of the TLS protocol. The certificateRefs field - is ignored in this mode. - - Support: Core - enum: - - Terminate - - Passthrough - type: string - options: - additionalProperties: - description: |- - AnnotationValue is the value of an annotation in Gateway API. This is used - for validation of maps such as TLS options. This roughly matches Kubernetes - annotation validation, although the length validation in that case is based - on the entire size of the annotations struct. - maxLength: 4096 - minLength: 0 - type: string - description: |- - Options are a list of key/value pairs to enable extended TLS - configuration for each implementation. For example, configuring the - minimum TLS version or supported cipher suites. - - A set of common keys MAY be defined by the API in the future. To avoid - any ambiguity, implementation-specific definitions MUST use - domain-prefixed names, such as `example.com/my-custom-option`. - Un-prefixed names are reserved for key names defined by Gateway API. - - Support: Implementation-specific - maxProperties: 16 - type: object - type: object - x-kubernetes-validations: - - message: certificateRefs or options must be specified when - mode is Terminate - rule: 'self.mode == ''Terminate'' ? size(self.certificateRefs) - > 0 || size(self.options) > 0 : true' - required: - - name - - port - - protocol - type: object - maxItems: 64 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - x-kubernetes-validations: - - message: tls must not be specified for protocols ['HTTP', 'TCP', - 'UDP'] - rule: 'self.all(l, l.protocol in [''HTTP'', ''TCP'', ''UDP''] ? - !has(l.tls) : true)' - - message: tls mode must be Terminate for protocol HTTPS - rule: 'self.all(l, (l.protocol == ''HTTPS'' && has(l.tls)) ? (l.tls.mode - == '''' || l.tls.mode == ''Terminate'') : true)' - - message: tls mode must be set for protocol TLS - rule: 'self.all(l, (l.protocol == ''TLS'' ? has(l.tls) && has(l.tls.mode) - && l.tls.mode != '''' : true))' - - message: hostname must not be specified for protocols ['TCP', 'UDP'] - rule: 'self.all(l, l.protocol in [''TCP'', ''UDP''] ? (!has(l.hostname) - || l.hostname == '''') : true)' - - message: Listener name must be unique within the Gateway - rule: self.all(l1, self.exists_one(l2, l1.name == l2.name)) - - message: Combination of port, protocol and hostname must be unique - for each listener - rule: 'self.all(l1, !has(l1.port) || self.exists_one(l2, has(l2.port) - && l1.port == l2.port && l1.protocol == l2.protocol && (has(l1.hostname) - && has(l2.hostname) ? l1.hostname == l2.hostname : !has(l1.hostname) - && !has(l2.hostname))))' - parentRef: - description: ParentRef references the Gateway that the listeners are - attached to. - properties: - group: - default: gateway.networking.k8s.io - description: Group is the group of the referent. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: Kind is kind of the referent. For example "Gateway". - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. If not present, - the namespace of the referent is assumed to be the same as - the namespace of the referring object. - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - required: - - listeners - - parentRef - type: object - status: - default: - conditions: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Accepted - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: Status defines the current state of ListenerSet. - properties: - conditions: - default: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Accepted - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: |- - Conditions describe the current conditions of the ListenerSet. - - Implementations MUST express ListenerSet conditions using the - `ListenerSetConditionType` and `ListenerSetConditionReason` - constants so that operators and tools can converge on a common - vocabulary to describe ListenerSet state. - - Known condition types are: - - * "Accepted" - * "Programmed" - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - listeners: - description: Listeners provide status for each unique listener port - defined in the Spec. - items: - description: ListenerStatus is the status associated with a Listener. - properties: - attachedRoutes: - description: |- - AttachedRoutes represents the total number of Routes that have been - successfully attached to this Listener. - - Successful attachment of a Route to a Listener is based solely on the - combination of the AllowedRoutes field on the corresponding Listener - and the Route's ParentRefs field. A Route is successfully attached to - a Listener when it is selected by the Listener's AllowedRoutes field - AND the Route has a valid ParentRef selecting the whole Gateway - resource or a specific Listener as a parent resource (more detail on - attachment semantics can be found in the documentation on the various - Route kinds ParentRefs fields). Listener status does not impact - successful attachment, i.e. the AttachedRoutes field count MUST be set - for Listeners, even if the Accepted condition of an individual Listener is set - to "False". The AttachedRoutes number represents the number of Routes with - the Accepted condition set to "True" that have been attached to this Listener. - Routes with any other value for the Accepted condition MUST NOT be included - in this count. - - Uses for this field include troubleshooting Route attachment and - measuring blast radius/impact of changes to a Listener. - format: int32 - type: integer - conditions: - description: Conditions describe the current condition of this - listener. - items: - description: Condition contains details for one aspect of - the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, - Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - name: - description: Name is the name of the Listener that this status - corresponds to. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - supportedKinds: - description: |- - SupportedKinds is the list indicating the Kinds supported by this - listener. This MUST represent the kinds supported by an implementation for - that Listener configuration. - - If kinds are specified in Spec that are not supported, they MUST NOT - appear in this list and an implementation MUST set the "ResolvedRefs" - condition to "False" with the "InvalidRouteKinds" reason. If both valid - and invalid Route kinds are specified, the implementation MUST - reference the valid Route kinds that have been specified. - items: - description: RouteGroupKind indicates the group and kind of - a Route resource. - properties: - group: - default: gateway.networking.k8s.io - description: Group is the group of the Route. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is the kind of the Route. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - required: - - kind - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - required: - - attachedRoutes - - conditions - - name - type: object - maxItems: 64 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: referencegrants.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -16901,169 +12468,6 @@ spec: singular: referencegrant scope: Namespaced versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: |- - ReferenceGrant identifies kinds of resources in other namespaces that are - trusted to reference the specified kinds of resources in the same namespace - as the policy. - - Each ReferenceGrant can be used to represent a unique trust relationship. - Additional Reference Grants can be used to add to the set of trusted - sources of inbound references for the namespace they are defined within. - - All cross-namespace references in Gateway API (with the exception of cross-namespace - Gateway-route attachment) require a ReferenceGrant. - - ReferenceGrant is a form of runtime verification allowing users to assert - which cross-namespace object references are permitted. Implementations that - support ReferenceGrant MUST NOT permit cross-namespace references which have - no grant, and MUST respond to the removal of a grant by revoking the access - that the grant allowed. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of ReferenceGrant. - properties: - from: - description: |- - From describes the trusted namespaces and kinds that can reference the - resources described in "To". Each entry in this list MUST be considered - to be an additional place that references can be valid from, or to put - this another way, entries MUST be combined using OR. - - Support: Core - items: - description: ReferenceGrantFrom describes trusted namespaces and - kinds. - properties: - group: - description: |- - Group is the group of the referent. - When empty, the Kubernetes core API group is inferred. - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: |- - Kind is the kind of the referent. Although implementations may support - additional resources, the following types are part of the "Core" - support level for this field. - - When used to permit a SecretObjectReference: - - * Gateway - - When used to permit a BackendObjectReference: - - * GRPCRoute - * HTTPRoute - * TCPRoute - * TLSRoute - * UDPRoute - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - namespace: - description: |- - Namespace is the namespace of the referent. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - namespace - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - to: - description: |- - To describes the resources that may be referenced by the resources - described in "From". Each entry in this list MUST be considered to be an - additional place that references can be valid to, or to put this another - way, entries MUST be combined using OR. - - Support: Core - items: - description: |- - ReferenceGrantTo describes what Kinds are allowed as targets of the - references. - properties: - group: - description: |- - Group is the group of the referent. - When empty, the Kubernetes core API group is inferred. - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: |- - Kind is the kind of the referent. Although implementations may support - additional resources, the following types are part of the "Core" - support level for this field: - - * Secret when used to permit a SecretObjectReference - * Service when used to permit a BackendObjectReference - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. When unspecified, this policy - refers to all resources of the specified Group and Kind in the local - namespace. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - required: - - from - - to - type: object - type: object - served: true - storage: false - subresources: {} - additionalPrinterColumns: - jsonPath: .metadata.creationTimestamp name: Age @@ -17168,7 +12572,6 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic to: description: |- To describes the resources that may be referenced by the resources @@ -17218,7 +12621,6 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic required: - from - to @@ -17238,9 +12640,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: tcproutes.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -17286,7 +12689,7 @@ spec: description: Spec defines the desired state of TCPRoute. properties: parentRefs: - description: |- + description: |+ ParentRefs references the resources (usually Gateways) that a Route wants to be attached to. Note that the referenced parent resource needs to allow this for the attachment to be complete. For Gateways, that means @@ -17348,6 +12751,11 @@ spec: connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. + + + + + items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered @@ -17497,7 +12905,6 @@ spec: type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: sectionName or port must be specified when parentRefs includes 2 or more references to the same parent @@ -17522,14 +12929,16 @@ spec: || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port == p2.port)))) rules: - description: Rules are a list of TCP matchers and actions. + description: |+ + Rules are a list of TCP matchers and actions. + items: description: TCPRouteRule is the configuration for a given rule. properties: backendRefs: description: |- BackendRefs defines the backend(s) where matching requests should be - sent. If unspecified or invalid (refers to a nonexistent resource or a + sent. If unspecified or invalid (refers to a non-existent resource or a Service with no endpoints), the underlying implementation MUST actively reject connection attempts to this backend. Connection rejections must respect weight; if an invalid backend is requested to have 80% of @@ -17552,6 +12961,7 @@ spec: namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + When the BackendRef points to a Kubernetes Service, implementations SHOULD honor the appProtocol field if it is set for the target Service Port. @@ -17567,6 +12977,7 @@ spec: protocol then the backend is considered invalid. Implementations MUST set the "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + Note that when the BackendTLSPolicy object is enabled by the implementation, there are some extra rules about validity to consider here. See the fields @@ -17662,7 +13073,6 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic name: description: |- Name is the name of the route rule. This name MUST be unique within a Route if it is set. @@ -17672,35 +13082,14 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - required: - - backendRefs type: object maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: Rule name must be unique within the route rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) && l1.name == l2.name)) - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string required: - rules type: object @@ -17746,9 +13135,9 @@ spec: There are a number of cases where the "Accepted" condition may not be set due to lack of controller visibility, that includes when: - * The Route refers to a nonexistent parent. + * The Route refers to a non-existent parent. * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. + * The Route is in a namespace the controller does not have access to. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -17967,13 +13356,11 @@ spec: - name type: object required: - - conditions - controllerName - parentRef type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic required: - parents type: object @@ -17995,9 +13382,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: tlsroutes.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -18014,7 +13402,7 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1 + name: v1alpha2 schema: openAPIV3Schema: description: |- @@ -18024,781 +13412,6 @@ spec: If you need to forward traffic to a single target for a TLS listener, you could choose to use a TCPRoute with a TLS listener. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of TLSRoute. - properties: - hostnames: - description: |- - Hostnames defines a set of SNI hostnames that should match against the - SNI attribute of TLS ClientHello message in TLS handshake. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed in SNI hostnames per RFC 6066. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - items: - description: |- - Hostname is the fully qualified domain name of a network host. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - Hostname can be "precise" which is a domain name without the terminating - dot of a network host (e.g. "foo.example.com") or "wildcard", which is a - domain name prefixed with a single wildcard label (e.g. `*.example.com`). - - Note that as per RFC1035 and RFC1123, a *label* must consist of lower case - alphanumeric characters or '-', and must start and end with an alphanumeric - character. No other punctuation is allowed. - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: Hostnames cannot contain an IP - rule: self.all(h, !isIP(h)) - - message: Hostnames must be valid based on RFC-1123 - rule: 'self.all(h, !h.contains(''*'') ? h.matches(''^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$'') - : true)' - - message: Wildcards on hostnames must be the first label, and the - rest of hostname must be valid based on RFC-1123 - rule: 'self.all(h, h.contains(''*'') ? (h.startsWith(''*.'') && - h.substring(2).matches(''^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$'')) - : true)' - parentRefs: - description: |- - ParentRefs references the resources (usually Gateways) that a Route wants - to be attached to. Note that the referenced parent resource needs to - allow this for the attachment to be complete. For Gateways, that means - the Gateway needs to allow attachment from Routes of this kind and - namespace. For Services, that means the Service must either be in the same - namespace for a "producer" route, or the mesh implementation must support - and allow "consumer" routes for the referenced Service. ReferenceGrant is - not applicable for governing ParentRefs to Services - it is not possible to - create a "producer" route for a Service in a different namespace from the - Route. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - ParentRefs must be _distinct_. This means either that: - - * They select different objects. If this is the case, then parentRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, `namespace`, and `name` must - be unique across all parentRef entries in the Route. - * They do not select different objects, but for each optional field used, - each ParentRef that selects the same object must set the same set of - optional fields to different values. If one ParentRef sets a - combination of optional fields, all must set the same combination. - - Some examples: - - * If one ParentRef sets `sectionName`, all ParentRefs referencing the - same object must also set `sectionName`. - * If one ParentRef sets `port`, all ParentRefs referencing the same - object must also set `port`. - * If one ParentRef sets `sectionName` and `port`, all ParentRefs - referencing the same object must also set `sectionName` and `port`. - - It is possible to separately reference multiple distinct objects that may - be collapsed by an implementation. For example, some implementations may - choose to merge compatible Gateway Listeners together. If that is the - case, the list of routes attached to those resources should also be - merged. - - Note that for ParentRefs that cross namespace boundaries, there are specific - rules. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example, - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable other kinds of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - items: - description: |- - ParentReference identifies an API object (usually a Gateway) that can be considered - a parent of this resource (usually a route). There are two kinds of parent resources - with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - maxItems: 32 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: sectionName or port must be specified when parentRefs includes - 2 or more references to the same parent - rule: 'self.all(p1, self.all(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) - || p1.__namespace__ == '''') && (!has(p2.__namespace__) || p2.__namespace__ - == '''')) || (has(p1.__namespace__) && has(p2.__namespace__) && - p1.__namespace__ == p2.__namespace__)) ? ((!has(p1.sectionName) - || p1.sectionName == '''') == (!has(p2.sectionName) || p2.sectionName - == '''') && (!has(p1.port) || p1.port == 0) == (!has(p2.port) - || p2.port == 0)): true))' - - message: sectionName or port must be unique when parentRefs includes - 2 or more references to the same parent - rule: self.all(p1, self.exists_one(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) - || p1.__namespace__ == '') && (!has(p2.__namespace__) || p2.__namespace__ - == '')) || (has(p1.__namespace__) && has(p2.__namespace__) && - p1.__namespace__ == p2.__namespace__ )) && (((!has(p1.sectionName) - || p1.sectionName == '') && (!has(p2.sectionName) || p2.sectionName - == '')) || ( has(p1.sectionName) && has(p2.sectionName) && p1.sectionName - == p2.sectionName)) && (((!has(p1.port) || p1.port == 0) && (!has(p2.port) - || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port - == p2.port)))) - rules: - description: Rules are a list of actions. - items: - description: TLSRouteRule is the configuration for a given rule. - properties: - backendRefs: - description: |- - BackendRefs defines the backend(s) where matching requests should be - sent. If unspecified or invalid (refers to a nonexistent resource or - a Service with no endpoints), the rule performs no forwarding; if no - filters are specified that would result in a response being sent, the - underlying implementation must actively reject request attempts to this - backend, by rejecting the connection. Request rejections must respect - weight; if an invalid backend is requested to have 80% of requests, then - 80% of requests must be rejected instead. - - Support: Core for Kubernetes Service - - Support: Extended for Kubernetes ServiceImport - - Support: Implementation-specific for any other resource - - Support for weight: Extended - items: - description: |- - BackendRef defines how a Route should forward a request to a Kubernetes - resource. - - Note that when a namespace different than the local namespace is specified, a - ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - - When the BackendRef points to a Kubernetes Service, implementations SHOULD - honor the appProtocol field if it is set for the target Service Port. - - Implementations supporting appProtocol SHOULD recognize the Kubernetes - Standard Application Protocols defined in KEP-3726. - - If a Service appProtocol isn't specified, an implementation MAY infer the - backend protocol through its own means. Implementations MAY infer the - protocol from the Route type referring to the backend Service. - - If a Route is not able to send traffic to the backend using the specified - protocol then the backend is considered invalid. Implementations MUST set the - "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. - - - Note that when the BackendTLSPolicy object is enabled by the implementation, - there are some extra rules about validity to consider here. See the fields - where this struct is used for more information about the exact behavior. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - weight: - default: 1 - description: |- - Weight specifies the proportion of requests forwarded to the referenced - backend. This is computed as weight/(sum of all weights in this - BackendRefs list). For non-zero values, there may be some epsilon from - the exact proportion defined here depending on the precision an - implementation supports. Weight is not a percentage and the sum of - weights does not need to equal 100. - - If only one backend is specified and it has a weight greater than 0, 100% - of the traffic is forwarded to that backend. If weight is set to 0, no - traffic should be forwarded for this entry. If unspecified, weight - defaults to 1. - - Support for this field varies based on the context where used. - format: int32 - maximum: 1000000 - minimum: 0 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') - ? has(self.port) : true' - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - name: - description: Name is the name of the route rule. This name MUST - be unique within a Route if it is set. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - backendRefs - type: object - maxItems: 1 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string - required: - - hostnames - - rules - type: object - status: - description: Status defines the current state of TLSRoute. - properties: - parents: - description: |- - Parents is a list of parent resources (usually Gateways) that are - associated with the route, and the status of the route with respect to - each parent. When this route attaches to a parent, the controller that - manages the parent must add an entry to this list when the controller - first sees the route and should update the entry as appropriate when the - route or gateway is modified. - - Note that parent references that cannot be resolved by an implementation - of this API will not be added to this list. Implementations of this API - can only populate Route status for the Gateways/parent resources they are - responsible for. - - A maximum of 32 Gateways will be represented in this list. An empty list - means the route has not been attached to any Gateway. - items: - description: |- - RouteParentStatus describes the status of a route with respect to an - associated Parent. - properties: - conditions: - description: |- - Conditions describes the status of the route with respect to the Gateway. - Note that the route's availability is also subject to the Gateway's own - status conditions and listener status. - - If the Route's ParentRef specifies an existing Gateway that supports - Routes of this kind AND that Gateway's controller has sufficient access, - then that Gateway's controller MUST set the "Accepted" condition on the - Route, to indicate whether the route has been accepted or rejected by the - Gateway, and why. - - A Route MUST be considered "Accepted" if at least one of the Route's - rules is implemented by the Gateway. - - There are a number of cases where the "Accepted" condition may not be set - due to lack of controller visibility, that includes when: - - * The Route refers to a nonexistent parent. - * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. - items: - description: Condition contains details for one aspect of - the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, - Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - controllerName: - description: |- - ControllerName is a domain/path string that indicates the name of the - controller that wrote this status. This corresponds with the - controllerName field on GatewayClass. - - Example: "example.net/gateway-controller". - - The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are - valid Kubernetes names - (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). - - Controllers MUST populate this field when writing status. Controllers should ensure that - entries to status populated with their ControllerName are cleaned up when they are no - longer necessary. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ - type: string - parentRef: - description: |- - ParentRef corresponds with a ParentRef in the spec that this - RouteParentStatus struct describes the status of. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - required: - - conditions - - controllerName - - parentRef - type: object - maxItems: 32 - type: array - x-kubernetes-list-type: atomic - required: - - parents - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - deprecated: true - deprecationWarning: The v1alpha2 version of TLSRoute has been deprecated and will - be removed in a future release of the API. Please upgrade to v1. - name: v1alpha2 - schema: - openAPIV3Schema: - description: |- - The TLSRoute resource is similar to TCPRoute, but can be configured - to match against TLS-specific metadata. This allows more flexibility - in matching streams for a given TLS listener. properties: apiVersion: description: |- @@ -18877,9 +13490,8 @@ spec: type: string maxItems: 16 type: array - x-kubernetes-list-type: atomic parentRefs: - description: |- + description: |+ ParentRefs references the resources (usually Gateways) that a Route wants to be attached to. Note that the referenced parent resource needs to allow this for the attachment to be complete. For Gateways, that means @@ -18941,6 +13553,11 @@ spec: connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. + + + + + items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered @@ -19090,7 +13707,6 @@ spec: type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: sectionName or port must be specified when parentRefs includes 2 or more references to the same parent @@ -19115,20 +13731,23 @@ spec: || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port == p2.port)))) rules: - description: Rules are a list of TLS matchers and actions. + description: |+ + Rules are a list of TLS matchers and actions. + items: description: TLSRouteRule is the configuration for a given rule. properties: backendRefs: description: |- BackendRefs defines the backend(s) where matching requests should be - sent. If unspecified or invalid (refers to a nonexistent resource or + sent. If unspecified or invalid (refers to a non-existent resource or a Service with no endpoints), the rule performs no forwarding; if no filters are specified that would result in a response being sent, the underlying implementation must actively reject request attempts to this - backend, by rejecting the connection. Request rejections must respect - weight; if an invalid backend is requested to have 80% of requests, then - 80% of requests must be rejected instead. + backend, by rejecting the connection or returning a 500 status code. + Request rejections must respect weight; if an invalid backend is + requested to have 80% of requests, then 80% of requests must be rejected + instead. Support: Core for Kubernetes Service @@ -19147,6 +13766,7 @@ spec: namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + When the BackendRef points to a Kubernetes Service, implementations SHOULD honor the appProtocol field if it is set for the target Service Port. @@ -19162,6 +13782,7 @@ spec: protocol then the backend is considered invalid. Implementations MUST set the "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + Note that when the BackendTLSPolicy object is enabled by the implementation, there are some extra rules about validity to consider here. See the fields @@ -19257,43 +13878,23 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic name: - description: Name is the name of the route rule. This name MUST - be unique within a Route if it is set. + description: |- + Name is the name of the route rule. This name MUST be unique within a Route if it is set. + + Support: Extended maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - required: - - backendRefs type: object maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: Rule name must be unique within the route rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) && l1.name == l2.name)) - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string required: - rules type: object @@ -19339,9 +13940,9 @@ spec: There are a number of cases where the "Accepted" condition may not be set due to lack of controller visibility, that includes when: - * The Route refers to a nonexistent parent. + * The Route refers to a non-existent parent. * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. + * The Route is in a namespace the controller does not have access to. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -19560,13 +14161,11 @@ spec: - name type: object required: - - conditions - controllerName - parentRef type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic required: - parents type: object @@ -19574,785 +14173,7 @@ spec: - spec type: object served: true - storage: false - subresources: - status: {} - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - deprecated: true - deprecationWarning: The v1alpha3 version of TLSRoute has been deprecated and will - be removed in a future release of the API. Please upgrade to v1. - name: v1alpha3 - schema: - openAPIV3Schema: - description: |- - The TLSRoute resource is similar to TCPRoute, but can be configured - to match against TLS-specific metadata. This allows more flexibility - in matching streams for a given TLS listener. - - If you need to forward traffic to a single target for a TLS listener, you - could choose to use a TCPRoute with a TLS listener. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of TLSRoute. - properties: - hostnames: - description: |- - Hostnames defines a set of SNI hostnames that should match against the - SNI attribute of TLS ClientHello message in TLS handshake. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed in SNI hostnames per RFC 6066. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - items: - description: |- - Hostname is the fully qualified domain name of a network host. This matches - the RFC 1123 definition of a hostname with 2 notable exceptions: - - 1. IPs are not allowed. - 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard - label must appear by itself as the first label. - - Hostname can be "precise" which is a domain name without the terminating - dot of a network host (e.g. "foo.example.com") or "wildcard", which is a - domain name prefixed with a single wildcard label (e.g. `*.example.com`). - - Note that as per RFC1035 and RFC1123, a *label* must consist of lower case - alphanumeric characters or '-', and must start and end with an alphanumeric - character. No other punctuation is allowed. - maxLength: 253 - minLength: 1 - pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: Hostnames cannot contain an IP - rule: self.all(h, !isIP(h)) - - message: Hostnames must be valid based on RFC-1123 - rule: 'self.all(h, !h.contains(''*'') ? h.matches(''^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$'') - : true)' - - message: Wildcards on hostnames must be the first label, and the - rest of hostname must be valid based on RFC-1123 - rule: 'self.all(h, h.contains(''*'') ? (h.startsWith(''*.'') && - h.substring(2).matches(''^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$'')) - : true)' - parentRefs: - description: |- - ParentRefs references the resources (usually Gateways) that a Route wants - to be attached to. Note that the referenced parent resource needs to - allow this for the attachment to be complete. For Gateways, that means - the Gateway needs to allow attachment from Routes of this kind and - namespace. For Services, that means the Service must either be in the same - namespace for a "producer" route, or the mesh implementation must support - and allow "consumer" routes for the referenced Service. ReferenceGrant is - not applicable for governing ParentRefs to Services - it is not possible to - create a "producer" route for a Service in a different namespace from the - Route. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - ParentRefs must be _distinct_. This means either that: - - * They select different objects. If this is the case, then parentRef - entries are distinct. In terms of fields, this means that the - multi-part key defined by `group`, `kind`, `namespace`, and `name` must - be unique across all parentRef entries in the Route. - * They do not select different objects, but for each optional field used, - each ParentRef that selects the same object must set the same set of - optional fields to different values. If one ParentRef sets a - combination of optional fields, all must set the same combination. - - Some examples: - - * If one ParentRef sets `sectionName`, all ParentRefs referencing the - same object must also set `sectionName`. - * If one ParentRef sets `port`, all ParentRefs referencing the same - object must also set `port`. - * If one ParentRef sets `sectionName` and `port`, all ParentRefs - referencing the same object must also set `sectionName` and `port`. - - It is possible to separately reference multiple distinct objects that may - be collapsed by an implementation. For example, some implementations may - choose to merge compatible Gateway Listeners together. If that is the - case, the list of routes attached to those resources should also be - merged. - - Note that for ParentRefs that cross namespace boundaries, there are specific - rules. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example, - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable other kinds of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - items: - description: |- - ParentReference identifies an API object (usually a Gateway) that can be considered - a parent of this resource (usually a route). There are two kinds of parent resources - with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - This API may be extended in the future to support additional kinds of parent - resources. - - The API object must be valid in the cluster; the Group and Kind must - be registered in the cluster for this reference to be valid. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - maxItems: 32 - type: array - x-kubernetes-list-type: atomic - x-kubernetes-validations: - - message: sectionName or port must be specified when parentRefs includes - 2 or more references to the same parent - rule: 'self.all(p1, self.all(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) - || p1.__namespace__ == '''') && (!has(p2.__namespace__) || p2.__namespace__ - == '''')) || (has(p1.__namespace__) && has(p2.__namespace__) && - p1.__namespace__ == p2.__namespace__)) ? ((!has(p1.sectionName) - || p1.sectionName == '''') == (!has(p2.sectionName) || p2.sectionName - == '''') && (!has(p1.port) || p1.port == 0) == (!has(p2.port) - || p2.port == 0)): true))' - - message: sectionName or port must be unique when parentRefs includes - 2 or more references to the same parent - rule: self.all(p1, self.exists_one(p2, p1.group == p2.group && p1.kind - == p2.kind && p1.name == p2.name && (((!has(p1.__namespace__) - || p1.__namespace__ == '') && (!has(p2.__namespace__) || p2.__namespace__ - == '')) || (has(p1.__namespace__) && has(p2.__namespace__) && - p1.__namespace__ == p2.__namespace__ )) && (((!has(p1.sectionName) - || p1.sectionName == '') && (!has(p2.sectionName) || p2.sectionName - == '')) || ( has(p1.sectionName) && has(p2.sectionName) && p1.sectionName - == p2.sectionName)) && (((!has(p1.port) || p1.port == 0) && (!has(p2.port) - || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port - == p2.port)))) - rules: - description: Rules are a list of actions. - items: - description: TLSRouteRule is the configuration for a given rule. - properties: - backendRefs: - description: |- - BackendRefs defines the backend(s) where matching requests should be - sent. If unspecified or invalid (refers to a nonexistent resource or - a Service with no endpoints), the rule performs no forwarding; if no - filters are specified that would result in a response being sent, the - underlying implementation must actively reject request attempts to this - backend, by rejecting the connection. Request rejections must respect - weight; if an invalid backend is requested to have 80% of requests, then - 80% of requests must be rejected instead. - - Support: Core for Kubernetes Service - - Support: Extended for Kubernetes ServiceImport - - Support: Implementation-specific for any other resource - - Support for weight: Extended - items: - description: |- - BackendRef defines how a Route should forward a request to a Kubernetes - resource. - - Note that when a namespace different than the local namespace is specified, a - ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - - When the BackendRef points to a Kubernetes Service, implementations SHOULD - honor the appProtocol field if it is set for the target Service Port. - - Implementations supporting appProtocol SHOULD recognize the Kubernetes - Standard Application Protocols defined in KEP-3726. - - If a Service appProtocol isn't specified, an implementation MAY infer the - backend protocol through its own means. Implementations MAY infer the - protocol from the Route type referring to the backend Service. - - If a Route is not able to send traffic to the backend using the specified - protocol then the backend is considered invalid. Implementations MUST set the - "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. - - - Note that when the BackendTLSPolicy object is enabled by the implementation, - there are some extra rules about validity to consider here. See the fields - where this struct is used for more information about the exact behavior. - properties: - group: - default: "" - description: |- - Group is the group of the referent. For example, "gateway.networking.k8s.io". - When unspecified or empty string, core API group is inferred. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Service - description: |- - Kind is the Kubernetes resource kind of the referent. For example - "Service". - - Defaults to "Service" when not specified. - - ExternalName services can refer to CNAME DNS records that may live - outside of the cluster and as such are difficult to reason about in - terms of conformance. They also may not be safe to forward to (see - CVE-2021-25740 for more information). Implementations SHOULD NOT - support ExternalName Services. - - Support: Core (Services with a type other than ExternalName) - - Support: Implementation-specific (Services with type ExternalName) - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the backend. When unspecified, the local - namespace is inferred. - - Note that when a namespace different than the local namespace is specified, - a ReferenceGrant object is required in the referent namespace to allow that - namespace's owner to accept the reference. See the ReferenceGrant - documentation for details. - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port specifies the destination port number to use for this resource. - Port is required when the referent is a Kubernetes Service. In this - case, the port number is the service port number, not the target port. - For other resources, destination port might be derived from the referent - resource or this field. - format: int32 - maximum: 65535 - minimum: 1 - type: integer - weight: - default: 1 - description: |- - Weight specifies the proportion of requests forwarded to the referenced - backend. This is computed as weight/(sum of all weights in this - BackendRefs list). For non-zero values, there may be some epsilon from - the exact proportion defined here depending on the precision an - implementation supports. Weight is not a percentage and the sum of - weights does not need to equal 100. - - If only one backend is specified and it has a weight greater than 0, 100% - of the traffic is forwarded to that backend. If weight is set to 0, no - traffic should be forwarded for this entry. If unspecified, weight - defaults to 1. - - Support for this field varies based on the context where used. - format: int32 - maximum: 1000000 - minimum: 0 - type: integer - required: - - name - type: object - x-kubernetes-validations: - - message: Must have port for Service reference - rule: '(size(self.group) == 0 && self.kind == ''Service'') - ? has(self.port) : true' - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - name: - description: Name is the name of the route rule. This name MUST - be unique within a Route if it is set. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - backendRefs - type: object - maxItems: 1 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string - required: - - hostnames - - rules - type: object - status: - description: Status defines the current state of TLSRoute. - properties: - parents: - description: |- - Parents is a list of parent resources (usually Gateways) that are - associated with the route, and the status of the route with respect to - each parent. When this route attaches to a parent, the controller that - manages the parent must add an entry to this list when the controller - first sees the route and should update the entry as appropriate when the - route or gateway is modified. - - Note that parent references that cannot be resolved by an implementation - of this API will not be added to this list. Implementations of this API - can only populate Route status for the Gateways/parent resources they are - responsible for. - - A maximum of 32 Gateways will be represented in this list. An empty list - means the route has not been attached to any Gateway. - items: - description: |- - RouteParentStatus describes the status of a route with respect to an - associated Parent. - properties: - conditions: - description: |- - Conditions describes the status of the route with respect to the Gateway. - Note that the route's availability is also subject to the Gateway's own - status conditions and listener status. - - If the Route's ParentRef specifies an existing Gateway that supports - Routes of this kind AND that Gateway's controller has sufficient access, - then that Gateway's controller MUST set the "Accepted" condition on the - Route, to indicate whether the route has been accepted or rejected by the - Gateway, and why. - - A Route MUST be considered "Accepted" if at least one of the Route's - rules is implemented by the Gateway. - - There are a number of cases where the "Accepted" condition may not be set - due to lack of controller visibility, that includes when: - - * The Route refers to a nonexistent parent. - * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. - items: - description: Condition contains details for one aspect of - the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, - Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - controllerName: - description: |- - ControllerName is a domain/path string that indicates the name of the - controller that wrote this status. This corresponds with the - controllerName field on GatewayClass. - - Example: "example.net/gateway-controller". - - The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are - valid Kubernetes names - (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). - - Controllers MUST populate this field when writing status. Controllers should ensure that - entries to status populated with their ControllerName are cleaned up when they are no - longer necessary. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ - type: string - parentRef: - description: |- - ParentRef corresponds with a ParentRef in the spec that this - RouteParentStatus struct describes the status of. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - required: - - conditions - - controllerName - - parentRef - type: object - maxItems: 32 - type: array - x-kubernetes-list-type: atomic - required: - - parents - type: object - required: - - spec - type: object - served: true - storage: false + storage: true subresources: status: {} status: @@ -20366,9 +14187,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/3328 + gateway.networking.k8s.io/bundle-version: v1.2.0 gateway.networking.k8s.io/channel: experimental + creationTimestamp: null name: udproutes.gateway.networking.k8s.io spec: group: gateway.networking.k8s.io @@ -20414,7 +14236,7 @@ spec: description: Spec defines the desired state of UDPRoute. properties: parentRefs: - description: |- + description: |+ ParentRefs references the resources (usually Gateways) that a Route wants to be attached to. Note that the referenced parent resource needs to allow this for the attachment to be complete. For Gateways, that means @@ -20476,6 +14298,11 @@ spec: connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. + + + + + items: description: |- ParentReference identifies an API object (usually a Gateway) that can be considered @@ -20625,7 +14452,6 @@ spec: type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: sectionName or port must be specified when parentRefs includes 2 or more references to the same parent @@ -20650,14 +14476,16 @@ spec: || p2.port == 0)) || (has(p1.port) && has(p2.port) && p1.port == p2.port)))) rules: - description: Rules are a list of UDP matchers and actions. + description: |+ + Rules are a list of UDP matchers and actions. + items: description: UDPRouteRule is the configuration for a given rule. properties: backendRefs: description: |- BackendRefs defines the backend(s) where matching requests should be - sent. If unspecified or invalid (refers to a nonexistent resource or a + sent. If unspecified or invalid (refers to a non-existent resource or a Service with no endpoints), the underlying implementation MUST actively reject connection attempts to this backend. Packet drops must respect weight; if an invalid backend is requested to have 80% of @@ -20680,6 +14508,7 @@ spec: namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + When the BackendRef points to a Kubernetes Service, implementations SHOULD honor the appProtocol field if it is set for the target Service Port. @@ -20695,6 +14524,7 @@ spec: protocol then the backend is considered invalid. Implementations MUST set the "ResolvedRefs" condition to "False" with the "UnsupportedProtocol" reason. + Note that when the BackendTLSPolicy object is enabled by the implementation, there are some extra rules about validity to consider here. See the fields @@ -20790,7 +14620,6 @@ spec: maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic name: description: |- Name is the name of the route rule. This name MUST be unique within a Route if it is set. @@ -20800,35 +14629,14 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - required: - - backendRefs type: object maxItems: 16 minItems: 1 type: array - x-kubernetes-list-type: atomic x-kubernetes-validations: - message: Rule name must be unique within the route rule: self.all(l1, !has(l1.name) || self.exists_one(l2, has(l2.name) && l1.name == l2.name)) - useDefaultGateways: - description: |- - UseDefaultGateways indicates the default Gateway scope to use for this - Route. If unset (the default) or set to None, the Route will not be - attached to any default Gateway; if set, it will be attached to any - default Gateway supporting the named scope, subject to the usual rules - about which Routes a Gateway is allowed to claim. - - Think carefully before using this functionality! The set of default - Gateways supporting the requested scope can change over time without - any notice to the Route author, and in many situations it will not be - appropriate to request a default Gateway for a given Route -- for - example, a Route with specific security requirements should almost - certainly not use a default Gateway. - enum: - - All - - None - type: string required: - rules type: object @@ -20874,9 +14682,9 @@ spec: There are a number of cases where the "Accepted" condition may not be set due to lack of controller visibility, that includes when: - * The Route refers to a nonexistent parent. + * The Route refers to a non-existent parent. * The Route is of a type that the controller does not support. - * The Route is in a namespace to which the controller does not have access. + * The Route is in a namespace the controller does not have access to. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -21095,13 +14903,11 @@ spec: - name type: object required: - - conditions - controllerName - parentRef type: object maxItems: 32 type: array - x-kubernetes-list-type: atomic required: - parents type: object @@ -21118,926 +14924,3 @@ status: plural: "" conditions: null storedVersions: null ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 - gateway.networking.k8s.io/channel: experimental - labels: - gateway.networking.k8s.io/policy: Direct - name: xbackendtrafficpolicies.gateway.networking.x-k8s.io -spec: - group: gateway.networking.x-k8s.io - names: - categories: - - gateway-api - kind: XBackendTrafficPolicy - listKind: XBackendTrafficPolicyList - plural: xbackendtrafficpolicies - shortNames: - - xbtrafficpolicy - singular: xbackendtrafficpolicy - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - XBackendTrafficPolicy defines the configuration for how traffic to a - target backend should be handled. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of BackendTrafficPolicy. - properties: - retryConstraint: - description: |- - RetryConstraint defines the configuration for when to allow or prevent - further retries to a target backend, by dynamically calculating a 'retry - budget'. This budget is calculated based on the percentage of incoming - traffic composed of retries over a given time interval. Once the budget - is exceeded, additional retries will be rejected. - - For example, if the retry budget interval is 10 seconds, there have been - 1000 active requests in the past 10 seconds, and the allowed percentage - of requests that can be retried is 20% (the default), then 200 of those - requests may be composed of retries. Active requests will only be - considered for the duration of the interval when calculating the retry - budget. Retrying the same original request multiple times within the - retry budget interval will lead to each retry being counted towards - calculating the budget. - - Configuring a RetryConstraint in BackendTrafficPolicy is compatible with - HTTPRoute Retry settings for each HTTPRouteRule that targets the same - backend. While the HTTPRouteRule Retry stanza can specify whether a - request will be retried, and the number of retry attempts each client - may perform, RetryConstraint helps prevent cascading failures such as - retry storms during periods of consistent failures. - - After the retry budget has been exceeded, additional retries to the - backend MUST return a 503 response to the client. - - Additional configurations for defining a constraint on retries MAY be - defined in the future. - - Support: Extended - properties: - budget: - default: - interval: 10s - percent: 20 - description: Budget holds the details of the retry budget configuration. - properties: - interval: - default: 10s - description: |- - Interval defines the duration in which requests will be considered - for calculating the budget for retries. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - x-kubernetes-validations: - - message: interval cannot be greater than one hour or less - than one second - rule: '!(duration(self) < duration(''1s'') || duration(self) - > duration(''1h''))' - percent: - default: 20 - description: |- - Percent defines the maximum percentage of active requests that may - be made up of retries. - - Support: Extended - maximum: 100 - minimum: 0 - type: integer - type: object - minRetryRate: - default: - count: 10 - interval: 1s - description: |- - MinRetryRate defines the minimum rate of retries that will be allowable - over a specified duration of time. - - The effective overall minimum rate of retries targeting the backend - service may be much higher, as there can be any number of clients which - are applying this setting locally. - - This ensures that requests can still be retried during periods of low - traffic, where the budget for retries may be calculated as a very low - value. - - Support: Extended - properties: - count: - description: |- - Count specifies the number of requests per time interval. - - Support: Extended - maximum: 1000000 - minimum: 1 - type: integer - interval: - description: |- - Interval specifies the divisor of the rate of requests, the amount of - time during which the given count of requests occur. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - x-kubernetes-validations: - - message: interval cannot be greater than one hour - rule: '!(duration(self) == duration(''0s'') || duration(self) - > duration(''1h''))' - type: object - type: object - sessionPersistence: - description: |- - SessionPersistence defines and configures session persistence - for the backend. - - Support: Extended - properties: - absoluteTimeout: - description: |- - AbsoluteTimeout defines the absolute timeout of the persistent - session. Once the AbsoluteTimeout duration has elapsed, the - session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - cookieConfig: - description: |- - CookieConfig provides configuration settings that are specific - to cookie-based session persistence. - - Support: Core - properties: - lifetimeType: - default: Session - description: |- - LifetimeType specifies whether the cookie has a permanent or - session-based lifetime. A permanent cookie persists until its - specified expiry time, defined by the Expires or Max-Age cookie - attributes, while a session cookie is deleted when the current - session ends. - - When set to "Permanent", AbsoluteTimeout indicates the - cookie's lifetime via the Expires or Max-Age cookie attributes - and is required. - - When set to "Session", AbsoluteTimeout indicates the - absolute lifetime of the cookie tracked by the gateway and - is optional. - - Defaults to "Session". - - Support: Core for "Session" type - - Support: Extended for "Permanent" type - enum: - - Permanent - - Session - type: string - type: object - idleTimeout: - description: |- - IdleTimeout defines the idle timeout of the persistent session. - Once the session has been idle for more than the specified - IdleTimeout duration, the session becomes invalid. - - Support: Extended - pattern: ^([0-9]{1,5}(h|m|s|ms)){1,4}$ - type: string - sessionName: - description: |- - SessionName defines the name of the persistent session token - which may be reflected in the cookie or the header. Users - should avoid reusing session names to prevent unintended - consequences, such as rejection or unpredictable behavior. - - Support: Implementation-specific - maxLength: 128 - type: string - type: - default: Cookie - description: |- - Type defines the type of session persistence such as through - the use of a header or cookie. Defaults to cookie based session - persistence. - - Support: Core for "Cookie" type - - Support: Extended for "Header" type - enum: - - Cookie - - Header - type: string - type: object - x-kubernetes-validations: - - message: AbsoluteTimeout must be specified when cookie lifetimeType - is Permanent - rule: '!has(self.cookieConfig) || !has(self.cookieConfig.lifetimeType) - || self.cookieConfig.lifetimeType != ''Permanent'' || has(self.absoluteTimeout)' - - message: cookieConfig can only be set with type Cookie - rule: '!has(self.cookieConfig) || self.type == ''Cookie''' - targetRefs: - description: |- - TargetRefs identifies API object(s) to apply this policy to. - Currently, Backends (A grouping of like endpoints such as Service, - ServiceImport, or any implementation-specific backendRef) are the only - valid API target references. - - Currently, a TargetRef cannot be scoped to a specific port on a - Service. - items: - description: |- - LocalPolicyTargetReference identifies an API object to apply a direct or - inherited policy to. This should be used as part of Policy resources - that can target Gateway API resources. For more information on how this - policy attachment model works, and a sample Policy resource, refer to - the policy attachment documentation for Gateway API. - properties: - group: - description: Group is the group of the target resource. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the target resource. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the target resource. - maxLength: 253 - minLength: 1 - type: string - required: - - group - - kind - - name - type: object - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - group - - kind - - name - x-kubernetes-list-type: map - required: - - targetRefs - type: object - status: - description: Status defines the current state of BackendTrafficPolicy. - properties: - ancestors: - description: |- - Ancestors is a list of ancestor resources (usually Gateways) that are - associated with the policy, and the status of the policy with respect to - each ancestor. When this policy attaches to a parent, the controller that - manages the parent and the ancestors MUST add an entry to this list when - the controller first sees the policy and SHOULD update the entry as - appropriate when the relevant ancestor is modified. - - Note that choosing the relevant ancestor is left to the Policy designers; - an important part of Policy design is designing the right object level at - which to namespace this status. - - Note also that implementations MUST ONLY populate ancestor status for - the Ancestor resources they are responsible for. Implementations MUST - use the ControllerName field to uniquely identify the entries in this list - that they are responsible for. - - Note that to achieve this, the list of PolicyAncestorStatus structs - MUST be treated as a map with a composite key, made up of the AncestorRef - and ControllerName fields combined. - - A maximum of 16 ancestors will be represented in this list. An empty list - means the Policy is not relevant for any ancestors. - - If this slice is full, implementations MUST NOT add further entries. - Instead they MUST consider the policy unimplementable and signal that - on any related resources such as the ancestor that would be referenced - here. For example, if this list was full on BackendTLSPolicy, no - additional Gateways would be able to reference the Service targeted by - the BackendTLSPolicy. - items: - description: |- - PolicyAncestorStatus describes the status of a route with respect to an - associated Ancestor. - - Ancestors refer to objects that are either the Target of a policy or above it - in terms of object hierarchy. For example, if a policy targets a Service, the - Policy's Ancestors are, in order, the Service, the HTTPRoute, the Gateway, and - the GatewayClass. Almost always, in this hierarchy, the Gateway will be the most - useful object to place Policy status on, so we recommend that implementations - SHOULD use Gateway as the PolicyAncestorStatus object unless the designers - have a _very_ good reason otherwise. - - In the context of policy attachment, the Ancestor is used to distinguish which - resource results in a distinct application of this policy. For example, if a policy - targets a Service, it may have a distinct result per attached Gateway. - - Policies targeting the same resource may have different effects depending on the - ancestors of those resources. For example, different Gateways targeting the same - Service may have different capabilities, especially if they have different underlying - implementations. - - For example, in BackendTLSPolicy, the Policy attaches to a Service that is - used as a backend in a HTTPRoute that is itself attached to a Gateway. - In this case, the relevant object for status is the Gateway, and that is the - ancestor object referred to in this status. - - Note that a parent is also an ancestor, so for objects where the parent is the - relevant object for status, this struct SHOULD still be used. - - This struct is intended to be used in a slice that's effectively a map, - with a composite key made up of the AncestorRef and the ControllerName. - properties: - ancestorRef: - description: |- - AncestorRef corresponds with a ParentRef in the spec that this - PolicyAncestorStatus struct describes the status of. - properties: - group: - default: gateway.networking.k8s.io - description: |- - Group is the group of the referent. - When unspecified, "gateway.networking.k8s.io" is inferred. - To set the core API group (such as for a "Service" kind referent), - Group must be explicitly set to "" (empty string). - - Support: Core - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - default: Gateway - description: |- - Kind is kind of the referent. - - There are two kinds of parent resources with "Core" support: - - * Gateway (Gateway conformance profile) - * Service (Mesh conformance profile, ClusterIP Services only) - - Support for other resources is Implementation-Specific. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: |- - Name is the name of the referent. - - Support: Core - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. When unspecified, this refers - to the local namespace of the Route. - - Note that there are specific rules for ParentRefs which cross namespace - boundaries. Cross-namespace references are only valid if they are explicitly - allowed by something in the namespace they are referring to. For example: - Gateway has the AllowedRoutes field, and ReferenceGrant provides a - generic way to enable any other kind of cross-namespace reference. - - - ParentRefs from a Route to a Service in the same namespace are "producer" - routes, which apply default routing rules to inbound connections from - any namespace to the Service. - - ParentRefs from a Route to a Service in a different namespace are - "consumer" routes, and these routing rules are only applied to outbound - connections originating from the same namespace as the Route, for which - the intended destination of the connections are a Service targeted as a - ParentRef of the Route. - - - Support: Core - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - description: |- - Port is the network port this Route targets. It can be interpreted - differently based on the type of parent resource. - - When the parent resource is a Gateway, this targets all listeners - listening on the specified port that also support this kind of Route(and - select this Route). It's not recommended to set `Port` unless the - networking behaviors specified in a Route must apply to a specific port - as opposed to a listener(s) whose port(s) may be changed. When both Port - and SectionName are specified, the name and port of the selected listener - must match both specified values. - - - When the parent resource is a Service, this targets a specific port in the - Service spec. When both Port (experimental) and SectionName are specified, - the name and port of the selected port must match both specified values. - - - Implementations MAY choose to support other parent resources. - Implementations supporting other types of parent resources MUST clearly - document how/if Port is interpreted. - - For the purpose of status, an attachment is considered successful as - long as the parent resource accepts it partially. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment - from the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, - the Route MUST be considered detached from the Gateway. - - Support: Extended - format: int32 - maximum: 65535 - minimum: 1 - type: integer - sectionName: - description: |- - SectionName is the name of a section within the target resource. In the - following resources, SectionName is interpreted as the following: - - * Gateway: Listener name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - * Service: Port name. When both Port (experimental) and SectionName - are specified, the name and port of the selected listener must match - both specified values. - - Implementations MAY choose to support attaching Routes to other resources. - If that is the case, they MUST clearly document how SectionName is - interpreted. - - When unspecified (empty string), this will reference the entire resource. - For the purpose of status, an attachment is considered successful if at - least one section in the parent resource accepts it. For example, Gateway - listeners can restrict which Routes can attach to them by Route kind, - namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from - the referencing Route, the Route MUST be considered successfully - attached. If no Gateway listeners accept attachment from this Route, the - Route MUST be considered detached from the Gateway. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - name - type: object - conditions: - description: Conditions describes the status of the Policy with - respect to the given Ancestor. - items: - description: Condition contains details for one aspect of - the current state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, - Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - minItems: 1 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - controllerName: - description: |- - ControllerName is a domain/path string that indicates the name of the - controller that wrote this status. This corresponds with the - controllerName field on GatewayClass. - - Example: "example.net/gateway-controller". - - The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are - valid Kubernetes names - (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). - - Controllers MUST populate this field when writing status. Controllers should ensure that - entries to status populated with their ControllerName are cleaned up when they are no - longer necessary. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ - type: string - required: - - ancestorRef - - conditions - - controllerName - type: object - maxItems: 16 - type: array - x-kubernetes-list-type: atomic - required: - - ancestors - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/gateway-api/pull/4530 - gateway.networking.k8s.io/bundle-version: v1.5.1 - gateway.networking.k8s.io/channel: experimental - name: xmeshes.gateway.networking.x-k8s.io -spec: - group: gateway.networking.x-k8s.io - names: - categories: - - gateway-api - kind: XMesh - listKind: XMeshList - plural: xmeshes - shortNames: - - mesh - singular: xmesh - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .status.conditions[?(@.type=="Accepted")].status - name: Accepted - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: XMesh defines mesh-wide characteristics of a GAMMA-compliant - service mesh. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of XMesh. - properties: - controllerName: - description: |- - ControllerName is the name of a controller that is managing Gateway API - resources for mesh traffic management. The value of this field MUST be a - domain prefixed path. - - Example: "example.com/awesome-mesh". - - This field is not mutable and cannot be empty. - - Support: Core - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ - type: string - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf - description: - description: Description optionally provides a human-readable description - of a Mesh. - maxLength: 64 - type: string - parametersRef: - description: |- - ParametersRef is an optional reference to a resource that contains - implementation-specific configuration for this Mesh. If no - implementation-specific parameters are needed, this field MUST be - omitted. - - ParametersRef can reference a standard Kubernetes resource, i.e. - ConfigMap, or an implementation-specific custom resource. The resource - can be cluster-scoped or namespace-scoped. - - If the referent cannot be found, refers to an unsupported kind, or when - the data within that resource is malformed, the Mesh MUST be rejected - with the "Accepted" status condition set to "False" and an - "InvalidParameters" reason. - - Support: Implementation-specific - properties: - group: - description: Group is the group of the referent. - maxLength: 253 - pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - kind: - description: Kind is kind of the referent. - maxLength: 63 - minLength: 1 - pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ - type: string - name: - description: Name is the name of the referent. - maxLength: 253 - minLength: 1 - type: string - namespace: - description: |- - Namespace is the namespace of the referent. - This field is required when referring to a Namespace-scoped resource and - MUST be unset when referring to a Cluster-scoped resource. - maxLength: 63 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - group - - kind - - name - type: object - required: - - controllerName - type: object - status: - default: - conditions: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Accepted - description: Status defines the current state of XMesh. - properties: - conditions: - default: - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Accepted - - lastTransitionTime: "1970-01-01T00:00:00Z" - message: Waiting for controller - reason: Pending - status: Unknown - type: Programmed - description: |- - Conditions is the current status from the controller for - this Mesh. - - Controllers should prefer to publish conditions using values - of MeshConditionType for the type of each Condition. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - supportedFeatures: - description: |- - SupportedFeatures is the set of features the Mesh support. - It MUST be sorted in ascending alphabetical order by the Name key. - items: - properties: - name: - description: |- - FeatureName is used to describe distinct features that are covered by - conformance tests. - type: string - required: - - name - type: object - maxItems: 64 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicy -metadata: - annotations: - gateway.networking.k8s.io/bundle-version: v1.5.0-dev - gateway.networking.k8s.io/channel: standard - name: safe-upgrades.gateway.networking.k8s.io -spec: - failurePolicy: Fail - matchConstraints: - resourceRules: - - apiGroups: - - apiextensions.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - '*' - validations: - - expression: object.spec.group != 'gateway.networking.k8s.io' || oldObject == null - || ( has(object.metadata.annotations) && object.metadata.annotations.exists(k, - k == 'gateway.networking.k8s.io/channel') && object.metadata.annotations['gateway.networking.k8s.io/channel'] - == 'standard' ) || ( oldObject != null && has(oldObject.metadata.annotations) - && oldObject.metadata.annotations.exists(k, k == 'gateway.networking.k8s.io/channel') - && oldObject.metadata.annotations['gateway.networking.k8s.io/channel'] == 'experimental' - ) - message: Installing experimental CRDs on top of standard channel CRDs is prohibited - by default. Uninstall ValidatingAdmissionPolicy safe-upgrades.gateway.networking.k8s.io - to install experimental CRDs on top of standard channel CRDs. - reason: Invalid - - expression: object.spec.group != 'gateway.networking.k8s.io' || (has(object.metadata.annotations) - && object.metadata.annotations.exists(k, k == 'gateway.networking.k8s.io/bundle-version') - && !matches(object.metadata.annotations['gateway.networking.k8s.io/bundle-version'], - 'v1.[0-4].\\d+') && !matches(object.metadata.annotations['gateway.networking.k8s.io/bundle-version'], - 'v0')) - message: Installing CRDs with version before v1.5.0 is prohibited by default. - Uninstall ValidatingAdmissionPolicy safe-upgrades.gateway.networking.k8s.io - to install older versions. - reason: Invalid ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - annotations: - gateway.networking.k8s.io/bundle-version: v1.5.0-dev - gateway.networking.k8s.io/channel: standard - name: safe-upgrades.gateway.networking.k8s.io -spec: - matchResources: - resourceRules: - - apiGroups: - - apiextensions.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - customresourcedefinitions - policyName: safe-upgrades.gateway.networking.k8s.io - validationActions: - - Deny diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md new file mode 100644 index 00000000..b6f170c8 --- /dev/null +++ b/packages/system/gpu-operator/examples/README.md @@ -0,0 +1,121 @@ +# GPU operator — native pod workload on Talos (reference) + +The files in this directory are **not** templates. They are reference +artifacts that document one working configuration for running GPU +workloads directly in pods on a Talos-based Cozystack cluster, together +with the DCGM metrics needed by the `gpu/gpu-performance` Grafana +dashboard. + +The out-of-the-box `values-talos.yaml` for this package targets the +sandbox (VFIO passthrough to KubeVirt VMs) scenario. The files here +illustrate an alternative — running CUDA workloads in regular pods with +the NVIDIA device plugin — and the workarounds it currently requires on +Talos. + +## Files + +- [`values-native-talos.yaml`](./values-native-talos.yaml) — Cozystack + `Package` values that disable sandbox workloads, enable the device + plugin, point `hostPaths.driverInstallDir` at the staging location + used by the compat DaemonSet, and wire DCGM to the custom metrics + ConfigMap. +- [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` + with a DCGM metrics CSV that adds profiling, ECC, throttling and + energy counters on top of the upstream defaults. The CSV is a + superset needed for full coverage of the `gpu/gpu-performance` + dashboard. Which parts are actually required depends on which + dashboards you ship — see the table below. +- [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet + that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc + tree into a path where the NVIDIA GPU Operator validator expects + them. See the "Why the compat DaemonSet exists" section below. + +## Why these are reference, not templates + +Shipping these as first-class templates would silently impose +assumptions that do not hold for every user: + +- Whether the NVIDIA Talos system extension is installed on the nodes. +- Whether GPUs are exposed directly to pods or passed through to VMs. +- The exact path the installed driver ends up at (depends on the + extension version and Talos release). + +The sandbox-oriented `values-talos.yaml` remains the default. Operators +who want native pod GPU workloads can start from this directory and +adapt as needed. + +## Why the compat DaemonSet exists + +The NVIDIA GPU Operator validator checks for `libnvidia-ml.so.1` and +`bin/nvidia-smi` in the path given by `hostPaths.driverInstallDir`. +Talos installs them under `/usr/local/glibc/usr/lib/` and +`/usr/local/bin/`, which the validator does not look at. Until upstream +addresses [NVIDIA/gpu-operator#1687][1], the DaemonSet copies those +files into a directory the validator does inspect and creates the +`.driver-ctr-ready` flag file so the validator proceeds. + +[1]: https://github.com/NVIDIA/gpu-operator/issues/1687 + +The compat DaemonSet runs privileged and bind-mounts host paths, so +the target namespace must allow privileged pods. On clusters that +enforce the Kubernetes Pod Security Standards at `baseline` or +`restricted`, label the namespace with +`pod-security.kubernetes.io/enforce: privileged` (and the matching +`audit`/`warn` labels if the admission webhook is configured to +surface violations) before applying the manifest. + +## Dashboards and what DCGM metrics they need + +Five GPU dashboards live under `gpu/*` in +`packages/system/monitoring/dashboards-infra.list`. All of them share +`packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` as +their source of aggregated series. The recording rules are safe to +ship on any cluster — they evaluate to empty series when DCGM is not +scraped, or when optional counters are missing. + +What each dashboard needs on top of the upstream DCGM Exporter +[`default-counters.csv`][default-csv]: + +| Dashboard | Scope | Needs beyond defaults | +| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- | +| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | +| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | +| `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | +| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `kube_node_status_allocatable` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | +| `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | + +`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` +are already in the upstream default set for the pinned DCGM Exporter +version, so the tensor-saturation and engine-active panels work without +any CSV override. The three counters listed in the table — throttling +violations and the power management limit — are the only extras the +tracked dashboards need. The recording rules in +`gpu-recording.rules.yaml` consume utilization, FB used, power, +temperature and the tensor-active profiling counter from the default +set, plus `DCGM_FI_DEV_POWER_VIOLATION` and +`DCGM_FI_DEV_THERMAL_VIOLATION` — used by the +`gpu.recording.efficiency.1m` group to derive the +`gpu:power_throttle_fraction:rate5m` and +`gpu:thermal_throttle_fraction:rate5m` series consumed by the +throttling panels on the efficiency and fleet dashboards. + +The `gpu.recording.throttle.validation.5m` group additionally ships the +`GPUThrottleFractionOverOne` alert (severity `warning`) as a regression +detector: it fires when either throttle-fraction series exceeds 1.0, +which would indicate that DCGM changed the scale/divisor of the +underlying violation counters and the recording rules need to be +re-derived. + +## Verification status + +The minimum-CSV claims above are verified by +`hack/check-gpu-recording-rules.bats`, which cross-checks every +`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules +against the union of the upstream default set (snapshotted at +`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version) +and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM +Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml` +is bumped, refresh the snapshot from the matching tag of the +[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test. + +[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml new file mode 100644 index 00000000..ef2b0b88 --- /dev/null +++ b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml @@ -0,0 +1,88 @@ +# Custom DCGM Exporter metrics CSV. Referenced from +# examples/values-native-talos.yaml via dcgmExporter.config.name. +# +# Extends the upstream default set with profiling counters, ECC, page +# retirement, row remap, energy and throttling violations — everything +# the gpu/gpu-performance dashboard and the GPU recording rules in +# monitoring-agents expect. +apiVersion: v1 +kind: ConfigMap +metadata: + name: dcgm-custom-metrics + namespace: cozy-gpu-operator +data: + dcgm-metrics.csv: | + # Format + # If line starts with a '#' it is considered a comment + # DCGM FIELD, Prometheus metric type, help message + + # Identity + DCGM_FI_DRIVER_VERSION, label, Driver version. + + # Clocks + DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). + DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). + + # Temperature + DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). + DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). + + # Power + DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). + DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Current power management limit (in W). + DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). + + # PCIE + DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. + + # Utilization (the sample period varies depending on the product) + DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). + DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). + DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). + DCGM_FI_DEV_DEC_UTIL, gauge, Decoder utilization (in %). + + # Errors and violations + DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. + + # Memory usage + DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). + DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB). + + # ECC (supported on datacenter-class GPUs such as A10) + DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. + DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. + DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. + DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. + + # Retired pages + DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. + DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. + DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. + + # Row remapping (not applicable to GDDR6 but left for datacenter GPUs) + DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors. + DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors. + DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed. + + # Throttle / violation counters (crucial for SLA and tenant monitoring) + DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (us per docs; ns on DCGM 3.x). + DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (us per docs; ns on DCGM 3.x). + + # NVLink — DCGM silently drops this metric on GPUs without NVLink, so + # enabling it unconditionally is safe and keeps this file reusable + # across GPU families. + DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. + + # DCP (profiling) metrics — supported from Ampere onwards. + DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. + DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. + DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. + DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. + DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. + DCGM_FI_PROF_PCIE_TX_BYTES, counter, The number of bytes of active PCIe tx (transmit) data including both header and payload. + DCGM_FI_PROF_PCIE_RX_BYTES, counter, The number of bytes of active PCIe rx (read) data including both header and payload. diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml new file mode 100644 index 00000000..fc9d5981 --- /dev/null +++ b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml @@ -0,0 +1,109 @@ +# Workaround for https://github.com/NVIDIA/gpu-operator/issues/1687 +# +# On Talos, the NVIDIA system extension installs driver files under +# /usr/local/glibc/usr/lib/ and /usr/local/bin/. The GPU Operator +# validator only looks for libnvidia-ml.so.1 and bin/nvidia-smi under +# the path configured as hostPaths.driverInstallDir. This DaemonSet +# stages the required files into /var/nvidia-driver on each node and +# creates the .driver-ctr-ready flag so the validator proceeds. +# +# Paired with examples/values-native-talos.yaml, which sets +# hostPaths.driverInstallDir to /var/nvidia-driver. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-driver-compat + namespace: cozy-gpu-operator + labels: + app: nvidia-driver-compat +spec: + selector: + matchLabels: + app: nvidia-driver-compat + template: + metadata: + labels: + app: nvidia-driver-compat + # DaemonSet rather than a one-shot Job: staging is idempotent per-node, + # must re-run after every Talos reboot (hostPath contents are wiped on + # reboot when the system extension re-installs), and the pause container + # keeps the pod visible for operator observability (kubectl get pods). + spec: + priorityClassName: system-node-critical + # Restrict to GPU nodes only. Without this the DaemonSet schedules onto + # every node (including control-plane and CPU-only workers) and burns + # resources on hosts where the compat tree is meaningless. + # The label is published by Node Feature Discovery / GPU Operator's NFD + # subchart; if NFD is disabled, replace this with whatever label your + # cluster uses to mark GPU hosts. + nodeSelector: + nvidia.com/gpu.present: "true" + # Tolerate-all is intentionally broad: the nodeSelector above already + # confines scheduling to GPU nodes, so the "Exists" toleration only + # matters when those nodes carry extra taints (dedicated=gpu:NoSchedule, + # nvidia.com/gpu:NoSchedule from the GPU Operator's default policy, + # etc.). This mirrors the stance of the upstream nvidia-driver-daemonset + # and nvidia-device-plugin DaemonSets — blanket tolerations plus a + # narrow nodeSelector. + tolerations: + - operator: Exists + initContainers: + - name: create-driver-tree + image: busybox:1.37 + command: + - sh + - -c + - | + set -e + GLIBC_LIB="/host/usr/local/glibc/usr/lib" + SRC_BIN="/host/usr/local/bin" + DST="/host/var/nvidia-driver" + + mkdir -p "$DST/bin" + + [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ] || { + echo "missing $GLIBC_LIB/libnvidia-ml.so.1" >&2 + exit 1 + } + [ -f "$SRC_BIN/nvidia-smi" ] || { + echo "missing $SRC_BIN/nvidia-smi" >&2 + exit 1 + } + + cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" + echo "Copied libnvidia-ml.so.1" + + cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" + chmod +x "$DST/bin/nvidia-smi" + echo "Copied nvidia-smi" + + mkdir -p /host/run/nvidia/validations + touch /host/run/nvidia/validations/.driver-ctr-ready + echo "Created driver-ctr-ready flag" + echo "Done" + securityContext: + privileged: true + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: host-root + mountPath: /host + containers: + - name: pause + image: registry.k8s.io/pause:3.10 + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + cpu: 50m + memory: 16Mi + volumes: + - name: host-root + hostPath: + path: / diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml new file mode 100644 index 00000000..ef7b2891 --- /dev/null +++ b/packages/system/gpu-operator/examples/values-native-talos.yaml @@ -0,0 +1,52 @@ +# Cozystack Package values for running GPU workloads natively in pods +# on Talos. This is the counterpart to values-talos.yaml, which targets +# the sandbox (VFIO passthrough) scenario. +# +# Prerequisites: +# - NVIDIA Talos system extension installed on GPU nodes. +# - examples/nvidia-driver-compat.yaml deployed to stage driver files +# where the gpu-operator validator looks for them. +# - examples/dcgm-custom-metrics.yaml applied so DCGM Exporter exports +# the full set of metrics used by the dashboard and recording rules. +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.gpu-operator +spec: + variant: default + components: + gpu-operator: + values: + gpu-operator: + # The compat DaemonSet stages libnvidia-ml.so.1 and nvidia-smi + # under /var/nvidia-driver. Point the validator at that path. + hostPaths: + driverInstallDir: "/var/nvidia-driver" + # Disable the sandbox path — workloads run in pods, not VMs. + sandboxWorkloads: + enabled: false + vfioManager: + enabled: false + # The Talos extension provides the driver and runtime hooks, + # so the operator's own toolkit and driver components must be + # switched off to avoid conflicting installations. + toolkit: + enabled: false + devicePlugin: + enabled: true + # Export full set of DCGM metrics using the ConfigMap in + # examples/dcgm-custom-metrics.yaml. + dcgmExporter: + serviceMonitor: + enabled: true + # Matches the 1m evaluation cadence of GPU recording rules with + # 4x oversampling headroom. DCGM_FI_PROF_* counters have + # documented overhead concerns below 1s; 15s is safe. + interval: "15s" + honorLabels: true + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + action: replace + config: + name: dcgm-custom-metrics diff --git a/packages/extra/gateway/Chart.yaml b/packages/system/hami/Chart.yaml similarity index 52% rename from packages/extra/gateway/Chart.yaml rename to packages/system/hami/Chart.yaml index 6e42a379..3fcf4c5d 100644 --- a/packages/extra/gateway/Chart.yaml +++ b/packages/system/hami/Chart.yaml @@ -1,5 +1,3 @@ apiVersion: v2 -name: gateway -description: Cozystack per-tenant Gateway API resource backed by Cilium -type: application +name: cozy-hami version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile new file mode 100644 index 00000000..83663a66 --- /dev/null +++ b/packages/system/hami/Makefile @@ -0,0 +1,43 @@ +export NAME=hami +export NAMESPACE=cozy-$(NAME) + +include ../../../hack/common-envs.mk +include ../../../hack/package.mk + +# When bumping the HAMi version, run `make update` and then review +# the resulting diff in `charts/hami/`. The recipe below reproduces the +# top-level vendoring overrides automatically: +# +# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver +# path requires kubelet DRA support that cozystack does not enable +# and has no upstream fix tracked. See commit 3c5521e. +# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does +# not try to re-pull hami-dra at build time. See commit 2734dc0. +# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream +# values.yaml since the corresponding code paths are gone. PSP is +# removed from Kubernetes 1.25+ and is unused by cozystack. +# +# Template-level patches are NOT reproduced automatically: +# +# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks +# that need to be removed because the dra value is gone (commit +# 2734dc0 stripped them). +# * device-plugin/monitorservice.yaml uses `indent` with leading +# whitespace; it must be rewritten to `nindent` for the labels block +# to render correctly when devicePlugin.service.labels is set +# (commit 3685254). +# +# After `make update`, run `git diff -- charts/hami/templates/` and +# replay these template patches against the new upstream version, then +# verify with `helm unittest`. If upstream restructured the affected +# files, the patches may need to be redesigned rather than reapplied. + +update: + rm -rf charts + helm repo add hami-charts https://project-hami.github.io/HAMi/ + helm repo update hami-charts + helm pull hami-charts/hami --untar --untardir charts + rm -rf charts/hami/charts/hami-dra + yq --inplace '.dependencies = []' charts/hami/Chart.yaml + rm -f charts/hami/Chart.lock + yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md new file mode 100644 index 00000000..68669ac7 --- /dev/null +++ b/packages/system/hami/README.md @@ -0,0 +1,82 @@ +# HAMi — GPU Virtualization Middleware + +[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. + +## Architecture + +HAMi consists of four components: + +- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` +- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic +- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API +- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation + +## Prerequisites + +- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) +- NVIDIA driver >= 440 on host nodes +- nvidia-container-toolkit configured as the default container runtime +- GPU nodes labeled with `gpu=on` + +## Known Limitations + +### glibc < 2.34 requirement for workload containers + +HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. + +**This limitation affects workload containers only**, not the host OS or HAMi's own components. + +| Distribution | glibc | Result | +| --------------- | ----- | -------------------------------------------- | +| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | +| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | +| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | +| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | +| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | + +Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. + +**Upstream tracking issues:** + +- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level +- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above + +### musl libc (Alpine) incompatibility + +HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. + +## Usage + +Enable HAMi in your tenant Kubernetes cluster values: + +```yaml +addons: + gpuOperator: + enabled: true + hami: + enabled: true +``` + +When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. + +### Requesting fractional GPU resources + +```yaml +resources: + limits: + nvidia.com/gpu: 1 + nvidia.com/gpumem: 3000 # 3000 MB of GPU memory + nvidia.com/gpucores: 30 # 30% of GPU compute cores +``` + +## Parameters + +Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. + +| Name | Description | Default | +| --- | --- | --- | +| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | +| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | +| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | +| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | +| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml new file mode 100644 index 00000000..55f32ab6 --- /dev/null +++ b/packages/system/hami/charts/hami/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 2.8.1 +dependencies: [] +description: Heterogeneous AI Computing Virtualization Middleware +keywords: +- vgpu +- gpu +kubeVersion: '>= 1.18.0-0' +maintainers: +- email: archlitchi@gmail.com + name: limengxuan +- email: xiaozhang0210@hotmail.com + name: zhangxiao +name: hami +sources: +- https://github.com/Project-HAMi/HAMi +type: application +version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md new file mode 100644 index 00000000..e8217290 --- /dev/null +++ b/packages/system/hami/charts/hami/README.md @@ -0,0 +1,237 @@ +# HAMi Helm Chart Values Documentation + +This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. + +## Global Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | +| `global.imageTag` | Image tag | `"v2.8.1"` | +| `global.gpuHookPath` | GPU Hook path | `/usr/local` | +| `global.labels` | Global labels | `{}` | +| `global.annotations` | Global annotations | `{}` | +| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | +| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | +| `nameOverride` | Name override | `""` | +| `fullnameOverride` | Full name override | `""` | +| `namespaceOverride` | Namespace override | `""` | + +## Resource Name Configuration + +### NVIDIA GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | +| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | +| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | +| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | +| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | + +### Cambricon MLU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | +| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | +| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | + +### Hygon DCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | +| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | +| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | + +### Metax GPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | +| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | +| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | +| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | + +### Enflame GCU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | +| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | + +### Kunlunxin XPU Resources +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | + +## Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `schedulerName` | Scheduler name | `"hami-scheduler"` | +| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | +| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | +| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | +| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | +| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | +| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | +| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | +| `scheduler.leaderElect` | Whether to enable leader election | `true` | +| `scheduler.replicas` | Number of replicas | `1` | + +### Kube Scheduler Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | +| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | +| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | +| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | +| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | +| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | +| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | +| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | + +### Extender Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | +| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | +| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | +| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | +| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | +| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | + +### Admission Webhook Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | +| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | +| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | +| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | +| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | +| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | +| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | + +### TLS Certificate Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | +| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | +| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | +| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | +| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | +| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | +| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | +| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | +| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | + +### Scheduler Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `scheduler.service.type` | Service type | `NodePort` | +| `scheduler.service.httpPort` | HTTP port | `443` | +| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | +| `scheduler.service.monitorPort` | Monitor port | `31993` | +| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | + +## Device Plugin Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | +| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | +| `devicePlugin.image.tag` | Device plugin image tag | `""` | +| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | +| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | + +### Monitor Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | +| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | +| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | +| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | +| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | +| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | +| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | +| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | + +### Device Plugin Other Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | +| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | +| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | +| `devicePlugin.runtimeClassName` | Runtime class name | `""` | +| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | +| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | +| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | +| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | +| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | +| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | +| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | +| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | + +### Device Plugin Service Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.service.type` | Service type | `NodePort` | +| `devicePlugin.service.httpPort` | HTTP port | `31992` | + +### Device Plugin Deployment Configuration + +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | +| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | +| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | +| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | +| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | + +## Device Configuration + +### AWS Neuron +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | + +### Kunlunxin +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.kunlun.enabled` | Whether to enable | `true` | +| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | + +### Mthreads +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.mthreads.enabled` | Whether to enable | `true` | +| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | + +### NVIDIA +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | +| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | + +### Huawei Ascend +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.ascend.enabled` | Whether to enable | `false` | +| `devices.ascend.image` | Image | `""` | +| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | +| `devices.ascend.extraArgs` | Extra arguments | `[]` | +| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | +| `devices.ascend.tolerations` | Tolerations | `[]` | +| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | + +### Iluvatar +| Parameter | Description | Default Value | +|-----------|-------------|---------------| +| `devices.iluvatar.enabled` | Whether to enable | `false` | +| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt new file mode 100644 index 00000000..15bb1218 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/NOTES.txt @@ -0,0 +1,3 @@ +** Please be patient while the chart is being deployed ** +Resource name: {{ .Values.resourceName }} + diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl new file mode 100644 index 00000000..b68018e4 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_commons.tpl @@ -0,0 +1,49 @@ +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .tag }} + {{- $tag = .tag | toString -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl new file mode 100644 index 00000000..ffcc61da --- /dev/null +++ b/packages/system/hami/charts/hami/templates/_helpers.tpl @@ -0,0 +1,163 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "hami-vgpu.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "hami-vgpu.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "hami-vgpu.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +The app name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler" -}} +{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The app name for DevicePlugin +*/}} +{{- define "hami-vgpu.device-plugin" -}} +{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + The app name for MockDevicePlugin + */}} +{{- define "hami-vgpu.mock-device-plugin" -}} +{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The tls secret name for Scheduler +*/}} +{{- define "hami-vgpu.scheduler.tls" -}} +{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The webhook name +*/}} +{{- define "hami-vgpu.scheduler.webhook" -}} +{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "hami-vgpu.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "hami-vgpu.labels" -}} +helm.sh/chart: {{ include "hami-vgpu.chart" . }} +{{ include "hami-vgpu.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "hami-vgpu.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* + Resolve the tag for kubeScheduler. +*/}} +{{- define "resolvedKubeSchedulerTag" -}} +{{- if .Values.scheduler.kubeScheduler.image.tag }} +{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} +{{- else }} +{{- include "strippedKubeVersion" . | trim -}} +{{- end }} +{{- end }} + +{{/* + Return the stripped Kubernetes version string by removing extra parts after semantic version number. + v1.31.1+k3s1 -> v1.31.1 + v1.30.8-eks-2d5f260 -> v1.30.8 + v1.31.1 -> v1.31.1 +*/}} +{{- define "strippedKubeVersion" -}} +{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} +{{- end -}} + +{{- define "hami.scheduler.kubeScheduler.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} +{{- end -}} + +{{- define "hami.scheduler.extender.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.devicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.mockDevicePlugin.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} +{{- end -}} + +{{- define "hami.devicePlugin.monitor.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} +{{- end -}} + +{{- define "hami.scheduler.patch.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.extender.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.devicePlugin.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} +{{- end -}} + +{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} +{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml new file mode 100644 index 00000000..74631e24 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | +{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml new file mode 100644 index 00000000..a0dc4ff3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml @@ -0,0 +1,55 @@ +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + app.kubernetes.io/component: hami-mock-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - image: {{ include "hami.mockDevicePlugin.image" . }} + imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} + name: hami-mock-dp-cntr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - ./k8s-device-plugin + - -v=5 + - --device-config-file=/device-config.yaml + volumeMounts: + - name: dp + mountPath: /var/lib/kubelet/device-plugins + - name: sys + mountPath: /sys + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + volumes: + - name: dp + hostPath: + path: /var/lib/kubelet/device-plugins + - name: sys + hostPath: + path: /sys + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml new file mode 100644 index 00000000..1f4c24f3 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml @@ -0,0 +1,262 @@ +{{- if .Values.devicePlugin.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + updateStrategy: + {{- with .Values.devicePlugin.updateStrategy }} + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-device-plugin + hami.io/webhook: ignore + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.devicePlugin.podAnnotations }} + {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.devicePlugin.runtimeClassName }} + runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} + {{- end }} + serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} + priorityClassName: system-node-critical + hostPID: true + hostNetwork: true + {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + initContainers: + - name: toolkit-validation + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + securityContext: + privileged: true + runAsUser: 0 + command: ["sh", "-c"] + args: + - | + echo "Waiting for NVIDIA Toolkit to be ready..." + until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do + echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." + sleep 5 + done + echo "NVIDIA Toolkit is ready!" + volumeMounts: + - name: nvidia-validations + mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} + mountPropagation: HostToContainer + readOnly: true + {{- end }} + containers: + - name: device-plugin + image: {{ include "hami.devicePlugin.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} + lifecycle: + postStart: + exec: + command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] + command: + - nvidia-device-plugin + - --config-file=/device-config.yaml + - --mig-strategy={{ .Values.devicePlugin.migStrategy }} + - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} + {{- range .Values.devicePlugin.extraArgs }} + - {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_MIG_MONITOR_DEVICES + value: all + - name: DEVICE_LIST_STRATEGY + value: {{ .Values.devicePlugin.deviceListStrategy }} + - name: HOOK_PATH + value: {{ .Values.global.gpuHookPath }} + {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} + - name: PASS_DEVICE_SPECS + value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: NVIDIA_DRIVER_ROOT + value: {{ .Values.devicePlugin.nvidiaDriverRoot }} + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} + - name: NVIDIA_CDI_HOOK_PATH + value: {{ .Values.devicePlugin.nvidiaHookPath }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} + - name: GDRCOPY_ENABLED + value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} + - name: GDS_ENABLED + value: {{ .Values.devicePlugin.gdsEnabled | quote }} + {{- end }} + {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} + - name: MOFED_ENABLED + value: {{ .Values.devicePlugin.mofedEnabled | quote }} + {{- end }} + {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} + - name: ENABLE_TOPOLOGY_SCORE + value: "true" + {{- end }} + {{- with .Values.devicePlugin.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + resources: + {{- toYaml .Values.devicePlugin.resources | nindent 12 }} + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + - name: lib + mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} + - name: usrbin + mountPath: /usrbin + - name: deviceconfig + mountPath: /config + - name: hosttmp + mountPath: /tmp + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + - name: cdi-root + mountPath: /var/run/cdi + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + # We always mount the driver root at /driver-root in the container. + # This is required for CDI detection to work correctly. + - name: driver-root + mountPath: /driver-root + readOnly: true + {{- end }} + - name: vgpu-monitor + image: {{ include "hami.devicePlugin.monitor.image" . }} + imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} + command: + - "vGPUmonitor" + {{- range .Values.devicePlugin.monitor.extraArgs }} + - {{ . }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_MIG_MONITOR_DEVICES + value: "all" + - name: HOOK_PATH + value: "{{ .Values.global.gpuHookPath }}/vgpu" + - name: HAMI_RESYNC_INTERVAL + value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} + {{- with .Values.devicePlugin.monitor.extraEnvs }} + {{- . | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} + volumeMounts: + - name: ctrs + mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: dockers + mountPath: /run/docker + - name: containerds + mountPath: /run/containerd + - name: sysinfo + mountPath: /sysinfo + - name: hostvar + mountPath: /hostvar + - name: hosttmp + mountPath: /tmp + volumes: + - name: ctrs + hostPath: + path: {{ .Values.devicePlugin.monitor.ctrPath }} + - name: hosttmp + hostPath: + path: /tmp + - name: dockers + hostPath: + path: /run/docker + - name: containerds + hostPath: + path: /run/containerd + - name: device-plugin + hostPath: + path: {{ .Values.devicePlugin.pluginPath }} + - name: lib + hostPath: + path: {{ .Values.devicePlugin.libPath }} + {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} + - name: nvidia-validations + hostPath: + path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} + type: DirectoryOrCreate + {{- end }} + {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} + - name: driver-root + hostPath: + path: {{ .Values.devicePlugin.nvidiaDriverRoot }} + type: Directory + {{- end }} + - name: cdi-root + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + - name: usrbin + hostPath: + path: /usr/bin + - name: sysinfo + hostPath: + path: /sys + - name: hostvar + hostPath: + path: /var + - name: deviceconfig + configMap: + name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.devicePlugin.nvidiaNodeSelector }} + nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.devicePlugin.tolerations }} + tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml new file mode 100644 index 00000000..6ac757f2 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml @@ -0,0 +1,28 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - create + - watch + - list + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - list + - patch +{{- end -}} + diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml new file mode 100644 index 00000000..2f0a14ba --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.device-plugin" . }}-monitor +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml new file mode 100644 index 00000000..88f1b214 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml @@ -0,0 +1,29 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.device-plugin" . }}-monitor + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.devicePlugin.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler + annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: monitorport + port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 + targetPort: 9394 + {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-device-plugin + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml new file mode 100644 index 00000000..6e3c2a44 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.devicePlugin.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-device-plugin" + {{- include "hami-vgpu.labels" . | nindent 4 }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml new file mode 100644 index 00000000..ebcc2c9c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: {{ .Values.devicePlugin.runtimeClassName }} + annotations: + helm.sh/hook: pre-install,pre-upgrade +handler: nvidia +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml new file mode 100644 index 00000000..e6d28721 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml @@ -0,0 +1,31 @@ +{{- if .Values.scheduler.admissionWebhook.enabled -}} +{{- if .Values.scheduler.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-serving-cert + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + dnsNames: + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc + - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + secretName: {{ include "hami-vgpu.scheduler.tls" . }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +spec: + selfSigned: {} +{{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml new file mode 100644 index 00000000..81c4fddf --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml @@ -0,0 +1,34 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods", "configmaps"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "patch", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "update", "list", "patch"] +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml new file mode 100644 index 00000000..a81d425c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml @@ -0,0 +1,47 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-kube + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-volume + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml new file mode 100644 index 00000000..109ecbce --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml @@ -0,0 +1,142 @@ +{{- if .Values.scheduler.kubeScheduler.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.json: | + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + {{- if .Values.scheduler.admissionWebhook.enabled }} + "urlPrefix": "https://127.0.0.1:443", + "enableHttps": true, + "tlsConfig": { + "insecure": true + }, + {{- else }} + "urlPrefix": "http://127.0.0.1:80", + "enableHttps": false, + {{- end }} + "filterVerb": "filter", + "bindVerb": "bind", + "weight": 1, + "nodeCacheCapable": true, + "httpTimeout": 30000000000, + "managedResources": [ + {{- range .Values.devices.amd.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + { + "name": "{{ . }}", + "ignoredByScheduler": true + }, + {{- end }} + {{- end }} + { + "name": "{{ .Values.resourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourceMemPercentage }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.resourcePriority }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.mluResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceMem }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.dcuResourceCores }}", + "ignoredByScheduler": true + }, + { + "name": "metax-tech.com/gpu", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceName }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceCore }}", + "ignoredByScheduler": true + }, + { + "name": "{{ .Values.metaxResourceMem }}", + "ignoredByScheduler": true + } + ], + "ignoreable": false + } + ] + } +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml new file mode 100644 index 00000000..6f6db097 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml @@ -0,0 +1,102 @@ +{{- if .Values.scheduler.kubeScheduler.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-newversion + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + config.yaml: | + {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} + apiVersion: kubescheduler.config.k8s.io/v1 + {{- else }} + apiVersion: kubescheduler.config.k8s.io/v1beta2 + {{- end }} + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: false + profiles: + - schedulerName: {{ .Values.schedulerName }} + extenders: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - urlPrefix: "https://127.0.0.1:443" + enableHTTPS: true + tlsConfig: + insecure: true + {{- else }} + - urlPrefix: "http://127.0.0.1:80" + enableHTTPS: false + {{- end }} + filterVerb: filter + bindVerb: bind + nodeCacheCapable: true + weight: 1 + httpTimeout: 30s + managedResources: + - name: {{ .Values.resourceName }} + ignoredByScheduler: true + - name: {{ .Values.resourceMem }} + ignoredByScheduler: true + - name: {{ .Values.resourceCores }} + ignoredByScheduler: true + - name: {{ .Values.resourceMemPercentage }} + ignoredByScheduler: true + - name: {{ .Values.resourcePriority }} + ignoredByScheduler: true + - name: {{ .Values.mluResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceName }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceMem }} + ignoredByScheduler: true + - name: {{ .Values.dcuResourceCores }} + ignoredByScheduler: true + - name: "metax-tech.com/gpu" + ignoredByScheduler: true + - name: {{ .Values.metaxResourceName }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceCore }} + ignoredByScheduler: true + - name: {{ .Values.metaxResourceMem }} + ignoredByScheduler: true + {{- if .Values.devices.ascend.enabled }} + {{- range .Values.devices.ascend.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.mthreads.enabled }} + {{- range .Values.devices.mthreads.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.enflame.enabled }} + {{- range .Values.devices.enflame.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- if .Values.devices.kunlun.enabled }} + {{- range .Values.devices.kunlun.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.awsneuron.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + {{- range .Values.devices.iluvatar.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} + {{- end }} + {{- range .Values.devices.amd.customresources }} + - name: {{ . }} + ignoredByScheduler: true + {{- end }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml new file mode 100644 index 00000000..1d89e189 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml @@ -0,0 +1,225 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- with .Values.global.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.global.annotations }} + annotations: {{ toYaml .Values.global.annotations | nindent 4}} + {{- end }} +spec: + {{- if .Values.scheduler.leaderElect }} + replicas: {{ .Values.scheduler.replicas }} + {{- else }} + replicas: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} + hami.io/webhook: ignore + annotations: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} + {{- else }} + checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} + {{- end }} + checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.podAnnotations }} + {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "hami-vgpu.scheduler" . }} + priorityClassName: system-node-critical + {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} + containers: + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: kube-scheduler + image: {{ include "hami.scheduler.kubeScheduler.image" . }} + imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} + command: + - kube-scheduler + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} + - {{ . }} + {{- end }} + {{- else }} + - --scheduler-name={{ .Values.schedulerName }} + {{- range .Values.scheduler.kubeScheduler.extraArgs }} + - {{ . }} + {{- end }} + {{- end }} + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + resources: + {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} + volumeMounts: + - name: scheduler-config + mountPath: /config + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + failureThreshold: 8 + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + {{- end }} + - name: vgpu-scheduler-extender + image: {{ include "hami.scheduler.extender.image" . }} + imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} + env: + {{- if .Values.scheduler.nodeLockExpire }} + - name: HAMI_NODELOCK_EXPIRE + value: "{{ .Values.scheduler.nodeLockExpire }}" + {{- end }} + {{- if .Values.global.managedNodeSelectorEnable }} + {{- range $key, $value := .Values.global.managedNodeSelector }} + - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - scheduler + {{- if .Values.scheduler.admissionWebhook.enabled }} + - --http_bind=0.0.0.0:443 + - --cert_file=/tls/tls.crt + - --key_file=/tls/tls.key + {{- else }} + - --http_bind=0.0.0.0:80 + {{- end }} + - --scheduler-name={{ .Values.schedulerName }} + - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} + - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} + - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} + - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} + - --device-config-file=/device-config.yaml + - --leader-elect={{ .Values.scheduler.leaderElect }} + - --leader-elect-resource-name={{ .Values.schedulerName }} + - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} + {{- if .Values.devices.ascend.enabled }} + - --enable-ascend=true + {{- end }} + {{- if .Values.devices.iluvatar.enabled }} + - --enable-iluvatar=true + {{- end }} + {{- if .Values.scheduler.nodeLabelSelector }} + - --node-label-selector={{- $first := true -}} + {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} + {{- if not $first }},{{ end -}} + {{- $key }}={{ $value -}} + {{- $first = false -}} + {{- end -}} + {{- end }} + {{- range .Values.scheduler.extender.extraArgs }} + - {{ . }} + {{- end }} + ports: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: https + containerPort: 443 + protocol: TCP + {{- else }} + - name: http + containerPort: 80 + protocol: TCP + {{- end }} + - name: metrics + containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} + protocol: TCP + resources: + {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} + volumeMounts: + - name: device-config + mountPath: /device-config.yaml + subPath: device-config.yaml + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + mountPath: /tls + {{- end }} + {{- if .Values.scheduler.livenessProbe }} + livenessProbe: + httpGet: + path: /healthz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + readinessProbe: + httpGet: + path: /readyz + {{- if .Values.scheduler.admissionWebhook.enabled }} + port: https + scheme: HTTPS + {{- else }} + port: http + scheme: HTTP + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + timeoutSeconds: 5 + {{- end }} + {{- if .Values.scheduler.leaderElect }} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hami-scheduler + topologyKey: "kubernetes.io/hostname" + {{- end }} + volumes: + {{- if .Values.scheduler.admissionWebhook.enabled }} + - name: tls-config + secret: + secretName: {{ template "hami-vgpu.scheduler.tls" . }} + {{- end }} + {{- if .Values.scheduler.kubeScheduler.enabled }} + - name: scheduler-config + configMap: + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + name: {{ template "hami-vgpu.scheduler" . }}-newversion + {{- else }} + name: {{ template "hami-vgpu.scheduler" . }} + {{- end }} + {{- end }} + - name: device-config + configMap: + name: {{ include "hami-vgpu.scheduler" . }}-device + {{- if .Values.scheduler.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.tolerations }} + tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.nodeName }} + nodeName: {{ .Values.scheduler.nodeName }} + {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml new file mode 100644 index 00000000..873b813b --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml @@ -0,0 +1,408 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "hami-vgpu.scheduler" . }}-device + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} +data: + device-config.yaml: |- + {{- if .Files.Glob "files/device-config.yaml" }} + {{- .Files.Get "files/device-config.yaml" | nindent 4}} + {{- else }} + nvidia: + resourceCountName: {{ .Values.resourceName }} + resourceMemoryName: {{ .Values.resourceMem }} + resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} + resourceCoreName: {{ .Values.resourceCores }} + resourcePriorityName: {{ .Values.resourcePriority }} + overwriteEnv: false + defaultMemory: 0 + defaultCores: 0 + defaultGPUNum: 1 + memoryFactor: 1 + deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} + deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} + deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} + gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} + libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} + runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" + knownMigGeometries: + - models: [ "A30" ] + allowedGeometries: + - + - name: 1g.6gb + core: 25 + memory: 6144 + count: 4 + - + - name: 2g.12gb + core: 50 + memory: 12288 + count: 2 + - + - name: 4g.24gb + core: 100 + memory: 24576 + count: 1 + - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] + allowedGeometries: + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 7 + - + - name: 1g.5gb + core: 14 + memory: 5120 + count: 1 + - name: 2g.10gb + core: 28 + memory: 10240 + count: 3 + - + - name: 3g.20gb + core: 42 + memory: 20480 + count: 2 + - + - name: 7g.40gb + core: 100 + memory: 40960 + count: 1 + - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.79gb + core: 100 + memory: 80896 + count: 1 + - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] + allowedGeometries: + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 7 + - + - name: 1g.10gb + core: 14 + memory: 10240 + count: 1 + - name: 2g.20gb + core: 28 + memory: 20480 + count: 3 + - + - name: 3g.40gb + core: 42 + memory: 40960 + count: 2 + - + - name: 7g.80gb + core: 100 + memory: 81920 + count: 1 + - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.47gb + core: 42 + memory: 48128 + count: 2 + - + - name: 7g.94gb + core: 100 + memory: 96256 + count: 1 + - models: [ "H20", "H100 on GH200"] + allowedGeometries: + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 7 + - + - name: 1g.12gb + core: 14 + memory: 12288 + count: 1 + - name: 2g.24gb + core: 28 + memory: 24576 + count: 3 + - + - name: 3g.48gb + core: 42 + memory: 49152 + count: 2 + - + - name: 7g.96gb + core: 100 + memory: 98304 + count: 1 + - models: [ "H200 NVL", "H200-SXM5"] + allowedGeometries: + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 7 + - + - name: 1g.18gb + core: 14 + memory: 18432 + count: 1 + - name: 2g.35gb + core: 28 + memory: 35840 + count: 3 + - + - name: 3g.71gb + core: 42 + memory: 72704 + count: 2 + - + - name: 7g.141gb + core: 100 + memory: 144384 + count: 1 + - models: [ "B200" ] + allowedGeometries: + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 7 + - + - name: 1g.23gb + core: 14 + memory: 23552 + count: 1 + - name: 2g.45gb + core: 28 + memory: 46080 + count: 3 + - + - name: 3g.90gb + core: 42 + memory: 92160 + count: 2 + - + - name: 7g.180gb + core: 100 + memory: 184320 + count: 1 + cambricon: + resourceCountName: {{ .Values.mluResourceName }} + resourceMemoryName: {{ .Values.mluResourceMem }} + resourceCoreName: {{ .Values.mluResourceCores }} + hygon: + resourceCountName: {{ .Values.dcuResourceName }} + resourceMemoryName: {{ .Values.dcuResourceMem }} + resourceCoreName: {{ .Values.dcuResourceCores }} + memoryFactor: 1 + metax: + resourceCountName: "metax-tech.com/gpu" + resourceVCountName: {{ .Values.metaxResourceName }} + resourceVMemoryName: {{ .Values.metaxResourceMem }} + resourceVCoreName: {{ .Values.metaxResourceCore }} + sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} + enflame: + resourceNameGCU: "enflame.com/gcu" + resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} + resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} + mthreads: + resourceCountName: "mthreads.com/vgpu" + resourceMemoryName: "mthreads.com/sgpu-memory" + resourceCoreName: "mthreads.com/sgpu-core" + iluvatars: + - chipName: MR-V100 + commonWord: MR-V100 + resourceCountName: iluvatar.ai/MR-V100-vgpu + resourceMemoryName: iluvatar.ai/MR-V100.vMem + resourceCoreName: iluvatar.ai/MR-V100.vCore + - chipName: MR-V50 + commonWord: MR-V50 + resourceCountName: iluvatar.ai/MR-V50-vgpu + resourceMemoryName: iluvatar.ai/MR-V50.vMem + resourceCoreName: iluvatar.ai/MR-V50.vCore + - chipName: BI-V150 + commonWord: BI-V150 + resourceCountName: iluvatar.ai/BI-V150-vgpu + resourceMemoryName: iluvatar.ai/BI-V150.vMem + resourceCoreName: iluvatar.ai/BI-V150.vCore + - chipName: BI-V100 + commonWord: BI-V100 + resourceCountName: iluvatar.ai/BI-V100-vgpu + resourceMemoryName: iluvatar.ai/BI-V100.vMem + resourceCoreName: iluvatar.ai/BI-V100.vCore + kunlun: + resourceCountName: {{ .Values.kunlunResourceName }} + resourceVCountName: {{ .Values.kunlunResourceVCountName }} + resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} + awsneuron: + resourceCountName: "aws.amazon.com/neuron" + resourceCoreName: "aws.amazon.com/neuroncore" + amd: + resourceCountName: "amd.com/gpu" + vnpus: + - chipName: 910A + commonWord: Ascend910A + resourceName: huawei.com/Ascend910A + resourceMemoryName: huawei.com/Ascend910A-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 30 + templates: + - name: vir02 + memory: 2184 + aiCore: 2 + - name: vir04 + memory: 4369 + aiCore: 4 + - name: vir08 + memory: 8738 + aiCore: 8 + - name: vir16 + memory: 17476 + aiCore: 16 + - chipName: 910B2 + commonWord: Ascend910B2 + resourceName: huawei.com/Ascend910B2 + resourceMemoryName: huawei.com/Ascend910B2-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 24 + aiCPU: 6 + templates: + - name: vir03_1c_8g + memory: 8192 + aiCore: 3 + aiCPU: 1 + - name: vir06_1c_16g + memory: 16384 + aiCore: 6 + aiCPU: 1 + - name: vir12_3c_32g + memory: 32768 + aiCore: 12 + aiCPU: 3 + - chipName: 910B3 + commonWord: Ascend910B3 + resourceName: huawei.com/Ascend910B3 + resourceMemoryName: huawei.com/Ascend910B3-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_16g + memory: 16384 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_32g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4-1 + commonWord: Ascend910B4-1 + resourceName: huawei.com/Ascend910B4-1 + resourceMemoryName: huawei.com/Ascend910B4-1-memory + memoryAllocatable: 65536 + memoryCapacity: 65536 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. + # The memory is used for scheduling so the correct values must be set. + # Template vir05_1c_8g actually provides 16GB memory, + - name: vir05_1c_8g + memory: 16384 + aiCore: 5 + aiCPU: 1 + # Template vir10_3c_16g actually provides 32GB memory + - name: vir10_3c_16g + memory: 32768 + aiCore: 10 + aiCPU: 3 + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + memoryAllocatable: 32768 + memoryCapacity: 32768 + memoryFactor: 1 + aiCore: 20 + aiCPU: 7 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 + aiCPU: 1 + - name: vir10_3c_16g + memory: 16384 + aiCore: 10 + aiCPU: 3 + - chipName: 310P3 + commonWord: Ascend310P + resourceName: huawei.com/Ascend310P + resourceMemoryName: huawei.com/Ascend310P-memory + memoryAllocatable: 21527 + memoryCapacity: 24576 + memoryFactor: 1 + aiCore: 8 + aiCPU: 7 + templates: + - name: vir01 + memory: 3072 + aiCore: 1 + aiCPU: 1 + - name: vir02 + memory: 6144 + aiCore: 2 + aiCPU: 2 + - name: vir04 + memory: 12288 + aiCore: 4 + aiCPU: 4 + {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml new file mode 100644 index 00000000..77e891cc --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + #- validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml new file mode 100644 index 00000000..2b82f926 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml new file mode 100644 index 00000000..0e36d95f --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml @@ -0,0 +1,68 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-create + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: create + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - create + - --cert-name=tls.crt + - --key-name=tls.key + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} + {{- else }} + - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} + {{- end }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml new file mode 100644 index 00000000..ce52042c --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission-patch + {{- if .Values.scheduler.patch.podAnnotations }} + annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "hami-vgpu.labels" . | nindent 8 }} + app.kubernetes.io/component: admission-webhook + hami.io/webhook: ignore + spec: + {{- if .Values.scheduler.patch.priorityClassName }} + priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} + {{- end }} + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} + {{- else }} + {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} + {{- end }} + containers: + - name: patch + {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} + image: {{ include "hami.scheduler.patch.new.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} + {{- else }} + image: {{ include "hami.scheduler.patch.image" . }} + imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} + {{- end }} + args: + - patch + - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} + - --namespace={{ include "hami-vgpu.namespace" . }} + - --patch-validating=false + - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} + restartPolicy: OnFailure + serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission + {{- if .Values.scheduler.patch.nodeSelector }} + nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.scheduler.patch.tolerations }} + tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.scheduler.patch.runAsUser }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml new file mode 100644 index 00000000..56682f8e --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml new file mode 100644 index 00000000..7239b128 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml new file mode 100644 index 00000000..857c6a8d --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.fullname" . }}-admission + namespace: {{ include "hami-vgpu.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "hami-vgpu.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml new file mode 100644 index 00000000..5f7d7e44 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml new file mode 100644 index 00000000..96e175a1 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "hami-vgpu.scheduler" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "hami-vgpu.mock-device-plugin" . }} +subjects: + - kind: ServiceAccount + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml new file mode 100644 index 00000000..d7538fed --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/service.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.labels" . | nindent 4 }} + {{- if .Values.scheduler.service.labels }} + {{ toYaml .Values.scheduler.service.labels | indent 4 }} + {{- end }} + {{- if .Values.scheduler.service.annotations }} + annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort + ports: + - name: http + port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 + targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} + {{- end }} + protocol: TCP + - name: monitor + port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 + targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} + {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort + nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} + {{- end }} + protocol: TCP + selector: + app.kubernetes.io/component: hami-scheduler + {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml new file mode 100644 index 00000000..0435c003 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + labels: + app.kubernetes.io/component: "hami-scheduler" + {{- include "hami-vgpu.labels" . | nindent 4 }} +--- +{{- if .Values.mockDevicePlugin.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "hami-vgpu.mock-device-plugin" . }} + namespace: {{ include "hami-vgpu.namespace" . }} +{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml new file mode 100644 index 00000000..db9f8029 --- /dev/null +++ b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml @@ -0,0 +1,57 @@ +{{- if .Values.scheduler.admissionWebhook.enabled -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + {{- if .Values.scheduler.certManager.enabled }} + annotations: + cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert + {{- end }} + name: {{ include "hami-vgpu.scheduler.webhook" . }} +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} + url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} + {{- else }} + service: + name: {{ include "hami-vgpu.scheduler" . }} + namespace: {{ include "hami-vgpu.namespace" . }} + path: /webhook + port: {{ .Values.scheduler.service.httpPort }} + {{- end }} + failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} + matchPolicy: Equivalent + name: vgpu.hami.io + namespaceSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} + - key: kubernetes.io/metadata.name + operator: NotIn + values: + {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} + {{- end }} + objectSelector: + matchExpressions: + - key: hami.io/webhook + operator: NotIn + values: + - ignore + reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + scope: '*' + sideEffects: None + timeoutSeconds: 10 +{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml new file mode 100644 index 00000000..01caca0b --- /dev/null +++ b/packages/system/hami/charts/hami/values.yaml @@ -0,0 +1,455 @@ +## @section Global configuration +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker image pull secrets +global: + ## @param global.imageRegistry Global Docker image registry + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## @param global.imagePullSecrets Global Docker image pull secrets + imagePullSecrets: [] + imageTag: "v2.8.1" + gpuHookPath: /usr/local + labels: {} + annotations: {} + managedNodeSelectorEnable: false + managedNodeSelector: + usage: "gpu" + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +#Nvidia GPU Parameters +resourceName: "nvidia.com/gpu" +resourceMem: "nvidia.com/gpumem" +resourceMemPercentage: "nvidia.com/gpumem-percentage" +resourceCores: "nvidia.com/gpucores" +resourcePriority: "nvidia.com/priority" + +#MLU Parameters +mluResourceName: "cambricon.com/vmlu" +mluResourceMem: "cambricon.com/mlu.smlu.vmemory" +mluResourceCores: "cambricon.com/mlu.smlu.vcore" + +#Hygon DCU Parameters +dcuResourceName: "hygon.com/dcunum" +dcuResourceMem: "hygon.com/dcumem" +dcuResourceCores: "hygon.com/dcucores" + +#Metax sGPU Parameters +metaxResourceName: "metax-tech.com/sgpu" +metaxResourceCore: "metax-tech.com/vcore" +metaxResourceMem: "metax-tech.com/vmemory" +metaxsGPUTopologyAware: "false" + +#Enflame VGCU Parameters +enflameResourceNameVGCU: "enflame.com/vgcu" +enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" + +#Kunlun XPU Parameters +kunlunResourceName: "kunlunxin.com/xpu" +kunlunResourceVCountName: "kunlunxin.com/vxpu" +kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" + +schedulerName: "hami-scheduler" + +scheduler: + # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. + # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default + # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. + nodeName: "" + #nodeLabelSelector: + # "gpu": "on" + overwriteEnv: "false" + defaultSchedulerPolicy: + nodeSchedulerPolicy: binpack + gpuSchedulerPolicy: spread + metricsBindAddress: ":9395" + # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it + forceOverwriteDefaultScheduler: true + livenessProbe: false + leaderElect: true + # when leaderElect is true, replicas is available, otherwise replicas is 1. + replicas: 1 + kubeScheduler: + # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. + enabled: true + ## @param image.registry kube scheduler image registry + ## @param image.repository kube scheduler image repository + ## @param image.tag kube scheduler image tag (immutable tags are recommended) + ## @param image.pullPolicy kube scheduler image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "registry.cn-hangzhou.aliyuncs.com" + repository: "google_containers/kube-scheduler" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraNewArgs: + - --config=/config/config.yaml + - -v=4 + extraArgs: + - --policy-config-file=/config/config.json + - -v=4 + extender: + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary, + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + extraArgs: + - --debug + - -v=4 + nodeLockExpire: "5m" + podAnnotations: {} + tolerations: [] + #serviceAccountName: "hami-vgpu-scheduler-sa" + admissionWebhook: + # If set to false, the admission webhook is not installed and any pods that should use HAMi must be + # configured to use the right 'schedulerName' and other device-specific configurations. + enabled: true + customURL: + enabled: false + # must be an endpoint using https. + # should generate host certs here + host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ + port: 31998 + path: /webhook + whitelistNamespaces: + # Specify the namespaces that the webhook will not be applied to. + # - default + # - kube-system + # - istio-system + reinvocationPolicy: Never + failurePolicy: Ignore + ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. + ## If enabled, always takes precedence over options 2. + certManager: + enabled: false + ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. + ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. + patch: + enabled: true + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "jettech/kube-webhook-certgen" + tag: "v1.5.2" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## @param image.registry scheduler extender image registry + ## @param image.repository scheduler extender image repository + ## @param image.tag scheduler extender image tag (immutable tags are recommended) + ## @param image.pullPolicy scheduler extender image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + imageNew: + registry: "docker.io" + repository: "liangjw/kube-webhook-certgen" + tag: "v1.1.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + tolerations: [] + runAsUser: 2000 + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 443 # HTTP port + schedulerPort: 31998 # NodePort for HTTP + monitorPort: 31993 # Monitoring port + monitorTargetPort: 9395 + httpTargetPort: 443 + labels: {} + annotations: {} + +devicePlugin: + enabled: true + gpuOperatorToolkitReady: + enabled: false + hostPath: "/run/nvidia/validations" + ## @param image.registry devicePlugin image registry + ## @param image.repository devicePlugin image repository + ## @param image.tag devicePlugin image tag (immutable tags are recommended) + ## @param image.pullPolicy devicePlugin image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + monitor: + ## @param image.registry monitor image registry + ## @param image.repository monitor image repository + ## @param image.tag monitor image tag (immutable tags are recommended) + ## @param image.pullPolicy monitor image pull policy + ## @param image.pullSecrets Specify docker-registry secret names as an array + image: + registry: "docker.io" + repository: "projecthami/hami" + tag: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ctrPath: /usr/local/vgpu/containers + resyncInterval: "5m" + extraArgs: + - -v=4 + extraEnvs: {} + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + + deviceSplitCount: 10 + deviceMemoryScaling: 1 + deviceCoreScaling: 1 + # Node configuration for device plugin, Priority: externalConfigName > config > default config + nodeConfiguration: + # If you want to use a custom config.json, you can set the content here. + # If this is set, it will override the default config.json(An example is as follows). + config: | + { + "nodeconfig": [ + { + "name": "your-node-name", + "operatingmode": "hami-core", + "devicememoryscaling": 1, + "devicesplitcount": 10, + "migstrategy": "none", + "filterdevices": { + "uuid": [], + "index": [] + } + } + ] + } + # If you want to use an existing ConfigMap, you can set the name here. + # If this is set, the chart will not create the ConfigMap and will use the existing one. + externalConfigName: "" + # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs + runtimeClassName: "" + # Whether to create runtime class, name comes from runtimeClassName when it is set + createRuntimeClass: false + migStrategy: "none" + disablecorelimit: "false" + passDeviceSpecsEnabled: false + deviceListStrategy: "envvar" + nvidiaHookPath: null + nvidiaDriverRoot: null + gdrcopyEnabled: null + gdsEnabled: null + mofedEnabled: null + + extraArgs: + - -v=4 + extraEnvs: {} + service: + type: NodePort # Default type is NodePort, can be changed to ClusterIP + httpPort: 31992 + labels: {} + annotations: {} + + pluginPath: /var/lib/kubelet/device-plugins + libPath: /usr/local/vgpu + + podAnnotations: {} + nvidiaNodeSelector: + gpu: "on" + tolerations: [] + # The updateStrategy for DevicePlugin DaemonSet. + # If you want to update the DaemonSet by manual, set type as "OnDelete". + # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. + # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + + resources: {} + # If you do want to specify resources, uncomment the following lines, adjust them as necessary. + # and remove the curly braces after 'resources:'. +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 100m +# memory: 100Mi + +mockDevicePlugin: + enabled: false + image: + registry: "docker.io" + repository: "projecthami/mock-device-plugin" + tag: "1.0.1" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] +devices: + amd: + customresources: + - amd.com/gpu + - amd.com/gpu-memory + awsneuron: + customresources: + - aws.amazon.com/neuron + - aws.amazon.com/neuroncore + kunlun: + enabled: true + customresources: + - kunlunxin.com/xpu + - kunlunxin.com/vxpu + - kunlunxin.com/vxpu-memory + enflame: + enabled: true + customresources: + - enflame.com/vgcu + - enflame.com/vgcu-percentage + - enflame.com/gcu + mthreads: + enabled: true + customresources: + - mthreads.com/vgpu + nvidia: + gpuCorePolicy: default + libCudaLogLevel: 1 + ascend: + enabled: false + image: "" + imagePullPolicy: IfNotPresent + extraArgs: [] + nodeSelector: + ascend: "on" + tolerations: [] + customresources: + - huawei.com/Ascend910A + - huawei.com/Ascend910A-memory + - huawei.com/Ascend910B2 + - huawei.com/Ascend910B2-memory + - huawei.com/Ascend910B3 + - huawei.com/Ascend910B3-memory + - huawei.com/Ascend910B4 + - huawei.com/Ascend910B4-memory + - huawei.com/Ascend910B4-1 + - huawei.com/Ascend910B4-1-memory + - huawei.com/Ascend310P + - huawei.com/Ascend310P-memory + iluvatar: + enabled: false + customresources: + - iluvatar.ai/BI-V100-vgpu + - iluvatar.ai/BI-V100.vCore + - iluvatar.ai/BI-V100.vMem + - iluvatar.ai/BI-V150-vgpu + - iluvatar.ai/BI-V150.vCore + - iluvatar.ai/BI-V150.vMem + - iluvatar.ai/MR-V100-vgpu + - iluvatar.ai/MR-V100.vCore + - iluvatar.ai/MR-V100.vMem + - iluvatar.ai/MR-V50-vgpu + - iluvatar.ai/MR-V50.vCore + - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml new file mode 100644 index 00000000..76e8d631 --- /dev/null +++ b/packages/system/hami/values.yaml @@ -0,0 +1,17 @@ +hami: + scheduler: + kubeScheduler: + image: + registry: registry.k8s.io + repository: kube-scheduler + devicePlugin: + runtimeClassName: nvidia + updateStrategy: + type: RollingUpdate + # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml + # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} + nodeConfiguration: + config: | + { + "nodeconfig": [] + } diff --git a/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml b/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml index 743f255d..192470dd 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 0.0.5 +appVersion: 0.0.6 description: A Helm chart for robotlb (loadbalancer on hetzner cloud). name: robotlb type: application diff --git a/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml b/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml index 4fd71366..41b4661a 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "robotlb.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicas }} + replicas: 1 selector: matchLabels: {{- include "robotlb.selectorLabels" . | nindent 6 }} diff --git a/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml b/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml index 3a7b9334..76bac249 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml @@ -3,7 +3,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "robotlb.fullname" . }}-cr -rules: {{- toYaml .Values.serviceAccount.permissions | nindent 2 }} +rules: + {{- toYaml .Values.serviceAccount.permissions | nindent 2 }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/hetzner-robotlb/charts/robotlb/values.yaml b/packages/system/hetzner-robotlb/charts/robotlb/values.yaml index 739c7b73..4365f677 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/values.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/values.yaml @@ -36,6 +36,9 @@ serviceAccount: - apiGroups: [""] resources: [nodes, pods] verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] podAnnotations: {} podLabels: {} diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 4de74113..6ba649f0 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -8,9 +8,32 @@ kamaji: resources: limits: cpu: 200m - memory: 500Mi + memory: 512Mi requests: cpu: 100m - memory: 100Mi + memory: 256Mi + startupProbe: + httpGet: + path: /healthz + port: healthcheck + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /healthz + port: healthcheck + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /readyz + port: healthcheck + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 extraArgs: - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.3.0@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d diff --git a/packages/system/keycloak/templates/httproute.yaml b/packages/system/keycloak/templates/httproute.yaml deleted file mode 100644 index 594260f0..00000000 --- a/packages/system/keycloak/templates/httproute.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} -{{- $ingressHost := .Values.ingress.host | default (printf "keycloak.%s" $host) }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if eq $gatewayEnabled "true" }} -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: keycloak -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $exposeIngress }} - sectionName: https - hostnames: - - {{ $ingressHost | quote }} - rules: - - backendRefs: - - name: keycloak-http - port: 80 -{{- end }} diff --git a/packages/system/keycloak/templates/ingress.yaml b/packages/system/keycloak/templates/ingress.yaml index 07000f51..7f9fd476 100644 --- a/packages/system/keycloak/templates/ingress.yaml +++ b/packages/system/keycloak/templates/ingress.yaml @@ -3,8 +3,6 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} {{- $clusterIssuer := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if ne $gatewayEnabled "true" }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -35,4 +33,3 @@ spec: name: keycloak-http port: name: http -{{- end }} diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 5e9e8f94..f81eaff0 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -2,13 +2,27 @@ apiVersion: cozystack.io/v1alpha1 kind: ApplicationDefinition metadata: name: kubernetes + annotations: + # Kubernetes tenants boot a Kamaji control plane whose admin-kubeconfig + # Secret is provisioned asynchronously. Cold Kamaji start (image pull + + # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed + # Flux helm-controller's default wait budget, causing remediation loops + # that uninstall the Cluster CR. This override is applied by cozystack-api + # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout. + # + # Coupling: the wait-for-kubeconfig init container in + # packages/apps/kubernetes/templates/_helpers.tpl hard-codes a 10m + # deadline chosen to stay strictly below this value so the pod's + # CrashLoopBackOff surfaces before flux remediation fires. If this + # annotation is raised, update that init deadline correspondingly. + release.cozystack.io/helm-install-timeout: "15m" spec: application: kind: Kubernetes singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: @@ -26,7 +40,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] secrets: exclude: [] include: diff --git a/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-ingress.yaml b/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-ingress.yaml index dcfc44bb..ee89953f 100644 --- a/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-ingress.yaml +++ b/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-ingress.yaml @@ -1,10 +1,9 @@ {{- $host := index .Values._cluster "root-host" }} {{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "cdi-uploadproxy" $exposeServices) (ne $gatewayEnabled "true") }} +{{- if and (has "cdi-uploadproxy" $exposeServices) }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-tlsroute.yaml b/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-tlsroute.yaml deleted file mode 100644 index 0f76ac0f..00000000 --- a/packages/system/kubevirt-cdi/templates/cdi-uploadproxy-tlsroute.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} -{{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "cdi-uploadproxy" $exposeServices) (eq $gatewayEnabled "true") }} -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: TLSRoute -metadata: - name: cdi-uploadproxy -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $exposeIngress }} - sectionName: tls-cdi-uploadproxy - hostnames: - - {{ printf "cdi-uploadproxy.%s" $host | quote }} - rules: - - backendRefs: - - name: cdi-uploadproxy - port: 443 -{{- end }} diff --git a/packages/system/kubevirt/templates/vm-exportproxy-ingress.yaml b/packages/system/kubevirt/templates/vm-exportproxy-ingress.yaml index fc19e16f..a089f5ef 100644 --- a/packages/system/kubevirt/templates/vm-exportproxy-ingress.yaml +++ b/packages/system/kubevirt/templates/vm-exportproxy-ingress.yaml @@ -1,9 +1,8 @@ {{- $host := index .Values._cluster "root-host" }} {{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "vm-exportproxy" $exposeServices) (ne $gatewayEnabled "true") }} +{{- if and (has "vm-exportproxy" $exposeServices) }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/packages/system/kubevirt/templates/vm-exportproxy-tlsroute.yaml b/packages/system/kubevirt/templates/vm-exportproxy-tlsroute.yaml deleted file mode 100644 index d2bcbfb7..00000000 --- a/packages/system/kubevirt/templates/vm-exportproxy-tlsroute.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} -{{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $gatewayEnabled := (index .Values._cluster "gateway-enabled") | default "false" }} -{{- if and (has "vm-exportproxy" $exposeServices) (eq $gatewayEnabled "true") }} -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: TLSRoute -metadata: - name: vm-exportproxy -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: cozystack - namespace: {{ $exposeIngress }} - sectionName: tls-vm-exportproxy - hostnames: - - {{ printf "vm-exportproxy.%s" $host | quote }} - rules: - - backendRefs: - - name: vm-exportproxy - port: 443 -{{- end }} diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index d5bada31..cdc1cf01 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,8 +4,13 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk +.PHONY: test + image: image-lineage-controller-webhook +test: + helm unittest . + image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md new file mode 100644 index 00000000..1e43c9c8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/README.md @@ -0,0 +1,67 @@ +# lineage-controller-webhook + +Cozystack system package for the **lineage controller webhook** — a mutating +admission webhook that stamps "lineage" labels onto tenant workloads, linking +each resource back to the Cozystack `Application` that ultimately owns it. + +The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, +`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and +`workloadmonitors.cozystack.io` outside system namespaces. For each request it +walks the ownership graph upward (Kubernetes `ownerReferences`, then the +`HelmRelease` Flux installed the resource with, then the Cozystack +`Application`-derived CRD that produced the HelmRelease) and writes the +discovered application's group, kind and name as labels +(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. +Those labels let the aggregated API server, the Cozystack dashboard, and other +lineage-aware consumers reason about which application a resource belongs to. + +The webhook serves TLS on port 9443 with a cert-manager issued certificate, and +runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above +is gated on the webhook being reachable. + +## Topology + +The chart ships a single shape, modelled on `cozystack-api`: + +- **Deployment** with two replicas (override via `replicas`). +- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` + (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). + Soft means: the pod lands on a control-plane node when one is reachable, and + on any worker otherwise — no override needed for managed Kubernetes, + Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't + visible. +- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on + tainted control-plane nodes when the soft affinity is satisfiable. +- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread + across nodes when possible (best-effort). +- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At + `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` + it caps disruption to one pod. +- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver + prefers a webhook endpoint on its own node when one exists, and transparently + falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on + older clusters the field is silently ignored and traffic uses default + cluster-wide distribution. + +## Parameters + +All keys live under the top-level `lineageControllerWebhook:` map. + +| Name | Description | Value | +| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | +| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | +| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | +| `replicas` | Deployment replica count | `2` | +| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | + +### `localK8sAPIEndpoint.enabled` (deprecated) + +When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and +`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on +its own node. It was originally added to avoid latency on the +webhook-to-apiserver path, but it is only valid when the pod is actually +scheduled on an apiserver-bearing node — which the chart's soft control-plane +affinity no longer guarantees. With this flag enabled and the pod scheduled +off a control-plane node, the controller will crash-loop dialing a non- +apiserver hostIP. Slated for removal once the latency motivation is addressed +in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml new file mode 100644 index 00000000..0786c8ff --- /dev/null +++ b/packages/system/lineage-controller-webhook/templates/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: lineage-controller-webhook +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index c5df90fb..fbb3ad54 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - internalTrafficPolicy: Local + trafficDistribution: PreferClose type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/workload.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/daemonset.yaml rename to packages/system/lineage-controller-webhook/templates/workload.yaml index 860aee6e..097919c6 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/workload.yaml @@ -1,10 +1,11 @@ apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: + replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -13,13 +14,25 @@ spec: labels: app: lineage-controller-webhook spec: - {{- with .Values.lineageControllerWebhook.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} + serviceAccountName: lineage-controller-webhook tolerations: - operator: Exists - serviceAccountName: lineage-controller-webhook + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml new file mode 100644 index 00000000..3cdbddd8 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/service_test.yaml @@ -0,0 +1,30 @@ +suite: lineage-controller-webhook service +templates: + - templates/service.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.trafficDistribution + value: PreferClose + - notExists: + path: spec.internalTrafficPolicy + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: 9443 + - equal: + path: spec.selector.app + value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml new file mode 100644 index 00000000..bcb24014 --- /dev/null +++ b/packages/system/lineage-controller-webhook/tests/workload_test.yaml @@ -0,0 +1,99 @@ +suite: lineage-controller-webhook workload + PDB +templates: + - templates/workload.yaml + - templates/pdb.yaml + +release: + name: lineage-controller-webhook + namespace: cozy-system + +tests: + # ---- Default Deployment shape ---------------------------------------------- + + - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations + template: templates/workload.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.tolerations + value: + - operator: Exists + - equal: + path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution + value: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - notExists: + path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution + - equal: + path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey + value: kubernetes.io/hostname + + - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) + template: templates/workload.yaml + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + # ---- PDB -------------------------------------------------------------------- + + - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + + - it: PDB still renders at replicas=1 (no-op, but consistent shape) + set: + lineageControllerWebhook.replicas: 1 + template: templates/pdb.yaml + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.maxUnavailable + value: 1 + + # ---- Replica override ------------------------------------------------------- + + - it: replicas value drives spec.replicas + set: + lineageControllerWebhook.replicas: 5 + template: templates/workload.yaml + asserts: + - equal: + path: spec.replicas + value: 5 + + # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- + + - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars + set: + lineageControllerWebhook.localK8sAPIEndpoint.enabled: true + template: templates/workload.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - contains: + path: spec.template.spec.containers[0].env + content: + name: KUBERNETES_SERVICE_PORT + value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index c5671548..e56d332d 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,10 +1,13 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 debug: false + replicas: 2 + # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and + # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its + # own node — only valid when the pod is actually scheduled on an apiserver- + # bearing node. Now that the chart's nodeAffinity to control-plane is soft + # (preferred, not required), the pod can land off-control-plane and crash- + # loop dialing a non-apiserver hostIP. Slated for removal once webhook + # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - enabled: true - # nodeSelector for the DaemonSet - # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" - # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" - nodeSelector: - node-role.kubernetes.io/control-plane: "" + enabled: false diff --git a/packages/system/linstor-gui/Makefile b/packages/system/linstor-gui/Makefile index e202f4cc..b408491d 100644 --- a/packages/system/linstor-gui/Makefile +++ b/packages/system/linstor-gui/Makefile @@ -12,18 +12,13 @@ image: image-linstor-gui image-linstor-gui: docker buildx build images/linstor-gui \ - --provenance false \ - --builder=$(BUILDER) \ - --platform=linux/amd64,linux/arm64 \ --build-arg LINSTOR_GUI_VERSION=$(LINSTOR_GUI_VERSION) \ --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)) \ --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)-$(TAG)) \ --cache-from type=registry,ref=$(REGISTRY)/linstor-gui:latest \ --cache-to type=inline \ --metadata-file images/linstor-gui.json \ - --push=$(PUSH) \ - --label "org.opencontainers.image.source=https://github.com/cozystack/cozystack" \ - --load=$(LOAD) + $(BUILDX_ARGS) REPOSITORY="$(REGISTRY)/linstor-gui" \ yq -i '.image.repository = strenv(REPOSITORY)' values.yaml TAG="$(call settag,$(LINSTOR_GUI_VERSION))@$$(yq e '."containerimage.digest"' images/linstor-gui.json -o json -r)" \ diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 2ad4def9..8d566678 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -5,7 +5,7 @@ include ../../../hack/common-envs.mk include ../../../hack/package.mk LINSTOR_VERSION ?= 1.33.2 -LINSTOR_CSI_VERSION ?= v1.10.5 +LINSTOR_CSI_VERSION ?= v1.10.6 image: image-piraeus-server image-linstor-csi diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile index 6cfd44aa..df5d179d 100644 --- a/packages/system/linstor/images/linstor-csi/Dockerfile +++ b/packages/system/linstor/images/linstor-csi/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.25 AS builder -ARG VERSION=v1.10.5 +ARG VERSION=v1.10.6 ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1 ARG TARGETARCH ARG TARGETOS diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff index ff0aec0c..0e272670 100644 --- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff +++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff @@ -1,25 +1,7 @@ diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index f544493..98e7fde 100644 +index ac4651d..46d565d 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go -@@ -181,7 +181,7 @@ func LogLevel(s string) func(*Linstor) error { - func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, error) { - var vols []volume.VolumeStatus - -- resourcesByName := make(map[string][]lapi.Resource) -+ resourcesByName := make(map[string][]lapi.ResourceWithVolumes) - - resDefs, err := s.client.ResourceDefinitions.GetAll(ctx, lapi.RDGetAllRequest{WithVolumeDefinitions: true}) - if err != nil { -@@ -194,7 +194,7 @@ func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, - } - - for i := range allResources { -- resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i].Resource) -+ resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i]) - } - - for _, rd := range resDefs { @@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu return err } @@ -35,7 +17,7 @@ index f544493..98e7fde 100644 logger.Debug("reconcile extra properties") err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1280,6 +1288,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v +@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v return err } @@ -50,7 +32,7 @@ index f544493..98e7fde 100644 logger.Debug("reconcile extra properties") err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1470,9 +1486,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) } @@ -62,7 +44,7 @@ index f544493..98e7fde 100644 for _, snapNode := range snap.Nodes { if err := s.NodeAvailable(ctx, snapNode); err != nil { logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") -@@ -1480,13 +1495,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu } if slices.Contains(preferredNodes, snapNode) { @@ -92,7 +74,7 @@ index f544493..98e7fde 100644 } if selectedNode == "" { -@@ -1679,6 +1704,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In +@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In return nil } @@ -207,40 +189,8 @@ index f544493..98e7fde 100644 // FindSnapsByID searches the snapshot in the backend func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { snapshotId, err := volume.ParseSnapshotId(id) -@@ -2173,7 +2306,7 @@ func (s *Linstor) Status(ctx context.Context, volId string) ([]string, *csi.Volu - "volume": volId, - }).Debug("getting assignments") - -- ress, err := s.client.Resources.GetAll(ctx, volId) -+ ress, err := s.client.Resources.GetResourceView(ctx, &lapi.ListOpts{Resource: []string{volId}}) - if err != nil { - return nil, nil, fmt.Errorf("failed to list resources for '%s': %w", volId, err) - } -@@ -2261,13 +2394,20 @@ func GetSnapshotRemoteAndReadiness(snap *lapi.Snapshot) (string, bool, error) { - return "", slices.Contains(snap.Flags, lapiconsts.FlagSuccessful), nil - } - --func NodesAndConditionFromResources(ress []lapi.Resource) ([]string, *csi.VolumeCondition) { -+func NodesAndConditionFromResources(ress []lapi.ResourceWithVolumes) ([]string, *csi.VolumeCondition) { - var allNodes, abnormalNodes []string - - for i := range ress { - res := &ress[i] - -- allNodes = append(allNodes, res.NodeName) -+ // A resource is a CSI publish target if any of its volumes were created -+ // by ControllerPublishVolume, identified by the temporary-diskless-attach property. -+ if slices.ContainsFunc(res.Volumes, func(v lapi.Volume) bool { -+ createdFor, ok := v.Props[linstor.PropertyCreatedFor] -+ return ok && createdFor == linstor.CreatedForTemporaryDisklessAttach -+ }) { -+ allNodes = append(allNodes, res.NodeName) -+ } - - if res.State == nil { - abnormalNodes = append(abnormalNodes, res.NodeName) diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go -index 39acd95..aed18ab 100644 +index 39acd95..54d1dfb 100644 --- a/pkg/volume/parameter.go +++ b/pkg/volume/parameter.go @@ -50,6 +50,7 @@ const ( diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff new file mode 100644 index 00000000..279c636b --- /dev/null +++ b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff @@ -0,0 +1,132 @@ +diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go +index ac4651d..9d75c79 100644 +--- a/pkg/client/linstor.go ++++ b/pkg/client/linstor.go +@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool) + } + + if otherResInUse > 0 && rwxBlock { +- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{ ++ rdProps := map[string]string{ + linstor.PropertyAllowTwoPrimaries: "yes", +- }} ++ } + +- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) ++ // DRBD requires Protocol C whenever allow-two-primaries is enabled. ++ // If the resource is configured with Protocol A or B (typically ++ // through its resource-group), drbdadm adjust on the satellites ++ // fails with "Protocol C required" (errno 139) and live migration ++ // cannot proceed. Override Protocol to C on the resource-definition ++ // itself so it applies to every connection (including diskless ++ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly ++ // the override we installed without disturbing operator-set props. ++ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId) ++ if protoErr != nil { ++ s.log.WithError(protoErr).WithField("volume", volId). ++ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach") ++ } else if proto == "A" || proto == "B" { ++ s.log.WithFields(logrus.Fields{ ++ "volume": volId, ++ "protocol": proto, ++ }).Info("installing Protocol=C override on resource-definition for dual-attach") ++ ++ rdProps[linstor.PropertyDrbdNetProtocol] = "C" ++ rdProps[linstor.PropertyCsiProtocolOverride] = "yes" ++ } ++ ++ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ ++ OverrideProps: rdProps, ++ }) + if err != nil { + return "", err + } +@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { + } + + if resInUse == 1 { +- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{ +- linstor.PropertyAllowTwoPrimaries, +- }} ++ deleteProps := []string{linstor.PropertyAllowTwoPrimaries} ++ ++ // Drop the Protocol=C override only if Attach installed it (gated by ++ // our marker), so an operator-set Protocol property on the ++ // resource-definition is preserved. ++ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId) ++ if getErr != nil { ++ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal") ++ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" { ++ log.Info("removing Protocol=C override from resource-definition") + +- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) ++ deleteProps = append(deleteProps, ++ linstor.PropertyDrbdNetProtocol, ++ linstor.PropertyCsiProtocolOverride, ++ ) ++ } ++ ++ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ ++ DeleteProps: deleteProps, ++ }) + if err != nil { + return nil404(err) + } +@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { + return nil404(s.client.Resources.Delete(ctx, volId, node)) + } + ++// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C") ++// that LINSTOR will hand to drbdadm for the given resource. Resource-definition ++// properties take precedence; otherwise the value is inherited from the ++// resource-group. An empty string is returned when neither level sets the ++// property (LINSTOR's compiled-in default is "C"). ++func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) { ++ rd, err := s.client.ResourceDefinitions.Get(ctx, volId) ++ if err != nil { ++ return "", err ++ } ++ ++ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok { ++ return v, nil ++ } ++ ++ if rd.ResourceGroupName == "" { ++ return "", nil ++ } ++ ++ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName) ++ if err != nil { ++ return "", err ++ } ++ ++ return rg.Props[linstor.PropertyDrbdNetProtocol], nil ++} ++ + // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology. + func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) { + log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments) +diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go +index 9a5f79c..c8bc9c3 100644 +--- a/pkg/linstor/const.go ++++ b/pkg/linstor/const.go +@@ -44,6 +44,19 @@ const ( + // PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration. + PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries" + ++ // PropertyDrbdNetProtocol is the DRBD network replication protocol option ++ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever ++ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with ++ // "Protocol C required" (errno 139). ++ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol" ++ ++ // PropertyCsiProtocolOverride marks a resource-connection where this driver ++ // has installed a temporary Protocol=C override to make a Protocol-A/B ++ // resource compatible with allow-two-primaries during live migration. The ++ // marker lets us safely remove only the overrides we set, without touching ++ // connection-level overrides installed by the operator. ++ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override" ++ + // CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer + // needed. + CreatedForTemporaryDisklessAttach = "temporary-diskless-attach" diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml new file mode 100644 index 00000000..5e0f9ea0 --- /dev/null +++ b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml @@ -0,0 +1,172 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: alerts-gpu-recording.rules +spec: + groups: + - name: gpu.recording.cluster.1m + interval: 1m + params: {} + rules: + - record: cluster:gpu_count:total + expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) + # Kube-allocated GPU count: GPUs requested by *active* pods — Pending + # and Running only. Source of truth for "what tenants asked for" — used + # for capacity planning and billing. Includes system pods so it stays + # consistent with cluster:gpu_count:total when computing :free. + # + # Phase filter via group_left() on kube_pod_status_phase matches the + # canonical pattern from k8s.rules.container_resource.yaml. Without it + # kube-state-metrics keeps series for Failed/Succeeded pods until the + # apiserver garbage-collects them, which inflates billing hours and + # can push :free negative. + - record: cluster:gpu_count:allocated + expr: | + sum( + kube_pod_container_resource_requests{resource="nvidia_com_gpu"} + * on (namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + # clamp_min guards against transients at DCGM/kube-state-metrics + # restart where cluster:gpu_count:total dips before :allocated catches + # up, or rare label drift between the two sources. + - record: cluster:gpu_count:free + expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0) + # Cluster-layer filter asymmetry — intentional, not a bug: + # + # * cluster:gpu_count:total and cluster:gpu_power_watts:sum do NOT + # filter cozy-*/kube-* namespaces because they describe the + # physical hardware fleet. A GPU draws power and occupies a + # slot regardless of which namespace's pod happens to hold it; + # excluding infra pods would under-report raw capacity and + # break capacity-planning math. + # + # No namespace filter — DCGM exporter metrics carry the exporter's + # own namespace, not the workload namespace. Filtering by namespace + # here would silently drop all series. + - record: cluster:gpu_util:avg + expr: avg(DCGM_FI_DEV_GPU_UTIL) + - record: cluster:gpu_power_watts:sum + expr: sum(DCGM_FI_DEV_POWER_USAGE) + + - name: gpu.recording.node.1m + interval: 1m + params: {} + rules: + # Kube-requested GPU count per namespace — billable/allocation view. + # This is the only namespace-level rule that works correctly because + # kube_pod_container_resource_requests carries the real workload + # namespace, unlike DCGM exporter metrics which carry the exporter's + # own namespace (cozy-gpu-operator). + - record: namespace:gpu_count:allocated + expr: | + sum by (namespace) ( + kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"} + * on (namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + # Hardware metrics aggregated per node (Hostname). DCGM exporter + # metrics carry a capital-H "Hostname" label identifying the node + # but no usable workload namespace — so hardware telemetry is + # aggregated at node/GPU granularity, not namespace. + - record: node:gpu_util:avg + expr: avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL) + - record: node:tensor_active:avg + expr: avg by (Hostname) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) + - record: node:fb_used_bytes:sum + expr: sum by (Hostname) (DCGM_FI_DEV_FB_USED) * 1048576 + - record: node:power_watts:sum + expr: sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) + + # Known gap: there is no lower-bound sanity alert for under-reporting. + # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other + # direction — e.g. from the current nanoseconds back to microseconds, + # making rate()/1e9 produce values around 0.001 instead of real + # fractions — the throttle signal would silently collapse to near-zero + # instead of plateauing at 1.0. GPUThrottleFractionOverOne below catches + # the >1 drift; the <<1 drift would require correlation with independent + # signals (power draw vs TDP, clock dips) that we do not yet wire up. + # Documented here so future maintainers know this is a known gap, not + # an oversight. + - name: gpu.recording.efficiency.1m + interval: 1m + params: {} + rules: + # Tensor hardware saturation per GPU — the honest "am I using the GPU" + # metric for AI/LLM workloads. Aggregated at GPU level (Hostname + + # gpu + UUID) because DCGM metrics don't carry workload namespace. + # Stored as a 0..1 ratio; consumers multiply by 100 at display time. + - record: gpu:tensor_saturation:avg5m + expr: | + avg by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE[5m]) + ) + + # Power efficiency per GPU — utilization per watt, reveals + # unoptimized workloads. Explicit on(...) pins the matching set + # to GPU-identifying labels. + - record: gpu:util_per_watt:avg5m + expr: | + max by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m]) + ) + / on (Hostname, gpu, UUID) + clamp_min( + max by (Hostname, gpu, UUID) ( + avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]) + ), + 1 + ) + + # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. + # DCGM_FI_DEV_*_VIOLATION is documented as µs but on A10/DCGM 3.x the + # counter grows in nanoseconds in practice — divide by 1e9 to get a + # 0..1 fraction (verified empirically when /1e6 yielded >100× reality). + # clamp_max protects against rate() artefacts at counter resets. + # + # max by (Hostname, gpu, UUID) collapses duplicate series from + # dcgm-exporter's pod-mapping labels (one physical GPU may appear + # under multiple pod/container labels during restarts or under + # MIG/MPS). Throttling is a physical-GPU property; taking max + # keeps consumer avg(...) honest — a naive avg() would dilute the + # signal by the number of pods sharing the GPU. + - record: gpu:power_throttle_fraction:rate5m + expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9), 1) + + # Fraction of time thermal-throttled. + - record: gpu:thermal_throttle_fraction:rate5m + expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1) + + # Regression watch — the /1e9 divisor on *_VIOLATION was derived + # empirically against DCGM 3.x on A10. If a future DCGM version + # restores the documented µs units, pre-clamp values would exceed 1.0 + # while clamp_max(..., 1) silently masks the drift — recorded + # throttle fractions would plateau at 1.0 and consumers would think + # every GPU is always throttled. This alert fires on that condition + # so we can rescale to /1e6 before dashboards start lying. + - name: gpu.recording.throttle.validation.5m + interval: 5m + params: {} + rules: + - alert: GPUThrottleFractionOverOne + expr: | + max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9) > 1 + or + max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9) > 1 + for: 15m + labels: + severity: warning + component: gpu-monitoring + annotations: + summary: "DCGM throttle fraction exceeds 1.0 pre-clamp on {{ $labels.Hostname }}/{{ $labels.gpu }}" + description: | + gpu:{power,thermal}_throttle_fraction:rate5m clamps to 1.0, but the raw + rate(DCGM_FI_DEV_*_VIOLATION)/1e9 value is already >1 on this GPU — the + empirical nanosecond assumption is broken. Likely the DCGM exporter + version changed and the counter now grows in its documented µs units. + Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in + gpu.recording.efficiency.1m before dashboards plateau at 100% throttle. + verified_on: "NVIDIA A10, DCGM 3.x — other GPU families (H100, L40S) may require a different divisor" + runbook: "If this alert fires, DCGM may have changed POWER_VIOLATION/THERMAL_VIOLATION counter units. Recalibrate by comparing raw counter rate against known throttle events; see gpu-recording.rules.yaml comments." diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index b5b9f52a..4c7e494d 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -32,3 +32,8 @@ hubble/overview hubble/dns-namespace hubble/l7-http-metrics hubble/network-overview +gpu/gpu-performance +gpu/gpu-efficiency +gpu/gpu-quotas +gpu/gpu-fleet +gpu/gpu-tenants diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index f6d7ddaa..5279f8a7 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add cnpg https://cloudnative-pg.github.io/charts diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml new file mode 100644 index 00000000..51a8ba81 --- /dev/null +++ b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml @@ -0,0 +1,148 @@ +{{- /* + Post-install gate: block the HelmRelease from reporting Ready until the + cnpg admission webhook actually serves through the cluster Service. Helm + --wait on the controller pod passes once its readinessProbe passes, but + EndpointSlice propagation and kube-proxy/cilium data-plane programming + can lag by a second or two — long enough for any HelmRelease that + depends on postgres-operator (e.g. cozy-keycloak, tenant Postgres apps) + to fire its own install and have kube-apiserver hit the mcluster.cnpg.io + mutating webhook with "dial tcp :443: connect: connection refused". + + The Job uses the apiserver service proxy, which exercises the same + endpoint-resolution and apiserver-initiated pod dial that the admission + webhook path uses. Once /readyz answers through the proxy the data-plane + race is resolved. It does not verify the webhook's TLS CA bundle, so + this gate is scoped to reachability regressions, not cert rotation. + + The service name and port name are hardcoded literals. Upstream cnpg + pins the service name in charts/cloudnative-pg/values.yaml with a + comment "DO NOT CHANGE THE SERVICE NAME as it is currently used to + generate the certificate and can not be configured". The port name is + fixed in charts/cloudnative-pg/templates/service.yaml (ports[0].name: + webhook-server). If a future `make update` ever changes either literal + upstream, the sync-check helm-unittest test + (tests/webhook-ready-hook_test.yaml) renders the subchart Service and + fails if the literal drifts — forcing this template to be updated in + the same change. +*/}} +{{- $_ := required "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" .Values.webhookReady.image.repository -}} +{{- $_ := required "webhookReady.image.tag must be set for the post-install readiness Job" .Values.webhookReady.image.tag -}} +{{- /* $svcName and $portName are hardcoded literals; see header comment. */ -}} +{{- $svcName := "cnpg-webhook-service" -}} +{{- /* $portName is the service port NAME, not number — matches ports[0].name in the vendored subchart's Service. */ -}} +{{- $portName := "webhook-server" -}} +{{- $resourceName := printf "https:%s:%s" $svcName $portName -}} +{{- $maxAttempts := .Values.webhookReady.maxAttempts | default 60 -}} +{{- $sleepSeconds := .Values.webhookReady.sleepSeconds | default 2 -}} +{{- /* Derive activeDeadlineSeconds from retries + 60s slack so a values override that raises maxAttempts doesn't get silently cut. */ -}} +{{- $deadline := add (mul (int $maxAttempts) (int $sleepSeconds)) 60 -}} +{{- $image := printf "%s:%s" .Values.webhookReady.image.repository .Values.webhookReady.image.tag -}} +{{- if .Values.webhookReady.image.digest }} +{{- $image = printf "%s@%s" $image .Values.webhookReady.image.digest -}} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +rules: + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: [{{ $resourceName | quote }}] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-webhook-ready +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-webhook-ready + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ .Values.webhookReady.backoffLimit | default 2 }} + activeDeadlineSeconds: {{ $deadline }} + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-webhook-ready + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: wait + image: {{ $image }} + imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + command: + - sh + - -c + - | + set -e + ns={{ .Release.Namespace }} + proxy="/api/v1/namespaces/${ns}/services/{{ $resourceName }}/proxy/readyz" + max_attempts={{ $maxAttempts }} + sleep_seconds={{ $sleepSeconds }} + i=0 + last_err="" + until last_err=$(kubectl get --raw "$proxy" 2>&1 >/dev/null); do + i=$((i + 1)) + if [ $i -gt $max_attempts ]; then + echo "timeout: cnpg webhook did not respond through the apiserver proxy after ${max_attempts} attempts (${sleep_seconds}s each)" + echo "last error: ${last_err}" + exit 1 + fi + if [ $((i % 10)) -eq 1 ]; then + echo "attempt $i/${max_attempts}: ${last_err}" + fi + sleep "$sleep_seconds" + done + echo "cnpg webhook is ready" diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml new file mode 100644 index 00000000..3253c663 --- /dev/null +++ b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml @@ -0,0 +1,328 @@ +suite: cnpg webhook post-install readiness gate + +templates: + - templates/webhook-ready-hook.yaml + - charts/cloudnative-pg/templates/service.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: renders four hook objects (SA + Role + RoleBinding + Job) + template: templates/webhook-ready-hook.yaml + asserts: + - hasDocuments: + count: 4 + + - it: every rendered object carries post-install and post-upgrade hook annotations + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + + - it: RBAC is created before the Job (hook-weight ordering) + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 0 + equal: + path: kind + value: ServiceAccount + - documentIndex: 0 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 1 + equal: + path: kind + value: Role + - documentIndex: 1 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 2 + equal: + path: kind + value: RoleBinding + - documentIndex: 2 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + - documentIndex: 3 + equal: + path: kind + value: Job + - documentIndex: 3 + equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "10" + + - it: RBAC resourceName matches the exact proxy URL segment the Job probes + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 1 + equal: + path: rules[0].resources[0] + value: services/proxy + - documentIndex: 1 + equal: + path: rules[0].resourceNames[0] + value: https:cnpg-webhook-service:webhook-server + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "/api/v1/namespaces/\\$\\{ns\\}/services/https:cnpg-webhook-service:webhook-server/proxy/readyz" + + - it: hardcoded service name in the hook matches the vendored cnpg subchart Service (drift guard for make update) + template: charts/cloudnative-pg/templates/service.yaml + asserts: + - equal: + path: metadata.name + value: cnpg-webhook-service + - equal: + path: spec.ports[0].name + value: webhook-server + + - it: Job calls kubectl get --raw on the proxy path + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "kubectl get --raw" + + - it: Job image is digest-pinned when webhookReady.image.digest is set + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + + - it: Job image falls back to tag-only when digest is not configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].image + value: docker.io/clastix/kubectl:v1.32 + + - it: backoffLimit defaults to 2 so transient pod-level failures retry instead of killing the HelmRelease + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 2 + + - it: backoffLimit is overridable + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + backoffLimit: 5 + asserts: + - documentIndex: 3 + equal: + path: spec.backoffLimit + value: 5 + + - it: chart render fails when webhookReady.image.repository is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: "" + tag: v1.32 + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" + + - it: chart render fails when webhookReady.image.tag is empty + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: "" + asserts: + - failedTemplate: + errorMessage: "webhookReady.image.tag must be set for the post-install readiness Job" + + - it: retry loop bounds default when blanked so a wiped override still produces a working Job + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: null + sleepSeconds: null + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=60" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=2" + + - it: Job pod runs non-root with seccomp RuntimeDefault for restricted-PSA clusters + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + + - it: wait container declares resource requests and limits for predictable scheduling + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: 10m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 32Mi + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.cpu + value: 100m + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 64Mi + + - it: Job container drops all capabilities and runs read-only rootfs + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].securityContext.capabilities.drop[0] + value: ALL + + - it: imagePullPolicy is IfNotPresent when digest-pinned, Always when tag-only + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: imagePullPolicy is Always when no digest is configured + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: "" + asserts: + - documentIndex: 3 + equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: retry loop captures and surfaces the last kubectl error message on timeout + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last_err=\$\(kubectl get --raw' + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'last error: \$\{last_err\}' + + - it: retry loop error message stays in sync when maxAttempts is bumped + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 90 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "max_attempts=90" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "sleep_seconds=3" + - documentIndex: 3 + matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'after \$\{max_attempts\} attempts' + + - it: activeDeadlineSeconds scales with retry bounds so an override raise does not silently cut + template: templates/webhook-ready-hook.yaml + set: + webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + maxAttempts: 180 + sleepSeconds: 3 + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: activeDeadlineSeconds defaults include 60s slack over the default retry window + template: templates/webhook-ready-hook.yaml + asserts: + - documentIndex: 3 + equal: + path: spec.activeDeadlineSeconds + value: 180 diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index cae3dc53..c408220e 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -3,3 +3,30 @@ cloudnative-pg: create: true image: tag: "1.27.3" +# Image used by the post-install webhook-readiness Job (see templates/webhook-ready-hook.yaml). +# Any image with kubectl on PATH works; the Job calls the apiserver service proxy with the +# hook ServiceAccount's token to confirm the mcluster.cnpg.io webhook is reachable end-to-end +# before the HelmRelease reports Ready, closing the "connection refused" bootstrap race. +# +# The tag is digest-pinned so an upstream retag does not change what runs on every install +# and upgrade across the fleet. Refresh by resolving the current manifest-list digest +# (`docker manifest inspect docker.io/clastix/kubectl:v1.32`) and updating `digest` below. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +webhookReady: + image: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + # Retry loop bounds for the readiness probe. Defaults total ~120s wall clock. + # Both are `default`-coerced in the template so an override that blanks them still + # produces a working Job. + maxAttempts: 60 + sleepSeconds: 2 + # Pod-level retry budget for transient node/registry failures (image pull rate limit, + # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. + # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all + # pod retries) are ANDed, so activeDeadlineSeconds is the shorter of the two gates with + # the defaults above: the 60*2+60 = 180s deadline cuts the Job before backoffLimit=2 + # ever matters if pod-level failures eat more than ~60s of the budget. Raise + # maxAttempts/sleepSeconds alongside backoffLimit when tuning for slow image pulls. + backoffLimit: 2 diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index c74c5783..72d013f1 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -8,7 +8,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: @@ -33,7 +33,7 @@ spec: # labelSelector: # helm.toolkit.fluxcd.io/name: "{reqs[0]['metadata','name']}" - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"], ["spec", "bootstrap", "serverName"]] secrets: exclude: [] include: diff --git a/packages/system/tenant-rd/cozyrds/tenant.yaml b/packages/system/tenant-rd/cozyrds/tenant.yaml index 5d352335..9af11af0 100644 --- a/packages/system/tenant-rd/cozyrds/tenant.yaml +++ b/packages/system/tenant-rd/cozyrds/tenant.yaml @@ -8,7 +8,7 @@ spec: singular: tenant plural: tenants openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access tenant services (defaults to using the tenant name as a subdomain for its parent tenant host).","type":"string","default":""},"etcd":{"description":"Deploy own Etcd cluster.","type":"boolean","default":false},"monitoring":{"description":"Deploy own Monitoring Stack.","type":"boolean","default":false},"ingress":{"description":"Deploy own Ingress Controller.","type":"boolean","default":false},"gateway":{"description":"Deploy own Gateway API Gateway (backed by Cilium Gateway API controller).","type":"boolean","default":false},"seaweedfs":{"description":"Deploy own SeaweedFS.","type":"boolean","default":false},"schedulingClass":{"description":"The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads.","type":"string","default":""},"resourceQuotas":{"description":"Define resource quotas for the tenant.","type":"object","default":{},"additionalProperties":{"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}} + {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access tenant services (defaults to using the tenant name as a subdomain for its parent tenant host).","type":"string","default":""},"etcd":{"description":"Deploy own Etcd cluster.","type":"boolean","default":false},"monitoring":{"description":"Deploy own Monitoring Stack.","type":"boolean","default":false},"ingress":{"description":"Deploy own Ingress Controller.","type":"boolean","default":false},"seaweedfs":{"description":"Deploy own SeaweedFS.","type":"boolean","default":false},"schedulingClass":{"description":"The name of a SchedulingClass CR to apply scheduling constraints for this tenant's workloads.","type":"string","default":""},"resourceQuotas":{"description":"Define resource quotas for the tenant.","type":"object","default":{},"additionalProperties":{"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}} release: prefix: tenant- labels: @@ -23,7 +23,7 @@ spec: plural: Tenants description: Separated tenant namespace icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQwMykiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDAzKSI+CjxwYXRoIGQ9Ik03MiAyOUM2Ni4zOTI2IDI5IDYxLjAxNDggMzEuMjM4OCA1Ny4wNDk3IDM1LjIyNEM1My4wODQ3IDM5LjIwOTEgNTAuODU3MSA0NC42MTQxIDUwLjg1NzEgNTAuMjVDNTAuODU3MSA1NS44ODU5IDUzLjA4NDcgNjEuMjkwOSA1Ny4wNDk3IDY1LjI3NkM2MS4wMTQ4IDY5LjI2MTIgNjYuMzkyNiA3MS41IDcyIDcxLjVDNzcuNjA3NCA3MS41IDgyLjk4NTIgNjkuMjYxMiA4Ni45NTAzIDY1LjI3NkM5MC45MTUzIDYxLjI5MDkgOTMuMTQyOSA1NS44ODU5IDkzLjE0MjkgNTAuMjVDOTMuMTQyOSA0NC42MTQxIDkwLjkxNTMgMzkuMjA5MSA4Ni45NTAzIDM1LjIyNEM4Mi45ODUyIDMxLjIzODggNzcuNjA3NCAyOSA3MiAyOVpNNjAuOTgyNiA4My4zMDM3QzYwLjQ1NCA4Mi41ODk4IDU5LjU5NTEgODIuMTkxNCA1OC43MTk2IDgyLjI3NDRDNDUuMzg5NyA4My43MzU0IDM1IDk1LjEwNzQgMzUgMTA4LjkwM0MzNSAxMTEuNzI2IDM3LjI3OTUgMTE0IDQwLjA3MSAxMTRIMTAzLjkyOUMxMDYuNzM3IDExNCAxMDkgMTExLjcwOSAxMDkgMTA4LjkwM0MxMDkgOTUuMTA3NCA5OC42MTAzIDgzLjc1MiA4NS4yNjM4IDgyLjI5MUM4NC4zODg0IDgyLjE5MTQgODMuNTI5NSA4Mi42MDY0IDgzLjAwMDkgODMuMzIwM0w3NC4wOTc4IDk1LjI0MDJDNzMuMDQwNiA5Ni42NTE0IDcwLjkyNjMgOTYuNjUxNCA2OS44NjkyIDk1LjI0MDJMNjAuOTY2MSA4My4zMjAzTDYwLjk4MjYgODMuMzAzN1oiIGZpbGw9ImJsYWNrIi8+CjwvZz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82ODdfMzQwMyIgeDE9IjcyIiB5MT0iMTQ0IiB4Mj0iLTEuMjgxN2UtMDUiIHkyPSI0IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wIHN0b3AtY29sb3I9IiNDMEQ2RkYiLz4KPHN0b3Agb2Zmc2V0PSIwLjMiIHN0b3AtY29sb3I9IiNDNERBRkYiLz4KPHN0b3Agb2Zmc2V0PSIwLjY1IiBzdG9wLWNvbG9yPSIjRDNFOUZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iI0U5RkZGRiIvPgo8L2xpbmVhckdyYWRpZW50Pgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzY4N18zNDAzIj4KPHJlY3Qgd2lkdGg9Ijc0IiBoZWlnaHQ9Ijg1IiBmaWxsPSJ3aGl0ZSIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMzUgMjkpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg== - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "etcd"], ["spec", "monitoring"], ["spec", "ingress"], ["spec", "gateway"], ["spec", "seaweedfs"], ["spec", "schedulingClass"], ["spec", "resourceQuotas"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "etcd"], ["spec", "monitoring"], ["spec", "ingress"], ["spec", "seaweedfs"], ["spec", "schedulingClass"], ["spec", "resourceQuotas"]] secrets: exclude: [] include: [] diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e93fa86b..e56bd9dd 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -8,7 +8,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQ1NCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDU0KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMDBCM0ZGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8L2c+CjxwYXRoIGQ9Ik03Mi41Mjc1IDUzLjgzNjdDNzIuNDQzMSA1My44MzUxIDcyLjM2MDUgNTMuODEyMiA3Mi4yODczIDUzLjc3MDFMNTYuNDY5OSA0NC43OTM0QzU2LjM5ODMgNDQuNzUxOSA1Ni4zMzg4IDQ0LjY5MjMgNTYuMjk3MyA0NC42MjA3QzU2LjI1NTkgNDQuNTQ5IDU2LjIzMzggNDQuNDY3OCA1Ni4yMzM0IDQ0LjM4NUM1Ni4yMzM0IDQ0LjIxNjkgNTYuMzI1OCA0NC4wNjE3IDU2LjQ2OTkgNDMuOTc4NUw3Mi4xOTEyIDM1LjA2MDlDNzIuMjYzNyAzNS4wMjEgNzIuMzQ1IDM1IDcyLjQyNzcgMzVDNzIuNTEwNSAzNSA3Mi41OTE4IDM1LjAyMSA3Mi42NjQzIDM1LjA2MDlMODguNDg3MiA0NC4wMzk1Qzg4LjU1OTEgNDQuMDgwMSA4OC42MTg4IDQ0LjEzOTIgODguNjYgNDQuMjEwN0M4OC43MDEzIDQ0LjI4MjIgODguNzIyNyA0NC4zNjM1IDg4LjcyMTkgNDQuNDQ2Qzg4LjcyMjUgNDQuNTI4NSA4OC43MDEgNDQuNjA5NyA4OC42NTk4IDQ0LjY4MTJDODguNjE4NSA0NC43NTI2IDg4LjU1ODkgNDQuODExOCA4OC40ODcyIDQ0Ljg1MjVMNzIuNzcxNCA1My43NjgzQzcyLjY5NzIgNTMuODExNCA3Mi42MTMzIDUzLjgzNDkgNzIuNTI3NSA1My44MzY3IiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBvcGFjaXR5PSIwLjciIGQ9Ik03MC4yNTUzIDc1LjY1MTdDNzAuMTcxIDc1LjY1MzUgNzAuMDg3OCA3NS42MzE3IDcwLjAxNTEgNzUuNTg4OEw1NC4yNDU4IDY2LjY0MTdDNTQuMTcxNSA2Ni42MDI0IDU0LjEwOTUgNjYuNTQzNiA1NC4wNjYxIDY2LjQ3MTZDNTQuMDIyOCA2Ni4zOTk3IDU0IDY2LjMxNzMgNTQgNjYuMjMzM1Y0OC4yNzhDNTQgNDguMTA4IDU0LjA5MjQgNDcuOTU0NiA1NC4yNDM5IDQ3Ljg2OTZDNTQuMzE3MiA0Ny44MjcxIDU0LjQwMDQgNDcuODA0NyA1NC40ODUxIDQ3LjgwNDdDNTQuNTY5NyA0Ny44MDQ3IDU0LjY1MjkgNDcuODI3MSA1NC43MjYyIDQ3Ljg2OTZMNzAuNDkzNyA1Ni44MTMxQzcwLjU2NDIgNTYuODU2NSA3MC42MjI1IDU2LjkxNyA3MC42NjMyIDU2Ljk4OTFDNzAuNzAzOSA1Ny4wNjEyIDcwLjcyNTcgNTcuMTQyNCA3MC43MjY1IDU3LjIyNTFWNzUuMTgwNUM3MC43MjU5IDc1LjI2MjggNzAuNzA0MiA3NS4zNDM2IDcwLjY2MzUgNzUuNDE1MUM3MC42MjI3IDc1LjQ4NjYgNzAuNTY0MiA3NS41NDY0IDcwLjQ5MzcgNzUuNTg4OEM3MC40MjA2IDc1LjYyOTEgNzAuMzM4NyA3NS42NTA3IDcwLjI1NTMgNzUuNjUxNyIgZmlsbD0id2hpdGUiLz4KPHBhdGggb3BhY2l0eT0iMC40IiBkPSJNNzQuNzE5OCA3NS42NTExQzc0LjYzMzMgNzUuNjUxMiA3NC41NDgyIDc1LjYyOTYgNzQuNDcyMiA3NS41ODgzQzc0LjQwMTYgNzUuNTQ2MSA3NC4zNDMyIDc1LjQ4NjIgNzQuMzAyNyA3NS40MTQ3Qzc0LjI2MjMgNzUuMzQzMSA3NC4yNDExIDc1LjI2MjIgNzQuMjQxMiA3NS4xOFY1Ny4zMzczQzc0LjI0MTIgNTcuMTcxIDc0LjMzMzYgNTcuMDE1OCA3NC40NzIyIDU2LjkyOUw5MC4yMzk3IDQ3Ljk4NTVDOTAuMzExOSA0Ny45NDM4IDkwLjM5MzggNDcuOTIxOSA5MC40NzcxIDQ3LjkyMTlDOTAuNTYwNSA0Ny45MjE5IDkwLjY0MjQgNDcuOTQzOCA5MC43MTQ2IDQ3Ljk4NTVDOTAuNzg3NiA0OC4wMjU1IDkwLjg0ODUgNDguMDg0MiA5MC44OTExIDQ4LjE1NTdDOTAuOTMzNyA0OC4yMjcyIDkwLjk1NjMgNDguMzA4OCA5MC45NTY2IDQ4LjM5MlY2Ni4yMzI4QzkwLjk1NyA2Ni4zMTY0IDkwLjkzNDcgNjYuMzk4NSA5MC44OTIxIDY2LjQ3MDRDOTAuODQ5NSA2Ni41NDI0IDkwLjc4ODEgNjYuNjAxNCA5MC43MTQ2IDY2LjY0MTFMNzQuOTUyNiA3NS41ODgzQzc0Ljg4MjUgNzUuNjMwNyA3NC44MDE4IDc1LjY1MjUgNzQuNzE5OCA3NS42NTExIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zNDU0IiB4MT0iMTYxIiB5MT0iMTgwIiB4Mj0iMy41OTI4NGUtMDciIHkyPSI0Ljk5OTk4IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wLz4KPHN0b3Agb2Zmc2V0PSIxIiBzdG9wLWNvbG9yPSIjNTk1NjU2Ii8+CjwvbGluZWFyR3JhZGllbnQ+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNjg3XzM0NTQiPgo8cmVjdCB3aWR0aD0iOTQiIGhlaWdodD0iOTQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgyNSAyNSkiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "externalAllowICMP"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] secrets: exclude: [] include: [] diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index f7791332..7ebbbc89 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -160,6 +160,34 @@ func (o *CozyServerOptions) Complete() error { // Convert to ResourceConfig o.ResourceConfig = &config.ResourceConfig{} for _, crd := range crdList.Items { + release := config.ReleaseConfig{ + Prefix: crd.Spec.Release.Prefix, + Labels: crd.Spec.Release.Labels, + ChartRef: config.ChartRefConfig{ + Kind: crd.Spec.Release.ChartRef.Kind, + Name: crd.Spec.Release.ChartRef.Name, + Namespace: crd.Spec.Release.ChartRef.Namespace, + }, + } + // Per-Application HelmRelease Install/Upgrade timeout. Applications + // whose parent chart contains asynchronously-provisioned resources + // the chart itself depends on (for example, the Kamaji-provisioned + // admin-kubeconfig Secret for Kubernetes tenants) need a longer + // wait budget than the Flux default. Consumed by the REST storage + // layer when building the HelmRelease Spec. The parser rejects + // units Flux would reject at webhook time, so a bad annotation + // surfaces as a loud startup failure instead of a silent drop to + // defaults. + d, err := config.ParseHelmInstallTimeoutAnnotation( + crd.Annotations[config.HelmInstallTimeoutAnnotation], + ) + if err != nil { + return fmt.Errorf( + "ApplicationDefinition %q has invalid %s annotation: %w", + crd.Name, config.HelmInstallTimeoutAnnotation, err, + ) + } + release.HelmInstallTimeout = d resource := config.Resource{ Application: config.ApplicationConfig{ Kind: crd.Spec.Application.Kind, @@ -168,15 +196,7 @@ func (o *CozyServerOptions) Complete() error { ShortNames: []string{}, // TODO: implement shortnames OpenAPISchema: crd.Spec.Application.OpenAPISchema, }, - Release: config.ReleaseConfig{ - Prefix: crd.Spec.Release.Prefix, - Labels: crd.Spec.Release.Labels, - ChartRef: config.ChartRefConfig{ - Kind: crd.Spec.Release.ChartRef.Kind, - Name: crd.Spec.Release.ChartRef.Name, - Namespace: crd.Spec.Release.ChartRef.Namespace, - }, - }, + Release: release, } o.ResourceConfig.Resources = append(o.ResourceConfig.Resources, resource) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e123e2c..21de27b2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,6 +16,48 @@ limitations under the License. package config +import ( + "fmt" + "regexp" + "time" +) + +// HelmInstallTimeoutAnnotation is the ApplicationDefinition metadata +// annotation key that overrides the Flux HelmRelease Install.Timeout and +// Upgrade.Timeout for a given Application kind. +const HelmInstallTimeoutAnnotation = "release.cozystack.io/helm-install-timeout" + +// helmTimeoutPattern mirrors the CRD validation pattern used by Flux +// helm-controller on HelmReleaseSpec.Install.Timeout (ms/s/m/h units only). +// time.ParseDuration accepts ns/us/µs, but Flux rejects them - parsing here +// with the same shape avoids feeding the controller a value it will later +// reject at webhook time. See +// github.com/fluxcd/helm-controller/api/v2 HelmReleaseSpec.Install.Timeout +// in the go module cache. +var helmTimeoutPattern = regexp.MustCompile(`^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$`) + +// ParseHelmInstallTimeoutAnnotation parses the value of the +// release.cozystack.io/helm-install-timeout annotation. The empty string is +// treated as "unset" and returns (0, nil) so callers can leave +// HelmInstallTimeout zeroed and let flux defaults apply. Values accepted by +// time.ParseDuration but rejected by Flux (ns/us/µs) return a helpful +// error instead of silently parsing and failing later at HelmRelease +// admission. +func ParseHelmInstallTimeoutAnnotation(raw string) (time.Duration, error) { + if raw == "" { + return 0, nil + } + if !helmTimeoutPattern.MatchString(raw) { + return 0, fmt.Errorf("must match %s (Flux accepts ms/s/m/h units only), got %q", + helmTimeoutPattern, raw) + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("time.ParseDuration(%q): %w", raw, err) + } + return d, nil +} + // ResourceConfig represents the structure of the configuration file. type ResourceConfig struct { Resources []Resource `yaml:"resources"` @@ -41,6 +83,12 @@ type ReleaseConfig struct { Prefix string `yaml:"prefix"` Labels map[string]string `yaml:"labels"` ChartRef ChartRefConfig `yaml:"chartRef"` + // HelmInstallTimeout overrides the Flux HelmRelease Install.Timeout and + // Upgrade.Timeout for this Application kind. When zero, flux defaults + // apply. Populated from the + // release.cozystack.io/helm-install-timeout annotation on the + // ApplicationDefinition at start-up. + HelmInstallTimeout time.Duration `yaml:"helmInstallTimeout,omitempty"` } // ChartRefConfig references a Flux source artifact for the Helm chart. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 00000000..eb6990af --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "strings" + "testing" + "time" +) + +// Cover the annotation parser used by cozystack-api at startup. The parser +// is consumed by pkg/cmd/server/start.go on every ApplicationDefinition; a +// typo here silently drops back to flux defaults and the Kubernetes tenant +// race described in cozystack#2412 reappears, so the table must exercise: +// - the unset path (empty string treated as "no override"), +// - every unit Flux accepts (ms, s, m, h), +// - compound forms (the CRD pattern accepts repeats), +// - units time.ParseDuration accepts but Flux rejects (ns, us, µs), +// - outright garbage. +func TestParseHelmInstallTimeoutAnnotation(t *testing.T) { + cases := []struct { + name string + input string + want time.Duration + wantErr bool + errMatch string + }{ + { + name: "empty string leaves flux defaults in place", + input: "", + want: 0, + }, + { + name: "minutes", + input: "15m", + want: 15 * time.Minute, + }, + { + name: "hours", + input: "1h", + want: time.Hour, + }, + { + name: "seconds", + input: "45s", + want: 45 * time.Second, + }, + { + name: "milliseconds", + input: "500ms", + want: 500 * time.Millisecond, + }, + { + name: "compound hour and minutes", + input: "2h30m", + want: 2*time.Hour + 30*time.Minute, + }, + { + name: "decimal minutes", + input: "1.5m", + want: 90 * time.Second, + }, + { + name: "nanoseconds rejected - Flux CRD pattern excludes ns", + input: "500ns", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "microseconds rejected - Flux CRD pattern excludes us", + input: "500us", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "microseconds unicode rejected", + input: "500µs", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "bare digits rejected", + input: "15", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "garbage rejected", + input: "abc", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + { + name: "negative rejected", + input: "-15m", + wantErr: true, + errMatch: "Flux accepts ms/s/m/h units only", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseHelmInstallTimeoutAnnotation(tc.input) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error, got duration=%v", got) + } + if tc.errMatch != "" && !strings.Contains(err.Error(), tc.errMatch) { + t.Errorf("error %q does not contain %q", err.Error(), tc.errMatch) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.want { + t.Errorf("got %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 896f3eb7..d233ecc3 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -181,16 +181,6 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation return nil, apierrors.NewBadRequest(err.Error()) } - // Run the genericapiserver-supplied admission chain (mutating + validating - // webhooks and ValidatingAdmissionPolicies) before persisting anything. - // Custom REST handlers must invoke this hook explicitly — unlike - // genericregistry.Store, which wires it automatically. - if createValidation != nil { - if err := createValidation(ctx, obj); err != nil { - return nil, err - } - } - // Convert Application to HelmRelease helmRelease, err := r.ConvertApplicationToHelmRelease(app) if err != nil { @@ -600,22 +590,6 @@ func (r *REST) Delete(ctx context.Context, name string, deleteValidation rest.Va return nil, false, apierrors.NewNotFound(r.gvr.GroupResource(), name) } - // Run the genericapiserver-supplied admission chain (validating webhooks - // and ValidatingAdmissionPolicies) on the resolved Application before - // removing the underlying HelmRelease. Custom REST handlers must invoke - // this hook explicitly — unlike genericregistry.Store, which wires it - // automatically. - if deleteValidation != nil { - converted, convErr := r.ConvertHelmReleaseToApplication(ctx, helmRelease) - if convErr != nil { - klog.Errorf("Failed to convert HelmRelease %s to Application for delete admission: %v", helmReleaseName, convErr) - return nil, false, convErr - } - if err := deleteValidation(ctx, &converted); err != nil { - return nil, false, err - } - } - klog.V(6).Infof("Deleting HelmRelease %s in namespace %s", helmReleaseName, namespace) // Delete the HelmRelease corresponding to the Application @@ -1554,6 +1528,24 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, } + // Per-Application HelmRelease wait budget. The mechanism is generic: + // an ApplicationDefinition that sets + // release.cozystack.io/helm-install-timeout gets Install.Timeout and + // Upgrade.Timeout populated from ReleaseConfig.HelmInstallTimeout + // (parsed at startup). Applications that leave it unset keep flux + // defaults so their failed installs remediate on the normal cadence. + // Today only kubernetes-rd carries the annotation because the + // Kubernetes Application's parent chart contains CAPI/Kamaji + // resources whose admin-kubeconfig Secret is provisioned + // asynchronously and Kamaji cold-start routinely exceeds flux's + // default wait budget; any future kind with the same shape can opt + // in by setting the same annotation. + if r.releaseConfig.HelmInstallTimeout > 0 { + timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout} + helmRelease.Spec.Install.Timeout = &timeout + helmRelease.Spec.Upgrade.Timeout = &timeout + } + return helmRelease, nil } diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go new file mode 100644 index 00000000..85ac0b97 --- /dev/null +++ b/pkg/registry/apps/application/rest_timeout_test.go @@ -0,0 +1,116 @@ +package application + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + "github.com/cozystack/cozystack/pkg/config" +) + +func newRESTForTimeout(kind, prefix string, helmInstallTimeout time.Duration) *REST { + return &REST{ + kindName: kind, + releaseConfig: config.ReleaseConfig{ + Prefix: prefix, + ChartRef: config.ChartRefConfig{ + Kind: "HelmChart", + Name: "x", + Namespace: "cozy-system", + }, + HelmInstallTimeout: helmInstallTimeout, + }, + } +} + +// Table-driven: every Application kind carries a per-CRD HelmRelease wait +// budget. The Kubernetes kind's parent chart contains CAPI/Kamaji resources +// whose admin-kubeconfig Secret is provisioned asynchronously, so its +// ApplicationDefinition sets release.cozystack.io/helm-install-timeout=15m +// (or longer). Other kinds leave the annotation unset and keep flux defaults +// so their failed installs remediate on the normal cadence. The test must +// cover both paths: a kind with the timeout set and one without. +func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing.T) { + cases := []struct { + name string + kind string + prefix string + configured time.Duration + wantSet bool + }{ + { + name: "Kubernetes kind with 15m configured gets Install and Upgrade Timeout", + kind: "Kubernetes", + prefix: "kubernetes-", + configured: 15 * time.Minute, + wantSet: true, + }, + { + // Fictional kind on purpose: the test is about the unset path + // regardless of which real Application kind ends up needing a + // timeout override. Using a real kind name would create false + // coupling to that Application's ApplicationDefinition. + name: "unrelated kind without configured timeout keeps flux defaults", + kind: "PlaceholderKindForDefaults", + prefix: "placeholder-", + configured: 0, + wantSet: false, + }, + { + name: "arbitrary future kind with 20m configured gets 20m", + kind: "TalosCluster", + prefix: "talos-", + configured: 20 * time.Minute, + wantSet: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := newRESTForTimeout(tc.kind, tc.prefix, tc.configured) + app := &appsv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, + } + + hr, err := r.convertApplicationToHelmRelease(app) + if err != nil { + t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) + } + + if hr.Spec.Install == nil || hr.Spec.Upgrade == nil { + t.Fatalf("Spec.Install/Upgrade must be non-nil") + } + + if tc.wantSet { + if hr.Spec.Install.Timeout == nil { + t.Fatalf("Spec.Install.Timeout must be set when HelmInstallTimeout=%v", tc.configured) + } + if hr.Spec.Install.Timeout.Duration != tc.configured { + t.Errorf("Spec.Install.Timeout = %v, want %v", hr.Spec.Install.Timeout.Duration, tc.configured) + } + if hr.Spec.Upgrade.Timeout == nil { + t.Fatalf("Spec.Upgrade.Timeout must be set when HelmInstallTimeout=%v", tc.configured) + } + if hr.Spec.Upgrade.Timeout.Duration != tc.configured { + t.Errorf("Spec.Upgrade.Timeout = %v, want %v", hr.Spec.Upgrade.Timeout.Duration, tc.configured) + } + } else { + if hr.Spec.Install.Timeout != nil { + t.Errorf("Spec.Install.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Install.Timeout.Duration) + } + if hr.Spec.Upgrade.Timeout != nil { + t.Errorf("Spec.Upgrade.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Upgrade.Timeout.Duration) + } + } + + if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { + t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) + } + if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { + t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) + } + }) + } +} diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index f1ed3fab..4ee97eb0 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -17,6 +17,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -24,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" @@ -123,8 +126,18 @@ func (r *REST) Get( return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) } + // Check if user has access to this namespace + hasAccess, err := r.hasAccessToNamespace(ctx, name) + if err != nil { + return nil, err + } + if !hasAccess { + // Return Forbidden to follow standard K8s RBAC behavior + return nil, apierrors.NewForbidden(r.gvr.GroupResource(), name, fmt.Errorf("access denied")) + } + ns := &corev1.Namespace{} - err := r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) + err = r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) if err != nil { return nil, err } @@ -143,11 +156,33 @@ func (r *REST) Get( // ----------------------------------------------------------------------------- func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) { + // Extract user identity once for the lifetime of the watch — it does not + // change between events and rebuilding it per event is wasteful. + u, ok := request.UserFrom(ctx) + if !ok { + return nil, apierrors.NewUnauthorized("user missing in context") + } + username := u.GetName() + groups := make(map[string]struct{}) + for _, group := range u.GetGroups() { + groups[group] = struct{}{} + } + nsList := &corev1.NamespaceList{} - nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: &metav1.ListOptions{ + + // Build upstream watch options with field and label selectors + rawOpts := &metav1.ListOptions{ Watch: true, ResourceVersion: opts.ResourceVersion, - }}) + } + if opts.FieldSelector != nil { + rawOpts.FieldSelector = opts.FieldSelector.String() + } + if opts.LabelSelector != nil { + rawOpts.LabelSelector = opts.LabelSelector.String() + } + + nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: rawOpts}) if err != nil { return nil, err } @@ -189,6 +224,30 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch continue } + // Apply defensive filtering for field and label selectors + if opts.FieldSelector != nil { + if !opts.FieldSelector.Matches(fields.Set{"metadata.name": ns.Name}) { + continue + } + } + if opts.LabelSelector != nil { + if !opts.LabelSelector.Matches(labels.Set(ns.Labels)) { + continue + } + } + + // Check if user has access to this namespace using the cached + // identity — avoids re-extracting user/groups on every event. + hasAccess, err := r.hasAccessToNamespaceForUser(ctx, ns.Name, username, groups) + if err != nil { + klog.ErrorS(err, "Failed to check access for namespace in watch", "namespace", ns.Name) + continue + } + if !hasAccess { + // User doesn't have access, skip this event + continue + } + out := &corev1alpha1.TenantNamespace{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1alpha1.SchemeGroupVersion.String(), @@ -309,6 +368,26 @@ func (r *REST) makeList(src *corev1.NamespaceList, allowed []string) *corev1alph return out } +// matchesSubject checks if a RoleBinding subject matches the user's identity. +// It handles Group, User, and ServiceAccount subjects with proper namespace fallback. +func matchesSubject(subj rbacv1.Subject, bindingNamespace, username string, groups map[string]struct{}) bool { + switch subj.Kind { + case "Group": + _, ok := groups[subj.Name] + return ok + case "User": + return subj.Name == username + case "ServiceAccount": + saNamespace := subj.Namespace + if saNamespace == "" { + saNamespace = bindingNamespace + } + return username == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) + default: + return false + } +} + func (r *REST) filterAccessible( ctx context.Context, names []string, @@ -347,22 +426,9 @@ func (r *REST) filterAccessible( subjectLoop: for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - switch subj.Kind { - case "Group": - if _, ok = groups[subj.Name]; ok { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } - case "User": - if subj.Name == u.GetName() { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } - case "ServiceAccount": - if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", subj.Namespace, subj.Name) { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop - } + if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { + allowedNameSet[rbs.Items[i].Namespace] = struct{}{} + break subjectLoop } } } @@ -373,6 +439,60 @@ func (r *REST) filterAccessible( return allowed, nil } +// hasAccessToNamespace checks if the user has access to a single namespace. +// This is optimized for Get/Watch operations where we check one namespace at a time. +// It lists RoleBindings only in the target namespace instead of all cluster RoleBindings. +func (r *REST) hasAccessToNamespace( + ctx context.Context, + namespace string, +) (bool, error) { + u, ok := request.UserFrom(ctx) + if !ok { + return false, fmt.Errorf("user missing in context") + } + groups := make(map[string]struct{}) + for _, group := range u.GetGroups() { + groups[group] = struct{}{} + } + return r.hasAccessToNamespaceForUser(ctx, namespace, u.GetName(), groups) +} + +// hasAccessToNamespaceForUser is the inner check that does not re-extract user +// identity from context. Use this in hot paths (e.g. the Watch loop) where the +// caller has already cached the user name and groups. +func (r *REST) hasAccessToNamespaceForUser( + ctx context.Context, + namespace, username string, + groups map[string]struct{}, +) (bool, error) { + // Check privileged groups + if _, ok := groups["system:masters"]; ok { + return true, nil + } + if _, ok := groups["cozystack-cluster-admin"]; ok { + return true, nil + } + + // List RoleBindings only in the target namespace + rbs := &rbacv1.RoleBindingList{} + err := r.c.List(ctx, rbs, client.InNamespace(namespace)) + if err != nil { + return false, fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err) + } + + // Check if user is in any RoleBinding subjects + for i := range rbs.Items { + for j := range rbs.Items[i].Subjects { + subj := rbs.Items[i].Subjects[j] + if matchesSubject(subj, rbs.Items[i].Namespace, username, groups) { + return true, nil + } + } + } + + return false, nil +} + // ----------------------------------------------------------------------------- // Boiler-plate // ----------------------------------------------------------------------------- diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index 7f2979bc..52d2a075 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -3,10 +3,20 @@ package tenantnamespace import ( + "context" "testing" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" ) func TestMakeListSortsAlphabetically(t *testing.T) { @@ -38,3 +48,466 @@ func TestMakeListSortsAlphabetically(t *testing.T) { } } } + +// Security tests for IDOR fix + +func TestHasAccessToNamespace_WithUserAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected user to have access, but got false") + } +} + +func TestHasAccessToNamespace_WithoutAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // RoleBinding for different user + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if hasAccess { + t.Error("expected user to NOT have access, but got true") + } +} + +func TestHasAccessToNamespace_WithGroupAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "Group", Name: "test-group", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated", "test-group"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected user to have access via group, but got false") + } +} + +func TestHasAccessToNamespace_SystemMasters(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "admin", + Groups: []string{"system:masters"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected system:masters to have access, but got false") + } +} + +func TestHasAccessToNamespace_CozyAdminGroup(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "cozy-admin", + Groups: []string{"cozystack-cluster-admin"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected cozystack-cluster-admin to have access, but got false") + } +} + +func TestHasAccessToNamespace_ServiceAccount(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "test-sa", Namespace: "tenant-test"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "system:serviceaccount:tenant-test:test-sa", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected service account to have access, but got false") + } +} + +func TestHasAccessToNamespace_ServiceAccountEmptyNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // ServiceAccount subject with empty namespace should default to RoleBinding namespace + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: "test-sa", Namespace: ""}, // Empty namespace + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "system:serviceaccount:tenant-test:test-sa", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !hasAccess { + t.Error("expected service account with empty namespace to have access, but got false") + } +} + +func TestGet_WithAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tn, ok := obj.(*corev1alpha1.TenantNamespace) + if !ok { + t.Fatalf("expected *TenantNamespace, got %T", obj) + } + if tn.Name != "tenant-test" { + t.Errorf("expected name %q, got %q", "tenant-test", tn.Name) + } + if tn.Kind != "TenantNamespace" { + t.Errorf("expected Kind=TenantNamespace, got %q", tn.Kind) + } + if tn.APIVersion != corev1alpha1.SchemeGroupVersion.String() { + t.Errorf("expected APIVersion=%q, got %q", corev1alpha1.SchemeGroupVersion.String(), tn.APIVersion) + } +} + +func TestGet_WithoutAccess(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, + } + + // RoleBinding for different user + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-binding", + Namespace: "tenant-test", + }, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "test-role", + }, + } + + client := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(ns, rb). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:authenticated"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) + if err == nil { + t.Fatal("expected error, got nil") + } + if obj != nil { + t.Errorf("expected nil object, got %v", obj) + } + + // Verify it returns Forbidden to follow standard K8s RBAC behavior + if !apierrors.IsForbidden(err) { + t.Errorf("expected Forbidden error, got %v", err) + } +} + +func TestGet_NonTenantNamespace(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + + client := fake.NewClientBuilder(). + WithScheme(scheme). + Build() + + r := &REST{ + c: client, + gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, + } + + u := &user.DefaultInfo{ + Name: "test-user", + Groups: []string{"system:masters"}, + } + ctx := request.WithUser(context.Background(), u) + + obj, err := r.Get(ctx, "default", &metav1.GetOptions{}) + if err == nil { + t.Fatal("expected error for non-tenant namespace, got nil") + } + if obj != nil { + t.Errorf("expected nil object, got %v", obj) + } + if !apierrors.IsNotFound(err) { + t.Errorf("expected NotFound error, got %v", err) + } +}